def get_base_config(key, default=None, conf_name=SERVICE_CONF):
local_path = conf_realpath(f'local.{conf_name}')
if os.path.exists(local_path):
local_config = file_utils.load_yaml_conf(local_path)
if isinstance(local_config, dict) and key in local_config:
return local_config[key]
config = file_utils.load_yaml_conf(conf_realpath(conf_name))
if isinstance(config, dict):
return config.get(key, default)
def from_conf(
federation_session_id: str,
party: Party,
runtime_conf: dict,
rabbitmq_config: dict,
):
LOGGER.debug(f"rabbitmq_config: {rabbitmq_config}")
host = rabbitmq_config.get("host")
port = rabbitmq_config.get("port")
mng_port = rabbitmq_config.get("mng_port")
base_user = rabbitmq_config.get("user")
base_password = rabbitmq_config.get("password")
federation_info = runtime_conf.get("job_parameters",
{}).get("federation_info", {})
union_name = federation_info.get("union_name")
policy_id = federation_info.get("policy_id")
rabbitmq_run = runtime_conf.get("job_parameters",
{}).get("rabbitmq_run", {})
LOGGER.debug(f"rabbitmq_run: {rabbitmq_run}")
max_message_size = rabbitmq_run.get("max_message_size",
DEFAULT_MESSAGE_MAX_SIZE)
LOGGER.debug(f"set max message size to {max_message_size} Bytes")
rabbit_manager = RabbitManager(base_user, base_password,
f"{host}:{mng_port}", rabbitmq_run)
rabbit_manager.create_user(union_name, policy_id)
route_table_path = rabbitmq_config.get("route_table")
if route_table_path is None:
route_table_path = "conf/rabbitmq_route_table.yaml"
route_table = file_utils.load_yaml_conf(conf_path=route_table_path)
mq = MQ(host, port, union_name, policy_id, route_table)
return Federation(federation_session_id, party, mq, rabbit_manager,
max_message_size)
def template_download():
min_data = request.json.get("min_data", False) if request.json else False
memory_file = io.BytesIO()
dir_dict = {}
template_info = file_utils.load_yaml_conf(TEMPLATE_INFO_PATH)
data_dir = template_info.get("template_data", {}).get("base_dir")
min_data_file = template_info.get("template_data", {}).get("min_data", [])
for name, dir_name in template_info.get("template_path", {}).items():
dir_dict[name] = os.path.join(get_fate_flow_directory(), dir_name)
delete_dir_list = []
for name, dir_list in template_info.get("delete_path").items():
for dir_name in dir_list:
delete_dir_list.append(os.path.join(dir_dict[name], dir_name))
tar = tarfile.open(fileobj=memory_file, mode='w:gz')
for name, base_dir in dir_dict.items():
for root, dir, files in os.walk(base_dir):
for file in files:
if min_data:
if data_dir in root and file not in min_data_file:
continue
if root in delete_dir_list:
continue
full_path = os.path.join(root, file)
rel_path = os.path.join(name,
os.path.relpath(full_path, base_dir))
tar.add(full_path, rel_path)
tar.close()
memory_file.seek(0)
return send_file(memory_file,
attachment_filename=f'template.tar.gz',
as_attachment=True)
def update_config(key, value, conf_name=SERVICE_CONF):
conf_path = conf_realpath(conf_name=conf_name)
if not os.path.isabs(conf_path):
conf_path = os.path.join(file_utils.get_project_base_directory(), conf_path)
with filelock.FileLock(os.path.join(os.path.dirname(conf_path), ".lock")):
config = file_utils.load_yaml_conf(conf_path=conf_path) or dict()
config[key] = value
file_utils.rewrite_yaml_conf(conf_path=conf_path, config=config)
def from_conf(
federation_session_id: str,
party: Party,
runtime_conf: dict,
pulsar_config: dict,
):
LOGGER.debug(f"pulsar_config: {pulsar_config}")
host = pulsar_config.get("host", "localhost")
port = pulsar_config.get("port", "6650")
mng_port = pulsar_config.get("mng_port", "8080")
topic_ttl = int(pulsar_config.get("topic_ttl", 0))
cluster = pulsar_config.get("cluster", DEFAULT_CLUSTER)
# tenant name should be unified between parties
tenant = pulsar_config.get("tenant", DEFAULT_TENANT)
# pulsaar runtime config
pulsar_run = runtime_conf.get("job_parameters",
{}).get("pulsar_run", {})
LOGGER.debug(f"pulsar_run: {pulsar_run}")
max_message_size = pulsar_run.get("max_message_size",
DEFAULT_MESSAGE_MAX_SIZE)
LOGGER.debug(f"set max message size to {max_message_size} Bytes")
# topic ttl could be overwritten by run time config
topic_ttl = int(pulsar_run.get("topic_ttl", topic_ttl))
# pulsar not use user and password so far
# TODO add credential to connections
base_user = pulsar_config.get("user")
base_password = pulsar_config.get("password")
pulsar_manager = PulsarManager(host=host,
port=mng_port,
runtime_config=pulsar_run)
# init tenant
tenant_info = pulsar_manager.get_tenant(tenant=tenant).json()
if tenant_info.get("allowedClusters") is None:
pulsar_manager.create_tenant(tenant=tenant,
admins=[],
clusters=[cluster])
route_table_path = pulsar_config.get("route_table")
if route_table_path is None:
route_table_path = "conf/pulsar_route_table.yaml"
route_table = file_utils.load_yaml_conf(conf_path=route_table_path)
mq = MQ(host, port, mng_port, route_table)
return Federation(
federation_session_id,
party,
mq,
pulsar_manager,
max_message_size,
topic_ttl,
cluster,
tenant,
)
def load(cls):
path = Path(
file_utils.get_project_base_directory()) / 'conf' / SERVICE_CONF
conf = file_utils.load_yaml_conf(path)
if not isinstance(conf, dict):
raise ValueError('invalid config file')
local_path = path.with_name(f'local.{SERVICE_CONF}')
if local_path.exists():
local_conf = file_utils.load_yaml_conf(local_path)
if not isinstance(local_conf, dict):
raise ValueError('invalid local config file')
conf.update(local_conf)
cls.LINKIS_SPARK_CONFIG = conf.get('fate_on_spark',
{}).get('linkis_spark')
for k, v in conf.items():
if isinstance(v, dict):
setattr(cls, k.upper(), v)
def load(cls):
conf = file_utils.load_yaml_conf(FATE_FLOW_JOB_DEFAULT_CONFIG_PATH)
if not isinstance(conf, dict):
raise ValueError("invalid config file")
for k, v in conf.items():
if hasattr(cls, k):
setattr(cls, k, v)
else:
stat_logger.warning(f"job default config not supported {k}")
return cls.get_all()
def get_data_table_count(path):
config_path = os.path.join(path, "config.yaml")
config = file_utils.load_yaml_conf(conf_path=config_path)
count = 0
if config:
if config.get("type") != "vision":
raise Exception(f"can not support this type {config.get('type')}")
ext = config.get("inputs").get("ext")
base_dir = os.path.join(path, "images")
for file_name in os.listdir(base_dir):
if file_name.endswith(ext):
count += 1
return count
def from_conf(federation_session_id: str, party: Party, runtime_conf: dict,
pulsar_config: dict):
LOGGER.debug(f"pulsar_config: {pulsar_config}")
host = pulsar_config.get("host", 'localhost')
port = pulsar_config.get("port", '6650')
mng_port = pulsar_config.get("mng_port", '8080')
topic_ttl = int(pulsar_config.get("topic_ttl", 0))
# pulsaar runtime config
pulsar_run = runtime_conf.get('job_parameters',
{}).get('pulsar_run', {})
LOGGER.debug(f'pulsar_run: {pulsar_run}')
max_message_size = pulsar_run.get('max_message_size',
DEFAULT_MESSAGE_MAX_SIZE)
LOGGER.debug(f'set max message size to {max_message_size} Bytes')
# topic ttl could be overwritten by run time config
topic_ttl = int(pulsar_run.get('topic_ttl', topic_ttl))
# pulsar not use user and password so far
# TODO add credential to connections
base_user = pulsar_config.get('user')
base_password = pulsar_config.get('password')
pulsar_manager = PulsarManager(host=host,
port=mng_port,
runtime_config=pulsar_run)
# init tenant
tenant = pulsar_manager.get_tenant(tenant=DEFAULT_TENANT).json()
if tenant.get('allowedClusters') is None:
pulsar_manager.create_tenant(tenant=DEFAULT_TENANT,
admins=[],
clusters=['standalone'])
route_table_path = pulsar_config.get("route_table")
if route_table_path is None:
route_table_path = "conf/pulsar_route_table.yaml"
route_table = file_utils.load_yaml_conf(conf_path=route_table_path)
mq = MQ(host, port, route_table)
return Federation(federation_session_id, party, mq, pulsar_manager,
max_message_size, topic_ttl)
def update_config(key, value, conf_name=SERVICE_CONF):
config = file_utils.load_yaml_conf(conf_path=conf_realpath(
conf_name=conf_name)) or dict()
config[key] = value
file_utils.rewrite_yaml_conf(conf_path=conf_realpath(conf_name=conf_name),
config=config)
def get_base_config(key, default=None, conf_name=SERVICE_CONF):
base_config = file_utils.load_yaml_conf(conf_path=conf_realpath(
conf_name=conf_name)) or dict()
return base_config.get(key, default)