def get_base_config(key, default=None, conf_name=SERVICE_CONF): local_path = conf_realpath(f'local.{conf_name}') if os.path.exists(local_path): local_config = file_utils.load_yaml_conf(local_path) if isinstance(local_config, dict) and key in local_config: return local_config[key] config = file_utils.load_yaml_conf(conf_realpath(conf_name)) if isinstance(config, dict): return config.get(key, default)
def from_conf( federation_session_id: str, party: Party, runtime_conf: dict, rabbitmq_config: dict, ): LOGGER.debug(f"rabbitmq_config: {rabbitmq_config}") host = rabbitmq_config.get("host") port = rabbitmq_config.get("port") mng_port = rabbitmq_config.get("mng_port") base_user = rabbitmq_config.get("user") base_password = rabbitmq_config.get("password") federation_info = runtime_conf.get("job_parameters", {}).get("federation_info", {}) union_name = federation_info.get("union_name") policy_id = federation_info.get("policy_id") rabbitmq_run = runtime_conf.get("job_parameters", {}).get("rabbitmq_run", {}) LOGGER.debug(f"rabbitmq_run: {rabbitmq_run}") max_message_size = rabbitmq_run.get("max_message_size", DEFAULT_MESSAGE_MAX_SIZE) LOGGER.debug(f"set max message size to {max_message_size} Bytes") rabbit_manager = RabbitManager(base_user, base_password, f"{host}:{mng_port}", rabbitmq_run) rabbit_manager.create_user(union_name, policy_id) route_table_path = rabbitmq_config.get("route_table") if route_table_path is None: route_table_path = "conf/rabbitmq_route_table.yaml" route_table = file_utils.load_yaml_conf(conf_path=route_table_path) mq = MQ(host, port, union_name, policy_id, route_table) return Federation(federation_session_id, party, mq, rabbit_manager, max_message_size)
def template_download(): min_data = request.json.get("min_data", False) if request.json else False memory_file = io.BytesIO() dir_dict = {} template_info = file_utils.load_yaml_conf(TEMPLATE_INFO_PATH) data_dir = template_info.get("template_data", {}).get("base_dir") min_data_file = template_info.get("template_data", {}).get("min_data", []) for name, dir_name in template_info.get("template_path", {}).items(): dir_dict[name] = os.path.join(get_fate_flow_directory(), dir_name) delete_dir_list = [] for name, dir_list in template_info.get("delete_path").items(): for dir_name in dir_list: delete_dir_list.append(os.path.join(dir_dict[name], dir_name)) tar = tarfile.open(fileobj=memory_file, mode='w:gz') for name, base_dir in dir_dict.items(): for root, dir, files in os.walk(base_dir): for file in files: if min_data: if data_dir in root and file not in min_data_file: continue if root in delete_dir_list: continue full_path = os.path.join(root, file) rel_path = os.path.join(name, os.path.relpath(full_path, base_dir)) tar.add(full_path, rel_path) tar.close() memory_file.seek(0) return send_file(memory_file, attachment_filename=f'template.tar.gz', as_attachment=True)
def update_config(key, value, conf_name=SERVICE_CONF): conf_path = conf_realpath(conf_name=conf_name) if not os.path.isabs(conf_path): conf_path = os.path.join(file_utils.get_project_base_directory(), conf_path) with filelock.FileLock(os.path.join(os.path.dirname(conf_path), ".lock")): config = file_utils.load_yaml_conf(conf_path=conf_path) or dict() config[key] = value file_utils.rewrite_yaml_conf(conf_path=conf_path, config=config)
def from_conf( federation_session_id: str, party: Party, runtime_conf: dict, pulsar_config: dict, ): LOGGER.debug(f"pulsar_config: {pulsar_config}") host = pulsar_config.get("host", "localhost") port = pulsar_config.get("port", "6650") mng_port = pulsar_config.get("mng_port", "8080") topic_ttl = int(pulsar_config.get("topic_ttl", 0)) cluster = pulsar_config.get("cluster", DEFAULT_CLUSTER) # tenant name should be unified between parties tenant = pulsar_config.get("tenant", DEFAULT_TENANT) # pulsaar runtime config pulsar_run = runtime_conf.get("job_parameters", {}).get("pulsar_run", {}) LOGGER.debug(f"pulsar_run: {pulsar_run}") max_message_size = pulsar_run.get("max_message_size", DEFAULT_MESSAGE_MAX_SIZE) LOGGER.debug(f"set max message size to {max_message_size} Bytes") # topic ttl could be overwritten by run time config topic_ttl = int(pulsar_run.get("topic_ttl", topic_ttl)) # pulsar not use user and password so far # TODO add credential to connections base_user = pulsar_config.get("user") base_password = pulsar_config.get("password") pulsar_manager = PulsarManager(host=host, port=mng_port, runtime_config=pulsar_run) # init tenant tenant_info = pulsar_manager.get_tenant(tenant=tenant).json() if tenant_info.get("allowedClusters") is None: pulsar_manager.create_tenant(tenant=tenant, admins=[], clusters=[cluster]) route_table_path = pulsar_config.get("route_table") if route_table_path is None: route_table_path = "conf/pulsar_route_table.yaml" route_table = file_utils.load_yaml_conf(conf_path=route_table_path) mq = MQ(host, port, mng_port, route_table) return Federation( federation_session_id, party, mq, pulsar_manager, max_message_size, topic_ttl, cluster, tenant, )
def load(cls): path = Path( file_utils.get_project_base_directory()) / 'conf' / SERVICE_CONF conf = file_utils.load_yaml_conf(path) if not isinstance(conf, dict): raise ValueError('invalid config file') local_path = path.with_name(f'local.{SERVICE_CONF}') if local_path.exists(): local_conf = file_utils.load_yaml_conf(local_path) if not isinstance(local_conf, dict): raise ValueError('invalid local config file') conf.update(local_conf) cls.LINKIS_SPARK_CONFIG = conf.get('fate_on_spark', {}).get('linkis_spark') for k, v in conf.items(): if isinstance(v, dict): setattr(cls, k.upper(), v)
def load(cls): conf = file_utils.load_yaml_conf(FATE_FLOW_JOB_DEFAULT_CONFIG_PATH) if not isinstance(conf, dict): raise ValueError("invalid config file") for k, v in conf.items(): if hasattr(cls, k): setattr(cls, k, v) else: stat_logger.warning(f"job default config not supported {k}") return cls.get_all()
def get_data_table_count(path): config_path = os.path.join(path, "config.yaml") config = file_utils.load_yaml_conf(conf_path=config_path) count = 0 if config: if config.get("type") != "vision": raise Exception(f"can not support this type {config.get('type')}") ext = config.get("inputs").get("ext") base_dir = os.path.join(path, "images") for file_name in os.listdir(base_dir): if file_name.endswith(ext): count += 1 return count
def from_conf(federation_session_id: str, party: Party, runtime_conf: dict, pulsar_config: dict): LOGGER.debug(f"pulsar_config: {pulsar_config}") host = pulsar_config.get("host", 'localhost') port = pulsar_config.get("port", '6650') mng_port = pulsar_config.get("mng_port", '8080') topic_ttl = int(pulsar_config.get("topic_ttl", 0)) # pulsaar runtime config pulsar_run = runtime_conf.get('job_parameters', {}).get('pulsar_run', {}) LOGGER.debug(f'pulsar_run: {pulsar_run}') max_message_size = pulsar_run.get('max_message_size', DEFAULT_MESSAGE_MAX_SIZE) LOGGER.debug(f'set max message size to {max_message_size} Bytes') # topic ttl could be overwritten by run time config topic_ttl = int(pulsar_run.get('topic_ttl', topic_ttl)) # pulsar not use user and password so far # TODO add credential to connections base_user = pulsar_config.get('user') base_password = pulsar_config.get('password') pulsar_manager = PulsarManager(host=host, port=mng_port, runtime_config=pulsar_run) # init tenant tenant = pulsar_manager.get_tenant(tenant=DEFAULT_TENANT).json() if tenant.get('allowedClusters') is None: pulsar_manager.create_tenant(tenant=DEFAULT_TENANT, admins=[], clusters=['standalone']) route_table_path = pulsar_config.get("route_table") if route_table_path is None: route_table_path = "conf/pulsar_route_table.yaml" route_table = file_utils.load_yaml_conf(conf_path=route_table_path) mq = MQ(host, port, route_table) return Federation(federation_session_id, party, mq, pulsar_manager, max_message_size, topic_ttl)
def update_config(key, value, conf_name=SERVICE_CONF): config = file_utils.load_yaml_conf(conf_path=conf_realpath( conf_name=conf_name)) or dict() config[key] = value file_utils.rewrite_yaml_conf(conf_path=conf_realpath(conf_name=conf_name), config=config)
def get_base_config(key, default=None, conf_name=SERVICE_CONF): base_config = file_utils.load_yaml_conf(conf_path=conf_realpath( conf_name=conf_name)) or dict() return base_config.get(key, default)