def init(cls):
if USE_AUTHENTICATION:
# init local storage
stat_logger.info('init local authorization library')
file_dir = os.path.join(file_utils.get_python_base_directory(),
'fate_flow')
os.makedirs(file_dir, exist_ok=True)
PrivilegeAuth.local_storage_file = os.path.join(
file_dir, 'authorization_config.json')
if not os.path.exists(PrivilegeAuth.local_storage_file):
with open(PrivilegeAuth.local_storage_file, 'w') as fp:
fp.write(json.dumps({}))
# init whitelist
PrivilegeAuth.command_whitelist = PRIVILEGE_COMMAND_WHITELIST
# init ALL_PERMISSION
component_path = os.path.join(
file_utils.get_python_base_directory(), 'federatedml', 'conf',
'setting_conf')
command_file_path = os.path.join(
file_utils.get_python_base_directory(), 'fate_flow', 'apps',
'party_app.py')
stat_logger.info(
'search commands from {}'.format(command_file_path))
search_command(command_file_path)
stat_logger.info(
'search components from {}'.format(component_path))
search_component(component_path)
def monkey_patch(cls):
package_name = "monkey_patch"
package_path = os.path.join(file_utils.get_python_base_directory(), "fate_flow", package_name)
if not os.path.exists(package_path):
return
for f in os.listdir(package_path):
f_path = os.path.join(file_utils.get_python_base_directory(), "fate_flow", package_name, f)
if not os.path.isdir(f_path) or "__pycache__" in f_path:
continue
patch_module = importlib.import_module("fate_flow." + package_name + '.' + f + '.monkey_patch')
patch_module.patch_all()
def get_job_dsl_parser(dsl=None,
runtime_conf=None,
pipeline_dsl=None,
train_runtime_conf=None):
parser_version = str(runtime_conf.get('dsl_version', '1'))
dsl_parser = get_dsl_parser_by_version(parser_version)
default_runtime_conf_path = os.path.join(
file_utils.get_python_base_directory(),
*['federatedml', 'conf', 'default_runtime_conf'])
setting_conf_path = os.path.join(file_utils.get_python_base_directory(),
*['federatedml', 'conf', 'setting_conf'])
job_type = JobRuntimeConfigAdapter(runtime_conf).get_job_type()
dsl_parser.run(dsl=dsl,
runtime_conf=runtime_conf,
pipeline_dsl=pipeline_dsl,
pipeline_runtime_conf=train_runtime_conf,
default_runtime_conf_prefix=default_runtime_conf_path,
setting_conf_prefix=setting_conf_path,
mode=job_type)
return dsl_parser
def __init__(self):
database_config = DATABASE.copy()
db_name = database_config.pop("name")
if WORK_MODE == WorkMode.STANDALONE:
db_file_path = os.path.join(file_utils.get_python_base_directory(),
'fate_flow', 'fate_flow_sqlite.db')
self.database_connection = APSWDatabase(db_file_path)
elif WORK_MODE == WorkMode.CLUSTER:
self.database_connection = PooledMySQLDatabase(
db_name, **database_config)
else:
raise Exception('can not init database')
def create_pipelined_model(self):
if os.path.exists(self.model_path):
raise Exception(
"Model creation failed because it has already been created, model cache path is {}"
.format(self.model_path))
else:
os.makedirs(self.model_path, exist_ok=False)
for path in [self.variables_index_path, self.variables_data_path]:
os.makedirs(path, exist_ok=False)
shutil.copytree(
os.path.join(file_utils.get_python_base_directory(), "federatedml",
"protobuf", "proto"), self.define_proto_path)
with open(self.define_meta_path, "w", encoding="utf-8") as fw:
yaml.dump({"describe": "This is the model definition meta"},
fw,
Dumper=yaml.RoundTripDumper)
def get_proto_buffer_class(cls, buffer_name):
package_path = os.path.join(file_utils.get_python_base_directory(),
'federatedml', 'protobuf', 'generated')
package_python_path = 'federatedml.protobuf.generated'
for f in os.listdir(package_path):
if f.startswith('.'):
continue
try:
proto_module = importlib.import_module(package_python_path +
'.' + f.rstrip('.py'))
for name, obj in inspect.getmembers(proto_module):
if inspect.isclass(obj) and name == buffer_name:
return obj
except Exception as e:
stat_logger.warning(e)
else:
return None
def test_tracking(self):
with open(
os.path.join(file_utils.get_python_base_directory(),
self.dsl_path), 'r') as f:
dsl_data = json.load(f)
with open(
os.path.join(file_utils.get_python_base_directory(),
self.config_path), 'r') as f:
config_data = json.load(f)
config_data['job_parameters']['work_mode'] = WORK_MODE
config_data["initiator"]["party_id"] = self.guest_party_id
config_data["role"] = {
"guest": [self.guest_party_id],
"host": [self.host_party_id],
"arbiter": [self.host_party_id]
}
response = requests.post("/".join([self.server_url, 'job', 'submit']),
json={
'job_dsl': dsl_data,
'job_runtime_conf': config_data
})
self.assertTrue(response.status_code in [200, 201])
self.assertTrue(int(response.json()['retcode']) == 0)
job_id = response.json()['jobId']
job_info = {'f_status': 'running'}
for i in range(60):
response = requests.post("/".join(
[self.server_url, 'job', 'query']),
json={
'job_id': job_id,
'role': 'guest'
})
self.assertTrue(response.status_code in [200, 201])
job_info = response.json()['data'][0]
if EndStatus.contains(job_info['f_status']):
break
time.sleep(self.sleep_time)
print('waiting job run success, the job has been running for {}s'.
format((i + 1) * self.sleep_time))
self.assertTrue(job_info['f_status'] == JobStatus.SUCCESS)
os.makedirs(self.success_job_dir, exist_ok=True)
with open(os.path.join(self.success_job_dir, job_id), 'w') as fw:
json.dump(job_info, fw)
self.assertTrue(
os.path.exists(os.path.join(self.success_job_dir, job_id)))
# test_component_parameters
test_component(self, 'component/parameters')
# test_component_metric_all
test_component(self, 'component/metric/all')
# test_component_metric
test_component(self, 'component/metrics')
# test_component_output_model
test_component(self, 'component/output/model')
# test_component_output_data_download
test_component(self, 'component/output/data')
# test_component_output_data_download
test_component(self, 'component/output/data/download')
# test_job_data_view
test_component(self, 'job/data_view')
def test_job_operation(self):
# submit
with open(
os.path.join(file_utils.get_python_base_directory(),
self.dsl_path), 'r') as f:
dsl_data = json.load(f)
with open(
os.path.join(file_utils.get_python_base_directory(),
self.config_path), 'r') as f:
config_data = json.load(f)
config_data["job_parameters"]["work_mode"] = WORK_MODE
config_data["initiator"]["party_id"] = self.guest_party_id
config_data["role"] = {
"guest": [self.guest_party_id],
"host": [self.host_party_id],
"arbiter": [self.host_party_id]
}
response = requests.post("/".join([self.server_url, 'job', 'submit']),
json={
'job_dsl': dsl_data,
'job_runtime_conf': config_data
})
self.assertTrue(response.status_code in [200, 201])
self.assertTrue(int(response.json()['retcode']) == 0)
job_id = response.json()['jobId']
# query
response = requests.post("/".join([self.server_url, 'job', 'query']),
json={
'job_id': job_id,
'role': 'guest'
})
self.assertTrue(int(response.json()['retcode']) == 0)
job_info = response.json()['data'][0]
# note
response = requests.post("/".join([self.server_url, 'job', 'update']),
json={
'job_id': job_id,
'role': job_info['f_role'],
'party_id': job_info['f_party_id'],
'notes': 'unittest'
})
self.assertTrue(int(response.json()['retcode']) == 0)
# config
response = requests.post("/".join([self.server_url, 'job', 'config']),
json={
'job_id': job_id,
'role': job_info['f_role'],
'party_id': job_info['f_party_id']
})
self.assertTrue(int(response.json()['retcode']) == 0)
time.sleep(30)
# stop
response = requests.post("/".join([self.server_url, 'job', 'stop']),
json={'job_id': job_id})
self.assertTrue(int(response.json()['retcode']) == 0)
# logs
with closing(
requests.get("/".join([self.server_url, 'job', 'log']),
json={'job_id': job_id},
stream=True)) as response:
self.assertTrue(response.status_code in [200, 201])
# query task
response = requests.post("/".join(
[self.server_url, 'job', '/task/query']),
json={'job_id': job_id})
self.assertTrue(int(response.json()['retcode']) == 0)
# query data viw
response = requests.post("/".join(
[self.server_url, 'job', '/data/view/query']),
json={'job_id': job_id})
self.assertTrue(int(response.json()['retcode']) == 0)
class PrivilegeAuth(object):
privilege_cache = {}
file_dir = os.path.join(file_utils.get_python_base_directory(),
'fate_flow')
local_storage_file = os.path.join(file_dir, 'authorization_config.json')
USE_LOCAL_STORAGE = True
ALL_PERMISSION = {
'privilege_role': ['guest', 'host', 'arbiter'],
'privilege_command': ['create', 'stop', 'run'],
'privilege_component': []
}
command_whitelist = None
@classmethod
def authentication_privilege(cls,
src_party_id,
src_role,
request_path,
party_id_index,
role_index,
command,
component_index=None):
if not src_party_id:
src_party_id = 0
src_party_id = str(src_party_id)
if src_party_id == PrivilegeAuth.get_dest_party_id(
request_path, party_id_index):
return
stat_logger.info("party {} role {} start authentication".format(
src_party_id, src_role))
privilege_dic = PrivilegeAuth.get_authentication_items(
request_path, role_index, command)
for privilege_type, value in privilege_dic.items():
if value and privilege_type == 'privilege_component':
continue
if value in PrivilegeAuth.command_whitelist:
continue
if value:
PrivilegeAuth.authentication_privilege_do(
value, src_party_id, src_role, privilege_type)
stat_logger.info('party {} role {} authenticated success'.format(
src_party_id, src_role))
return True
@classmethod
def authentication_component(cls, job_dsl, src_party_id, src_role,
party_id, component_name):
if USE_AUTHENTICATION:
if not src_party_id:
src_party_id = 0
src_party_id = str(src_party_id)
if src_party_id == party_id:
return
stat_logger.info(
'party {} role {} start authenticated component'.format(
src_party_id, src_role))
component_module = get_component_module(component_name, job_dsl)
PrivilegeAuth.authentication_privilege_do(
component_module,
src_party_id,
src_role,
privilege_type="privilege_component")
stat_logger.info(
'party {} role {} authenticated component success'.format(
src_party_id, src_role))
return True
@classmethod
def authentication_privilege_do(cls, value, src_party_id, src_role,
privilege_type):
if value not in PrivilegeAuth.privilege_cache.get(
src_party_id, {}).get(src_role, {}).get(privilege_type, []):
if value not in PrivilegeAuth.get_permission_config(
src_party_id, src_role).get(privilege_type, []):
stat_logger.info('{} {} not authorized'.format(
privilege_type.split('_')[1], value))
raise Exception('{} {} not authorized'.format(
privilege_type.split('_')[1], value))
@classmethod
def get_permission_config(cls, src_party_id, src_role, use_local=True):
if PrivilegeAuth.USE_LOCAL_STORAGE:
return PrivilegeAuth.read_local_storage(src_party_id, src_role)
else:
return PrivilegeAuth.read_cloud_config_center(
src_party_id, src_role)
@classmethod
def read_local_storage(cls, src_party_id, src_role):
with open(PrivilegeAuth.local_storage_file) as fp:
local_storage_conf = json.load(fp)
PrivilegeAuth.privilege_cache = local_storage_conf
return local_storage_conf.get(src_party_id, {}).get(src_role, {})
@classmethod
def get_new_permission_config(cls, src_party_id, src_role, privilege_role,
privilege_command, privilege_component,
privilege_dataset, delete, src_user,
dest_user):
with open(PrivilegeAuth.local_storage_file) as f:
stat_logger.info(
"add permissions: src_party_id {} src_role {} privilege_role {} privilege_command {} privilege_component {} privilege_dataset {}"
.format(src_party_id, src_role, privilege_role,
privilege_command, privilege_component,
privilege_dataset))
json_data = json.load(f)
local_storage = json_data
privilege_dic = PrivilegeAuth.get_privilege_dic(
privilege_role, privilege_command, privilege_component)
for privilege_type, values in privilege_dic.items():
if values:
for value in values.split(','):
value = value.strip()
if value == 'all':
value = PrivilegeAuth.ALL_PERMISSION[
privilege_type]
if value not in PrivilegeAuth.ALL_PERMISSION.get(privilege_type) and \
value != PrivilegeAuth.ALL_PERMISSION.get(privilege_type):
stat_logger.info(
'{} does not exist in the permission {}'.
format(value,
privilege_type.split('_')[1]))
raise Exception(
'{} does not exist in the permission {}'.
format(value,
privilege_type.split('_')[1]))
if not delete:
if local_storage.get(src_party_id, {}):
if local_storage.get(src_party_id).get(
src_role, {}):
if local_storage.get(src_party_id).get(
src_role).get(privilege_type):
local_storage[src_party_id][src_role][privilege_type].append(value) \
if isinstance(value, str) \
else local_storage[src_party_id][src_role][privilege_type].extend(value)
else:
local_storage[src_party_id][src_role][privilege_type] = [value] \
if isinstance(value, str) \
else value
else:
local_storage[src_party_id][src_role] = {privilege_type: [value]} \
if isinstance(value, str) \
else {privilege_type: value}
else:
local_storage[src_party_id] = {src_role: {privilege_type: [value]}} \
if isinstance(value, str)\
else {src_role: {privilege_type: value}}
local_storage[src_party_id][src_role][privilege_type] = \
list(set(local_storage[src_party_id][src_role][privilege_type]))
else:
try:
if isinstance(value, str):
local_storage[src_party_id][src_role][
privilege_type].remove(value)
else:
local_storage[src_party_id][src_role][
privilege_type] = []
except:
stat_logger.info(
'{} {} is not authorized ,it cannot be deleted'
.format(
privilege_type.split('_')[1], value
) if isinstance(value, str) else
"No permission to delete")
raise Exception(
'{} {} is not authorized ,it cannot be deleted'
.format(
privilege_type.split('_')[1], value
) if isinstance(value, str) else
"No permission to delete")
# privilege dataset
if privilege_dataset:
if not src_user or not dest_user:
raise Exception(
"Required parameters: src_user and dest_user")
if not delete:
cls.set_default_timeout(privilege_dataset)
if local_storage.get(src_party_id, {}):
if local_storage.get(src_party_id).get(src_role, {}):
if local_storage.get(src_party_id).get(
src_role).get("privilege_dataset"):
if local_storage.get(src_party_id).get(
src_role).get("privilege_dataset").get(
src_user, {}):
if local_storage.get(src_party_id).get(
src_role).get(
"privilege_dataset").get(
src_user,
{}).get(dest_user, []):
for table in privilege_dataset:
privilege_dataset_table_list = [
_[0] for _ in
local_storage[src_party_id]
[src_role]["privilege_dataset"]
[src_user][dest_user]
]
if table[
0] not in privilege_dataset_table_list:
local_storage[src_party_id][
src_role][
"privilege_dataset"][
src_user][
dest_user].append(
table)
else:
local_storage[src_party_id][
src_role][
"privilege_dataset"][
src_user][dest_user][
privilege_dataset_table_list
.index(
table[0]
)] = table
else:
local_storage[src_party_id][src_role][
"privilege_dataset"][src_user] = {
dest_user: privilege_dataset
}
else:
local_storage[src_party_id][src_role][
"privilege_dataset"][src_user] = {
dest_user: privilege_dataset
}
else:
local_storage[src_party_id][src_role][
"privilege_dataset"] = {
src_user: {
dest_user: privilege_dataset
}
}
else:
local_storage[src_party_id][src_role] = {
"privilege_dataset": {
src_user: {
dest_user: privilege_dataset
}
}
}
else:
local_storage[src_party_id] = {
src_role: {
"privilege_dataset": {
src_user: {
dest_user: privilege_dataset
}
}
}
}
else:
if isinstance(privilege_dataset, list):
for table in privilege_dataset:
user_privilege_dataset = local_storage[
src_party_id][src_role]["privilege_dataset"][
src_user][dest_user]
for _table in user_privilege_dataset:
if _table[0] == table:
user_privilege_dataset.remove(_table)
elif privilege_dataset in ["all", "ALL", "*"]:
local_storage[src_party_id][src_role][
"privilege_dataset"][src_user][dest_user] = []
stat_logger.info('add permission successfully')
f.close()
return local_storage
@classmethod
def rewrite_local_storage(cls, new_json):
with open(PrivilegeAuth.local_storage_file, 'w') as fp:
PrivilegeAuth.privilege_cache = new_json
json.dump(new_json, fp, indent=4, separators=(',', ': '))
fp.close()
@classmethod
def set_default_timeout(cls, privilege_dataset):
for index, table in enumerate(privilege_dataset):
if isinstance(table, list):
table[1] = int(time.time()) + int(table[1])
else:
privilege_dataset[index] = [
table,
int(time.time()) + AUTHENTICATION_DEFAULT_TIMEOUT
if USE_DEFAULT_TIMEOUT else None
]
@classmethod
def read_cloud_config_center(cls, src_party_id, src_role):
pass
@classmethod
def write_cloud_config_center(cls, ):
pass
@classmethod
def get_authentication_items(cls, request_path, role_index, command):
dest_role = request_path.split('/')[role_index]
component = None
return PrivilegeAuth.get_privilege_dic(dest_role, command, component)
@classmethod
def get_dest_party_id(cls, request_path, party_id_index):
return request_path.split('/')[party_id_index]
@classmethod
def get_privilege_dic(cls, privilege_role, privilege_command,
privilege_component):
return {
'privilege_role': privilege_role,
'privilege_command': privilege_command,
'privilege_component': privilege_component
}
@classmethod
def init(cls):
if USE_AUTHENTICATION:
# init local storage
stat_logger.info('init local authorization library')
file_dir = os.path.join(file_utils.get_python_base_directory(),
'fate_flow')
os.makedirs(file_dir, exist_ok=True)
PrivilegeAuth.local_storage_file = os.path.join(
file_dir, 'authorization_config.json')
if not os.path.exists(PrivilegeAuth.local_storage_file):
with open(PrivilegeAuth.local_storage_file, 'w') as fp:
fp.write(json.dumps({}))
# init whitelist
PrivilegeAuth.command_whitelist = PRIVILEGE_COMMAND_WHITELIST
# init ALL_PERMISSION
component_path = os.path.join(
file_utils.get_python_base_directory(), 'federatedml', 'conf',
'setting_conf')
search_command()
stat_logger.info(
'search components from {}'.format(component_path))
search_component(component_path)
