def load_model(config_data):
logger.info(config_data)
for serving in config_data.get('servings'):
with grpc.insecure_channel(serving) as channel:
stub = model_service_pb2_grpc.ModelServiceStub(channel)
load_model_request = model_service_pb2.PublishRequest()
for role_name, role_partys in config_data.get("role").items():
for _party_id in role_partys:
load_model_request.role[role_name].partyId.append(
_party_id)
for role_name, role_model_config in config_data.get(
"model").items():
for _party_id, role_party_model_config in role_model_config.items(
):
load_model_request.model[role_name].roleModelInfo[
_party_id].tableName = role_party_model_config[
'table_name']
load_model_request.model[role_name].roleModelInfo[
_party_id].namespace = role_party_model_config[
'namespace']
logger.info('request serving: {} load model'.format(serving))
load_model_request.local.role = config_data.get('local').get(
'role')
load_model_request.local.partyId = config_data.get('local').get(
'party_id')
print(load_model_request)
logger.info(load_model_request)
response = stub.publishLoad(load_model_request)
logger.info('{} {} load model status: {}'.format(
load_model_request.local.role,
load_model_request.local.partyId, response.statusCode))
def bind_model_service(config_data):
service_id = config_data.get('service_id')
initiator_role = config_data['initiator']['role']
initiator_party_id = config_data['initiator']['party_id']
model_id = config_data['job_parameters']['model_id']
model_version = config_data['job_parameters']['model_version']
status = True
for serving in config_data.get('servings'):
with grpc.insecure_channel(serving) as channel:
stub = model_service_pb2_grpc.ModelServiceStub(channel)
publish_model_request = model_service_pb2.PublishRequest()
publish_model_request.serviceId = service_id
for role_name, role_party in config_data.get("role").items():
publish_model_request.role[role_name].partyId.extend(
role_party)
publish_model_request.model[initiator_role].roleModelInfo[
initiator_party_id].tableName = model_version
publish_model_request.model[initiator_role].roleModelInfo[
initiator_party_id].namespace = model_utils.gen_party_model_id(
model_id, initiator_role, initiator_party_id)
publish_model_request.local.role = initiator_role
publish_model_request.local.partyId = initiator_party_id
stat_logger.info(publish_model_request)
response = stub.publishBind(publish_model_request)
stat_logger.info(response)
if response.statusCode != 0:
status = False
return status, service_id
def publish_online(config_data):
for serving in config_data.get('servings'):
with grpc.insecure_channel(serving) as channel:
stub = model_service_pb2_grpc.ModelServiceStub(channel)
request = model_service_pb2.PublishRequest()
request.myPartyId = config_data.get("partyId")
for party_id, model in config_data.get("models").items():
request.models[int(party_id)].name = model["name"]
request.models[int(party_id)].namespace = model["namespace"]
response = stub.publishOnline(request)
print(response)
def load_model(config_data):
for serving in config_data.get('servings'):
with grpc.insecure_channel(serving) as channel:
stub = model_service_pb2_grpc.ModelServiceStub(channel)
request = model_service_pb2.PublishRequest()
request.myPartyId = config_data.get("my_party_id")
for party_id, model in config_data.get("models").items():
request.models[int(party_id)].name = model["name"]
request.models[int(party_id)].namespace = model["namespace"]
response = stub.publishLoad(request)
logger.info(
"party_id: {}, serving server: {}, load status: {}".format(
request.myPartyId, serving, response.statusCode))
def load_model(config_data):
stat_logger.info(config_data)
if not config_data.get('servings'):
return 100, 'Please configure servings address'
for serving in config_data.get('servings'):
with grpc.insecure_channel(serving) as channel:
stub = model_service_pb2_grpc.ModelServiceStub(channel)
load_model_request = model_service_pb2.PublishRequest()
for role_name, role_partys in config_data.get("role").items():
for _party_id in role_partys:
load_model_request.role[role_name].partyId.append(
_party_id)
for role_name, role_model_config in config_data.get(
"model").items():
for _party_id, role_party_model_config in role_model_config.items(
):
load_model_request.model[role_name].roleModelInfo[
_party_id].tableName = role_party_model_config[
'model_version']
load_model_request.model[role_name].roleModelInfo[
_party_id].namespace = role_party_model_config[
'model_id']
stat_logger.info('request serving: {} load model'.format(serving))
load_model_request.local.role = config_data.get('local').get(
'role')
load_model_request.local.partyId = config_data.get('local').get(
'party_id')
load_model_request.loadType = config_data['job_parameters'].get(
"load_type", "FATEFLOW")
load_model_request.filePath = config_data['job_parameters'].get(
"file_path", "")
stat_logger.info(load_model_request)
response = stub.publishLoad(load_model_request)
stat_logger.info('{} {} load model status: {}'.format(
load_model_request.local.role,
load_model_request.local.partyId, response.statusCode))
if response.statusCode != 0:
return response.statusCode, '{} {}'.format(
response.message, response.error)
return 0, 'success'
def publish_online(config_data):
_role = config_data.get('local').get('role')
_party_id = config_data.get('local').get('party_id')
for serving in config_data.get('servings'):
with grpc.insecure_channel(serving) as channel:
stub = model_service_pb2_grpc.ModelServiceStub(channel)
publish_model_request = model_service_pb2.PublishRequest()
for role_name, role_party in config_data.get("role").items():
publish_model_request.role[role_name].partyId.extend(
role_party)
for role_name, role_model_config in config_data.get(
"model").items():
if role_name != _role:
continue
if role_model_config.get(str(_party_id)):
table_config = copy.deepcopy(
role_model_config.get(str(_party_id)))
table_config['scene_id'] = config_data.get('scene_id')
table_config['local'] = {
'role': _role,
'party_id': _party_id
}
table_config['role'] = config_data.get('role')
table_config['data_type'] = 'model'
table_config['gen_table_info'] = True
table_name, namespace = dtable_utils.get_table_info(
config=table_config)
publish_model_request.model[_role].roleModelInfo[
_party_id].tableName = table_name
publish_model_request.model[_role].roleModelInfo[
_party_id].namespace = namespace
publish_model_request.local.role = _role
publish_model_request.local.partyId = _party_id
logger.info(publish_model_request)
response = stub.publishOnline(publish_model_request)
logger.info(response)
def load_model(config_data):
default_table_config = dict()
default_table_config['scene_id'] = config_data.get('scene_id')
default_table_config['role'] = config_data.get('role')
default_table_config['data_type'] = 'model'
default_table_config['gen_table_info'] = True
logger.info(config_data)
for serving in config_data.get('servings'):
with grpc.insecure_channel(serving) as channel:
stub = model_service_pb2_grpc.ModelServiceStub(channel)
load_model_request = model_service_pb2.PublishRequest()
model_table_name = ''
for role_name, role_party in config_data.get("role").items():
for _party_id in role_party:
load_model_request.role[role_name].partyId.append(
_party_id)
if _party_id == PARTY_ID:
# get model table name
# the model table names automatically generated by all parties are the same
local_party_model_config = config_data.get(
'model').get(role_name, {}).get(str(_party_id))
if local_party_model_config:
table_config = copy.deepcopy(default_table_config)
table_config.update(local_party_model_config)
table_config['local'] = {
'role': role_name,
'party_id': PARTY_ID
}
table_name, namespace = dtable_utils.get_table_info(
config=table_config)
model_table_name = table_name
load_model_request.model[role_name].roleModelInfo[
int(_party_id)].tableName = table_name
load_model_request.model[role_name].roleModelInfo[
int(_party_id)].namespace = namespace
logger.info('load another party model')
for role_name, role_model_config in config_data.get(
"model").items():
for _party_id, role_party_model_config in role_model_config.items(
):
if _party_id == str(PARTY_ID) or not model_table_name:
continue
table_config = copy.deepcopy(default_table_config)
table_config['local'] = {
'role': role_name,
'party_id': _party_id
}
table_config.update(role_party_model_config)
table_config['table_name'] = table_config[
'table_name'] if table_config.get(
'table_name') else model_table_name
table_name, namespace = dtable_utils.get_table_info(
config=table_config)
load_model_request.model[role_name].roleModelInfo[int(
_party_id)].tableName = table_name
load_model_request.model[role_name].roleModelInfo[int(
_party_id)].namespace = namespace
logger.info('request serving: {} load model'.format(serving))
for role_name, role_party in config_data.get("role").items():
if role_name == 'arbiter':
continue
for _party_id in role_party:
if _party_id == PARTY_ID:
load_model_request.local.role = role_name
load_model_request.local.partyId = _party_id
logger.info(load_model_request)
response = stub.publishLoad(load_model_request)
logger.info('{} {} load model status: {}'.format(
role_name, _party_id, response.statusCode))