def save_feature_data(kv_data,
scene_id,
my_party_id,
partner_party_id,
my_role,
commit_log="",
commit_id=None,
tag="",
branch="master"):
version_table, data_table_info, scene_key, parent, commit_id = save_version(
"feature_version",
scene_id=scene_id,
my_party_id=my_party_id,
partner_party_id=partner_party_id,
my_role=my_role,
commit_id=commit_id,
tag=tag,
branch=branch)
data_table_info = gen_data_table_info("feature_data",
scene_key=scene_key,
commit_id=commit_id)
data_table = get_data_table(data_table_info=data_table_info,
create_if_missing=True)
data_table.put_all(kv_data)
# save version info
data_table_info["commitLog"] = commit_log
save_version_info(version_table=version_table,
branch=branch,
commit_id=commit_id,
parent=parent,
data_table_info=data_table_info)
return commit_id
def save_model(buffer_type, proto_buffer, commit_log="", branch="master", name='', namespace=''):
version_table, data_table_info, scene_key, parent, commit_id = save_version("model_version", branch=branch)
data_table_info = gen_data_table_info("model_data", scene_key=scene_key, commit_id=commit_id)
data_table = get_data_table(data_table_info=data_table_info, create_if_missing=True)
# todo: model slice?
data_table.put(buffer_type, proto_buffer.SerializeToString(), use_serialize=False)
# save model version info
data_table_info["commitLog"] = commit_log
save_version_info(version_table=version_table, branch=branch, commit_id=commit_id, parent=parent, data_table_info=data_table_info)
return commit_id
def read_feature_header(commit_id=None, tag=None, branch="master"):
version_table, data_table_info, scene_key, parent, commit_id = read_version(
"feature_header_version", commit_id=commit_id, tag=tag, branch=branch)
if commit_id:
# Maybe param commit id or get commit id by current branch commit
data_table_info = data_table_info if data_table_info else gen_data_table_info(
"feature_header", scene_key=scene_key, commit_id=commit_id)
data_table = get_data_table(data_table_info=data_table_info,
create_if_missing=False)
return json_loads(data_table.get(
"features", use_serialize=False)), json_loads(
data_table.get("labels", use_serialize=False))
else:
return None, None
def read_feature_meta(commit_id=None, tag=None, branch="master"):
version_table, data_table_info, scene_key, parent, commit_id = read_version(
"feature_version", commit_id=commit_id, tag=tag, branch=branch)
feature_meta = dict()
if commit_id:
# Maybe param commit id or get commit id by current branch commit
data_table_info = data_table_info if data_table_info else gen_data_table_info(
"feature_meta", scene_key=scene_key, commit_id=commit_id)
data_table = get_data_table(data_table_info=data_table_info,
create_if_missing=False)
for k, v in data_table.collect(use_serialize=False):
feature_meta[k] = bytes_string(v)
return feature_meta
else:
return feature_meta
def read_model(buffer_type, proto_buffer, commit_id=None, tag=None, branch="master", name='', namespace=''):
version_table, data_table_info, scene_key, parent, commit_id = read_version("model_version",
commit_id=commit_id,
tag=tag,
branch=branch)
if commit_id:
# Maybe param commit id or get commit id by current branch commit
data_table_info = data_table_info if data_table_info else gen_data_table_info("model_data", scene_key=scene_key, commit_id=commit_id)
data_table = get_data_table(data_table_info=data_table_info, create_if_missing=False)
# todo: model slice?
buffer_bytes = data_table.get(buffer_type, use_serialize=False)
proto_buffer.ParseFromString(buffer_bytes)
return True
else:
return False
def save_feature_meta(kv_meta,
commit_log="",
commit_id=None,
tag="",
branch="master"):
version_table, data_table_info, scene_key, parent, commit_id = save_version(
"feature_version", commit_id=commit_id, tag=tag, branch=branch)
data_table_info = gen_data_table_info("feature_meta",
scene_key=scene_key,
commit_id=commit_id)
data_table = get_data_table(data_table_info=data_table_info,
create_if_missing=True)
for k, v in kv_meta.items():
data_table.put(k, json_dumps(v), use_serialize=False)
# save version info
data_table_info["commitLog"] = commit_log
save_version_info(version_table=version_table,
branch=branch,
commit_id=commit_id,
parent=parent,
data_table_info=data_table_info)
return commit_id
def get_feature_data_table(scene_id=None,
my_party_id=None,
partner_party_id=None,
my_role=None,
commit_id=None,
tag=None,
branch="master"):
version_table, data_table_info, scene_key, parent, commit_id = read_version(
"feature_version",
scene_id=scene_id,
my_party_id=my_party_id,
partner_party_id=partner_party_id,
my_role=my_role,
commit_id=commit_id,
tag=tag,
branch=branch)
if commit_id:
data_table_info = data_table_info if data_table_info else gen_data_table_info(
"feature_data", scene_key=scene_key, commit_id=commit_id)
return get_data_table(data_table_info=data_table_info,
create_if_missing=False)
else:
return None
def save_feature_header(features,
labels,
commit_log="",
commit_id=None,
tag=None,
branch="master"):
version_table, data_table_info, scene_key, parent, commit_id = save_version(
"feature_header_version", commit_id=commit_id, tag=tag, branch=branch)
data_table_info = gen_data_table_info("feature_header",
scene_key=scene_key,
commit_id=commit_id)
data_table = get_data_table(data_table_info=data_table_info,
create_if_missing=True)
data_table.put("features", json_dumps(features), use_serialize=False)
data_table.put("labels", json_dumps(labels), use_serialize=False)
# save version info
data_table_info["commitLog"] = commit_log
save_version_info(version_table=version_table,
branch=branch,
commit_id=commit_id,
parent=parent,
data_table_info=data_table_info)
return commit_id