def save_feature_data(kv_data, scene_id, my_party_id, partner_party_id, my_role, commit_log="", commit_id=None, tag="", branch="master"): version_table, data_table_info, scene_key, parent, commit_id = save_version( "feature_version", scene_id=scene_id, my_party_id=my_party_id, partner_party_id=partner_party_id, my_role=my_role, commit_id=commit_id, tag=tag, branch=branch) data_table_info = gen_data_table_info("feature_data", scene_key=scene_key, commit_id=commit_id) data_table = get_data_table(data_table_info=data_table_info, create_if_missing=True) data_table.put_all(kv_data) # save version info data_table_info["commitLog"] = commit_log save_version_info(version_table=version_table, branch=branch, commit_id=commit_id, parent=parent, data_table_info=data_table_info) return commit_id
def save_model(buffer_type, proto_buffer, commit_log="", branch="master", name='', namespace=''): version_table, data_table_info, scene_key, parent, commit_id = save_version("model_version", branch=branch) data_table_info = gen_data_table_info("model_data", scene_key=scene_key, commit_id=commit_id) data_table = get_data_table(data_table_info=data_table_info, create_if_missing=True) # todo: model slice? data_table.put(buffer_type, proto_buffer.SerializeToString(), use_serialize=False) # save model version info data_table_info["commitLog"] = commit_log save_version_info(version_table=version_table, branch=branch, commit_id=commit_id, parent=parent, data_table_info=data_table_info) return commit_id
def read_feature_header(commit_id=None, tag=None, branch="master"): version_table, data_table_info, scene_key, parent, commit_id = read_version( "feature_header_version", commit_id=commit_id, tag=tag, branch=branch) if commit_id: # Maybe param commit id or get commit id by current branch commit data_table_info = data_table_info if data_table_info else gen_data_table_info( "feature_header", scene_key=scene_key, commit_id=commit_id) data_table = get_data_table(data_table_info=data_table_info, create_if_missing=False) return json_loads(data_table.get( "features", use_serialize=False)), json_loads( data_table.get("labels", use_serialize=False)) else: return None, None
def read_feature_meta(commit_id=None, tag=None, branch="master"): version_table, data_table_info, scene_key, parent, commit_id = read_version( "feature_version", commit_id=commit_id, tag=tag, branch=branch) feature_meta = dict() if commit_id: # Maybe param commit id or get commit id by current branch commit data_table_info = data_table_info if data_table_info else gen_data_table_info( "feature_meta", scene_key=scene_key, commit_id=commit_id) data_table = get_data_table(data_table_info=data_table_info, create_if_missing=False) for k, v in data_table.collect(use_serialize=False): feature_meta[k] = bytes_string(v) return feature_meta else: return feature_meta
def read_model(buffer_type, proto_buffer, commit_id=None, tag=None, branch="master", name='', namespace=''): version_table, data_table_info, scene_key, parent, commit_id = read_version("model_version", commit_id=commit_id, tag=tag, branch=branch) if commit_id: # Maybe param commit id or get commit id by current branch commit data_table_info = data_table_info if data_table_info else gen_data_table_info("model_data", scene_key=scene_key, commit_id=commit_id) data_table = get_data_table(data_table_info=data_table_info, create_if_missing=False) # todo: model slice? buffer_bytes = data_table.get(buffer_type, use_serialize=False) proto_buffer.ParseFromString(buffer_bytes) return True else: return False
def save_feature_meta(kv_meta, commit_log="", commit_id=None, tag="", branch="master"): version_table, data_table_info, scene_key, parent, commit_id = save_version( "feature_version", commit_id=commit_id, tag=tag, branch=branch) data_table_info = gen_data_table_info("feature_meta", scene_key=scene_key, commit_id=commit_id) data_table = get_data_table(data_table_info=data_table_info, create_if_missing=True) for k, v in kv_meta.items(): data_table.put(k, json_dumps(v), use_serialize=False) # save version info data_table_info["commitLog"] = commit_log save_version_info(version_table=version_table, branch=branch, commit_id=commit_id, parent=parent, data_table_info=data_table_info) return commit_id
def get_feature_data_table(scene_id=None, my_party_id=None, partner_party_id=None, my_role=None, commit_id=None, tag=None, branch="master"): version_table, data_table_info, scene_key, parent, commit_id = read_version( "feature_version", scene_id=scene_id, my_party_id=my_party_id, partner_party_id=partner_party_id, my_role=my_role, commit_id=commit_id, tag=tag, branch=branch) if commit_id: data_table_info = data_table_info if data_table_info else gen_data_table_info( "feature_data", scene_key=scene_key, commit_id=commit_id) return get_data_table(data_table_info=data_table_info, create_if_missing=False) else: return None
def save_feature_header(features, labels, commit_log="", commit_id=None, tag=None, branch="master"): version_table, data_table_info, scene_key, parent, commit_id = save_version( "feature_header_version", commit_id=commit_id, tag=tag, branch=branch) data_table_info = gen_data_table_info("feature_header", scene_key=scene_key, commit_id=commit_id) data_table = get_data_table(data_table_info=data_table_info, create_if_missing=True) data_table.put("features", json_dumps(features), use_serialize=False) data_table.put("labels", json_dumps(labels), use_serialize=False) # save version info data_table_info["commitLog"] = commit_log save_version_info(version_table=version_table, branch=branch, commit_id=commit_id, parent=parent, data_table_info=data_table_info) return commit_id