def _deserialize_operand(operand: kv_pb2.Operand, include_key=False, use_serialize=True):
if operand.value and len(operand.value) > 0:
if use_serialize:
return (_EggRoll.value_serdes.deserialize(operand.key), _EggRoll.value_serdes.deserialize(
operand.value)) if include_key else _EggRoll.value_serdes.deserialize(operand.value)
else:
return (bytes_to_string(operand.key), operand.value) if include_key else operand.value
return None
def import_data(request_data):
config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme)
client = CosS3Client(config)
file_name = request_data.get('sourcePath')
response = client.get_object(
Bucket=bucket,
Key=file_name
)
fp = response['Body'].get_raw_stream()
while True:
line = bytes_to_string(fp.readline())
if not line:
break
values = line.rstrip('\n').split('\t')
yield (values[0], list_feature_to_fate_str(values[1:]))
def _merge(cursors, use_serialize=True):
''' Merge sorted iterators. '''
entries = []
for _id, it in enumerate(cursors):
if it.next():
key, value = it.item()
entries.append([key, value, _id, it])
else:
it.close()
heapify(entries)
while entries:
key, value, _, it = entry = entries[0]
if use_serialize:
yield c_pickle.loads(key), c_pickle.loads(value)
else:
yield bytes_to_string(key), value
if it.next():
entry[0], entry[1] = it.item()
heapreplace(entries, entry)
else:
_, _, _, it = heappop(entries)
it.close()
def get_branch_current_commit(version_table, branch_name):
try:
return bytes_to_string(
version_table.get(branch_name, use_serialize=False))
except:
return None
def __generate_operand(kvs: Iterable, use_serialize=True):
for k, v in kvs:
yield kv_pb2.Operand(key=_EggRoll.value_serdes.serialize(k)
if use_serialize else bytes_to_string(k),
value=_EggRoll.value_serdes.serialize(v)
if use_serialize else v)