Exemplo n.º 1
0
 def _deserialize_operand(operand: kv_pb2.Operand, include_key=False, use_serialize=True):
     if operand.value and len(operand.value) > 0:
         if use_serialize:
             return (_EggRoll.value_serdes.deserialize(operand.key), _EggRoll.value_serdes.deserialize(
                 operand.value)) if include_key else _EggRoll.value_serdes.deserialize(operand.value)
         else:
             return (bytes_to_string(operand.key), operand.value) if include_key else operand.value
     return None
Exemplo n.º 2
0
 def import_data(request_data):
     config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme)
     client = CosS3Client(config)
     file_name = request_data.get('sourcePath')
     response = client.get_object(
         Bucket=bucket,
         Key=file_name
     )
     fp = response['Body'].get_raw_stream()
     while True:
         line = bytes_to_string(fp.readline())
         if not line:
             break
         values = line.rstrip('\n').split('\t')
         yield (values[0], list_feature_to_fate_str(values[1:]))
Exemplo n.º 3
0
 def _merge(cursors, use_serialize=True):
     ''' Merge sorted iterators. '''
     entries = []
     for _id, it in enumerate(cursors):
         if it.next():
             key, value = it.item()
             entries.append([key, value, _id, it])
         else:
             it.close()
     heapify(entries)
     while entries:
         key, value, _, it = entry = entries[0]
         if use_serialize:
             yield c_pickle.loads(key), c_pickle.loads(value)
         else:
             yield bytes_to_string(key), value
         if it.next():
             entry[0], entry[1] = it.item()
             heapreplace(entries, entry)
         else:
             _, _, _, it = heappop(entries)
             it.close()
def get_branch_current_commit(version_table, branch_name):
    try:
        return bytes_to_string(
            version_table.get(branch_name, use_serialize=False))
    except:
        return None
Exemplo n.º 5
0
 def __generate_operand(kvs: Iterable, use_serialize=True):
     for k, v in kvs:
         yield kv_pb2.Operand(key=_EggRoll.value_serdes.serialize(k)
                              if use_serialize else bytes_to_string(k),
                              value=_EggRoll.value_serdes.serialize(v)
                              if use_serialize else v)