def dh_infer_label(object_id , ai_label, score): try: dh.wait_shards_ready(project_name, tuple_topic) # print("shards all ready!!!") # print("=======================================\n\n") topic_result = dh.get_topic(project_name, tuple_topic) # print(topic_result) if topic_result.record_type != RecordType.TUPLE: print("topic type illegal!") sys.exit(-1) # print("=======================================\n\n") record_schema = topic_result.record_schema # records0 = [] record1 = TupleRecord(schema=record_schema) record1.set_value('object_id', object_id) record1.set_value('ai_label', ai_label) record1.set_value('score', float(score)) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records0.append(record1) dh.put_records(project_name, tuple_topic, records0) except DatahubException as e: print(e) sys.exit(-1)
def test_put_tuple_record_pb_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) records = [] record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema, values=[-9223372036854775808, 'yc1', 10.01, True, -62135798400000000]) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord(schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.partition_key = 'TestPartitionKey' records.append(record2) with HTTMock(datahub_pb_api_mock): put_result = dh2.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []
def test_put_tuple_record_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) records = [] record0 = TupleRecord( schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema) record1.values = [ -9223372036854775808, 'yc1', 10.01, True, -62135798400000000 ] record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord( schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.set_value(0, 9223372036854775807) record2.set_value('string_field', 'yc1') record2.partition_key = 'TestPartitionKey' records.append(record2) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/put/topics/success/shards' with HTTMock(gen_mock_api(check)): put_result = dh.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []
record_schema = topic_result.record_schema records0 = [] record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record0.shard_id = '0' record0.put_attribute('AK', '47') records0.append(record0) record1 = TupleRecord(schema=record_schema) record1.set_value('bigint_field', 2) record1.set_value('string_field', 'yc2') record1.set_value('double_field', None) record1.set_value('bool_field', False) record1.set_value('time_field', 1455869335000011) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records0.append(record1) record2 = TupleRecord(schema=record_schema) record2.set_value(0, 3) record2.set_value(1, 'yc3') record2.set_value(2, 1.1) record2.set_value(3, False) record2.set_value(4, 1455869335000011) record2.attributes = {'key': 'value'} record2.partition_key = 'TestPartitionKey' records0.append(record2) put_result = dh.put_records(project_name, tuple_topic_name, records0) print(put_result)
def test_put_tuple_record_pb_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) records = [] record0 = TupleRecord( schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema, values=[ -9223372036854775808, 'yc1', 10.01, True, -62135798400000000 ]) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord( schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.partition_key = 'TestPartitionKey' records.append(record2) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/put/topics/success/shards' crc, compute_crc, pb_str = unwrap_pb_frame(request.body) pb_put_record_request = PutRecordsRequest() pb_put_record_request.ParseFromString(pb_str) assert len(pb_put_record_request.records) == 3 assert len(pb_put_record_request.records[0].data.data) == 5 assert pb_put_record_request.records[0].data.data[0].value == b'1' assert pb_put_record_request.records[0].data.data[ 1].value == b'yc1' assert pb_put_record_request.records[0].data.data[ 2].value == b'10.01' assert pb_put_record_request.records[0].data.data[ 3].value == b'true' assert pb_put_record_request.records[0].data.data[ 4].value == b'253402271999000000' assert len(pb_put_record_request.records[1].data.data) == 5 assert pb_put_record_request.records[1].data.data[ 0].value == b'-9223372036854775808' assert pb_put_record_request.records[1].data.data[ 1].value == b'yc1' assert pb_put_record_request.records[1].data.data[ 2].value == b'10.01' assert pb_put_record_request.records[1].data.data[ 3].value == b'true' assert pb_put_record_request.records[1].data.data[ 4].value == b'-62135798400000000' assert len(pb_put_record_request.records[2].data.data) == 5 assert pb_put_record_request.records[2].data.data[ 0].value == b'9223372036854775807' assert pb_put_record_request.records[2].data.data[ 1].value == b'yc1' assert pb_put_record_request.records[2].data.data[ 2].value == b'10.01' assert pb_put_record_request.records[2].data.data[ 3].value == b'true' assert pb_put_record_request.records[2].data.data[ 4].value == b'1455869335000000' with HTTMock(gen_pb_mock_api(check)): put_result = dh2.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []