def dh_infer_label(object_id , ai_label, score):
try:
dh.wait_shards_ready(project_name, tuple_topic)
# print("shards all ready!!!")
# print("=======================================\n\n")
topic_result = dh.get_topic(project_name, tuple_topic)
# print(topic_result)
if topic_result.record_type != RecordType.TUPLE:
print("topic type illegal!")
sys.exit(-1)
# print("=======================================\n\n")
record_schema = topic_result.record_schema
#
records0 = []
record1 = TupleRecord(schema=record_schema)
record1.set_value('object_id', object_id)
record1.set_value('ai_label', ai_label)
record1.set_value('score', float(score))
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records0.append(record1)
dh.put_records(project_name, tuple_topic, records0)
except DatahubException as e:
print(e)
sys.exit(-1)
def test_put_tuple_record_pb_success(self):
project_name = 'put'
topic_name = 'success'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
records = []
record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000])
record0.shard_id = '0'
record0.shard_id = '0'
records.append(record0)
record1 = TupleRecord(schema=record_schema,
values=[-9223372036854775808, 'yc1', 10.01, True, -62135798400000000])
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records.append(record1)
record2 = TupleRecord(schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000])
record2.partition_key = 'TestPartitionKey'
records.append(record2)
with HTTMock(datahub_pb_api_mock):
put_result = dh2.put_records(project_name, topic_name, records)
assert put_result.failed_record_count == 0
assert put_result.failed_records == []
def test_put_tuple_record_success(self):
project_name = 'put'
topic_name = 'success'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
records = []
record0 = TupleRecord(
schema=record_schema,
values=[1, 'yc1', 10.01, True, 253402271999000000])
record0.shard_id = '0'
record0.shard_id = '0'
records.append(record0)
record1 = TupleRecord(schema=record_schema)
record1.values = [
-9223372036854775808, 'yc1', 10.01, True, -62135798400000000
]
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records.append(record1)
record2 = TupleRecord(
schema=record_schema,
values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000])
record2.set_value(0, 9223372036854775807)
record2.set_value('string_field', 'yc1')
record2.partition_key = 'TestPartitionKey'
records.append(record2)
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/put/topics/success/shards'
with HTTMock(gen_mock_api(check)):
put_result = dh.put_records(project_name, topic_name, records)
assert put_result.failed_record_count == 0
assert put_result.failed_records == []
record_schema = topic_result.record_schema
records0 = []
record0 = TupleRecord(schema=record_schema,
values=[1, 'yc1', 10.01, True, 1455869335000000])
record0.shard_id = '0'
record0.put_attribute('AK', '47')
records0.append(record0)
record1 = TupleRecord(schema=record_schema)
record1.set_value('bigint_field', 2)
record1.set_value('string_field', 'yc2')
record1.set_value('double_field', None)
record1.set_value('bool_field', False)
record1.set_value('time_field', 1455869335000011)
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records0.append(record1)
record2 = TupleRecord(schema=record_schema)
record2.set_value(0, 3)
record2.set_value(1, 'yc3')
record2.set_value(2, 1.1)
record2.set_value(3, False)
record2.set_value(4, 1455869335000011)
record2.attributes = {'key': 'value'}
record2.partition_key = 'TestPartitionKey'
records0.append(record2)
put_result = dh.put_records(project_name, tuple_topic_name, records0)
print(put_result)
def test_put_tuple_record_pb_success(self):
project_name = 'put'
topic_name = 'success'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
records = []
record0 = TupleRecord(
schema=record_schema,
values=[1, 'yc1', 10.01, True, 253402271999000000])
record0.shard_id = '0'
record0.shard_id = '0'
records.append(record0)
record1 = TupleRecord(schema=record_schema,
values=[
-9223372036854775808, 'yc1', 10.01, True,
-62135798400000000
])
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records.append(record1)
record2 = TupleRecord(
schema=record_schema,
values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000])
record2.partition_key = 'TestPartitionKey'
records.append(record2)
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/put/topics/success/shards'
crc, compute_crc, pb_str = unwrap_pb_frame(request.body)
pb_put_record_request = PutRecordsRequest()
pb_put_record_request.ParseFromString(pb_str)
assert len(pb_put_record_request.records) == 3
assert len(pb_put_record_request.records[0].data.data) == 5
assert pb_put_record_request.records[0].data.data[0].value == b'1'
assert pb_put_record_request.records[0].data.data[
1].value == b'yc1'
assert pb_put_record_request.records[0].data.data[
2].value == b'10.01'
assert pb_put_record_request.records[0].data.data[
3].value == b'true'
assert pb_put_record_request.records[0].data.data[
4].value == b'253402271999000000'
assert len(pb_put_record_request.records[1].data.data) == 5
assert pb_put_record_request.records[1].data.data[
0].value == b'-9223372036854775808'
assert pb_put_record_request.records[1].data.data[
1].value == b'yc1'
assert pb_put_record_request.records[1].data.data[
2].value == b'10.01'
assert pb_put_record_request.records[1].data.data[
3].value == b'true'
assert pb_put_record_request.records[1].data.data[
4].value == b'-62135798400000000'
assert len(pb_put_record_request.records[2].data.data) == 5
assert pb_put_record_request.records[2].data.data[
0].value == b'9223372036854775807'
assert pb_put_record_request.records[2].data.data[
1].value == b'yc1'
assert pb_put_record_request.records[2].data.data[
2].value == b'10.01'
assert pb_put_record_request.records[2].data.data[
3].value == b'true'
assert pb_put_record_request.records[2].data.data[
4].value == b'1455869335000000'
with HTTMock(gen_pb_mock_api(check)):
put_result = dh2.put_records(project_name, topic_name, records)
assert put_result.failed_record_count == 0
assert put_result.failed_records == []
