def test_put_tuple_record_success(self):
project_name = 'put'
topic_name = 'success'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
records = []
record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000])
record0.shard_id = '0'
record0.shard_id = '0'
records.append(record0)
record1 = TupleRecord(schema=record_schema)
record1.values = [-9223372036854775808, 'yc1', 10.01, True, -62135798400000000]
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records.append(record1)
record2 = TupleRecord(schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000])
record2.set_value(0, 9223372036854775807)
record2.set_value('string_field', 'yc1')
record2.partition_key = 'TestPartitionKey'
records.append(record2)
with HTTMock(datahub_api_mock):
put_result = dh.put_records(project_name, topic_name, records)
assert put_result.failed_record_count == 0
assert put_result.failed_records == []
def test_build_tuple_record_allow_null(self):
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
], [False, True, False, True, True])
try:
record0 = TupleRecord(
schema=record_schema,
values=[1, 'yc1', None, True, 253402271999000000])
except InvalidParameterException:
pass
else:
raise Exception(
'build record success with none value of field not allowed null'
)
record1 = TupleRecord(schema=record_schema)
try:
record1.set_value(0, None)
except InvalidParameterException:
pass
else:
raise Exception(
'set record success with none value of field not allowd null')
def dh_infer_label(object_id , ai_label, score):
try:
dh.wait_shards_ready(project_name, tuple_topic)
# print("shards all ready!!!")
# print("=======================================\n\n")
topic_result = dh.get_topic(project_name, tuple_topic)
# print(topic_result)
if topic_result.record_type != RecordType.TUPLE:
print("topic type illegal!")
sys.exit(-1)
# print("=======================================\n\n")
record_schema = topic_result.record_schema
#
records0 = []
record1 = TupleRecord(schema=record_schema)
record1.set_value('object_id', object_id)
record1.set_value('ai_label', ai_label)
record1.set_value('score', float(score))
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records0.append(record1)
dh.put_records(project_name, tuple_topic, records0)
except DatahubException as e:
print(e)
sys.exit(-1)
while True:
records = []
record0 = TupleRecord(schema=topic.record_schema,
values=[1, 'yc1', 10.01, True, 1455869335000000])
record0.shard_id = shards[0].shard_id
record0.put_attribute('AK', '47')
records.append(record0)
record1 = TupleRecord(schema=topic.record_schema)
record1.values = [1, 'yc1', 10.01, True, 1455869335000000]
record1.shard_id = shards[1].shard_id
records.append(record1)
record2 = TupleRecord(schema=topic.record_schema)
record2.set_value(0, 3)
record2.set_value(1, 'yc3')
record2.set_value('double_field', 10.03)
record2.set_value('bool_field', False)
record2.set_value('time_field', 1455869335000013)
record2.shard_id = shards[2].shard_id
records.append(record2)
failed_indexs = dh.put_records(project_name, topic_name, records)
print("put tuple %d records, failed list: %s" %
(len(records), failed_indexs))
# failed_indexs如果非空最好对failed record再进行重试
print("=======================================\n\n")
except DatahubException as e:
print(traceback.format_exc())
if topic_result.record_type != RecordType.TUPLE:
print("topic type illegal!")
sys.exit(-1)
print("=======================================\n\n")
record_schema = topic_result.record_schema
records0 = []
record0 = TupleRecord(schema=record_schema,
values=[1, 'yc1', 10.01, True, 1455869335000000])
record0.shard_id = '0'
record0.put_attribute('AK', '47')
records0.append(record0)
record1 = TupleRecord(schema=record_schema)
record1.set_value('bigint_field', 2)
record1.set_value('string_field', 'yc2')
record1.set_value('double_field', None)
record1.set_value('bool_field', False)
record1.set_value('time_field', 1455869335000011)
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records0.append(record1)
record2 = TupleRecord(schema=record_schema)
record2.set_value(0, 3)
record2.set_value(1, 'yc3')
record2.set_value(2, 1.1)
record2.set_value(3, False)
record2.set_value(4, 1455869335000011)
record2.attributes = {'key': 'value'}
record2.partition_key = 'TestPartitionKey'
'perf test')
except ResourceExistException as e:
print("topic %s already exist!" % args.topic)
print("=======================================\n\n")
write_request_count = 0
write_suc_reord_count = 0
write_fail_record_count = 0
with Timer() as t:
for i in range(0, args.round):
records = []
for j in range(0, args.batch):
record = TupleRecord(schema=record_schema)
# record['bigint_field'] = 2
record.set_value('string_field', data)
# record['double_field'] = 10.02
# record['bool_field'] = False
# record['time_field'] = 1455869335000011
record.shard_id = '0'
records.append(record)
put_result = dh.put_records(args.project, args.topic, records)
write_request_count += 1
write_suc_reord_count += (len(records) -
put_result.failed_record_count)
write_fail_record_count += put_result.failed_record_count
# print "%d put %d records, fail %d" %(write_request_count, len(records), len(fail_records))
print("===============result==================")
print("write_request_count:%d, %f/s" %