def test_put_tuple_record_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) records = [] record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema) record1.values = [-9223372036854775808, 'yc1', 10.01, True, -62135798400000000] record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord(schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.set_value(0, 9223372036854775807) record2.set_value('string_field', 'yc1') record2.partition_key = 'TestPartitionKey' records.append(record2) with HTTMock(datahub_api_mock): put_result = dh.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []
def test_build_tuple_record_allow_null(self): record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ], [False, True, False, True, True]) try: record0 = TupleRecord( schema=record_schema, values=[1, 'yc1', None, True, 253402271999000000]) except InvalidParameterException: pass else: raise Exception( 'build record success with none value of field not allowed null' ) record1 = TupleRecord(schema=record_schema) try: record1.set_value(0, None) except InvalidParameterException: pass else: raise Exception( 'set record success with none value of field not allowd null')
def dh_infer_label(object_id , ai_label, score): try: dh.wait_shards_ready(project_name, tuple_topic) # print("shards all ready!!!") # print("=======================================\n\n") topic_result = dh.get_topic(project_name, tuple_topic) # print(topic_result) if topic_result.record_type != RecordType.TUPLE: print("topic type illegal!") sys.exit(-1) # print("=======================================\n\n") record_schema = topic_result.record_schema # records0 = [] record1 = TupleRecord(schema=record_schema) record1.set_value('object_id', object_id) record1.set_value('ai_label', ai_label) record1.set_value('score', float(score)) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records0.append(record1) dh.put_records(project_name, tuple_topic, records0) except DatahubException as e: print(e) sys.exit(-1)
while True: records = [] record0 = TupleRecord(schema=topic.record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record0.shard_id = shards[0].shard_id record0.put_attribute('AK', '47') records.append(record0) record1 = TupleRecord(schema=topic.record_schema) record1.values = [1, 'yc1', 10.01, True, 1455869335000000] record1.shard_id = shards[1].shard_id records.append(record1) record2 = TupleRecord(schema=topic.record_schema) record2.set_value(0, 3) record2.set_value(1, 'yc3') record2.set_value('double_field', 10.03) record2.set_value('bool_field', False) record2.set_value('time_field', 1455869335000013) record2.shard_id = shards[2].shard_id records.append(record2) failed_indexs = dh.put_records(project_name, topic_name, records) print("put tuple %d records, failed list: %s" % (len(records), failed_indexs)) # failed_indexs如果非空最好对failed record再进行重试 print("=======================================\n\n") except DatahubException as e: print(traceback.format_exc())
if topic_result.record_type != RecordType.TUPLE: print("topic type illegal!") sys.exit(-1) print("=======================================\n\n") record_schema = topic_result.record_schema records0 = [] record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record0.shard_id = '0' record0.put_attribute('AK', '47') records0.append(record0) record1 = TupleRecord(schema=record_schema) record1.set_value('bigint_field', 2) record1.set_value('string_field', 'yc2') record1.set_value('double_field', None) record1.set_value('bool_field', False) record1.set_value('time_field', 1455869335000011) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records0.append(record1) record2 = TupleRecord(schema=record_schema) record2.set_value(0, 3) record2.set_value(1, 'yc3') record2.set_value(2, 1.1) record2.set_value(3, False) record2.set_value(4, 1455869335000011) record2.attributes = {'key': 'value'} record2.partition_key = 'TestPartitionKey'
'perf test') except ResourceExistException as e: print("topic %s already exist!" % args.topic) print("=======================================\n\n") write_request_count = 0 write_suc_reord_count = 0 write_fail_record_count = 0 with Timer() as t: for i in range(0, args.round): records = [] for j in range(0, args.batch): record = TupleRecord(schema=record_schema) # record['bigint_field'] = 2 record.set_value('string_field', data) # record['double_field'] = 10.02 # record['bool_field'] = False # record['time_field'] = 1455869335000011 record.shard_id = '0' records.append(record) put_result = dh.put_records(args.project, args.topic, records) write_request_count += 1 write_suc_reord_count += (len(records) - put_result.failed_record_count) write_fail_record_count += put_result.failed_record_count # print "%d put %d records, fail %d" %(write_request_count, len(records), len(fail_records)) print("===============result==================") print("write_request_count:%d, %f/s" %