def test_put_tuple_record_pb_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) records = [] record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema, values=[-9223372036854775808, 'yc1', 10.01, True, -62135798400000000]) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord(schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.partition_key = 'TestPartitionKey' records.append(record2) with HTTMock(datahub_pb_api_mock): put_result = dh2.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []
def test_put_tuple_record_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) records = [] record0 = TupleRecord( schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema) record1.values = [ -9223372036854775808, 'yc1', 10.01, True, -62135798400000000 ] record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord( schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.set_value(0, 9223372036854775807) record2.set_value('string_field', 'yc1') record2.partition_key = 'TestPartitionKey' records.append(record2) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/put/topics/success/shards' with HTTMock(gen_mock_api(check)): put_result = dh.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []
record1.set_value('bigint_field', 2) record1.set_value('string_field', 'yc2') record1.set_value('double_field', None) record1.set_value('bool_field', False) record1.set_value('time_field', 1455869335000011) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records0.append(record1) record2 = TupleRecord(schema=record_schema) record2.set_value(0, 3) record2.set_value(1, 'yc3') record2.set_value(2, 1.1) record2.set_value(3, False) record2.set_value(4, 1455869335000011) record2.attributes = {'key': 'value'} record2.partition_key = 'TestPartitionKey' records0.append(record2) put_result = dh.put_records(project_name, tuple_topic_name, records0) print(put_result) print("put tuple %d records, failed count: %d" % (len(records0), put_result.failed_record_count)) # failed_record_count如果大于0最好对failed record再进行重试 print("=======================================\n\n") except DatahubException as e: print(e) sys.exit(-1) # ===================== put blob records ===================== try:
def test_put_tuple_record_pb_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) records = [] record0 = TupleRecord( schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema, values=[ -9223372036854775808, 'yc1', 10.01, True, -62135798400000000 ]) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord( schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.partition_key = 'TestPartitionKey' records.append(record2) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/put/topics/success/shards' crc, compute_crc, pb_str = unwrap_pb_frame(request.body) pb_put_record_request = PutRecordsRequest() pb_put_record_request.ParseFromString(pb_str) assert len(pb_put_record_request.records) == 3 assert len(pb_put_record_request.records[0].data.data) == 5 assert pb_put_record_request.records[0].data.data[0].value == b'1' assert pb_put_record_request.records[0].data.data[ 1].value == b'yc1' assert pb_put_record_request.records[0].data.data[ 2].value == b'10.01' assert pb_put_record_request.records[0].data.data[ 3].value == b'true' assert pb_put_record_request.records[0].data.data[ 4].value == b'253402271999000000' assert len(pb_put_record_request.records[1].data.data) == 5 assert pb_put_record_request.records[1].data.data[ 0].value == b'-9223372036854775808' assert pb_put_record_request.records[1].data.data[ 1].value == b'yc1' assert pb_put_record_request.records[1].data.data[ 2].value == b'10.01' assert pb_put_record_request.records[1].data.data[ 3].value == b'true' assert pb_put_record_request.records[1].data.data[ 4].value == b'-62135798400000000' assert len(pb_put_record_request.records[2].data.data) == 5 assert pb_put_record_request.records[2].data.data[ 0].value == b'9223372036854775807' assert pb_put_record_request.records[2].data.data[ 1].value == b'yc1' assert pb_put_record_request.records[2].data.data[ 2].value == b'10.01' assert pb_put_record_request.records[2].data.data[ 3].value == b'true' assert pb_put_record_request.records[2].data.data[ 4].value == b'1455869335000000' with HTTMock(gen_pb_mock_api(check)): put_result = dh2.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []