def test_get_cursor(self): project_name = "cursor_test_p" topic_name = "cursor_test_t%d_1" % int(time.time()) record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, 3, 7, record_schema, '1') dh.wait_shards_ready(project_name, topic_name) except ResourceExistException: pass # put tuple records record = TupleRecord( schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' record.put_attribute('AK', '47') records = [record for i in range(0, 3)] put_record_result = dh.put_records(project_name, topic_name, records) print(put_record_result) assert put_record_result.failed_record_count == 0 time.sleep(5) # ======================= get cursor ======================= cursor_oldest = dh.get_cursor(project_name, topic_name, '0', CursorType.OLDEST) cursor_latest = dh.get_cursor(project_name, topic_name, '0', CursorType.LATEST) cursor_sequence_1 = dh.get_cursor(project_name, topic_name, '0', CursorType.SEQUENCE, 0) cursor_sequence_2 = dh.get_cursor(project_name, topic_name, '0', CursorType.SEQUENCE, 2) cursor_system_time = dh.get_cursor(project_name, topic_name, '0', CursorType.SYSTEM_TIME, 0) print(cursor_system_time) # assert cursor_oldest.cursor == cursor_sequence_1.cursor # assert cursor_latest.cursor == cursor_sequence_2.cursor # assert cursor_oldest.cursor == cursor_system_time.cursor finally: clean_topic(dh, project_name) dh.delete_project(project_name)
def test_update_subscription_offset(self): project_name = "offset_test_p%d_2" % int(time.time()) topic_name = "offset_test_t%d_2" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') print(create_result) sub_id = create_result.sub_id # ======================= update subscription offset ======================= result = dh.init_and_get_subscription_offset(project_name, topic_name, sub_id, '0') print(result) offsets = result.offsets offsets['0'].sequence += 1 offsets['0'].timestamp += 1 dh.update_subscription_offset(project_name, topic_name, sub_id, offsets) offsets2 = { '0': OffsetWithSession( offsets['0'].sequence + 1, offsets['0'].timestamp + 1, offsets['0'].version, offsets['0'].session_id ) } dh.update_subscription_offset(project_name, topic_name, sub_id, offsets2) offsets3 = { '0': { 'Sequence': 1, 'Timestamp': 1, 'Version': offsets['0'].version, 'SessionId': offsets['0'].session_id } } dh.update_subscription_offset(project_name, topic_name, sub_id, offsets3) dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def test_get_exist_topic(self): project_name_2 = "topic_test_p%d_3" % int(time.time()) topic_name_1 = "topic_test_t%d_1" % int(time.time()) topic_name_2 = "topic_test_t%d_2" % int(time.time()) record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field', 'decimal_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP, FieldType.DECIMAL ]) try: dh.create_project(project_name_2, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name_2, topic_name_1, 3, 7, record_schema, '1') except ResourceExistException: pass try: dh.create_blob_topic(project_name_2, topic_name_2, 3, 7, '2') except ResourceExistException: pass # ======================= get topic ======================= topic_result_1 = dh.get_topic(project_name_2, topic_name_1) print(topic_result_1) assert topic_result_1.project_name == project_name_2 assert topic_result_1.topic_name == topic_name_1 assert topic_result_1.comment == '1' assert topic_result_1.life_cycle == 7 assert topic_result_1.record_type == RecordType.TUPLE for index, field in enumerate( topic_result_1.record_schema.field_list): assert field.name == record_schema.field_list[index].name assert field.type == record_schema.field_list[index].type assert field.allow_null == record_schema.field_list[ index].allow_null assert topic_result_1.shard_count == 3 topic_result_2 = dh.get_topic(project_name_2, topic_name_2) print(topic_result_2) assert topic_result_2.topic_name == topic_name_2 assert topic_result_2.project_name == project_name_2 assert topic_result_2.comment == '2' assert topic_result_2.life_cycle == 7 assert topic_result_2.record_type == RecordType.BLOB assert topic_result_2.shard_count == 3 finally: clean_topic(dh, project_name_2) dh.delete_project(project_name_2)
def test_get_subscription_offset(self): project_name = "offset_test_p%d_1" % int(time.time()) topic_name = "offset_test_t%d_1" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') print(create_result) sub_id = create_result.sub_id # ======================= get subscription ======================= dh.init_and_get_subscription_offset(project_name, topic_name, sub_id, '0') result = dh.get_subscription_offset(project_name, topic_name, sub_id, '0') print(result) assert result.offsets['0'].sequence >= 0 assert result.offsets['0'].timestamp >= 0 assert result.offsets['0'].version >= 0 result = dh.get_subscription_offset(project_name, topic_name, sub_id, ['0']) print(result) assert result.offsets['0'].sequence >= 0 assert result.offsets['0'].timestamp >= 0 assert result.offsets['0'].version >= 0 result = dh.get_subscription_offset(project_name, topic_name, sub_id) print(result) assert result.offsets['0'].sequence >= 0 assert result.offsets['0'].timestamp >= 0 assert result.offsets['0'].version >= 0 result = dh.get_subscription_offset(project_name, topic_name, sub_id, []) print(result) assert result.offsets['0'].sequence >= 0 assert result.offsets['0'].timestamp >= 0 assert result.offsets['0'].version >= 0 dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def test_split_merge_shard(self): project_name = "shard_test_p" topic_name_1 = 'shard_test_t%d_1' % int(time.time()) record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name_1, 1, 7, record_schema, '1') except ResourceExistException: pass time.sleep(5) # ======================= split shard ======================= new_shards = dh.split_shard(project_name, topic_name_1, '0', '66666666666666666666666666666666').new_shards assert new_shards[0].shard_id == '1' assert new_shards[1].shard_id == '2' assert new_shards[0].end_hash_key == '66666666666666666666666666666666' assert new_shards[1].begin_hash_key == '66666666666666666666666666666666' dh.wait_shards_ready(project_name, topic_name_1) shard_list = dh.list_shard(project_name, topic_name_1).shards print(shard_list) for shard in shard_list: if shard.shard_id == '0': assert shard.state == ShardState.CLOSED else: assert shard.state == ShardState.ACTIVE time.sleep(5) # ======================= merge shard ======================= merge_result = dh.merge_shard(project_name, topic_name_1, '1', '2') print(merge_result) assert merge_result.shard_id == '3' dh.wait_shards_ready(project_name, topic_name_1) shard_list = dh.list_shard(project_name, topic_name_1).shards print(shard_list) for shard in shard_list: if shard.shard_id in ('0', '1', '2'): assert shard.state == ShardState.CLOSED else: assert shard.state == ShardState.ACTIVE finally: clean_topic(dh, project_name) dh.delete_project(project_name)
def test_build_record_with_invalid_value(self): record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: record = TupleRecord(schema=record_schema, values=['a', 'yc1', 10.01, True, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[-9223372036854775809, 'yc1', 10.01, True, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[9223372036854775808, 'yc1', 10.01, True, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=['1', 'yc1', 'a', True, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, 2, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, -62135798400000001]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, -253402271999000001]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!')
def test_schema(self): project_name_0 = "topic_test_p%d_0" % int(time.time()) topic_name_0 = "topic_test_t%d_0" % int(time.time()) topic_name_1 = "topic_test_t%d_1" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema_0 = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) record_schema_1 = RecordSchema() record_schema_1.add_field(Field('bigint_field', FieldType.BIGINT)) record_schema_1.add_field(Field('string_field', FieldType.STRING, False)) record_schema_1.add_field(Field('double_field', FieldType.DOUBLE, False)) record_schema_1.add_field(Field('bool_field', FieldType.BOOLEAN)) record_schema_1.add_field(Field('event_time1', FieldType.TIMESTAMP)) print(record_schema_0) print(RecordSchema()) try: dh.create_project(project_name_0, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name_0, topic_name_0, shard_count, life_cycle, record_schema_0, '') except ResourceExistException: pass result = dh.get_topic(project_name_0, topic_name_0) print(result) for index, field in enumerate(result.record_schema.field_list): assert field.name == record_schema_0.field_list[index].name assert field.type == record_schema_0.field_list[index].type assert field.allow_null == record_schema_0.field_list[index].allow_null try: dh.create_tuple_topic(project_name_0, topic_name_1, shard_count, life_cycle, record_schema_1, '') except ResourceExistException: pass result = dh.get_topic(project_name_0, topic_name_1) print(result) for index, field in enumerate(result.record_schema.field_list): assert field.name == record_schema_1.field_list[index].name assert field.type == record_schema_1.field_list[index].type assert field.allow_null == record_schema_1.field_list[index].allow_null finally: clean_topic(dh, project_name_0) dh.delete_project(project_name_0)
def test_build_schema_success(self): record_schema_0 = RecordSchema.from_lists([ 'tinyint_field', 'smallint_field', 'integer_field', 'bigint_field', 'string_field', 'float_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.TINYINT, FieldType.SMALLINT, FieldType.INTEGER, FieldType.BIGINT, FieldType.STRING, FieldType.FLOAT, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ], [False, True, True, True, True, True, True, True, True]) record_schema_1 = RecordSchema([]) record_schema_1.add_field( Field('tinyint_field', FieldType.TINYINT, False)) record_schema_1.add_field(Field('smallint_field', FieldType.SMALLINT)) record_schema_1.add_field(Field('integer_field', FieldType.INTEGER)) record_schema_1.add_field(Field('bigint_field', FieldType.BIGINT)) record_schema_1.add_field(Field('string_field', FieldType.STRING)) record_schema_1.add_field(Field('float_field', FieldType.FLOAT)) record_schema_1.add_field(Field('double_field', FieldType.DOUBLE)) record_schema_1.add_field(Field('bool_field', FieldType.BOOLEAN)) record_schema_1.add_field(Field('event_time1', FieldType.TIMESTAMP)) fields = [] fields.append(Field('tinyint_field', FieldType.TINYINT, False)) fields.append(Field('smallint_field', FieldType.SMALLINT)) fields.append(Field('integer_field', FieldType.INTEGER)) fields.append(Field('bigint_field', FieldType.BIGINT)) fields.append(Field('string_field', FieldType.STRING)) fields.append(Field('float_field', FieldType.FLOAT)) fields.append(Field('double_field', FieldType.DOUBLE)) fields.append(Field('bool_field', FieldType.BOOLEAN)) fields.append(Field('event_time1', FieldType.TIMESTAMP)) record_schema_2 = RecordSchema(fields) for index in range(0, len(record_schema_0.field_list)): assert record_schema_0.field_list[ index].name == record_schema_1.field_list[index].name assert record_schema_0.field_list[ index].type == record_schema_1.field_list[index].type assert record_schema_0.field_list[ index].allow_null == record_schema_1.field_list[ index].allow_null assert record_schema_0.field_list[ index].name == record_schema_2.field_list[index].name assert record_schema_0.field_list[ index].type == record_schema_2.field_list[index].type assert record_schema_0.field_list[ index].allow_null == record_schema_2.field_list[ index].allow_null
def test_update_subscription_state(self): project_name = "topic_test_p%d_3" % int(time.time()) topic_name = "topic_test_t%d_3" % int(time.time()) shard_count = 3 life_cycle = 7 record_type = RecordType.TUPLE record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') assert create_result.sub_id # ======================= update subscription state ======================= dh.update_subscription_state(project_name, topic_name, create_result.sub_id, SubscriptionState.INACTIVE) result = dh.get_subscription(project_name, topic_name, create_result.sub_id) assert result.state == SubscriptionState.INACTIVE dh.update_subscription_state(project_name, topic_name, create_result.sub_id, SubscriptionState.ACTIVE) result = dh.get_subscription(project_name, topic_name, create_result.sub_id) assert result.state == SubscriptionState.ACTIVE dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def test_get_subscription(self): project_name = "topic_test_p%d_1" % int(time.time()) topic_name = "topic_test_t%d_1" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') # ======================= get subscription ======================= result = dh.get_subscription(project_name, topic_name, create_result.sub_id) print(result) assert result.comment == 'comment' assert result.create_time > 0 assert result.is_owner is True assert result.last_modify_time > 0 assert result.state == SubscriptionState.ACTIVE assert result.sub_id == create_result.sub_id assert result.topic_name == topic_name assert result.type == 0 dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def test_put_data_record_with_empty_topic_name(self): project_name = 'valid' topic_name = '' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' try: put_result = dh.put_records(project_name, topic_name, [record]) except InvalidParameterException: pass else: raise Exception('put data record success with empty topic name!')
def test_update_topic(self): project_name = "topic_test_t" topic_name_1 = "topic_test_t%d_1" % int(time.time()) topic_name_2 = "topic_test_t%d_2" % int(time.time()) record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name_1, 3, 7, record_schema, '1') except ResourceExistException: pass try: dh.create_blob_topic(project_name, topic_name_2, 3, 7, '2') except ResourceExistException: pass time.sleep(1) # ======================= update topic ======================= dh.update_topic(project_name, topic_name_1, 7, "2") topic_result_1 = dh.get_topic(project_name, topic_name_1) assert topic_result_1.life_cycle == 7 assert topic_result_1.comment == "2" try: dh.update_topic(project_name, topic_name_2, 5, "new comment") except LimitExceededException: pass # topic_result_2 = dh.get_topic(project_name, topic_name_2) # assert topic_result_2.comment == "new comment" finally: clean_topic(dh, project_name) dh.delete_project(project_name)
def test_get_record_with_empty_shard_id(self): project_name = 'valid' topic_name = 'valid' shard_id = '' limit_num = 10 cursor = '20000000000000000000000000fb0021' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num) except InvalidParameterException: pass else: raise Exception('get data record success with empty shard id!')
def test_put_malformed_tuple_record(self): project_name = 'put' topic_name = 'malformed' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' try: with HTTMock(datahub_api_mock): put_result = dh.put_records(project_name, topic_name, [record]) except InvalidParameterException: pass else: raise Exception('put malformed tuple record success!')
def test_put_data_record_with_unexisted_topic_name(self): project_name = 'valid' topic_name = 'unexisted' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' try: with HTTMock(datahub_api_mock): put_result = dh.put_records(project_name, topic_name, [record]) except ResourceNotFoundException: pass else: raise Exception('put data record success with unexisted topic name!')
def test_get_record_with_unexisted_shard_id(self): project_name = 'valid' topic_name = 'valid' shard_id = '0' limit_num = 10 cursor = '20000000000000000000000000fb0021' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: with HTTMock(datahub_api_mock): get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num) except ResourceNotFoundException: pass else: raise Exception('get data record success with unexisted shard id!')
def test_create_topic_with_unexisted_project_name(self): project_name = 'unexisted' topic_name = 'valid' shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/unexisted/topics/valid' content = json.loads(request.body) assert content['Comment'] == 'comment' assert content['Lifecycle'] == 7 assert content['ShardCount'] == 3 if content['RecordType'] == 'TUPLE': schema = json.loads(content['RecordSchema']) assert len(schema['fields']) == 5 assert schema['fields'][0]['type'] == 'bigint' assert schema['fields'][0]['name'] == 'bigint_field' assert schema['fields'][1]['type'] == 'string' assert schema['fields'][1]['name'] == 'string_field' assert schema['fields'][2]['type'] == 'double' assert schema['fields'][2]['name'] == 'double_field' assert schema['fields'][3]['type'] == 'boolean' assert schema['fields'][3]['name'] == 'bool_field' assert schema['fields'][4]['type'] == 'timestamp' assert schema['fields'][4]['name'] == 'event_time1' try: with HTTMock(gen_mock_api(check)): dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment') except ResourceNotFoundException: pass else: raise Exception('create success with unexisted project name!') try: with HTTMock(gen_mock_api(check)): dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment') except ResourceNotFoundException: pass else: raise Exception('create success with unexisted project name!')
def test_create_topic_success(self): project_name = 'success' topic_name = 'success' shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) with HTTMock(datahub_api_mock): dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment') dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment')
def test_get_topic_success(self): project_name = 'success' topic_name = 'tuple' def check(request): assert request.method == 'GET' assert request.url == 'http://endpoint/projects/success/topics/tuple' with HTTMock(gen_mock_api(check)): tuple_topic_result = dh.get_topic(project_name, topic_name) assert tuple_topic_result.project_name == project_name assert tuple_topic_result.topic_name == topic_name assert tuple_topic_result.comment == 'tuple' assert tuple_topic_result.shard_count == 3 assert tuple_topic_result.life_cycle == 7 assert tuple_topic_result.record_type == RecordType.TUPLE assert tuple_topic_result.create_time == 1525312823 assert tuple_topic_result.last_modify_time == 1525312823 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) for index in range(0, len(record_schema.field_list)): assert record_schema.field_list[index].name == tuple_topic_result.record_schema.field_list[index].name assert record_schema.field_list[index].type == tuple_topic_result.record_schema.field_list[index].type assert record_schema.field_list[index].allow_null == tuple_topic_result.record_schema.field_list[ index].allow_null topic_name = 'blob' def check(request): assert request.method == 'GET' assert request.url == 'http://endpoint/projects/success/topics/blob' with HTTMock(gen_mock_api(check)): blob_topic_result = dh.get_topic(project_name, topic_name) assert blob_topic_result.project_name == project_name assert blob_topic_result.topic_name == topic_name assert blob_topic_result.comment == 'blob' assert blob_topic_result.shard_count == 3 assert blob_topic_result.life_cycle == 7 assert blob_topic_result.record_type == RecordType.BLOB assert blob_topic_result.create_time == 1525344044 assert blob_topic_result.last_modify_time == 1525344044
def test_put_tuple_record_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) records = [] record0 = TupleRecord( schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema) record1.values = [ -9223372036854775808, 'yc1', 10.01, True, -62135798400000000 ] record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord( schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.set_value(0, 9223372036854775807) record2.set_value('string_field', 'yc1') record2.partition_key = 'TestPartitionKey' records.append(record2) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/put/topics/success/shards' with HTTMock(gen_mock_api(check)): put_result = dh.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []
def test_build_tuple_record_allow_null(self): record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP], [False, True, False, True, True]) try: record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', None, True, 253402271999000000]) except InvalidParameterException: pass else: raise Exception('build record success with none value of field not allowed null') record1 = TupleRecord(schema=record_schema) try: record1.set_value(0, None) except InvalidParameterException: pass else: raise Exception('set record success with none value of field not allowd null')
def test_update_topic_with_invalid_life_cycle(self): project_name = 'valid' topic_name = 'valid' shard_count = 3 life_cycle = 0 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment') except InvalidParameterException: pass else: raise Exception('update success with invalid life cycle!') try: dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment') except InvalidParameterException: pass else: raise Exception('update success with invalid life cycle!')
def test_get_tuple_record_pb_success(self): project_name = 'get' topic_name = 'tuple' shard_id = '0' limit_num = 10 cursor = '20000000000000000000000000fb0021' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) with HTTMock(datahub_pb_api_mock): get_result = dh2.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num) print(get_result) print(get_result.records[0]) assert get_result.next_cursor == '200000000000000000000000018c0030' assert get_result.record_count == 3 assert get_result.start_seq == 0 assert len(get_result.records) == 3 assert get_result.records[0].system_time == 1527161792134 assert get_result.records[0].values == (99, 'yc1', 10.01, True, 1455869335000000) assert get_result.records[0].attributes == {}
def test_get_tuple_record_pb_success(self): project_name = 'get' topic_name = 'tuple' shard_id = '0' limit_num = 10 cursor = '20000000000000000000000000fb0021' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/get/topics/tuple/shards/0' crc, compute_crc, pb_str = unwrap_pb_frame(request.body) pb_get_record_request = GetRecordsRequest() pb_get_record_request.ParseFromString(pb_str) assert pb_get_record_request.cursor == '20000000000000000000000000fb0021' assert pb_get_record_request.limit == 10 with HTTMock(gen_pb_mock_api(check)): get_result = dh2.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num) print(get_result) print(get_result.records[0]) assert get_result.next_cursor == '200000000000000000000000018c0030' assert get_result.record_count == 3 assert get_result.start_seq == 0 assert len(get_result.records) == 3 assert get_result.records[0].system_time == 1527161792134 assert get_result.records[0].values == (99, 'yc1', 10.01, True, 1455869335000000) assert get_result.records[0].attributes == {} assert get_result.records[2].values == (99, 'yc2', 10.02, False, 1455869335000011)
def test_create_and_delete_subscription(self): project_name = "topic_test_p%d_0" % int(time.time()) topic_name = "topic_test_t%d_0" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass # ======================= create subscription ======================= result = dh.create_subscription(project_name, topic_name, 'comment') print(result) assert result.sub_id # ======================= delete subscription ======================= dh.delete_subscription(project_name, topic_name, result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def create_all_topic(self): for k, v in conf_aliyun_datahub['topics'].items(): topic_name = k shard_count = v['shard_count'] life_cycle = v['life_cycle'] record_schema = RecordSchema.from_lists(v['record_schema'][0], v['record_schema'][1], v['record_schema'][2]) try: self.datahub.create_tuple_topic(self.project_name, topic_name, shard_count, life_cycle, record_schema, topic_name) logger.debug(self.to_string() + "create_tuple_topic({0}, {1})".format( self.project_name, topic_name)) except ResourceExistException: logger.debug( self.to_string() + "create_tuple_topic({0}, {1}) ResourceExistException". format(self.project_name, topic_name)) except Exception: logger.info(traceback.format_exc()) raise
def test_get_tuple_record_success(self): project_name = 'get' topic_name = 'tuple' shard_id = '0' limit_num = 10 cursor = '20000000000000000000000000fb0021' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/get/topics/tuple/shards/0' content = json.loads(request.body) assert content['Limit'] == 10 assert content['Action'] == 'sub' assert content['Cursor'] == '20000000000000000000000000fb0021' with HTTMock(gen_mock_api(check)): get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num) print(get_result) print(get_result.records[0]) assert get_result.next_cursor == '20000000000000000000000000830010' assert get_result.record_count == 1 assert get_result.start_seq == 0 assert len(get_result.records) == 1 assert get_result.records[0].system_time == 1526293795168 assert get_result.records[0].values == (1, 'yc1', 10.01, False, 1455869335000000) assert get_result.records[0].attributes == {"string": "string"}
def test_put_data_record_with_unexisted_topic_name(self): project_name = 'valid' topic_name = 'unexisted' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' try: def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/valid/topics/unexisted/shards' content = json.loads(request.body) assert content['Action'] == 'pub' assert len(content['Records']) == 1 assert len(content['Records'][0]['Data']) == 5 assert content['Records'][0]['Data'][0] == '1' assert content['Records'][0]['Data'][1] == 'yc1' assert content['Records'][0]['Data'][2] == '1.001e+01' assert content['Records'][0]['Data'][3] == 'true' assert content['Records'][0]['Data'][4] == '1455869335000000' with HTTMock(gen_mock_api(check)): put_result = dh.put_records(project_name, topic_name, [record]) except ResourceNotFoundException: pass else: raise Exception( 'put data record success with unexisted topic name!')
print("=======================================\n\n") except ResourceExistException: print("project already exist!") print("=======================================\n\n") except Exception as e: print(traceback.format_exc()) sys.exit(-1) # ===================== create topic ===================== # --------------------- wild_label topic --------------------- tuple_topic = "smart_sync_infer" shard_count = 1 life_cycle = 7 record_schema = RecordSchema.from_lists( ['object_id', 'ai_label', 'score'], [FieldType.STRING, FieldType.STRING, FieldType.DOUBLE]) try: dh.create_tuple_topic(project_name, tuple_topic, shard_count, life_cycle, record_schema, comment) print("create traffic topic success!") print("=======================================\n\n") except ResourceExistException: print("topic already exist!") print("=======================================\n\n") except Exception as e: print(traceback.format_exc()) sys.exit(-1) # ===================== get topic ===================== topic_result = dh.get_topic(project_name, tuple_topic)
def test_list_subscription(self): project_name = "topic_test_p%d_4" % int(time.time()) topic_name = "topic_test_t%d_4" % int(time.time()) shard_count = 3 life_cycle = 7 record_type = RecordType.TUPLE record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') assert create_result.sub_id # ======================= query subscription state ======================= result = dh.list_subscription(project_name, topic_name, '', 1, 10) print(result) assert result.total_count == 1 subscription = result.subscriptions[0] assert subscription.comment == 'comment' assert subscription.create_time > 0 assert subscription.is_owner is True assert subscription.last_modify_time > 0 assert subscription.state == SubscriptionState.ACTIVE assert subscription.sub_id == create_result.sub_id assert subscription.topic_name == topic_name assert subscription.type == 0 try: dh.list_subscription(project_name, topic_name, '', 0, 1) except InvalidParameterException: pass else: raise Exception( 'query subscription success with invalid page index') try: dh.list_subscription(project_name, topic_name, '', 1, -1) except InvalidParameterException: pass else: raise Exception( 'query subscription success with invalid page size') dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)