def test_build_schema_with_invalid_type(self): try: record_schema_0 = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ 'int', FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) except InvalidParameterException: pass else: raise Exception('build schema success with wrong filed type!') try: record_schema_1 = RecordSchema() record_schema_1.add_field(Field('string_field', 'str')) except InvalidParameterException: pass else: raise Exception('build schema success with wrong filed type!') try: fields = [] fields.append(Field('bigint_field', FieldType.BIGINT)) fields.append(Field('string_field', FieldType.STRING)) fields.append(Field('double_field', FieldType.DOUBLE)) fields.append(Field('bool_field', FieldType.BOOLEAN)) fields.append(Field('event_time1', 'time')) except InvalidParameterException: pass else: raise Exception('build schema success with wrong filed type!')
def test_update_topic_with_invalid_life_cycle(self): project_name = 'valid' topic_name = 'valid' shard_count = 3 life_cycle = 0 record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment') except InvalidParameterException: pass else: raise Exception('update success with invalid life cycle!') try: dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment') except InvalidParameterException: pass else: raise Exception('update success with invalid life cycle!')
def test_list_topic(self): project_name_0 = "topic_test_p%d_0" % int(time.time()) topic_name_0 = "topic_test_t%d_0" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name_0, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name_0, topic_name_0, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass # ======================= list topic ======================= result = dh.list_topic(project_name_0) print(result) assert topic_name_0 in result.topic_names finally: clean_topic(dh, project_name_0) dh.delete_project(project_name_0)
def test_create_topic_already_existed(self): project_name = 'existed' topic_name = 'existed' shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: with HTTMock(datahub_api_mock): dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment') except ResourceExistException: pass else: raise Exception('create success with topic already existed!') try: with HTTMock(datahub_api_mock): dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment') except ResourceExistException: pass else: raise Exception('create success with topic already existed!')
def test_put_data_record_with_limit_exceeded(self): project_name = 'put' topic_name = 'limit_exceeded' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' try: def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/put/topics/limit_exceeded/shards' content = json.loads(request.body) assert content['Action'] == 'pub' assert len(content['Records']) == 1 assert len(content['Records'][0]['Data']) == 5 assert content['Records'][0]['Data'][0] == '1' assert content['Records'][0]['Data'][1] == 'yc1' assert content['Records'][0]['Data'][2] == '1.001e+01' assert content['Records'][0]['Data'][3] == 'true' assert content['Records'][0]['Data'][4] == '1455869335000000' with HTTMock(gen_mock_api(check)): put_result = dh.put_records(project_name, topic_name, [record]) except LimitExceededException: pass else: raise Exception('put data record success with limit exceeded!')
def test_append_field(self): project_name_6 = "topic_test_p%d_6" % int(time.time()) topic_name_1 = "topic_test_t%d_1" % int(time.time()) record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name_6, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name_6, topic_name_1, 3, 7, record_schema, '1') except ResourceExistException: pass time.sleep(1) # ======================= append field ======================= dh.append_field(project_name_6, topic_name_1, 'append', FieldType.BOOLEAN) finally: clean_topic(dh, project_name_6) dh.delete_project(project_name_6)
def test_list_shard(self): project_name = "shard_test_p" topic_name_0 = 'shard_test_t%d_0' % int(time.time()) record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name_0, 3, 7, record_schema, '1') except ResourceExistException: pass # ======================= list shard ======================= shard_results = dh.list_shard(project_name, topic_name_0) print(shard_results) assert len(shard_results.shards) == 3 assert shard_results.shards[0].left_shard_id != '' assert shard_results.shards[0].right_shard_id != '' finally: clean_topic(dh, project_name) dh.delete_project(project_name)
def test_create_topic_with_invalid_topic_name(self): project_name = 'valid' invalid_topic_names = ["", "1invalid", "_invalid", "!invalid", "invalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalid" "invalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalid"] shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) for invalid_topic_name in invalid_topic_names: try: dh.create_tuple_topic(project_name, invalid_topic_name, shard_count, life_cycle, record_schema, 'comment') except InvalidParameterException: pass else: raise Exception('create success with invalid topic name!') for invalid_topic_name in invalid_topic_names: try: dh.create_blob_topic(project_name, invalid_topic_name, shard_count, life_cycle, 'comment') except InvalidParameterException: pass else: raise Exception('create success with invalid topic name!')
def test_put_tuple_record_pb_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) records = [] record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema, values=[-9223372036854775808, 'yc1', 10.01, True, -62135798400000000]) record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord(schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.partition_key = 'TestPartitionKey' records.append(record2) with HTTMock(datahub_pb_api_mock): put_result = dh2.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []
def test_get_record_with_unexisted_shard_id(self): project_name = 'valid' topic_name = 'valid' shard_id = '0' limit_num = 10 cursor = '20000000000000000000000000fb0021' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/valid/topics/valid/shards/0' content = json.loads(request.body) assert content['Limit'] == 10 assert content['Action'] == 'sub' assert content['Cursor'] == '20000000000000000000000000fb0021' with HTTMock(gen_mock_api(check)): get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num) except ResourceNotFoundException: pass else: raise Exception('get data record success with unexisted shard id!')
def test_build_tuple_record_allow_null(self): record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ], [False, True, False, True, True]) try: record0 = TupleRecord( schema=record_schema, values=[1, 'yc1', None, True, 253402271999000000]) except InvalidParameterException: pass else: raise Exception( 'build record success with none value of field not allowed null' ) record1 = TupleRecord(schema=record_schema) try: record1.set_value(0, None) except InvalidParameterException: pass else: raise Exception( 'set record success with none value of field not allowd null')
def test_create_and_delete_topic(self): project_name_1 = "topic_test_p%d_1" % int(time.time()) topic_name_1 = "topic_test_t%d_1" % int(time.time()) topic_name_2 = "topic_test_t%d_2" % int(time.time()) record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field', 'decimal_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP, FieldType.DECIMAL]) try: dh.create_project(project_name_1, '') except ResourceExistException: pass # make sure project wil be deleted try: # ======================= create topic ======================= try: dh.create_tuple_topic(project_name_1, topic_name_1, 3, 7, record_schema, '1') except ResourceExistException: pass try: dh.create_blob_topic(project_name_1, topic_name_2, 3, 7, '2') except ResourceExistException: pass finally: # ======================= delete topic ======================= clean_topic(dh, project_name_1) dh.delete_project(project_name_1)
def test_create_topic_success(self): project_name = 'success' topic_name = 'success' shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/success/topics/success' content = json.loads(request.body) assert content['Comment'] == 'comment' assert content['Lifecycle'] == 7 assert content['ShardCount'] == 3 if content['RecordType'] == 'TUPLE': schema = json.loads(content['RecordSchema']) assert len(schema['fields']) == 5 assert schema['fields'][0]['type'] == 'bigint' assert schema['fields'][0]['name'] == 'bigint_field' assert schema['fields'][1]['type'] == 'string' assert schema['fields'][1]['name'] == 'string_field' assert schema['fields'][2]['type'] == 'double' assert schema['fields'][2]['name'] == 'double_field' assert schema['fields'][3]['type'] == 'boolean' assert schema['fields'][3]['name'] == 'bool_field' assert schema['fields'][4]['type'] == 'timestamp' assert schema['fields'][4]['name'] == 'event_time1' with HTTMock(gen_mock_api(check)): dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment') dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment')
def test_update_subscription_offset(self): project_name = "offset_test_p%d_2" % int(time.time()) topic_name = "offset_test_t%d_2" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') print(create_result) sub_id = create_result.sub_id # ======================= update subscription offset ======================= result = dh.init_and_get_subscription_offset(project_name, topic_name, sub_id, '0') print(result) offsets = result.offsets offsets['0'].sequence += 1 offsets['0'].timestamp += 1 dh.update_subscription_offset(project_name, topic_name, sub_id, offsets) offsets2 = { '0': OffsetWithSession( offsets['0'].sequence + 1, offsets['0'].timestamp + 1, offsets['0'].version, offsets['0'].session_id ) } dh.update_subscription_offset(project_name, topic_name, sub_id, offsets2) offsets3 = { '0': { 'Sequence': 1, 'Timestamp': 1, 'Version': offsets['0'].version, 'SessionId': offsets['0'].session_id } } dh.update_subscription_offset(project_name, topic_name, sub_id, offsets3) dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def test_get_cursor(self): project_name = "cursor_test_p" topic_name = "cursor_test_t%d_1" % int(time.time()) record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, 3, 7, record_schema, '1') dh.wait_shards_ready(project_name, topic_name) except ResourceExistException: pass # put tuple records record = TupleRecord( schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' record.put_attribute('AK', '47') records = [record for i in range(0, 3)] put_record_result = dh.put_records(project_name, topic_name, records) print(put_record_result) assert put_record_result.failed_record_count == 0 time.sleep(5) # ======================= get cursor ======================= cursor_oldest = dh.get_cursor(project_name, topic_name, '0', CursorType.OLDEST) cursor_latest = dh.get_cursor(project_name, topic_name, '0', CursorType.LATEST) cursor_sequence_1 = dh.get_cursor(project_name, topic_name, '0', CursorType.SEQUENCE, 0) cursor_sequence_2 = dh.get_cursor(project_name, topic_name, '0', CursorType.SEQUENCE, 2) cursor_system_time = dh.get_cursor(project_name, topic_name, '0', CursorType.SYSTEM_TIME, 0) print(cursor_system_time) # assert cursor_oldest.cursor == cursor_sequence_1.cursor # assert cursor_latest.cursor == cursor_sequence_2.cursor # assert cursor_oldest.cursor == cursor_system_time.cursor finally: clean_topic(dh, project_name) dh.delete_project(project_name)
def test_get_subscription_offset(self): project_name = "offset_test_p%d_1" % int(time.time()) topic_name = "offset_test_t%d_1" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') print(create_result) sub_id = create_result.sub_id # ======================= get subscription ======================= dh.init_and_get_subscription_offset(project_name, topic_name, sub_id, '0') result = dh.get_subscription_offset(project_name, topic_name, sub_id, '0') print(result) assert result.offsets['0'].sequence >= 0 assert result.offsets['0'].timestamp >= 0 assert result.offsets['0'].version >= 0 result = dh.get_subscription_offset(project_name, topic_name, sub_id, ['0']) print(result) assert result.offsets['0'].sequence >= 0 assert result.offsets['0'].timestamp >= 0 assert result.offsets['0'].version >= 0 result = dh.get_subscription_offset(project_name, topic_name, sub_id) print(result) assert result.offsets['0'].sequence >= 0 assert result.offsets['0'].timestamp >= 0 assert result.offsets['0'].version >= 0 result = dh.get_subscription_offset(project_name, topic_name, sub_id, []) print(result) assert result.offsets['0'].sequence >= 0 assert result.offsets['0'].timestamp >= 0 assert result.offsets['0'].version >= 0 dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def test_get_exist_topic(self): project_name_2 = "topic_test_p%d_3" % int(time.time()) topic_name_1 = "topic_test_t%d_1" % int(time.time()) topic_name_2 = "topic_test_t%d_2" % int(time.time()) record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field', 'decimal_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP, FieldType.DECIMAL ]) try: dh.create_project(project_name_2, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name_2, topic_name_1, 3, 7, record_schema, '1') except ResourceExistException: pass try: dh.create_blob_topic(project_name_2, topic_name_2, 3, 7, '2') except ResourceExistException: pass # ======================= get topic ======================= topic_result_1 = dh.get_topic(project_name_2, topic_name_1) print(topic_result_1) assert topic_result_1.project_name == project_name_2 assert topic_result_1.topic_name == topic_name_1 assert topic_result_1.comment == '1' assert topic_result_1.life_cycle == 7 assert topic_result_1.record_type == RecordType.TUPLE for index, field in enumerate( topic_result_1.record_schema.field_list): assert field.name == record_schema.field_list[index].name assert field.type == record_schema.field_list[index].type assert field.allow_null == record_schema.field_list[ index].allow_null assert topic_result_1.shard_count == 3 topic_result_2 = dh.get_topic(project_name_2, topic_name_2) print(topic_result_2) assert topic_result_2.topic_name == topic_name_2 assert topic_result_2.project_name == project_name_2 assert topic_result_2.comment == '2' assert topic_result_2.life_cycle == 7 assert topic_result_2.record_type == RecordType.BLOB assert topic_result_2.shard_count == 3 finally: clean_topic(dh, project_name_2) dh.delete_project(project_name_2)
def test_split_merge_shard(self): project_name = "shard_test_p" topic_name_1 = 'shard_test_t%d_1' % int(time.time()) record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name_1, 1, 7, record_schema, '1') except ResourceExistException: pass time.sleep(5) # ======================= split shard ======================= new_shards = dh.split_shard(project_name, topic_name_1, '0', '66666666666666666666666666666666').new_shards assert new_shards[0].shard_id == '1' assert new_shards[1].shard_id == '2' assert new_shards[0].end_hash_key == '66666666666666666666666666666666' assert new_shards[1].begin_hash_key == '66666666666666666666666666666666' dh.wait_shards_ready(project_name, topic_name_1) shard_list = dh.list_shard(project_name, topic_name_1).shards print(shard_list) for shard in shard_list: if shard.shard_id == '0': assert shard.state == ShardState.CLOSED else: assert shard.state == ShardState.ACTIVE time.sleep(5) # ======================= merge shard ======================= merge_result = dh.merge_shard(project_name, topic_name_1, '1', '2') print(merge_result) assert merge_result.shard_id == '3' dh.wait_shards_ready(project_name, topic_name_1) shard_list = dh.list_shard(project_name, topic_name_1).shards print(shard_list) for shard in shard_list: if shard.shard_id in ('0', '1', '2'): assert shard.state == ShardState.CLOSED else: assert shard.state == ShardState.ACTIVE finally: clean_topic(dh, project_name) dh.delete_project(project_name)
def test_build_record_with_invalid_value(self): record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: record = TupleRecord(schema=record_schema, values=['a', 'yc1', 10.01, True, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[-9223372036854775809, 'yc1', 10.01, True, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[9223372036854775808, 'yc1', 10.01, True, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=['1', 'yc1', 'a', True, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, 2, 1455869335000000]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, -62135798400000001]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!') try: record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, -253402271999000001]) except InvalidParameterException: pass else: raise Exception('build record success with invalid value!')
def test_update_subscription_state(self): project_name = "topic_test_p%d_3" % int(time.time()) topic_name = "topic_test_t%d_3" % int(time.time()) shard_count = 3 life_cycle = 7 record_type = RecordType.TUPLE record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') assert create_result.sub_id # ======================= update subscription state ======================= dh.update_subscription_state(project_name, topic_name, create_result.sub_id, SubscriptionState.INACTIVE) result = dh.get_subscription(project_name, topic_name, create_result.sub_id) assert result.state == SubscriptionState.INACTIVE dh.update_subscription_state(project_name, topic_name, create_result.sub_id, SubscriptionState.ACTIVE) result = dh.get_subscription(project_name, topic_name, create_result.sub_id) assert result.state == SubscriptionState.ACTIVE dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def test_get_subscription(self): project_name = "topic_test_p%d_1" % int(time.time()) topic_name = "topic_test_t%d_1" % int(time.time()) shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '') except ResourceExistException: pass create_result = dh.create_subscription(project_name, topic_name, 'comment') # ======================= get subscription ======================= result = dh.get_subscription(project_name, topic_name, create_result.sub_id) print(result) assert result.comment == 'comment' assert result.create_time > 0 assert result.is_owner is True assert result.last_modify_time > 0 assert result.state == SubscriptionState.ACTIVE assert result.sub_id == create_result.sub_id assert result.topic_name == topic_name assert result.type == 0 dh.delete_subscription(project_name, topic_name, create_result.sub_id) finally: clean_topic(dh, project_name, True) dh.delete_project(project_name)
def test_put_data_record_with_empty_topic_name(self): project_name = 'valid' topic_name = '' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' try: put_result = dh.put_records(project_name, topic_name, [record]) except InvalidParameterException: pass else: raise Exception('put data record success with empty topic name!')
def test_update_topic(self): project_name = "topic_test_t" topic_name_1 = "topic_test_t%d_1" % int(time.time()) topic_name_2 = "topic_test_t%d_2" % int(time.time()) record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) try: dh.create_project(project_name, '') except ResourceExistException: pass # make sure project wil be deleted try: try: dh.create_tuple_topic(project_name, topic_name_1, 3, 7, record_schema, '1') except ResourceExistException: pass try: dh.create_blob_topic(project_name, topic_name_2, 3, 7, '2') except ResourceExistException: pass time.sleep(1) # ======================= update topic ======================= dh.update_topic(project_name, topic_name_1, 7, "2") topic_result_1 = dh.get_topic(project_name, topic_name_1) assert topic_result_1.life_cycle == 7 assert topic_result_1.comment == "2" try: dh.update_topic(project_name, topic_name_2, 5, "new comment") except LimitExceededException: pass # topic_result_2 = dh.get_topic(project_name, topic_name_2) # assert topic_result_2.comment == "new comment" finally: clean_topic(dh, project_name) dh.delete_project(project_name)
def test_put_malformed_tuple_record(self): project_name = 'put' topic_name = 'malformed' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' try: with HTTMock(datahub_api_mock): put_result = dh.put_records(project_name, topic_name, [record]) except InvalidParameterException: pass else: raise Exception('put malformed tuple record success!')
def test_put_data_record_with_unexisted_topic_name(self): project_name = 'valid' topic_name = 'unexisted' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000]) record.shard_id = '0' try: with HTTMock(datahub_api_mock): put_result = dh.put_records(project_name, topic_name, [record]) except ResourceNotFoundException: pass else: raise Exception('put data record success with unexisted topic name!')
def test_get_record_with_empty_shard_id(self): project_name = 'valid' topic_name = 'valid' shard_id = '' limit_num = 10 cursor = '20000000000000000000000000fb0021' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num) except InvalidParameterException: pass else: raise Exception('get data record success with empty shard id!')
def test_get_record_with_unexisted_shard_id(self): project_name = 'valid' topic_name = 'valid' shard_id = '0' limit_num = 10 cursor = '20000000000000000000000000fb0021' record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) try: with HTTMock(datahub_api_mock): get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num) except ResourceNotFoundException: pass else: raise Exception('get data record success with unexisted shard id!')
def test_create_topic_success(self): project_name = 'success' topic_name = 'success' shard_count = 3 life_cycle = 7 record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) with HTTMock(datahub_api_mock): dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment') dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment')
def test_get_topic_success(self): project_name = 'success' topic_name = 'tuple' def check(request): assert request.method == 'GET' assert request.url == 'http://endpoint/projects/success/topics/tuple' with HTTMock(gen_mock_api(check)): tuple_topic_result = dh.get_topic(project_name, topic_name) assert tuple_topic_result.project_name == project_name assert tuple_topic_result.topic_name == topic_name assert tuple_topic_result.comment == 'tuple' assert tuple_topic_result.shard_count == 3 assert tuple_topic_result.life_cycle == 7 assert tuple_topic_result.record_type == RecordType.TUPLE assert tuple_topic_result.create_time == 1525312823 assert tuple_topic_result.last_modify_time == 1525312823 record_schema = RecordSchema.from_lists( ['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'], [FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP]) for index in range(0, len(record_schema.field_list)): assert record_schema.field_list[index].name == tuple_topic_result.record_schema.field_list[index].name assert record_schema.field_list[index].type == tuple_topic_result.record_schema.field_list[index].type assert record_schema.field_list[index].allow_null == tuple_topic_result.record_schema.field_list[ index].allow_null topic_name = 'blob' def check(request): assert request.method == 'GET' assert request.url == 'http://endpoint/projects/success/topics/blob' with HTTMock(gen_mock_api(check)): blob_topic_result = dh.get_topic(project_name, topic_name) assert blob_topic_result.project_name == project_name assert blob_topic_result.topic_name == topic_name assert blob_topic_result.comment == 'blob' assert blob_topic_result.shard_count == 3 assert blob_topic_result.life_cycle == 7 assert blob_topic_result.record_type == RecordType.BLOB assert blob_topic_result.create_time == 1525344044 assert blob_topic_result.last_modify_time == 1525344044
def test_put_tuple_record_success(self): project_name = 'put' topic_name = 'success' record_schema = RecordSchema.from_lists([ 'bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field' ], [ FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP ]) records = [] record0 = TupleRecord( schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000]) record0.shard_id = '0' record0.shard_id = '0' records.append(record0) record1 = TupleRecord(schema=record_schema) record1.values = [ -9223372036854775808, 'yc1', 10.01, True, -62135798400000000 ] record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD' records.append(record1) record2 = TupleRecord( schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000]) record2.set_value(0, 9223372036854775807) record2.set_value('string_field', 'yc1') record2.partition_key = 'TestPartitionKey' records.append(record2) def check(request): assert request.method == 'POST' assert request.url == 'http://endpoint/projects/put/topics/success/shards' with HTTMock(gen_mock_api(check)): put_result = dh.put_records(project_name, topic_name, records) assert put_result.failed_record_count == 0 assert put_result.failed_records == []