def test_get_cursor(self):
project_name = "cursor_test_p"
topic_name = "cursor_test_t%d_1" % int(time.time())
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, 3, 7,
record_schema, '1')
dh.wait_shards_ready(project_name, topic_name)
except ResourceExistException:
pass
# put tuple records
record = TupleRecord(
schema=record_schema,
values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
record.put_attribute('AK', '47')
records = [record for i in range(0, 3)]
put_record_result = dh.put_records(project_name, topic_name,
records)
print(put_record_result)
assert put_record_result.failed_record_count == 0
time.sleep(5)
# ======================= get cursor =======================
cursor_oldest = dh.get_cursor(project_name, topic_name, '0',
CursorType.OLDEST)
cursor_latest = dh.get_cursor(project_name, topic_name, '0',
CursorType.LATEST)
cursor_sequence_1 = dh.get_cursor(project_name, topic_name, '0',
CursorType.SEQUENCE, 0)
cursor_sequence_2 = dh.get_cursor(project_name, topic_name, '0',
CursorType.SEQUENCE, 2)
cursor_system_time = dh.get_cursor(project_name, topic_name, '0',
CursorType.SYSTEM_TIME, 0)
print(cursor_system_time)
# assert cursor_oldest.cursor == cursor_sequence_1.cursor
# assert cursor_latest.cursor == cursor_sequence_2.cursor
# assert cursor_oldest.cursor == cursor_system_time.cursor
finally:
clean_topic(dh, project_name)
dh.delete_project(project_name)
def test_update_subscription_offset(self):
project_name = "offset_test_p%d_2" % int(time.time())
topic_name = "offset_test_t%d_2" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name, 'comment')
print(create_result)
sub_id = create_result.sub_id
# ======================= update subscription offset =======================
result = dh.init_and_get_subscription_offset(project_name, topic_name, sub_id, '0')
print(result)
offsets = result.offsets
offsets['0'].sequence += 1
offsets['0'].timestamp += 1
dh.update_subscription_offset(project_name, topic_name, sub_id, offsets)
offsets2 = {
'0': OffsetWithSession(
offsets['0'].sequence + 1,
offsets['0'].timestamp + 1,
offsets['0'].version,
offsets['0'].session_id
)
}
dh.update_subscription_offset(project_name, topic_name, sub_id, offsets2)
offsets3 = {
'0': {
'Sequence': 1,
'Timestamp': 1,
'Version': offsets['0'].version,
'SessionId': offsets['0'].session_id
}
}
dh.update_subscription_offset(project_name, topic_name, sub_id, offsets3)
dh.delete_subscription(project_name, topic_name, create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def test_get_exist_topic(self):
project_name_2 = "topic_test_p%d_3" % int(time.time())
topic_name_1 = "topic_test_t%d_1" % int(time.time())
topic_name_2 = "topic_test_t%d_2" % int(time.time())
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field', 'decimal_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP, FieldType.DECIMAL
])
try:
dh.create_project(project_name_2, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name_2, topic_name_1, 3, 7,
record_schema, '1')
except ResourceExistException:
pass
try:
dh.create_blob_topic(project_name_2, topic_name_2, 3, 7, '2')
except ResourceExistException:
pass
# ======================= get topic =======================
topic_result_1 = dh.get_topic(project_name_2, topic_name_1)
print(topic_result_1)
assert topic_result_1.project_name == project_name_2
assert topic_result_1.topic_name == topic_name_1
assert topic_result_1.comment == '1'
assert topic_result_1.life_cycle == 7
assert topic_result_1.record_type == RecordType.TUPLE
for index, field in enumerate(
topic_result_1.record_schema.field_list):
assert field.name == record_schema.field_list[index].name
assert field.type == record_schema.field_list[index].type
assert field.allow_null == record_schema.field_list[
index].allow_null
assert topic_result_1.shard_count == 3
topic_result_2 = dh.get_topic(project_name_2, topic_name_2)
print(topic_result_2)
assert topic_result_2.topic_name == topic_name_2
assert topic_result_2.project_name == project_name_2
assert topic_result_2.comment == '2'
assert topic_result_2.life_cycle == 7
assert topic_result_2.record_type == RecordType.BLOB
assert topic_result_2.shard_count == 3
finally:
clean_topic(dh, project_name_2)
dh.delete_project(project_name_2)
def test_get_subscription_offset(self):
project_name = "offset_test_p%d_1" % int(time.time())
topic_name = "offset_test_t%d_1" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name, 'comment')
print(create_result)
sub_id = create_result.sub_id
# ======================= get subscription =======================
dh.init_and_get_subscription_offset(project_name, topic_name, sub_id, '0')
result = dh.get_subscription_offset(project_name, topic_name, sub_id, '0')
print(result)
assert result.offsets['0'].sequence >= 0
assert result.offsets['0'].timestamp >= 0
assert result.offsets['0'].version >= 0
result = dh.get_subscription_offset(project_name, topic_name, sub_id, ['0'])
print(result)
assert result.offsets['0'].sequence >= 0
assert result.offsets['0'].timestamp >= 0
assert result.offsets['0'].version >= 0
result = dh.get_subscription_offset(project_name, topic_name, sub_id)
print(result)
assert result.offsets['0'].sequence >= 0
assert result.offsets['0'].timestamp >= 0
assert result.offsets['0'].version >= 0
result = dh.get_subscription_offset(project_name, topic_name, sub_id, [])
print(result)
assert result.offsets['0'].sequence >= 0
assert result.offsets['0'].timestamp >= 0
assert result.offsets['0'].version >= 0
dh.delete_subscription(project_name, topic_name, create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def test_split_merge_shard(self):
project_name = "shard_test_p"
topic_name_1 = 'shard_test_t%d_1' % int(time.time())
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name_1, 1, 7, record_schema, '1')
except ResourceExistException:
pass
time.sleep(5)
# ======================= split shard =======================
new_shards = dh.split_shard(project_name, topic_name_1, '0',
'66666666666666666666666666666666').new_shards
assert new_shards[0].shard_id == '1'
assert new_shards[1].shard_id == '2'
assert new_shards[0].end_hash_key == '66666666666666666666666666666666'
assert new_shards[1].begin_hash_key == '66666666666666666666666666666666'
dh.wait_shards_ready(project_name, topic_name_1)
shard_list = dh.list_shard(project_name, topic_name_1).shards
print(shard_list)
for shard in shard_list:
if shard.shard_id == '0':
assert shard.state == ShardState.CLOSED
else:
assert shard.state == ShardState.ACTIVE
time.sleep(5)
# ======================= merge shard =======================
merge_result = dh.merge_shard(project_name, topic_name_1, '1', '2')
print(merge_result)
assert merge_result.shard_id == '3'
dh.wait_shards_ready(project_name, topic_name_1)
shard_list = dh.list_shard(project_name, topic_name_1).shards
print(shard_list)
for shard in shard_list:
if shard.shard_id in ('0', '1', '2'):
assert shard.state == ShardState.CLOSED
else:
assert shard.state == ShardState.ACTIVE
finally:
clean_topic(dh, project_name)
dh.delete_project(project_name)
def test_build_record_with_invalid_value(self):
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
record = TupleRecord(schema=record_schema, values=['a', 'yc1', 10.01, True, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema,
values=[-9223372036854775809, 'yc1', 10.01, True, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema,
values=[9223372036854775808, 'yc1', 10.01, True, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema, values=['1', 'yc1', 'a', True, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, 2, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, -62135798400000001])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, -253402271999000001])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
def test_schema(self):
project_name_0 = "topic_test_p%d_0" % int(time.time())
topic_name_0 = "topic_test_t%d_0" % int(time.time())
topic_name_1 = "topic_test_t%d_1" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema_0 = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
record_schema_1 = RecordSchema()
record_schema_1.add_field(Field('bigint_field', FieldType.BIGINT))
record_schema_1.add_field(Field('string_field', FieldType.STRING, False))
record_schema_1.add_field(Field('double_field', FieldType.DOUBLE, False))
record_schema_1.add_field(Field('bool_field', FieldType.BOOLEAN))
record_schema_1.add_field(Field('event_time1', FieldType.TIMESTAMP))
print(record_schema_0)
print(RecordSchema())
try:
dh.create_project(project_name_0, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name_0, topic_name_0, shard_count, life_cycle, record_schema_0, '')
except ResourceExistException:
pass
result = dh.get_topic(project_name_0, topic_name_0)
print(result)
for index, field in enumerate(result.record_schema.field_list):
assert field.name == record_schema_0.field_list[index].name
assert field.type == record_schema_0.field_list[index].type
assert field.allow_null == record_schema_0.field_list[index].allow_null
try:
dh.create_tuple_topic(project_name_0, topic_name_1, shard_count, life_cycle, record_schema_1, '')
except ResourceExistException:
pass
result = dh.get_topic(project_name_0, topic_name_1)
print(result)
for index, field in enumerate(result.record_schema.field_list):
assert field.name == record_schema_1.field_list[index].name
assert field.type == record_schema_1.field_list[index].type
assert field.allow_null == record_schema_1.field_list[index].allow_null
finally:
clean_topic(dh, project_name_0)
dh.delete_project(project_name_0)
def test_build_schema_success(self):
record_schema_0 = RecordSchema.from_lists([
'tinyint_field', 'smallint_field', 'integer_field', 'bigint_field',
'string_field', 'float_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.TINYINT, FieldType.SMALLINT, FieldType.INTEGER,
FieldType.BIGINT, FieldType.STRING, FieldType.FLOAT,
FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP
], [False, True, True, True, True, True, True, True, True])
record_schema_1 = RecordSchema([])
record_schema_1.add_field(
Field('tinyint_field', FieldType.TINYINT, False))
record_schema_1.add_field(Field('smallint_field', FieldType.SMALLINT))
record_schema_1.add_field(Field('integer_field', FieldType.INTEGER))
record_schema_1.add_field(Field('bigint_field', FieldType.BIGINT))
record_schema_1.add_field(Field('string_field', FieldType.STRING))
record_schema_1.add_field(Field('float_field', FieldType.FLOAT))
record_schema_1.add_field(Field('double_field', FieldType.DOUBLE))
record_schema_1.add_field(Field('bool_field', FieldType.BOOLEAN))
record_schema_1.add_field(Field('event_time1', FieldType.TIMESTAMP))
fields = []
fields.append(Field('tinyint_field', FieldType.TINYINT, False))
fields.append(Field('smallint_field', FieldType.SMALLINT))
fields.append(Field('integer_field', FieldType.INTEGER))
fields.append(Field('bigint_field', FieldType.BIGINT))
fields.append(Field('string_field', FieldType.STRING))
fields.append(Field('float_field', FieldType.FLOAT))
fields.append(Field('double_field', FieldType.DOUBLE))
fields.append(Field('bool_field', FieldType.BOOLEAN))
fields.append(Field('event_time1', FieldType.TIMESTAMP))
record_schema_2 = RecordSchema(fields)
for index in range(0, len(record_schema_0.field_list)):
assert record_schema_0.field_list[
index].name == record_schema_1.field_list[index].name
assert record_schema_0.field_list[
index].type == record_schema_1.field_list[index].type
assert record_schema_0.field_list[
index].allow_null == record_schema_1.field_list[
index].allow_null
assert record_schema_0.field_list[
index].name == record_schema_2.field_list[index].name
assert record_schema_0.field_list[
index].type == record_schema_2.field_list[index].type
assert record_schema_0.field_list[
index].allow_null == record_schema_2.field_list[
index].allow_null
def test_update_subscription_state(self):
project_name = "topic_test_p%d_3" % int(time.time())
topic_name = "topic_test_t%d_3" % int(time.time())
shard_count = 3
life_cycle = 7
record_type = RecordType.TUPLE
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name,
'comment')
assert create_result.sub_id
# ======================= update subscription state =======================
dh.update_subscription_state(project_name, topic_name,
create_result.sub_id,
SubscriptionState.INACTIVE)
result = dh.get_subscription(project_name, topic_name,
create_result.sub_id)
assert result.state == SubscriptionState.INACTIVE
dh.update_subscription_state(project_name, topic_name,
create_result.sub_id,
SubscriptionState.ACTIVE)
result = dh.get_subscription(project_name, topic_name,
create_result.sub_id)
assert result.state == SubscriptionState.ACTIVE
dh.delete_subscription(project_name, topic_name,
create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def test_get_subscription(self):
project_name = "topic_test_p%d_1" % int(time.time())
topic_name = "topic_test_t%d_1" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name,
'comment')
# ======================= get subscription =======================
result = dh.get_subscription(project_name, topic_name,
create_result.sub_id)
print(result)
assert result.comment == 'comment'
assert result.create_time > 0
assert result.is_owner is True
assert result.last_modify_time > 0
assert result.state == SubscriptionState.ACTIVE
assert result.sub_id == create_result.sub_id
assert result.topic_name == topic_name
assert result.type == 0
dh.delete_subscription(project_name, topic_name,
create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def test_put_data_record_with_empty_topic_name(self):
project_name = 'valid'
topic_name = ''
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
try:
put_result = dh.put_records(project_name, topic_name, [record])
except InvalidParameterException:
pass
else:
raise Exception('put data record success with empty topic name!')
def test_update_topic(self):
project_name = "topic_test_t"
topic_name_1 = "topic_test_t%d_1" % int(time.time())
topic_name_2 = "topic_test_t%d_2" % int(time.time())
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name_1, 3, 7,
record_schema, '1')
except ResourceExistException:
pass
try:
dh.create_blob_topic(project_name, topic_name_2, 3, 7, '2')
except ResourceExistException:
pass
time.sleep(1)
# ======================= update topic =======================
dh.update_topic(project_name, topic_name_1, 7, "2")
topic_result_1 = dh.get_topic(project_name, topic_name_1)
assert topic_result_1.life_cycle == 7
assert topic_result_1.comment == "2"
try:
dh.update_topic(project_name, topic_name_2, 5, "new comment")
except LimitExceededException:
pass
# topic_result_2 = dh.get_topic(project_name, topic_name_2)
# assert topic_result_2.comment == "new comment"
finally:
clean_topic(dh, project_name)
dh.delete_project(project_name)
def test_get_record_with_empty_shard_id(self):
project_name = 'valid'
topic_name = 'valid'
shard_id = ''
limit_num = 10
cursor = '20000000000000000000000000fb0021'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num)
except InvalidParameterException:
pass
else:
raise Exception('get data record success with empty shard id!')
def test_put_malformed_tuple_record(self):
project_name = 'put'
topic_name = 'malformed'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
try:
with HTTMock(datahub_api_mock):
put_result = dh.put_records(project_name, topic_name, [record])
except InvalidParameterException:
pass
else:
raise Exception('put malformed tuple record success!')
def test_put_data_record_with_unexisted_topic_name(self):
project_name = 'valid'
topic_name = 'unexisted'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
try:
with HTTMock(datahub_api_mock):
put_result = dh.put_records(project_name, topic_name, [record])
except ResourceNotFoundException:
pass
else:
raise Exception('put data record success with unexisted topic name!')
def test_get_record_with_unexisted_shard_id(self):
project_name = 'valid'
topic_name = 'valid'
shard_id = '0'
limit_num = 10
cursor = '20000000000000000000000000fb0021'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
with HTTMock(datahub_api_mock):
get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num)
except ResourceNotFoundException:
pass
else:
raise Exception('get data record success with unexisted shard id!')
def test_create_topic_with_unexisted_project_name(self):
project_name = 'unexisted'
topic_name = 'valid'
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/unexisted/topics/valid'
content = json.loads(request.body)
assert content['Comment'] == 'comment'
assert content['Lifecycle'] == 7
assert content['ShardCount'] == 3
if content['RecordType'] == 'TUPLE':
schema = json.loads(content['RecordSchema'])
assert len(schema['fields']) == 5
assert schema['fields'][0]['type'] == 'bigint'
assert schema['fields'][0]['name'] == 'bigint_field'
assert schema['fields'][1]['type'] == 'string'
assert schema['fields'][1]['name'] == 'string_field'
assert schema['fields'][2]['type'] == 'double'
assert schema['fields'][2]['name'] == 'double_field'
assert schema['fields'][3]['type'] == 'boolean'
assert schema['fields'][3]['name'] == 'bool_field'
assert schema['fields'][4]['type'] == 'timestamp'
assert schema['fields'][4]['name'] == 'event_time1'
try:
with HTTMock(gen_mock_api(check)):
dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment')
except ResourceNotFoundException:
pass
else:
raise Exception('create success with unexisted project name!')
try:
with HTTMock(gen_mock_api(check)):
dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment')
except ResourceNotFoundException:
pass
else:
raise Exception('create success with unexisted project name!')
def test_create_topic_success(self):
project_name = 'success'
topic_name = 'success'
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
with HTTMock(datahub_api_mock):
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, 'comment')
dh.create_blob_topic(project_name, topic_name, shard_count,
life_cycle, 'comment')
def test_get_topic_success(self):
project_name = 'success'
topic_name = 'tuple'
def check(request):
assert request.method == 'GET'
assert request.url == 'http://endpoint/projects/success/topics/tuple'
with HTTMock(gen_mock_api(check)):
tuple_topic_result = dh.get_topic(project_name, topic_name)
assert tuple_topic_result.project_name == project_name
assert tuple_topic_result.topic_name == topic_name
assert tuple_topic_result.comment == 'tuple'
assert tuple_topic_result.shard_count == 3
assert tuple_topic_result.life_cycle == 7
assert tuple_topic_result.record_type == RecordType.TUPLE
assert tuple_topic_result.create_time == 1525312823
assert tuple_topic_result.last_modify_time == 1525312823
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
for index in range(0, len(record_schema.field_list)):
assert record_schema.field_list[index].name == tuple_topic_result.record_schema.field_list[index].name
assert record_schema.field_list[index].type == tuple_topic_result.record_schema.field_list[index].type
assert record_schema.field_list[index].allow_null == tuple_topic_result.record_schema.field_list[
index].allow_null
topic_name = 'blob'
def check(request):
assert request.method == 'GET'
assert request.url == 'http://endpoint/projects/success/topics/blob'
with HTTMock(gen_mock_api(check)):
blob_topic_result = dh.get_topic(project_name, topic_name)
assert blob_topic_result.project_name == project_name
assert blob_topic_result.topic_name == topic_name
assert blob_topic_result.comment == 'blob'
assert blob_topic_result.shard_count == 3
assert blob_topic_result.life_cycle == 7
assert blob_topic_result.record_type == RecordType.BLOB
assert blob_topic_result.create_time == 1525344044
assert blob_topic_result.last_modify_time == 1525344044
def test_put_tuple_record_success(self):
project_name = 'put'
topic_name = 'success'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
records = []
record0 = TupleRecord(
schema=record_schema,
values=[1, 'yc1', 10.01, True, 253402271999000000])
record0.shard_id = '0'
record0.shard_id = '0'
records.append(record0)
record1 = TupleRecord(schema=record_schema)
record1.values = [
-9223372036854775808, 'yc1', 10.01, True, -62135798400000000
]
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records.append(record1)
record2 = TupleRecord(
schema=record_schema,
values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000])
record2.set_value(0, 9223372036854775807)
record2.set_value('string_field', 'yc1')
record2.partition_key = 'TestPartitionKey'
records.append(record2)
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/put/topics/success/shards'
with HTTMock(gen_mock_api(check)):
put_result = dh.put_records(project_name, topic_name, records)
assert put_result.failed_record_count == 0
assert put_result.failed_records == []
def test_build_tuple_record_allow_null(self):
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP],
[False, True, False, True, True])
try:
record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', None, True, 253402271999000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with none value of field not allowed null')
record1 = TupleRecord(schema=record_schema)
try:
record1.set_value(0, None)
except InvalidParameterException:
pass
else:
raise Exception('set record success with none value of field not allowd null')
def test_update_topic_with_invalid_life_cycle(self):
project_name = 'valid'
topic_name = 'valid'
shard_count = 3
life_cycle = 0
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment')
except InvalidParameterException:
pass
else:
raise Exception('update success with invalid life cycle!')
try:
dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment')
except InvalidParameterException:
pass
else:
raise Exception('update success with invalid life cycle!')
def test_get_tuple_record_pb_success(self):
project_name = 'get'
topic_name = 'tuple'
shard_id = '0'
limit_num = 10
cursor = '20000000000000000000000000fb0021'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
with HTTMock(datahub_pb_api_mock):
get_result = dh2.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num)
print(get_result)
print(get_result.records[0])
assert get_result.next_cursor == '200000000000000000000000018c0030'
assert get_result.record_count == 3
assert get_result.start_seq == 0
assert len(get_result.records) == 3
assert get_result.records[0].system_time == 1527161792134
assert get_result.records[0].values == (99, 'yc1', 10.01, True, 1455869335000000)
assert get_result.records[0].attributes == {}
def test_get_tuple_record_pb_success(self):
project_name = 'get'
topic_name = 'tuple'
shard_id = '0'
limit_num = 10
cursor = '20000000000000000000000000fb0021'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/get/topics/tuple/shards/0'
crc, compute_crc, pb_str = unwrap_pb_frame(request.body)
pb_get_record_request = GetRecordsRequest()
pb_get_record_request.ParseFromString(pb_str)
assert pb_get_record_request.cursor == '20000000000000000000000000fb0021'
assert pb_get_record_request.limit == 10
with HTTMock(gen_pb_mock_api(check)):
get_result = dh2.get_tuple_records(project_name, topic_name,
shard_id, record_schema, cursor,
limit_num)
print(get_result)
print(get_result.records[0])
assert get_result.next_cursor == '200000000000000000000000018c0030'
assert get_result.record_count == 3
assert get_result.start_seq == 0
assert len(get_result.records) == 3
assert get_result.records[0].system_time == 1527161792134
assert get_result.records[0].values == (99, 'yc1', 10.01, True,
1455869335000000)
assert get_result.records[0].attributes == {}
assert get_result.records[2].values == (99, 'yc2', 10.02, False,
1455869335000011)
def test_create_and_delete_subscription(self):
project_name = "topic_test_p%d_0" % int(time.time())
topic_name = "topic_test_t%d_0" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, '')
except ResourceExistException:
pass
# ======================= create subscription =======================
result = dh.create_subscription(project_name, topic_name,
'comment')
print(result)
assert result.sub_id
# ======================= delete subscription =======================
dh.delete_subscription(project_name, topic_name, result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def create_all_topic(self):
for k, v in conf_aliyun_datahub['topics'].items():
topic_name = k
shard_count = v['shard_count']
life_cycle = v['life_cycle']
record_schema = RecordSchema.from_lists(v['record_schema'][0],
v['record_schema'][1],
v['record_schema'][2])
try:
self.datahub.create_tuple_topic(self.project_name, topic_name,
shard_count, life_cycle,
record_schema, topic_name)
logger.debug(self.to_string() +
"create_tuple_topic({0}, {1})".format(
self.project_name, topic_name))
except ResourceExistException:
logger.debug(
self.to_string() +
"create_tuple_topic({0}, {1}) ResourceExistException".
format(self.project_name, topic_name))
except Exception:
logger.info(traceback.format_exc())
raise
def test_get_tuple_record_success(self):
project_name = 'get'
topic_name = 'tuple'
shard_id = '0'
limit_num = 10
cursor = '20000000000000000000000000fb0021'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/get/topics/tuple/shards/0'
content = json.loads(request.body)
assert content['Limit'] == 10
assert content['Action'] == 'sub'
assert content['Cursor'] == '20000000000000000000000000fb0021'
with HTTMock(gen_mock_api(check)):
get_result = dh.get_tuple_records(project_name, topic_name,
shard_id, record_schema, cursor,
limit_num)
print(get_result)
print(get_result.records[0])
assert get_result.next_cursor == '20000000000000000000000000830010'
assert get_result.record_count == 1
assert get_result.start_seq == 0
assert len(get_result.records) == 1
assert get_result.records[0].system_time == 1526293795168
assert get_result.records[0].values == (1, 'yc1', 10.01, False,
1455869335000000)
assert get_result.records[0].attributes == {"string": "string"}
def test_put_data_record_with_unexisted_topic_name(self):
project_name = 'valid'
topic_name = 'unexisted'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
record = TupleRecord(schema=record_schema,
values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
try:
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/valid/topics/unexisted/shards'
content = json.loads(request.body)
assert content['Action'] == 'pub'
assert len(content['Records']) == 1
assert len(content['Records'][0]['Data']) == 5
assert content['Records'][0]['Data'][0] == '1'
assert content['Records'][0]['Data'][1] == 'yc1'
assert content['Records'][0]['Data'][2] == '1.001e+01'
assert content['Records'][0]['Data'][3] == 'true'
assert content['Records'][0]['Data'][4] == '1455869335000000'
with HTTMock(gen_mock_api(check)):
put_result = dh.put_records(project_name, topic_name, [record])
except ResourceNotFoundException:
pass
else:
raise Exception(
'put data record success with unexisted topic name!')
print("=======================================\n\n")
except ResourceExistException:
print("project already exist!")
print("=======================================\n\n")
except Exception as e:
print(traceback.format_exc())
sys.exit(-1)
# ===================== create topic =====================
# --------------------- wild_label topic ---------------------
tuple_topic = "smart_sync_infer"
shard_count = 1
life_cycle = 7
record_schema = RecordSchema.from_lists(
['object_id', 'ai_label', 'score'],
[FieldType.STRING, FieldType.STRING, FieldType.DOUBLE])
try:
dh.create_tuple_topic(project_name, tuple_topic, shard_count, life_cycle, record_schema, comment)
print("create traffic topic success!")
print("=======================================\n\n")
except ResourceExistException:
print("topic already exist!")
print("=======================================\n\n")
except Exception as e:
print(traceback.format_exc())
sys.exit(-1)
# ===================== get topic =====================
topic_result = dh.get_topic(project_name, tuple_topic)
def test_list_subscription(self):
project_name = "topic_test_p%d_4" % int(time.time())
topic_name = "topic_test_t%d_4" % int(time.time())
shard_count = 3
life_cycle = 7
record_type = RecordType.TUPLE
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name,
'comment')
assert create_result.sub_id
# ======================= query subscription state =======================
result = dh.list_subscription(project_name, topic_name, '', 1, 10)
print(result)
assert result.total_count == 1
subscription = result.subscriptions[0]
assert subscription.comment == 'comment'
assert subscription.create_time > 0
assert subscription.is_owner is True
assert subscription.last_modify_time > 0
assert subscription.state == SubscriptionState.ACTIVE
assert subscription.sub_id == create_result.sub_id
assert subscription.topic_name == topic_name
assert subscription.type == 0
try:
dh.list_subscription(project_name, topic_name, '', 0, 1)
except InvalidParameterException:
pass
else:
raise Exception(
'query subscription success with invalid page index')
try:
dh.list_subscription(project_name, topic_name, '', 1, -1)
except InvalidParameterException:
pass
else:
raise Exception(
'query subscription success with invalid page size')
dh.delete_subscription(project_name, topic_name,
create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)