def test_build_schema_with_invalid_type(self):
try:
record_schema_0 = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
'int', FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN,
FieldType.TIMESTAMP
])
except InvalidParameterException:
pass
else:
raise Exception('build schema success with wrong filed type!')
try:
record_schema_1 = RecordSchema()
record_schema_1.add_field(Field('string_field', 'str'))
except InvalidParameterException:
pass
else:
raise Exception('build schema success with wrong filed type!')
try:
fields = []
fields.append(Field('bigint_field', FieldType.BIGINT))
fields.append(Field('string_field', FieldType.STRING))
fields.append(Field('double_field', FieldType.DOUBLE))
fields.append(Field('bool_field', FieldType.BOOLEAN))
fields.append(Field('event_time1', 'time'))
except InvalidParameterException:
pass
else:
raise Exception('build schema success with wrong filed type!')
def test_update_topic_with_invalid_life_cycle(self):
project_name = 'valid'
topic_name = 'valid'
shard_count = 3
life_cycle = 0
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, 'comment')
except InvalidParameterException:
pass
else:
raise Exception('update success with invalid life cycle!')
try:
dh.create_blob_topic(project_name, topic_name, shard_count,
life_cycle, 'comment')
except InvalidParameterException:
pass
else:
raise Exception('update success with invalid life cycle!')
def test_list_topic(self):
project_name_0 = "topic_test_p%d_0" % int(time.time())
topic_name_0 = "topic_test_t%d_0" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name_0, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name_0, topic_name_0, shard_count, life_cycle, record_schema, '')
except ResourceExistException:
pass
# ======================= list topic =======================
result = dh.list_topic(project_name_0)
print(result)
assert topic_name_0 in result.topic_names
finally:
clean_topic(dh, project_name_0)
dh.delete_project(project_name_0)
def test_create_topic_already_existed(self):
project_name = 'existed'
topic_name = 'existed'
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
with HTTMock(datahub_api_mock):
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, 'comment')
except ResourceExistException:
pass
else:
raise Exception('create success with topic already existed!')
try:
with HTTMock(datahub_api_mock):
dh.create_blob_topic(project_name, topic_name, shard_count,
life_cycle, 'comment')
except ResourceExistException:
pass
else:
raise Exception('create success with topic already existed!')
def test_put_data_record_with_limit_exceeded(self):
project_name = 'put'
topic_name = 'limit_exceeded'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
record = TupleRecord(schema=record_schema,
values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
try:
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/put/topics/limit_exceeded/shards'
content = json.loads(request.body)
assert content['Action'] == 'pub'
assert len(content['Records']) == 1
assert len(content['Records'][0]['Data']) == 5
assert content['Records'][0]['Data'][0] == '1'
assert content['Records'][0]['Data'][1] == 'yc1'
assert content['Records'][0]['Data'][2] == '1.001e+01'
assert content['Records'][0]['Data'][3] == 'true'
assert content['Records'][0]['Data'][4] == '1455869335000000'
with HTTMock(gen_mock_api(check)):
put_result = dh.put_records(project_name, topic_name, [record])
except LimitExceededException:
pass
else:
raise Exception('put data record success with limit exceeded!')
def test_append_field(self):
project_name_6 = "topic_test_p%d_6" % int(time.time())
topic_name_1 = "topic_test_t%d_1" % int(time.time())
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name_6, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name_6, topic_name_1, 3, 7, record_schema, '1')
except ResourceExistException:
pass
time.sleep(1)
# ======================= append field =======================
dh.append_field(project_name_6, topic_name_1, 'append', FieldType.BOOLEAN)
finally:
clean_topic(dh, project_name_6)
dh.delete_project(project_name_6)
def test_list_shard(self):
project_name = "shard_test_p"
topic_name_0 = 'shard_test_t%d_0' % int(time.time())
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name_0, 3, 7, record_schema, '1')
except ResourceExistException:
pass
# ======================= list shard =======================
shard_results = dh.list_shard(project_name, topic_name_0)
print(shard_results)
assert len(shard_results.shards) == 3
assert shard_results.shards[0].left_shard_id != ''
assert shard_results.shards[0].right_shard_id != ''
finally:
clean_topic(dh, project_name)
dh.delete_project(project_name)
def test_create_topic_with_invalid_topic_name(self):
project_name = 'valid'
invalid_topic_names = ["", "1invalid", "_invalid", "!invalid",
"invalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalid"
"invalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalidinvalid"]
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
for invalid_topic_name in invalid_topic_names:
try:
dh.create_tuple_topic(project_name, invalid_topic_name, shard_count, life_cycle, record_schema,
'comment')
except InvalidParameterException:
pass
else:
raise Exception('create success with invalid topic name!')
for invalid_topic_name in invalid_topic_names:
try:
dh.create_blob_topic(project_name, invalid_topic_name, shard_count, life_cycle, 'comment')
except InvalidParameterException:
pass
else:
raise Exception('create success with invalid topic name!')
def test_put_tuple_record_pb_success(self):
project_name = 'put'
topic_name = 'success'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
records = []
record0 = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 253402271999000000])
record0.shard_id = '0'
record0.shard_id = '0'
records.append(record0)
record1 = TupleRecord(schema=record_schema,
values=[-9223372036854775808, 'yc1', 10.01, True, -62135798400000000])
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records.append(record1)
record2 = TupleRecord(schema=record_schema, values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000])
record2.partition_key = 'TestPartitionKey'
records.append(record2)
with HTTMock(datahub_pb_api_mock):
put_result = dh2.put_records(project_name, topic_name, records)
assert put_result.failed_record_count == 0
assert put_result.failed_records == []
def test_get_record_with_unexisted_shard_id(self):
project_name = 'valid'
topic_name = 'valid'
shard_id = '0'
limit_num = 10
cursor = '20000000000000000000000000fb0021'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/valid/topics/valid/shards/0'
content = json.loads(request.body)
assert content['Limit'] == 10
assert content['Action'] == 'sub'
assert content['Cursor'] == '20000000000000000000000000fb0021'
with HTTMock(gen_mock_api(check)):
get_result = dh.get_tuple_records(project_name, topic_name,
shard_id, record_schema,
cursor, limit_num)
except ResourceNotFoundException:
pass
else:
raise Exception('get data record success with unexisted shard id!')
def test_build_tuple_record_allow_null(self):
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
], [False, True, False, True, True])
try:
record0 = TupleRecord(
schema=record_schema,
values=[1, 'yc1', None, True, 253402271999000000])
except InvalidParameterException:
pass
else:
raise Exception(
'build record success with none value of field not allowed null'
)
record1 = TupleRecord(schema=record_schema)
try:
record1.set_value(0, None)
except InvalidParameterException:
pass
else:
raise Exception(
'set record success with none value of field not allowd null')
def test_create_and_delete_topic(self):
project_name_1 = "topic_test_p%d_1" % int(time.time())
topic_name_1 = "topic_test_t%d_1" % int(time.time())
topic_name_2 = "topic_test_t%d_2" % int(time.time())
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field', 'decimal_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP,
FieldType.DECIMAL])
try:
dh.create_project(project_name_1, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
# ======================= create topic =======================
try:
dh.create_tuple_topic(project_name_1, topic_name_1, 3, 7, record_schema, '1')
except ResourceExistException:
pass
try:
dh.create_blob_topic(project_name_1, topic_name_2, 3, 7, '2')
except ResourceExistException:
pass
finally:
# ======================= delete topic =======================
clean_topic(dh, project_name_1)
dh.delete_project(project_name_1)
def test_create_topic_success(self):
project_name = 'success'
topic_name = 'success'
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/success/topics/success'
content = json.loads(request.body)
assert content['Comment'] == 'comment'
assert content['Lifecycle'] == 7
assert content['ShardCount'] == 3
if content['RecordType'] == 'TUPLE':
schema = json.loads(content['RecordSchema'])
assert len(schema['fields']) == 5
assert schema['fields'][0]['type'] == 'bigint'
assert schema['fields'][0]['name'] == 'bigint_field'
assert schema['fields'][1]['type'] == 'string'
assert schema['fields'][1]['name'] == 'string_field'
assert schema['fields'][2]['type'] == 'double'
assert schema['fields'][2]['name'] == 'double_field'
assert schema['fields'][3]['type'] == 'boolean'
assert schema['fields'][3]['name'] == 'bool_field'
assert schema['fields'][4]['type'] == 'timestamp'
assert schema['fields'][4]['name'] == 'event_time1'
with HTTMock(gen_mock_api(check)):
dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, 'comment')
dh.create_blob_topic(project_name, topic_name, shard_count, life_cycle, 'comment')
def test_update_subscription_offset(self):
project_name = "offset_test_p%d_2" % int(time.time())
topic_name = "offset_test_t%d_2" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name, 'comment')
print(create_result)
sub_id = create_result.sub_id
# ======================= update subscription offset =======================
result = dh.init_and_get_subscription_offset(project_name, topic_name, sub_id, '0')
print(result)
offsets = result.offsets
offsets['0'].sequence += 1
offsets['0'].timestamp += 1
dh.update_subscription_offset(project_name, topic_name, sub_id, offsets)
offsets2 = {
'0': OffsetWithSession(
offsets['0'].sequence + 1,
offsets['0'].timestamp + 1,
offsets['0'].version,
offsets['0'].session_id
)
}
dh.update_subscription_offset(project_name, topic_name, sub_id, offsets2)
offsets3 = {
'0': {
'Sequence': 1,
'Timestamp': 1,
'Version': offsets['0'].version,
'SessionId': offsets['0'].session_id
}
}
dh.update_subscription_offset(project_name, topic_name, sub_id, offsets3)
dh.delete_subscription(project_name, topic_name, create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def test_get_cursor(self):
project_name = "cursor_test_p"
topic_name = "cursor_test_t%d_1" % int(time.time())
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, 3, 7,
record_schema, '1')
dh.wait_shards_ready(project_name, topic_name)
except ResourceExistException:
pass
# put tuple records
record = TupleRecord(
schema=record_schema,
values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
record.put_attribute('AK', '47')
records = [record for i in range(0, 3)]
put_record_result = dh.put_records(project_name, topic_name,
records)
print(put_record_result)
assert put_record_result.failed_record_count == 0
time.sleep(5)
# ======================= get cursor =======================
cursor_oldest = dh.get_cursor(project_name, topic_name, '0',
CursorType.OLDEST)
cursor_latest = dh.get_cursor(project_name, topic_name, '0',
CursorType.LATEST)
cursor_sequence_1 = dh.get_cursor(project_name, topic_name, '0',
CursorType.SEQUENCE, 0)
cursor_sequence_2 = dh.get_cursor(project_name, topic_name, '0',
CursorType.SEQUENCE, 2)
cursor_system_time = dh.get_cursor(project_name, topic_name, '0',
CursorType.SYSTEM_TIME, 0)
print(cursor_system_time)
# assert cursor_oldest.cursor == cursor_sequence_1.cursor
# assert cursor_latest.cursor == cursor_sequence_2.cursor
# assert cursor_oldest.cursor == cursor_system_time.cursor
finally:
clean_topic(dh, project_name)
dh.delete_project(project_name)
def test_get_subscription_offset(self):
project_name = "offset_test_p%d_1" % int(time.time())
topic_name = "offset_test_t%d_1" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count, life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name, 'comment')
print(create_result)
sub_id = create_result.sub_id
# ======================= get subscription =======================
dh.init_and_get_subscription_offset(project_name, topic_name, sub_id, '0')
result = dh.get_subscription_offset(project_name, topic_name, sub_id, '0')
print(result)
assert result.offsets['0'].sequence >= 0
assert result.offsets['0'].timestamp >= 0
assert result.offsets['0'].version >= 0
result = dh.get_subscription_offset(project_name, topic_name, sub_id, ['0'])
print(result)
assert result.offsets['0'].sequence >= 0
assert result.offsets['0'].timestamp >= 0
assert result.offsets['0'].version >= 0
result = dh.get_subscription_offset(project_name, topic_name, sub_id)
print(result)
assert result.offsets['0'].sequence >= 0
assert result.offsets['0'].timestamp >= 0
assert result.offsets['0'].version >= 0
result = dh.get_subscription_offset(project_name, topic_name, sub_id, [])
print(result)
assert result.offsets['0'].sequence >= 0
assert result.offsets['0'].timestamp >= 0
assert result.offsets['0'].version >= 0
dh.delete_subscription(project_name, topic_name, create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def test_get_exist_topic(self):
project_name_2 = "topic_test_p%d_3" % int(time.time())
topic_name_1 = "topic_test_t%d_1" % int(time.time())
topic_name_2 = "topic_test_t%d_2" % int(time.time())
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field', 'decimal_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP, FieldType.DECIMAL
])
try:
dh.create_project(project_name_2, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name_2, topic_name_1, 3, 7,
record_schema, '1')
except ResourceExistException:
pass
try:
dh.create_blob_topic(project_name_2, topic_name_2, 3, 7, '2')
except ResourceExistException:
pass
# ======================= get topic =======================
topic_result_1 = dh.get_topic(project_name_2, topic_name_1)
print(topic_result_1)
assert topic_result_1.project_name == project_name_2
assert topic_result_1.topic_name == topic_name_1
assert topic_result_1.comment == '1'
assert topic_result_1.life_cycle == 7
assert topic_result_1.record_type == RecordType.TUPLE
for index, field in enumerate(
topic_result_1.record_schema.field_list):
assert field.name == record_schema.field_list[index].name
assert field.type == record_schema.field_list[index].type
assert field.allow_null == record_schema.field_list[
index].allow_null
assert topic_result_1.shard_count == 3
topic_result_2 = dh.get_topic(project_name_2, topic_name_2)
print(topic_result_2)
assert topic_result_2.topic_name == topic_name_2
assert topic_result_2.project_name == project_name_2
assert topic_result_2.comment == '2'
assert topic_result_2.life_cycle == 7
assert topic_result_2.record_type == RecordType.BLOB
assert topic_result_2.shard_count == 3
finally:
clean_topic(dh, project_name_2)
dh.delete_project(project_name_2)
def test_split_merge_shard(self):
project_name = "shard_test_p"
topic_name_1 = 'shard_test_t%d_1' % int(time.time())
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name_1, 1, 7, record_schema, '1')
except ResourceExistException:
pass
time.sleep(5)
# ======================= split shard =======================
new_shards = dh.split_shard(project_name, topic_name_1, '0',
'66666666666666666666666666666666').new_shards
assert new_shards[0].shard_id == '1'
assert new_shards[1].shard_id == '2'
assert new_shards[0].end_hash_key == '66666666666666666666666666666666'
assert new_shards[1].begin_hash_key == '66666666666666666666666666666666'
dh.wait_shards_ready(project_name, topic_name_1)
shard_list = dh.list_shard(project_name, topic_name_1).shards
print(shard_list)
for shard in shard_list:
if shard.shard_id == '0':
assert shard.state == ShardState.CLOSED
else:
assert shard.state == ShardState.ACTIVE
time.sleep(5)
# ======================= merge shard =======================
merge_result = dh.merge_shard(project_name, topic_name_1, '1', '2')
print(merge_result)
assert merge_result.shard_id == '3'
dh.wait_shards_ready(project_name, topic_name_1)
shard_list = dh.list_shard(project_name, topic_name_1).shards
print(shard_list)
for shard in shard_list:
if shard.shard_id in ('0', '1', '2'):
assert shard.state == ShardState.CLOSED
else:
assert shard.state == ShardState.ACTIVE
finally:
clean_topic(dh, project_name)
dh.delete_project(project_name)
def test_build_record_with_invalid_value(self):
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
record = TupleRecord(schema=record_schema, values=['a', 'yc1', 10.01, True, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema,
values=[-9223372036854775809, 'yc1', 10.01, True, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema,
values=[9223372036854775808, 'yc1', 10.01, True, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema, values=['1', 'yc1', 'a', True, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, 2, 1455869335000000])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, -62135798400000001])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
try:
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, -253402271999000001])
except InvalidParameterException:
pass
else:
raise Exception('build record success with invalid value!')
def test_update_subscription_state(self):
project_name = "topic_test_p%d_3" % int(time.time())
topic_name = "topic_test_t%d_3" % int(time.time())
shard_count = 3
life_cycle = 7
record_type = RecordType.TUPLE
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name,
'comment')
assert create_result.sub_id
# ======================= update subscription state =======================
dh.update_subscription_state(project_name, topic_name,
create_result.sub_id,
SubscriptionState.INACTIVE)
result = dh.get_subscription(project_name, topic_name,
create_result.sub_id)
assert result.state == SubscriptionState.INACTIVE
dh.update_subscription_state(project_name, topic_name,
create_result.sub_id,
SubscriptionState.ACTIVE)
result = dh.get_subscription(project_name, topic_name,
create_result.sub_id)
assert result.state == SubscriptionState.ACTIVE
dh.delete_subscription(project_name, topic_name,
create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def test_get_subscription(self):
project_name = "topic_test_p%d_1" % int(time.time())
topic_name = "topic_test_t%d_1" % int(time.time())
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, '')
except ResourceExistException:
pass
create_result = dh.create_subscription(project_name, topic_name,
'comment')
# ======================= get subscription =======================
result = dh.get_subscription(project_name, topic_name,
create_result.sub_id)
print(result)
assert result.comment == 'comment'
assert result.create_time > 0
assert result.is_owner is True
assert result.last_modify_time > 0
assert result.state == SubscriptionState.ACTIVE
assert result.sub_id == create_result.sub_id
assert result.topic_name == topic_name
assert result.type == 0
dh.delete_subscription(project_name, topic_name,
create_result.sub_id)
finally:
clean_topic(dh, project_name, True)
dh.delete_project(project_name)
def test_put_data_record_with_empty_topic_name(self):
project_name = 'valid'
topic_name = ''
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
try:
put_result = dh.put_records(project_name, topic_name, [record])
except InvalidParameterException:
pass
else:
raise Exception('put data record success with empty topic name!')
def test_update_topic(self):
project_name = "topic_test_t"
topic_name_1 = "topic_test_t%d_1" % int(time.time())
topic_name_2 = "topic_test_t%d_2" % int(time.time())
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
try:
dh.create_project(project_name, '')
except ResourceExistException:
pass
# make sure project wil be deleted
try:
try:
dh.create_tuple_topic(project_name, topic_name_1, 3, 7,
record_schema, '1')
except ResourceExistException:
pass
try:
dh.create_blob_topic(project_name, topic_name_2, 3, 7, '2')
except ResourceExistException:
pass
time.sleep(1)
# ======================= update topic =======================
dh.update_topic(project_name, topic_name_1, 7, "2")
topic_result_1 = dh.get_topic(project_name, topic_name_1)
assert topic_result_1.life_cycle == 7
assert topic_result_1.comment == "2"
try:
dh.update_topic(project_name, topic_name_2, 5, "new comment")
except LimitExceededException:
pass
# topic_result_2 = dh.get_topic(project_name, topic_name_2)
# assert topic_result_2.comment == "new comment"
finally:
clean_topic(dh, project_name)
dh.delete_project(project_name)
def test_put_malformed_tuple_record(self):
project_name = 'put'
topic_name = 'malformed'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
try:
with HTTMock(datahub_api_mock):
put_result = dh.put_records(project_name, topic_name, [record])
except InvalidParameterException:
pass
else:
raise Exception('put malformed tuple record success!')
def test_put_data_record_with_unexisted_topic_name(self):
project_name = 'valid'
topic_name = 'unexisted'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
record = TupleRecord(schema=record_schema, values=[1, 'yc1', 10.01, True, 1455869335000000])
record.shard_id = '0'
try:
with HTTMock(datahub_api_mock):
put_result = dh.put_records(project_name, topic_name, [record])
except ResourceNotFoundException:
pass
else:
raise Exception('put data record success with unexisted topic name!')
def test_get_record_with_empty_shard_id(self):
project_name = 'valid'
topic_name = 'valid'
shard_id = ''
limit_num = 10
cursor = '20000000000000000000000000fb0021'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num)
except InvalidParameterException:
pass
else:
raise Exception('get data record success with empty shard id!')
def test_get_record_with_unexisted_shard_id(self):
project_name = 'valid'
topic_name = 'valid'
shard_id = '0'
limit_num = 10
cursor = '20000000000000000000000000fb0021'
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'time_field'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
try:
with HTTMock(datahub_api_mock):
get_result = dh.get_tuple_records(project_name, topic_name, shard_id, record_schema, cursor, limit_num)
except ResourceNotFoundException:
pass
else:
raise Exception('get data record success with unexisted shard id!')
def test_create_topic_success(self):
project_name = 'success'
topic_name = 'success'
shard_count = 3
life_cycle = 7
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'event_time1'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
with HTTMock(datahub_api_mock):
dh.create_tuple_topic(project_name, topic_name, shard_count,
life_cycle, record_schema, 'comment')
dh.create_blob_topic(project_name, topic_name, shard_count,
life_cycle, 'comment')
def test_get_topic_success(self):
project_name = 'success'
topic_name = 'tuple'
def check(request):
assert request.method == 'GET'
assert request.url == 'http://endpoint/projects/success/topics/tuple'
with HTTMock(gen_mock_api(check)):
tuple_topic_result = dh.get_topic(project_name, topic_name)
assert tuple_topic_result.project_name == project_name
assert tuple_topic_result.topic_name == topic_name
assert tuple_topic_result.comment == 'tuple'
assert tuple_topic_result.shard_count == 3
assert tuple_topic_result.life_cycle == 7
assert tuple_topic_result.record_type == RecordType.TUPLE
assert tuple_topic_result.create_time == 1525312823
assert tuple_topic_result.last_modify_time == 1525312823
record_schema = RecordSchema.from_lists(
['bigint_field', 'string_field', 'double_field', 'bool_field', 'event_time1'],
[FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE, FieldType.BOOLEAN, FieldType.TIMESTAMP])
for index in range(0, len(record_schema.field_list)):
assert record_schema.field_list[index].name == tuple_topic_result.record_schema.field_list[index].name
assert record_schema.field_list[index].type == tuple_topic_result.record_schema.field_list[index].type
assert record_schema.field_list[index].allow_null == tuple_topic_result.record_schema.field_list[
index].allow_null
topic_name = 'blob'
def check(request):
assert request.method == 'GET'
assert request.url == 'http://endpoint/projects/success/topics/blob'
with HTTMock(gen_mock_api(check)):
blob_topic_result = dh.get_topic(project_name, topic_name)
assert blob_topic_result.project_name == project_name
assert blob_topic_result.topic_name == topic_name
assert blob_topic_result.comment == 'blob'
assert blob_topic_result.shard_count == 3
assert blob_topic_result.life_cycle == 7
assert blob_topic_result.record_type == RecordType.BLOB
assert blob_topic_result.create_time == 1525344044
assert blob_topic_result.last_modify_time == 1525344044
def test_put_tuple_record_success(self):
project_name = 'put'
topic_name = 'success'
record_schema = RecordSchema.from_lists([
'bigint_field', 'string_field', 'double_field', 'bool_field',
'time_field'
], [
FieldType.BIGINT, FieldType.STRING, FieldType.DOUBLE,
FieldType.BOOLEAN, FieldType.TIMESTAMP
])
records = []
record0 = TupleRecord(
schema=record_schema,
values=[1, 'yc1', 10.01, True, 253402271999000000])
record0.shard_id = '0'
record0.shard_id = '0'
records.append(record0)
record1 = TupleRecord(schema=record_schema)
record1.values = [
-9223372036854775808, 'yc1', 10.01, True, -62135798400000000
]
record1.hash_key = '4FFFFFFFFFFFFFFD7FFFFFFFFFFFFFFD'
records.append(record1)
record2 = TupleRecord(
schema=record_schema,
values=[9223372036854775807, 'yc1', 10.01, True, 1455869335000000])
record2.set_value(0, 9223372036854775807)
record2.set_value('string_field', 'yc1')
record2.partition_key = 'TestPartitionKey'
records.append(record2)
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/put/topics/success/shards'
with HTTMock(gen_mock_api(check)):
put_result = dh.put_records(project_name, topic_name, records)
assert put_result.failed_record_count == 0
assert put_result.failed_records == []
