def test_update_connector(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"),
("mm", "%M")])
connector_config = OdpsConnectorConfig(odps_project_name,
system_time_table_name,
odps_endpoint, tunnel_endpoint,
odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15,
partition_config)
try:
dh.create_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config)
except LimitExceededException:
pass
time.sleep(1)
new_odps_project_name = "1"
new_system_time_table_name = "2"
new_odps_endpoint = "3"
new_tunnel_endpoint = "4"
new_odps_access_id = "5"
new_odps_access_key = "6"
new_partition_config = OrderedDict([("pt", "%Y%m%d"), ("ct", "%H%M")])
new_connector_config = OdpsConnectorConfig(
new_odps_project_name, new_system_time_table_name,
new_odps_endpoint, new_tunnel_endpoint, new_odps_access_id,
new_odps_access_key, PartitionMode.USER_DEFINE, 30,
new_partition_config)
dh.update_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS,
new_connector_config)
config = dh.get_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS).config
assert config.project_name == new_odps_project_name
assert config.table_name == new_system_time_table_name
assert config.odps_endpoint == new_odps_endpoint
assert config.tunnel_endpoint == new_tunnel_endpoint
assert config.partition_mode == PartitionMode.USER_DEFINE
for k, v in new_partition_config.items():
assert config.partition_config.get(k) == v
dh.delete_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS)
def test_create_connector_success(self):
project_name = 'success'
topic_name = 'success'
column_fields = ['field_a', 'field_b']
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"),
("mm", "%M")])
connector_config = OdpsConnectorConfig(
'connector_project_name', 'connector_table_name', 'dps_endpoint',
'tunnel_endpoint', 'connector_access_id', 'connector_access_key',
PartitionMode.SYSTEM_TIME, 0, partition_config)
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/success/topics/success/connectors/sink_odps'
content = json.loads(request.body)
print(request.body)
assert content['Action'] == 'Create'
assert content['ColumnFields'] == column_fields
assert content['SinkStartTime'] == -1
assert content['Config'][
'PartitionMode'] == PartitionMode.SYSTEM_TIME.value
assert content['Config']['Project'] == 'connector_project_name'
assert content['Config']['TunnelEndpoint'] == 'tunnel_endpoint'
assert content['Config']['Table'] == 'connector_table_name'
assert content['Config']['OdpsEndpoint'] == 'dps_endpoint'
assert content['Config']['AccessId'] == 'connector_access_id'
assert content['Config']['AccessKey'] == 'connector_access_key'
assert content['Config']['PartitionConfig']['ds'] == '%Y%m%d'
assert content['Config']['PartitionConfig']['hh'] == '%H'
assert content['Config']['PartitionConfig']['mm'] == '%M'
with HTTMock(gen_mock_api(check)):
dh.create_connector(project_name, topic_name,
ConnectorType.SINK_ODPS, column_fields,
connector_config)
def test_reload_connector(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"),
("mm", "%M")])
connector_config = OdpsConnectorConfig(odps_project_name,
system_time_table_name,
odps_endpoint, tunnel_endpoint,
odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15,
partition_config)
try:
dh.create_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config)
except LimitExceededException:
pass
dh.reload_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS,
'0')
dh.reload_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS)
dh.delete_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS)
def test_get_connector_done_time(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"),
("mm", "%M")])
connector_config = OdpsConnectorConfig(odps_project_name,
system_time_table_name,
odps_endpoint, tunnel_endpoint,
odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15,
partition_config)
try:
dh.create_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config)
except LimitExceededException:
pass
time.sleep(1)
result = dh.get_connector_done_time(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS)
print(result)
assert result.done_time >= 0
assert result.time_zone == 'Asia/Shanghai'
assert result.time_window == 900
dh.delete_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS)
def test_append_connector_field(self):
connector_config = OdpsConnectorConfig(odps_project_name,
user_define_table_name,
odps_endpoint, tunnel_endpoint,
odps_access_id, odps_access_key,
PartitionMode.USER_DEFINE, 15,
{})
try:
dh.create_connector(connector_test_project_name,
user_define_topic_name,
ConnectorType.SINK_ODPS, user_define_fields,
connector_config)
except LimitExceededException:
pass
connector_result = dh.get_connector(connector_test_project_name,
user_define_topic_name,
ConnectorType.SINK_ODPS)
assert connector_result.column_fields == user_define_fields
dh.append_connector_field(connector_test_project_name,
user_define_topic_name,
ConnectorType.SINK_ODPS,
str(odps_connector_column_fields[1]))
connector_result = dh.get_connector(connector_test_project_name,
user_define_topic_name,
ConnectorType.SINK_ODPS)
assert connector_result.column_fields == user_define_fields + odps_connector_column_fields[
1:2]
dh.delete_connector(connector_test_project_name,
user_define_topic_name, ConnectorType.SINK_ODPS)
def test_get_odps_connector(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"),
("mm", "%M")])
connector_config = OdpsConnectorConfig(odps_project_name,
system_time_table_name,
odps_endpoint, tunnel_endpoint,
odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15,
partition_config)
try:
dh.create_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config)
except LimitExceededException:
pass
result = dh.get_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS)
print(result)
print(result.config)
print(result.config.partition_config)
assert result.type == ConnectorType.SINK_ODPS
assert result.column_fields == odps_connector_column_fields
assert result.state == ConnectorState.CONNECTOR_RUNNING or result.state == ConnectorState.CONNECTOR_CREATED
assert result.config.project_name == odps_project_name
assert result.config.table_name == system_time_table_name
assert result.config.odps_endpoint == odps_endpoint
assert result.config.tunnel_endpoint == tunnel_endpoint
dh.delete_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS)
def test_get_odps_connector(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"), ("mm", "%M")])
connector_config = OdpsConnectorConfig(odps_project_name, system_time_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15, partition_config)
try:
dh.create_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config)
except ResourceExistException:
pass
result = dh.get_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS)
print(result)
print(result.config)
print(result.config.partition_config)
assert result.type == ConnectorType.SINK_ODPS
assert result.column_fields == odps_connector_column_fields
assert result.state == ConnectorState.CONNECTOR_RUNNING
assert result.config.project_name == odps_project_name
assert result.config.table_name == system_time_table_name
assert result.config.odps_endpoint == odps_endpoint
assert result.config.tunnel_endpoint == tunnel_endpoint
for shard_context in result.shard_contexts:
assert shard_context.shard_id != ''
assert shard_context.current_sequence >= -1
dh.delete_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS)
def test_create_connector_success(self):
project_name = 'success'
topic_name = 'success'
column_fields = ['field_a', 'field_b']
connector_config = OdpsConnectorConfig('onnector_project_name', 'connector_table_name', 'dps_endpoint',
'tunnel_endpoint', 'connector_access_id', 'connector_access_key',
PartitionMode.SYSTEM_TIME, 0, {})
with HTTMock(datahub_api_mock):
dh.create_connector(project_name, topic_name, ConnectorType.SINK_ODPS, column_fields, connector_config)
def test_create_odps_connector(self):
partition_config = OrderedDict([
("ds", "%Y%m%d"),
("hh", "%H"),
("mm", "%M")
])
# system time mode
connector_config_0 = OdpsConnectorConfig(odps_project_name, system_time_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15, partition_config)
try:
dh.create_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config_0)
except ResourceExistException:
pass
dh.delete_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS)
# event time mode
connector_config_1 = OdpsConnectorConfig(odps_project_name, event_time_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.EVENT_TIME, 15, partition_config)
try:
dh.create_connector(connector_test_project_name, event_time_topic_name, ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config_1)
except ResourceExistException:
pass
dh.delete_connector(connector_test_project_name, event_time_topic_name, ConnectorType.SINK_ODPS)
# user define mode
connector_config_2 = OdpsConnectorConfig(odps_project_name, user_define_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.USER_DEFINE, 15, {})
try:
dh.create_connector(connector_test_project_name, user_define_topic_name, ConnectorType.SINK_ODPS,
odps_connector_column_fields + user_define_fields, connector_config_2)
except ResourceExistException:
pass
dh.delete_connector(connector_test_project_name, user_define_topic_name, ConnectorType.SINK_ODPS)
def test_append_connector_field(self):
connector_config = OdpsConnectorConfig(odps_project_name, user_define_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.USER_DEFINE, 15, {})
try:
dh.create_connector(connector_test_project_name, user_define_topic_name, ConnectorType.SINK_ODPS,
user_define_fields, connector_config)
except ResourceExistException:
pass
dh.append_connector_field(connector_test_project_name, user_define_topic_name, ConnectorType.SINK_ODPS,
odps_connector_column_fields[1])
dh.delete_connector(connector_test_project_name, user_define_topic_name, ConnectorType.SINK_ODPS)
def test_create_connector_with_empty_project_name(self):
project_name = ''
topic_name = 'valid'
column_fields = ['field_a', 'field_b']
connector_config = OdpsConnectorConfig('onnector_project_name', 'connector_table_name', 'dps_endpoint',
'tunnel_endpoint', 'connector_access_id', 'connector_access_key',
PartitionMode.SYSTEM_TIME, 0, {})
try:
dh.create_connector(project_name, topic_name, ConnectorType.SINK_ODPS, column_fields, connector_config)
except InvalidParameterException:
pass
else:
raise Exception('create data connector success with empty project name!')
def test_create_connector_with_invalid_parameter(self):
project_name = 'valid'
topic_name = 'valid'
column_fields = ['field_a', 'field_b', 'invalid']
connector_config = OdpsConnectorConfig('onnector_project_name', 'connector_table_name_invalid', 'dps_endpoint',
'tunnel_endpoint', 'connector_access_id', 'connector_access_key',
PartitionMode.SYSTEM_TIME, 0, {})
try:
with HTTMock(datahub_api_mock):
dh.create_connector(project_name, topic_name, ConnectorType.SINK_ODPS,
column_fields, connector_config)
except InvalidParameterException:
pass
else:
raise Exception('create data connector success with unexisted project name!')
def test_update_connector_state(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"), ("mm", "%M")])
connector_config = OdpsConnectorConfig(odps_project_name, system_time_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15, partition_config)
try:
dh.create_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config)
except ResourceExistException:
pass
states = [ConnectorState.CONNECTOR_RUNNING, ConnectorState.CONNECTOR_PAUSED]
for state in states:
dh.update_connector_state(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS,
state)
time.sleep(5)
dh.delete_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS)
def test_get_connector_shard_status(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"), ("mm", "%M")])
connector_config = OdpsConnectorConfig(odps_project_name, system_time_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15, partition_config)
try:
dh.create_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config)
except ResourceExistException:
pass
result = dh.get_connector_shard_status(connector_test_project_name, system_time_topic_name,
ConnectorType.SINK_ODPS, '0')
print(result)
assert result.start_sequence >= 0
assert result.end_sequence > 0
assert result.current_sequence >= -1
assert result.record_time >= 0
assert result.update_time > 0
assert result.state == ConnectorShardStatus.CONTEXT_PLANNED
dh.delete_connector(connector_test_project_name, system_time_topic_name, ConnectorType.SINK_ODPS)
def test_create_connector_with_invalid_parameter(self):
project_name = 'valid'
topic_name = 'valid'
column_fields = ['field_a', 'field_b', 'invalid']
connector_config = OdpsConnectorConfig(
'connector_project_name', 'connector_table_name_invalid',
'dps_endpoint', 'tunnel_endpoint', 'connector_access_id',
'connector_access_key', PartitionMode.SYSTEM_TIME, 0, {})
try:
def check(request):
assert request.method == 'POST'
assert request.url == 'http://endpoint/projects/valid/topics/valid/connectors/sink_odps'
content = json.loads(request.body)
assert content['Action'] == 'Create'
assert content['ColumnFields'] == column_fields
assert content['SinkStartTime'] == -1
assert content['Config'][
'PartitionMode'] == PartitionMode.SYSTEM_TIME.value
assert content['Config']['Project'] == 'connector_project_name'
assert content['Config']['TunnelEndpoint'] == 'tunnel_endpoint'
assert content['Config'][
'Table'] == 'connector_table_name_invalid'
assert content['Config']['OdpsEndpoint'] == 'dps_endpoint'
assert content['Config']['AccessId'] == 'connector_access_id'
assert content['Config']['AccessKey'] == 'connector_access_key'
with HTTMock(gen_mock_api(check)):
dh.create_connector(project_name, topic_name,
ConnectorType.SINK_ODPS, column_fields,
connector_config)
except InvalidParameterException:
pass
else:
raise Exception(
'create data connector success with unexisted project name!')
def test_update_connector_state(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"),
("mm", "%M")])
connector_config = OdpsConnectorConfig(odps_project_name,
system_time_table_name,
odps_endpoint, tunnel_endpoint,
odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15,
partition_config)
try:
dh.create_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config)
except LimitExceededException:
pass
connector_result = dh.get_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS)
assert connector_result.state == ConnectorState.CONNECTOR_RUNNING
dh.update_connector_state(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
ConnectorState.CONNECTOR_STOPPED)
connector_result = dh.get_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS)
assert connector_result.state == ConnectorState.CONNECTOR_STOPPED
dh.update_connector_offset(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS, '0',
ConnectorOffset(1999, 1582801630000))
shard_result = dh.get_connector_shard_status(
connector_test_project_name, system_time_topic_name,
ConnectorType.SINK_ODPS)
assert shard_result.shard_status_infos['0'].current_sequence == 1999
assert shard_result.shard_status_infos[
'0'].current_timestamp == 1582801630000
dh.update_connector_offset(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS, '',
ConnectorOffset(2999, 1582801630001))
shard_result = dh.get_connector_shard_status(
connector_test_project_name, system_time_topic_name,
ConnectorType.SINK_ODPS)
assert shard_result.shard_status_infos['0'].current_sequence == 2999
assert shard_result.shard_status_infos[
'0'].current_timestamp == 1582801630001
assert shard_result.shard_status_infos['1'].current_sequence == 2999
assert shard_result.shard_status_infos[
'1'].current_timestamp == 1582801630001
dh.update_connector_state(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
ConnectorState.CONNECTOR_RUNNING)
connector_result = dh.get_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS)
assert connector_result.state == ConnectorState.CONNECTOR_RUNNING
dh.delete_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS)
def test_get_connector_shard_status(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"),
("mm", "%M")])
now = time.time()
connector_config = OdpsConnectorConfig(odps_project_name,
system_time_table_name,
odps_endpoint, tunnel_endpoint,
odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15,
partition_config)
try:
dh.create_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields, connector_config,
10000000)
except LimitExceededException:
pass
result = dh.get_connector_shard_status(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS, '0')
print(result)
assert len(result.shard_status_infos) == 1
assert '0' in result.shard_status_infos
assert result.shard_status_infos['0'].current_sequence >= -1
assert result.shard_status_infos['0'].current_timestamp == 10000000
assert result.shard_status_infos['0'].done_time >= 0
assert result.shard_status_infos['0'].last_error_message == 'Loading'
assert result.shard_status_infos[
'0'].state == ConnectorShardStatus.CONTEXT_PLANNED
assert result.shard_status_infos['0'].update_time >= now - 1
assert result.shard_status_infos['0'].discard_count >= 0
result = dh.get_connector_shard_status(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS)
print(result)
assert len(result.shard_status_infos) == 2
assert '0' in result.shard_status_infos
assert result.shard_status_infos['0'].current_sequence >= -1
assert result.shard_status_infos['0'].current_timestamp == 10000000
assert result.shard_status_infos['0'].done_time >= 0
assert result.shard_status_infos['0'].last_error_message == 'Loading'
assert result.shard_status_infos[
'0'].state == ConnectorShardStatus.CONTEXT_PLANNED
assert result.shard_status_infos['0'].update_time >= now - 1
assert result.shard_status_infos['0'].discard_count >= 0
assert '1' in result.shard_status_infos
assert result.shard_status_infos['0'].current_sequence >= -1
assert result.shard_status_infos['0'].current_timestamp == 10000000
assert result.shard_status_infos['0'].done_time >= 0
assert result.shard_status_infos['0'].last_error_message == 'Loading'
assert result.shard_status_infos[
'0'].state == ConnectorShardStatus.CONTEXT_PLANNED
assert result.shard_status_infos['0'].update_time >= now - 1
assert result.shard_status_infos['0'].discard_count >= 0
dh.delete_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS)
def test_create_odps_connector(self):
partition_config = OrderedDict([("ds", "%Y%m%d"), ("hh", "%H"),
("mm", "%M")])
# system time mode
connector_config_0 = OdpsConnectorConfig(
odps_project_name, system_time_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.SYSTEM_TIME, 15, partition_config)
try:
dh.delete_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS)
except ResourceNotFoundException:
pass
result = dh.create_connector(connector_test_project_name,
system_time_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields,
connector_config_0)
print('create odps connector SYSTEM_TIME: ', result)
connector_id = result.connector_id
list_result = dh.list_connector(connector_test_project_name,
system_time_topic_name)
print('list connector: ', list_result)
assert len(list_result.connector_names) == 1
assert len(list_result.connector_ids) == 1
assert list_result.connector_ids[0] == connector_id
connector_result = dh.get_connector(connector_test_project_name,
system_time_topic_name,
connector_id)
print('get odps connector: ', connector_result)
assert connector_result.connector_id == connector_id
assert connector_result.sub_id != ''
assert connector_result.type == ConnectorType.SINK_ODPS
assert connector_result.create_time > 0
assert connector_result.cluster_addr != ''
assert connector_result.state == ConnectorState.CONNECTOR_RUNNING
assert connector_result.column_fields == odps_connector_column_fields
assert connector_result.config.project_name == odps_project_name
assert connector_result.config.table_name == system_time_table_name
assert connector_result.config.odps_endpoint == odps_endpoint
assert connector_result.config.tunnel_endpoint == tunnel_endpoint
assert connector_result.config.partition_mode == PartitionMode.SYSTEM_TIME
assert connector_result.config.time_range == 15
assert connector_result.config.partition_config == partition_config
assert 'SubscriptionId' in connector_result.extra_config
dh.delete_connector(connector_test_project_name,
system_time_topic_name, ConnectorType.SINK_ODPS)
# event time mode
connector_config_1 = OdpsConnectorConfig(
odps_project_name, event_time_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.EVENT_TIME, 15, partition_config)
try:
dh.delete_connector(connector_test_project_name,
event_time_topic_name, ConnectorType.SINK_ODPS)
except ResourceNotFoundException:
pass
result = dh.create_connector(connector_test_project_name,
event_time_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields,
connector_config_1)
print('create odps connector EVENT_TIME: ', result)
connector_id = result.connector_id
connector_result = dh.get_connector(connector_test_project_name,
event_time_topic_name,
connector_id)
print('get odps connector: ', connector_result)
assert connector_result.connector_id == connector_id
assert connector_result.sub_id != ''
assert connector_result.type == ConnectorType.SINK_ODPS
assert connector_result.create_time > 0
assert connector_result.cluster_addr != ''
assert connector_result.state == ConnectorState.CONNECTOR_RUNNING
assert connector_result.column_fields == odps_connector_column_fields
assert connector_result.config.project_name == odps_project_name
assert connector_result.config.table_name == event_time_table_name
assert connector_result.config.odps_endpoint == odps_endpoint
assert connector_result.config.tunnel_endpoint == tunnel_endpoint
assert connector_result.config.partition_mode == PartitionMode.EVENT_TIME
assert connector_result.config.time_range == 15
assert connector_result.config.partition_config == partition_config
assert 'SubscriptionId' in connector_result.extra_config
dh.delete_connector(connector_test_project_name, event_time_topic_name,
ConnectorType.SINK_ODPS)
# user define mode
connector_config_2 = OdpsConnectorConfig(
odps_project_name, user_define_table_name, odps_endpoint,
tunnel_endpoint, odps_access_id, odps_access_key,
PartitionMode.USER_DEFINE, 15)
try:
dh.delete_connector(connector_test_project_name,
user_define_topic_name,
ConnectorType.SINK_ODPS)
except ResourceNotFoundException:
pass
result = dh.create_connector(
connector_test_project_name, user_define_topic_name,
ConnectorType.SINK_ODPS,
odps_connector_column_fields + user_define_fields,
connector_config_2, 100)
print('create odps connector USER_DEFINE: ', result)
connector_id = result.connector_id
connector_result = dh.get_connector(connector_test_project_name,
user_define_topic_name,
ConnectorType.SINK_ODPS)
print('get odps connector: ', connector_result)
assert connector_result.connector_id == connector_id
assert connector_result.sub_id != ''
assert connector_result.type == ConnectorType.SINK_ODPS
assert connector_result.create_time > 0
assert connector_result.cluster_addr != ''
assert connector_result.column_fields == odps_connector_column_fields + user_define_fields
assert connector_result.config.project_name == odps_project_name
assert connector_result.config.table_name == user_define_table_name
assert connector_result.config.odps_endpoint == odps_endpoint
assert connector_result.config.tunnel_endpoint == tunnel_endpoint
assert connector_result.config.partition_mode == PartitionMode.USER_DEFINE
assert connector_result.config.time_range == 15
dh.delete_connector(connector_test_project_name,
user_define_topic_name, ConnectorType.SINK_ODPS)