def test_get_tables_from_properties(self):
"""Test getting selected tables from tap properties JSON"""
# Load MySQL and Postgres properties JSON
mysql_properties = utils.load_json(
'{}/properties_mysql.json'.format(RESOURCES_DIR))
postgres_properties = utils.load_json(
'{}/properties_postgres.json'.format(RESOURCES_DIR))
# Get list of selected tables
# MySQL and Postgres schemas defined at different keys. get_tables_from_properties function
# should detect and extract correctly
mysql_tables = utils.get_tables_from_properties(mysql_properties)
postgres_tables = utils.get_tables_from_properties(postgres_properties)
# MySQL schema
assert mysql_tables == \
[
'mysql_source_db.address',
'mysql_source_db.order',
'mysql_source_db.weight_unit'
]
assert postgres_tables == \
[
'public.city',
'public.country'
]
def test_get_bookmark_for_table_postgresl(self):
"""Test bookmark extractors for Postgres taps"""
# Load Postgres properties JSON
postgres_properties = utils.load_json(
'{}/properties_postgres.json'.format(RESOURCES_DIR))
# Postgres: public.countrylanguage is LOG_BASED
assert utils.get_bookmark_for_table('public.countrylanguage',
postgres_properties,
PostgresMock()) == {
'lsn': '16/B374D848',
'version': 1
}
# Postgres: postgres_source_db.public.city is INCREMENTAL
assert utils.get_bookmark_for_table('public.city',
postgres_properties,
PostgresMock(),
dbname='postgres_source_db') == {
'replication_key': 'id',
'replication_key_value':
123456,
'version': 1
}
# Postgres: postgres_source_db.public.foo not exists
assert utils.get_bookmark_for_table('public.foo',
postgres_properties,
PostgresMock(),
dbname='postgres_source_db') == {}
def test_get_bookmark_for_table_mysql(self):
"""Test bookmark extractors for MySQL taps"""
# Load MySQL and Postgres properties JSON
mysql_properties = utils.load_json(
'{}/properties_mysql.json'.format(RESOURCES_DIR))
# MySQL: mysql_source_db.order is LOG_BASED
assert utils.get_bookmark_for_table('mysql_source_db.order',
mysql_properties, MySqlMock()) == {
'log_file':
'mysqld-bin.000001',
'log_pos': '123456',
'version': 1
}
# MySQL: mysql_source_db.address is INCREMENTAL
assert utils.get_bookmark_for_table('mysql_source_db.address',
mysql_properties, MySqlMock()) == {
'replication_key':
'date_updated',
'replication_key_value':
123456,
'version': 1
}
# MySQL mysql_source_db.foo not exists
assert utils.get_bookmark_for_table('mysql_source_db.foo',
mysql_properties,
MySqlMock()) == {}
def test_get_bookmark_for_table_postgresl(self):
"""Test bookmark extractors for Postgres taps"""
# Load Postgres properties JSON
postgres_properties = utils.load_json(
"{}/properties_postgres.json".format(RESOURCES_DIR))
# Postgres: public.countrylanguage is LOG_BASED
assert utils.get_bookmark_for_table("public.countrylanguage",
postgres_properties,
PostgresMock()) == {
"lsn": "16/B374D848",
"version": 1
}
# Postgres: postgres_source_db.public.city is INCREMENTAL
assert utils.get_bookmark_for_table("public.city",
postgres_properties,
PostgresMock(),
dbname="postgres_source_db") == {
"replication_key": "id",
"replication_key_value":
123456,
"version": 1
}
# Postgres: postgres_source_db.public.foo not exists
assert utils.get_bookmark_for_table("public.foo",
postgres_properties,
PostgresMock(),
dbname="postgres_source_db") == {}
def test_get_bookmark_for_table_mysql(self):
"""Test bookmark extractors for MySQL taps"""
# Load MySQL and Postgres properties JSON
mysql_properties = utils.load_json(
"{}/properties_mysql.json".format(RESOURCES_DIR))
# MySQL: mysql_source_db.order is LOG_BASED
assert utils.get_bookmark_for_table("mysql_source_db.order",
mysql_properties, MySqlMock()) == {
"log_file":
"mysqld-bin.000001",
"log_pos": "123456",
"version": 1
}
# MySQL: mysql_source_db.address is INCREMENTAL
assert utils.get_bookmark_for_table("mysql_source_db.address",
mysql_properties, MySqlMock()) == {
"replication_key":
"date_updated",
"replication_key_value":
123456,
"version": 1
}
# MySQL mysql_source_db.foo not exists
assert utils.get_bookmark_for_table("mysql_source_db.foo",
mysql_properties,
MySqlMock()) == {}
def test_get_tables_from_properties_for_s3_csv(self):
properties = utils.load_json(
'{}/properties_s3_csv.json'.format(RESOURCES_DIR))
s3_csv_tables = utils.get_tables_from_properties(properties)
# MySQL schema
assert s3_csv_tables == {
'applications',
'candidate_survey_questions',
'interviews',
}
def test_get_bookmark_for_table_tap_s3_csv(self):
"""Test bookmark extractors for S3 CSV taps"""
# Load properties JSON
properties = utils.load_json(
'{}/properties_s3_csv.json'.format(RESOURCES_DIR))
# applications is INCREMENTAL
assert utils.get_bookmark_for_table('applications', properties,
S3CsvMock()) == {
'modified_since':
'2019-11-15T07:39:44.171098',
}
# candidate_survey_questions is Full table
assert utils.get_bookmark_for_table('candidate_survey_questions',
properties, S3CsvMock()) == {}
# foo not exists
assert utils.get_bookmark_for_table('foo', properties,
S3CsvMock()) == {}
