def test_get_tables_from_properties(self): """Test getting selected tables from tap properties JSON""" # Load MySQL and Postgres properties JSON mysql_properties = utils.load_json( '{}/properties_mysql.json'.format(RESOURCES_DIR)) postgres_properties = utils.load_json( '{}/properties_postgres.json'.format(RESOURCES_DIR)) # Get list of selected tables # MySQL and Postgres schemas defined at different keys. get_tables_from_properties function # should detect and extract correctly mysql_tables = utils.get_tables_from_properties(mysql_properties) postgres_tables = utils.get_tables_from_properties(postgres_properties) # MySQL schema assert mysql_tables == \ [ 'mysql_source_db.address', 'mysql_source_db.order', 'mysql_source_db.weight_unit' ] assert postgres_tables == \ [ 'public.city', 'public.country' ]
def test_get_bookmark_for_table_postgresl(self): """Test bookmark extractors for Postgres taps""" # Load Postgres properties JSON postgres_properties = utils.load_json( '{}/properties_postgres.json'.format(RESOURCES_DIR)) # Postgres: public.countrylanguage is LOG_BASED assert utils.get_bookmark_for_table('public.countrylanguage', postgres_properties, PostgresMock()) == { 'lsn': '16/B374D848', 'version': 1 } # Postgres: postgres_source_db.public.city is INCREMENTAL assert utils.get_bookmark_for_table('public.city', postgres_properties, PostgresMock(), dbname='postgres_source_db') == { 'replication_key': 'id', 'replication_key_value': 123456, 'version': 1 } # Postgres: postgres_source_db.public.foo not exists assert utils.get_bookmark_for_table('public.foo', postgres_properties, PostgresMock(), dbname='postgres_source_db') == {}
def test_get_bookmark_for_table_mysql(self): """Test bookmark extractors for MySQL taps""" # Load MySQL and Postgres properties JSON mysql_properties = utils.load_json( '{}/properties_mysql.json'.format(RESOURCES_DIR)) # MySQL: mysql_source_db.order is LOG_BASED assert utils.get_bookmark_for_table('mysql_source_db.order', mysql_properties, MySqlMock()) == { 'log_file': 'mysqld-bin.000001', 'log_pos': '123456', 'version': 1 } # MySQL: mysql_source_db.address is INCREMENTAL assert utils.get_bookmark_for_table('mysql_source_db.address', mysql_properties, MySqlMock()) == { 'replication_key': 'date_updated', 'replication_key_value': 123456, 'version': 1 } # MySQL mysql_source_db.foo not exists assert utils.get_bookmark_for_table('mysql_source_db.foo', mysql_properties, MySqlMock()) == {}
def test_get_bookmark_for_table_postgresl(self): """Test bookmark extractors for Postgres taps""" # Load Postgres properties JSON postgres_properties = utils.load_json( "{}/properties_postgres.json".format(RESOURCES_DIR)) # Postgres: public.countrylanguage is LOG_BASED assert utils.get_bookmark_for_table("public.countrylanguage", postgres_properties, PostgresMock()) == { "lsn": "16/B374D848", "version": 1 } # Postgres: postgres_source_db.public.city is INCREMENTAL assert utils.get_bookmark_for_table("public.city", postgres_properties, PostgresMock(), dbname="postgres_source_db") == { "replication_key": "id", "replication_key_value": 123456, "version": 1 } # Postgres: postgres_source_db.public.foo not exists assert utils.get_bookmark_for_table("public.foo", postgres_properties, PostgresMock(), dbname="postgres_source_db") == {}
def test_get_bookmark_for_table_mysql(self): """Test bookmark extractors for MySQL taps""" # Load MySQL and Postgres properties JSON mysql_properties = utils.load_json( "{}/properties_mysql.json".format(RESOURCES_DIR)) # MySQL: mysql_source_db.order is LOG_BASED assert utils.get_bookmark_for_table("mysql_source_db.order", mysql_properties, MySqlMock()) == { "log_file": "mysqld-bin.000001", "log_pos": "123456", "version": 1 } # MySQL: mysql_source_db.address is INCREMENTAL assert utils.get_bookmark_for_table("mysql_source_db.address", mysql_properties, MySqlMock()) == { "replication_key": "date_updated", "replication_key_value": 123456, "version": 1 } # MySQL mysql_source_db.foo not exists assert utils.get_bookmark_for_table("mysql_source_db.foo", mysql_properties, MySqlMock()) == {}
def test_get_tables_from_properties_for_s3_csv(self): properties = utils.load_json( '{}/properties_s3_csv.json'.format(RESOURCES_DIR)) s3_csv_tables = utils.get_tables_from_properties(properties) # MySQL schema assert s3_csv_tables == { 'applications', 'candidate_survey_questions', 'interviews', }
def test_get_bookmark_for_table_tap_s3_csv(self): """Test bookmark extractors for S3 CSV taps""" # Load properties JSON properties = utils.load_json( '{}/properties_s3_csv.json'.format(RESOURCES_DIR)) # applications is INCREMENTAL assert utils.get_bookmark_for_table('applications', properties, S3CsvMock()) == { 'modified_since': '2019-11-15T07:39:44.171098', } # candidate_survey_questions is Full table assert utils.get_bookmark_for_table('candidate_survey_questions', properties, S3CsvMock()) == {} # foo not exists assert utils.get_bookmark_for_table('foo', properties, S3CsvMock()) == {}