def test_run(self): self.setUpTestEnvironment(COMPRESSION_FOLDER_PATH) runner.run_check_job_and_check_status(self) found_catalogs = menagerie.get_catalogs(self.conn_id) self.assertEqual( len(found_catalogs), 1, msg="unable to locate schemas for connection {}".format( self.conn_id)) found_catalog_names = set( map(lambda c: c['tap_stream_id'], found_catalogs)) subset = self.expected_check_streams().issubset(found_catalog_names) self.assertTrue( subset, msg="Expected check streams are not subset of discovered catalog") # Clear state before our run menagerie.set_state(self.conn_id, {}) self.select_specific_catalog(found_catalogs, "gz_file_having_empty_csv") runner.run_sync_job_and_check_status(self) expected_records = 0 # Verify actual rows were synced records = runner.get_upserts_from_target_output() self.assertEqual(expected_records, len(records))
def test_run(self): runner.run_check_job_and_check_status(self) found_catalogs = menagerie.get_catalogs(self.conn_id) self.check_all_streams_in_catalogs(found_catalogs) self.select_found_catalogs(found_catalogs) # clear state and run the actual sync menagerie.set_state(self.conn_id, {}) runner.run_sync_job_and_check_status(self) self.check_output_record_counts() max_bookmarks_from_records = runner.get_max_bookmarks_from_target(self) state = menagerie.get_state(self.conn_id) bookmarks = state.get("bookmarks", {}) self.check_bookmarks(bookmarks, max_bookmarks_from_records) self.check_offsets(bookmarks) self.look_for_unexpected_bookmarks(bookmarks) self.assertIsNone(state.get("currently_syncing"))
def test_run(self): self.setUpTestEnvironment() runner.run_check_job_and_check_status(self) found_catalogs = menagerie.get_catalogs(self.conn_id) self.assertEqual( len(found_catalogs), len(self.expected_check_streams()), msg="unable to locate schemas for connection {}".format( self.conn_id)) found_catalog_names = set( map(lambda c: c['tap_stream_id'], found_catalogs)) subset = self.expected_check_streams().issubset(found_catalog_names) self.assertTrue( subset, msg="Expected check streams are not subset of discovered catalog") # Clear state before our run menagerie.set_state(self.conn_id, {}) self.select_found_catalogs(found_catalogs) runner.run_sync_job_and_check_status(self) no_csv_records = 998 no_jsonl_records = 10 no_gz_has_csv_records = 998 no_gz_has_jsonl_records = 2 no_zip_records = 40 expected_records = no_csv_records + no_jsonl_records + no_gz_has_csv_records + no_gz_has_jsonl_records + no_zip_records with open( utils.get_resources_path( "output_csv_records.json", ALL_SUPPORTED_FOLDER_PATH)) as json_file: expected_csv_records = simplejson.load(json_file, use_decimal=True).get( "records", []) with open( utils.get_resources_path( "output_jsonl_records.json", ALL_SUPPORTED_FOLDER_PATH)) as json_file: expected_jsonl_records = simplejson.load(json_file, use_decimal=True).get( "records", []) with open( utils.get_resources_path( "output_gz_csv_records.json", ALL_SUPPORTED_FOLDER_PATH)) as json_file: expected_gz_has_csv_records = simplejson.load( json_file, use_decimal=True).get("records", []) with open( utils.get_resources_path( "output_gz_jsonl_records.json", ALL_SUPPORTED_FOLDER_PATH)) as json_file: expected_gz_has_jsonl_records = simplejson.load( json_file, use_decimal=True).get("records", []) with open( utils.get_resources_path( "output_zip_records.json", ALL_SUPPORTED_FOLDER_PATH)) as json_file: expected_zip_records = simplejson.load(json_file, use_decimal=True).get( "records", []) synced_records = runner.get_records_from_target_output() csv_upsert_messages = [ m for m in synced_records.get('all_support_csv').get('messages') if m['action'] == 'upsert' ] jsonl_upsert_messages = [ m for m in synced_records.get('all_support_jsonl').get('messages') if m['action'] == 'upsert' ] gz_with_csv_upsert_messages = [ m for m in synced_records.get('all_support_gz_has_csv').get( 'messages') if m['action'] == 'upsert' ] gz_with_jsonl_upsert_messages = [ m for m in synced_records.get('all_support_gz_has_jsonl').get( 'messages') if m['action'] == 'upsert' ] zip_upsert_messages = [ m for m in synced_records.get('all_support_zip').get('messages') if m['action'] == 'upsert' ] csv_records = [message.get('data') for message in csv_upsert_messages] jsonl_records = [ message.get('data') for message in jsonl_upsert_messages ] gz_has_csv_records = [ message.get('data') for message in gz_with_csv_upsert_messages ] gz_has_jsonl_records = [ message.get('data') for message in gz_with_jsonl_upsert_messages ] zip_records = [message.get('data') for message in zip_upsert_messages] no_records = len(csv_records) + len(jsonl_records) + len( gz_has_csv_records) + len(gz_has_jsonl_records) + len(zip_records) self.assertEqual(expected_records, no_records) self.assertEquals(expected_csv_records, csv_records) self.assertEquals(expected_jsonl_records, jsonl_records) self.assertEquals(expected_gz_has_csv_records, gz_has_csv_records) self.assertEquals(expected_gz_has_jsonl_records, gz_has_jsonl_records) self.assertEquals(expected_zip_records, zip_records)