def test_extract_dependency_check_bulk(self): """ Testing that if no dependencies are in a state that doesn't stop an extract from being loaded, then the extract is loaded. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) dependency = ExtractDependency( child_extract_id=dependent_extract.extract.extract_id, parent_extract_id=self.extract.extract.extract_id, ) self.session.add(dependency) self.session.commit() self.extract.change_extract_status("loaded") extract_trackers = [dependent_extract, self.extract] given_result = dependent_extract.extract_dependency_check( extracts=extract_trackers) expected_result = False self.assertEqual(expected_result, given_result)
def test_extract_dependency_check_bulk_in_list(self): """ Testing that even if dependencies are in a state that stops an extract from being loaded, the extract status can still be changed because it is in the bulk extract list. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) dependency = ExtractDependency( child_extract_id=dependent_extract.extract.extract_id, parent_extract_id=self.extract.extract.extract_id, ) self.session.add(dependency) self.session.commit() self.extract.change_extract_status("loading") extracts = [dependent_extract, self.extract] given_result = dependent_extract.extract_dependency_check( extracts=extracts) expected_result = False self.assertEqual(expected_result, given_result)
def test_extract_dependency_check_blocked(self): """ Testing that if a dependency is in a state that stops an extract from being loaded, then the extract triggers an error blocking the file from being processed. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) dependency = ExtractDependency( child_extract_id=dependent_extract.extract.extract_id, parent_extract_id=self.extract.extract.extract_id, ) self.session.add(dependency) self.session.commit() self.extract.change_extract_status("loading") with self.assertRaises(Exception) as context: dependent_extract.extract_dependency_check() clean_filename = str(Path("/home/test/extract_dir/Dependent File.csv")) return self.assertTrue( "Extract files that extract %s is dependent on have not been loaded," " are being created, or are in the process of loading." % clean_filename in str(context.exception))
def setUp(self): """ Creating an initial extract tracking record for testing. :return: """ self.extract = ExtractTracker( process_run=self.process_run, filename="test_extract_filename.csv", location_name="Test Location", location_path="/home/test/extract_dir", )
def test_ensure_nulls_caught_on_instantiation(self): """ With the adding of the ability of having a extract_id we have to allow for filename to be nullable. If ExtractTracker is instantiated without filename provided an error should be raised. :return: """ with self.assertRaises(Exception) as context: ExtractTracker(process_run=self.extract.process_run) return self.assertTrue( "Filename must be provided." in str(context.exception))
def test_set_file_type_invalid(self): """Testing that when an extract is created with an invalid file type, an error is thrown.""" with self.assertRaises(Exception) as context: ExtractTracker( process_run=self.process_run, filename="Compression Type File.csv", location_name="Test Location", location_path="/home/test/extract_dir", filetype="zap", ) self.assertTrue( "zap is not a valid file type" in str(context.exception))
def test_set_file_type(self): """Testing that when an extract is create with a valid file type, the type is associated correctly.""" extract = ExtractTracker( process_run=self.process_run, filename="Compression Type File.csv", location_name="Test Location", location_path="/home/test/extract_dir", filetype="Comma Separated Values", ) given_result = extract.filetype.extract_filetype_code expected_result = "csv" self.assertEqual(expected_result, given_result)
def test_initialization_no_location_no_location_path(self): """ Testing that if no location or location path is set, an error is thrown. :return: """ with self.assertRaises(Exception) as context: # Running registration a second time to mimic job being run twice ExtractTracker(process_run=self.process_run, filename="test_extract_filename.csv") return self.assertTrue( "A location object or location_path must be provided." in str( context.exception))
def test_set_compression_type(self): """Testing that when an extract is created with a compression type, the type is associated correctly.""" extract = ExtractTracker( process_run=self.process_run, filename="Compression Type File.csv", location_name="Test Location", location_path="/home/test/extract_dir", compression_type="zip", ) given_result = extract.compression_type.extract_compression_type expected_result = "zip" self.assertEqual(expected_result, given_result)
def test_derive_location_name_s3(self): """ Testing that if a location name is not provided, one is created from the s3 path provided. :return: """ extract = ExtractTracker( process_run=self.process_run, filename="test_extract_filename2.csv", location_path="https://test-test.s3.amazonaws.com/test/extract_dir", ) location = self.session.query(Location).filter( Location.location_id == extract.extract.extract_location_id) given_result = location[0].location_name expected_result = "s3 test-test - extract_dir" self.assertEqual(expected_result, given_result)
def test_derive_location_name_local_path(self): """ Testing that if a location name is not provided, one is created from the local path provided. :return: """ extract = ExtractTracker( process_run=self.process_run, filename="test_extract_filename2.csv", location_path="/home/test/extract_dir2", ) location = self.session.query(Location).filter( Location.location_id == extract.extract.extract_location_id) given_result = location[0].location_name expected_result = "local - extract_dir2" self.assertEqual(expected_result, given_result)
def test_extract_tracker_with_extract_id(self): """ Testing that when providing a extract_id to ExtractTracker, the instance is returned instead of being created. :return: """ extract_id = self.extract.extract.extract_id new_extract_tracker = ExtractTracker(extract_id=extract_id, process_run=self.process_run) expected_filename = self.extract.extract.extract_filename given_filename = new_extract_tracker.extract.extract_filename expected_location = self.extract.location.location_name given_location = new_extract_tracker.location.location_name expected_compression_type = self.extract.compression_type_id given_compression_type = new_extract_tracker.compression_type_id expected_filetype = self.extract.extract.extract_filetype given_filetype = new_extract_tracker.extract.extract_filetype expected_full_filename = self.extract.full_filename given_full_filename = new_extract_tracker.full_filename expected_dataset_types = self.extract.dataset_types given_dataset_types = new_extract_tracker.dataset_types expected_process = self.extract.extract_process.process_tracking_id given_process = new_extract_tracker.extract_process.process_tracking_id expected_sources = self.extract.sources given_sources = new_extract_tracker.sources self.assertEqual(expected_filename, given_filename) self.assertEqual(expected_location, given_location) self.assertEqual(expected_compression_type, given_compression_type) self.assertEqual(expected_filetype, given_filetype) self.assertEqual(expected_full_filename, given_full_filename) self.assertEqual(expected_dataset_types, given_dataset_types) self.assertEqual(expected_process, given_process) self.assertEqual(expected_sources, given_sources)
def test_add_dependency_parent(self): """ Testing that a parent extract dependency is created when adding dependency to extract. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) self.extract.add_dependency(dependency_type="parent", dependency=dependent_extract) given_result = (self.session.query(ExtractDependency).filter( ExtractDependency.child_extract_id == self.extract.extract.extract_id).count()) expected_result = 1 self.assertEqual(expected_result, given_result)
def test_register_extract_sources_source_objects(self): """ Testing that source objects that are part of the process are also registering to the extract. The source object is also associated to the given location of the extract. :return: """ process_run = ProcessTracker( process_name="Testing Extract Tracking Source Objects", process_type="Load", actor_name="UnitTesting", tool_name="Spark", source_objects={"Unittests": ["Table1"]}, targets="Unittests", dataset_types="Category 1", ) source_object_extract = ExtractTracker( process_run=process_run, filename="test_extract_filename.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) given_result = ( self.session.query(ExtractSourceObject).join(Extract).filter( Extract.extract_filename == source_object_extract.extract.extract_filename).count()) expected_result = 1 location_given_result = ( self.session.query(SourceObjectLocation).filter( SourceObjectLocation.location_id == source_object_extract.extract.extract_location_id).count()) location_expected_result = 1 self.assertEqual(expected_result, given_result) self.assertEqual(location_expected_result, location_given_result)
class TestExtractTracker(unittest.TestCase): @classmethod def setUpClass(cls): cls.process_tracker = ProcessTracker( process_name="Testing Extract Tracking", process_type="Load", actor_name="UnitTesting", tool_name="Spark", sources="Unittests", targets="Unittests", dataset_types="Category 1", ) cls.process_run = cls.process_tracker data_store = cls.process_tracker.data_store cls.session = data_store.session cls.data_store_type = data_store.data_store_type @classmethod def tearDownClass(cls): cls.session.query(ClusterProcess).delete() cls.session.query(DatasetType) cls.session.query(ErrorTracking).delete() cls.session.query(ExtractProcess).delete() cls.session.query(ProcessDatasetType).delete() cls.session.query(ProcessTracking).delete() cls.session.query(ProcessSource).delete() cls.session.query(ProcessSourceObject).delete() cls.session.query(ProcessTarget).delete() cls.session.query(ProcessDependency).delete() cls.session.query(Process).delete() cls.session.commit() cls.session.close() def setUp(self): """ Creating an initial extract tracking record for testing. :return: """ self.extract = ExtractTracker( process_run=self.process_run, filename="test_extract_filename.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) def tearDown(self): """ Need to clean up tables to return them to pristine state for other tests. :return: """ self.session.query(SourceLocation).delete() self.session.query(SourceObjectLocation).delete() self.session.query(ExtractSource).delete() self.session.query(ExtractSourceObject).delete() self.session.query(ExtractDatasetType).delete() self.session.query(ExtractDependency).delete() self.session.query(ExtractProcess).delete() self.session.query(Extract).delete() self.session.query(Location).delete() self.session.commit() def test_add_dependency_parent(self): """ Testing that a parent extract dependency is created when adding dependency to extract. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) self.extract.add_dependency(dependency_type="parent", dependency=dependent_extract) given_result = (self.session.query(ExtractDependency).filter( ExtractDependency.child_extract_id == self.extract.extract.extract_id).count()) expected_result = 1 self.assertEqual(expected_result, given_result) def test_add_dependency_child(self): """ Testing that a child extract dependency is created when adding dependency to extract. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) self.extract.add_dependency(dependency_type="child", dependency=dependent_extract) given_result = (self.session.query(ExtractDependency).filter( ExtractDependency.parent_extract_id == self.extract.extract.extract_id).count()) expected_result = 1 self.assertEqual(expected_result, given_result) def test_add_dependency_invalid_type(self): """ Ensuring that when a dependency is added, only 'parent' and 'child' are accepted. :return: """ with self.assertRaises(Exception) as context: self.extract.add_dependency(dependency_type="blarg", dependency=self.extract.extract) return self.assertTrue("blarg is an invalid extract dependency type." in str(context.exception)) def test_initialization_no_location_no_location_path(self): """ Testing that if no location or location path is set, an error is thrown. :return: """ with self.assertRaises(Exception) as context: # Running registration a second time to mimic job being run twice ExtractTracker(process_run=self.process_run, filename="test_extract_filename.csv") return self.assertTrue( "A location object or location_path must be provided." in str( context.exception)) def test_change_extract_status(self): """ Testing that when changing the extract status, the extract record and extract process record updates successfully. :return: """ extract_id = self.extract.extract.extract_id self.extract.change_extract_status("ready") extract_record = self.session.query(Extract).filter( Extract.extract_id == extract_id) extract_process_record = self.session.query(ExtractProcess).filter( ExtractProcess.extract_tracking_id == extract_id) given_result = [ extract_record[0].extract_status_id, extract_process_record[0].extract_process_status_id, ] expected_result = [ self.extract.extract_status_ready, self.extract.extract_status_ready, ] self.assertEqual(expected_result, given_result) def test_change_extract_status_invalid_type(self): """ When trying to change a extract's status and the status is an invalid type, throw and error. :return: """ with self.assertRaises(Exception) as context: # Running registration a second time to mimic job being run twice self.extract.change_extract_status(new_status="blarg") return self.assertTrue("blarg is not a valid extract status type. " "Please add the status to extract_status_lkup" in str(context.exception)) def test_change_extract_status_initialization(self): """ Testing that when the extract is first being worked on by a process, the status is set to 'initializing' :return: """ extract_id = self.extract.extract.extract_id self.extract.retrieve_extract_process() extract_process_record = self.session.query(ExtractProcess).filter( ExtractProcess.extract_tracking_id == extract_id) given_result = extract_process_record[0].extract_process_status_id expected_result = self.extract.extract_status_initializing self.assertEqual(expected_result, given_result) def test_derive_location_name_local_path(self): """ Testing that if a location name is not provided, one is created from the local path provided. :return: """ extract = ExtractTracker( process_run=self.process_run, filename="test_extract_filename2.csv", location_path="/home/test/extract_dir2", ) location = self.session.query(Location).filter( Location.location_id == extract.extract.extract_location_id) given_result = location[0].location_name expected_result = "local - extract_dir2" self.assertEqual(expected_result, given_result) def test_derive_location_name_s3(self): """ Testing that if a location name is not provided, one is created from the s3 path provided. :return: """ extract = ExtractTracker( process_run=self.process_run, filename="test_extract_filename2.csv", location_path="https://test-test.s3.amazonaws.com/test/extract_dir", ) location = self.session.query(Location).filter( Location.location_id == extract.extract.extract_location_id) given_result = location[0].location_name expected_result = "s3 test-test - extract_dir" self.assertEqual(expected_result, given_result) def test_extract_dependency_check(self): """ Testing that if no dependencies are in a state that doesn't stop an extract from being loaded, then the extract is loaded. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) dependency = ExtractDependency( child_extract_id=dependent_extract.extract.extract_id, parent_extract_id=self.extract.extract.extract_id, ) self.session.add(dependency) self.session.commit() self.extract.change_extract_status("loaded") given_result = dependent_extract.extract_dependency_check() expected_result = False self.assertEqual(expected_result, given_result) def test_extract_dependency_check_blocked(self): """ Testing that if a dependency is in a state that stops an extract from being loaded, then the extract triggers an error blocking the file from being processed. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) dependency = ExtractDependency( child_extract_id=dependent_extract.extract.extract_id, parent_extract_id=self.extract.extract.extract_id, ) self.session.add(dependency) self.session.commit() self.extract.change_extract_status("loading") with self.assertRaises(Exception) as context: dependent_extract.extract_dependency_check() clean_filename = str(Path("/home/test/extract_dir/Dependent File.csv")) return self.assertTrue( "Extract files that extract %s is dependent on have not been loaded," " are being created, or are in the process of loading." % clean_filename in str(context.exception)) def test_extract_dependency_check_bulk(self): """ Testing that if no dependencies are in a state that doesn't stop an extract from being loaded, then the extract is loaded. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) dependency = ExtractDependency( child_extract_id=dependent_extract.extract.extract_id, parent_extract_id=self.extract.extract.extract_id, ) self.session.add(dependency) self.session.commit() self.extract.change_extract_status("loaded") extract_trackers = [dependent_extract, self.extract] given_result = dependent_extract.extract_dependency_check( extracts=extract_trackers) expected_result = False self.assertEqual(expected_result, given_result) def test_extract_dependency_check_bulk_in_list(self): """ Testing that even if dependencies are in a state that stops an extract from being loaded, the extract status can still be changed because it is in the bulk extract list. :return: """ dependent_extract = ExtractTracker( process_run=self.process_run, filename="Dependent File.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) dependency = ExtractDependency( child_extract_id=dependent_extract.extract.extract_id, parent_extract_id=self.extract.extract.extract_id, ) self.session.add(dependency) self.session.commit() self.extract.change_extract_status("loading") extracts = [dependent_extract, self.extract] given_result = dependent_extract.extract_dependency_check( extracts=extracts) expected_result = False self.assertEqual(expected_result, given_result) def test_location_name_provided(self): """ Testing that if a location name is provided (like with default extract), one is not created. :return: """ location = self.session.query(Location).filter( Location.location_id == self.extract.extract.extract_location_id) given_result = location[0].location_name expected_result = "Test Location" self.assertEqual(expected_result, given_result) def test_register_extract_dataset_types(self): """ Testing that dataset types that are part of the process are also registering to the extract. :return: """ given_result = ( self.session.query(ExtractDatasetType).join(Extract).filter( Extract.extract_filename == self.extract.extract.extract_filename).count()) expected_result = 1 self.assertEqual(expected_result, given_result) def test_register_extract_sources(self): """ Testing that sources that are part of the process are also registering to the extract. The source is also associated to the given location of the extract. :return: """ given_result = (self.session.query(ExtractSource).join(Extract).filter( Extract.extract_filename == self.extract.extract.extract_filename).count()) expected_result = 1 location_given_result = (self.session.query(SourceLocation).filter( SourceLocation.location_id == self.extract.extract.extract_location_id).count()) location_expected_result = 1 self.assertEqual(expected_result, given_result) self.assertEqual(location_expected_result, location_given_result) def test_register_extract_sources_source_objects(self): """ Testing that source objects that are part of the process are also registering to the extract. The source object is also associated to the given location of the extract. :return: """ process_run = ProcessTracker( process_name="Testing Extract Tracking Source Objects", process_type="Load", actor_name="UnitTesting", tool_name="Spark", source_objects={"Unittests": ["Table1"]}, targets="Unittests", dataset_types="Category 1", ) source_object_extract = ExtractTracker( process_run=process_run, filename="test_extract_filename.csv", location_name="Test Location", location_path="/home/test/extract_dir", ) given_result = ( self.session.query(ExtractSourceObject).join(Extract).filter( Extract.extract_filename == source_object_extract.extract.extract_filename).count()) expected_result = 1 location_given_result = ( self.session.query(SourceObjectLocation).filter( SourceObjectLocation.location_id == source_object_extract.extract.extract_location_id).count()) location_expected_result = 1 self.assertEqual(expected_result, given_result) self.assertEqual(location_expected_result, location_given_result) def test_set_extract_low_high_dates_write(self): """ Testing that low and high dates are set for the write audit fields. :return: """ low_date = utilities.timestamp_converter( data_store_type=self.data_store_type, timestamp=datetime.now()) - timedelta(hours=1) high_date = utilities.timestamp_converter( data_store_type=self.data_store_type, timestamp=datetime.now()) self.extract.set_extract_low_high_dates(low_date=low_date, high_date=high_date, audit_type="write") given_dates = self.session.query( Extract.extract_write_low_date_time, Extract.extract_write_high_date_time).filter( Extract.extract_id == self.extract.extract.extract_id) expected_result = [low_date, high_date] given_result = [ given_dates[0].extract_write_low_date_time, given_dates[0].extract_write_high_date_time, ] self.assertEqual(expected_result, given_result) def test_set_extract_low_high_dates_load(self): """ Testing that low and high dates are set for the load audit fields. :return: """ low_date = utilities.timestamp_converter( data_store_type=self.data_store_type, timestamp=datetime.now()) - timedelta(hours=1) high_date = utilities.timestamp_converter( data_store_type=self.data_store_type, timestamp=datetime.now()) self.extract.set_extract_low_high_dates(low_date=low_date, high_date=high_date, audit_type="load") given_dates = self.session.query( Extract.extract_load_low_date_time, Extract.extract_load_high_date_time).filter( Extract.extract_id == self.extract.extract.extract_id) expected_result = [low_date, high_date] given_result = [ given_dates[0].extract_load_low_date_time, given_dates[0].extract_load_high_date_time, ] self.assertEqual(expected_result, given_result) def test_set_extract_low_high_dates_invalid_type(self): """ Testing that the set_extract_low_high_dates method does not allow invalid types. :return: """ low_date = utilities.timestamp_converter( data_store_type=self.data_store_type, timestamp=datetime.now()) - timedelta(hours=1) high_date = utilities.timestamp_converter( data_store_type=self.data_store_type, timestamp=datetime.now()) with self.assertRaises(Exception) as context: self.extract.set_extract_low_high_dates(low_date=low_date, high_date=high_date, audit_type="blarg") return self.assertTrue( "blarg is not a valid audit_type." in str(context.exception)) def test_set_extract_record_count_write(self): """ Testing that the record count is set for the write record count column. :return: """ records = 10000 self.extract.set_extract_record_count(num_records=records, audit_type="write") given_result = self.session.query( Extract.extract_write_record_count).filter( Extract.extract_id == self.extract.extract.extract_id) self.assertEqual(records, given_result[0].extract_write_record_count) def test_set_extract_record_count_load(self): """ Testing that the record count is set for the load record count column. :return: """ records = 10000 self.extract.set_extract_record_count(num_records=records, audit_type="load") given_result = self.session.query( Extract.extract_load_record_count).filter( Extract.extract_id == self.extract.extract.extract_id) self.assertEqual(records, given_result[0].extract_load_record_count) def test_set_extract_record_count_invalid_type(self): """ Testing that the set_extract_record_count method does not allow invalid types. :return: """ with self.assertRaises(Exception) as context: self.extract.set_extract_record_count(num_records=100000, audit_type="blarg") return self.assertTrue( "blarg is not a valid audit_type." in str(context.exception)) def test_set_compression_type(self): """Testing that when an extract is created with a compression type, the type is associated correctly.""" extract = ExtractTracker( process_run=self.process_run, filename="Compression Type File.csv", location_name="Test Location", location_path="/home/test/extract_dir", compression_type="zip", ) given_result = extract.compression_type.extract_compression_type expected_result = "zip" self.assertEqual(expected_result, given_result) def test_set_compression_type_invalid(self): """Testing that when an extract is created with an invalid compression type, an error is thrown.""" with self.assertRaises(Exception) as context: ExtractTracker( process_run=self.process_run, filename="Compression Type File.csv", location_name="Test Location", location_path="/home/test/extract_dir", compression_type="zap", ) self.assertTrue( "zap is not a valid compression type" in str(context.exception)) def test_set_file_type(self): """Testing that when an extract is create with a valid file type, the type is associated correctly.""" extract = ExtractTracker( process_run=self.process_run, filename="Compression Type File.csv", location_name="Test Location", location_path="/home/test/extract_dir", filetype="Comma Separated Values", ) given_result = extract.filetype.extract_filetype_code expected_result = "csv" self.assertEqual(expected_result, given_result) def test_set_file_type_invalid(self): """Testing that when an extract is created with an invalid file type, an error is thrown.""" with self.assertRaises(Exception) as context: ExtractTracker( process_run=self.process_run, filename="Compression Type File.csv", location_name="Test Location", location_path="/home/test/extract_dir", filetype="zap", ) self.assertTrue( "zap is not a valid file type" in str(context.exception)) def test_extract_tracker_with_extract_id(self): """ Testing that when providing a extract_id to ExtractTracker, the instance is returned instead of being created. :return: """ extract_id = self.extract.extract.extract_id new_extract_tracker = ExtractTracker(extract_id=extract_id, process_run=self.process_run) expected_filename = self.extract.extract.extract_filename given_filename = new_extract_tracker.extract.extract_filename expected_location = self.extract.location.location_name given_location = new_extract_tracker.location.location_name expected_compression_type = self.extract.compression_type_id given_compression_type = new_extract_tracker.compression_type_id expected_filetype = self.extract.extract.extract_filetype given_filetype = new_extract_tracker.extract.extract_filetype expected_full_filename = self.extract.full_filename given_full_filename = new_extract_tracker.full_filename expected_dataset_types = self.extract.dataset_types given_dataset_types = new_extract_tracker.dataset_types expected_process = self.extract.extract_process.process_tracking_id given_process = new_extract_tracker.extract_process.process_tracking_id expected_sources = self.extract.sources given_sources = new_extract_tracker.sources self.assertEqual(expected_filename, given_filename) self.assertEqual(expected_location, given_location) self.assertEqual(expected_compression_type, given_compression_type) self.assertEqual(expected_filetype, given_filetype) self.assertEqual(expected_full_filename, given_full_filename) self.assertEqual(expected_dataset_types, given_dataset_types) self.assertEqual(expected_process, given_process) self.assertEqual(expected_sources, given_sources) def test_ensure_nulls_caught_on_instantiation(self): """ With the adding of the ability of having a extract_id we have to allow for filename to be nullable. If ExtractTracker is instantiated without filename provided an error should be raised. :return: """ with self.assertRaises(Exception) as context: ExtractTracker(process_run=self.extract.process_run) return self.assertTrue( "Filename must be provided." in str(context.exception)) def test_file_size_splitter_bytes(self): """ Testing that if provided a file size in bytes, the file size in GB will be returned. :return: """ given_result = self.extract.file_size_splitter(file_size="1048576B") given_result = {given_result[0], given_result[1]} expected_result = {1, "GB"} self.assertEqual(expected_result, given_result) def test_file_size_splitter_mb(self): """ Testing that if provided a file size in MB, the file size will not be modified. :return: """ given_result = self.extract.file_size_splitter(file_size="1024MB") given_result = {given_result[0], given_result[1]} expected_result = {1024, "MB"} self.assertEqual(expected_result, given_result) def test_file_size_splitter_gb(self): """ Testing that if provided a file size in GB, the file size will not be modified. :return: """ given_result = self.extract.file_size_splitter(file_size="50GB") given_result = {given_result[0], given_result[1]} expected_result = {50, "GB"} self.assertEqual(expected_result, given_result) def test_file_size_splitter_no_measure(self): """ Testing that if provided a file size without a measure, the file size will be assumed to be bytes and returned in GB. :return: """ given_result = self.extract.file_size_splitter(file_size="1048576") given_result = {given_result[0], given_result[1]} expected_result = {1, "GB"} self.assertEqual(expected_result, given_result) def test_file_size_splitter_invalid_measure(self): """ Testing that if provided a file size with an invalid measure, an exception will be thrown. :return: """ with self.assertRaises(Exception) as context: self.extract.file_size_splitter(file_size="1024ZXB") self.assertTrue( "Unsupported measure detected. Please provide file size in bytes, MB, or GB. Measure provided was: ZXB" in str(context.exception))