def test_execute(self, mock_call): chain = StetlTestCase.get_chain(self.etl) chain.run() self.assertTrue(mock_call.called) self.assertEqual(1, mock_call.call_count) # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 5) self.assertEqual(list[0], 'ogr2ogr') # Compare command line arguments with config section = StetlTestCase.get_section(chain, -1) file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path') # Certain options should not occur self.assertFalse('-spat' in list) self.assertFalse('-lco' in list) # Destination format self.assertTrue('-f' in list) f_idx = list.index('-f') dest_format = self.etl.configdict.get(section, 'dest_format') self.assertEqual(list[f_idx + 1], dest_format) # Destination datasource dest_data_source = self.etl.configdict.get(section, 'dest_data_source') self.assertEqual(list[f_idx + 2], dest_data_source) # Source datasource self.assertEqual(list[-1], file_path)
def test_execute(self, mock_call): chain = StetlTestCase.get_chain(self.etl) chain.run() self.assertTrue(mock_call.called) self.assertEqual(1, mock_call.call_count) # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 5) self.assertEqual(list[0], 'ogr2ogr') # Compare command line arguments with config section = StetlTestCase.get_section(chain, -1) file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path') # Certain options should not occur self.assertFalse('-spat' in list) self.assertFalse('-lco' in list) # Destination format self.assertTrue('-f' in list) f_idx = list.index('-f') dest_format = self.etl.configdict.get(section, 'dest_format') self.assertEqual(list[f_idx + 1], dest_format) # Destination datasource dest_data_source = self.etl.configdict.get(section, 'dest_data_source') self.assertEqual(list[f_idx + 2], dest_data_source) # Source datasource self.assertEqual(list[-1], file_path)
def test_execute_zip(self, mock_after_chain_invoke): chain = StetlTestCase.get_chain(self.etl) chain.run() # ZIP file contains two GML files, both should be extracted; count is 3 because of final # call, so the ZipArchiveExpander can indicate that no more files can be found. self.assertTrue(mock_after_chain_invoke.called) self.assertEqual(2, mock_after_chain_invoke.call_count) # Check if temp dir exists section = StetlTestCase.get_section(chain, 2) target_dir = self.etl.configdict.get(section, 'target_dir') self.assertTrue(os.path.exists(target_dir)) file_objects = os.listdir(target_dir) # 3 XML files in archive self.assertEqual(3, len(file_objects)) for file_object in file_objects: file_object_path = os.path.join(target_dir, file_object) self.assertTrue(file_object.startswith('0221LIG')) self.assertTrue(file_object.endswith('.xml')) self.assertTrue(os.path.exists(file_object_path)) os.remove(file_object_path)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.stringfilter.StringSubstitutionFilter', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, -1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.outputs.execoutput.Ogr2OgrExecOutput', class_name)
def test_execute_vsizip(self, mock_after_chain_invoke): chain = StetlTestCase.get_chain(self.etl, index=1) chain.run() # ZIP file contains one XML file and is extracted; count is 2 because of final # call, so the VsiFileExtractor can indicate that no more files can be found. self.assertTrue(mock_after_chain_invoke.called) self.assertEqual(2, mock_after_chain_invoke.call_count) # Check if temp file exists section = StetlTestCase.get_section(chain, 2) target_dir = self.etl.configdict.get(section, 'target_dir') self.assertTrue(os.path.exists(target_dir)) file_objects = os.listdir(target_dir) # 2 XML files in archive self.assertEqual(2, len(file_objects)) for file_object in file_objects: file_object_path = os.path.join(target_dir, file_object) self.assertTrue(file_object.startswith('0221WPL')) self.assertTrue(file_object.endswith('.xml')) self.assertTrue(os.path.exists(file_object_path)) os.remove(file_object_path)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.filters.execfilter.CommandExecFilter', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.zipfileextractor.ZipFileExtractor', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.filters.sieve.AttrValueRecordSieve', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.inputs.fileinput.XmlElementStreamerFileInput', class_name)
def test_execute(self): chain = StetlTestCase.get_chain(self.etl) chain.run() # Check if temp file exists section = StetlTestCase.get_section(chain, 1) file_path = self.etl.configdict.get(section, 'file_path') self.assertTrue(os.path.exists(file_path)) # Compare temp file to input file byte by byte section = StetlTestCase.get_section(chain) orig_file_path = self.etl.configdict.get(section, 'file_path') with open(file_path, 'rb') as f: file = f.read() with open(orig_file_path, 'rb') as f: orig_file = f.read() self.assertEqual(file, orig_file) # Remove temp file os.remove(file_path)
def test_execute(self): chain = StetlTestCase.get_chain(self.etl) chain.run() # Check if temp file exists section = StetlTestCase.get_section(chain, 1) file_path = self.etl.configdict.get(section, 'file_path') self.assertTrue(os.path.exists(file_path)) # Compare temp file to input file byte by byte section = StetlTestCase.get_section(chain) orig_file_path = self.etl.configdict.get(section, 'file_path') with open(file_path, 'rb') as f: file = f.read() with open(orig_file_path, 'rb') as f: orig_file = f.read() self.assertEqual(file, orig_file) # Remove temp file os.remove(file_path)
def test_execute_no_cleanup(self, mock_call): chain = StetlTestCase.get_chain(self.etl, 7) file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path') chain.run() # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 5) # Check if input file still exists self.assertTrue(os.path.exists(file_path))
def test_execute(self): # Read content of input file chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) fn = self.etl.configdict.get(section, 'file_path') with open(fn, 'r') as f: contents = f.read() self.etl.run() self.assertGreater(len(sys.stdout.getvalue()), 0) # Assert includes last linebreak from stdout, due to print function self.assertEqual(sys.stdout.getvalue(), contents + '\n')
def test_execute_no_cleanup(self, mock_call): chain = StetlTestCase.get_chain(self.etl, 7) file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path') chain.run() # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 5) # Check if input file still exists self.assertTrue(os.path.exists(file_path))
def test_execute(self, mock_tx_execute): # Read content of input file chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) fn = self.etl.configdict.get(section, 'file_path') with open(fn, 'r') as f: contents = f.read() self.etl.run() self.assertTrue(mock_tx_execute.called) self.assertEqual(1, mock_tx_execute.call_count) args, kwargs = mock_tx_execute.call_args self.assertEqual(contents, args[1])
def test_execute(self, mock_call): # Read content of input file chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) fn = self.etl.configdict.get(section, 'file_path') with open(fn, 'r') as f: contents = f.read() self.etl.run() self.assertTrue(mock_call.called) self.assertEqual(1, mock_call.call_count) args, kwargs = mock_call.call_args self.assertEqual(contents, args[0])
def test_execute_all_xml(self, mock_after_chain_invoke): chain = StetlTestCase.get_chain(self.etl, index=2) chain.run() # ZIP file contains 18 XML files in various zipfiles and is extracted; count is 19 because of final # call, so the VsiFileExtractor can indicate that no more files can be found. self.assertTrue(mock_after_chain_invoke.called) self.assertEqual(19, mock_after_chain_invoke.call_count) # Check if temp file exists section = StetlTestCase.get_section(chain, 1) file_path = self.etl.configdict.get(section, 'file_path') self.assertTrue(os.path.exists(file_path)) os.remove(file_path)
def test_execute(self, mock_after_chain_invoke): chain = StetlTestCase.get_chain(self.etl) chain.run() # ZIP file contains two GML files, both should be extracted; count is 3 because of final # call, so the ZipFileExtractor can indicate that no more files can be found. self.assertTrue(mock_after_chain_invoke.called) self.assertEqual(3, mock_after_chain_invoke.call_count) # Check if temp file exists section = StetlTestCase.get_section(chain, 1) file_path = self.etl.configdict.get(section, 'file_path') self.assertTrue(os.path.exists(file_path)) os.remove(file_path)
def test_execute(self, mock_after_chain_invoke): chain = StetlTestCase.get_chain(self.etl) chain.run() # ZIP file contains two GML files, both should be extracted; count is 3 because of final # call, so the ZipFileExtractor can indicate that no more files can be found. self.assertTrue(mock_after_chain_invoke.called) self.assertEqual(3, mock_after_chain_invoke.call_count) # Check if temp file exists section = StetlTestCase.get_section(chain, 1) file_path = self.etl.configdict.get(section, 'file_path') self.assertTrue(os.path.exists(file_path)) os.remove(file_path)
def test_execute_extent(self, mock_call): chain = StetlTestCase.get_chain(self.etl, 2) chain.run() # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 10) # Check spatial extent section = StetlTestCase.get_section(chain, -1) self.assertTrue('-spat' in list) spat_idx = list.index('-spat') spatial_extent = self.etl.configdict.get(section, 'spatial_extent') self.assertEqual(spatial_extent.split(), list[spat_idx + 1:spat_idx + 5])
def test_execute_extent(self, mock_call): chain = StetlTestCase.get_chain(self.etl, 2) chain.run() # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 10) # Check spatial extent section = StetlTestCase.get_section(chain, -1) self.assertTrue('-spat' in list) spat_idx = list.index('-spat') spatial_extent = self.etl.configdict.get(section, 'spatial_extent') self.assertEqual(spatial_extent.split(), list[spat_idx + 1:spat_idx + 5])
def test_execute(self): # Read content of input file chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) fn = self.etl.configdict.get(section, 'file_path') with open(fn, 'r') as f: contents = f.read() # Invoke first component of chain chain.first_comp.do_init() packet = Packet() packet.init() packet.component = chain.first_comp chain.first_comp.before_invoke(packet) packet = chain.first_comp.invoke(packet) self.assertEqual(packet.data, contents)
def test_legacy_class(self): cfg_dict = { 'config_file': os.path.join(self.curr_dir, 'configs/zipfileextractordeprecated.cfg') } self.etl = ETL(cfg_dict) chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') # Deprecated class name in config self.assertEqual('stetl.filters.zipfileextractor.ZipFileExtractor', class_name) # Assigned to new version in fileextractor module! self.assertTrue(isinstance(chain.first_comp.next, ZipFileExtractor))
def test_execute_gfs(self, mock_call): chain = StetlTestCase.get_chain(self.etl, 4) chain.run() # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 5) # Check if GFS file exists, and clean it up file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path') file_ext = os.path.splitext(file_path) gfs_path = file_ext[0] + '.gfs' self.assertTrue(os.path.exists(gfs_path)) os.remove(gfs_path) self.assertFalse(os.path.exists(gfs_path))
def test_execute_cleanup(self, mock_call): # Copy test file to temporary location, because it will be cleaned up orig_file_path = self.etl.configdict.get(StetlTestCase.get_section(StetlTestCase.get_chain(self.etl)), 'file_path') orig_file_ext = os.path.splitext(orig_file_path) temp_file_path = orig_file_ext[0] + "_temp" + orig_file_ext[1] shutil.copy(orig_file_path, temp_file_path) chain = StetlTestCase.get_chain(self.etl, 5) chain.run() # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 5) # Check if temp file has been removed self.assertFalse(os.path.exists(temp_file_path))
def test_execute(self): # Read content of input file chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) fn = self.etl.configdict.get(section, 'file_path') with open(fn, 'r') as f: contents = f.read() # Invoke first component of chain chain.first_comp.do_init() packet = Packet() packet.init() packet.component = chain.first_comp chain.first_comp.before_invoke(packet) packet = chain.first_comp.invoke(packet) self.assertEqual(packet.data, contents)
def test_execute_gfs(self, mock_call): chain = StetlTestCase.get_chain(self.etl, 4) chain.run() # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 5) # Check if GFS file exists, and clean it up file_path = self.etl.configdict.get(StetlTestCase.get_section(chain), 'file_path') file_ext = os.path.splitext(file_path) gfs_path = file_ext[0] + '.gfs' self.assertTrue(os.path.exists(gfs_path)) os.remove(gfs_path) self.assertFalse(os.path.exists(gfs_path))
def test_execute_cleanup(self, mock_call): # Copy test file to temporary location, because it will be cleaned up orig_file_path = self.etl.configdict.get( StetlTestCase.get_section(StetlTestCase.get_chain(self.etl)), 'file_path') orig_file_ext = os.path.splitext(orig_file_path) temp_file_path = orig_file_ext[0] + "_temp" + orig_file_ext[1] shutil.copy(orig_file_path, temp_file_path) chain = StetlTestCase.get_chain(self.etl, 5) chain.run() # Check command line args, kwargs = mock_call.call_args list = self.parse_command(args[0]) self.assertEqual(len(list), 5) # Check if temp file has been removed self.assertFalse(os.path.exists(temp_file_path))
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.xmlassembler.XmlAssembler', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, -1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('outputs.dboutput.PostgresDbOutput', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('inputs.fileinput.LineStreamerFileInput', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.sieve.AttrValueRecordSieve', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('inputs.fileinput.JsonFileInput', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.filters.zipfileextractor.ZipFileExtractor', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.filters.regexfilter.RegexFilter', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.execfilter.CommandExecFilter', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, -1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('outputs.execoutput.Ogr2OgrExecOutput', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, -1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('outputs.dboutput.PostgresDbOutput', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.packetwriter.PacketWriter', class_name)
def check_execution_result(self, chain, result, with_namespace): section = StetlTestCase.get_section(chain, 1) element_tags = self.etl.configdict.get(section, 'element_tags') pattern = "<Element %s%s " % ('{[^}]+}' if with_namespace else '', element_tags) for elem in result: self.assertIsNotNone(re.match(pattern, elem))
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 2) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.filters.archiveexpander.ZipArchiveExpander', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.inputs.ogrinput.OgrInput', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.inputs.ogrinput.OgrInput', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, -1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.outputs.standardoutput.StandardOutput', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('stetl.filters.xmlelementreader.XmlElementReader', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.packetwriter.PacketWriter', class_name)
def check_execution_result(self, chain, result, with_namespace): section = StetlTestCase.get_section(chain, 1) element_tags = self.etl.configdict.get(section, 'element_tags') pattern = "<Element %s%s " % ('{[^}]+}' if with_namespace else '', element_tags) for elem in result: self.assertIsNotNone(re.match(pattern, elem))
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.stringfilter.StringSubstitutionFilter', class_name)
def test_class(self): chain = StetlTestCase.get_chain(self.etl) section = StetlTestCase.get_section(chain, 1) class_name = self.etl.configdict.get(section, 'class') self.assertEqual('filters.xmlassembler.XmlAssembler', class_name)