def setUp(self):
super(StandardOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/standardoutput.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(ConfigTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/copy_in_out.cfg')}
self.etl = ETL(cfg_dict)
def main():
"""The `main` function, to be called from commandline, like `python src/main.py -c etl.cfg`.
Args:
-c --config <config_file> the Stetl config file.
-s --section <section_name> the section in the Stetl config (ini) file to execute (default is [etl]).
-a --args <arglist> sero or more substitutable args for symbolic, {arg}, values in Stetl config file, in format -a arg1=foo -a arg2=bar etc.
-d --doc <class> Get component documentation like its configuration parameters, e.g. stetl --doc stetl.inputs.fileinput.FileInput
-v --version Show the current version of stelt and exit
-h --help get help info
"""
# Pass arguments explicitly, facilitates testing
args = parse_args(sys.argv[1:])
if args.version:
print('Stetl version: ', __version__)
exit()
if args.config_file:
# Do the ETL
etl = ETL(vars(args), args.config_args)
etl.run()
elif args.doc_args:
print_doc(args.doc_args)
else:
print('Unknown option, try stetl -h for help')
def setUp(self):
super(XmlElementReaderTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/xmlelementreader.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(GlobFileInputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/globfileinput.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(VsiFilterTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/vsifiltertest.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(ConfigTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {"config_file": os.path.join(curr_dir, "configs/copy_in_out.cfg")}
self.etl = ETL(cfg_dict)
def setUp(self):
super(StringConcatFilterTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/stringconcatfilter.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(MergerMultiInputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/mergermultiinput.cfg')}
self.etl = ETL(cfg_dict)
class ConfigTest(StetlTestCase):
"""Basic configuration tests"""
def setUp(self):
super(ConfigTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {"config_file": os.path.join(curr_dir, "configs/copy_in_out.cfg")}
self.etl = ETL(cfg_dict)
def test_type(self):
self.assertEqual(self.etl.configdict.get("etl", "chains"), "input_xml_file|output_std")
def test_run(self):
self.etl.run()
class ConfigTest(StetlTestCase):
"""Basic configuration tests"""
def setUp(self):
super(ConfigTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/copy_in_out.cfg')}
self.etl = ETL(cfg_dict)
def test_type(self):
self.assertEqual(self.etl.configdict.get('etl', 'chains'), 'input_xml_file|output_std')
def test_run(self):
self.etl.run()
def setUp(self):
super(ZipArchiveExpanderTest, self).setUp()
# Initialize Stetl
self.curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(self.curr_dir, 'configs/ziparchiveexpander.cfg')}
self.etl = ETL(cfg_dict)
def test_args_dict(self):
args_dict = {'in_file': 'infile.txt', 'out_file': 'outfile.txt'}
etl = ETL(self.cfg_dict, args_dict)
# Test args substitution from args_dict
self.assertEqual(etl.configdict.get('input_file', 'file_path'),
'infile.txt')
self.assertEqual(etl.configdict.get('output_file', 'file_path'),
'outfile.txt')
def setUp(self):
super(PacketWriterTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir, 'configs/packetwriter.cfg')
}
self.etl = ETL(cfg_dict)
def setUp(self):
super(CommandExecFilterTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir,
'configs/commandexecfilter.cfg')
}
self.etl = ETL(cfg_dict)
def setUp(self):
super(Ogr2OgrExecOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir,
'configs/ogr2ogrexecoutput.cfg')
}
self.etl = ETL(cfg_dict)
def setUp(self):
super(ZipFileExtractorTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir,
'configs/zipfileextractor.cfg')
}
self.etl = ETL(cfg_dict)
class CommandExecOutputTest(StetlTestCase):
"""Unit tests for CommandExecOutput"""
def setUp(self):
super(CommandExecOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir,
'configs/commandexecoutput.cfg')
}
self.etl = ETL(cfg_dict)
def test_class(self):
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, -1)
class_name = self.etl.configdict.get(section, 'class')
self.assertEqual('outputs.execoutput.CommandExecOutput', class_name)
def test_instance(self):
chain = StetlTestCase.get_chain(self.etl)
self.assertTrue(isinstance(chain.cur_comp, CommandExecOutput))
@mock.patch('subprocess.call', autospec=True)
def test_execute(self, mock_call):
# Read content of input file
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain)
fn = self.etl.configdict.get(section, 'file_path')
with open(fn, 'r') as f:
contents = f.read()
self.etl.run()
self.assertTrue(mock_call.called)
self.assertEqual(1, mock_call.call_count)
args, kwargs = mock_call.call_args
self.assertEqual(contents, args[0])
class PostgresDbOutputTest(StetlTestCase):
"""Unit tests for PostgresDbOutput"""
def setUp(self):
super(PostgresDbOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/postgresdboutput.cfg')}
self.etl = ETL(cfg_dict)
def test_class(self):
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, -1)
class_name = self.etl.configdict.get(section, 'class')
self.assertEqual('outputs.dboutput.PostgresDbOutput', class_name)
def test_instance(self):
chain = StetlTestCase.get_chain(self.etl)
self.assertTrue(isinstance(chain.cur_comp, PostgresDbOutput))
@mock.patch('stetl.postgis.PostGIS.tx_execute', autospec=True)
def test_execute(self, mock_tx_execute):
# Read content of input file
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain)
fn = self.etl.configdict.get(section, 'file_path')
with open(fn, 'r') as f:
contents = f.read()
self.etl.run()
self.assertTrue(mock_tx_execute.called)
self.assertEqual(1, mock_tx_execute.call_count)
args, kwargs = mock_tx_execute.call_args
self.assertEqual(contents, args[1])
def test_args_dict_env_override(self):
args_dict = {'in_file': 'infile.txt', 'out_file': 'outfile.txt'}
# Override in OS env
os.environ['stetl_in_file'] = 'env_infile.txt'
etl = ETL(self.cfg_dict, args_dict)
# Test args substitution from args_dict
self.assertEqual(etl.configdict.get('input_file', 'file_path'),
os.environ['stetl_in_file'])
self.assertEqual(etl.configdict.get('output_file', 'file_path'),
'outfile.txt')
class StandardOutputTest(StetlTestCase):
"""Unit tests for StandardOutput"""
def setUp(self):
super(StandardOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir, 'configs/standardoutput.cfg')
}
self.etl = ETL(cfg_dict)
def test_class(self):
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, -1)
class_name = self.etl.configdict.get(section, 'class')
self.assertEqual('outputs.standardoutput.StandardOutput', class_name)
def test_instance(self):
chain = StetlTestCase.get_chain(self.etl)
self.assertTrue(isinstance(chain.cur_comp, StandardOutput))
def test_execute(self):
# Read content of input file
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain)
fn = self.etl.configdict.get(section, 'file_path')
with open(fn, 'r') as f:
contents = f.read()
self.etl.run()
self.assertGreater(sys.stdout.getvalue(), 0)
# Assert includes last linebreak from stdout, due to print function
self.assertEqual(sys.stdout.getvalue(), contents + '\n')
class StandardOutputTest(StetlTestCase):
"""Unit tests for StandardOutput"""
def setUp(self):
super(StandardOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/standardoutput.cfg')}
self.etl = ETL(cfg_dict)
def test_class(self):
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, -1)
class_name = self.etl.configdict.get(section, 'class')
self.assertEqual('stetl.outputs.standardoutput.StandardOutput', class_name)
def test_instance(self):
chain = StetlTestCase.get_chain(self.etl)
self.assertTrue(isinstance(chain.cur_comp, StandardOutput))
def test_execute(self):
# Read content of input file
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain)
fn = self.etl.configdict.get(section, 'file_path')
with open(fn, 'r') as f:
contents = f.read()
self.etl.run()
self.assertGreater(len(sys.stdout.getvalue()), 0)
# Assert includes last linebreak from stdout, due to print function
self.assertEqual(sys.stdout.getvalue(), contents + '\n')
def test_legacy_class(self):
cfg_dict = {
'config_file':
os.path.join(self.curr_dir,
'configs/zipfileextractordeprecated.cfg')
}
self.etl = ETL(cfg_dict)
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, 1)
class_name = self.etl.configdict.get(section, 'class')
# Deprecated class name in config
self.assertEqual('stetl.filters.zipfileextractor.ZipFileExtractor',
class_name)
# Assigned to new version in fileextractor module!
self.assertTrue(isinstance(chain.first_comp.next, ZipFileExtractor))
def test_args_dict_env_all(self):
"""
Substitute ALL args from OS env.
:return:
"""
# Set all args in in OS env
os.environ['stetl_in_file'] = 'env_infile.txt'
os.environ['stetl_out_file'] = 'env_outfile.txt'
args_dict = None
etl = ETL(self.cfg_dict, args_dict)
# Test args substitution from args_dict
self.assertEqual(etl.configdict.get('input_file', 'file_path'),
os.environ['stetl_in_file'])
self.assertEqual(etl.configdict.get('output_file', 'file_path'),
os.environ['stetl_out_file'])
def process(args): etl = ETL(args, args) etl.run()
def test_type(self):
cfg_dict = {'config_file': 'tests/configs/copy_in_out.cfg'}
etl = ETL(cfg_dict)
self.failUnlessEqual(etl.configdict.get('etl', 'chains'),
'input_xml_file|output_std')
def process(args):
etl = ETL(args, args)
etl.run()
