def main():
"""The `main` function, to be called from commandline, like `python src/main.py -c etl.cfg`.
Args:
-c --config <config_file> the Stetl config file.
-s --section <section_name> the section in the Stetl config (ini) file to execute (default is [etl]).
-a --args <arglist> sero or more substitutable args for symbolic, {arg}, values in Stetl config file, in format -a arg1=foo -a arg2=bar etc.
-d --doc <class> Get component documentation like its configuration parameters, e.g. stetl --doc stetl.inputs.fileinput.FileInput
-v --version Show the current version of stelt and exit
-h --help get help info
"""
# Pass arguments explicitly, facilitates testing
args = parse_args(sys.argv[1:])
if args.version:
print('Stetl version: ', __version__)
exit()
if args.config_file:
# Do the ETL
etl = ETL(vars(args), args.config_args)
etl.run()
elif args.doc_args:
print_doc(args.doc_args)
else:
print('Unknown option, try stetl -h for help')
class ConfigTest(StetlTestCase):
"""Basic configuration tests"""
def setUp(self):
super(ConfigTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {"config_file": os.path.join(curr_dir, "configs/copy_in_out.cfg")}
self.etl = ETL(cfg_dict)
def test_type(self):
self.assertEqual(self.etl.configdict.get("etl", "chains"), "input_xml_file|output_std")
def test_run(self):
self.etl.run()
class ConfigTest(StetlTestCase):
"""Basic configuration tests"""
def setUp(self):
super(ConfigTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/copy_in_out.cfg')}
self.etl = ETL(cfg_dict)
def test_type(self):
self.assertEqual(self.etl.configdict.get('etl', 'chains'), 'input_xml_file|output_std')
def test_run(self):
self.etl.run()
class CommandExecOutputTest(StetlTestCase):
"""Unit tests for CommandExecOutput"""
def setUp(self):
super(CommandExecOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir,
'configs/commandexecoutput.cfg')
}
self.etl = ETL(cfg_dict)
def test_class(self):
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, -1)
class_name = self.etl.configdict.get(section, 'class')
self.assertEqual('outputs.execoutput.CommandExecOutput', class_name)
def test_instance(self):
chain = StetlTestCase.get_chain(self.etl)
self.assertTrue(isinstance(chain.cur_comp, CommandExecOutput))
@mock.patch('subprocess.call', autospec=True)
def test_execute(self, mock_call):
# Read content of input file
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain)
fn = self.etl.configdict.get(section, 'file_path')
with open(fn, 'r') as f:
contents = f.read()
self.etl.run()
self.assertTrue(mock_call.called)
self.assertEqual(1, mock_call.call_count)
args, kwargs = mock_call.call_args
self.assertEqual(contents, args[0])
class PostgresDbOutputTest(StetlTestCase):
"""Unit tests for PostgresDbOutput"""
def setUp(self):
super(PostgresDbOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/postgresdboutput.cfg')}
self.etl = ETL(cfg_dict)
def test_class(self):
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, -1)
class_name = self.etl.configdict.get(section, 'class')
self.assertEqual('outputs.dboutput.PostgresDbOutput', class_name)
def test_instance(self):
chain = StetlTestCase.get_chain(self.etl)
self.assertTrue(isinstance(chain.cur_comp, PostgresDbOutput))
@mock.patch('stetl.postgis.PostGIS.tx_execute', autospec=True)
def test_execute(self, mock_tx_execute):
# Read content of input file
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain)
fn = self.etl.configdict.get(section, 'file_path')
with open(fn, 'r') as f:
contents = f.read()
self.etl.run()
self.assertTrue(mock_tx_execute.called)
self.assertEqual(1, mock_tx_execute.call_count)
args, kwargs = mock_tx_execute.call_args
self.assertEqual(contents, args[1])
class StandardOutputTest(StetlTestCase):
"""Unit tests for StandardOutput"""
def setUp(self):
super(StandardOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir, 'configs/standardoutput.cfg')
}
self.etl = ETL(cfg_dict)
def test_class(self):
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, -1)
class_name = self.etl.configdict.get(section, 'class')
self.assertEqual('outputs.standardoutput.StandardOutput', class_name)
def test_instance(self):
chain = StetlTestCase.get_chain(self.etl)
self.assertTrue(isinstance(chain.cur_comp, StandardOutput))
def test_execute(self):
# Read content of input file
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain)
fn = self.etl.configdict.get(section, 'file_path')
with open(fn, 'r') as f:
contents = f.read()
self.etl.run()
self.assertGreater(sys.stdout.getvalue(), 0)
# Assert includes last linebreak from stdout, due to print function
self.assertEqual(sys.stdout.getvalue(), contents + '\n')
class StandardOutputTest(StetlTestCase):
"""Unit tests for StandardOutput"""
def setUp(self):
super(StandardOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/standardoutput.cfg')}
self.etl = ETL(cfg_dict)
def test_class(self):
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, -1)
class_name = self.etl.configdict.get(section, 'class')
self.assertEqual('stetl.outputs.standardoutput.StandardOutput', class_name)
def test_instance(self):
chain = StetlTestCase.get_chain(self.etl)
self.assertTrue(isinstance(chain.cur_comp, StandardOutput))
def test_execute(self):
# Read content of input file
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain)
fn = self.etl.configdict.get(section, 'file_path')
with open(fn, 'r') as f:
contents = f.read()
self.etl.run()
self.assertGreater(len(sys.stdout.getvalue()), 0)
# Assert includes last linebreak from stdout, due to print function
self.assertEqual(sys.stdout.getvalue(), contents + '\n')
def process(args):
etl = ETL(args, args)
etl.run()
def process(args): etl = ETL(args, args) etl.run()
