def main():
"""The `main` function, to be called from commandline, like `python src/main.py -c etl.cfg`.
Args:
-c --config <config_file> the Stetl config file.
-s --section <section_name> the section in the Stetl config (ini) file to execute (default is [etl]).
-a --args <arglist> sero or more substitutable args for symbolic, {arg}, values in Stetl config file, in format -a arg1=foo -a arg2=bar etc.
-d --doc <class> Get component documentation like its configuration parameters, e.g. stetl --doc stetl.inputs.fileinput.FileInput
-v --version Show the current version of stelt and exit
-h --help get help info
"""
# Pass arguments explicitly, facilitates testing
args = parse_args(sys.argv[1:])
if args.version:
print('Stetl version: ', __version__)
exit()
if args.config_file:
# Do the ETL
etl = ETL(vars(args), args.config_args)
etl.run()
elif args.doc_args:
print_doc(args.doc_args)
else:
print('Unknown option, try stetl -h for help')
def setUp(self):
super(ChainTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/copy_in_out.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(GlobFileInputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/globfileinput.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(StringConcatFilterTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/stringconcatfilter.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(VsiFilterTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/vsifiltertest.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(MergerMultiInputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/mergermultiinput.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(StandardOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/standardoutput.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(ZipArchiveExpanderTest, self).setUp()
# Initialize Stetl
self.curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(self.curr_dir, 'configs/ziparchiveexpander.cfg')}
self.etl = ETL(cfg_dict)
def setUp(self):
super(XmlElementReaderTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {'config_file': os.path.join(curr_dir, 'configs/xmlelementreader.cfg')}
self.etl = ETL(cfg_dict)
def test_args_dict(self):
args_dict = {'in_file': 'infile.txt', 'out_file': 'outfile.txt'}
etl = ETL(self.cfg_dict, args_dict)
# Test args substitution from args_dict
self.assertEqual(etl.configdict.get('input_file', 'file_path'),
'infile.txt')
self.assertEqual(etl.configdict.get('output_file', 'file_path'),
'outfile.txt')
def setUp(self):
super(PacketWriterTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir, 'configs/packetwriter.cfg')
}
self.etl = ETL(cfg_dict)
def setUp(self):
super(CommandExecFilterTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir,
'configs/commandexecfilter.cfg')
}
self.etl = ETL(cfg_dict)
def setUp(self):
super(Ogr2OgrExecOutputTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir,
'configs/ogr2ogrexecoutput.cfg')
}
self.etl = ETL(cfg_dict)
def setUp(self):
super(ZipFileExtractorTest, self).setUp()
# Initialize Stetl
curr_dir = os.path.dirname(os.path.realpath(__file__))
cfg_dict = {
'config_file': os.path.join(curr_dir,
'configs/zipfileextractor.cfg')
}
self.etl = ETL(cfg_dict)
def test_args_dict_env_override(self):
args_dict = {'in_file': 'infile.txt', 'out_file': 'outfile.txt'}
# Override in OS env
os.environ['stetl_in_file'] = 'env_infile.txt'
etl = ETL(self.cfg_dict, args_dict)
# Test args substitution from args_dict
self.assertEqual(etl.configdict.get('input_file', 'file_path'),
os.environ['stetl_in_file'])
self.assertEqual(etl.configdict.get('output_file', 'file_path'),
'outfile.txt')
def test_legacy_class(self):
cfg_dict = {
'config_file':
os.path.join(self.curr_dir,
'configs/zipfileextractordeprecated.cfg')
}
self.etl = ETL(cfg_dict)
chain = StetlTestCase.get_chain(self.etl)
section = StetlTestCase.get_section(chain, 1)
class_name = self.etl.configdict.get(section, 'class')
# Deprecated class name in config
self.assertEqual('stetl.filters.zipfileextractor.ZipFileExtractor',
class_name)
# Assigned to new version in fileextractor module!
self.assertTrue(isinstance(chain.first_comp.next, ZipFileExtractor))
def test_args_dict_env_all(self):
"""
Substitute ALL args from OS env.
:return:
"""
# Set all args in in OS env
os.environ['stetl_in_file'] = 'env_infile.txt'
os.environ['stetl_out_file'] = 'env_outfile.txt'
args_dict = None
etl = ETL(self.cfg_dict, args_dict)
# Test args substitution from args_dict
self.assertEqual(etl.configdict.get('input_file', 'file_path'),
os.environ['stetl_in_file'])
self.assertEqual(etl.configdict.get('output_file', 'file_path'),
os.environ['stetl_out_file'])
def test_type(self):
cfg_dict = {'config_file': 'tests/configs/copy_in_out.cfg'}
etl = ETL(cfg_dict)
self.failUnlessEqual(etl.configdict.get('etl', 'chains'),
'input_xml_file|output_std')
def process(args):
etl = ETL(args, args)
etl.run()
