def __init__(self, configdict, section):
FileInput.__init__(self, configdict, section, produces=FORMAT.string)
self.file = None
# Optional formatting of content according to Python String.format()
# Input file should have substitutable values like {schema} {foo}
# format_args should be of the form format_args = schema:test foo:bar
if self.format_args:
# Convert string to dict: http://stackoverflow.com/a/1248990
self.format_args = Util.string_to_dict(self.format_args, ':')
def __init__(self, configdict, section):
StringFilter.__init__(self,
configdict,
section,
consumes=FORMAT.string,
produces=FORMAT.string)
# Convert string to dict: http://stackoverflow.com/a/1248990
self.format_args_dict = Util.string_to_dict(self.format_args,
self.separator)
def __init__(self, configdict, section):
FileInput.__init__(self, configdict, section, produces=FORMAT.string)
self.file = None
# Optional formatting of content according to Python String.format()
# Input file should have substitutable values like {schema} {foo}
# format_args should be of the form format_args = schema:test foo:bar
if self.format_args:
# Convert string to dict: http://stackoverflow.com/a/1248990
self.format_args = Util.string_to_dict(self.format_args, ':')
def __init__(self, configdict, section):
StringFilter.__init__(self, configdict, section, consumes=FORMAT.string, produces=FORMAT.string)
# Formatting of content according to Python String.format()
# String should have substitutable values like {schema} {foo}
# format_args should be of the form format_args = schema:test foo:bar ...
self.format_args = self.cfg.get('format_args')
# Convert string to dict: http://stackoverflow.com/a/1248990
self.format_args_dict = Util.string_to_dict(self.format_args, ':')
def execute_cmd(self, cmd):
env_vars = Util.string_to_dict(self.env_args, self.env_separator)
old_environ = os.environ.copy()
try:
os.environ.update(env_vars)
log.info("executing cmd=%s" % Util.safe_string_value(cmd))
subprocess.call(cmd, shell=True)
log.info("execute done")
finally:
os.environ = old_environ
def execute_cmd(self, cmd):
env_vars = Util.string_to_dict(self.env_args, self.env_separator)
old_environ = os.environ.copy()
try:
os.environ.update(env_vars)
log.info("executing cmd=%s" % Util.safe_string_value(cmd))
subprocess.call(cmd, shell=True)
log.info("execute done")
finally:
os.environ = old_environ
def execute_cmd(self, cmd):
env_vars = Util.string_to_dict(self.env_args, self.env_separator)
old_environ = os.environ.copy()
try:
os.environ.update(env_vars)
log.info("executing cmd=%s" % cmd)
result = subprocess.check_output(cmd, shell=True)
log.info("execute done")
return result
finally:
os.environ = old_environ
def execute_cmd(self, cmd):
env_vars = Util.string_to_dict(self.env_args, self.env_separator)
old_environ = os.environ.copy()
try:
os.environ.update(env_vars)
log.info("executing cmd=%s" % cmd)
result = subprocess.check_output(cmd, shell=True)
log.info("execute done")
return result
finally:
os.environ = old_environ
def __init__(self, configdict, section):
StringFilter.__init__(self,
configdict,
section,
consumes=FORMAT.string,
produces=FORMAT.string)
# Formatting of content according to Python String.format()
# String should have substitutable values like {schema} {foo}
# format_args should be of the form format_args = schema:test foo:bar ...
self.format_args = self.cfg.get('format_args')
# Convert string to dict: http://stackoverflow.com/a/1248990
self.format_args_dict = Util.string_to_dict(self.format_args, ':')
def parse_args(args_list):
log.info("Stetl version = %s" % __version__)
argparser = argparse.ArgumentParser(description='Invoke Stetl')
argparser.add_argument('-c ', '--config', type=str, help='ETL config file in .ini format', dest='config_file',
required=False)
argparser.add_argument('-s ', '--section', type=str, help='Section in the config file to execute, default is [etl]',
dest='config_section', required=False)
argparser.add_argument('-a ', '--args', type=str,
help='Arguments or .properties files to be substituted for symbolic {argN}s in Stetl config file,\
as -a "arg1=foo arg2=bar" and/or -a args.properties, multiple -a options are possible',
dest='config_args', required=False, action='append')
argparser.add_argument('-d ', '--doc', type=str,
help='Get component documentation like its configuration parameters, e.g. stetl doc stetl.inputs.fileinput.FileInput',
dest='doc_args', required=False)
argparser.add_argument('-v', '--version',
action='store_true',
help='Show current version of stetl and exit',
required=False)
args = argparser.parse_args(args_list)
if args.config_args:
args_total = dict()
for arg in args.config_args:
if os.path.isfile(arg):
log.info('Found args file at: %s' % arg)
args_total = Util.merge_two_dicts(args_total, Util.propsfile_to_dict(arg))
else:
# Convert string to dict: http://stackoverflow.com/a/1248990
args_total = Util.merge_two_dicts(args_total, Util.string_to_dict(arg))
args.config_args = args_total
return args
def __init__(self, configdict, section):
StringFilter.__init__(self, configdict, section, consumes=FORMAT.string, produces=FORMAT.string)
# Convert string to dict: http://stackoverflow.com/a/1248990
self.format_args_dict = Util.string_to_dict(self.format_args, self.separator)
