def __init__(self, configdict, section): FileInput.__init__(self, configdict, section, produces=FORMAT.string) self.file = None # Optional formatting of content according to Python String.format() # Input file should have substitutable values like {schema} {foo} # format_args should be of the form format_args = schema:test foo:bar if self.format_args: # Convert string to dict: http://stackoverflow.com/a/1248990 self.format_args = Util.string_to_dict(self.format_args, ':')
def __init__(self, configdict, section): StringFilter.__init__(self, configdict, section, consumes=FORMAT.string, produces=FORMAT.string) # Convert string to dict: http://stackoverflow.com/a/1248990 self.format_args_dict = Util.string_to_dict(self.format_args, self.separator)
def __init__(self, configdict, section): FileInput.__init__(self, configdict, section, produces=FORMAT.string) self.file = None # Optional formatting of content according to Python String.format() # Input file should have substitutable values like {schema} {foo} # format_args should be of the form format_args = schema:test foo:bar if self.format_args: # Convert string to dict: http://stackoverflow.com/a/1248990 self.format_args = Util.string_to_dict(self.format_args, ':')
def __init__(self, configdict, section): StringFilter.__init__(self, configdict, section, consumes=FORMAT.string, produces=FORMAT.string) # Formatting of content according to Python String.format() # String should have substitutable values like {schema} {foo} # format_args should be of the form format_args = schema:test foo:bar ... self.format_args = self.cfg.get('format_args') # Convert string to dict: http://stackoverflow.com/a/1248990 self.format_args_dict = Util.string_to_dict(self.format_args, ':')
def execute_cmd(self, cmd): env_vars = Util.string_to_dict(self.env_args, self.env_separator) old_environ = os.environ.copy() try: os.environ.update(env_vars) log.info("executing cmd=%s" % Util.safe_string_value(cmd)) subprocess.call(cmd, shell=True) log.info("execute done") finally: os.environ = old_environ
def execute_cmd(self, cmd): env_vars = Util.string_to_dict(self.env_args, self.env_separator) old_environ = os.environ.copy() try: os.environ.update(env_vars) log.info("executing cmd=%s" % Util.safe_string_value(cmd)) subprocess.call(cmd, shell=True) log.info("execute done") finally: os.environ = old_environ
def execute_cmd(self, cmd): env_vars = Util.string_to_dict(self.env_args, self.env_separator) old_environ = os.environ.copy() try: os.environ.update(env_vars) log.info("executing cmd=%s" % cmd) result = subprocess.check_output(cmd, shell=True) log.info("execute done") return result finally: os.environ = old_environ
def execute_cmd(self, cmd): env_vars = Util.string_to_dict(self.env_args, self.env_separator) old_environ = os.environ.copy() try: os.environ.update(env_vars) log.info("executing cmd=%s" % cmd) result = subprocess.check_output(cmd, shell=True) log.info("execute done") return result finally: os.environ = old_environ
def __init__(self, configdict, section): StringFilter.__init__(self, configdict, section, consumes=FORMAT.string, produces=FORMAT.string) # Formatting of content according to Python String.format() # String should have substitutable values like {schema} {foo} # format_args should be of the form format_args = schema:test foo:bar ... self.format_args = self.cfg.get('format_args') # Convert string to dict: http://stackoverflow.com/a/1248990 self.format_args_dict = Util.string_to_dict(self.format_args, ':')
def parse_args(args_list): log.info("Stetl version = %s" % __version__) argparser = argparse.ArgumentParser(description='Invoke Stetl') argparser.add_argument('-c ', '--config', type=str, help='ETL config file in .ini format', dest='config_file', required=False) argparser.add_argument('-s ', '--section', type=str, help='Section in the config file to execute, default is [etl]', dest='config_section', required=False) argparser.add_argument('-a ', '--args', type=str, help='Arguments or .properties files to be substituted for symbolic {argN}s in Stetl config file,\ as -a "arg1=foo arg2=bar" and/or -a args.properties, multiple -a options are possible', dest='config_args', required=False, action='append') argparser.add_argument('-d ', '--doc', type=str, help='Get component documentation like its configuration parameters, e.g. stetl doc stetl.inputs.fileinput.FileInput', dest='doc_args', required=False) argparser.add_argument('-v', '--version', action='store_true', help='Show current version of stetl and exit', required=False) args = argparser.parse_args(args_list) if args.config_args: args_total = dict() for arg in args.config_args: if os.path.isfile(arg): log.info('Found args file at: %s' % arg) args_total = Util.merge_two_dicts(args_total, Util.propsfile_to_dict(arg)) else: # Convert string to dict: http://stackoverflow.com/a/1248990 args_total = Util.merge_two_dicts(args_total, Util.string_to_dict(arg)) args.config_args = args_total return args
def __init__(self, configdict, section): StringFilter.__init__(self, configdict, section, consumes=FORMAT.string, produces=FORMAT.string) # Convert string to dict: http://stackoverflow.com/a/1248990 self.format_args_dict = Util.string_to_dict(self.format_args, self.separator)