def __init__(self, params): """ Constructor """ Base_Type.__init__(self) self.print_only = False if params: if 'files' in params: self.addall_config_files(params['files']) if 'file' in params: self.add_config_file(params['file']) if 'run_date' in params: self.run_date = params['run_date'] if params.get('print_only',False): self.print_only = True self.force_name = params.get('force',None) if 'log_file' in params: directory = os.path.dirname(os.path.realpath(params['log_file'])) if not os.path.exists(directory): os.makedirs(directory) self.log_file = params['log_file'] else: run_path = os.path.dirname(os.path.realpath(sys.argv[0])) directory = os.path.join(run_path,'log') if not os.path.exists(directory): os.makedirs(directory) self.log_file = os.path.join(directory,'rawdata.log') self.setup_logging() self.log_message("Kicking Off Raw Data Processor", log_type='main', status='start', step='root', name='raw_processor')
def __init__(self, params): """ Constructor """ Base_Type.__init__(self) self.return_status = 0 self.return_results = None self.entry = params.get('entry',None) self.name = params.get('name',None) self.message = None self.csv_files = {} self.param_dict = {} self.temp_writer = None self.temp_csvfile = None
def __init__(self, params): """ Constructor """ Base_Type.__init__(self) if 'run_date' in params: self.run_date = RawDataUtilities.date_from_string(params['run_date']) self.entry_name = params.get('entry_name',None) self.total_time = 0.0 self.entries[0] = [] self.entries[1] = [] #self.set_uuid(params.get('uuid','---none---')) self.log_message("Initialization Complete", log_type='main', status='running', step='load configs',name='config_file_reader',log_level=self.log_info())
def __init__(self, params): """ Constructor """ Base_Type.__init__(self) self.print_only = False if params: if 'files' in params: self.addall_config_files(params['files']) if 'file' in params: self.add_config_file(params['file']) if 'run_date' in params: self.run_date = params['run_date'] if params.get('print_only', False): self.print_only = True self.force_name = params.get('force', None) if 'log_file' in params: directory = os.path.dirname( os.path.realpath(params['log_file'])) if not os.path.exists(directory): os.makedirs(directory) self.log_file = params['log_file'] else: run_path = os.path.dirname(os.path.realpath(sys.argv[0])) directory = os.path.join(run_path, 'log') if not os.path.exists(directory): os.makedirs(directory) self.log_file = os.path.join(directory, 'rawdata.log') self.setup_logging() self.log_message("Kicking Off Raw Data Processor", log_type='main', status='start', step='root', name='raw_processor')
def __init__(self, params): """ Constructor """ Base_Type.__init__(self) ## class variables self.updates={} self.return_val = None self.description = None self.connection_type = None self.last_processed = None self.num_run = 0 self.message = None self.no_run_reason = 'Schedule Not Ready' ### valid options... Default is Append # Append - Add records to the end # Overlay - replace the current with the new # New - Add a new file with the run number on the end Most current result has the name with no number self.file_write=None ## Valid Options... Default is Temp CSV file # Dataframe # ListList # DictionaryList # TempCSV # None self.temp_results=None self.out_filename = None self.src_implementation = None self.run_frequency = None ## location where results are written self.working_directory = None self.options = {} self.today_dt = None self.source_file = None self.instance = None self.temp_csv = None self.csv = None ## required fields self.today_dt = params['run_date'] self.name = params['name'] self.source_file = params['source_file'] self.description = params['description'] self.src_implementation = params['src_implementation'] self.working_directory = params['working_directory'] ## optional fields with defaults self.dependencies = [] if (params.get('dependencies', None)) == None else params['dependencies'].split(",") self.dependencies.append('kicker') self.successors= [] if (params.get('successors', None)) == None else params['successors'].split(",") self.connection_type = params.get('connection_type','none') # if it hasn't run before it will be empty self.first_run = False if params.get('last_processed',None): self.last_processed = params['last_processed'] else: self.last_processed = RawDataUtilities.string_from_date(self.today_dt) self.first_run = True self.last_processed = RawDataUtilities.date_from_string(self.last_processed) self.num_run = int(params.get('num_run',0)) self.out_filename = params.get('out_filename',self.name) self.run_frequency = params.get('run_frequency','Daily') self.temp_results = params.get('temp_results','TempCSV') self.file_write = params.get('file_write','Append') #self.entry_type = params.get('entry_type', 'Connector') self.last_run = params.get('last_run','success') self.instantiate_instance = True if params.get("instantiate","true") == "true" else False ## parameters to pass down to the entry implementation self.options = params.get('non_std',{}) self.updates['last_processed'] = RawDataUtilities.string_from_date(self.today_dt) self.updates['num_run'] = str(self.num_run + 1) self.order = 0 self.ready = True if self.instantiate_instance and self.get_instance(): self.log_message("Initialization Complete (success): "+self.name, log_type='entry', status='running', step='load configs',name='config_entry',log_level=self.log_info()) else: if self.instantiate_instance: self.ready = False self.log_message("Initialization Complete (failure): "+self.name, log_type='entry', status='running', step='load configs',name='config_entry',log_level=self.log_info())
def __init__(self, params): """ Constructor """ Base_Type.__init__(self) ## class variables self.updates = {} self.return_val = None self.description = None self.connection_type = None self.last_processed = None self.num_run = 0 self.message = None self.no_run_reason = 'Schedule Not Ready' ### valid options... Default is Append # Append - Add records to the end # Overlay - replace the current with the new # New - Add a new file with the run number on the end Most current result has the name with no number self.file_write = None ## Valid Options... Default is Temp CSV file # Dataframe # ListList # DictionaryList # TempCSV # None self.temp_results = None self.out_filename = None self.src_implementation = None self.run_frequency = None ## location where results are written self.working_directory = None self.options = {} self.today_dt = None self.source_file = None self.instance = None self.temp_csv = None self.csv = None ## required fields self.today_dt = params['run_date'] self.name = params['name'] self.source_file = params['source_file'] self.description = params['description'] self.src_implementation = params['src_implementation'] self.working_directory = params['working_directory'] ## optional fields with defaults self.dependencies = [] if (params.get( 'dependencies', None)) == None else params['dependencies'].split(",") self.dependencies.append('kicker') self.successors = [] if (params.get( 'successors', None)) == None else params['successors'].split(",") self.connection_type = params.get('connection_type', 'none') # if it hasn't run before it will be empty self.first_run = False if params.get('last_processed', None): self.last_processed = params['last_processed'] else: self.last_processed = RawDataUtilities.string_from_date( self.today_dt) self.first_run = True self.last_processed = RawDataUtilities.date_from_string( self.last_processed) self.num_run = int(params.get('num_run', 0)) self.out_filename = params.get('out_filename', self.name) self.run_frequency = params.get('run_frequency', 'Daily') self.temp_results = params.get('temp_results', 'TempCSV') self.file_write = params.get('file_write', 'Append') #self.entry_type = params.get('entry_type', 'Connector') self.last_run = params.get('last_run', 'success') self.instantiate_instance = True if params.get( "instantiate", "true") == "true" else False ## parameters to pass down to the entry implementation self.options = params.get('non_std', {}) self.updates['last_processed'] = RawDataUtilities.string_from_date( self.today_dt) self.updates['num_run'] = str(self.num_run + 1) self.order = 0 self.ready = True if self.instantiate_instance and self.get_instance(): self.log_message("Initialization Complete (success): " + self.name, log_type='entry', status='running', step='load configs', name='config_entry', log_level=self.log_info()) else: if self.instantiate_instance: self.ready = False self.log_message("Initialization Complete (failure): " + self.name, log_type='entry', status='running', step='load configs', name='config_entry', log_level=self.log_info())