def execute_entries(self, print_order=False): """ iterate through entries and run them """ self.order_entries() if print_order: self.log_message("Printing order of steps",status='start',step='execute entries') self.print_the_order() self.log_message("Printing order of steps",status='complete') return 0 self.ent_result = {} start_time = RawDataUtilities.get_now() self.log_message("Now preparing to run config entries",status='start',step='execute entries') for i in xrange(len(self.entries)): if len(self.entries[i]) > 0: self.log_message('processing all configs of type: '+str(self.entries[i][0].get_entry_type())+", now..",status='running',name='congig_file_reader') for ent in self.entries[i]: if self.do_run(ent): self.log_message(' About to run: '+ent.name, name=ent.name) print "" if ent.name == 'kicker' else '... running '+ent.name ent.execute_entry() self.total_time = RawDataUtilities.get_diff(start_time,seconds=True) self.ent_result[ent.name] = ent.return_val ent.updates['last_processed'] = RawDataUtilities.string_from_date(RawDataUtilities.add_seconds_to_date(self.run_date, self.total_time)) else: if ent.no_run_reason: print "NOT RUNNING: "+ent.name + " ("+ent.no_run_reason+")" self.log_message("All entries have run",status='complete') total_ret = self.log_summary(self.update_all_configs()) return total_ret
def execute_connection(self): """ Re-read the class description :) parameters needed: file to strip (strip_file): assumption is that it will be found in the working_directory date field to use (date_field): Most common date formats can be used.. recommended is MM/DD/YYYY or 01/01/2015 strip parameters (strip_criteria): pass in 2 elements (separated by a comma), Unit, Date Unit (Day, Month, Year) example strip_criteria = 6,Month (default is 1,Month) by default this will be from the run date (which defaults to today's date unless set otherwise), but can be from the "last_processed" date by passing in the optional parameter "use_last_processed = true" another option is to simply pass in the date to use for stripping.. strip_date = 01/01/2015 Anything occuring before (not equal to) that date will be pulled off of the file the stripped off records will be saved under the connector name passed in the parameters """ filename = self.param_dict['strip_file'] date_field = self.param_dict['date_field'] units, date_unit = self.param_dict.get('strip_criteria', '1,Month').split(",") use_last = True if self.param_dict.get( "use_last_processed", "false").upper == "TRUE" else False strip_date = self.entry.last_processed if use_last else RawDataUtilities.date_from_string( self.param_dict.get( "strip_date", RawDataUtilities.string_from_date(self.entry.today_dt))) match_date = self.get_match_date(units, date_unit, strip_date) file_in = os.path.join(self.entry.working_directory, filename) with open(file_in) as csvfile: reader = csv.DictReader(csvfile) self.setup_csv_temp_writer(self.get_temp_csv_name(), reader.fieldnames) for row in reader: compare_date = RawDataUtilities.date_from_string( row[date_field]) # should subtract the date on the file from match date # Example any dates less than 10/1/2014 # Compare date = 09/1/2014 # Difference is less than 0 diff = RawDataUtilities.get_diff(match_date, compare_date) if diff < 0: self.write_temp_rec(row) self.close_temp_csv() return 0
def convert_value(self, attr_v, convert): if isinstance(attr_v, datetime): str_date = RawDataUtilities.string_from_date(attr_v, str_format=convert) return "'"+str_date+"'" elif convert == 'str': return "'"+str(attr_v)+"'" # if the attribute isn't a string, no conversion will be done elif isinstance(attr_v,str): if convert == 'int': return int(attr_v) else: # assuming that it is float return float(attr_v) else: return attr_v
def convert_value(self, attr_v, convert): if isinstance(attr_v, datetime): str_date = RawDataUtilities.string_from_date(attr_v, str_format=convert) return "'" + str_date + "'" elif convert == "str": return "'" + str(attr_v) + "'" # if the attribute isn't a string, no conversion will be done elif isinstance(attr_v, str): if convert == "int": return int(attr_v) else: # assuming that it is float return float(attr_v) else: return attr_v
def execute_connection(self): """ Re-read the class description :) parameters needed: file to strip (strip_file): assumption is that it will be found in the working_directory date field to use (date_field): Most common date formats can be used.. recommended is MM/DD/YYYY or 01/01/2015 strip parameters (strip_criteria): pass in 2 elements (separated by a comma), Unit, Date Unit (Day, Month, Year) example strip_criteria = 6,Month (default is 1,Month) by default this will be from the run date (which defaults to today's date unless set otherwise), but can be from the "last_processed" date by passing in the optional parameter "use_last_processed = true" another option is to simply pass in the date to use for stripping.. strip_date = 01/01/2015 Anything occuring before (not equal to) that date will be pulled off of the file the stripped off records will be saved under the connector name passed in the parameters """ filename = self.param_dict["strip_file"] date_field = self.param_dict["date_field"] units, date_unit = self.param_dict.get("strip_criteria", "1,Month").split(",") use_last = True if self.param_dict.get("use_last_processed", "false").upper == "TRUE" else False strip_date = ( self.entry.last_processed if use_last else RawDataUtilities.date_from_string( self.param_dict.get("strip_date", RawDataUtilities.string_from_date(self.entry.today_dt)) ) ) match_date = self.get_match_date(units, date_unit, strip_date) file_in = os.path.join(self.entry.working_directory, filename) with open(file_in) as csvfile: reader = csv.DictReader(csvfile) self.setup_csv_temp_writer(self.get_temp_csv_name(), reader.fieldnames) for row in reader: compare_date = RawDataUtilities.date_from_string(row[date_field]) # should subtract the date on the file from match date # Example any dates less than 10/1/2014 # Compare date = 09/1/2014 # Difference is less than 0 diff = RawDataUtilities.get_diff(match_date, compare_date) if diff < 0: self.write_temp_rec(row) self.close_temp_csv() return 0
def __init__(self, params): """ Constructor """ Base_Type.__init__(self) ## class variables self.updates={} self.return_val = None self.description = None self.connection_type = None self.last_processed = None self.num_run = 0 self.message = None self.no_run_reason = 'Schedule Not Ready' ### valid options... Default is Append # Append - Add records to the end # Overlay - replace the current with the new # New - Add a new file with the run number on the end Most current result has the name with no number self.file_write=None ## Valid Options... Default is Temp CSV file # Dataframe # ListList # DictionaryList # TempCSV # None self.temp_results=None self.out_filename = None self.src_implementation = None self.run_frequency = None ## location where results are written self.working_directory = None self.options = {} self.today_dt = None self.source_file = None self.instance = None self.temp_csv = None self.csv = None ## required fields self.today_dt = params['run_date'] self.name = params['name'] self.source_file = params['source_file'] self.description = params['description'] self.src_implementation = params['src_implementation'] self.working_directory = params['working_directory'] ## optional fields with defaults self.dependencies = [] if (params.get('dependencies', None)) == None else params['dependencies'].split(",") self.dependencies.append('kicker') self.successors= [] if (params.get('successors', None)) == None else params['successors'].split(",") self.connection_type = params.get('connection_type','none') # if it hasn't run before it will be empty self.first_run = False if params.get('last_processed',None): self.last_processed = params['last_processed'] else: self.last_processed = RawDataUtilities.string_from_date(self.today_dt) self.first_run = True self.last_processed = RawDataUtilities.date_from_string(self.last_processed) self.num_run = int(params.get('num_run',0)) self.out_filename = params.get('out_filename',self.name) self.run_frequency = params.get('run_frequency','Daily') self.temp_results = params.get('temp_results','TempCSV') self.file_write = params.get('file_write','Append') #self.entry_type = params.get('entry_type', 'Connector') self.last_run = params.get('last_run','success') self.instantiate_instance = True if params.get("instantiate","true") == "true" else False ## parameters to pass down to the entry implementation self.options = params.get('non_std',{}) self.updates['last_processed'] = RawDataUtilities.string_from_date(self.today_dt) self.updates['num_run'] = str(self.num_run + 1) self.order = 0 self.ready = True if self.instantiate_instance and self.get_instance(): self.log_message("Initialization Complete (success): "+self.name, log_type='entry', status='running', step='load configs',name='config_entry',log_level=self.log_info()) else: if self.instantiate_instance: self.ready = False self.log_message("Initialization Complete (failure): "+self.name, log_type='entry', status='running', step='load configs',name='config_entry',log_level=self.log_info())
def __init__(self, params): """ Constructor """ Base_Type.__init__(self) ## class variables self.updates = {} self.return_val = None self.description = None self.connection_type = None self.last_processed = None self.num_run = 0 self.message = None self.no_run_reason = 'Schedule Not Ready' ### valid options... Default is Append # Append - Add records to the end # Overlay - replace the current with the new # New - Add a new file with the run number on the end Most current result has the name with no number self.file_write = None ## Valid Options... Default is Temp CSV file # Dataframe # ListList # DictionaryList # TempCSV # None self.temp_results = None self.out_filename = None self.src_implementation = None self.run_frequency = None ## location where results are written self.working_directory = None self.options = {} self.today_dt = None self.source_file = None self.instance = None self.temp_csv = None self.csv = None ## required fields self.today_dt = params['run_date'] self.name = params['name'] self.source_file = params['source_file'] self.description = params['description'] self.src_implementation = params['src_implementation'] self.working_directory = params['working_directory'] ## optional fields with defaults self.dependencies = [] if (params.get( 'dependencies', None)) == None else params['dependencies'].split(",") self.dependencies.append('kicker') self.successors = [] if (params.get( 'successors', None)) == None else params['successors'].split(",") self.connection_type = params.get('connection_type', 'none') # if it hasn't run before it will be empty self.first_run = False if params.get('last_processed', None): self.last_processed = params['last_processed'] else: self.last_processed = RawDataUtilities.string_from_date( self.today_dt) self.first_run = True self.last_processed = RawDataUtilities.date_from_string( self.last_processed) self.num_run = int(params.get('num_run', 0)) self.out_filename = params.get('out_filename', self.name) self.run_frequency = params.get('run_frequency', 'Daily') self.temp_results = params.get('temp_results', 'TempCSV') self.file_write = params.get('file_write', 'Append') #self.entry_type = params.get('entry_type', 'Connector') self.last_run = params.get('last_run', 'success') self.instantiate_instance = True if params.get( "instantiate", "true") == "true" else False ## parameters to pass down to the entry implementation self.options = params.get('non_std', {}) self.updates['last_processed'] = RawDataUtilities.string_from_date( self.today_dt) self.updates['num_run'] = str(self.num_run + 1) self.order = 0 self.ready = True if self.instantiate_instance and self.get_instance(): self.log_message("Initialization Complete (success): " + self.name, log_type='entry', status='running', step='load configs', name='config_entry', log_level=self.log_info()) else: if self.instantiate_instance: self.ready = False self.log_message("Initialization Complete (failure): " + self.name, log_type='entry', status='running', step='load configs', name='config_entry', log_level=self.log_info())