def execute_entries(self, print_order=False): """ iterate through entries and run them """ self.order_entries() if print_order: self.log_message("Printing order of steps",status='start',step='execute entries') self.print_the_order() self.log_message("Printing order of steps",status='complete') return 0 self.ent_result = {} start_time = RawDataUtilities.get_now() self.log_message("Now preparing to run config entries",status='start',step='execute entries') for i in xrange(len(self.entries)): if len(self.entries[i]) > 0: self.log_message('processing all configs of type: '+str(self.entries[i][0].get_entry_type())+", now..",status='running',name='congig_file_reader') for ent in self.entries[i]: if self.do_run(ent): self.log_message(' About to run: '+ent.name, name=ent.name) print "" if ent.name == 'kicker' else '... running '+ent.name ent.execute_entry() self.total_time = RawDataUtilities.get_diff(start_time,seconds=True) self.ent_result[ent.name] = ent.return_val ent.updates['last_processed'] = RawDataUtilities.string_from_date(RawDataUtilities.add_seconds_to_date(self.run_date, self.total_time)) else: if ent.no_run_reason: print "NOT RUNNING: "+ent.name + " ("+ent.no_run_reason+")" self.log_message("All entries have run",status='complete') total_ret = self.log_summary(self.update_all_configs()) return total_ret
def execute_connection(self): """ Re-read the class description :) parameters needed: file to strip (strip_file): assumption is that it will be found in the working_directory date field to use (date_field): Most common date formats can be used.. recommended is MM/DD/YYYY or 01/01/2015 strip parameters (strip_criteria): pass in 2 elements (separated by a comma), Unit, Date Unit (Day, Month, Year) example strip_criteria = 6,Month (default is 1,Month) by default this will be from the run date (which defaults to today's date unless set otherwise), but can be from the "last_processed" date by passing in the optional parameter "use_last_processed = true" another option is to simply pass in the date to use for stripping.. strip_date = 01/01/2015 Anything occuring before (not equal to) that date will be pulled off of the file the stripped off records will be saved under the connector name passed in the parameters """ filename = self.param_dict['strip_file'] date_field = self.param_dict['date_field'] units, date_unit = self.param_dict.get('strip_criteria', '1,Month').split(",") use_last = True if self.param_dict.get( "use_last_processed", "false").upper == "TRUE" else False strip_date = self.entry.last_processed if use_last else RawDataUtilities.date_from_string( self.param_dict.get( "strip_date", RawDataUtilities.string_from_date(self.entry.today_dt))) match_date = self.get_match_date(units, date_unit, strip_date) file_in = os.path.join(self.entry.working_directory, filename) with open(file_in) as csvfile: reader = csv.DictReader(csvfile) self.setup_csv_temp_writer(self.get_temp_csv_name(), reader.fieldnames) for row in reader: compare_date = RawDataUtilities.date_from_string( row[date_field]) # should subtract the date on the file from match date # Example any dates less than 10/1/2014 # Compare date = 09/1/2014 # Difference is less than 0 diff = RawDataUtilities.get_diff(match_date, compare_date) if diff < 0: self.write_temp_rec(row) self.close_temp_csv() return 0
def run_now(self,run_date=None): """ Determine if this entry should be run today """ if self.last_run == 'success': diff_date = self.today_dt if run_date: diff_date = run_date freq = self.frequencies[self.run_frequency] num_since_last_run = RawDataUtilities.get_diff(self.last_processed,diff_date) if num_since_last_run >= freq or self.first_run or freq < 0: return True else: self.no_run_reason = "Last Run Ended In Failure" return False
def execute_connection(self): """ Re-read the class description :) parameters needed: file to strip (strip_file): assumption is that it will be found in the working_directory date field to use (date_field): Most common date formats can be used.. recommended is MM/DD/YYYY or 01/01/2015 strip parameters (strip_criteria): pass in 2 elements (separated by a comma), Unit, Date Unit (Day, Month, Year) example strip_criteria = 6,Month (default is 1,Month) by default this will be from the run date (which defaults to today's date unless set otherwise), but can be from the "last_processed" date by passing in the optional parameter "use_last_processed = true" another option is to simply pass in the date to use for stripping.. strip_date = 01/01/2015 Anything occuring before (not equal to) that date will be pulled off of the file the stripped off records will be saved under the connector name passed in the parameters """ filename = self.param_dict["strip_file"] date_field = self.param_dict["date_field"] units, date_unit = self.param_dict.get("strip_criteria", "1,Month").split(",") use_last = True if self.param_dict.get("use_last_processed", "false").upper == "TRUE" else False strip_date = ( self.entry.last_processed if use_last else RawDataUtilities.date_from_string( self.param_dict.get("strip_date", RawDataUtilities.string_from_date(self.entry.today_dt)) ) ) match_date = self.get_match_date(units, date_unit, strip_date) file_in = os.path.join(self.entry.working_directory, filename) with open(file_in) as csvfile: reader = csv.DictReader(csvfile) self.setup_csv_temp_writer(self.get_temp_csv_name(), reader.fieldnames) for row in reader: compare_date = RawDataUtilities.date_from_string(row[date_field]) # should subtract the date on the file from match date # Example any dates less than 10/1/2014 # Compare date = 09/1/2014 # Difference is less than 0 diff = RawDataUtilities.get_diff(match_date, compare_date) if diff < 0: self.write_temp_rec(row) self.close_temp_csv() return 0
def run_now(self, run_date=None): """ Determine if this entry should be run today """ if self.last_run == 'success': diff_date = self.today_dt if run_date: diff_date = run_date freq = self.frequencies[self.run_frequency] num_since_last_run = RawDataUtilities.get_diff( self.last_processed, diff_date) if num_since_last_run >= freq or self.first_run or freq < 0: return True else: self.no_run_reason = "Last Run Ended In Failure" return False