Esempio n. 1
0
 def execute_entries(self, print_order=False):
     """
     iterate through entries and run them
     """
     self.order_entries()
     if print_order:
         self.log_message("Printing order of steps",status='start',step='execute entries')
         self.print_the_order()
         self.log_message("Printing order of steps",status='complete')
         return 0
     self.ent_result = {}
     start_time = RawDataUtilities.get_now()
     self.log_message("Now preparing to run config entries",status='start',step='execute entries')
     for i in xrange(len(self.entries)):
         if len(self.entries[i]) > 0:
             self.log_message('processing all configs of type: '+str(self.entries[i][0].get_entry_type())+", now..",status='running',name='congig_file_reader')
         for ent in self.entries[i]:
             if self.do_run(ent):
                 self.log_message(' About to run: '+ent.name, name=ent.name)
                 print "" if ent.name == 'kicker' else '... running '+ent.name
                 ent.execute_entry()
                 self.total_time = RawDataUtilities.get_diff(start_time,seconds=True)
                 self.ent_result[ent.name] = ent.return_val
                 ent.updates['last_processed'] = RawDataUtilities.string_from_date(RawDataUtilities.add_seconds_to_date(self.run_date, self.total_time))
             else:
                 if ent.no_run_reason:
                     print "NOT RUNNING: "+ent.name + " ("+ent.no_run_reason+")"
                 
     
     self.log_message("All entries have run",status='complete')
     total_ret = self.log_summary(self.update_all_configs())
     return total_ret
Esempio n. 2
0
    def execute_connection(self):
        """
        Re-read the class description :)
        
        parameters needed:
        file to strip (strip_file): assumption is that it will be found in the working_directory
        date field to use (date_field):  Most common date formats can be used.. recommended is MM/DD/YYYY or 01/01/2015
        strip parameters (strip_criteria):  pass in 2 elements (separated by a comma), Unit, Date Unit (Day, Month, Year)  example strip_criteria = 6,Month 
            (default is 1,Month)
        
        by default this will be from the run date (which defaults to today's date unless set otherwise), but can be from the "last_processed" date by passing in the optional parameter 
            "use_last_processed = true"
        
        another option is to simply pass in the date to use for stripping.. strip_date = 01/01/2015 
        
        Anything occuring before (not equal to) that date will be pulled off of the file
        
        the stripped off records will be saved under the connector name passed in the parameters        
        """
        filename = self.param_dict['strip_file']
        date_field = self.param_dict['date_field']
        units, date_unit = self.param_dict.get('strip_criteria',
                                               '1,Month').split(",")
        use_last = True if self.param_dict.get(
            "use_last_processed", "false").upper == "TRUE" else False
        strip_date = self.entry.last_processed if use_last else RawDataUtilities.date_from_string(
            self.param_dict.get(
                "strip_date",
                RawDataUtilities.string_from_date(self.entry.today_dt)))

        match_date = self.get_match_date(units, date_unit, strip_date)

        file_in = os.path.join(self.entry.working_directory, filename)
        with open(file_in) as csvfile:
            reader = csv.DictReader(csvfile)
            self.setup_csv_temp_writer(self.get_temp_csv_name(),
                                       reader.fieldnames)
            for row in reader:
                compare_date = RawDataUtilities.date_from_string(
                    row[date_field])
                # should subtract the date on the file from match date
                # Example any dates less than 10/1/2014
                # Compare date = 09/1/2014
                # Difference is less than 0
                diff = RawDataUtilities.get_diff(match_date, compare_date)
                if diff < 0:
                    self.write_temp_rec(row)
            self.close_temp_csv()
        return 0
Esempio n. 3
0
 def run_now(self,run_date=None):
     """
     Determine if this entry should be run today
     """   
     if self.last_run == 'success':
         diff_date = self.today_dt
         if run_date:
             diff_date = run_date
         freq = self.frequencies[self.run_frequency]
         num_since_last_run = RawDataUtilities.get_diff(self.last_processed,diff_date)
         if num_since_last_run >= freq or self.first_run or freq < 0:
             return True
     else:
         self.no_run_reason = "Last Run Ended In Failure"
     return False
Esempio n. 4
0
    def execute_connection(self):
        """
        Re-read the class description :)
        
        parameters needed:
        file to strip (strip_file): assumption is that it will be found in the working_directory
        date field to use (date_field):  Most common date formats can be used.. recommended is MM/DD/YYYY or 01/01/2015
        strip parameters (strip_criteria):  pass in 2 elements (separated by a comma), Unit, Date Unit (Day, Month, Year)  example strip_criteria = 6,Month 
            (default is 1,Month)
        
        by default this will be from the run date (which defaults to today's date unless set otherwise), but can be from the "last_processed" date by passing in the optional parameter 
            "use_last_processed = true"
        
        another option is to simply pass in the date to use for stripping.. strip_date = 01/01/2015 
        
        Anything occuring before (not equal to) that date will be pulled off of the file
        
        the stripped off records will be saved under the connector name passed in the parameters        
        """
        filename = self.param_dict["strip_file"]
        date_field = self.param_dict["date_field"]
        units, date_unit = self.param_dict.get("strip_criteria", "1,Month").split(",")
        use_last = True if self.param_dict.get("use_last_processed", "false").upper == "TRUE" else False
        strip_date = (
            self.entry.last_processed
            if use_last
            else RawDataUtilities.date_from_string(
                self.param_dict.get("strip_date", RawDataUtilities.string_from_date(self.entry.today_dt))
            )
        )

        match_date = self.get_match_date(units, date_unit, strip_date)

        file_in = os.path.join(self.entry.working_directory, filename)
        with open(file_in) as csvfile:
            reader = csv.DictReader(csvfile)
            self.setup_csv_temp_writer(self.get_temp_csv_name(), reader.fieldnames)
            for row in reader:
                compare_date = RawDataUtilities.date_from_string(row[date_field])
                # should subtract the date on the file from match date
                # Example any dates less than 10/1/2014
                # Compare date = 09/1/2014
                # Difference is less than 0
                diff = RawDataUtilities.get_diff(match_date, compare_date)
                if diff < 0:
                    self.write_temp_rec(row)
            self.close_temp_csv()
        return 0
Esempio n. 5
0
 def run_now(self, run_date=None):
     """
     Determine if this entry should be run today
     """
     if self.last_run == 'success':
         diff_date = self.today_dt
         if run_date:
             diff_date = run_date
         freq = self.frequencies[self.run_frequency]
         num_since_last_run = RawDataUtilities.get_diff(
             self.last_processed, diff_date)
         if num_since_last_run >= freq or self.first_run or freq < 0:
             return True
     else:
         self.no_run_reason = "Last Run Ended In Failure"
     return False