map(lambda r: r.run(self, checks_to_run), self.job_template.rules) [source.close_connection() for source in self.job_template.data_sources] # Dedupe checks_to_run even against checks. Expect of tuples of format (DataSource, table_name_string, Check) seen = set() seen_add = seen.add checks_to_run = [c for c in checks_to_run if not ((c[0].id, c[1], c[2].id) in seen or seen_add((c[0].id, c[1], c[2].id)))] if len(checks_to_run) > 0: # Bucketize checks based on parallelization chosen. Each bucket runs sequentially. checks_by_parallelization = self.get_checks_by_parallelization(checks_to_run) # Run each bucket of checks in a separate celery worker, by turning each subarray into an array of celery run check # job signatures, and then splatting each array of run check signatures into a chain(requiring them to be done one # at a time in each chain), then you group all chains together so they run in parallel. Each chain is a worker. # Then finally you call register finished when all done. separate_queues = [map(lambda c: celery_jobs.job_runs.run_check.si(c[0].id, c[1], c[2].id, self.id), chks) for chks in checks_by_parallelization] sep_chains = [chain(*queue) for queue in separate_queues] print sep_chains group_of_chains = (group(*sep_chains) | celery_jobs.job_runs.register_finished.s(self.id)).apply_async() else: self.set_finished() except Exception: self.set_failed() db_session.add(log) db_session.commit() timestamps_triggers(JobRun)
and all it's checks and rules cloned as well. """ self.parent_job_template_id = self.id self.read_only = True rules = self.rules ds = self.data_sources # Grab before expunging. db_session.expunge(self) make_transient(self) [r.become_read_only_clone() for r in rules] [d.become_read_only_clone() for d in ds] self.id = None db_session.add(self) db_session.commit() self.rules = rules self.data_sources = ds def checks(self): seen = set() seen_add = seen.add all_checks = np.array(map(lambda r: r.checks, self.rules)).flatten() checks = [c for c in all_checks if not c.id in seen or seen_add(c.id)] return checks timestamps_triggers(JobTemplate)
{"traceback": traceback.format_exc()}) def add_log(self, event, message, metadata={}): if not self.log: self.log = [] self.log.append(self.__class__.new_event(event, message, metadata)) # http://stackoverflow.com/questions/30088089/sqlalchemy-json-typedecorator-not-saving-correctly-issues-with-session-commit flag_modified(self, "log") @event.listens_for(HasLogs, "mapper_configured", propagate=True) def setup_listener(mapper, class_): name = class_.__name__ loggable_type = name.lower() class_.logs = relationship( Log, primaryjoin=and_(class_.id == foreign(remote(Log.loggable_id)), Log.loggable_type == loggable_type), backref=backref("parent_%s" % loggable_type, primaryjoin=remote(class_.id) == foreign( Log.loggable_id)), cascade="all, delete-orphan") @event.listens_for(class_.logs, "append") def append_address(target, value, initiator): value.loggable_type = loggable_type timestamps_triggers(Log)
return host def config(self): return { "host": self.host, "port": self.port, "user": self.user, "password": self.password, "dbname": self.dbname, "data_source_type": self.data_source_type.value } def open_connection(self): klazz = eval(self.config()['data_source_type'].title() + "Connection") self.db = klazz(**self.config()) def close_connection(self): self.db.close() def tables(self): return self.db.tables(self.schemas) def col_present(self, table, column): return self.db.col_present(table, column) def count(self, table): return self.db.count(table) timestamps_triggers(DataSource)
metadata["log_metadata"]) db_session.add(job_run) if (job_run.status in [ JobRunStatus.failed, JobRunStatus.cancelled, JobRunStatus.rejected ]): log.add_log( "cancelled", "Check cancelled due to Job Run Status of %s caused by some other worker." % (job_run.status)) else: chk_class = eval( camelize(str(self.check_type.value)) + "Check") check = chk_class(metadata) check.run() log.add_log("finished", "Check Ended", metadata["log_metadata"]) except Exception as e: print str(sys.exc_info()) log.new_error_event("Check Failed due to Error", metadata["log_metadata"]) db_session.commit() raise db_session.commit() timestamps_triggers(Check)
tables_and_sources = [] [tables_and_sources.append([source, source.tables()]) for source in job_run.job_template.data_sources] return tables_and_sources def run(self, job_run, checks_to_run, tables_and_sources = None): tables_and_sources = self.all_tables_with_source(job_run) if tables_and_sources == None else tables_and_sources log = self.get_log(job_run=job_run) log.add_log("creation", "Begin Rule Check") try: log.add_log("check", "Checking %s condition for conditional %s" % (self.condition.value, self.conditional)) # Whittle down the [source, [table1, table2...]] to same structure but only with tables matching condition. tables_and_sources_matching = [getattr(self, self.condition.value)(self.conditional, source_and_tables[0], source_and_tables[1], job_run) for source_and_tables in tables_and_sources] # Add tuples like (source, table, checkObj) to checks_to_run array by iterating over every source, # every table in that source, every check for that table. Triple generator magic. [[[checks_to_run.append((source_and_tables[0], table, check)) for check in self.checks] for table in source_and_tables[1]] for source_and_tables in tables_and_sources_matching] # Now allow any children rules to apply to tables that have been matched by this rule: [child.run(job_run, checks_to_run, tables_and_sources_matching) for child in self.children] log.add_log("finished", "Rule Check Ended") except Exception: print str(sys.exc_info()) log.new_error_event() raise timestamps_triggers(Rule)