def _insert_match(self, obj0, obj1, check=False): """ Insert the match data for a single kim object into `matches` """ cursor = self.conn.cursor() te,mo = [str(i) for i in [obj0, obj1]] # make sure it is in runner, subject order if database.get_leader(te).lower() != 'te': te,mo = mo,te # if we have this item already, return its index o = cursor.execute("SELECT * FROM matches WHERE test==? AND model==?", (te, mo)).fetchone() if o is not None: return o[0] # if it actually isn't a match, don't put it in! if check and not kimapi.valid_match(kimobjects.kim_obj(te), kimobjects.kim_obj(mo)): return None # insert if not present cursor.execute( "INSERT INTO matches VALUES (NULL, ?, ?)", (str(te), str(mo)) ) matchid = cursor.lastrowid self.conn.commit() return matchid
def dependents_naive(te, mo): out = [] for test in kimobjects.Test.all(): deps = test.runtime_dependencies(mo) for dep in deps: if hasattr(dep, '__iter__'): tmpte, tmpmo = list2obj(dep) if te == tmpte and mo == tmpmo: if kimapi.valid_match(test, mo): out.append([test, mo]) return out
def push_jobs(self, update): """ Push all of the jobs that need to be done given an update """ kimid = update['kimid'] status = update['status'] priority_factor = self.priority_to_number(update['priority']) name,leader,num,version = database.parse_kim_code(kimid) # try to build the kimid before sending jobs # if self.make_object(kimid) == 0: # rsync_tools.director_build_write(kimid) # else: # self.logger.error("Could not build %r", kimid) # self.bsd.use(TUBE_ERRORS) # self.bsd.put(simplejson.dumps({"error": "Could not build %r" % kimid})) # return self.make_all() if leader=="VT": # for every test launch test = kimobjects.VerificationTest(kimid) models = kimobjects.Test.all() tests = [test]*ll(models) elif leader=="VM": #for all of the models, run a job test = kimobjects.VerificationModel(kimid) models = kimobjects.Model.all() tests = [test]*ll(models) else: if status == "approved": if leader=="TE": # for all of the models, add a job test = kimobjects.Test(kimid) models = list(test.models) tests = [test]*ll(models) elif leader=="MO": # for all of the tests, add a job model = kimobjects.Model(kimid) tests = list(model.tests) models = [model]*ll(tests) elif leader=="TD": # if it is a new version of an existing test driver, hunt # down all of the tests that use it and launch their # corresponding jobs driver = kimobjects.TestDriver(kimid) temp_tests = list(driver.tests) models = [] tests = [] for t in temp_tests: tmodels = list(t.models) if len(tmodels) > 0: models.extend(tmodels) tests.extend([t]*ll(tmodels)) elif leader=="MD": # if this is a new version, hunt down all of the models # that rely on it and recompute their results driver = kimobjects.ModelDriver(kimid) temp_models = list(driver.models) tests = [] models = [] for m in temp_models: mtests = list(m.tests) if len(mtests) > 0: tests.extend(mtests) models.extend([m]*ll(mtests)) else: self.logger.error("Tried to update an invalid KIM ID!: %r",kimid) checkmatch = True if status == "pending": if leader=="TE": # a pending test rsync_tools.director_test_verification_read(kimid) self.make_all() # run against all test verifications tests = list(kimobjects.VertificationTest.all()) models = [kimobjects.Test(kimid, search=False)]*ll(tests) elif leader=="MO": # a pending model rsync_tools.director_model_verification_read(kimid) self.make_all() # run against all model verifications tests = list(kimobjects.VertificationModel.all()) models = [kimobjects.Model(kimid, search=False)]*ll(tests) elif leader=="TD": # a pending test driver pass elif leader=="MD": # a pending model driver pass else: self.logger.error("Tried to update an invalid KIM ID!: %r",kimid) checkmatch = False if checkmatch: for test, model in zip(tests,models): if kimapi.valid_match(test,model): priority = int(priority_factor*database.test_model_to_priority(test,model) * 1000000) self.check_dependencies_and_push(test,model,priority,status) else: for test, model in zip(tests,models): priority = int(priority_factor*database.test_model_to_priority(test,model) * 1000000) self.check_dependencies_and_push(test,model,priority,status)
def push_jobs(self, update): """ Push all of the jobs that need to be done given an update """ self.make_all() kimid = update['kimid'] status = update['status'] priority_factor = self.priority_to_number(update['priority']) if database.isuuid(kimid): priority = int(priority_factor * 1000000) self.check_dependencies_and_push(kimid, priority, status) return name, leader, num, version = database.parse_kim_code(kimid) checkmatch = False if leader == "VT": # for every test launch test = kimobjects.TestVerification(kimid) models = list(kimobjects.Test.all()) tests = [test] * ll(models) elif leader == "VM": #for all of the models, run a job test = kimobjects.ModelVerification(kimid) models = list(kimobjects.Model.all()) tests = [test] * ll(models) else: if status == "approved": if leader == "TE": # for all of the models, add a job test = kimobjects.Test(kimid) models = list(test.models) tests = [test] * ll(models) elif leader == "MO": # for all of the tests, add a job model = kimobjects.Model(kimid) tests = list(model.tests) models = [model] * ll(tests) elif leader == "TD": # if it is a new version of an existing test driver, hunt # down all of the tests that use it and launch their # corresponding jobs driver = kimobjects.TestDriver(kimid) temp_tests = list(driver.tests) models = [] tests = [] for t in temp_tests: tmodels = list(t.models) if len(tmodels) > 0: models.extend(tmodels) tests.extend([t] * ll(tmodels)) elif leader == "MD": # if this is a new version, hunt down all of the models # that rely on it and recompute their results driver = kimobjects.ModelDriver(kimid) temp_models = list(driver.models) tests = [] models = [] for m in temp_models: mtests = list(m.tests) if len(mtests) > 0: tests.extend(mtests) models.extend([m] * ll(mtests)) else: self.logger.error("Tried to update an invalid KIM ID!: %r", kimid) checkmatch = True if status == "pending": rsync_tools.director_pending_read(kimid) self.make_all() if leader == "TE": # run against all test verifications tests = list(kimobjects.VertificationTest.all()) models = [kimobjects.Test(kimid, search=False)] * ll(tests) elif leader == "MO": # run against all model verifications tests = list(kimobjects.VertificationModel.all()) models = [kimobjects.Model(kimid, search=False) ] * ll(tests) elif leader == "TD": # a pending test driver pass elif leader == "MD": # a pending model driver pass else: self.logger.error("Tried to update an invalid KIM ID!: %r", kimid) checkmatch = False for test, model in zip(tests, models): if not checkmatch or (checkmatch and kimapi.valid_match(test, model)): priority = int(priority_factor * database.test_model_to_priority(test, model) * 1000000) self.check_dependencies_and_push((test, model), priority, status)
def tests(self): """ Return a generator of the valid matching tests that match this model """ return ( test for test in Test.all() if kimapi.valid_match(test,self) )
def subjects(self): """ Return a generator for all of the valid subjects """ return (subject for subject in self.subject_type.all() if kimapi.valid_match(self,subject) )
def run(self, job, jobid, depends=[], status='approved'): self.logger.info("Worker got job request %r", jobid) self.check_kimid(job[0]) self.check_kimid(job[1]) # check to see if this is a verifier or an actual test pending = (status == 'pending') self.logger.info("Rsyncing %r %r", job, depends) locker.lock_rsync_read(job, depends, pending) runner_kcode, subject_kcode = job runner = kimobjects.kim_obj(runner_kcode) subject = kimobjects.kim_obj(subject_kcode) self.logger.info("Building %r %r", runner, subject) locker.lock_build(runner) locker.lock_build(subject) if runner.kim_code_leader == 'TE' and subject.kim_code_leader == 'MO': # First check that the thing we are trying to compute is a match! if not kimapi.valid_match(runner, subject): self.logger.error("Invalid match (%r, %r)", runner, subject) raise cf.PipelineRuntimeError( "Invalid match reached worker (%r, %r)" % (runner, subject) ) self.logger.info("Running (%r,%r)", runner, subject) comp = compute.Computation(runner, subject, result_code=jobid) # The next bit of logic is complicated in order to get the job to # resend upon SystemExit or KeyboardInterrupt errors, but for no other # reason. error = None try: comp.run(extrainfo=self.boxinfo) except (KeyboardInterrupt, SystemExit) as e: raise e except Exception as error: self.logger.error("Encountered errors when running %r" % jobid) else: self.logger.debug("Sending result message back") # Try to rsync the results back to the gateway. If this fails, it is # not really the Test's fault. Therefore, treat this as another type of # failure (RsyncRuntimeError) so that the task is sent back into the # queue for a retry if comp.result_path: try: locker.lock_rsync_write(jobid, comp.result_path) self.send_result(jobid, comp.uuid) except (KeyboardInterrupt, SystemExit) as e: raise e except Exception as e: self.logger.exception("Worker could not rsync results! %r", e) raise cf.RsyncRuntimeError() if error: raise error self.logger.info("Worker completed job %r", jobid)