def sync_pending(self, kimid): """ Get an object and its drivers from pending area """ rsync_tools.director_pending_read(kimid) for driver in kimobjects.kim_obj(kimid).drivers: rsync_tools.director_pending_read(kimid)
def precheck(self, kimid, status='pending'): """ Perform pre-checks on a new object """ errors = [] if cf.DIRECTOR_NOSYNC: errors.append( 'Precheck failed: DIRECTOR_NOSYNC must be set to False' ) self.logger.error('%r', errors) return errors approved = (status == 'approved') self.logger.info("Running pre-checks for KIM item %r" % kimid) # try to actually get the kimobject if not cf.DIRECTOR_NOSYNC: if approved: # Read from Gateway's local 'precheck' repository to local 'precheck' repository rsync_tools.director_approved_read(precheck=2) else: rsync_tools.director_pending_read(kimid, precheck=2) # Can we use our ORM? # NOTE: This is redundant with a check done on the Gateway, but it's not expensive try: obj = kimobjects.kim_obj(kimid, precheck=True) except Exception as e: errors.append( 'Could not initialize KIMObjects ORM:\n%r' % e ) self.logger.error('%r', errors) return errors # try to get the drivers so we can start to build, etc try: if not cf.DIRECTOR_NOSYNC: drivers = list(obj.drivers) self.logger.info("Retrieving drivers for %r, %r" % (obj, drivers)) for driver in drivers: rsync_tools.director_pending_read(str(driver), precheck=2) except Exception as e: errors.append( 'Could not find drivers associated with object:\n%r' % e ) self.logger.error('%r', errors) return errors # can it be built? try: obj.make(precheck=True) except Exception as e: errors.append( 'KIMObject could not be built using `make`:\n%r' % e ) self.logger.error('%r', errors) return errors def _assert(condition): if not condition: raise AssertionError() checks_orm = [ [lambda o: o.drivers, 'Searching for drivers failed'], [lambda o: _assert(o.kimspec), 'Does not contain kimspec'], [lambda o: _assert(o.kim_api_version), 'No KIM API version specified'], [lambda o: _assert(o.pipeline_api_version), 'No Pipeline API version specified'] ] checks_runner = [ [lambda t: t.processed_infile(next(kimobjects.Test.all(precheck=True))), 'Could not template pipeline.stdin file'], [lambda t: t.runtime_dependencies('blank'), 'Could not template dependencies file'], [lambda t: list(t.matches), 'Matches could not be generated, exceptions'], [lambda t: _assert(list(t.matches)), 'No valid matches found in system'] ] checks_subject = [ [lambda t: list(t.matches), 'Matches could not be generated, errors.'], [lambda t: _assert(list(t.matches)), 'No valid matches found in system'] ] def _run_checks(check_list, *args): for check in check_list: try: check[0](*args) except Exception as e: errors.append(check[1]) self.logger.error('%s:\n%r' % (check[1], e)) _run_checks(checks_orm, obj) if isinstance(obj, kimobjects.Runner): _run_checks(checks_runner, obj) if isinstance(obj, kimobjects.Subject): _run_checks(checks_subject, obj) if errors: self.logger.error( "Returning errors for precheck of %r:\n%r" % (kimid, errors) ) else: self.logger.info("No errors found during precheck for %r" % kimid) return errors
def push_jobs(self, update): """ Push all of the jobs that need to be done given an update """ self.make_all() kimid = update['kimid'] status = update['status'] priority_factor = self.priority_to_number(update['priority']) if database.isuuid(kimid): priority = int(priority_factor * 1000000) self.check_dependencies_and_push(kimid, priority, status) return name, leader, num, version = database.parse_kim_code(kimid) checkmatch = False if leader == "VT": # for every test launch test = kimobjects.TestVerification(kimid) models = list(kimobjects.Test.all()) tests = [test] * ll(models) elif leader == "VM": #for all of the models, run a job test = kimobjects.ModelVerification(kimid) models = list(kimobjects.Model.all()) tests = [test] * ll(models) else: if status == "approved": if leader == "TE": # for all of the models, add a job test = kimobjects.Test(kimid) models = list(test.models) tests = [test] * ll(models) elif leader == "MO": # for all of the tests, add a job model = kimobjects.Model(kimid) tests = list(model.tests) models = [model] * ll(tests) elif leader == "TD": # if it is a new version of an existing test driver, hunt # down all of the tests that use it and launch their # corresponding jobs driver = kimobjects.TestDriver(kimid) temp_tests = list(driver.tests) models = [] tests = [] for t in temp_tests: tmodels = list(t.models) if len(tmodels) > 0: models.extend(tmodels) tests.extend([t] * ll(tmodels)) elif leader == "MD": # if this is a new version, hunt down all of the models # that rely on it and recompute their results driver = kimobjects.ModelDriver(kimid) temp_models = list(driver.models) tests = [] models = [] for m in temp_models: mtests = list(m.tests) if len(mtests) > 0: tests.extend(mtests) models.extend([m] * ll(mtests)) else: self.logger.error("Tried to update an invalid KIM ID!: %r", kimid) checkmatch = True if status == "pending": rsync_tools.director_pending_read(kimid) self.make_all() if leader == "TE": # run against all test verifications tests = list(kimobjects.VertificationTest.all()) models = [kimobjects.Test(kimid, search=False)] * ll(tests) elif leader == "MO": # run against all model verifications tests = list(kimobjects.VertificationModel.all()) models = [kimobjects.Model(kimid, search=False) ] * ll(tests) elif leader == "TD": # a pending test driver pass elif leader == "MD": # a pending model driver pass else: self.logger.error("Tried to update an invalid KIM ID!: %r", kimid) checkmatch = False for test, model in zip(tests, models): if not checkmatch or (checkmatch and kimapi.valid_match(test, model)): priority = int(priority_factor * database.test_model_to_priority(test, model) * 1000000) self.check_dependencies_and_push((test, model), priority, status)