def sync_pending(self, kimid):
""" Get an object and its drivers from pending area """
rsync_tools.director_pending_read(kimid)
for driver in kimobjects.kim_obj(kimid).drivers:
rsync_tools.director_pending_read(kimid)
def precheck(self, kimid, status='pending'):
""" Perform pre-checks on a new object """
errors = []
if cf.DIRECTOR_NOSYNC:
errors.append(
'Precheck failed: DIRECTOR_NOSYNC must be set to False'
)
self.logger.error('%r', errors)
return errors
approved = (status == 'approved')
self.logger.info("Running pre-checks for KIM item %r" % kimid)
# try to actually get the kimobject
if not cf.DIRECTOR_NOSYNC:
if approved:
# Read from Gateway's local 'precheck' repository to local 'precheck' repository
rsync_tools.director_approved_read(precheck=2)
else:
rsync_tools.director_pending_read(kimid, precheck=2)
# Can we use our ORM?
# NOTE: This is redundant with a check done on the Gateway, but it's not expensive
try:
obj = kimobjects.kim_obj(kimid, precheck=True)
except Exception as e:
errors.append(
'Could not initialize KIMObjects ORM:\n%r' % e
)
self.logger.error('%r', errors)
return errors
# try to get the drivers so we can start to build, etc
try:
if not cf.DIRECTOR_NOSYNC:
drivers = list(obj.drivers)
self.logger.info("Retrieving drivers for %r, %r" % (obj, drivers))
for driver in drivers:
rsync_tools.director_pending_read(str(driver), precheck=2)
except Exception as e:
errors.append(
'Could not find drivers associated with object:\n%r' % e
)
self.logger.error('%r', errors)
return errors
# can it be built?
try:
obj.make(precheck=True)
except Exception as e:
errors.append(
'KIMObject could not be built using `make`:\n%r' % e
)
self.logger.error('%r', errors)
return errors
def _assert(condition):
if not condition:
raise AssertionError()
checks_orm = [
[lambda o: o.drivers, 'Searching for drivers failed'],
[lambda o: _assert(o.kimspec), 'Does not contain kimspec'],
[lambda o: _assert(o.kim_api_version), 'No KIM API version specified'],
[lambda o: _assert(o.pipeline_api_version), 'No Pipeline API version specified']
]
checks_runner = [
[lambda t: t.processed_infile(next(kimobjects.Test.all(precheck=True))),
'Could not template pipeline.stdin file'],
[lambda t: t.runtime_dependencies('blank'), 'Could not template dependencies file'],
[lambda t: list(t.matches), 'Matches could not be generated, exceptions'],
[lambda t: _assert(list(t.matches)), 'No valid matches found in system']
]
checks_subject = [
[lambda t: list(t.matches), 'Matches could not be generated, errors.'],
[lambda t: _assert(list(t.matches)), 'No valid matches found in system']
]
def _run_checks(check_list, *args):
for check in check_list:
try:
check[0](*args)
except Exception as e:
errors.append(check[1])
self.logger.error('%s:\n%r' % (check[1], e))
_run_checks(checks_orm, obj)
if isinstance(obj, kimobjects.Runner):
_run_checks(checks_runner, obj)
if isinstance(obj, kimobjects.Subject):
_run_checks(checks_subject, obj)
if errors:
self.logger.error(
"Returning errors for precheck of %r:\n%r" % (kimid, errors)
)
else:
self.logger.info("No errors found during precheck for %r" % kimid)
return errors
def push_jobs(self, update):
""" Push all of the jobs that need to be done given an update """
self.make_all()
kimid = update['kimid']
status = update['status']
priority_factor = self.priority_to_number(update['priority'])
if database.isuuid(kimid):
priority = int(priority_factor * 1000000)
self.check_dependencies_and_push(kimid, priority, status)
return
name, leader, num, version = database.parse_kim_code(kimid)
checkmatch = False
if leader == "VT":
# for every test launch
test = kimobjects.TestVerification(kimid)
models = list(kimobjects.Test.all())
tests = [test] * ll(models)
elif leader == "VM":
#for all of the models, run a job
test = kimobjects.ModelVerification(kimid)
models = list(kimobjects.Model.all())
tests = [test] * ll(models)
else:
if status == "approved":
if leader == "TE":
# for all of the models, add a job
test = kimobjects.Test(kimid)
models = list(test.models)
tests = [test] * ll(models)
elif leader == "MO":
# for all of the tests, add a job
model = kimobjects.Model(kimid)
tests = list(model.tests)
models = [model] * ll(tests)
elif leader == "TD":
# if it is a new version of an existing test driver, hunt
# down all of the tests that use it and launch their
# corresponding jobs
driver = kimobjects.TestDriver(kimid)
temp_tests = list(driver.tests)
models = []
tests = []
for t in temp_tests:
tmodels = list(t.models)
if len(tmodels) > 0:
models.extend(tmodels)
tests.extend([t] * ll(tmodels))
elif leader == "MD":
# if this is a new version, hunt down all of the models
# that rely on it and recompute their results
driver = kimobjects.ModelDriver(kimid)
temp_models = list(driver.models)
tests = []
models = []
for m in temp_models:
mtests = list(m.tests)
if len(mtests) > 0:
tests.extend(mtests)
models.extend([m] * ll(mtests))
else:
self.logger.error("Tried to update an invalid KIM ID!: %r",
kimid)
checkmatch = True
if status == "pending":
rsync_tools.director_pending_read(kimid)
self.make_all()
if leader == "TE":
# run against all test verifications
tests = list(kimobjects.VertificationTest.all())
models = [kimobjects.Test(kimid, search=False)] * ll(tests)
elif leader == "MO":
# run against all model verifications
tests = list(kimobjects.VertificationModel.all())
models = [kimobjects.Model(kimid, search=False)
] * ll(tests)
elif leader == "TD":
# a pending test driver
pass
elif leader == "MD":
# a pending model driver
pass
else:
self.logger.error("Tried to update an invalid KIM ID!: %r",
kimid)
checkmatch = False
for test, model in zip(tests, models):
if not checkmatch or (checkmatch
and kimapi.valid_match(test, model)):
priority = int(priority_factor *
database.test_model_to_priority(test, model) *
1000000)
self.check_dependencies_and_push((test, model), priority,
status)