def run(self):
"""
Endless loop that waits for updates on the tube TUBE_UPDATES
The update is a json string for a dictionary with key-values pairs
that look like:
* kimid : any form of the kimid that is recognized by the database package
* priority : one of 'immediate', 'very high', 'high', 'normal', 'low', 'very low'
* status : one of 'approved', 'pending' where pending signifies it has just been
submitted and needs to go through VCs whereas approved is to initiate a full match
run after verification or update
"""
# connect and grab the job thread
self.connect()
self.bean.watch(cf.TUBE_UPDATES)
while True:
self.logger.info("Director Waiting for message...")
request = self.bean.reserve()
self.job = request
# make sure it doesn't come alive again soon
request.bury()
# got a request to update a model or test
# from the website (or other trusted place)
if request.stats()['tube'] == cf.TUBE_UPDATES:
# update the repository,send it out as a job to compute
try:
rsync_tools.director_approved_read()
self.push_jobs(simplejson.loads(request.body))
except Exception as e:
self.logger.exception("Director had an error on update")
request.delete()
self.job = None
def synchronize(self, fresh=False):
"""
Here, we bring a director up-to-date with the shared repository,
updating the objects and syncing the database to the filesystem.
* Pull the entire repository to make sure we are up to date
* Incrementally build only the relevant objects that have changed
* Place these items into the database (run matches, etc)
If `fresh` is true, go ahead and rebuild everything
During a complete re-run this function should take very little time.
"""
if cf.DIRECTOR_NOSYNC:
return
# if the director is just being brought up, clear out the built
# cache and rebuild everything from scratch. this happens before
# rsync so as to be agressive in synchronizing
if fresh:
self.logger.info("Rebuilding KIMObjects in repository...")
self.db.clear_build()
self.logger.info("Syncing database to filesystem...")
self.db.remove_missing()
# pull the repo using rsync from the shared respository
rsync_tools.director_approved_read()
self.logger.info("Attempting to build new KIMOBjects...")
for t in chain(kimobjects.TestDriver.all(), kimobjects.ModelDriver.all(),
kimobjects.Test.all(), kimobjects.Model.all()):
self.db.build(t)
self.logger.info("Attempting to add new KIMOBjects to database...")
for t in chain(kimobjects.TestDriver.all(), kimobjects.ModelDriver.all(),
kimobjects.Test.all(), kimobjects.Model.all()):
self.db.insert(t)
def precheck(self, kimid, status='pending'):
""" Perform pre-checks on a new object """
errors = []
if cf.DIRECTOR_NOSYNC:
errors.append(
'Precheck failed: DIRECTOR_NOSYNC must be set to False'
)
self.logger.error('%r', errors)
return errors
approved = (status == 'approved')
self.logger.info("Running pre-checks for KIM item %r" % kimid)
# try to actually get the kimobject
if not cf.DIRECTOR_NOSYNC:
if approved:
# Read from Gateway's local 'precheck' repository to local 'precheck' repository
rsync_tools.director_approved_read(precheck=2)
else:
rsync_tools.director_pending_read(kimid, precheck=2)
# Can we use our ORM?
# NOTE: This is redundant with a check done on the Gateway, but it's not expensive
try:
obj = kimobjects.kim_obj(kimid, precheck=True)
except Exception as e:
errors.append(
'Could not initialize KIMObjects ORM:\n%r' % e
)
self.logger.error('%r', errors)
return errors
# try to get the drivers so we can start to build, etc
try:
if not cf.DIRECTOR_NOSYNC:
drivers = list(obj.drivers)
self.logger.info("Retrieving drivers for %r, %r" % (obj, drivers))
for driver in drivers:
rsync_tools.director_pending_read(str(driver), precheck=2)
except Exception as e:
errors.append(
'Could not find drivers associated with object:\n%r' % e
)
self.logger.error('%r', errors)
return errors
# can it be built?
try:
obj.make(precheck=True)
except Exception as e:
errors.append(
'KIMObject could not be built using `make`:\n%r' % e
)
self.logger.error('%r', errors)
return errors
def _assert(condition):
if not condition:
raise AssertionError()
checks_orm = [
[lambda o: o.drivers, 'Searching for drivers failed'],
[lambda o: _assert(o.kimspec), 'Does not contain kimspec'],
[lambda o: _assert(o.kim_api_version), 'No KIM API version specified'],
[lambda o: _assert(o.pipeline_api_version), 'No Pipeline API version specified']
]
checks_runner = [
[lambda t: t.processed_infile(next(kimobjects.Test.all(precheck=True))),
'Could not template pipeline.stdin file'],
[lambda t: t.runtime_dependencies('blank'), 'Could not template dependencies file'],
[lambda t: list(t.matches), 'Matches could not be generated, exceptions'],
[lambda t: _assert(list(t.matches)), 'No valid matches found in system']
]
checks_subject = [
[lambda t: list(t.matches), 'Matches could not be generated, errors.'],
[lambda t: _assert(list(t.matches)), 'No valid matches found in system']
]
def _run_checks(check_list, *args):
for check in check_list:
try:
check[0](*args)
except Exception as e:
errors.append(check[1])
self.logger.error('%s:\n%r' % (check[1], e))
_run_checks(checks_orm, obj)
if isinstance(obj, kimobjects.Runner):
_run_checks(checks_runner, obj)
if isinstance(obj, kimobjects.Subject):
_run_checks(checks_subject, obj)
if errors:
self.logger.error(
"Returning errors for precheck of %r:\n%r" % (kimid, errors)
)
else:
self.logger.info("No errors found during precheck for %r" % kimid)
return errors
