def process_messages(self):
while 1:
request = self.bean.reserve()
tube = request.stats()['tube']
if tube == cf.TUBE_WEB_UPDATES:
logger.debug("processing %r" % request.body)
try:
job = json.loads(request.body)
approved = True if job['status'] == 'approved' else False
rsync_tools.gateway_full_read()
kimcode = job['kimid']
if database.isuuid(kimcode):
leader = database.uuid_type(kimcode)
insert_one_result(leader, kimcode)
else:
insert_one_object(kimcode)
self.bean.send_msg(cf.TUBE_UPDATES, request.body)
except Exception as e:
logger.error("%r" % e)
elif tube == cf.TUBE_RESULTS or tube == cf.TUBE_ERRORS:
logger.debug("processing %r" % request.body)
try:
kimcode = json.loads(request.body)['jobid']
leader = database.uuid_type(kimcode)
rsync_tools.gateway_write_result(leader, kimcode)
except Exception as e:
logger.error("%r" % e)
self.bean.send_msg(cf.TUBE_WEB_RESULTS, request.body)
request.delete()
def new(self, itemid, status='approved', priority='normal', force=False):
""" New KIM object submitted to web-app """
priority = network.transform_priority(priority)
leader = database.get_leader(itemid).lower()
approved = (status == 'approved')
self.rsync_kimid(itemid, approved=approved)
# insert this object into the database
if database.isuuid(itemid):
mongodb.insert_one_result(leader, itemid)
else:
mongodb.insert_one_object(itemid)
# pass the message along to the director
msg = network.director_update_message(
itemid=itemid, status=status, priority=priority, force=force
)
self.send_job('pipeline.tasks.director_run', kwargs=msg)
def run(self, itemid, status='approved', priority='normal', force=False):
"""
Complete a matching or dependency task given by `job`. This message
should be formed by calling `pipeline.network.director_update_message`
"""
priority = network.transform_priority(priority)
self.logger.info(
"Director got update message for %r (%r | %r | %r)" %
(itemid, status, priority, force)
)
# decide what to do with the update request based on what
# type of object it is
if database.isuuid(itemid):
self.process_test_result(uuid=itemid, priority=priority)
elif database.iskimcode(itemid):
if status == 'approved':
self.process_kimcode_approved(itemid, priority)
if status == 'pending':
self.process_kimcode_pending(itemid, priority)
self.logger.info("Director Waiting for message...")
def push_jobs(self, update):
""" Push all of the jobs that need to be done given an update """
self.make_all()
kimid = update['kimid']
status = update['status']
priority_factor = self.priority_to_number(update['priority'])
if database.isuuid(kimid):
priority = int(priority_factor * 1000000)
self.check_dependencies_and_push(kimid, priority, status)
return
name, leader, num, version = database.parse_kim_code(kimid)
checkmatch = False
if leader == "VT":
# for every test launch
test = kimobjects.TestVerification(kimid)
models = list(kimobjects.Test.all())
tests = [test] * ll(models)
elif leader == "VM":
#for all of the models, run a job
test = kimobjects.ModelVerification(kimid)
models = list(kimobjects.Model.all())
tests = [test] * ll(models)
else:
if status == "approved":
if leader == "TE":
# for all of the models, add a job
test = kimobjects.Test(kimid)
models = list(test.models)
tests = [test] * ll(models)
elif leader == "MO":
# for all of the tests, add a job
model = kimobjects.Model(kimid)
tests = list(model.tests)
models = [model] * ll(tests)
elif leader == "TD":
# if it is a new version of an existing test driver, hunt
# down all of the tests that use it and launch their
# corresponding jobs
driver = kimobjects.TestDriver(kimid)
temp_tests = list(driver.tests)
models = []
tests = []
for t in temp_tests:
tmodels = list(t.models)
if len(tmodels) > 0:
models.extend(tmodels)
tests.extend([t] * ll(tmodels))
elif leader == "MD":
# if this is a new version, hunt down all of the models
# that rely on it and recompute their results
driver = kimobjects.ModelDriver(kimid)
temp_models = list(driver.models)
tests = []
models = []
for m in temp_models:
mtests = list(m.tests)
if len(mtests) > 0:
tests.extend(mtests)
models.extend([m] * ll(mtests))
else:
self.logger.error("Tried to update an invalid KIM ID!: %r",
kimid)
checkmatch = True
if status == "pending":
rsync_tools.director_pending_read(kimid)
self.make_all()
if leader == "TE":
# run against all test verifications
tests = list(kimobjects.VertificationTest.all())
models = [kimobjects.Test(kimid, search=False)] * ll(tests)
elif leader == "MO":
# run against all model verifications
tests = list(kimobjects.VertificationModel.all())
models = [kimobjects.Model(kimid, search=False)
] * ll(tests)
elif leader == "TD":
# a pending test driver
pass
elif leader == "MD":
# a pending model driver
pass
else:
self.logger.error("Tried to update an invalid KIM ID!: %r",
kimid)
checkmatch = False
for test, model in zip(tests, models):
if not checkmatch or (checkmatch
and kimapi.valid_match(test, model)):
priority = int(priority_factor *
database.test_model_to_priority(test, model) *
1000000)
self.check_dependencies_and_push((test, model), priority,
status)
