def process_messages(self): while 1: request = self.bean.reserve() tube = request.stats()['tube'] if tube == cf.TUBE_WEB_UPDATES: logger.debug("processing %r" % request.body) try: job = json.loads(request.body) approved = True if job['status'] == 'approved' else False rsync_tools.gateway_full_read() kimcode = job['kimid'] if database.isuuid(kimcode): leader = database.uuid_type(kimcode) insert_one_result(leader, kimcode) else: insert_one_object(kimcode) self.bean.send_msg(cf.TUBE_UPDATES, request.body) except Exception as e: logger.error("%r" % e) elif tube == cf.TUBE_RESULTS or tube == cf.TUBE_ERRORS: logger.debug("processing %r" % request.body) try: kimcode = json.loads(request.body)['jobid'] leader = database.uuid_type(kimcode) rsync_tools.gateway_write_result(leader, kimcode) except Exception as e: logger.error("%r" % e) self.bean.send_msg(cf.TUBE_WEB_RESULTS, request.body) request.delete()
def new(self, itemid, status='approved', priority='normal', force=False): """ New KIM object submitted to web-app """ priority = network.transform_priority(priority) leader = database.get_leader(itemid).lower() approved = (status == 'approved') self.rsync_kimid(itemid, approved=approved) # insert this object into the database if database.isuuid(itemid): mongodb.insert_one_result(leader, itemid) else: mongodb.insert_one_object(itemid) # pass the message along to the director msg = network.director_update_message( itemid=itemid, status=status, priority=priority, force=force ) self.send_job('pipeline.tasks.director_run', kwargs=msg)
def run(self, itemid, status='approved', priority='normal', force=False): """ Complete a matching or dependency task given by `job`. This message should be formed by calling `pipeline.network.director_update_message` """ priority = network.transform_priority(priority) self.logger.info( "Director got update message for %r (%r | %r | %r)" % (itemid, status, priority, force) ) # decide what to do with the update request based on what # type of object it is if database.isuuid(itemid): self.process_test_result(uuid=itemid, priority=priority) elif database.iskimcode(itemid): if status == 'approved': self.process_kimcode_approved(itemid, priority) if status == 'pending': self.process_kimcode_pending(itemid, priority) self.logger.info("Director Waiting for message...")
def push_jobs(self, update): """ Push all of the jobs that need to be done given an update """ self.make_all() kimid = update['kimid'] status = update['status'] priority_factor = self.priority_to_number(update['priority']) if database.isuuid(kimid): priority = int(priority_factor * 1000000) self.check_dependencies_and_push(kimid, priority, status) return name, leader, num, version = database.parse_kim_code(kimid) checkmatch = False if leader == "VT": # for every test launch test = kimobjects.TestVerification(kimid) models = list(kimobjects.Test.all()) tests = [test] * ll(models) elif leader == "VM": #for all of the models, run a job test = kimobjects.ModelVerification(kimid) models = list(kimobjects.Model.all()) tests = [test] * ll(models) else: if status == "approved": if leader == "TE": # for all of the models, add a job test = kimobjects.Test(kimid) models = list(test.models) tests = [test] * ll(models) elif leader == "MO": # for all of the tests, add a job model = kimobjects.Model(kimid) tests = list(model.tests) models = [model] * ll(tests) elif leader == "TD": # if it is a new version of an existing test driver, hunt # down all of the tests that use it and launch their # corresponding jobs driver = kimobjects.TestDriver(kimid) temp_tests = list(driver.tests) models = [] tests = [] for t in temp_tests: tmodels = list(t.models) if len(tmodels) > 0: models.extend(tmodels) tests.extend([t] * ll(tmodels)) elif leader == "MD": # if this is a new version, hunt down all of the models # that rely on it and recompute their results driver = kimobjects.ModelDriver(kimid) temp_models = list(driver.models) tests = [] models = [] for m in temp_models: mtests = list(m.tests) if len(mtests) > 0: tests.extend(mtests) models.extend([m] * ll(mtests)) else: self.logger.error("Tried to update an invalid KIM ID!: %r", kimid) checkmatch = True if status == "pending": rsync_tools.director_pending_read(kimid) self.make_all() if leader == "TE": # run against all test verifications tests = list(kimobjects.VertificationTest.all()) models = [kimobjects.Test(kimid, search=False)] * ll(tests) elif leader == "MO": # run against all model verifications tests = list(kimobjects.VertificationModel.all()) models = [kimobjects.Model(kimid, search=False) ] * ll(tests) elif leader == "TD": # a pending test driver pass elif leader == "MD": # a pending model driver pass else: self.logger.error("Tried to update an invalid KIM ID!: %r", kimid) checkmatch = False for test, model in zip(tests, models): if not checkmatch or (checkmatch and kimapi.valid_match(test, model)): priority = int(priority_factor * database.test_model_to_priority(test, model) * 1000000) self.check_dependencies_and_push((test, model), priority, status)