def check_kimid(self, kimid): """ Validate the contents of the job message, particularly KIMIDs """ try: name,leader,num,version = database.parse_kim_code(kimid) except InvalidKIMID as e: # we were not given a valid kimid self.logger.error("Could not parse {} as a valid KIMID".format(kimid)) raise e
def Subject(kimid): name, leader, num, version = database.parse_kim_code(kimid) if leader == "TE": return Test(kimid) if leader == "MO": return Model(kimid) return None
def Verifier(kimid): name, leader, num, version = database.parse_kim_code(kimid) if leader == "VT": return VerificationTest(kimid) if leader == "VM": return VerificationModel(kimid) return None
def kimcode_to_dict(kimcode): name, leader, num, version = parse_kim_code(kimcode) foo = { "kim-name": name, "kim-leader": leader, "kim-num": num, "kim-version": int(version), "kim-short": "_".join((leader, num)) } return foo
def kimcode_to_dict(kimcode): name, leader, num, version = parse_kim_code(kimcode) leader = leader.lower() foo = { "name": name, "type": leader.lower(), "kimnum": num, "version": int(version), "shortcode": "_".join((leader.upper(), num)), "kimcode": kimcode, "path": os.path.join(leader.lower(), kimcode), "approved": True, '_id': kimcode, } if foo['type'] in ('te', 'mo', 'md', 'vt', 'vm'): foo['makeable'] = True if foo['type'] in ('te', 'vt', 'vm'): foo['runner'] = True if foo['type'] in ('te', 'mo'): foo['subject'] = True if foo['type'] in ('md', 'td'): foo['driver'] = True else: foo['driver'] = False specpath = os.path.join(REPO_DIR, leader, kimcode, "kimspec.ini") spec = configtojson(specpath) if foo['type'] == 'te': try: testresult = spec['kimspec'].get('TEST_DRIVER_NAME') if testresult: foo['driver'] = rmbadkeys(kimcode_to_dict(testresult)) except: pass if foo['type'] == 'mo': try: modeldriver = spec['kimspec'].get('MODEL_DRIVER_NAME') if modeldriver: foo['driver'] = rmbadkeys(kimcode_to_dict(modeldriver)) except: pass foo.update(spec) return foo
def kimfinder(kim): return database.kim_code_finder(database.parse_kim_code(kim))[0]
def latestversion(kim): return database.format_kim_code(*database.get_latest_version(database.parse_kim_code(kim)))
def kid_to_folder_wild(kid): """ Convert a kim_code to its directory """ name,leader,num,version = parse_kim_code(kid) path = os.path.join(leader.lower(),"*"+kid+"*") return path
def push_jobs(self, update): """ Push all of the jobs that need to be done given an update """ kimid = update['kimid'] status = update['status'] priority_factor = self.priority_to_number(update['priority']) name,leader,num,version = database.parse_kim_code(kimid) # try to build the kimid before sending jobs # if self.make_object(kimid) == 0: # rsync_tools.director_build_write(kimid) # else: # self.logger.error("Could not build %r", kimid) # self.bsd.use(TUBE_ERRORS) # self.bsd.put(simplejson.dumps({"error": "Could not build %r" % kimid})) # return self.make_all() if leader=="VT": # for every test launch test = kimobjects.VerificationTest(kimid) models = kimobjects.Test.all() tests = [test]*ll(models) elif leader=="VM": #for all of the models, run a job test = kimobjects.VerificationModel(kimid) models = kimobjects.Model.all() tests = [test]*ll(models) else: if status == "approved": if leader=="TE": # for all of the models, add a job test = kimobjects.Test(kimid) models = list(test.models) tests = [test]*ll(models) elif leader=="MO": # for all of the tests, add a job model = kimobjects.Model(kimid) tests = list(model.tests) models = [model]*ll(tests) elif leader=="TD": # if it is a new version of an existing test driver, hunt # down all of the tests that use it and launch their # corresponding jobs driver = kimobjects.TestDriver(kimid) temp_tests = list(driver.tests) models = [] tests = [] for t in temp_tests: tmodels = list(t.models) if len(tmodels) > 0: models.extend(tmodels) tests.extend([t]*ll(tmodels)) elif leader=="MD": # if this is a new version, hunt down all of the models # that rely on it and recompute their results driver = kimobjects.ModelDriver(kimid) temp_models = list(driver.models) tests = [] models = [] for m in temp_models: mtests = list(m.tests) if len(mtests) > 0: tests.extend(mtests) models.extend([m]*ll(mtests)) else: self.logger.error("Tried to update an invalid KIM ID!: %r",kimid) checkmatch = True if status == "pending": if leader=="TE": # a pending test rsync_tools.director_test_verification_read(kimid) self.make_all() # run against all test verifications tests = list(kimobjects.VertificationTest.all()) models = [kimobjects.Test(kimid, search=False)]*ll(tests) elif leader=="MO": # a pending model rsync_tools.director_model_verification_read(kimid) self.make_all() # run against all model verifications tests = list(kimobjects.VertificationModel.all()) models = [kimobjects.Model(kimid, search=False)]*ll(tests) elif leader=="TD": # a pending test driver pass elif leader=="MD": # a pending model driver pass else: self.logger.error("Tried to update an invalid KIM ID!: %r",kimid) checkmatch = False if checkmatch: for test, model in zip(tests,models): if kimapi.valid_match(test,model): priority = int(priority_factor*database.test_model_to_priority(test,model) * 1000000) self.check_dependencies_and_push(test,model,priority,status) else: for test, model in zip(tests,models): priority = int(priority_factor*database.test_model_to_priority(test,model) * 1000000) self.check_dependencies_and_push(test,model,priority,status)
def __init__(self,kim_code,search=True,subdir=None,abspath=None,precheck=False): """ Initialize a KIMObject given the kim_code, where partial kim codes are promoted if possible, if search is False, then don't look for existing ones Args: kim_code (str) A full or partial kim_code, i.e. one like: * "Full_Name_of_thing__TE_000000000000_000" * "TE_000000000000_000" * "TE_000000000000" * "Full_Name_of_thing__TE_000000000000" search (bool) Whether or not to search the directory structure for the fullest match, false is useful when creating new KIMObjects to avoid hitting a PipelineSearchError subdir (str) In order to point to a directory that does not follow that pattern /home/openkim/openkim-repository/{mo,md,te...}/KIM_CODE/KIM_CODE can provide the folder of /home/openkim/openkim-repository/{mo,md,te...}/SUBDIR/KIM_CODE """ logger.debug("Initializing a new KIMObject: %r", kim_code) name, leader, num, version = database.parse_kim_code(kim_code) # test to see if we have the right leader if self.required_leader: assert leader==self.required_leader,"{} not a valid KIM code for {}".format(kim_code, self.__class__.__name__) #grab the attributes self.kim_code_name = name self.kim_code_leader = leader self.kim_code_number = num self.kim_code_version = version if not search: self.kim_code = kim_code #if we were given everything, we are good to go elif name and leader and num and version: self.kim_code = database.format_kim_code(name,leader,num,version) #if we weren't given a name, see if one exists elif name is None and leader and num and version: name = database.look_for_name(leader,num,version) self.kim_code_name = name self.kim_code = database.format_kim_code(name,leader,num,version) #if we weren't given a version elif name and leader and num and version is None: name,leader,num,version = database.get_latest_version(name,leader,num) self.kim_code_version = version self.kim_code = database.format_kim_code(name,leader,num,version) #if we weren't given a name or version elif name is None and leader and num and version is None: name,leader,num,version = database.get_latest_version(name,leader,num) self.kim_code_name = name self.kim_code_version = version self.kim_code = database.format_kim_code(name,leader,num,version) self.kim_code_id = database.strip_name(self.kim_code) self.kim_code_short = database.strip_version(self.kim_code) # Determine where this KIMObject sits in the local repository if precheck: self.parent_dir = os.path.join(os.path.join(cf.LOCAL_REPOSITORY_PATH,'precheck'), self.kim_code_leader.lower()) else: self.parent_dir = os.path.join(cf.LOCAL_REPOSITORY_PATH, self.kim_code_leader.lower()) if subdir is not None: self.path = os.path.join(self.parent_dir, subdir) else: self.path = os.path.join(self.parent_dir, self.kim_code) if abspath is not None: self.path = abspath # assume the object is not built by default self.built = False
def kim_obj(kim_code, *args, **kwargs): """ Just given a kim_code try to make the right object, i.e. try to make a TE code a Test, etc. """ name,leader,num,version = database.parse_kim_code(kim_code) cls = code_to_model.get(leader, KIMObject) return cls(kim_code, *args, **kwargs)
def __init__(self,kim_code,search=True,subdir=None): """ Initialize a KIMObject given the kim_code, where partial kim codes are promoted if possible, if search is False, then don't look for existing ones Args: kim_code (str) A full or partial kim_code, i.e. one like: * "Full_Name_of_thing__TE_000000000000_000" * "TE_000000000000_000" * "TE_000000000000" * "Full_Name_of_thing__TE_000000000000" search (bool) Whether or not to search the directory structure for the fullest match, false is useful when creating new KIMObjects to avoid hitting a PipelineSearchError subdir (str) In order to point to a directory that does not follow that pattern /home/openkim/openkim-repository/{mo,md,te...}/KIM_CODE/KIM_CODE can provide the folder of /home/openkim/openkim-repository/{mo,md,te...}/SUBDIR/KIM_CODE """ logger.debug("Initializing a new KIMObject: %r", kim_code) name, leader, num, version = database.parse_kim_code(kim_code) # test to see if we have the right leader if self.required_leader: assert leader==self.required_leader,"{} not a valid KIM code for {}".format(kim_code, self.__class__.__name__) #grab the attributes self.kim_code_name = name self.kim_code_leader = leader self.kim_code_number = num self.kim_code_version = version if not search: self.kim_code = kim_code #if we were given everything, we are good to go elif name and leader and num and version: self.kim_code = database.format_kim_code(name,leader,num,version) #if we weren't given a name, see if one exists elif name is None and leader and num and version: name = database.look_for_name(leader,num,version) self.kim_code_name = name self.kim_code = database.format_kim_code(name,leader,num,version) #if we weren't given a version elif name and leader and num and version is None: name,leader,num,version = database.get_latest_version(name,leader,num) self.kim_code_version = version self.kim_code = database.format_kim_code(name,leader,num,version) #if we weren't given a name or version elif name is None and leader and num and version is None: name,leader,num,version = database.get_latest_version(name,leader,num) self.kim_code_name = name self.kim_code_version = version self.kim_code = database.format_kim_code(name,leader,num,version) self.kim_code_id = database.strip_name(self.kim_code) self.kim_code_short = database.strip_version(self.kim_code) self.parent_dir = os.path.join(cf.KIM_REPOSITORY_DIR, self.kim_code_leader.lower()) if subdir is not None: self.path = os.path.join(self.parent_dir, subdir) else: self.path = os.path.join(self.parent_dir, self.kim_code)
def stripversion(kim): kimtup = database.parse_kim_code(kim) newtup = ( kimtup.name, kimtup.leader, kimtup.num, None) return database.format_kim_code( *newtup )
def latestversion(kim): return database.format_kim_code( *database.get_latest_version(database.parse_kim_code(kim)))
def push_jobs(self, update): """ Push all of the jobs that need to be done given an update """ self.make_all() kimid = update['kimid'] status = update['status'] priority_factor = self.priority_to_number(update['priority']) if database.isuuid(kimid): priority = int(priority_factor * 1000000) self.check_dependencies_and_push(kimid, priority, status) return name, leader, num, version = database.parse_kim_code(kimid) checkmatch = False if leader == "VT": # for every test launch test = kimobjects.TestVerification(kimid) models = list(kimobjects.Test.all()) tests = [test] * ll(models) elif leader == "VM": #for all of the models, run a job test = kimobjects.ModelVerification(kimid) models = list(kimobjects.Model.all()) tests = [test] * ll(models) else: if status == "approved": if leader == "TE": # for all of the models, add a job test = kimobjects.Test(kimid) models = list(test.models) tests = [test] * ll(models) elif leader == "MO": # for all of the tests, add a job model = kimobjects.Model(kimid) tests = list(model.tests) models = [model] * ll(tests) elif leader == "TD": # if it is a new version of an existing test driver, hunt # down all of the tests that use it and launch their # corresponding jobs driver = kimobjects.TestDriver(kimid) temp_tests = list(driver.tests) models = [] tests = [] for t in temp_tests: tmodels = list(t.models) if len(tmodels) > 0: models.extend(tmodels) tests.extend([t] * ll(tmodels)) elif leader == "MD": # if this is a new version, hunt down all of the models # that rely on it and recompute their results driver = kimobjects.ModelDriver(kimid) temp_models = list(driver.models) tests = [] models = [] for m in temp_models: mtests = list(m.tests) if len(mtests) > 0: tests.extend(mtests) models.extend([m] * ll(mtests)) else: self.logger.error("Tried to update an invalid KIM ID!: %r", kimid) checkmatch = True if status == "pending": rsync_tools.director_pending_read(kimid) self.make_all() if leader == "TE": # run against all test verifications tests = list(kimobjects.VertificationTest.all()) models = [kimobjects.Test(kimid, search=False)] * ll(tests) elif leader == "MO": # run against all model verifications tests = list(kimobjects.VertificationModel.all()) models = [kimobjects.Model(kimid, search=False) ] * ll(tests) elif leader == "TD": # a pending test driver pass elif leader == "MD": # a pending model driver pass else: self.logger.error("Tried to update an invalid KIM ID!: %r", kimid) checkmatch = False for test, model in zip(tests, models): if not checkmatch or (checkmatch and kimapi.valid_match(test, model)): priority = int(priority_factor * database.test_model_to_priority(test, model) * 1000000) self.check_dependencies_and_push((test, model), priority, status)
def stripversion(kim): kimtup = database.parse_kim_code(kim) newtup = (kimtup.name, kimtup.leader, kimtup.num, None) return database.format_kim_code(*newtup)
def run(self): """ Start to listen, tunnels should be open and ready """ self.connect() self.bean.watch(cf.TUBE_JOBS) """ Endless loop that awaits jobs to run """ while True: self.logger.info("Waiting for jobs...") job = self.bean.reserve() self.job = job # if appears that there is a 120sec re-birth of jobs that have been reserved # and I do not want to put an artificial time limit, so let's bury jobs # when we get them job.bury() self.comm.send_msg("running", job.body) # update the repository, attempt to run the job and return the results to the director try: jobmsg = network.Message(string=job.body) pending = True if jobmsg == "pending" else False except simplejson.JSONDecodeError: # message is not JSON decodeable self.logger.error("Did not recieve valid JSON, {}".format( job.body)) job.delete() continue except KeyError: # message does not have the right keys self.logger.error( "Did not recieve a valid message, missing key: {}".format( job.body)) job.delete() continue self.jobmsg = jobmsg try: name, leader, num, version = database.parse_kim_code( jobmsg.job[0]) except InvalidKIMID as e: # we were not given a valid kimid self.logger.error("Could not parse {} as a valid KIMID".format( jobmsg.job[0])) self.job_message(jobmsg, errors=e, tube=cf.TUBE_ERRORS) job.delete() continue try: # check to see if this is a verifier or an actual test with self.rsynclock: self.logger.info("Rsyncing from repo %r", jobmsg.job + jobmsg.depends) rsync_tools.worker_read(*jobmsg.job, depends=jobmsg.depends, pending=pending) runner_kcode, subject_kcode = jobmsg.job runner = kimobjects.kim_obj(runner_kcode) subject = kimobjects.kim_obj(subject_kcode) self.builder.lock_build(runner) self.builder.lock_build(subject) self.logger.info("Running (%r,%r)", runner, subject) comp = compute.Computation(runner, subject, result_code=jobmsg.jobid) errormsg = None try: comp.run(extrainfo=self.boxinfo) except Exception as e: errormsg = e self.logger.exception("Errors occured, moving to er/") else: self.logger.debug("Sending result message back") finally: self.logger.info("Rsyncing results %r", jobmsg.jobid) with self.rsynclock: rsync_tools.worker_write(comp.result_path) if errormsg: self.job_message(jobmsg, errors=e, tube=cf.TUBE_ERRORS) else: self.job_message(jobmsg, tube=cf.TUBE_RESULTS) job.delete() except Exception as e: self.logger.exception( "Failed to initalize run, deleting... %r" % e) self.job_message(jobmsg, errors=e, tube=cf.TUBE_ERRORS) job.delete() self.job = None self.jobsmsg = None
def run_job(self): """ Endless loop that awaits jobs to run """ while True: with loglock: self.logger.info("Waiting for jobs...") job = self.bean.reserve() self.job = job # if appears that there is a 120sec re-birth of jobs that have been reserved # and I do not want to put an artificial time limit, so let's bury jobs # when we get them job.bury() self.comm.send_msg("running", job.body) # update the repository, attempt to run the job and return the results to the director try: jobmsg = Message(string=job.body) except simplejson.JSONDecodeError: # message is not JSON decodeable with loglock: self.logger.error("Did not recieve valid JSON, {}".format(job.body)) job.delete() continue except KeyError: # message does not have the right keys with loglock: self.logger.error("Did not recieve a valid message, missing key: {}".format(job.body)) job.delete() continue self.jobmsg = jobmsg # check to see if this is a verifier or an actual test try: name,leader,num,version = database.parse_kim_code(jobmsg.job[0]) except InvalidKIMID as e: # we were not given a valid kimid with loglock: self.logger.error("Could not parse {} as a valid KIMID".format(jobmsg.job[0])) self.job_message(jobmsg, errors=e, tube=TUBE_ERRORS) job.delete() continue if leader == "VT" or leader == "VM": try: with buildlock: with loglock: self.logger.info("rsyncing to repo %r", jobmsg.job+jobmsg.depends) rsync_tools.worker_verification_read(*jobmsg.job, depends=jobmsg.depends) self.make_all() verifier_kcode, subject_kcode = jobmsg.job verifier = kimobjects.Verifier(verifier_kcode) subject = kimobjects.Subject(subject_kcode) with loglock: self.logger.info("Running (%r,%r)",verifier,subject) comp = compute.Computation(verifier, subject) comp.run(jobmsg.jobid) result = kimobjects.Result(jobmsg.jobid).results with loglock: self.logger.info("rsyncing results %r", jobmsg.jobid) rsync_tools.worker_verification_write(jobmsg.jobid) with loglock: self.logger.info("sending result message back") self.job_message(jobmsg, results=result, tube=TUBE_RESULTS) job.delete() # could be that a dependency has not been met. # put it back on the queue to wait except PipelineDataMissing as e: if job.stats()['age'] < 5*PIPELINE_JOB_TIMEOUT: with loglock: self.logger.error("Run failed, missing data. Returning to queue... (%r)" % e) job.release(delay=PIPELINE_JOB_TIMEOUT) else: with loglock: self.logger.error("Run failed, missing data. Lifetime has expired, deleting (%r)" % e) job.delete() # another problem has occurred. just remove the job # and send the error back along the error queue except Exception as e: with loglock: self.logger.error("Run failed, deleting... %r" % e) self.job_message(jobmsg, errors=e, tube=TUBE_ERRORS) job.delete() else: try: with buildlock: with loglock: self.logger.info("rsyncing to repo %r %r", jobmsg.job,jobmsg.depends) rsync_tools.worker_test_result_read(*jobmsg.job, depends=jobmsg.depends) self.make_all() test_kcode, model_kcode = jobmsg.job test = kimobjects.Test(test_kcode) model = kimobjects.Model(model_kcode) with loglock: self.logger.info("Running (%r,%r)",test,model) comp = compute.Computation(test, model) comp.run(jobmsg.jobid) result = kimobjects.Result(jobmsg.jobid).results with loglock: self.logger.info("rsyncing results %r", jobmsg.jobid) rsync_tools.worker_test_result_write(jobmsg.jobid) with loglock: self.logger.info("sending result message back") self.job_message(jobmsg, results=result, tube=TUBE_RESULTS) job.delete() # could be that a dependency has not been met. # put it back on the queue to wait except PipelineDataMissing as e: if job.stats()['age'] < 5*PIPELINE_JOB_TIMEOUT: with loglock: self.logger.error("Run failed, missing data. Returning to queue... (%r)" % e) job.release(delay=PIPELINE_JOB_TIMEOUT) else: with loglock: self.logger.error("Run failed, missing data. Lifetime has expired, deleting (%r)" % e) job.delete() # another problem has occurred. just remove the job # and send the error back along the error queue except Exception as e: with loglock: self.logger.error("Run failed, deleting... %r" % e) self.job_message(jobmsg, errors="%r"%e, tube=TUBE_ERRORS) job.delete() self.job = None self.jobsmsg = None
def kid_to_folder(kid): """ Convert a kim_code to its directory """ name,leader,num,version = parse_kim_code(kid) path = os.path.join(leader.lower(),kid) return path