def check_kimid(self, kimid):
""" Validate the contents of the job message, particularly KIMIDs """
try:
name,leader,num,version = database.parse_kim_code(kimid)
except InvalidKIMID as e:
# we were not given a valid kimid
self.logger.error("Could not parse {} as a valid KIMID".format(kimid))
raise e
def Subject(kimid):
name, leader, num, version = database.parse_kim_code(kimid)
if leader == "TE":
return Test(kimid)
if leader == "MO":
return Model(kimid)
return None
def Verifier(kimid):
name, leader, num, version = database.parse_kim_code(kimid)
if leader == "VT":
return VerificationTest(kimid)
if leader == "VM":
return VerificationModel(kimid)
return None
def kimcode_to_dict(kimcode):
name, leader, num, version = parse_kim_code(kimcode)
foo = {
"kim-name": name,
"kim-leader": leader,
"kim-num": num,
"kim-version": int(version),
"kim-short": "_".join((leader, num))
}
return foo
def kimcode_to_dict(kimcode):
name, leader, num, version = parse_kim_code(kimcode)
leader = leader.lower()
foo = {
"name": name,
"type": leader.lower(),
"kimnum": num,
"version": int(version),
"shortcode": "_".join((leader.upper(), num)),
"kimcode": kimcode,
"path": os.path.join(leader.lower(), kimcode),
"approved": True,
'_id': kimcode,
}
if foo['type'] in ('te', 'mo', 'md', 'vt', 'vm'):
foo['makeable'] = True
if foo['type'] in ('te', 'vt', 'vm'):
foo['runner'] = True
if foo['type'] in ('te', 'mo'):
foo['subject'] = True
if foo['type'] in ('md', 'td'):
foo['driver'] = True
else:
foo['driver'] = False
specpath = os.path.join(REPO_DIR, leader, kimcode, "kimspec.ini")
spec = configtojson(specpath)
if foo['type'] == 'te':
try:
testresult = spec['kimspec'].get('TEST_DRIVER_NAME')
if testresult:
foo['driver'] = rmbadkeys(kimcode_to_dict(testresult))
except:
pass
if foo['type'] == 'mo':
try:
modeldriver = spec['kimspec'].get('MODEL_DRIVER_NAME')
if modeldriver:
foo['driver'] = rmbadkeys(kimcode_to_dict(modeldriver))
except:
pass
foo.update(spec)
return foo
def kimfinder(kim):
return database.kim_code_finder(database.parse_kim_code(kim))[0]
def latestversion(kim):
return database.format_kim_code(*database.get_latest_version(database.parse_kim_code(kim)))
def kid_to_folder_wild(kid):
""" Convert a kim_code to its directory """
name,leader,num,version = parse_kim_code(kid)
path = os.path.join(leader.lower(),"*"+kid+"*")
return path
def push_jobs(self, update):
""" Push all of the jobs that need to be done given an update """
kimid = update['kimid']
status = update['status']
priority_factor = self.priority_to_number(update['priority'])
name,leader,num,version = database.parse_kim_code(kimid)
# try to build the kimid before sending jobs
# if self.make_object(kimid) == 0:
# rsync_tools.director_build_write(kimid)
# else:
# self.logger.error("Could not build %r", kimid)
# self.bsd.use(TUBE_ERRORS)
# self.bsd.put(simplejson.dumps({"error": "Could not build %r" % kimid}))
# return
self.make_all()
if leader=="VT":
# for every test launch
test = kimobjects.VerificationTest(kimid)
models = kimobjects.Test.all()
tests = [test]*ll(models)
elif leader=="VM":
#for all of the models, run a job
test = kimobjects.VerificationModel(kimid)
models = kimobjects.Model.all()
tests = [test]*ll(models)
else:
if status == "approved":
if leader=="TE":
# for all of the models, add a job
test = kimobjects.Test(kimid)
models = list(test.models)
tests = [test]*ll(models)
elif leader=="MO":
# for all of the tests, add a job
model = kimobjects.Model(kimid)
tests = list(model.tests)
models = [model]*ll(tests)
elif leader=="TD":
# if it is a new version of an existing test driver, hunt
# down all of the tests that use it and launch their
# corresponding jobs
driver = kimobjects.TestDriver(kimid)
temp_tests = list(driver.tests)
models = []
tests = []
for t in temp_tests:
tmodels = list(t.models)
if len(tmodels) > 0:
models.extend(tmodels)
tests.extend([t]*ll(tmodels))
elif leader=="MD":
# if this is a new version, hunt down all of the models
# that rely on it and recompute their results
driver = kimobjects.ModelDriver(kimid)
temp_models = list(driver.models)
tests = []
models = []
for m in temp_models:
mtests = list(m.tests)
if len(mtests) > 0:
tests.extend(mtests)
models.extend([m]*ll(mtests))
else:
self.logger.error("Tried to update an invalid KIM ID!: %r",kimid)
checkmatch = True
if status == "pending":
if leader=="TE":
# a pending test
rsync_tools.director_test_verification_read(kimid)
self.make_all()
# run against all test verifications
tests = list(kimobjects.VertificationTest.all())
models = [kimobjects.Test(kimid, search=False)]*ll(tests)
elif leader=="MO":
# a pending model
rsync_tools.director_model_verification_read(kimid)
self.make_all()
# run against all model verifications
tests = list(kimobjects.VertificationModel.all())
models = [kimobjects.Model(kimid, search=False)]*ll(tests)
elif leader=="TD":
# a pending test driver
pass
elif leader=="MD":
# a pending model driver
pass
else:
self.logger.error("Tried to update an invalid KIM ID!: %r",kimid)
checkmatch = False
if checkmatch:
for test, model in zip(tests,models):
if kimapi.valid_match(test,model):
priority = int(priority_factor*database.test_model_to_priority(test,model) * 1000000)
self.check_dependencies_and_push(test,model,priority,status)
else:
for test, model in zip(tests,models):
priority = int(priority_factor*database.test_model_to_priority(test,model) * 1000000)
self.check_dependencies_and_push(test,model,priority,status)
def kimfinder(kim):
return database.kim_code_finder(database.parse_kim_code(kim))[0]
def __init__(self,kim_code,search=True,subdir=None,abspath=None,precheck=False):
""" Initialize a KIMObject given the kim_code, where partial kim codes are promoted if possible,
if search is False, then don't look for existing ones
Args:
kim_code (str)
A full or partial kim_code, i.e. one like:
* "Full_Name_of_thing__TE_000000000000_000"
* "TE_000000000000_000"
* "TE_000000000000"
* "Full_Name_of_thing__TE_000000000000"
search (bool)
Whether or not to search the directory structure for the fullest match,
false is useful when creating new KIMObjects to avoid hitting a PipelineSearchError
subdir (str)
In order to point to a directory that does not follow that pattern
/home/openkim/openkim-repository/{mo,md,te...}/KIM_CODE/KIM_CODE
can provide the folder of
/home/openkim/openkim-repository/{mo,md,te...}/SUBDIR/KIM_CODE
"""
logger.debug("Initializing a new KIMObject: %r", kim_code)
name, leader, num, version = database.parse_kim_code(kim_code)
# test to see if we have the right leader
if self.required_leader:
assert leader==self.required_leader,"{} not a valid KIM code for {}".format(kim_code, self.__class__.__name__)
#grab the attributes
self.kim_code_name = name
self.kim_code_leader = leader
self.kim_code_number = num
self.kim_code_version = version
if not search:
self.kim_code = kim_code
#if we were given everything, we are good to go
elif name and leader and num and version:
self.kim_code = database.format_kim_code(name,leader,num,version)
#if we weren't given a name, see if one exists
elif name is None and leader and num and version:
name = database.look_for_name(leader,num,version)
self.kim_code_name = name
self.kim_code = database.format_kim_code(name,leader,num,version)
#if we weren't given a version
elif name and leader and num and version is None:
name,leader,num,version = database.get_latest_version(name,leader,num)
self.kim_code_version = version
self.kim_code = database.format_kim_code(name,leader,num,version)
#if we weren't given a name or version
elif name is None and leader and num and version is None:
name,leader,num,version = database.get_latest_version(name,leader,num)
self.kim_code_name = name
self.kim_code_version = version
self.kim_code = database.format_kim_code(name,leader,num,version)
self.kim_code_id = database.strip_name(self.kim_code)
self.kim_code_short = database.strip_version(self.kim_code)
# Determine where this KIMObject sits in the local repository
if precheck:
self.parent_dir = os.path.join(os.path.join(cf.LOCAL_REPOSITORY_PATH,'precheck'),
self.kim_code_leader.lower())
else:
self.parent_dir = os.path.join(cf.LOCAL_REPOSITORY_PATH, self.kim_code_leader.lower())
if subdir is not None:
self.path = os.path.join(self.parent_dir, subdir)
else:
self.path = os.path.join(self.parent_dir, self.kim_code)
if abspath is not None:
self.path = abspath
# assume the object is not built by default
self.built = False
def kim_obj(kim_code, *args, **kwargs):
""" Just given a kim_code try to make the right object, i.e. try to make a TE code a Test, etc. """
name,leader,num,version = database.parse_kim_code(kim_code)
cls = code_to_model.get(leader, KIMObject)
return cls(kim_code, *args, **kwargs)
def __init__(self,kim_code,search=True,subdir=None):
""" Initialize a KIMObject given the kim_code, where partial kim codes are promoted if possible,
if search is False, then don't look for existing ones
Args:
kim_code (str)
A full or partial kim_code, i.e. one like:
* "Full_Name_of_thing__TE_000000000000_000"
* "TE_000000000000_000"
* "TE_000000000000"
* "Full_Name_of_thing__TE_000000000000"
search (bool)
Whether or not to search the directory structure for the fullest match,
false is useful when creating new KIMObjects to avoid hitting a PipelineSearchError
subdir (str)
In order to point to a directory that does not follow that pattern
/home/openkim/openkim-repository/{mo,md,te...}/KIM_CODE/KIM_CODE
can provide the folder of
/home/openkim/openkim-repository/{mo,md,te...}/SUBDIR/KIM_CODE
"""
logger.debug("Initializing a new KIMObject: %r", kim_code)
name, leader, num, version = database.parse_kim_code(kim_code)
# test to see if we have the right leader
if self.required_leader:
assert leader==self.required_leader,"{} not a valid KIM code for {}".format(kim_code, self.__class__.__name__)
#grab the attributes
self.kim_code_name = name
self.kim_code_leader = leader
self.kim_code_number = num
self.kim_code_version = version
if not search:
self.kim_code = kim_code
#if we were given everything, we are good to go
elif name and leader and num and version:
self.kim_code = database.format_kim_code(name,leader,num,version)
#if we weren't given a name, see if one exists
elif name is None and leader and num and version:
name = database.look_for_name(leader,num,version)
self.kim_code_name = name
self.kim_code = database.format_kim_code(name,leader,num,version)
#if we weren't given a version
elif name and leader and num and version is None:
name,leader,num,version = database.get_latest_version(name,leader,num)
self.kim_code_version = version
self.kim_code = database.format_kim_code(name,leader,num,version)
#if we weren't given a name or version
elif name is None and leader and num and version is None:
name,leader,num,version = database.get_latest_version(name,leader,num)
self.kim_code_name = name
self.kim_code_version = version
self.kim_code = database.format_kim_code(name,leader,num,version)
self.kim_code_id = database.strip_name(self.kim_code)
self.kim_code_short = database.strip_version(self.kim_code)
self.parent_dir = os.path.join(cf.KIM_REPOSITORY_DIR, self.kim_code_leader.lower())
if subdir is not None:
self.path = os.path.join(self.parent_dir, subdir)
else:
self.path = os.path.join(self.parent_dir, self.kim_code)
def kim_obj(kim_code, *args, **kwargs):
""" Just given a kim_code try to make the right object, i.e. try to make a TE code a Test, etc. """
name,leader,num,version = database.parse_kim_code(kim_code)
cls = code_to_model.get(leader, KIMObject)
return cls(kim_code, *args, **kwargs)
def stripversion(kim):
kimtup = database.parse_kim_code(kim)
newtup = ( kimtup.name, kimtup.leader, kimtup.num, None)
return database.format_kim_code( *newtup )
def latestversion(kim):
return database.format_kim_code(
*database.get_latest_version(database.parse_kim_code(kim)))
def push_jobs(self, update):
""" Push all of the jobs that need to be done given an update """
self.make_all()
kimid = update['kimid']
status = update['status']
priority_factor = self.priority_to_number(update['priority'])
if database.isuuid(kimid):
priority = int(priority_factor * 1000000)
self.check_dependencies_and_push(kimid, priority, status)
return
name, leader, num, version = database.parse_kim_code(kimid)
checkmatch = False
if leader == "VT":
# for every test launch
test = kimobjects.TestVerification(kimid)
models = list(kimobjects.Test.all())
tests = [test] * ll(models)
elif leader == "VM":
#for all of the models, run a job
test = kimobjects.ModelVerification(kimid)
models = list(kimobjects.Model.all())
tests = [test] * ll(models)
else:
if status == "approved":
if leader == "TE":
# for all of the models, add a job
test = kimobjects.Test(kimid)
models = list(test.models)
tests = [test] * ll(models)
elif leader == "MO":
# for all of the tests, add a job
model = kimobjects.Model(kimid)
tests = list(model.tests)
models = [model] * ll(tests)
elif leader == "TD":
# if it is a new version of an existing test driver, hunt
# down all of the tests that use it and launch their
# corresponding jobs
driver = kimobjects.TestDriver(kimid)
temp_tests = list(driver.tests)
models = []
tests = []
for t in temp_tests:
tmodels = list(t.models)
if len(tmodels) > 0:
models.extend(tmodels)
tests.extend([t] * ll(tmodels))
elif leader == "MD":
# if this is a new version, hunt down all of the models
# that rely on it and recompute their results
driver = kimobjects.ModelDriver(kimid)
temp_models = list(driver.models)
tests = []
models = []
for m in temp_models:
mtests = list(m.tests)
if len(mtests) > 0:
tests.extend(mtests)
models.extend([m] * ll(mtests))
else:
self.logger.error("Tried to update an invalid KIM ID!: %r",
kimid)
checkmatch = True
if status == "pending":
rsync_tools.director_pending_read(kimid)
self.make_all()
if leader == "TE":
# run against all test verifications
tests = list(kimobjects.VertificationTest.all())
models = [kimobjects.Test(kimid, search=False)] * ll(tests)
elif leader == "MO":
# run against all model verifications
tests = list(kimobjects.VertificationModel.all())
models = [kimobjects.Model(kimid, search=False)
] * ll(tests)
elif leader == "TD":
# a pending test driver
pass
elif leader == "MD":
# a pending model driver
pass
else:
self.logger.error("Tried to update an invalid KIM ID!: %r",
kimid)
checkmatch = False
for test, model in zip(tests, models):
if not checkmatch or (checkmatch
and kimapi.valid_match(test, model)):
priority = int(priority_factor *
database.test_model_to_priority(test, model) *
1000000)
self.check_dependencies_and_push((test, model), priority,
status)
def stripversion(kim):
kimtup = database.parse_kim_code(kim)
newtup = (kimtup.name, kimtup.leader, kimtup.num, None)
return database.format_kim_code(*newtup)
def run(self):
""" Start to listen, tunnels should be open and ready """
self.connect()
self.bean.watch(cf.TUBE_JOBS)
""" Endless loop that awaits jobs to run """
while True:
self.logger.info("Waiting for jobs...")
job = self.bean.reserve()
self.job = job
# if appears that there is a 120sec re-birth of jobs that have been reserved
# and I do not want to put an artificial time limit, so let's bury jobs
# when we get them
job.bury()
self.comm.send_msg("running", job.body)
# update the repository, attempt to run the job and return the results to the director
try:
jobmsg = network.Message(string=job.body)
pending = True if jobmsg == "pending" else False
except simplejson.JSONDecodeError:
# message is not JSON decodeable
self.logger.error("Did not recieve valid JSON, {}".format(
job.body))
job.delete()
continue
except KeyError:
# message does not have the right keys
self.logger.error(
"Did not recieve a valid message, missing key: {}".format(
job.body))
job.delete()
continue
self.jobmsg = jobmsg
try:
name, leader, num, version = database.parse_kim_code(
jobmsg.job[0])
except InvalidKIMID as e:
# we were not given a valid kimid
self.logger.error("Could not parse {} as a valid KIMID".format(
jobmsg.job[0]))
self.job_message(jobmsg, errors=e, tube=cf.TUBE_ERRORS)
job.delete()
continue
try:
# check to see if this is a verifier or an actual test
with self.rsynclock:
self.logger.info("Rsyncing from repo %r",
jobmsg.job + jobmsg.depends)
rsync_tools.worker_read(*jobmsg.job,
depends=jobmsg.depends,
pending=pending)
runner_kcode, subject_kcode = jobmsg.job
runner = kimobjects.kim_obj(runner_kcode)
subject = kimobjects.kim_obj(subject_kcode)
self.builder.lock_build(runner)
self.builder.lock_build(subject)
self.logger.info("Running (%r,%r)", runner, subject)
comp = compute.Computation(runner,
subject,
result_code=jobmsg.jobid)
errormsg = None
try:
comp.run(extrainfo=self.boxinfo)
except Exception as e:
errormsg = e
self.logger.exception("Errors occured, moving to er/")
else:
self.logger.debug("Sending result message back")
finally:
self.logger.info("Rsyncing results %r", jobmsg.jobid)
with self.rsynclock:
rsync_tools.worker_write(comp.result_path)
if errormsg:
self.job_message(jobmsg, errors=e, tube=cf.TUBE_ERRORS)
else:
self.job_message(jobmsg, tube=cf.TUBE_RESULTS)
job.delete()
except Exception as e:
self.logger.exception(
"Failed to initalize run, deleting... %r" % e)
self.job_message(jobmsg, errors=e, tube=cf.TUBE_ERRORS)
job.delete()
self.job = None
self.jobsmsg = None
def run_job(self):
""" Endless loop that awaits jobs to run """
while True:
with loglock:
self.logger.info("Waiting for jobs...")
job = self.bean.reserve()
self.job = job
# if appears that there is a 120sec re-birth of jobs that have been reserved
# and I do not want to put an artificial time limit, so let's bury jobs
# when we get them
job.bury()
self.comm.send_msg("running", job.body)
# update the repository, attempt to run the job and return the results to the director
try:
jobmsg = Message(string=job.body)
except simplejson.JSONDecodeError:
# message is not JSON decodeable
with loglock:
self.logger.error("Did not recieve valid JSON, {}".format(job.body))
job.delete()
continue
except KeyError:
# message does not have the right keys
with loglock:
self.logger.error("Did not recieve a valid message, missing key: {}".format(job.body))
job.delete()
continue
self.jobmsg = jobmsg
# check to see if this is a verifier or an actual test
try:
name,leader,num,version = database.parse_kim_code(jobmsg.job[0])
except InvalidKIMID as e:
# we were not given a valid kimid
with loglock:
self.logger.error("Could not parse {} as a valid KIMID".format(jobmsg.job[0]))
self.job_message(jobmsg, errors=e, tube=TUBE_ERRORS)
job.delete()
continue
if leader == "VT" or leader == "VM":
try:
with buildlock:
with loglock:
self.logger.info("rsyncing to repo %r", jobmsg.job+jobmsg.depends)
rsync_tools.worker_verification_read(*jobmsg.job, depends=jobmsg.depends)
self.make_all()
verifier_kcode, subject_kcode = jobmsg.job
verifier = kimobjects.Verifier(verifier_kcode)
subject = kimobjects.Subject(subject_kcode)
with loglock:
self.logger.info("Running (%r,%r)",verifier,subject)
comp = compute.Computation(verifier, subject)
comp.run(jobmsg.jobid)
result = kimobjects.Result(jobmsg.jobid).results
with loglock:
self.logger.info("rsyncing results %r", jobmsg.jobid)
rsync_tools.worker_verification_write(jobmsg.jobid)
with loglock:
self.logger.info("sending result message back")
self.job_message(jobmsg, results=result, tube=TUBE_RESULTS)
job.delete()
# could be that a dependency has not been met.
# put it back on the queue to wait
except PipelineDataMissing as e:
if job.stats()['age'] < 5*PIPELINE_JOB_TIMEOUT:
with loglock:
self.logger.error("Run failed, missing data. Returning to queue... (%r)" % e)
job.release(delay=PIPELINE_JOB_TIMEOUT)
else:
with loglock:
self.logger.error("Run failed, missing data. Lifetime has expired, deleting (%r)" % e)
job.delete()
# another problem has occurred. just remove the job
# and send the error back along the error queue
except Exception as e:
with loglock:
self.logger.error("Run failed, deleting... %r" % e)
self.job_message(jobmsg, errors=e, tube=TUBE_ERRORS)
job.delete()
else:
try:
with buildlock:
with loglock:
self.logger.info("rsyncing to repo %r %r", jobmsg.job,jobmsg.depends)
rsync_tools.worker_test_result_read(*jobmsg.job, depends=jobmsg.depends)
self.make_all()
test_kcode, model_kcode = jobmsg.job
test = kimobjects.Test(test_kcode)
model = kimobjects.Model(model_kcode)
with loglock:
self.logger.info("Running (%r,%r)",test,model)
comp = compute.Computation(test, model)
comp.run(jobmsg.jobid)
result = kimobjects.Result(jobmsg.jobid).results
with loglock:
self.logger.info("rsyncing results %r", jobmsg.jobid)
rsync_tools.worker_test_result_write(jobmsg.jobid)
with loglock:
self.logger.info("sending result message back")
self.job_message(jobmsg, results=result, tube=TUBE_RESULTS)
job.delete()
# could be that a dependency has not been met.
# put it back on the queue to wait
except PipelineDataMissing as e:
if job.stats()['age'] < 5*PIPELINE_JOB_TIMEOUT:
with loglock:
self.logger.error("Run failed, missing data. Returning to queue... (%r)" % e)
job.release(delay=PIPELINE_JOB_TIMEOUT)
else:
with loglock:
self.logger.error("Run failed, missing data. Lifetime has expired, deleting (%r)" % e)
job.delete()
# another problem has occurred. just remove the job
# and send the error back along the error queue
except Exception as e:
with loglock:
self.logger.error("Run failed, deleting... %r" % e)
self.job_message(jobmsg, errors="%r"%e, tube=TUBE_ERRORS)
job.delete()
self.job = None
self.jobsmsg = None
def kid_to_folder(kid):
""" Convert a kim_code to its directory """
name,leader,num,version = parse_kim_code(kid)
path = os.path.join(leader.lower(),kid)
return path
