def get_new_change_status_files():
"""Compare old and new status files from $MAST_CONTROL
and $MAST_SCRATCH.
Write an indication whether the status files have changed,
not changed. Use "archived" if a recipe has been archived or is
missing from SCRATCH for some other reason.
"""
rdict=dict()
mastcontrol=dirutil.get_mast_control_path()
mastscratch=dirutil.get_mast_scratch_path()
olddirs=list()
recipedirs=dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles"))
for recipedir in recipedirs:
ingreddirs = dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles",recipedir))
for ingreddir in ingreddirs:
olddirs.append("%s/%s" % (recipedir, ingreddir))
srecdirs = dirutil.immediate_subdirs(mastscratch)
for srecdir in srecdirs:
singreddirs = dirutil.immediate_subdirs(os.path.join(mastscratch,srecdir))
for singreddir in singreddirs:
csfile = os.path.join(mastscratch, srecdir, singreddir, "change_status.txt")
if os.path.isfile(csfile):
if not "%s/%s" % (srecdir, singreddir) in olddirs:
if not srecdir in rdict.keys():
rdict[srecdir]=dict()
rdict[srecdir]["MAIN"]="changed"
rdict[srecdir][singreddir]="send"
return rdict
def compare_old_and_new_change_status_files():
"""Compare old and new status files from $MAST_CONTROL
and $MAST_SCRATCH.
Write an indication whether the status files have changed,
not changed. Use "archived" if a recipe has been archived or is
missing from SCRATCH for some other reason.
"""
rdict=dict()
mastcontrol=dirutil.get_mast_control_path()
mastscratch=dirutil.get_mast_scratch_path()
recipedirs=dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles"))
for recipedir in recipedirs:
mystatus="unknown"
rdict[recipedir]=dict()
changelist=list()
if not os.path.exists(os.path.join(mastscratch,recipedir)):
mystatus="archived"
else:
ingreddirs = dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles",recipedir))
for ingreddir in ingreddirs:
scratchstatusfile = MASTFile(os.path.join(mastscratch,recipedir,ingreddir,"change_status.txt"))
controlstatusfile = MASTFile(os.path.join(mastcontrol,"changestatusfiles",recipedir,ingreddir,"change_status.txt"))
if scratchstatusfile.data == controlstatusfile.data:
mystatus="unchanged"
else:
mystatus="changed"
rdict[recipedir][ingreddir]="send"
rdict[recipedir]["MAIN"]=mystatus
return rdict
def get_new_change_status_files():
"""Compare old and new status files from $MAST_CONTROL
and $MAST_SCRATCH.
Write an indication whether the status files have changed,
not changed. Use "archived" if a recipe has been archived or is
missing from SCRATCH for some other reason.
"""
rdict = dict()
mastcontrol = dirutil.get_mast_control_path()
mastscratch = dirutil.get_mast_scratch_path()
olddirs = list()
recipedirs = dirutil.immediate_subdirs(
os.path.join(mastcontrol, "changestatusfiles"))
for recipedir in recipedirs:
ingreddirs = dirutil.immediate_subdirs(
os.path.join(mastcontrol, "changestatusfiles", recipedir))
for ingreddir in ingreddirs:
olddirs.append("%s/%s" % (recipedir, ingreddir))
srecdirs = dirutil.immediate_subdirs(mastscratch)
for srecdir in srecdirs:
singreddirs = dirutil.immediate_subdirs(
os.path.join(mastscratch, srecdir))
for singreddir in singreddirs:
csfile = os.path.join(mastscratch, srecdir, singreddir,
"change_status.txt")
if os.path.isfile(csfile):
if not "%s/%s" % (srecdir, singreddir) in olddirs:
if not srecdir in rdict.keys():
rdict[srecdir] = dict()
rdict[srecdir]["MAIN"] = "changed"
rdict[srecdir][singreddir] = "send"
return rdict
def zip_only_changed():
"""Zip only the changed recipes"""
mastcontrol = dirutil.get_mast_control_path()
mastscratch = dirutil.get_mast_scratch_path()
rchangefile = MASTFile(os.path.join(mastcontrol, "recipechangefile.txt"))
#import shutil
#shutil.rmtree(os.path.join(mastcontrol,"statusfiles"))
totarfile = MASTFile()
for ritem in rchangefile.data:
if ":send" in ritem:
recipe = ritem.split(":")[0]
ingred = ritem.split(":")[1]
totarfile.data.append(
os.path.join("MAST/SCRATCH", recipe, ingred) + "\n")
totarfile.data.append(
os.path.join("MAST/SCRATCH", recipe, "status.txt") + "\n")
elif ":archived" in ritem:
recipe = ritem.split(":")[0]
totarfile.data.append(os.path.join("MAST/ARCHIVE", recipe) + "\n")
elif ":changed" in ritem:
recipe = ritem.split(":")[0]
totarfile.data.append(
os.path.join("MAST/SCRATCH", recipe, "MAST_ERROR") + "\n")
totarfile.data.append("MAST/CONTROL\n")
totarfile.to_file(os.getcwd() + "/tarthis")
return
def save_old_change_status_files():
"""Save off old status files from $MAST_SCRATCH
into $MAST_CONTROL
"""
mastcontrol = dirutil.get_mast_control_path()
if not os.path.exists(os.path.join(mastcontrol, "changestatusfiles")):
os.mkdir(os.path.join(mastcontrol, "changestatusfiles"))
mastscratch = dirutil.get_mast_scratch_path()
recipedirs = dirutil.immediate_subdirs(mastscratch)
for recipedir in recipedirs:
ingreddirs = dirutil.immediate_subdirs(
os.path.join(mastscratch, recipedir))
for ingreddir in ingreddirs:
csfile = os.path.join(mastscratch, recipedir, ingreddir,
"change_status.txt")
if os.path.isfile(csfile):
changestatusfile = MASTFile(csfile)
trydir = os.path.join(mastcontrol, "changestatusfiles",
recipedir)
if not os.path.exists(trydir):
os.mkdir(trydir)
trydir2 = os.path.join(trydir, ingreddir)
if not os.path.exists(trydir2):
os.mkdir(trydir2)
changestatusfile.to_file("%s/change_status.txt" % trydir2)
return True
def save_ingredient_files(listfilename="",whichfilename=""):
"""Save off old files from ingredients in $MAST_SCRATCH
into $MAST_CONTROL
Args:
listfilename <str>: File name for the list of files to be saved
whichfilename <str>: Short file name to be saved (identical file
for all recipes)
"""
if listfilename == "":
raise NameError("No list file name given.")
if whichfilename == "":
raise NameError("No file name given for which files to save.")
mastcontrol=dirutil.get_mast_control_path()
if not os.path.exists(os.path.join(mastcontrol,listfilename)):
os.mkdir(os.path.join(mastcontrol,listfilename))
mastscratch=dirutil.get_mast_scratch_path()
recipedirs=dirutil.immediate_subdirs(mastscratch)
for recipedir in recipedirs:
recipefulldir=os.path.join(mastscratch,recipedir)
ingreddirs=dirutil.immediate_subdirs(recipefulldir)
for ingreddir in ingreddirs:
statfilepath = os.path.join(recipefulldir, ingreddir, whichfilename)
if os.path.exists(statfilepath):
statusfile = MASTFile(statfilepath)
trydir = os.path.join(mastcontrol,listfilename,recipedir,ingreddir)
if not os.path.exists(trydir):
os.mkdir(trydir)
statusfile.to_file("%s/%s" % (trydir, whichfilename))
return True
def compare_old_and_new_change_status_files():
"""Compare old and new status files from $MAST_CONTROL
and $MAST_SCRATCH.
Write an indication whether the status files have changed,
not changed. Use "archived" if a recipe has been archived or is
missing from SCRATCH for some other reason.
"""
rdict = dict()
mastcontrol = dirutil.get_mast_control_path()
mastscratch = dirutil.get_mast_scratch_path()
recipedirs = dirutil.immediate_subdirs(
os.path.join(mastcontrol, "changestatusfiles"))
for recipedir in recipedirs:
mystatus = "unknown"
rdict[recipedir] = dict()
changelist = list()
if not os.path.exists(os.path.join(mastscratch, recipedir)):
mystatus = "archived"
else:
ingreddirs = dirutil.immediate_subdirs(
os.path.join(mastcontrol, "changestatusfiles", recipedir))
for ingreddir in ingreddirs:
scratchstatusfile = MASTFile(
os.path.join(mastscratch, recipedir, ingreddir,
"change_status.txt"))
controlstatusfile = MASTFile(
os.path.join(mastcontrol, "changestatusfiles", recipedir,
ingreddir, "change_status.txt"))
if scratchstatusfile.data == controlstatusfile.data:
mystatus = "unchanged"
else:
mystatus = "changed"
rdict[recipedir][ingreddir] = "send"
rdict[recipedir]["MAIN"] = mystatus
return rdict
def compare_old_and_new_status_files():
"""Compare old and new status files from $MAST_CONTROL
and $MAST_SCRATCH.
Write an indication whether the status files have changed,
not changed. Use "archived" if a recipe has been archived or is
missing from SCRATCH for some other reason.
"""
rdict=dict()
mastcontrol=dirutil.get_mast_control_path()
mastscratch=dirutil.get_mast_scratch_path()
recipedirs=dirutil.immediate_subdirs(os.path.join(mastcontrol,"statusfiles"))
for recipedir in recipedirs:
mystatus="unknown"
rdict[recipedir]=dict()
changelist=list()
if not os.path.exists(os.path.join(mastscratch,recipedir)):
mystatus="archived"
else:
scratchstatusfile = MASTFile(os.path.join(mastscratch,recipedir,"status.txt"))
controlstatusfile = MASTFile(os.path.join(mastcontrol,"statusfiles",recipedir,"status.txt"))
if scratchstatusfile.data == controlstatusfile.data:
mystatus="unchanged"
else:
mystatus="changed"
myidx=0
while myidx < len(scratchstatusfile.data):
oldline = controlstatusfile.data[myidx]
newline = scratchstatusfile.data[myidx]
if "#" in oldline:
pass
else:
ingred = oldline.split(":")[0].strip()
oldstatus = oldline.split(":")[1].strip()
newstatus = newline.split(":")[1].strip()
if (oldstatus == "P") and (newstatus == "P"):
rdict[recipedir][ingred]="AVOID"
elif (oldstatus == "C") and (newstatus == "C"):
rdict[recipedir][ingred]="AVOID"
else:
rdict[recipedir][ingred]="send"
myidx = myidx + 1
rdict[recipedir]["MAIN"]=mystatus
recipedirs=dirutil.immedate_subdirs(os.path.join(mastcontrol,"changestatusfiles"))
for recipedir in recipedirs:
ingreddirs=dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles",recipedir))
for ingreddir in ingreddirs:
oldchangestatus=MASTFile(os.path.join(mastcontrol,"changestatusfiles",recipedir,ingreddir,"change_status.txt"))
newchangestatus=MASTFile(os.path.join(mastscratch,recipedir,ingreddir,"change_status.txt"))
if not (oldchangestatus.data == newchangestatus.data):
rdict[recipedir]["MAIN"]="changed"
rdict[recipedir][ingreddir]="send"
rchangefile = MASTFile()
for rdir in rdict.keys():
for key, value in rdict[rdir].iteritems():
rchangefile.data.append("%s:%s:%s\n" % (rdir,key,value))
rchangefile.to_file(os.path.join(mastcontrol,"recipechangefile.txt"))
return True
def compare_old_and_new_status_files():
"""Compare old and new status files from $MAST_CONTROL
and $MAST_SCRATCH.
Write an indication whether the status files have changed,
not changed. Use "archived" if a recipe has been archived or is
missing from SCRATCH for some other reason.
"""
rdict = dict()
mastcontrol = dirutil.get_mast_control_path()
mastscratch = dirutil.get_mast_scratch_path()
recipedirs = dirutil.immediate_subdirs(
os.path.join(mastcontrol, "statusfiles"))
for recipedir in recipedirs:
mystatus = "unknown"
rdict[recipedir] = dict()
changelist = list()
if not os.path.exists(os.path.join(mastscratch, recipedir)):
mystatus = "archived"
else:
scratchstatusfile = MASTFile(
os.path.join(mastscratch, recipedir, "status.txt"))
controlstatusfile = MASTFile(
os.path.join(mastcontrol, "statusfiles", recipedir,
"status.txt"))
if scratchstatusfile.data == controlstatusfile.data:
mystatus = "unchanged"
else:
mystatus = "changed"
myidx = 0
while myidx < len(scratchstatusfile.data):
oldline = controlstatusfile.data[myidx]
newline = scratchstatusfile.data[myidx]
if "#" in oldline:
pass
else:
ingred = oldline.split(":")[0].strip()
oldstatus = oldline.split(":")[1].strip()
newstatus = newline.split(":")[1].strip()
if (oldstatus == "P") and (newstatus == "P"):
rdict[recipedir][ingred] = "AVOID"
elif (oldstatus == "C") and (newstatus == "C"):
rdict[recipedir][ingred] = "AVOID"
else:
rdict[recipedir][ingred] = "send"
myidx = myidx + 1
rdict[recipedir]["MAIN"] = mystatus
rchangefile = MASTFile()
for rdir in rdict.keys():
for key, value in rdict[rdir].iteritems():
rchangefile.data.append("%s:%s:%s\n" % (rdir, key, value))
rchangefile.to_file(os.path.join(mastcontrol, "recipechangefile.txt"))
return True
def get_new_mast_error_files():
"""Get new MAST_ERROR files.
"""
rdict=dict()
mastcontrol=dirutil.get_mast_control_path()
mastscratch=dirutil.get_mast_scratch_path()
olddirs=list()
srecdirs = dirutil.immediate_subdirs(mastscratch)
for srecdir in srecdirs:
merrfile = os.path.join(mastscratch, srecdir, "MAST_ERROR")
if os.path.isfile(merrfile):
if not srecdir in rdict.keys():
rdict[srecdir] = dict()
rdict[srecdir]["MAIN"]="changed"
return rdict
def get_new_mast_error_files():
"""Get new MAST_ERROR files.
"""
rdict = dict()
mastcontrol = dirutil.get_mast_control_path()
mastscratch = dirutil.get_mast_scratch_path()
olddirs = list()
srecdirs = dirutil.immediate_subdirs(mastscratch)
for srecdir in srecdirs:
merrfile = os.path.join(mastscratch, srecdir, "MAST_ERROR")
if os.path.isfile(merrfile):
if not srecdir in rdict.keys():
rdict[srecdir] = dict()
rdict[srecdir]["MAIN"] = "changed"
return rdict
def save_old_status_files():
"""Save off old status files from $MAST_SCRATCH
into $MAST_CONTROL
"""
mastcontrol=dirutil.get_mast_control_path()
if not os.path.exists(os.path.join(mastcontrol,"statusfiles")):
os.mkdir(os.path.join(mastcontrol,"statusfiles"))
mastscratch=dirutil.get_mast_scratch_path()
recipedirs=dirutil.immediate_subdirs(mastscratch)
for recipedir in recipedirs:
statusfile = MASTFile(os.path.join(mastscratch,recipedir,"status.txt"))
trydir = os.path.join(mastcontrol,"statusfiles",recipedir)
if not os.path.exists(trydir):
os.mkdir(trydir)
statusfile.to_file("%s/status.txt" % trydir)
return True
def save_old_status_files():
"""Save off old status files from $MAST_SCRATCH
into $MAST_CONTROL
"""
mastcontrol = dirutil.get_mast_control_path()
if not os.path.exists(os.path.join(mastcontrol, "statusfiles")):
os.mkdir(os.path.join(mastcontrol, "statusfiles"))
mastscratch = dirutil.get_mast_scratch_path()
recipedirs = dirutil.immediate_subdirs(mastscratch)
for recipedir in recipedirs:
statusfile = MASTFile(
os.path.join(mastscratch, recipedir, "status.txt"))
trydir = os.path.join(mastcontrol, "statusfiles", recipedir)
if not os.path.exists(trydir):
os.mkdir(trydir)
statusfile.to_file("%s/status.txt" % trydir)
return True
def add_handler_for_control(logger):
"""Add a handler for control
Args:
logger <logging.Logger>: Logger
"""
logger.setLevel(logging.INFO)
#formatstr ='%(levelname)8s : %(name)10s: %(message)s'
formatstr ='%(levelname)8s : %(message)s'
formatter = logging.Formatter(formatstr)
controlhandler = logging.FileHandler(filename="%s/mast.log" % dirutil.get_mast_control_path())
controlhandler.setFormatter(formatter)
controlfilter = ControlFilter()
controlhandler.addFilter(controlfilter)
if not getattr(logger, 'has_control_handler', None):
logger.addHandler(controlhandler)
logger.has_control_handler = True
return logger
def validate_recipe_name(rname):
"""Validate the recipe name.
Use $MAST_CONTROL if recipe name is not found in $MAST_SCRATCH.
Args:
rname <str>: Recipe name candidate
Returns:
rnamevalid <str>: Validated recipe name
"""
#check existence of rname:
mast_scratch = dirutil.get_mast_scratch_path()
rdirs = os.listdir(mast_scratch)
rnamevalid = dirutil.get_mast_control_path()
for rdir in rdirs:
if rdir in rname:
rnamevalid = os.path.join(mast_scratch, rdir)
return rnamevalid
return rnamevalid
def write_to_submit_list(mydir):
"""Write an entry to the submission list in
$MAST_CONTROL/submitlist
Args:
mydir <str>: Directory which includes submission
script submit.sh for a single
calculation to be submitted to the
queue
"""
mast_control = dirutil.get_mast_control_path() #setting here instead of globally allows tests to run in isolated test_control folder
submitlist=os.path.join(mast_control, "submitlist")
if os.path.isfile(submitlist):
submitfile=MASTFile(submitlist)
else:
submitfile=MASTFile()
submitfile.data.append(mydir + "\n")
submitfile.to_file(submitlist)
return
def merge_dictionaries_and_write_recipechangefile():
rdict_status = compare_old_and_new_status_files()
rdict_change_status = compare_old_and_new_change_status_files()
rdict_new_change_status = get_new_change_status_files()
rdict_error_files = get_new_mast_error_files()
rchangefile = MASTFile()
rdict = dict()
for recdir in rdict_status.keys():
if not recdir in rdict.keys():
rdict[recdir] = dict()
for ingreddir in rdict_status[recdir].keys():
if not ingreddir in rdict[recdir].keys():
rdict[recdir][ingreddir] = rdict_status[recdir][ingreddir]
for recdir in rdict_change_status.keys():
if not recdir in rdict.keys():
rdict[recdir] = dict()
for ingreddir in rdict_change_status[recdir].keys():
if not ingreddir in rdict[recdir].keys():
rdict[recdir][ingreddir] = rdict_change_status[recdir][
ingreddir]
else:
if rdict_change_status[recdir][ingreddir] == "send":
rdict[recdir][ingreddir] = "send" #update to send
for recdir in rdict_new_change_status.keys():
if not recdir in rdict.keys():
rdict[recdir] = dict()
for ingreddir in rdict_new_change_status[recdir].keys():
if not ingreddir in rdict[recdir].keys():
rdict[recdir][ingreddir] = rdict_new_change_status[recdir][
ingreddir]
else:
if rdict_new_change_status[recdir][ingreddir] == "send":
rdict[recdir][ingreddir] = "send" #update to send
for recdir in rdict_error_files.keys():
if not recdir in rdict.keys():
rdict[recdir] = dict()
for ingreddir in rdict_error_files[recdir].keys():
rdict[recdir][ingreddir] = rdict_error_files[recdir][ingreddir]
for rdir in rdict.keys():
for key, value in rdict[rdir].iteritems():
rchangefile.data.append("%s:%s:%s\n" % (rdir, key, value))
mastcontrol = dirutil.get_mast_control_path()
rchangefile.to_file(os.path.join(mastcontrol, "recipechangefile.txt"))
return True
def config_for_recipe_and_control(rname):
rnamevalid = validate_recipe_name(rname)
formatstr ='%(asctime)s : %(module)10s: %(levelname)8s : %(message)s'
longformatter = logging.Formatter(formatstr)
formatstrshort ='%(levelname)8s : %(message)s'
shortformatter = logging.Formatter(formatstrshort)
recipehandler = logging.FileHandler(filename="%s/mast_recipe.log" % rnamevalid)
recipehandler.setFormatter(longformatter)
controlhandler = logging.FileHandler(filename="%s/mast.log" % dirutil.get_mast_control_path())
controlhandler.setFormatter(shortformatter)
logger=logging.getLogger('')
if not getattr(logger, 'has_recipe_handler', None):
logger.addHandler(recipehandler)
logger.has_recipe_handler = True
if not getattr(logger, 'has_control_handler', None):
logger.addHandler(controlhandler)
logger.has_control_handler = True
logger.setLevel(logging.INFO)
return logger
def write_to_submit_list(mydir):
"""Write an entry to the submission list in
$MAST_CONTROL/submitlist
Args:
mydir <str>: Directory which includes submission
script submit.sh for a single
calculation to be submitted to the
queue
"""
mast_control = dirutil.get_mast_control_path(
) #setting here instead of globally allows tests to run in isolated test_control folder
submitlist = os.path.join(mast_control, "submitlist")
if os.path.isfile(submitlist):
submitfile = MASTFile(submitlist)
else:
submitfile = MASTFile()
submitfile.data.append(mydir + "\n")
submitfile.to_file(submitlist)
return
def merge_dictionaries_and_write_recipechangefile():
rdict_status = compare_old_and_new_status_files()
rdict_change_status = compare_old_and_new_change_status_files()
rdict_new_change_status = get_new_change_status_files()
rdict_error_files = get_new_mast_error_files()
rchangefile = MASTFile()
rdict=dict()
for recdir in rdict_status.keys():
if not recdir in rdict.keys():
rdict[recdir]=dict()
for ingreddir in rdict_status[recdir].keys():
if not ingreddir in rdict[recdir].keys():
rdict[recdir][ingreddir] = rdict_status[recdir][ingreddir]
for recdir in rdict_change_status.keys():
if not recdir in rdict.keys():
rdict[recdir]=dict()
for ingreddir in rdict_change_status[recdir].keys():
if not ingreddir in rdict[recdir].keys():
rdict[recdir][ingreddir] = rdict_change_status[recdir][ingreddir]
else:
if rdict_change_status[recdir][ingreddir] == "send":
rdict[recdir][ingreddir] = "send" #update to send
for recdir in rdict_new_change_status.keys():
if not recdir in rdict.keys():
rdict[recdir]=dict()
for ingreddir in rdict_new_change_status[recdir].keys():
if not ingreddir in rdict[recdir].keys():
rdict[recdir][ingreddir] = rdict_new_change_status[recdir][ingreddir]
else:
if rdict_new_change_status[recdir][ingreddir] == "send":
rdict[recdir][ingreddir] = "send" #update to send
for recdir in rdict_error_files.keys():
if not recdir in rdict.keys():
rdict[recdir]=dict()
for ingreddir in rdict_error_files[recdir].keys():
rdict[recdir][ingreddir] = rdict_error_files[recdir][ingreddir]
for rdir in rdict.keys():
for key, value in rdict[rdir].iteritems():
rchangefile.data.append("%s:%s:%s\n" % (rdir,key,value))
mastcontrol=dirutil.get_mast_control_path()
rchangefile.to_file(os.path.join(mastcontrol,"recipechangefile.txt"))
return True
def zip_only_changed():
"""Zip only the changed recipes"""
mastcontrol=dirutil.get_mast_control_path()
mastscratch=dirutil.get_mast_scratch_path()
rchangefile = MASTFile(os.path.join(mastcontrol,"recipechangefile.txt"))
#import shutil
#shutil.rmtree(os.path.join(mastcontrol,"statusfiles"))
totarfile = MASTFile()
for ritem in rchangefile.data:
if ":send" in ritem:
recipe=ritem.split(":")[0]
ingred=ritem.split(":")[1]
totarfile.data.append(os.path.join("MAST/SCRATCH",recipe,ingred)+"\n")
totarfile.data.append(os.path.join("MAST/SCRATCH",recipe,"status.txt") + "\n")
elif ":archived" in ritem:
recipe=ritem.split(":")[0]
totarfile.data.append(os.path.join("MAST/ARCHIVE",recipe)+"\n")
totarfile.data.append("MAST/CONTROL\n")
totarfile.to_file(os.getcwd() + "/tarthis")
return
def add_mast_handler(logger, hname="mast.log"):
"""Add a handler and format string
Args:
logger <logging.Logger>: Logger for which to add the handler
hname <str>: handler name (file name)
"""
logger.setLevel(logging.INFO)
if os.getenv("MAST_DEBUG") == None:
pass
else:
logger.setLevel(logging.DEBUG)
formatstr ='%(levelname)8s: %(name)s: %(asctime)s: %(message)s'
formatter = logging.Formatter(formatstr)
monitorhandler = logging.FileHandler(filename=os.path.join(dirutil.get_mast_control_path(), hname))
monitorhandler.setFormatter(formatter)
#controlfilter = ControlFilter()
#controlhandler.addFilter(controlfilter)
if not getattr(logger, 'has_monitor_handler', None):
logger.addHandler(monitorhandler)
logger.has_monitor_handler = True
return logger
def save_old_change_status_files():
"""Save off old status files from $MAST_SCRATCH
into $MAST_CONTROL
"""
mastcontrol=dirutil.get_mast_control_path()
if not os.path.exists(os.path.join(mastcontrol,"changestatusfiles")):
os.mkdir(os.path.join(mastcontrol,"changestatusfiles"))
mastscratch=dirutil.get_mast_scratch_path()
recipedirs=dirutil.immediate_subdirs(mastscratch)
for recipedir in recipedirs:
ingreddirs=dirutil.immediate_subdirs(os.path.join(mastscratch,recipedir))
for ingreddir in ingreddirs:
csfile = os.path.join(mastscratch,recipedir,ingreddir,"change_status.txt")
if os.path.isfile(csfile):
changestatusfile = MASTFile(csfile)
trydir = os.path.join(mastcontrol,"changestatusfiles",recipedir)
if not os.path.exists(trydir):
os.mkdir(trydir)
trydir2 = os.path.join(trydir, ingreddir)
if not os.path.exists(trydir2):
os.mkdir(trydir2)
changestatusfile.to_file("%s/change_status.txt" % trydir2)
return True
#
# Maintainer: Tam Mayeshiba
# Last updated: 2014-04-25
##############################################################
# Purpose: Modify and get information out of submission scripts.
# 11/9/12 created
import os
import sys
import math #TA to calc num_nodes
import importlib
from MAST.utility.mastfile import MASTFile
from MAST.utility import MASTError
from MAST.utility import dirutil
mast_control = dirutil.get_mast_control_path()
mast_install = dirutil.get_mast_install_path()
mast_platform = dirutil.get_mast_platform()
#mysc = "submit.platforms.script_commands_%s" % mast_platform
#my_script_commands = importlib.import_module(mysc)
"""
Script commands
"""
def write_submit_script(keywords):
"""This script example is built on the following ingredient keywords,
and may require significant customization.
mast_processors
mast_ppn
# 10/25/12 TTM created
# 12/6/12 TTM added UKY DLX commands
import os
import sys
import math #TA to calc num_nodes
import time
import importlib
import subprocess
import shutil
import logging
from MAST.utility.mastfile import MASTFile
from MAST.utility import MASTError
from MAST.utility import dirutil
from MAST.utility import loggerutils
mast_control = dirutil.get_mast_control_path()
myqc = "MAST.submit.platforms.%s.queue_commands" % dirutil.get_mast_platform()
my_queue_commands = importlib.import_module(myqc)
def direct_shell_command(scriptname="submit.sh"):
myds = my_queue_commands.direct_shell_command()
return "%s %s" % (myds, scriptname)
def queue_submission_command(scriptname="submit.sh"):
myqs = my_queue_commands.queue_submission_command()
return "%s %s" % (myqs, scriptname)
def write_to_submit_list(mydir):
"""Write an entry to the submission list in
$MAST_CONTROL/submitlist
