def compare_old_and_new_change_status_files(): """Compare old and new status files from $MAST_CONTROL and $MAST_SCRATCH. Write an indication whether the status files have changed, not changed. Use "archived" if a recipe has been archived or is missing from SCRATCH for some other reason. """ rdict=dict() mastcontrol=dirutil.get_mast_control_path() mastscratch=dirutil.get_mast_scratch_path() recipedirs=dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles")) for recipedir in recipedirs: mystatus="unknown" rdict[recipedir]=dict() changelist=list() if not os.path.exists(os.path.join(mastscratch,recipedir)): mystatus="archived" else: ingreddirs = dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles",recipedir)) for ingreddir in ingreddirs: scratchstatusfile = MASTFile(os.path.join(mastscratch,recipedir,ingreddir,"change_status.txt")) controlstatusfile = MASTFile(os.path.join(mastcontrol,"changestatusfiles",recipedir,ingreddir,"change_status.txt")) if scratchstatusfile.data == controlstatusfile.data: mystatus="unchanged" else: mystatus="changed" rdict[recipedir][ingreddir]="send" rdict[recipedir]["MAIN"]=mystatus return rdict
def save_old_change_status_files(): """Save off old status files from $MAST_SCRATCH into $MAST_CONTROL """ mastcontrol = dirutil.get_mast_control_path() if not os.path.exists(os.path.join(mastcontrol, "changestatusfiles")): os.mkdir(os.path.join(mastcontrol, "changestatusfiles")) mastscratch = dirutil.get_mast_scratch_path() recipedirs = dirutil.immediate_subdirs(mastscratch) for recipedir in recipedirs: ingreddirs = dirutil.immediate_subdirs( os.path.join(mastscratch, recipedir)) for ingreddir in ingreddirs: csfile = os.path.join(mastscratch, recipedir, ingreddir, "change_status.txt") if os.path.isfile(csfile): changestatusfile = MASTFile(csfile) trydir = os.path.join(mastcontrol, "changestatusfiles", recipedir) if not os.path.exists(trydir): os.mkdir(trydir) trydir2 = os.path.join(trydir, ingreddir) if not os.path.exists(trydir2): os.mkdir(trydir2) changestatusfile.to_file("%s/change_status.txt" % trydir2) return True
def get_new_change_status_files(): """Compare old and new status files from $MAST_CONTROL and $MAST_SCRATCH. Write an indication whether the status files have changed, not changed. Use "archived" if a recipe has been archived or is missing from SCRATCH for some other reason. """ rdict = dict() mastcontrol = dirutil.get_mast_control_path() mastscratch = dirutil.get_mast_scratch_path() olddirs = list() recipedirs = dirutil.immediate_subdirs( os.path.join(mastcontrol, "changestatusfiles")) for recipedir in recipedirs: ingreddirs = dirutil.immediate_subdirs( os.path.join(mastcontrol, "changestatusfiles", recipedir)) for ingreddir in ingreddirs: olddirs.append("%s/%s" % (recipedir, ingreddir)) srecdirs = dirutil.immediate_subdirs(mastscratch) for srecdir in srecdirs: singreddirs = dirutil.immediate_subdirs( os.path.join(mastscratch, srecdir)) for singreddir in singreddirs: csfile = os.path.join(mastscratch, srecdir, singreddir, "change_status.txt") if os.path.isfile(csfile): if not "%s/%s" % (srecdir, singreddir) in olddirs: if not srecdir in rdict.keys(): rdict[srecdir] = dict() rdict[srecdir]["MAIN"] = "changed" rdict[srecdir][singreddir] = "send" return rdict
def save_ingredient_files(listfilename="",whichfilename=""): """Save off old files from ingredients in $MAST_SCRATCH into $MAST_CONTROL Args: listfilename <str>: File name for the list of files to be saved whichfilename <str>: Short file name to be saved (identical file for all recipes) """ if listfilename == "": raise NameError("No list file name given.") if whichfilename == "": raise NameError("No file name given for which files to save.") mastcontrol=dirutil.get_mast_control_path() if not os.path.exists(os.path.join(mastcontrol,listfilename)): os.mkdir(os.path.join(mastcontrol,listfilename)) mastscratch=dirutil.get_mast_scratch_path() recipedirs=dirutil.immediate_subdirs(mastscratch) for recipedir in recipedirs: recipefulldir=os.path.join(mastscratch,recipedir) ingreddirs=dirutil.immediate_subdirs(recipefulldir) for ingreddir in ingreddirs: statfilepath = os.path.join(recipefulldir, ingreddir, whichfilename) if os.path.exists(statfilepath): statusfile = MASTFile(statfilepath) trydir = os.path.join(mastcontrol,listfilename,recipedir,ingreddir) if not os.path.exists(trydir): os.mkdir(trydir) statusfile.to_file("%s/%s" % (trydir, whichfilename)) return True
def zip_only_changed(): """Zip only the changed recipes""" mastcontrol = dirutil.get_mast_control_path() mastscratch = dirutil.get_mast_scratch_path() rchangefile = MASTFile(os.path.join(mastcontrol, "recipechangefile.txt")) #import shutil #shutil.rmtree(os.path.join(mastcontrol,"statusfiles")) totarfile = MASTFile() for ritem in rchangefile.data: if ":send" in ritem: recipe = ritem.split(":")[0] ingred = ritem.split(":")[1] totarfile.data.append( os.path.join("MAST/SCRATCH", recipe, ingred) + "\n") totarfile.data.append( os.path.join("MAST/SCRATCH", recipe, "status.txt") + "\n") elif ":archived" in ritem: recipe = ritem.split(":")[0] totarfile.data.append(os.path.join("MAST/ARCHIVE", recipe) + "\n") elif ":changed" in ritem: recipe = ritem.split(":")[0] totarfile.data.append( os.path.join("MAST/SCRATCH", recipe, "MAST_ERROR") + "\n") totarfile.data.append("MAST/CONTROL\n") totarfile.to_file(os.getcwd() + "/tarthis") return
def compare_old_and_new_change_status_files(): """Compare old and new status files from $MAST_CONTROL and $MAST_SCRATCH. Write an indication whether the status files have changed, not changed. Use "archived" if a recipe has been archived or is missing from SCRATCH for some other reason. """ rdict = dict() mastcontrol = dirutil.get_mast_control_path() mastscratch = dirutil.get_mast_scratch_path() recipedirs = dirutil.immediate_subdirs( os.path.join(mastcontrol, "changestatusfiles")) for recipedir in recipedirs: mystatus = "unknown" rdict[recipedir] = dict() changelist = list() if not os.path.exists(os.path.join(mastscratch, recipedir)): mystatus = "archived" else: ingreddirs = dirutil.immediate_subdirs( os.path.join(mastcontrol, "changestatusfiles", recipedir)) for ingreddir in ingreddirs: scratchstatusfile = MASTFile( os.path.join(mastscratch, recipedir, ingreddir, "change_status.txt")) controlstatusfile = MASTFile( os.path.join(mastcontrol, "changestatusfiles", recipedir, ingreddir, "change_status.txt")) if scratchstatusfile.data == controlstatusfile.data: mystatus = "unchanged" else: mystatus = "changed" rdict[recipedir][ingreddir] = "send" rdict[recipedir]["MAIN"] = mystatus return rdict
def get_new_change_status_files(): """Compare old and new status files from $MAST_CONTROL and $MAST_SCRATCH. Write an indication whether the status files have changed, not changed. Use "archived" if a recipe has been archived or is missing from SCRATCH for some other reason. """ rdict=dict() mastcontrol=dirutil.get_mast_control_path() mastscratch=dirutil.get_mast_scratch_path() olddirs=list() recipedirs=dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles")) for recipedir in recipedirs: ingreddirs = dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles",recipedir)) for ingreddir in ingreddirs: olddirs.append("%s/%s" % (recipedir, ingreddir)) srecdirs = dirutil.immediate_subdirs(mastscratch) for srecdir in srecdirs: singreddirs = dirutil.immediate_subdirs(os.path.join(mastscratch,srecdir)) for singreddir in singreddirs: csfile = os.path.join(mastscratch, srecdir, singreddir, "change_status.txt") if os.path.isfile(csfile): if not "%s/%s" % (srecdir, singreddir) in olddirs: if not srecdir in rdict.keys(): rdict[srecdir]=dict() rdict[srecdir]["MAIN"]="changed" rdict[srecdir][singreddir]="send" return rdict
def __init__(self, **kwargs): MASTObj.__init__(self, ALLOWED_KEYS, **kwargs) self.section_end = '$end' self.delimiter = ' ' # how we're breaking up each line self.section_parsers = {\ 'mast' : self.parse_mast_section, 'structure' : self.parse_structure_section, 'scaling' : self.parse_scaling_section, 'ingredients' : self.parse_ingredients_section, 'defects' : self.parse_defects_section, 'recipe' : self.parse_recipe_section, 'neb' : self.parse_neb_section, 'chemical_potentials' : self.parse_chemical_potentials_section, 'summary' : self.parse_summary_section, 'personal_recipe' : self.parse_personal_recipe_section } scratchpath = dirutil.get_mast_scratch_path() inputlocation = os.path.dirname(self.keywords['inputfile']) if (inputlocation == ""): self.logger = loggerutils.get_mast_logger("mast input parser") elif scratchpath not in inputlocation: self.logger = loggerutils.get_mast_logger("mast input parser") else: self.logger = loggerutils.get_mast_logger("mast input parser %s" % inputlocation)
def compare_old_and_new_status_files(): """Compare old and new status files from $MAST_CONTROL and $MAST_SCRATCH. Write an indication whether the status files have changed, not changed. Use "archived" if a recipe has been archived or is missing from SCRATCH for some other reason. """ rdict=dict() mastcontrol=dirutil.get_mast_control_path() mastscratch=dirutil.get_mast_scratch_path() recipedirs=dirutil.immediate_subdirs(os.path.join(mastcontrol,"statusfiles")) for recipedir in recipedirs: mystatus="unknown" rdict[recipedir]=dict() changelist=list() if not os.path.exists(os.path.join(mastscratch,recipedir)): mystatus="archived" else: scratchstatusfile = MASTFile(os.path.join(mastscratch,recipedir,"status.txt")) controlstatusfile = MASTFile(os.path.join(mastcontrol,"statusfiles",recipedir,"status.txt")) if scratchstatusfile.data == controlstatusfile.data: mystatus="unchanged" else: mystatus="changed" myidx=0 while myidx < len(scratchstatusfile.data): oldline = controlstatusfile.data[myidx] newline = scratchstatusfile.data[myidx] if "#" in oldline: pass else: ingred = oldline.split(":")[0].strip() oldstatus = oldline.split(":")[1].strip() newstatus = newline.split(":")[1].strip() if (oldstatus == "P") and (newstatus == "P"): rdict[recipedir][ingred]="AVOID" elif (oldstatus == "C") and (newstatus == "C"): rdict[recipedir][ingred]="AVOID" else: rdict[recipedir][ingred]="send" myidx = myidx + 1 rdict[recipedir]["MAIN"]=mystatus recipedirs=dirutil.immedate_subdirs(os.path.join(mastcontrol,"changestatusfiles")) for recipedir in recipedirs: ingreddirs=dirutil.immediate_subdirs(os.path.join(mastcontrol,"changestatusfiles",recipedir)) for ingreddir in ingreddirs: oldchangestatus=MASTFile(os.path.join(mastcontrol,"changestatusfiles",recipedir,ingreddir,"change_status.txt")) newchangestatus=MASTFile(os.path.join(mastscratch,recipedir,ingreddir,"change_status.txt")) if not (oldchangestatus.data == newchangestatus.data): rdict[recipedir]["MAIN"]="changed" rdict[recipedir][ingreddir]="send" rchangefile = MASTFile() for rdir in rdict.keys(): for key, value in rdict[rdir].iteritems(): rchangefile.data.append("%s:%s:%s\n" % (rdir,key,value)) rchangefile.to_file(os.path.join(mastcontrol,"recipechangefile.txt")) return True
def __init__(self): self.scratch = dirutil.get_mast_scratch_path() self._ARCHIVE = dirutil.get_mast_archive_path() self.make_directories() self.logger = loggerutils.get_mast_logger('mast_monitor') self.logger.info("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") self.logger.info("\nMAST monitor started at %s.\n" % time.asctime()) self.logger.info("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%") self.run(1)
def compare_old_and_new_status_files(): """Compare old and new status files from $MAST_CONTROL and $MAST_SCRATCH. Write an indication whether the status files have changed, not changed. Use "archived" if a recipe has been archived or is missing from SCRATCH for some other reason. """ rdict = dict() mastcontrol = dirutil.get_mast_control_path() mastscratch = dirutil.get_mast_scratch_path() recipedirs = dirutil.immediate_subdirs( os.path.join(mastcontrol, "statusfiles")) for recipedir in recipedirs: mystatus = "unknown" rdict[recipedir] = dict() changelist = list() if not os.path.exists(os.path.join(mastscratch, recipedir)): mystatus = "archived" else: scratchstatusfile = MASTFile( os.path.join(mastscratch, recipedir, "status.txt")) controlstatusfile = MASTFile( os.path.join(mastcontrol, "statusfiles", recipedir, "status.txt")) if scratchstatusfile.data == controlstatusfile.data: mystatus = "unchanged" else: mystatus = "changed" myidx = 0 while myidx < len(scratchstatusfile.data): oldline = controlstatusfile.data[myidx] newline = scratchstatusfile.data[myidx] if "#" in oldline: pass else: ingred = oldline.split(":")[0].strip() oldstatus = oldline.split(":")[1].strip() newstatus = newline.split(":")[1].strip() if (oldstatus == "P") and (newstatus == "P"): rdict[recipedir][ingred] = "AVOID" elif (oldstatus == "C") and (newstatus == "C"): rdict[recipedir][ingred] = "AVOID" else: rdict[recipedir][ingred] = "send" myidx = myidx + 1 rdict[recipedir]["MAIN"] = mystatus rchangefile = MASTFile() for rdir in rdict.keys(): for key, value in rdict[rdir].iteritems(): rchangefile.data.append("%s:%s:%s\n" % (rdir, key, value)) rchangefile.to_file(os.path.join(mastcontrol, "recipechangefile.txt")) return True
def get_new_mast_error_files(): """Get new MAST_ERROR files. """ rdict=dict() mastcontrol=dirutil.get_mast_control_path() mastscratch=dirutil.get_mast_scratch_path() olddirs=list() srecdirs = dirutil.immediate_subdirs(mastscratch) for srecdir in srecdirs: merrfile = os.path.join(mastscratch, srecdir, "MAST_ERROR") if os.path.isfile(merrfile): if not srecdir in rdict.keys(): rdict[srecdir] = dict() rdict[srecdir]["MAIN"]="changed" return rdict
def get_new_mast_error_files(): """Get new MAST_ERROR files. """ rdict = dict() mastcontrol = dirutil.get_mast_control_path() mastscratch = dirutil.get_mast_scratch_path() olddirs = list() srecdirs = dirutil.immediate_subdirs(mastscratch) for srecdir in srecdirs: merrfile = os.path.join(mastscratch, srecdir, "MAST_ERROR") if os.path.isfile(merrfile): if not srecdir in rdict.keys(): rdict[srecdir] = dict() rdict[srecdir]["MAIN"] = "changed" return rdict
def save_old_status_files(): """Save off old status files from $MAST_SCRATCH into $MAST_CONTROL """ mastcontrol=dirutil.get_mast_control_path() if not os.path.exists(os.path.join(mastcontrol,"statusfiles")): os.mkdir(os.path.join(mastcontrol,"statusfiles")) mastscratch=dirutil.get_mast_scratch_path() recipedirs=dirutil.immediate_subdirs(mastscratch) for recipedir in recipedirs: statusfile = MASTFile(os.path.join(mastscratch,recipedir,"status.txt")) trydir = os.path.join(mastcontrol,"statusfiles",recipedir) if not os.path.exists(trydir): os.mkdir(trydir) statusfile.to_file("%s/status.txt" % trydir) return True
def validate_recipe_name(rname): """Validate the recipe name. Use $MAST_CONTROL if recipe name is not found in $MAST_SCRATCH. Args: rname <str>: Recipe name candidate Returns: rnamevalid <str>: Validated recipe name """ #check existence of rname: mast_scratch = dirutil.get_mast_scratch_path() rdirs = os.listdir(mast_scratch) rnamevalid = dirutil.get_mast_control_path() for rdir in rdirs: if rdir in rname: rnamevalid = os.path.join(mast_scratch, rdir) return rnamevalid return rnamevalid
def save_old_status_files(): """Save off old status files from $MAST_SCRATCH into $MAST_CONTROL """ mastcontrol = dirutil.get_mast_control_path() if not os.path.exists(os.path.join(mastcontrol, "statusfiles")): os.mkdir(os.path.join(mastcontrol, "statusfiles")) mastscratch = dirutil.get_mast_scratch_path() recipedirs = dirutil.immediate_subdirs(mastscratch) for recipedir in recipedirs: statusfile = MASTFile( os.path.join(mastscratch, recipedir, "status.txt")) trydir = os.path.join(mastcontrol, "statusfiles", recipedir) if not os.path.exists(trydir): os.mkdir(trydir) statusfile.to_file("%s/status.txt" % trydir) return True
def main(): recipe_name, script_head_dir = parse_arguments() mast_scratch = dirutil.get_mast_scratch_path() mymon = MASTMon() my_recipe_plan = mymon.set_up_recipe_plan(os.path.join(mast_scratch, recipe_name), 1) my_dag_contents=list() for iname in my_recipe_plan.ingredients: #all JOB lines need to be at top my_dag_contents.append("JOB %s submit.sh DIR %s\n" % (iname, iname)) for iname in my_recipe_plan.ingredients: my_dag_contents.append("SCRIPT PRE %s %s/mast_do_setup.sh %s %s\n" % (iname, script_head_dir, recipe_name, iname)) my_dag_contents.append("SCRIPT POST %s %s/mast_check_is_complete.sh %s %s\n" % (iname, script_head_dir, recipe_name, iname)) my_dag_contents.append("RETRY %s 5\n" % iname) ptc = list(my_recipe_plan.parents_to_check[iname]) for pname in ptc: my_dag_contents.append("PARENT %s CHILD %s\n" % (pname, iname)) my_dag_file = MASTFile() my_dag_file.data = my_dag_contents my_dag_file.to_file(os.path.join(mast_scratch, recipe_name, "recipe.dag")) #print_to_file(recipe_name, ing_name, "MAIN: is complete: %s" % is_complete) return 0
def zip_only_changed(): """Zip only the changed recipes""" mastcontrol=dirutil.get_mast_control_path() mastscratch=dirutil.get_mast_scratch_path() rchangefile = MASTFile(os.path.join(mastcontrol,"recipechangefile.txt")) #import shutil #shutil.rmtree(os.path.join(mastcontrol,"statusfiles")) totarfile = MASTFile() for ritem in rchangefile.data: if ":send" in ritem: recipe=ritem.split(":")[0] ingred=ritem.split(":")[1] totarfile.data.append(os.path.join("MAST/SCRATCH",recipe,ingred)+"\n") totarfile.data.append(os.path.join("MAST/SCRATCH",recipe,"status.txt") + "\n") elif ":archived" in ritem: recipe=ritem.split(":")[0] totarfile.data.append(os.path.join("MAST/ARCHIVE",recipe)+"\n") totarfile.data.append("MAST/CONTROL\n") totarfile.to_file(os.getcwd() + "/tarthis") return
def save_old_change_status_files(): """Save off old status files from $MAST_SCRATCH into $MAST_CONTROL """ mastcontrol=dirutil.get_mast_control_path() if not os.path.exists(os.path.join(mastcontrol,"changestatusfiles")): os.mkdir(os.path.join(mastcontrol,"changestatusfiles")) mastscratch=dirutil.get_mast_scratch_path() recipedirs=dirutil.immediate_subdirs(mastscratch) for recipedir in recipedirs: ingreddirs=dirutil.immediate_subdirs(os.path.join(mastscratch,recipedir)) for ingreddir in ingreddirs: csfile = os.path.join(mastscratch,recipedir,ingreddir,"change_status.txt") if os.path.isfile(csfile): changestatusfile = MASTFile(csfile) trydir = os.path.join(mastcontrol,"changestatusfiles",recipedir) if not os.path.exists(trydir): os.mkdir(trydir) trydir2 = os.path.join(trydir, ingreddir) if not os.path.exists(trydir2): os.mkdir(trydir2) changestatusfile.to_file("%s/change_status.txt" % trydir2) return True
def main(): recipe_name, script_head_dir = parse_arguments() mast_scratch = dirutil.get_mast_scratch_path() mymon = MASTMon() my_recipe_plan = mymon.set_up_recipe_plan( os.path.join(mast_scratch, recipe_name), 1) my_dag_contents = list() for iname in my_recipe_plan.ingredients: #all JOB lines need to be at top my_dag_contents.append("JOB %s submit.sh DIR %s\n" % (iname, iname)) for iname in my_recipe_plan.ingredients: my_dag_contents.append("SCRIPT PRE %s %s/mast_do_setup.sh %s %s\n" % (iname, script_head_dir, recipe_name, iname)) my_dag_contents.append( "SCRIPT POST %s %s/mast_check_is_complete.sh %s %s\n" % (iname, script_head_dir, recipe_name, iname)) my_dag_contents.append("RETRY %s 5\n" % iname) ptc = list(my_recipe_plan.parents_to_check[iname]) for pname in ptc: my_dag_contents.append("PARENT %s CHILD %s\n" % (pname, iname)) my_dag_file = MASTFile() my_dag_file.data = my_dag_contents my_dag_file.to_file(os.path.join(mast_scratch, recipe_name, "recipe.dag")) #print_to_file(recipe_name, ing_name, "MAIN: is complete: %s" % is_complete) return 0