def grid(args): #Check directory, token and grid usage (taken from merge.py) # check if we are in a output dirname if not ((os.path.isfile("./ConfigureTrain.C") or os.path.isfile("./MLTrainDefinition.cfg")) and os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")): raise ValueError("This command needs to be run from an output directory") if not utils.check_alien_token(): print("No valid alien token present. Run `alien-token-init` before nittygriddy") quit() ROOT.gROOT.LoadMacro(r'GetSetting.C') if ROOT.GetSetting("runmode") != "grid": raise ValueError("The data in this folder was not run on the grid!") #Get process name from working directory WorkingDir = os.getcwd() ProcessName = WorkingDir.split("/")[-1] if args.gridmode == 'resubmit': #Resubmit #Get list of failed jobs gbbox_split_cmd = ("gbbox ps -fs | grep " + ProcessName) #Get masterjobs which do not split (especially merging jobs) gbbox_cmd = ("gbbox ps -f | grep " + ProcessName) gbbox_ps = "" try: gbbox_ps += subprocess.check_output(gbbox_split_cmd, shell=True) except subprocess.CalledProcessError as e: print ("No split jobs to resubmit found for: {}".format(ProcessName)) try: gbbox_ps += subprocess.check_output(gbbox_cmd, shell=True) except subprocess.CalledProcessError as e: print ("No masterjobs to resubmit found for: {}".format(ProcessName)) if gbbox_ps == "": print ("No jobs / masterjobs found for resubmission!") quit() #Remove double jobs gbbox_set = set(gbbox_ps.splitlines()) for line in gbbox_set: JobNr = line.split()[1] JobNr = re.sub("\D", "",JobNr) JobStat = line.split()[2] if (JobStat[0] == "E" or JobStat[0] == "Z"): #Check the first letter of the job status for error (E) or zombie (Z) resub_cmd = ("gbbox resubmit " + JobNr) subprocess.call(resub_cmd, shell=True) else: print ("Job {} is in state: {}. Not resubmitting.".format(JobNr,JobStat)) elif args.gridmode == 'nuke': print ("Work in progress. Function still needs to be implemented.")
def merge(args): # check if we are in a output dirname if not ((os.path.isfile("./ConfigureTrain.C") or os.path.isfile("./MLTrainDefinition.cfg")) and os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")): raise ValueError( "This command needs to be run from an output directory") if not utils.check_alien_token(): print "No valid alien token present. Run `alien-token-init` before nittygriddy" quit() ROOT.gROOT.LoadMacro(r'GetSetting.C') if ROOT.GetSetting("runmode") != "grid": raise ValueError("The data in this folder was not run on the grid!") # root does not like it if the stdout is piped and it uses some shell functionality # thus, using call on the whole string and shell True if args.mergemode == 'online': cmd = r'root -b -l -q -x "run.C(\"merge_online\")"' subprocess.check_call(cmd, shell=True) elif args.mergemode == 'offline': cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"' subprocess.check_call(cmd, shell=True) if args.mergemode == 'download': workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) files = utils.find_latest_merge_results(alien_workdir) print "The following files will be downloaded:" pprint(files) for alien_path in files: local_path = alien_path.split(workdir)[-1].lstrip("/") try: utils.download_file(alien_path, local_path) except ValueError: print "{} exists and is not redownloaded.".format(local_path) except OSError, e: # If there was an error in the download, just print, but keep going print e.message
def test_downloaded_something(self): utils.check_alien_token() utils.download_dataset("LHC10h_AOD160", 0.001)
def test_alien_token_valid(self): self.assertTrue(utils.check_alien_token())
def test_downloaded_something(self): utils.check_alien_token() utils.download_dataset("LHC10h_AOD160", 0.001)
def test_alien_token_valid(self): self.assertTrue(utils.check_alien_token())
def merge(args): # check if we are in a output dirname if not ((os.path.isfile("./ConfigureTrain.C") or os.path.isfile("./MLTrainDefinition.cfg")) and os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")): raise ValueError("This command needs to be run from an output directory") if not utils.check_alien_token(): print("No valid alien token present. Run `alien-token-init` before nittygriddy") quit() ROOT.gROOT.LoadMacro(r'GetSetting.C') if ROOT.GetSetting("runmode") != "grid": raise ValueError("The data in this folder was not run on the grid!") # root does not like it if the stdout is piped and it uses some shell functionality # thus, using call on the whole string and shell True if args.mergemode == 'online': cmd = r'root -b -l -q -x "run.C(\"merge_online\")"' subprocess.check_call(cmd, shell=True) elif args.mergemode == 'offline': cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"' subprocess.check_call(cmd, shell=True) if args.mergemode == 'download': workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) files = utils.find_latest_merge_results(alien_workdir) print("The following files will be downloaded:") pprint(files) for alien_path in files: local_path = alien_path.split(workdir)[-1].lstrip("/") try: utils.download_file(alien_path, local_path) except ValueError: print("{} exists and is not redownloaded.".format(local_path)) except OSError as e: # If there was an error in the download, just print, but keep going print(e.message) if args.mergemode == 'clean': workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) files = utils.find_sources_of_merged_files(utils.find_latest_merge_results(alien_workdir)) folders = [os.path.split(f)[0] for f in files] if not folders: print("No files to be cleaned up") return print("The following folders will be __DELETED__:") pprint(folders) shall_delete = utils.yn_choice("Do you really want to delete these Folders? This may take a while") if shall_delete: # its not possible to pass all arguments at once (too many) # Hence, pass 10 at a time for slice_start in range(len(folders))[::10]: cmd = ['alien_rmdir'] + folders[slice_start:(slice_start + 10)] # alien_rmdir always seems to return a non-zero exit code... Yeah! subprocess.call(cmd) # see if the deletion was successful: left_files = utils.find_sources_of_merged_files(utils.find_latest_merge_results(alien_workdir)) if left_files: print("The following files could not be deleted:") pprint(left_files) if args.mergemode == 'unmerge': workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/Stage_*.xml')]) subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/AnalysisResults.root')]) subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/root_archive.zip')]) subprocess.call(['alien_rmdir', os.path.join(alien_workdir, 'output/*/Stage_*')])
def merge(args): # check if we are in a output dirname if not ((os.path.isfile("./ConfigureTrain.C") or os.path.isfile("./MLTrainDefinition.cfg")) and os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")): raise ValueError( "This command needs to be run from an output directory") if not utils.check_alien_token(): print( "No valid alien token present. Run `alien-token-init` before nittygriddy" ) quit() ROOT.gROOT.LoadMacro(r'GetSetting.C') if ROOT.GetSetting("runmode") != "grid": raise ValueError("The data in this folder was not run on the grid!") # root does not like it if the stdout is piped and it uses some shell functionality # thus, using call on the whole string and shell True if args.mergemode == 'online': cmd = r'root -b -l -q -x "run.C(\"merge_online\")"' subprocess.check_call(cmd, shell=True) elif args.mergemode == 'offline': cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"' subprocess.check_call(cmd, shell=True) if args.mergemode == 'download': workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) files = utils.find_latest_merge_results(alien_workdir) print("The following files will be downloaded:") pprint(files) for alien_path in files: local_path = alien_path.split(workdir)[-1].lstrip("/") try: utils.download_file(alien_path, local_path) except ValueError: print("{} exists and is not redownloaded.".format(local_path)) except OSError as e: # If there was an error in the download, just print, but keep going print(e.message) if args.mergemode == 'clean': workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) files = utils.find_sources_of_merged_files( utils.find_latest_merge_results(alien_workdir)) folders = [os.path.split(f)[0] for f in files] if not folders: print("No files to be cleaned up") return print("The following folders will be __DELETED__:") pprint(folders) shall_delete = utils.yn_choice( "Do you really want to delete these Folders? This may take a while" ) if shall_delete: # its not possible to pass all arguments at once (too many) # Hence, pass 10 at a time for slice_start in range(len(folders))[::10]: cmd = ['alien_rmdir'] + folders[slice_start:(slice_start + 10)] # alien_rmdir always seems to return a non-zero exit code... Yeah! subprocess.call(cmd) # see if the deletion was successful: left_files = utils.find_sources_of_merged_files( utils.find_latest_merge_results(alien_workdir)) if left_files: print("The following files could not be deleted:") pprint(left_files) if args.mergemode == 'unmerge': workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) subprocess.call( ['alien_rm', os.path.join(alien_workdir, 'output/*/Stage_*.xml')]) subprocess.call([ 'alien_rm', os.path.join(alien_workdir, 'output/*/AnalysisResults.root') ]) subprocess.call([ 'alien_rm', os.path.join(alien_workdir, 'output/*/root_archive.zip') ]) subprocess.call( ['alien_rmdir', os.path.join(alien_workdir, 'output/*/Stage_*')])
def merge(args): # check if we are in a output dirname if not ((os.path.isfile("./ConfigureTrain.C") or os.path.isfile("./MLTrainDefinition.cfg")) and os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")): raise ValueError( "This command needs to be run from an output directory") if not utils.check_alien_token(): print( "No valid alien token present. Run `alien-token-init` before nittygriddy" ) quit() ROOT.gROOT.LoadMacro(r"GetSetting.C") if ROOT.GetSetting("runmode") != "grid": raise ValueError("The data in this folder was not run on the grid!") # root does not like it if the stdout is piped and it uses some shell functionality # thus, using call on the whole string and shell True if args.mergemode == "online": cmd = r'root -b -l -q -x "run.C(\"merge_online\")"' subprocess.check_call(cmd, shell=True) elif args.mergemode == "offline": cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"' subprocess.check_call(cmd, shell=True) if args.mergemode == "download": workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) files = [] for i in args.files.split(): files += utils.find_latest_merge_results(alien_workdir, os.path.join("/", i)) print("The following files will be downloaded:") pprint(files) for alien_path in files: local_path = alien_path.split(workdir)[-1].lstrip("/") try: utils.download_file(alien_path, local_path) except ValueError: print("{} exists and is not redownloaded.".format(local_path)) except OSError as e: # If there was an error in the download, just print, but keep going print(e) if args.mergemode == "local": def count_all_ext(path, fname): flist = [] for root, dirs, files in os.walk(path): # print("r", root) # print("d", dirs) # print("f", files) for f in files: if fname != f: continue flist.append(root + "/" + f) # pprint(flist) return flist for i in args.files.split(): files = count_all_ext("output", fname=i) print("The following files will be locally merged into", i, ":") pprint(files) cmd = "hadd " + i for j in files: cmd += " " + j.strip() print(cmd) subprocess.check_call(cmd, shell=True) if args.mergemode == "clean": workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) latest_files = utils.find_latest_merge_results(alien_workdir) files = utils.find_sources_of_merged_files(latest_files) folders = [os.path.split(f)[0] for f in files] if not folders: print("No files to be cleaned up") return print("The following folders will be __DELETED__:") pprint(folders) shall_delete = utils.yn_choice( "Do you really want to delete these Folders? This may take a while" ) if shall_delete: # its not possible to pass all arguments at once (too many) # Hence, pass 10 at a time for slice_start in range(len(folders))[::10]: cmd = ["alien_rm", "-r" ] + folders[slice_start:(slice_start + 10)] # alien_rmdir always seems to return a non-zero exit code... Yeah! subprocess.call(cmd) # see if the deletion was successful: left_files = utils.find_sources_of_merged_files( utils.find_latest_merge_results(alien_workdir)) if left_files: print("The following files could not be deleted:") pprint(left_files) if args.mergemode == "unmerge": workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1] alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir) subprocess.call( ["alien_rm", os.path.join(alien_workdir, "output/*/Stage_*.xml")]) subprocess.call([ "alien_rm", os.path.join(alien_workdir, "output/*/AnalysisResults.root") ]) subprocess.call([ "alien_rm", os.path.join(alien_workdir, "output/*/root_archive.zip") ]) subprocess.call([ "alien_rm", "-r", os.path.join(alien_workdir, "output/*/Stage_*") ])