def grid(args):
#Check directory, token and grid usage (taken from merge.py)
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C") or
os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and
os.path.isfile("./run.C")):
raise ValueError("This command needs to be run from an output directory")
if not utils.check_alien_token():
print("No valid alien token present. Run `alien-token-init` before nittygriddy")
quit()
ROOT.gROOT.LoadMacro(r'GetSetting.C')
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
#Get process name from working directory
WorkingDir = os.getcwd()
ProcessName = WorkingDir.split("/")[-1]
if args.gridmode == 'resubmit':
#Resubmit
#Get list of failed jobs
gbbox_split_cmd = ("gbbox ps -fs | grep " + ProcessName)
#Get masterjobs which do not split (especially merging jobs)
gbbox_cmd = ("gbbox ps -f | grep " + ProcessName)
gbbox_ps = ""
try:
gbbox_ps += subprocess.check_output(gbbox_split_cmd, shell=True)
except subprocess.CalledProcessError as e:
print ("No split jobs to resubmit found for: {}".format(ProcessName))
try:
gbbox_ps += subprocess.check_output(gbbox_cmd, shell=True)
except subprocess.CalledProcessError as e:
print ("No masterjobs to resubmit found for: {}".format(ProcessName))
if gbbox_ps == "":
print ("No jobs / masterjobs found for resubmission!")
quit()
#Remove double jobs
gbbox_set = set(gbbox_ps.splitlines())
for line in gbbox_set:
JobNr = line.split()[1]
JobNr = re.sub("\D", "",JobNr)
JobStat = line.split()[2]
if (JobStat[0] == "E" or JobStat[0] == "Z"):
#Check the first letter of the job status for error (E) or zombie (Z)
resub_cmd = ("gbbox resubmit " + JobNr)
subprocess.call(resub_cmd, shell=True)
else:
print ("Job {} is in state: {}. Not resubmitting.".format(JobNr,JobStat))
elif args.gridmode == 'nuke':
print ("Work in progress. Function still needs to be implemented.")
def merge(args):
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C")
or os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")):
raise ValueError(
"This command needs to be run from an output directory")
if not utils.check_alien_token():
print "No valid alien token present. Run `alien-token-init` before nittygriddy"
quit()
ROOT.gROOT.LoadMacro(r'GetSetting.C')
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
# root does not like it if the stdout is piped and it uses some shell functionality
# thus, using call on the whole string and shell True
if args.mergemode == 'online':
cmd = r'root -b -l -q -x "run.C(\"merge_online\")"'
subprocess.check_call(cmd, shell=True)
elif args.mergemode == 'offline':
cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"'
subprocess.check_call(cmd, shell=True)
if args.mergemode == 'download':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_latest_merge_results(alien_workdir)
print "The following files will be downloaded:"
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print "{} exists and is not redownloaded.".format(local_path)
except OSError, e:
# If there was an error in the download, just print, but keep going
print e.message
def test_downloaded_something(self):
utils.check_alien_token()
utils.download_dataset("LHC10h_AOD160", 0.001)
def test_alien_token_valid(self):
self.assertTrue(utils.check_alien_token())
def test_downloaded_something(self):
utils.check_alien_token()
utils.download_dataset("LHC10h_AOD160", 0.001)
def test_alien_token_valid(self):
self.assertTrue(utils.check_alien_token())
def merge(args):
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C") or
os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and
os.path.isfile("./run.C")):
raise ValueError("This command needs to be run from an output directory")
if not utils.check_alien_token():
print("No valid alien token present. Run `alien-token-init` before nittygriddy")
quit()
ROOT.gROOT.LoadMacro(r'GetSetting.C')
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
# root does not like it if the stdout is piped and it uses some shell functionality
# thus, using call on the whole string and shell True
if args.mergemode == 'online':
cmd = r'root -b -l -q -x "run.C(\"merge_online\")"'
subprocess.check_call(cmd, shell=True)
elif args.mergemode == 'offline':
cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"'
subprocess.check_call(cmd, shell=True)
if args.mergemode == 'download':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_latest_merge_results(alien_workdir)
print("The following files will be downloaded:")
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print("{} exists and is not redownloaded.".format(local_path))
except OSError as e:
# If there was an error in the download, just print, but keep going
print(e.message)
if args.mergemode == 'clean':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_sources_of_merged_files(utils.find_latest_merge_results(alien_workdir))
folders = [os.path.split(f)[0] for f in files]
if not folders:
print("No files to be cleaned up")
return
print("The following folders will be __DELETED__:")
pprint(folders)
shall_delete = utils.yn_choice("Do you really want to delete these Folders? This may take a while")
if shall_delete:
# its not possible to pass all arguments at once (too many)
# Hence, pass 10 at a time
for slice_start in range(len(folders))[::10]:
cmd = ['alien_rmdir'] + folders[slice_start:(slice_start + 10)]
# alien_rmdir always seems to return a non-zero exit code... Yeah!
subprocess.call(cmd)
# see if the deletion was successful:
left_files = utils.find_sources_of_merged_files(utils.find_latest_merge_results(alien_workdir))
if left_files:
print("The following files could not be deleted:")
pprint(left_files)
if args.mergemode == 'unmerge':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/Stage_*.xml')])
subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/AnalysisResults.root')])
subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/root_archive.zip')])
subprocess.call(['alien_rmdir', os.path.join(alien_workdir, 'output/*/Stage_*')])
def merge(args):
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C")
or os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")):
raise ValueError(
"This command needs to be run from an output directory")
if not utils.check_alien_token():
print(
"No valid alien token present. Run `alien-token-init` before nittygriddy"
)
quit()
ROOT.gROOT.LoadMacro(r'GetSetting.C')
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
# root does not like it if the stdout is piped and it uses some shell functionality
# thus, using call on the whole string and shell True
if args.mergemode == 'online':
cmd = r'root -b -l -q -x "run.C(\"merge_online\")"'
subprocess.check_call(cmd, shell=True)
elif args.mergemode == 'offline':
cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"'
subprocess.check_call(cmd, shell=True)
if args.mergemode == 'download':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_latest_merge_results(alien_workdir)
print("The following files will be downloaded:")
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print("{} exists and is not redownloaded.".format(local_path))
except OSError as e:
# If there was an error in the download, just print, but keep going
print(e.message)
if args.mergemode == 'clean':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_sources_of_merged_files(
utils.find_latest_merge_results(alien_workdir))
folders = [os.path.split(f)[0] for f in files]
if not folders:
print("No files to be cleaned up")
return
print("The following folders will be __DELETED__:")
pprint(folders)
shall_delete = utils.yn_choice(
"Do you really want to delete these Folders? This may take a while"
)
if shall_delete:
# its not possible to pass all arguments at once (too many)
# Hence, pass 10 at a time
for slice_start in range(len(folders))[::10]:
cmd = ['alien_rmdir'] + folders[slice_start:(slice_start + 10)]
# alien_rmdir always seems to return a non-zero exit code... Yeah!
subprocess.call(cmd)
# see if the deletion was successful:
left_files = utils.find_sources_of_merged_files(
utils.find_latest_merge_results(alien_workdir))
if left_files:
print("The following files could not be deleted:")
pprint(left_files)
if args.mergemode == 'unmerge':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
subprocess.call(
['alien_rm',
os.path.join(alien_workdir, 'output/*/Stage_*.xml')])
subprocess.call([
'alien_rm',
os.path.join(alien_workdir, 'output/*/AnalysisResults.root')
])
subprocess.call([
'alien_rm',
os.path.join(alien_workdir, 'output/*/root_archive.zip')
])
subprocess.call(
['alien_rmdir',
os.path.join(alien_workdir, 'output/*/Stage_*')])
def merge(args):
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C")
or os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")):
raise ValueError(
"This command needs to be run from an output directory")
if not utils.check_alien_token():
print(
"No valid alien token present. Run `alien-token-init` before nittygriddy"
)
quit()
ROOT.gROOT.LoadMacro(r"GetSetting.C")
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
# root does not like it if the stdout is piped and it uses some shell functionality
# thus, using call on the whole string and shell True
if args.mergemode == "online":
cmd = r'root -b -l -q -x "run.C(\"merge_online\")"'
subprocess.check_call(cmd, shell=True)
elif args.mergemode == "offline":
cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"'
subprocess.check_call(cmd, shell=True)
if args.mergemode == "download":
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = []
for i in args.files.split():
files += utils.find_latest_merge_results(alien_workdir,
os.path.join("/", i))
print("The following files will be downloaded:")
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print("{} exists and is not redownloaded.".format(local_path))
except OSError as e:
# If there was an error in the download, just print, but keep going
print(e)
if args.mergemode == "local":
def count_all_ext(path, fname):
flist = []
for root, dirs, files in os.walk(path):
# print("r", root)
# print("d", dirs)
# print("f", files)
for f in files:
if fname != f:
continue
flist.append(root + "/" + f)
# pprint(flist)
return flist
for i in args.files.split():
files = count_all_ext("output", fname=i)
print("The following files will be locally merged into", i, ":")
pprint(files)
cmd = "hadd " + i
for j in files:
cmd += " " + j.strip()
print(cmd)
subprocess.check_call(cmd, shell=True)
if args.mergemode == "clean":
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
latest_files = utils.find_latest_merge_results(alien_workdir)
files = utils.find_sources_of_merged_files(latest_files)
folders = [os.path.split(f)[0] for f in files]
if not folders:
print("No files to be cleaned up")
return
print("The following folders will be __DELETED__:")
pprint(folders)
shall_delete = utils.yn_choice(
"Do you really want to delete these Folders? This may take a while"
)
if shall_delete:
# its not possible to pass all arguments at once (too many)
# Hence, pass 10 at a time
for slice_start in range(len(folders))[::10]:
cmd = ["alien_rm", "-r"
] + folders[slice_start:(slice_start + 10)]
# alien_rmdir always seems to return a non-zero exit code... Yeah!
subprocess.call(cmd)
# see if the deletion was successful:
left_files = utils.find_sources_of_merged_files(
utils.find_latest_merge_results(alien_workdir))
if left_files:
print("The following files could not be deleted:")
pprint(left_files)
if args.mergemode == "unmerge":
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
subprocess.call(
["alien_rm",
os.path.join(alien_workdir, "output/*/Stage_*.xml")])
subprocess.call([
"alien_rm",
os.path.join(alien_workdir, "output/*/AnalysisResults.root")
])
subprocess.call([
"alien_rm",
os.path.join(alien_workdir, "output/*/root_archive.zip")
])
subprocess.call([
"alien_rm", "-r",
os.path.join(alien_workdir, "output/*/Stage_*")
])