def test_find_merge_sources(self):
workdir = "nitty_test"
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
merge_results = utils.find_latest_merge_results(alien_workdir)
self.assertEqual(utils.find_sources_of_merged_files(merge_results),
['/alice/cern.ch/user/c/cbourjau/nitty_test/001/AnalysisResults.root',
'/alice/cern.ch/user/c/cbourjau/nitty_test/002/AnalysisResults.root'])
def test_find_merge_sources(self):
workdir = "nitty_test"
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
merge_results = utils.find_latest_merge_results(alien_workdir)
self.assertEqual(utils.find_sources_of_merged_files(merge_results), [
'/alice/cern.ch/user/c/cbourjau/nitty_test/001/AnalysisResults.root',
'/alice/cern.ch/user/c/cbourjau/nitty_test/002/AnalysisResults.root'
])
def find_latest_merge_results(self):
# a test dir set up under alien:///alice/cern.ch/user/c/cbourjau/
workdir = "nitty_test"
# try with not absolute workdir:
self.assertRaises(ValueError, utils.find_latest_merge_results, workdir)
# with absolut alien path
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
self.assertEqual(utils.find_latest_merge_results(alien_workdir),
['/alice/cern.ch/user/c/cbourjau/nitty_test/AnalysisResults.root'])
def find_latest_merge_results(self):
# a test dir set up under alien:///alice/cern.ch/user/c/cbourjau/
workdir = "nitty_test"
# try with not absolute workdir:
self.assertRaises(ValueError, utils.find_latest_merge_results, workdir)
# with absolut alien path
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
self.assertEqual(
utils.find_latest_merge_results(alien_workdir),
['/alice/cern.ch/user/c/cbourjau/nitty_test/AnalysisResults.root'])
def merge(args):
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C")
or os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")):
raise ValueError(
"This command needs to be run from an output directory")
if not utils.check_alien_token():
print "No valid alien token present. Run `alien-token-init` before nittygriddy"
quit()
ROOT.gROOT.LoadMacro(r'GetSetting.C')
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
# root does not like it if the stdout is piped and it uses some shell functionality
# thus, using call on the whole string and shell True
if args.mergemode == 'online':
cmd = r'root -b -l -q -x "run.C(\"merge_online\")"'
subprocess.check_call(cmd, shell=True)
elif args.mergemode == 'offline':
cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"'
subprocess.check_call(cmd, shell=True)
if args.mergemode == 'download':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_latest_merge_results(alien_workdir)
print "The following files will be downloaded:"
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print "{} exists and is not redownloaded.".format(local_path)
except OSError, e:
# If there was an error in the download, just print, but keep going
print e.message
def merge(args):
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C") or
os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and
os.path.isfile("./run.C")):
raise ValueError("This command needs to be run from an output directory")
if not utils.check_alien_token():
print("No valid alien token present. Run `alien-token-init` before nittygriddy")
quit()
ROOT.gROOT.LoadMacro(r'GetSetting.C')
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
# root does not like it if the stdout is piped and it uses some shell functionality
# thus, using call on the whole string and shell True
if args.mergemode == 'online':
cmd = r'root -b -l -q -x "run.C(\"merge_online\")"'
subprocess.check_call(cmd, shell=True)
elif args.mergemode == 'offline':
cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"'
subprocess.check_call(cmd, shell=True)
if args.mergemode == 'download':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_latest_merge_results(alien_workdir)
print("The following files will be downloaded:")
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print("{} exists and is not redownloaded.".format(local_path))
except OSError as e:
# If there was an error in the download, just print, but keep going
print(e.message)
if args.mergemode == 'clean':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_sources_of_merged_files(utils.find_latest_merge_results(alien_workdir))
folders = [os.path.split(f)[0] for f in files]
if not folders:
print("No files to be cleaned up")
return
print("The following folders will be __DELETED__:")
pprint(folders)
shall_delete = utils.yn_choice("Do you really want to delete these Folders? This may take a while")
if shall_delete:
# its not possible to pass all arguments at once (too many)
# Hence, pass 10 at a time
for slice_start in range(len(folders))[::10]:
cmd = ['alien_rmdir'] + folders[slice_start:(slice_start + 10)]
# alien_rmdir always seems to return a non-zero exit code... Yeah!
subprocess.call(cmd)
# see if the deletion was successful:
left_files = utils.find_sources_of_merged_files(utils.find_latest_merge_results(alien_workdir))
if left_files:
print("The following files could not be deleted:")
pprint(left_files)
if args.mergemode == 'unmerge':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/Stage_*.xml')])
subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/AnalysisResults.root')])
subprocess.call(['alien_rm', os.path.join(alien_workdir, 'output/*/root_archive.zip')])
subprocess.call(['alien_rmdir', os.path.join(alien_workdir, 'output/*/Stage_*')])
def merge(args):
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C")
or os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")):
raise ValueError(
"This command needs to be run from an output directory")
if not utils.check_alien_token():
print(
"No valid alien token present. Run `alien-token-init` before nittygriddy"
)
quit()
ROOT.gROOT.LoadMacro(r'GetSetting.C')
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
# root does not like it if the stdout is piped and it uses some shell functionality
# thus, using call on the whole string and shell True
if args.mergemode == 'online':
cmd = r'root -b -l -q -x "run.C(\"merge_online\")"'
subprocess.check_call(cmd, shell=True)
elif args.mergemode == 'offline':
cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"'
subprocess.check_call(cmd, shell=True)
if args.mergemode == 'download':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_latest_merge_results(alien_workdir)
print("The following files will be downloaded:")
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print("{} exists and is not redownloaded.".format(local_path))
except OSError as e:
# If there was an error in the download, just print, but keep going
print(e.message)
if args.mergemode == 'clean':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_sources_of_merged_files(
utils.find_latest_merge_results(alien_workdir))
folders = [os.path.split(f)[0] for f in files]
if not folders:
print("No files to be cleaned up")
return
print("The following folders will be __DELETED__:")
pprint(folders)
shall_delete = utils.yn_choice(
"Do you really want to delete these Folders? This may take a while"
)
if shall_delete:
# its not possible to pass all arguments at once (too many)
# Hence, pass 10 at a time
for slice_start in range(len(folders))[::10]:
cmd = ['alien_rmdir'] + folders[slice_start:(slice_start + 10)]
# alien_rmdir always seems to return a non-zero exit code... Yeah!
subprocess.call(cmd)
# see if the deletion was successful:
left_files = utils.find_sources_of_merged_files(
utils.find_latest_merge_results(alien_workdir))
if left_files:
print("The following files could not be deleted:")
pprint(left_files)
if args.mergemode == 'unmerge':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
subprocess.call(
['alien_rm',
os.path.join(alien_workdir, 'output/*/Stage_*.xml')])
subprocess.call([
'alien_rm',
os.path.join(alien_workdir, 'output/*/AnalysisResults.root')
])
subprocess.call([
'alien_rm',
os.path.join(alien_workdir, 'output/*/root_archive.zip')
])
subprocess.call(
['alien_rmdir',
os.path.join(alien_workdir, 'output/*/Stage_*')])
def merge(args):
# check if we are in a output dirname
if not ((os.path.isfile("./ConfigureTrain.C")
or os.path.isfile("./MLTrainDefinition.cfg")) and
os.path.isfile("./GetSetting.C") and os.path.isfile("./run.C")):
raise ValueError(
"This command needs to be run from an output directory")
if not utils.check_alien_token():
print(
"No valid alien token present. Run `alien-token-init` before nittygriddy"
)
quit()
ROOT.gROOT.LoadMacro(r"GetSetting.C")
if ROOT.GetSetting("runmode") != "grid":
raise ValueError("The data in this folder was not run on the grid!")
# root does not like it if the stdout is piped and it uses some shell functionality
# thus, using call on the whole string and shell True
if args.mergemode == "online":
cmd = r'root -b -l -q -x "run.C(\"merge_online\")"'
subprocess.check_call(cmd, shell=True)
elif args.mergemode == "offline":
cmd = r'root -b -l -q -x "run.C(\"merge_offline\")"'
subprocess.check_call(cmd, shell=True)
if args.mergemode == "download":
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = []
for i in args.files.split():
files += utils.find_latest_merge_results(alien_workdir,
os.path.join("/", i))
print("The following files will be downloaded:")
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print("{} exists and is not redownloaded.".format(local_path))
except OSError as e:
# If there was an error in the download, just print, but keep going
print(e)
if args.mergemode == "local":
def count_all_ext(path, fname):
flist = []
for root, dirs, files in os.walk(path):
# print("r", root)
# print("d", dirs)
# print("f", files)
for f in files:
if fname != f:
continue
flist.append(root + "/" + f)
# pprint(flist)
return flist
for i in args.files.split():
files = count_all_ext("output", fname=i)
print("The following files will be locally merged into", i, ":")
pprint(files)
cmd = "hadd " + i
for j in files:
cmd += " " + j.strip()
print(cmd)
subprocess.check_call(cmd, shell=True)
if args.mergemode == "clean":
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
latest_files = utils.find_latest_merge_results(alien_workdir)
files = utils.find_sources_of_merged_files(latest_files)
folders = [os.path.split(f)[0] for f in files]
if not folders:
print("No files to be cleaned up")
return
print("The following folders will be __DELETED__:")
pprint(folders)
shall_delete = utils.yn_choice(
"Do you really want to delete these Folders? This may take a while"
)
if shall_delete:
# its not possible to pass all arguments at once (too many)
# Hence, pass 10 at a time
for slice_start in range(len(folders))[::10]:
cmd = ["alien_rm", "-r"
] + folders[slice_start:(slice_start + 10)]
# alien_rmdir always seems to return a non-zero exit code... Yeah!
subprocess.call(cmd)
# see if the deletion was successful:
left_files = utils.find_sources_of_merged_files(
utils.find_latest_merge_results(alien_workdir))
if left_files:
print("The following files could not be deleted:")
pprint(left_files)
if args.mergemode == "unmerge":
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
subprocess.call(
["alien_rm",
os.path.join(alien_workdir, "output/*/Stage_*.xml")])
subprocess.call([
"alien_rm",
os.path.join(alien_workdir, "output/*/AnalysisResults.root")
])
subprocess.call([
"alien_rm",
os.path.join(alien_workdir, "output/*/root_archive.zip")
])
subprocess.call([
"alien_rm", "-r",
os.path.join(alien_workdir, "output/*/Stage_*")
])
pprint(files)
for alien_path in files:
local_path = alien_path.split(workdir)[-1].lstrip("/")
try:
utils.download_file(alien_path, local_path)
except ValueError:
print "{} exists and is not redownloaded.".format(local_path)
except OSError, e:
# If there was an error in the download, just print, but keep going
print e.message
if args.mergemode == 'clean':
workdir = os.path.dirname(os.path.abspath("./run.C")).split("/")[-1]
alien_workdir = os.path.join(utils.find_user_grid_dir(), workdir)
files = utils.find_sources_of_merged_files(
utils.find_latest_merge_results(alien_workdir))
folders = [os.path.split(f)[0] for f in files]
if not folders:
print "No files to be cleaned up"
return
print "The following folders will be __DELETED__:"
pprint(folders)
shall_delete = utils.yn_choice(
"Do you really want to delete these Folders? This may take a while"
)
if shall_delete:
# its not possible to pass all arguments at once (too many)
# Hence, pass 10 at a time
for slice_start in range(len(folders))[::10]:
cmd = ['alien_rmdir'] + folders[slice_start:(slice_start + 10)]
# alien_rmdir always seems to return a non-zero exit code... Yeah!