def set_output(self, null, null2):
out_path = self.in_dct["OUT_DIR"][0]
(path, __, directory) = out_path.rpartition("/")
#If the user provided path/directory
if not path:
#If the user provided directory, assume path is pwd
path = os.getcwd()
try:
with dR.cd(path):
pass
except:
err_str = "DQMHistComp Error: OUT_DIR path %s does not exist. Create this path or choose another before continuing"
self.problems.append(err_str % path)
self.status_good = False
else:
with dR.cd(path):
pwd = os.getcwd()
print pwd
if directory in os.listdir(pwd):
err_str = "DQMHistComp Error: OUT_DIR directory %s already exists at %s. Choose a different directory name before continuing"
self.problems.append(err_str % (directory, path))
self.status_good = False
return
else:
os.mkdir(directory)
self.out_path = path
self.out_dir = directory
return
def input_root(self, null, index):
#Get all root inputs, or root files in input directories
target_lst = self.input_files
file_dir = self.in_dct["INPUT"][index]
#.root file option
if ".root" in file_dir:
with dR.cd(self.rootfile_path):
pwd = os.getcwd()
if file_dir not in os.listdir(pwd):
self.problems.append("File %s/%s does not exist" % (self.rootfile_path, file_dir))
self.status_good = False
else:
target_lst.append(file_dir)
#Directory option: Get all root files in provided directory
else:
try:
with dR.cd("%s" % file_dir):
pass
except:
print "DQMHistComp Error: Could not access directory %s; confirm directory name and try again" % file_dir
self.status_good = False
else:
with dR.cd("%s" % file_dir):
pwd = os.getcwd()
target_lst += [i for i in os.listdir(pwd) if ".root" in i]
return
def delete_batch(self, null, null2):
# Remove batch .json file
if self.delete_done:
return
self.delete_batches += self.in_dct["DELETE_BATCH"]
with dR.cd("%s/%s" % (self.batch_path, self.batches_dir)):
for rem_batch in self.delete_batches:
try:
os.remove(rem_batch + ".json")
except OSError:
# The record was already deleted
# print "caught err", sys.exc_info()[1]
continue
# Edit the metadata on batches
with dR.cd(self.batch_path):
with open(self.metameta_fname, "r") as file_handle:
batch_lst = json.load(file_handle)
for rem_batch in self.delete_batches:
try:
batch_lst.remove(rem_batch)
except:
# The records were already deleted
# print "caught err", sys.exc_info()[1]
continue
with open(self.metameta_fname, "w+") as file_handle:
file_handle.write(json.dumps(batch_lst))
self.delete_done = True
return
def append_remove(self, mode, index):
# Stages appends/removes from a APPEND or REMOVE command
if mode == "append":
my_list = self.append_files
file_dir = self.in_dct["APPEND"][index]
elif mode == "remove":
my_list = self.remove_files
file_dir = self.in_dct["REMOVE"][index]
# .root file option
if ".root" in file_dir:
my_list.append(file_dir)
# Directory option: Get all root files in provided directory
else:
try:
with dR.cd("%s" % file_dir):
pass
except:
print "Error: Could not access directory %s; batch will not be created" % file_dir
self.status_good = False
else:
with dR.cd("%s" % file_dir):
pwd = os.getcwd()
my_list += [i for i in os.listdir(pwd) if ".root" in i]
def check_exists(self, path, filename, auto=True):
# Check if file at ~path/filename exists; if not, automatically construct it
try:
with dR.cd(path):
pwd = os.getcwd()
# Fails if directory path doesn't exists
except OSError:
self.problems.append("Directory %s does not exist" % path)
return False
else:
# Check for the file as is
if filename in os.listdir(pwd):
return True
else:
# If file wasn't present, automatically create the metafile
if auto:
print "Creating meta file at %s/%s" % (path, filename)
with dR.cd(path):
with open(filename, "w+") as file_handle:
file_handle.write(json.dumps([]))
return True
# If the file is necessary and not a metafile, set status to bad
else:
self.problems.append("File %s/%s does not exist" % (path, filename))
return False
def __write_out(self):
# Construct batch meta
with dR.cd("%s/%s" % (self.batch_path, self.batches_dir)):
if self.remake:
with open(self.batch_file, "w+") as file_handle:
new_batch = json.dumps(self.append_files)
file_handle.write(new_batch)
return
else:
# if this batch exists, edit it
try:
file_handle = open(self.batch_file, "rw")
# If the batch doesn't exist yet, create it using the remake option
except IOError:
self.remake = True
self.__write_out()
return
else:
# decoded is a loaded list of .root file names
decoded = json.load(file_handle)
file_handle.close()
# Append APPEND files
for app_file in self.append_files:
if app_file not in decoded:
decoded.append(app_file)
# Remove REMOVE files
for del_file in self.remove_files:
try:
decoded.remove(del_file)
except:
print "caught err", sys.exc_info()[1]
updated_batch = json.dumps(decoded)
with open(self.batch_file, "w+") as file_handle:
file_handle.write(updated_batch)
def range_organizer(self, index, quiet=False):
#Read and write ranges to the fileTools inst according to user setup
current_DT = dR.dctTools(self.master_tree_dct[index])
sister_db = self.master_file_dct[index].sister_db
#Read in range files to populate master_tree_dct
with dR.cd("%s/%s" % (self.rfile_path, self.rfile_directory)):
self.master_tree_dct[index] = current_DT.range_reader(sister_db, append=True)
self.safe_print("file_organizer(%i) reading range file %s" % (index, sister_db), quiet )
def __construct_ranges(self): #Construct ranges for newly submitted .root files constructed_lst = [] #Check which range files are already at rfile_path for rootfile in self.queue_construct: with dR.cd(self.rootfile_path): FT = dR.fileTools(rootfile, self.rfile_path, debug=self.debug) FT.construct_all_ranges() constructed_lst.append(rootfile) #Append to the rangefile metadata if any(constructed_lst): with dR.cd(self.rfile_path): with open(self.rfile_meta_fname, "r") as file_handle: rfmeta = json.load(file_handle) for new in constructed_lst: rfmeta.append(new) with open(self.rfile_meta_fname, "w+") as file_handle: file_handle.write(json.dumps(rfmeta))
def check_path(self, path):
try:
with dR.cd(path):
pass
except:
print "caught err", sys.exc_info()[1]
self.make_path_lst.append(path)
self.problems.append("Directory %s does not exist" % path)
return False
else:
return True
def make_paths(self):
# Construct the directory paths
sorted_mkpth = sorted(self.make_path_lst, key=lambda s: len(s))
print "Making directories at ", "; ".join(sorted_mkpth)
for pathname in sorted_mkpth:
# Construct the directory at path if it is not there
(place, __, dr) = pathname.rpartition("/")
with dR.cd(place):
pwd = os.getcwd()
if dr not in os.listdir(pwd):
os.mkdir(dr)
def __update_mm(self):
# Update the batch metameta
with dR.cd("%s" % self.batch_path):
# The first time, metameta may not exist
with open(self.metameta_fname, "r") as file_handle:
mm_decoded = json.load(file_handle)
# We must check for the presence of the current batch before we append
if self.batch not in mm_decoded:
mm_decoded.append(self.batch)
with open(self.metameta_fname, "w+") as file_handle:
file_handle.write(json.dumps(mm_decoded))
def __construct_ranges(self):
# Construct ranges for newly submitted .root files
# Check which range files are already at rfile_path
constructed_lst = []
with dR.cd("%s/%s" % (self.rfile_path, self.rfile_directory)):
pwd = os.getcwd()
for rootfile in self.append_files:
if rootfile in os.listdir(pwd):
continue
with dR.cd(self.rootfile_path):
FT = dR.fileTools(rootfile, self.rfile_path, debug=self.debug)
FT.construct_all_ranges()
constructed_lst.append(rootfile)
# Append to the rangefile metadata
if any(constructed_lst):
with dR.cd(self.rfile_path):
with open(self.rfile_meta_fname, "r") as file_handle:
rfmeta = json.load(file_handle)
for new in constructed_lst:
rfmeta.append(new)
with open(self.rfile_meta_fname, "w+") as file_handle:
file_handle.write(json.dumps(rfmeta))
def execute(self): #Executes methods consistent with user inputs #Prevent canvases from popping up R.gROOT.SetBatch(True) #Initialize ranges and filesets for i, fname in enumerate(self.file_list): self.file_organizer(i) self.range_organizer(i) #Construct master ranges self.master_range_finder() #Create histograms for each fileset with dR.cd(self.OUT_DIR): for i, fname in enumerate(self.file_list): self.hist_manager(i)
def master_range_finder(self, quiet=False):
#Find all db files in the rfiles directory and compile ranges into best_ranges
self.safe_print("master_range_finder() combining ranges", quiet)
#Only needs run once - compiles best ranges from the passed rfiles list
if self.master_ranges_found:
return
DT_list = []
#Get the rfiles from the repository
with dR.cd("%s/%s" % (self.rfile_path, self.rfile_directory)):
#Set up dctTools for ranges in provided rfiles
for db in self.rfiles:
DT_temp = dR.dctTools({})
DT_temp.range_reader(str(db), append=False)
DT_list.append(DT_temp)
master_DT = dR.dctTools({})
self.master_ranges = master_DT.get_best_ranges(*DT_list)
self.master_ranges_found = True
def dump_batch_all(self, mode, null):
# Print or write the contents of a single batch or a list of all batches
if mode == "dump_all_batches":
grabpath = self.batch_path
grabfile = self.metameta_fname
if mode == "dump_batch":
grabpath = "%s/%s" % (self.batch_path, self.batches_dir)
grabfile = self.batch_file
with dR.cd(grabpath):
with open(grabfile, "r") as file_handle:
grab_lst = json.load(file_handle)
grab_lst = json.dumps(grab_lst, sort_keys=True, indent=2)
# If the user gave 'shell', the function prints;
shell_true = self.in_dct[mode.upper()][0] == "shell"
if shell_true:
print grab_lst
return
# If the user gave a file write to the given file in the pwd
if not shell_true:
with open(self.in_dct[mode.upper()][0], "w+") as write_out:
write_out.write(grab_lst)
return
def set_rfile(self, mode, index, file_batch = None):
#Get all range file inputs from either .root file or a given batch
if mode == "rangefile":
target_lst = self.rfiles
#For using provided Rangefiles
if file_batch == None:
file_batch = self.in_dct["RANGEFILE"][index]
#Single .root file option
if ".root" in file_batch:
#Check batches repository for previous ranges
with dR.cd(self.batch_path):
with open(self.metameta_fname) as file_handle:
batch_lst = json.load(file_handle)
if file_batch in batch_lst:
target_lst.append(file_batch)
return
#Otherwise, check the rootfile path and prompt for construction
with dR.cd(self.rootfile_path):
pwd = os.getcwd()
if file_batch in os.listdir(pwd):
self.queue_construct.append(file_batch)
target_lst.append(file_batch)
return
else:
err_str = "DQMHistCompError: RANGEFILE %s/%s does not exist and could not be found in batch repository. Check that this file exists and place it in the rootfile path before continuing"
self.problems.append(err_str % (self.rootfile_path, file_batch))
self.status_good = False
return
#Batch option: Unpackage batch info from batch_meta
else:
batchfile = file_batch + ".json"
with dR.cd(self.batch_path):
with open(self.metameta_fname) as file_handle:
batch_lst = json.load(file_handle)
#Check meta for existance of batch
if file_batch not in batch_lst:
err_str = "DQMHistComp Error: Batch %s could not be found. Create this batch with BatchMake.py, or choose another batch before continuing" % file_batch
self.problems.append(err_str)
self.status_good = False
return
#Load the .root file contents of the batch to the master list
with dR.cd("%s/%s" % (self.batch_path, self.batches_dir)):
print batchfile
with open(batchfile) as file_handle:
batch_contents = json.load(file_handle)
target_lst += [i for i in batch_contents]
return
#Choose to use the input files in the ranging process
elif mode == "use_input_range":
if self.in_dct["USE_INPUT_RANGE"][0].lower() != "true":
self.use_input_range = False
return
#Just run the above methods over all the input files
else:
self.use_input_range = True
for i, file_dir in enumerate(self.in_dct["INPUT"]):
self.set_rfile("rangefile", i, file_batch=self.in_dct["INPUT"][i])
return
