def set_output(self, null, null2): out_path = self.in_dct["OUT_DIR"][0] (path, __, directory) = out_path.rpartition("/") #If the user provided path/directory if not path: #If the user provided directory, assume path is pwd path = os.getcwd() try: with dR.cd(path): pass except: err_str = "DQMHistComp Error: OUT_DIR path %s does not exist. Create this path or choose another before continuing" self.problems.append(err_str % path) self.status_good = False else: with dR.cd(path): pwd = os.getcwd() print pwd if directory in os.listdir(pwd): err_str = "DQMHistComp Error: OUT_DIR directory %s already exists at %s. Choose a different directory name before continuing" self.problems.append(err_str % (directory, path)) self.status_good = False return else: os.mkdir(directory) self.out_path = path self.out_dir = directory return
def input_root(self, null, index): #Get all root inputs, or root files in input directories target_lst = self.input_files file_dir = self.in_dct["INPUT"][index] #.root file option if ".root" in file_dir: with dR.cd(self.rootfile_path): pwd = os.getcwd() if file_dir not in os.listdir(pwd): self.problems.append("File %s/%s does not exist" % (self.rootfile_path, file_dir)) self.status_good = False else: target_lst.append(file_dir) #Directory option: Get all root files in provided directory else: try: with dR.cd("%s" % file_dir): pass except: print "DQMHistComp Error: Could not access directory %s; confirm directory name and try again" % file_dir self.status_good = False else: with dR.cd("%s" % file_dir): pwd = os.getcwd() target_lst += [i for i in os.listdir(pwd) if ".root" in i] return
def delete_batch(self, null, null2): # Remove batch .json file if self.delete_done: return self.delete_batches += self.in_dct["DELETE_BATCH"] with dR.cd("%s/%s" % (self.batch_path, self.batches_dir)): for rem_batch in self.delete_batches: try: os.remove(rem_batch + ".json") except OSError: # The record was already deleted # print "caught err", sys.exc_info()[1] continue # Edit the metadata on batches with dR.cd(self.batch_path): with open(self.metameta_fname, "r") as file_handle: batch_lst = json.load(file_handle) for rem_batch in self.delete_batches: try: batch_lst.remove(rem_batch) except: # The records were already deleted # print "caught err", sys.exc_info()[1] continue with open(self.metameta_fname, "w+") as file_handle: file_handle.write(json.dumps(batch_lst)) self.delete_done = True return
def append_remove(self, mode, index): # Stages appends/removes from a APPEND or REMOVE command if mode == "append": my_list = self.append_files file_dir = self.in_dct["APPEND"][index] elif mode == "remove": my_list = self.remove_files file_dir = self.in_dct["REMOVE"][index] # .root file option if ".root" in file_dir: my_list.append(file_dir) # Directory option: Get all root files in provided directory else: try: with dR.cd("%s" % file_dir): pass except: print "Error: Could not access directory %s; batch will not be created" % file_dir self.status_good = False else: with dR.cd("%s" % file_dir): pwd = os.getcwd() my_list += [i for i in os.listdir(pwd) if ".root" in i]
def check_exists(self, path, filename, auto=True): # Check if file at ~path/filename exists; if not, automatically construct it try: with dR.cd(path): pwd = os.getcwd() # Fails if directory path doesn't exists except OSError: self.problems.append("Directory %s does not exist" % path) return False else: # Check for the file as is if filename in os.listdir(pwd): return True else: # If file wasn't present, automatically create the metafile if auto: print "Creating meta file at %s/%s" % (path, filename) with dR.cd(path): with open(filename, "w+") as file_handle: file_handle.write(json.dumps([])) return True # If the file is necessary and not a metafile, set status to bad else: self.problems.append("File %s/%s does not exist" % (path, filename)) return False
def __write_out(self): # Construct batch meta with dR.cd("%s/%s" % (self.batch_path, self.batches_dir)): if self.remake: with open(self.batch_file, "w+") as file_handle: new_batch = json.dumps(self.append_files) file_handle.write(new_batch) return else: # if this batch exists, edit it try: file_handle = open(self.batch_file, "rw") # If the batch doesn't exist yet, create it using the remake option except IOError: self.remake = True self.__write_out() return else: # decoded is a loaded list of .root file names decoded = json.load(file_handle) file_handle.close() # Append APPEND files for app_file in self.append_files: if app_file not in decoded: decoded.append(app_file) # Remove REMOVE files for del_file in self.remove_files: try: decoded.remove(del_file) except: print "caught err", sys.exc_info()[1] updated_batch = json.dumps(decoded) with open(self.batch_file, "w+") as file_handle: file_handle.write(updated_batch)
def range_organizer(self, index, quiet=False): #Read and write ranges to the fileTools inst according to user setup current_DT = dR.dctTools(self.master_tree_dct[index]) sister_db = self.master_file_dct[index].sister_db #Read in range files to populate master_tree_dct with dR.cd("%s/%s" % (self.rfile_path, self.rfile_directory)): self.master_tree_dct[index] = current_DT.range_reader(sister_db, append=True) self.safe_print("file_organizer(%i) reading range file %s" % (index, sister_db), quiet )
def __construct_ranges(self): #Construct ranges for newly submitted .root files constructed_lst = [] #Check which range files are already at rfile_path for rootfile in self.queue_construct: with dR.cd(self.rootfile_path): FT = dR.fileTools(rootfile, self.rfile_path, debug=self.debug) FT.construct_all_ranges() constructed_lst.append(rootfile) #Append to the rangefile metadata if any(constructed_lst): with dR.cd(self.rfile_path): with open(self.rfile_meta_fname, "r") as file_handle: rfmeta = json.load(file_handle) for new in constructed_lst: rfmeta.append(new) with open(self.rfile_meta_fname, "w+") as file_handle: file_handle.write(json.dumps(rfmeta))
def check_path(self, path): try: with dR.cd(path): pass except: print "caught err", sys.exc_info()[1] self.make_path_lst.append(path) self.problems.append("Directory %s does not exist" % path) return False else: return True
def make_paths(self): # Construct the directory paths sorted_mkpth = sorted(self.make_path_lst, key=lambda s: len(s)) print "Making directories at ", "; ".join(sorted_mkpth) for pathname in sorted_mkpth: # Construct the directory at path if it is not there (place, __, dr) = pathname.rpartition("/") with dR.cd(place): pwd = os.getcwd() if dr not in os.listdir(pwd): os.mkdir(dr)
def __update_mm(self): # Update the batch metameta with dR.cd("%s" % self.batch_path): # The first time, metameta may not exist with open(self.metameta_fname, "r") as file_handle: mm_decoded = json.load(file_handle) # We must check for the presence of the current batch before we append if self.batch not in mm_decoded: mm_decoded.append(self.batch) with open(self.metameta_fname, "w+") as file_handle: file_handle.write(json.dumps(mm_decoded))
def __construct_ranges(self): # Construct ranges for newly submitted .root files # Check which range files are already at rfile_path constructed_lst = [] with dR.cd("%s/%s" % (self.rfile_path, self.rfile_directory)): pwd = os.getcwd() for rootfile in self.append_files: if rootfile in os.listdir(pwd): continue with dR.cd(self.rootfile_path): FT = dR.fileTools(rootfile, self.rfile_path, debug=self.debug) FT.construct_all_ranges() constructed_lst.append(rootfile) # Append to the rangefile metadata if any(constructed_lst): with dR.cd(self.rfile_path): with open(self.rfile_meta_fname, "r") as file_handle: rfmeta = json.load(file_handle) for new in constructed_lst: rfmeta.append(new) with open(self.rfile_meta_fname, "w+") as file_handle: file_handle.write(json.dumps(rfmeta))
def execute(self): #Executes methods consistent with user inputs #Prevent canvases from popping up R.gROOT.SetBatch(True) #Initialize ranges and filesets for i, fname in enumerate(self.file_list): self.file_organizer(i) self.range_organizer(i) #Construct master ranges self.master_range_finder() #Create histograms for each fileset with dR.cd(self.OUT_DIR): for i, fname in enumerate(self.file_list): self.hist_manager(i)
def master_range_finder(self, quiet=False): #Find all db files in the rfiles directory and compile ranges into best_ranges self.safe_print("master_range_finder() combining ranges", quiet) #Only needs run once - compiles best ranges from the passed rfiles list if self.master_ranges_found: return DT_list = [] #Get the rfiles from the repository with dR.cd("%s/%s" % (self.rfile_path, self.rfile_directory)): #Set up dctTools for ranges in provided rfiles for db in self.rfiles: DT_temp = dR.dctTools({}) DT_temp.range_reader(str(db), append=False) DT_list.append(DT_temp) master_DT = dR.dctTools({}) self.master_ranges = master_DT.get_best_ranges(*DT_list) self.master_ranges_found = True
def dump_batch_all(self, mode, null): # Print or write the contents of a single batch or a list of all batches if mode == "dump_all_batches": grabpath = self.batch_path grabfile = self.metameta_fname if mode == "dump_batch": grabpath = "%s/%s" % (self.batch_path, self.batches_dir) grabfile = self.batch_file with dR.cd(grabpath): with open(grabfile, "r") as file_handle: grab_lst = json.load(file_handle) grab_lst = json.dumps(grab_lst, sort_keys=True, indent=2) # If the user gave 'shell', the function prints; shell_true = self.in_dct[mode.upper()][0] == "shell" if shell_true: print grab_lst return # If the user gave a file write to the given file in the pwd if not shell_true: with open(self.in_dct[mode.upper()][0], "w+") as write_out: write_out.write(grab_lst) return
def set_rfile(self, mode, index, file_batch = None): #Get all range file inputs from either .root file or a given batch if mode == "rangefile": target_lst = self.rfiles #For using provided Rangefiles if file_batch == None: file_batch = self.in_dct["RANGEFILE"][index] #Single .root file option if ".root" in file_batch: #Check batches repository for previous ranges with dR.cd(self.batch_path): with open(self.metameta_fname) as file_handle: batch_lst = json.load(file_handle) if file_batch in batch_lst: target_lst.append(file_batch) return #Otherwise, check the rootfile path and prompt for construction with dR.cd(self.rootfile_path): pwd = os.getcwd() if file_batch in os.listdir(pwd): self.queue_construct.append(file_batch) target_lst.append(file_batch) return else: err_str = "DQMHistCompError: RANGEFILE %s/%s does not exist and could not be found in batch repository. Check that this file exists and place it in the rootfile path before continuing" self.problems.append(err_str % (self.rootfile_path, file_batch)) self.status_good = False return #Batch option: Unpackage batch info from batch_meta else: batchfile = file_batch + ".json" with dR.cd(self.batch_path): with open(self.metameta_fname) as file_handle: batch_lst = json.load(file_handle) #Check meta for existance of batch if file_batch not in batch_lst: err_str = "DQMHistComp Error: Batch %s could not be found. Create this batch with BatchMake.py, or choose another batch before continuing" % file_batch self.problems.append(err_str) self.status_good = False return #Load the .root file contents of the batch to the master list with dR.cd("%s/%s" % (self.batch_path, self.batches_dir)): print batchfile with open(batchfile) as file_handle: batch_contents = json.load(file_handle) target_lst += [i for i in batch_contents] return #Choose to use the input files in the ranging process elif mode == "use_input_range": if self.in_dct["USE_INPUT_RANGE"][0].lower() != "true": self.use_input_range = False return #Just run the above methods over all the input files else: self.use_input_range = True for i, file_dir in enumerate(self.in_dct["INPUT"]): self.set_rfile("rangefile", i, file_batch=self.in_dct["INPUT"][i]) return