def _untar_and_parse_lhe_file(filename, tags=None): # Untar event file new_filename, extension = os.path.splitext(filename) if extension == ".gz": if not os.path.exists(new_filename): call_command("gunzip -c {} > {}".format(filename, new_filename)) filename = new_filename for event, elem in ET.iterparse(filename): if tags and elem.tag not in tags: continue else: yield elem elem.clear()
def run_mg_pythia( mg_directory, mg_process_directory, proc_card_filename=None, run_card_file=None, param_card_file=None, reweight_card_file=None, pythia8_card_file=None, is_background=False, initial_command=None, log_file=None, ): # Copy cards if run_card_file is not None: shutil.copyfile(run_card_file, mg_process_directory + "/Cards/run_card.dat") if param_card_file is not None: shutil.copyfile(param_card_file, mg_process_directory + "/Cards/param_card.dat") if reweight_card_file is not None and not is_background: shutil.copyfile(reweight_card_file, mg_process_directory + "/Cards/reweight_card.dat") if pythia8_card_file is not None: shutil.copyfile(pythia8_card_file, mg_process_directory + "/Cards/pythia8_card.dat") # Find filenames for process card and script if proc_card_filename is None: for i in range(1000): proc_card_filename = mg_process_directory + "/Cards/start_event_generation_{}.mg5".format( i) if not os.path.isfile(proc_card_filename): break # MG commands shower_option = "OFF" if pythia8_card_file is None else "Pythia8" reweight_option = "OFF" if is_background else "ON" mg_commands = """ launch {} shower={} detector=OFF analysis=OFF madspin=OFF reweight={} done """.format(mg_process_directory, shower_option, reweight_option) with open(proc_card_filename, "w") as file: file.write(mg_commands) # Call MG5 or export into script if initial_command is None: initial_command = "" else: initial_command = initial_command + "; " _ = call_command(initial_command + mg_directory + "/bin/mg5_aMC " + proc_card_filename, log_file=log_file)
def extract_weight_order(filename, default_weight_label=None): # Untar event file new_filename, extension = os.path.splitext(filename) if extension == ".gz": if not os.path.exists(new_filename): call_command("gunzip -k {}".format(filename)) filename = new_filename with open(filename, encoding="latin-1") as file: for line in file: terms = line.replace('"', "").split() if len(terms) == 0 or terms[0] != "N": continue logger.debug("Parsing HepMC line: %s", line) n_benchmarks = int(terms[1]) if not len(terms) == n_benchmarks + 2: logger.warning( "Wrong number of weights in HepMC file. This is fine if the" " weights are parsed from the LHE file, but will lead to " "issues otherwise.") return None weight_labels = [] for term in terms[2:]: if term.startswith("id="): term = term[3:] term = term.partition("_MERGING=")[0] weight_labels.append(term) else: weight_labels.append(default_weight_label) logger.debug("Found weight labels in HepMC file: %s", weight_labels) return weight_labels # Default result (no reweighting, background scenario) logger.debug("Did not find weight labels in HepMC file") return [default_weight_label]
def run_mg_reweighting(mg_process_directory, run_name, reweight_card_file=None, initial_command=None, log_file=None): """ Runs MG reweighting. Parameters ---------- mg_process_directory : str Path to the MG process directory. run_name : str Run name. reweight_card_file : str or None, optional Path to the MadGraph reweight card. If None, the card present in the process folder is used. (Default value = None) initial_command : str or None, optional Initial shell commands that have to be executed before MG is run (e.g. to load a virtual environment). Default value: None. log_file : str or None, optional Path to a log file in which the MadGraph output is saved. Default value: None. Returns ------- bash_script_call : str How to call this script. """ # Preparations create_missing_folders([os.path.dirname(log_file)]) # Prepare run... logger.info("Starting reweighting of an existing sample in %s", mg_process_directory) # Copy cards if reweight_card_file is not None: shutil.copyfile(reweight_card_file, mg_process_directory + "/Cards/reweight_card.dat") # Call MG5 reweight feature if initial_command is None: initial_command = "" else: initial_command = initial_command + "; " _ = call_command("{}{}/bin/madevent reweight {} -f".format( initial_command, mg_process_directory, run_name), log_file=log_file)
def run_mg_reweighting(mg_process_directory, run_name, reweight_card_file=None, initial_command=None, log_file=None): """ Runs MG reweighting. Parameters ---------- mg_process_directory : str Path to the MG process directory. run_name : str Run name. reweight_card_file : str or None, optional Path to the MadGraph reweight card. If None, the card present in the process folder is used. (Default value = None) initial_command : str or None, optional Initial shell commands that have to be executed before MG is run (e.g. to load a virtual environment). Default value: None. log_file : str or None, optional Path to a log file in which the MadGraph output is saved. Default value: None. Returns ------- bash_script_call : str How to call this script. """ # Preparations if log_file is not None: Path(log_file).parent.mkdir(parents=True, exist_ok=True) # Prepare run... logger.info("Starting reweighting of an existing sample in %s", mg_process_directory) # Copy cards if reweight_card_file is not None: shutil.copyfile(reweight_card_file, f"{mg_process_directory}/Cards/reweight_card.dat") # Call MG5 reweight feature initial_command = f"{initial_command}; " if initial_command else "" _ = call_command( cmd= f"{initial_command}{mg_process_directory}/bin/madevent reweight {run_name} -f", log_file=log_file, )
def _untar_and_parse_lhe_file(filename): # Untar event file new_filename, extension = os.path.splitext(filename) if extension == ".gz": if not os.path.exists(new_filename): call_command("gunzip -c {} > {}".format(filename, new_filename)) filename = new_filename # In some cases, the LHE comments can contain bad characters with open(filename, "r") as file: lhe_content = file.read() lhe_lines = lhe_content.split("\n") for i, line in enumerate(lhe_lines): comment_pos = line.find("#") if comment_pos >= 0: lhe_lines[i] = line[:comment_pos] lhe_content = "\n".join(lhe_lines) # Parse XML tree root = ET.fromstring(lhe_content) return root, filename
def generate_mg_process(mg_directory, temp_directory, proc_card_file, mg_process_directory, initial_command=None, log_file=None): # MG commands temp_proc_card_file = temp_directory + "/generate.mg5" shutil.copyfile(proc_card_file, temp_proc_card_file) with open(temp_proc_card_file, "a") as myfile: myfile.write("\n\noutput " + mg_process_directory) # Call MG5 if initial_command is None: initial_command = "" else: initial_command = initial_command + "; " _ = call_command(initial_command + mg_directory + "/bin/mg5_aMC " + temp_proc_card_file, log_file=log_file)
def extract_weight_order(filename, default_weight_label=None): # Untar event file new_filename, extension = os.path.splitext(filename) if extension == ".gz": if not os.path.exists(new_filename): call_command("cp {} {}_bak".format(filename, filename)) call_command("gunzip {}".format(filename)) call_command("cp {}_bak {}".format(filename, filename)) filename = new_filename with open(filename, encoding="latin-1") as file: for line in file: terms = line.replace('"', "").split() if len(terms) == 0 or terms[0] != "N": continue n_benchmarks = int(terms[1]) assert len(terms) == n_benchmarks + 2 weight_labels = [] for term in terms[2:]: if term.startswith("id="): term = term[3:] term = term.partition("_MERGING=")[0] weight_labels.append(term) else: weight_labels.append(default_weight_label) logging.debug("Found weight labels in HEPMC file: %s", weight_labels) return weight_labels # Default result (no reweighting, background scenario) logging.debug("Did not find weight labels in HEPMC file") return [default_weight_label]
def run_delphes( delphes_directory, delphes_card_filename, hepmc_sample_filename, delphes_sample_filename=None, initial_command=None, log_file=None, overwrite_existing_delphes_root_file=True, delete_unzipped_file=True, ): """ Runs Delphes on a HepMC sample """ # Unzip event file filename = Path(hepmc_sample_filename).with_suffix("") extension = Path(hepmc_sample_filename).suffix to_delete = None if extension == ".gz": logger.debug("Unzipping %s", hepmc_sample_filename) if not filename.exists(): unzip_file(hepmc_sample_filename, filename) if delete_unzipped_file: to_delete = filename hepmc_sample_filename = str(filename) # Where to put Delphes sample if delphes_sample_filename is None: filename_prefix = filename.with_suffix("") for i in range(1, 1000): if i == 1: filename_candidate = f"{filename_prefix}_delphes.root" else: filename_candidate = f"{filename_prefix}_delphes_{i}.root" if not Path(filename_candidate).exists(): delphes_sample_filename = filename_candidate break elif overwrite_existing_delphes_root_file: delphes_sample_filename = filename_candidate Path(delphes_sample_filename).unlink() break assert delphes_sample_filename is not None, "Could not find filename for Delphes sample" assert Path(delphes_sample_filename).exists( ) is not True, "Could not find filename for Delphes sample" # Initial commands if initial_command is None: initial_command = "" else: initial_command = initial_command + "; " # Call Delphes _ = call_command( f"{initial_command}{delphes_directory}/DelphesHepMC " f"{delphes_card_filename} " f"{delphes_sample_filename} " f"{hepmc_sample_filename}", log_file=log_file, ) # Delete unzipped file if to_delete is not None: logger.debug("Deleting %s", to_delete) Path(to_delete).unlink() return delphes_sample_filename
def run_mg( mg_directory, mg_process_directory, proc_card_filename=None, run_card_file=None, param_card_file=None, reweight_card_file=None, pythia8_card_file=None, configuration_card_file=None, is_background=False, initial_command=None, log_file=None, explicit_python_call=False, order="LO", python_executable=None, ): """ Calls MadGraph to generate events. Parameters ---------- mg_directory : str Path to the MadGraph 5 base directory. mg_process_directory : str Path to the MG process directory. proc_card_filename : str or None, optional Filename for the MG command card that will be generated. If None, a default filename in the MG process directory will be chosen. run_card_file : str or None, optional Path to the MadGraph run card. If None, the card present in the process folder is used. Default value: None) param_card_file : str or None, optional Path to the MadGraph param card. If None, the card present in the process folder is used. Default value: None) reweight_card_file : str or None, optional Path to the MadGraph reweight card. If None, the card present in the process folder is used. (Default value = None) pythia8_card_file : str or None, optional Path to the MadGraph Pythia8 card. If None, Pythia is not run. Default value: None. configuration_card_file : str or None, optional Path to the MadGraph configuration card. If None, the card present in the process folder is used. (Default value: None). is_background : bool, optional Should be True for background processes, i.e. process in which the differential cross section does not depend on the parameters (and would be the same for all benchmarks). In this case, no reweighting is run, which can substantially speed up the event generation. Default value: False. initial_command : str or None, optional Initial shell commands that have to be executed before MG is run (e.g. to load a virtual environment). Default value: None. log_file : str or None, optional Path to a log file in which the MadGraph output is saved. Default value: None. explicit_python_call : bool, optional Calls `python2.7` instead of `python`. Returns ------- None """ # Preparations create_missing_folders([mg_process_directory, os.path.dirname(log_file)]) if proc_card_filename is not None: create_missing_folders([os.path.dirname(proc_card_filename)]) # Just run it already logger.info("Starting MadGraph and Pythia in %s", mg_process_directory) # Copy cards if run_card_file is not None: shutil.copyfile(run_card_file, mg_process_directory + "/Cards/run_card.dat") if param_card_file is not None: shutil.copyfile(param_card_file, mg_process_directory + "/Cards/param_card.dat") if reweight_card_file is not None and not is_background: shutil.copyfile(reweight_card_file, mg_process_directory + "/Cards/reweight_card.dat") if pythia8_card_file is not None and order == "LO": shutil.copyfile(pythia8_card_file, mg_process_directory + "/Cards/pythia8_card.dat") if pythia8_card_file is not None and order == "NLO": shutil.copyfile(pythia8_card_file, mg_process_directory + "/Cards/shower_card.dat") if configuration_card_file is not None: shutil.copyfile(configuration_card_file, mg_process_directory + "/Cards/me5_configuration.txt") # Find filenames for process card and script if proc_card_filename is None: for i in range(1000): proc_card_filename = mg_process_directory + "/Cards/start_event_generation_{}.mg5".format( i) if not os.path.isfile(proc_card_filename): break # MG commands shower_option = "OFF" if pythia8_card_file is None else "Pythia8" reweight_option = "OFF" if is_background else "ON" mg_commands = """ launch {} shower={} detector=OFF analysis=OFF madspin=OFF reweight={} done """.format(mg_process_directory, shower_option, reweight_option) with open(proc_card_filename, "w") as file: file.write(mg_commands) # Call MG5 if initial_command is None: initial_command = "" else: initial_command = initial_command + "; " # Explicitly call Python 2 if necessary if explicit_python_call: python_call = python_executable + " " if python_executable is not None else "python2.7 " else: python_call = "" _ = call_command(initial_command + python_call + mg_directory + "/bin/mg5_aMC " + proc_card_filename, log_file=log_file)
def generate_mg_process( mg_directory, temp_directory, proc_card_file, mg_process_directory, ufo_model_directory=None, log_file=None, initial_command=None, explicit_python_call=False, python_executable=None, ): """ Calls MadGraph to create the process folder. Parameters ---------- mg_directory : str Path to the MadGraph 5 directory. temp_directory : str Path to a directory for temporary files. proc_card_file : str Path to the process card that tells MadGraph how to generate the process. mg_process_directory : str Path to the MG process directory. ufo_model_directory : str or None, optional Path to a UFO model that is not yet installed. It will be copied to the MG directory before the process card is executed. Default value: None. initial_command : str or None, optional Initial bash commands that have to be executed before MG is run (e.g. to load the correct virtual environment). Default value: None. log_file : str or None, optional Path to a log file in which the MadGraph output is saved. Default value: None. explicit_python_call : bool, optional Calls `python2.7` instead of `python`. python_executable : None or str, optional Overwrites the default Python executable Returns ------- None """ # Preparations logger.info("Generating MadGraph process folder from %s at %s", proc_card_file, mg_process_directory) create_missing_folders( [temp_directory, mg_process_directory, os.path.dirname(log_file)]) if ufo_model_directory is not None: copy_ufo_model(ufo_model_directory, mg_directory) # MG commands temp_proc_card_file = temp_directory + "/generate.mg5" shutil.copyfile(proc_card_file, temp_proc_card_file) with open(temp_proc_card_file, "a") as myfile: myfile.write("\n\noutput " + mg_process_directory) # Call MG5 if initial_command is None: initial_command = "" else: initial_command = initial_command + "; " # Explicitly call Python 2 if necessary if explicit_python_call: python_call = python_executable + " " if python_executable is not None else "python2.7 " else: python_call = "" logger.info( "Calling MadGraph: %s", initial_command + python_call + mg_directory + "/bin/mg5_aMC " + temp_proc_card_file) _ = call_command(initial_command + python_call + mg_directory + "/bin/mg5_aMC " + temp_proc_card_file, log_file=log_file)
def run_delphes( delphes_directory, delphes_card_filename, hepmc_sample_filename, delphes_sample_filename=None, initial_command=None, log_file=None, overwrite_existing_delphes_root_file=True, delete_unzipped_file=True, ): """ Runs Delphes on a HepMC sample """ # Unzip event file filename, extension = os.path.splitext(hepmc_sample_filename) to_delete = None if extension == ".gz": logger.debug("Unzipping %s", hepmc_sample_filename) if not os.path.exists(filename): unzip_file(hepmc_sample_filename, filename) if delete_unzipped_file: to_delete = filename hepmc_sample_filename = filename # Where to put Delphes sample if delphes_sample_filename is None: filename_prefix = hepmc_sample_filename.replace(".hepmc.gz", "") filename_prefix = filename_prefix.replace(".hepmc", "") for i in range(1, 1000): if i == 1: filename_candidate = f"{filename_prefix}_delphes.root" else: filename_candidate = f"{filename_prefix}_delphes_{i}.root" if not os.path.exists(filename_candidate): delphes_sample_filename = filename_candidate break elif overwrite_existing_delphes_root_file: delphes_sample_filename = filename_candidate os.remove(delphes_sample_filename) break assert delphes_sample_filename is not None, "Could not find filename for Delphes sample" assert not os.path.exists( delphes_sample_filename ), "Could not find filename for Delphes sample" # Initial commands if initial_command is None: initial_command = "" else: initial_command = initial_command + "; " # Call Delphes _ = call_command( f"{initial_command}{delphes_directory}/DelphesHepMC " f"{delphes_card_filename} " f"{delphes_sample_filename} " f"{hepmc_sample_filename}", log_file=log_file, ) # Delete unzipped file if to_delete is not None: logger.debug("Deleting %s", to_delete) os.remove(to_delete) return delphes_sample_filename
def run_mg( mg_directory, mg_process_directory, proc_card_filename=None, run_card_file=None, param_card_file=None, reweight_card_file=None, pythia8_card_file=None, configuration_card_file=None, is_background=False, initial_command=None, log_file=None, order="LO", python_executable=None, ): """ Calls MadGraph to generate events. Parameters ---------- mg_directory : str Path to the MadGraph 5 base directory. mg_process_directory : str Path to the MG process directory. proc_card_filename : str or None, optional Filename for the MG command card that will be generated. If None, a default filename in the MG process directory will be chosen. run_card_file : str or None, optional Path to the MadGraph run card. If None, the card present in the process folder is used. Default value: None) param_card_file : str or None, optional Path to the MadGraph param card. If None, the card present in the process folder is used. Default value: None) reweight_card_file : str or None, optional Path to the MadGraph reweight card. If None, the card present in the process folder is used. (Default value = None) pythia8_card_file : str or None, optional Path to the MadGraph Pythia8 card. If None, Pythia is not run. Default value: None. configuration_card_file : str or None, optional Path to the MadGraph configuration card. If None, the card present in the process folder is used. (Default value: None). is_background : bool, optional Should be True for background processes, i.e. process in which the differential cross section does not depend on the parameters (and would be the same for all benchmarks). In this case, no reweighting is run, which can substantially speed up the event generation. Default value: False. initial_command : str or None, optional Initial shell commands that have to be executed before MG is run (e.g. to load a virtual environment). Default value: None. log_file : str or None, optional Path to a log file in which the MadGraph output is saved. Default value: None. python_executable : None or str, optional Overwrites the default Python executable Returns ------- None """ # Preparations Path(mg_process_directory).mkdir(parents=True, exist_ok=True) if log_file is not None: Path(log_file).parent.mkdir(parents=True, exist_ok=True) if proc_card_filename is not None: Path(proc_card_filename).parent.mkdir(parents=True, exist_ok=True) # Just run it already logger.info("Starting MadGraph and Pythia in %s", mg_process_directory) # Copy cards if run_card_file is not None: shutil.copyfile(run_card_file, f"{mg_process_directory}/Cards/run_card.dat") if param_card_file is not None: shutil.copyfile(param_card_file, f"{mg_process_directory}/Cards/param_card.dat") if reweight_card_file is not None and not is_background: shutil.copyfile(reweight_card_file, f"{mg_process_directory}/Cards/reweight_card.dat") if pythia8_card_file is not None and order == "LO": shutil.copyfile(pythia8_card_file, f"{mg_process_directory}/Cards/pythia8_card.dat") if pythia8_card_file is not None and order == "NLO": shutil.copyfile(pythia8_card_file, f"{mg_process_directory}/Cards/shower_card.dat") if configuration_card_file is not None: shutil.copyfile(configuration_card_file, f"{mg_process_directory}/Cards/me5_configuration.txt") # Find filenames for process card and script if proc_card_filename is None: for i in range(1000): proc_card_filename = f"{mg_process_directory}/Cards/start_event_generation_{i}.mg5" if not Path(proc_card_filename).is_file(): break # MG commands shower_option = "OFF" if pythia8_card_file is None else "Pythia8" reweight_option = "OFF" if is_background else "ON" mg_commands = """ launch {} shower={} detector=OFF analysis=OFF madspin=OFF reweight={} done """.format(mg_process_directory, shower_option, reweight_option) with open(proc_card_filename, "w") as file: file.write(mg_commands) # Call specific initial command and Python binary initial_command = f"{initial_command}; " if initial_command is not None else "" python_binary = f"{python_executable} " if python_executable is not None else "" command = f"{initial_command}{python_binary}{mg_directory}/bin/mg5_aMC {proc_card_filename}" logger.info(f"Calling MadGraph: {command}") _ = call_command(cmd=command, log_file=log_file)
def generate_mg_process( mg_directory, temp_directory, proc_card_file, mg_process_directory, ufo_model_directory=None, log_file=None, initial_command=None, python_executable=None, ): """ Calls MadGraph to create the process folder. Parameters ---------- mg_directory : str Path to the MadGraph 5 directory. temp_directory : str Path to a directory for temporary files. proc_card_file : str Path to the process card that tells MadGraph how to generate the process. mg_process_directory : str Path to the MG process directory. ufo_model_directory : str or None, optional Path to a UFO model that is not yet installed. It will be copied to the MG directory before the process card is executed. Default value: None. initial_command : str or None, optional Initial bash commands that have to be executed before MG is run (e.g. to load the correct virtual environment). Default value: None. log_file : str or None, optional Path to a log file in which the MadGraph output is saved. Default value: None. python_executable : None or str, optional Overwrites the default Python executable Returns ------- None """ # Preparations logger.info("Generating MadGraph process folder from %s at %s", proc_card_file, mg_process_directory) Path(temp_directory).mkdir(parents=True, exist_ok=True) Path(mg_process_directory).mkdir(parents=True, exist_ok=True) if log_file is not None: Path(log_file).parent.mkdir(parents=True, exist_ok=True) if ufo_model_directory is not None: copy_ufo_model(ufo_model_directory, mg_directory) # MG commands temp_proc_card_file = Path(temp_directory, "generate.mg5") shutil.copyfile(proc_card_file, temp_proc_card_file) with open(temp_proc_card_file, "a") as myfile: myfile.write(f"\n") myfile.write(f"\n") myfile.write(f"output {mg_process_directory}") # Call specific initial command and Python binary initial_command = f"{initial_command}; " if initial_command is not None else "" python_binary = f"{python_executable} " if python_executable is not None else "" command = f"{initial_command}{python_binary}{mg_directory}/bin/mg5_aMC {temp_proc_card_file}" logger.info(f"Calling MadGraph: {command}") _ = call_command(cmd=command, log_file=log_file)
def extract_observables_from_lhe_file(filename, sampling_benchmark, is_background, rescale_factor, observables, benchmark_names): """ Extracts observables and weights from a LHE file """ # Untar Event file new_filename, extension = os.path.splitext(filename) if extension == ".gz": if not os.path.exists(new_filename): call_command("gunzip -c {} > {}".format(filename, filename.strip(".gz"))) filename = new_filename # Load LHE file file = open(filename, "r") # Go to first event, also check if sum or avg is_average = False for line in file: if len(line.split()) > 2 and line.split()[1] == "=" and line.split( )[2] == "nevents": number_events_runcard = float(line.split()[0]) if len(line.split()) > 2 and line.split( )[2] == "event_norm" and line.split()[0] == "average": is_average = True if line.strip() == "</init>": break #Rescale by nevent if average if is_average: rescale_factor = rescale_factor / number_events_runcard #Sampling benchmark default for is_background=True if is_background: sampling_benchmark = "default" # Read events partons_all_events = [] weights_all_events = [] while True: end_of_file, event_partons, event_weights = _read_lhe_event( file, sampling_benchmark) if end_of_file: break weights_all_events.append(event_weights) partons_all_events.append(event_partons) # Rewrite weights weights = [] if is_background: for benchmarkname in benchmark_names: key_weights = [] for weight_event in weights_all_events: key_weights.append(weight_event["default"] * rescale_factor) weights.append(key_weights) weights = np.array(weights) else: for benchmarkname in benchmark_names: key_weights = [] for weight_event in weights_all_events: key_weights.append(weight_event[benchmarkname] * rescale_factor) weights.append(key_weights) weights = np.array(weights) # Obtain values for each observable in each event observable_values = OrderedDict() n_events = len(partons_all_events) for obs_name, obs_definition in six.iteritems(observables): values_this_observable = [] for event in range(n_events): variables = {"p": partons_all_events[event]} if isinstance(obs_definition, six.string_types): try: values_this_observable.append( eval(obs_definition, variables)) except Exception: values_this_observable.append(np.nan) else: try: values_this_observable.append( obs_definition(partons_all_events[event])) except RuntimeError: # (SyntaxError, NameError, TypeError, ZeroDivisionError, IndexError, RuntimeError): values_this_observable.append(np.nan) values_this_observable = np.array(values_this_observable, dtype=np.float) observable_values[obs_name] = values_this_observable return observable_values, weights