def read_config(config_file): """Read the given config file and return the parsed settings and channels""" config = configparser.ConfigParser() config.optionxform = str # preserve case of the options config.read(config_file.name) settings = list(config.items('settings')) settings = Settings(settings) if not config.has_section('channels'): print_color('[WARNING] No channels specified in %s' % config_file.name, 'YELLOW') return settings, [] lines = [line.rstrip('\n') for line in config_file.readlines() if not line.startswith('#') and line.split()] # last part excludes empty lines channels = lines[lines.index('[channels]')+1:] if not channels: print_color('[WARNING] No channels specified in %s' % config_file.name, 'YELLOW') return settings, [] delimiter = ' ' pattern = re.compile(r'''((?:[^%s"']|"[^"]*"|'[^']*')+)''' % delimiter) # split using delimiter outside of quotations [http://stackoverflow.com/a/2787064] chnl = [] for channel in channels: option = pattern.split(channel.lstrip(';'))[1::2] if len(option) != 3: print_error('[ERROR] Wrong number of arguments for channel %s' % option[0]) print(' This channel will be skipped') continue if option[1] is '0' or option[2] is '0': #print('Skip channel', format_channel(option[0], False)) print('Skip channel', option[0]) continue if not option[1].isdigit() or not option[2].isdigit(): print_error('[ERROR] The amount of files and events to simulate have to be integers!') print(' Check your settings for channel %s' % option[0]) continue chnl.append((option[0], int(option[1]), int(option[2]))) return settings, chnl
def __init__(self, settings=None): self.__settings = { 'OUTPUT_PATH': '.', 'MCGEN_DATA': 'mcgen', 'GEANT_DATA': 'geant', 'MCGEN_PREFIX': 'mcgen', 'GEANT_PREFIX': 'g4sim', 'LOG_DATA': 'log', 'A2_GEANT_PATH': '', 'MCGEN_ONLY': False, 'GEANT_ONLY': False, 'QSUB_BIN': 'qsub', 'QSUB_MAIL': 'a', 'QSUB_EXTRA': '', 'QUEUE': 'dflt', 'WALLTIME': '12:00:00', 'PRIORITY': 0, 'GENERATOR': 'Ant-pluto', 'GENERATOR_PATH': '', 'Emin': 1420, 'Emax': 1580, 'COCKTAIL_SETUP': '', 'COCKTAIL_BINNING': 0, 'GUN_THETA': '0 180', 'GUN_OPENING': '', 'GeantFlags': '', 'AddFlags': '', } if settings and len(settings[0]) == 2: for setting in settings: if setting[0] not in self.__settings: print_error('[ERROR] Setting "%s" not valid! Will be skipped.' % setting[0]) continue if not setting[1] and self.__settings[setting[0]]: print_color('[INFO] No value given for "{0}". Default value "{1}" will ' 'be used'.format(setting[0], self.__settings[setting[0]]), 'BLUE') continue if setting[0] in ('PRIORITY', 'COCKTAIL_BINNING'): if not setting[1].isdigit(): print_error('[ERROR] The given value for "%s" is not an integer, use ' 'default value %d instead.' % (setting[1], self.__settings[setting[1]])) continue self.__settings[setting[0]] = int(setting[1]) continue if setting[0] in ('Emin', 'Emax'): if not isfloat(setting[1]): print_error('[ERROR] The given energy "%s" is not a float! Use default ' 'value %.1f instead.' % (setting[1], self.__settings[setting[0]])) continue self.__settings[setting[0]] = float(setting[1]) continue self.__settings[setting[0]] = setting[1] elif settings and len(settings[0]) != 2: print_error('[ERROR] Given settings in the wrong format, ' 'list with tuples (setting, value) expected.')
def upload_all_videos(directory, video_dict): uploaded_video_ids = [] failed_videos = [] for video_file, action_upload in video_dict.items(): if action_upload: response, _ = upload_video(os.path.join(directory, video_file)) if response is not None: uploaded_video_ids.append(response["id"]) else: failed_videos.append(video_file) if failed_videos != []: color.print_color("Those uploads have failed:", color="YELLOW") color.print_color(', '.join(failed_videos), color="RED") else: print("Every video was uploaded successfully.") return uploaded_video_ids, failed_videos
def test_process(cmd, time=None): """Try to run a process and check its return code if something went wrong, print command output if time is given, kill process after time expired""" # use shell=True, otherwise the command passed to Popen to execute, # including cmd.split(' ', 1), produces errors (probably due to reaction string) proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Python 3.3 needed for timeout, most recent version on blaster is 3.1 # if time: # if time < 0: # print_error('given time is negative!') # return False # try: # outs, errs = proc.communicate(timeout=time) # if proc.returncode: # print_error('non-zero returncode for command: %s' % cmd) # print_color('error output:', 'YELLOW') # print(errs.decode('utf-8')) # print_color('standard output:', 'YELLOW') # print(outs.decode('utf-8')) # return False # else: # return True # except TimeoutExpired: # proc.kill() # outs, errs = proc.communicate() # print_error('[ERROR] given timeout expired, process killed') # print_color('error output:', 'YELLOW') # print(errs.decode('utf-8')) # print_color('standard output:', 'YELLOW') # print(outs.decode('utf-8')) # return False outs, errs = proc.communicate() if proc.returncode: print_error('[ERROR] non-zero returncode for command: %s' % cmd) print_color('error output:', 'YELLOW') print(errs.decode('utf-8')) print_color('standard output:', 'YELLOW') print(outs.decode('utf-8')) return False return True
def sanity_check_cocktail(settings): """Check if the given settings for Ant-cocktail seem okay""" setup = settings.get('COCKTAIL_SETUP') binning = int(settings.get('COCKTAIL_BINNING')) if setup and not setup.startswith('Setup_'): print_color("[WARNING] The specified detector setup doesn't start with 'Setup_'", 'YELLOW') if not setup and not binning: print_error('[ERROR] No Setup and no binning for the beam energy specified!') print_error(' Please make sure to provide one option') return False if setup and binning: print_error('[ERROR] You provided both a setup and energy binning for the Cocktail') print_error(' Please make sure to provide only one of these options') return False return True
def __init__(self, settings=None): self.__settings = { 'OUTPUT_PATH': '.', 'PLUTO_DATA': 'pluto', 'GEANT_DATA': 'geant', 'LOG_DATA': 'log', 'A2_GEANT_PATH': '/home/neiser/opt/a2geant', 'QSUB_BIN': 'qsub', 'QSUB_MAIL': 'a', 'QUEUE': 'dflt', 'WALLTIME': '12:00:00', 'PRIORITY': 0, 'Emin': 1420, 'Emax': 1580 } if settings and len(settings[0]) == 2: for setting in settings: if setting[0] not in self.__settings: print_error('[ERROR] Setting "%s" not valid! Will be skipped.' % setting[0]) continue if not setting[1]: print_color('[INFO] No value given for "{0}". Default value "{1}" will ' 'be used'.format(setting[0], self.__settings[setting[0]]), 'BLUE') continue if setting[0] == 'PRIORITY': if not setting[1].isdigit(): print_error('[ERROR] The given priority is not an integer, use ' 'default value %d instead.' % self.__settings[setting[1]]) continue self.__settings[setting[0]] = int(setting[1]) continue if setting[0] in ('Emin', 'Emax'): if not isfloat(setting[1]): print_error('[ERROR] The given energy "%s" is not a float! Use default ' 'value %.1f instead.' % (setting[1], self.__settings[setting[0]])) continue self.__settings[setting[0]] = float(setting[1]) continue self.__settings[setting[0]] = setting[1] elif settings and len(settings[0]) != 2: print_error('[ERROR] Given settings in the wrong format, ' 'list with tuples (setting, value) expected.')
def simulation_dialogue(): """Show a simulation dialogue in which the user can enter the channels to be simulated if they're not specified in the config file""" channels = [] positive_responses = ['y', 'Y', 'j', 'J', 'yes', 'Yes'] negative_responses = ['n', 'N', 'no', 'No'] allowed_responses = positive_responses + negative_responses opt = input("\nDo you want to enter channels which should be simulated? [y/n]: ") while opt not in allowed_responses: opt = input("You've entered an invalid response! Please try again: ") if opt in negative_responses: return channels while opt in positive_responses: if not channels: print('Please enter a channel which should be simulated:') print('Note: The syntax has to be exactly the Pluto syntax for the reaction,\n' ' e.g. "pi0 [g g]" for the decay of a pi0 into two photons.\n' ' The recoil proton is taken into account, type only the desired reaction') else: opt = input("Do you want to enter another channel? [y/n]: ") while opt not in allowed_responses: opt = input("You've entered an invalid response! Please try again: ") if opt in negative_responses: continue chan = 'p ' chan += input(chan) n_files = input_digit("How much files should be generated for this channel?", max_retries=3) if not n_files: print("This channel will be skipped") continue n_events = input_digit("How mich events should be generated per file?") if not n_events: print("This channel will be skipped") continue channels.append([chan, n_files, n_events]) print_color("You've entered %d channels for the simulation process" % len(channels), 'BLUE') return channels
def check_simulation_files(settings, channel): """Do some sanity checks if the existing simulation files seem to be okay and return the maximum file number""" pluto_files = [f for f in os.listdir(settings.get('PLUTO_DATA')) if f.startswith(PLUTO_PREFIX) and channel in f] geant_files = [f for f in os.listdir(settings.get('GEANT_DATA')) if f.startswith(GEANT_PREFIX) and channel in f] max_pluto = max_file_number(pluto_files) max_geant = max_file_number(geant_files) if max_geant > max_pluto: print_color(" [Warning]", 'YELLOW') print("There are more Geant4 simulation files than Pluto generated\nfiles for channel %s" % format_channel(channel, False)) input("Will continue by pressing any key ") elif max_geant < max_pluto: print_color(" [Warning]", 'YELLOW') print("There are more Pluto generated files than Geant4 simulated\nfiles for channel %s" % format_channel(channel, False)) input("Will continue by pressing any key ") return max(max_pluto, max_geant)
def get_decay_string(channel, level=1): """Get a decay string for a certain channel, print warning if proton is missing""" channel = channel.strip('"') # Ant-cocktail if channel.lower().startswith('cocktail'): return 'cocktail' # Ant-mcgun if channel.lower().startswith('gun:'): channel = channel.split(':')[-1].strip() channel = parse_pluto_string.get_initial_state(channel) channel = parse_pluto_string.particle_list_to_string(channel) return channel + '-gun' # default: assume Ant-pluto reaction channel syntax if channel.startswith('p '): channel = channel[2:] else: print_color('[WARNING] recoil proton missing in decay string: %s' % channel, 'WARNING') channel = parse_pluto_string.get_decay_string(channel, level) return channel
def insert_videos_playlist(uploaded_video_ids, playlist_id): playlist_insert_failed = [] print("\nuploaded_video_ids", uploaded_video_ids, end='\n\n') for video_id in uploaded_video_ids: try: response = add_to_playlist(playlist_id, video_id) if not 'id' in response: playlist_insert_failed.append(video_id) except Exception: playlist_insert_failed.append(video_id) print("http error when trying to insert video to playlist") print( "This video may be a duplicate or the playlist_id may be wrong" ) if playlist_insert_failed != []: color.print_color("Those playlist insertion have failed:", "YELLOW") color.print_color(", ".join(playlist_insert_failed), "RED") else: color.print_color( "Every video was successfully inserted into the playlist.", color="CYAN")
def check_binaries(settings, ant_pluto='', verbose=False): """Check if the needed binaries exist, return the absolute paths to them""" pluto, tid, geant = None, None, None # first of all check if the specified qsub binary exists if not find_executable(settings.get('QSUB_BIN')): print_error('[ERROR] The binary %s could not be found!' % settings.get('QSUB_BIN')) sys.exit(1) if ant_pluto: if verbose: print('Searching for Ant-pluto and Ant-addTID in %s' % ant_pluto) pluto = check_bin(ant_pluto, 'Ant-pluto') if pluto and verbose: print('Found Ant-pluto') tid = check_bin(ant_pluto, 'Ant-addTID') if tid and verbose: print('Found Ant-addTID') if not pluto: print_error('[ERROR] Ant-pluto not found in %s!' % ant_pluto) sys.exit(1) if not tid: print_error('[ERROR] Ant-addTID not found in %s!' % ant_pluto) sys.exit(1) else: pluto = find_executable('Ant-pluto') if not pluto: print_error('[ERROR] Ant-pluto not found!') if verbose: print("Ant-pluto couldn't be found within your $PATH variable") sys.exit(1) tid = find_executable('Ant-addTID') if not tid: print_error('[ERROR] Ant-addTID not found!') if verbose: print("Ant-addTID couldn't be found within your $PATH variable") sys.exit(1) else: pluto = abspath(pluto) tid = abspath(tid) if verbose: print('Ant-pluto found:', pluto) print('Ant-addTID found:', tid) geant_path = settings.get('A2_GEANT_PATH') if verbose: print('Searching for the A2 binary in %s' % geant_path) if not check_bin(geant_path, 'A2'): print_error('[ERROR] A2 Geant executable not found!') sys.exit(1) elif verbose: print('A2 executable found in %s' % geant_path) geant = check_bin(geant_path, 'runGeant.sh') if not geant: print_error('[ERROR] The runGeant.sh script could not be found or used!') sys.exit(1) # check if Geant version is used which can read in Pluto files (without pluto2mkin converter) if os.path.exists(get_path(geant_path, 'pluto2mkin')): print_error('[ERROR] pluto2mkin converter found in %s' % geant_path) print(" It's highly recommended to use the PlutoGen branch of the a2geant repository.") sys.exit(1) # check target length in A2 Geant4 DetectorSetup.mac geant_macros = get_path(geant_path, 'macros') if not check_file(geant_macros, 'DetectorSetup.mac'): print(" No 'DetectorSetup.mac' macro found in the Geant macros directory.") target_length = '' with open(get_path(geant_macros, 'DetectorSetup.mac'), 'r') as mac: for line in mac: if '/A2/det/setTargetLength' in line: target_length = line.split()[1] if float(target_length) < 10.: print_color("[WARNING] The target length specified in the 'DetectorSetup.mac' macro", 'YELLOW') print_color(' in the Geant macros directory is smaller than the usual lH2 target', 'YELLOW') print_color(' size: %s cm. If you consider to use a smeared z vertex, make sure' % target_length, 'YELLOW') print_color(' the specified target length is correctly set.', 'YELLOW') print() return pluto, tid, geant
def export(self, file_name, force=False): """Export the current settings to the given file""" file_name = os.path.expanduser(file_name) path = os.path.dirname(file_name) if not path: path = os.getcwd() file_name = os.path.join(path, file_name) if not os.path.exists(path) and not force: print_error('[ERROR] The path "%s" for exporting the settings does not exist!' % path) return False elif not os.path.exists(path): print_color('[INFO] The path "%s" does not exist, it will be created now.' % path, 'BLUE') try: os.makedirs(path) except OSError as exception: if exception.errno == errno.EACCES: print_error('[ERROR] You do not have the permission to create directories in "%s"' % os.path.dirname(path)) return False elif exception.errno != errno.EEXIST: raise if not os.access(path, os.W_OK): print_error('[ERROR] The given path "%s" os not writable!' % path) return False if os.path.isfile(file_name): if not force: print_error('[ERROR] The specified file "%s" already exists, aborting export.' % file_name) return False else: print_color('[INFO] The file "%s" already exists, it will be overwritten.' % file_name, 'BLUE') with open(file_name, 'w') as file: file.write('%s\n' % '[settings]') file.write('%s\n' % '# output path, current directory will be used if missing') file.write('%s = %s\n' % ('OUTPUT_PATH', self.__settings['OUTPUT_PATH'])) file.write('%s\n' % '# directory relative to output path above to store Pluto files') file.write('%s = %s\n' % ('PLUTO_DATA', self.__settings['PLUTO_DATA'])) file.write('%s\n' % '# relative path to output path above ' 'where the Geant4 files should be stored') file.write('%s = %s\n' % ('GEANT_DATA', self.__settings['GEANT_DATA'])) file.write('%s\n' % '# log directory relative to output path above, ' 'log will be used if empty') file.write('%s = %s\n' % ('LOG_DATA', self.__settings['LOG_DATA'])) file.write('%s = %s\n\n' % ('A2_GEANT_PATH', self.__settings['A2_GEANT_PATH'])) file.write('%s\n' % '# some default settings for jobs') file.write('%s = %s\n' % ('QSUB_BIN', self.__settings['QSUB_BIN'])) file.write('%s\n' % '# mail when job aborts (a), begins (b), ends (e), no mails (n)') file.write('%s = %s\n' % ('QSUB_MAIL', self.__settings['QSUB_MAIL'])) file.write('%s = %s\n' % ('QUEUE', self.__settings['QUEUE'])) file.write('%s = %s\n' % ('WALLTIME', self.__settings['WALLTIME'])) file.write('%s = %s\n\n' % ('PRIORITY', self.__settings['PRIORITY'])) file.write('%s\n' % '# simulation specific settings') file.write('%s\n' % '# minimum energy of the photon beam') file.write('%s: %s\n' % ('Emin', self.__settings['Emin'])) file.write('%s\n' % '# maximum energy of the photon beam') file.write('%s: %s\n\n' % ('Emax', self.__settings['Emax'])) file.write('%s\n' % '[channels]') file.write('%s\n' % '# channels which should be simulated, line has to start with ";", ' 'given in the syntax used in Pluto (do not forget the recoil proton!), ' 'the amount of files and the number of events per file') file.write('%s\n' % ';"p pi0 [g g]" 10 100000') return True
def print_commands(commands): for cmd in commands: print_color(" | ".join(cmd.get_aliases()), bcolors.OKGREEN, end='') print_color(" " + cmd.get_params_help(), bcolors.OKBLUE) print("\t" + cmd.get_description())
#!/usr/bin/python import color color.print_color("apple") #invalid #print_color("apple"); from color import print_color print_color("apple") #invalid #print_name("orange"); from color import * print_color("apple") print_name("orange")
def user_want_to_continue(directory, video_dict): color.print_color(WARN_MSG.format(directory), color="RED") color.print_color(', '.join(video_dict.keys()), color="YELLOW") return input(color.format_color(WARN_INPUT, color="CYAN")) in 'yY'
def main(): """Main function: process all information, prepare simulation process, submit jobs""" parser = argparse.ArgumentParser(description='Submit simulations on blaster, configuration ' 'is done via a config file "sim_settings" in ~ or . ' 'and/or a list of command line arguments') parser.add_argument('-c', '--config', nargs=1, metavar='config_file', dest='config',# required=True, type=lambda x: is_valid_file(parser, x), help='Optional: Specify a custom config file') parser.add_argument('-o', '--output', nargs=1, metavar='output_directory', type=lambda x: is_valid_dir(parser, x), help='Optional: Custom output directory') parser.add_argument('-a', '--ant-pluto', nargs=1, type=str, metavar='Ant-pluto binary path', help='Optional: If the Ant-pluto binary can\'t be found in your' '$PATH variable, use this option to specify the path manually') parser.add_argument('-l', '--list', nargs='?', const=True, help='List the amount of existing files per channel' 'in the output directory and exit; if "all" is specified' 'as an optional argument the amount of events will be listed') parser.add_argument('-e', '--example-config', nargs='?', const='example_settings', help='Export an example of default settings and exit. ' 'If no file is specified, the output will be written to "example_settings"') parser.add_argument('-w', '--walltime', type=int, nargs=1, metavar='walltime', help='Walltime for jobs, time in hours') parser.add_argument('-q', '--queue', type=str, nargs=1, metavar='queue', help='Queue which should be used for the jobs') parser.add_argument('-f', '--force', action='store_true', help='Force creation of directories if they do not exist' 'or overwrite existing files') parser.add_argument('-v', '--verbose', action='store_true', help='Print additional output') args = parser.parse_args() verbose = args.verbose force = args.force settings = None if args.example_config: print_color('[INFO] Write example settings to file "%s"' % args.example_config, 'BLUE') settings = Settings() if not settings.export(args.example_config, force): print_error('[ERROR] Creating example settings "%s" failed.' % args.example_config) sys.exit(0) config = None channels = [] if args.config: config = args.config[0] if verbose: print('Use config file %s' % config) else: if verbose: print('Try to find a default config file') config = find_config(verbose=verbose) if not config: print_color('No config file found, use default values.', 'YELLOW') settings = Settings() if config: print_color('The config file "%s" will be used.' % config, 'BLUE') if verbose: print('Read config file %s' % config) with open(config, 'r') as conf: settings, channels = read_config(conf) if args.list: if verbose and args.list == 'all': print('Trying to determine the total amount of simulated events') if args.output: settings.set('OUTPUT_PATH', get_path(args.output[0])) list_file_amount(settings, args.list == 'all') sys.exit(0) if verbose: print('The following settings will be used:') settings.print() if channels: print('The following channels have been found:') print(channels) if args.output: if not check_path(args.output[0], force): sys.exit('The output directory %s cannot be used' % args.output[0]) output = get_path(args.output[0]) print_color('Setting custom output directory: %s' % output, 'GREEN') settings.set('OUTPUT_PATH', output) ant_pluto = '' if args.ant_pluto: ant_pluto = args.ant_pluto[0] print_color('Use custom Ant-pluto path %s' % ant_pluto, 'GREEN') if not check_directories(settings, ant_pluto, force, verbose): sys.exit(1) pluto, tid, geant = check_binaries(settings, ant_pluto, verbose) if not pluto or not tid or not geant: sys.exit(1) if not channels: print_color('[Warning] No channels specified in the config file', 'YELLOW') channels = simulation_dialogue() if not channels: sys.exit('No channels entered for simulation, exiting.') if args.walltime: print_color('Setting custom walltime to %d hours' % args.walltime[0], 'GREEN') settings.set('WALLTIME', args.walltime[0]) if args.queue: print_color('Setting custom queue to %s' % args.queue[0], 'GREEN') settings.set('QUEUE', args.queue[0]) if verbose: print('This are the updated settings:') settings.print() if verbose: print('Determining the existing amount of files...') simulation = [] for channel in channels: decay_string = get_decay_string(channel[0]) max_number = check_simulation_files(settings, decay_string) simulation.append([decay_string, channel[0], channel[1], channel[2], max_number]) print_color(str(len(simulation)) + ' channels configured. ' 'The following simulation will take place:', 'BLUE') total_files, total_events = 0, 0 for channel, _, files, events, _ in simulation: total = files*events print("{0:<20s} {1:>4d} files per {2:>4s} events (total {3:>4s} events)" .format(format_channel(channel), files, unit_prefix(events), unit_prefix(total))) total_files += files total_events += total print(" Total %s events in %d files" % (unit_prefix(total_events), total_files)) print(" Files will be stored in " + settings.get('OUTPUT_PATH')) # start the job submission print('Start submitting jobs, total', total_files) submit_jobs(settings, simulation, pluto, tid, geant, total_files) print_color('Done!', 'GREEN')
def check_binaries(settings, generator_path='', verbose=False): """Check if the needed binaries exist, return the absolute paths to them""" generator, geant = None, None # first of all check if the specified qsub binary exists if not find_executable(settings.get('QSUB_BIN')): print_error('[ERROR] The binary %s could not be found!' % settings.get('QSUB_BIN')) sys.exit(1) generator = settings.get('GENERATOR') generator_path = settings.get('GENERATOR_PATH') if generator_path: if verbose: print('Searching for MC generator in %s' % generator_path) generator = check_bin(generator_path, generator) if generator and verbose: print('Found %s' % settings.get('GENERATOR')) if not generator: print_error('[ERROR] %s not found in %s!' % (settings.get('GENERATOR'), generator_path)) sys.exit(1) else: generator = find_executable(generator) if not generator: print_error('[ERROR] %s not found!' % settings.get('GENERATOR')) if verbose: print("%s couldn't be found within your $PATH variable" % settings.get('GENERATOR')) sys.exit(1) else: generator = abspath(generator) if verbose: print('%s found: %s', (settings.get('GENERATOR'), generator)) geant_path = settings.get('A2_GEANT_PATH') if geant_path: if verbose: print('Searching for the A2 binary in %s' % geant_path) if not check_bin(geant_path, 'A2'): print_error('[ERROR] A2 Geant executable not found!') sys.exit(1) elif verbose: print('A2 executable found in %s' % geant_path) if check_file(geant_path, 'runA2Geant'): geant = check_bin(geant_path, 'runA2Geant') else: geant = check_bin(geant_path, 'runGeant.sh') if not geant: print_error('[ERROR] The runA2Geant or runGeant.sh script could not be found or used!') sys.exit(1) else: geant = find_executable('A2') # fallback solution if not in $PATH if not geant: print_color("[WARNING] The A2 Geant binary couldn't be found within your $PATH variable", 'YELLOW') print(' try to use fallback solution') fallback = '/home/neiser/opt/a2geant/build/bin' if check_file(fallback, 'A2'): geant = check_bin(fallback, 'A2') if not geant: print_error('[ERROR] A2 executable not found!') if verbose: print("The A2 Geant binary couldn't be found neither in your $PATH variable nor in the fallback solution") sys.exit(1) else: if verbose: print('A2 executable found:', geant) geant_path = dirname(abspath(geant)) if check_file(geant_path, 'runA2Geant'): geant = check_bin(geant_path, 'runA2Geant') else: geant = check_bin(geant_path, 'runGeant.sh') if not geant: print_error('[ERROR] The runA2Geant or runGeant.sh script could not be found or used!') sys.exit(1) # check if Geant version is used which can read in Pluto files (without pluto2mkin converter) if os.path.exists(get_path(geant_path, 'pluto2mkin')): print_error('[ERROR] pluto2mkin converter found in %s' % geant_path) print(" It's highly recommended to use the Ant branch of the a2geant repository.") sys.exit(1) # check target length in A2 Geant4 DetectorSetup.mac geant_base = geant_path.replace('/build/bin', '') geant_macros = get_path(geant_base, 'macros') if not check_file(geant_macros, 'DetectorSetup.mac'): print(" No 'DetectorSetup.mac' macro found in the Geant macros directory.") target_length = '' with open(get_path(geant_macros, 'DetectorSetup.mac'), 'r') as mac: for line in mac: if '/A2/det/setTargetLength' in line: target_length = line.split()[1] if float(target_length) < 10.: print_color("[WARNING] The target length specified in the 'DetectorSetup.mac' macro", 'YELLOW') print_color(' in the Geant macros directory is smaller than the usual lH2 target', 'YELLOW') print_color(' size: %s cm. If you consider to use a smeared z vertex, make sure' % target_length, 'YELLOW') print_color(' the specified target length is correctly set.', 'YELLOW') print() return generator, geant
def check_playlist_valid(playlist_id): if playlist_id is None: color.print_color("Couldn't create the playlist. Aborting", color="RED") exit(-1)
def check_user_want_to_continue(directory, video_dict): if not user_want_to_continue(directory, video_dict): color.print_color("aborting... see you next time", color="RED") exit(0)
def export(self, file_name, force=False): """Export the current settings to the given file""" file_name = os.path.expanduser(file_name) path = os.path.dirname(file_name) if not path: path = os.getcwd() file_name = os.path.join(path, file_name) if not os.path.exists(path) and not force: print_error( '[ERROR] The path "%s" for exporting the settings does not exist!' % path) return False elif not os.path.exists(path): print_color( '[INFO] The path "%s" does not exist, it will be created now.' % path, 'BLUE') try: os.makedirs(path) except OSError as exception: if exception.errno == errno.EACCES: print_error( '[ERROR] You do not have the permission to create directories in "%s"' % os.path.dirname(path)) return False elif exception.errno != errno.EEXIST: raise if not os.access(path, os.W_OK): print_error('[ERROR] The given path "%s" os not writable!' % path) return False if os.path.isfile(file_name): if not force: print_error( '[ERROR] The specified file "%s" already exists, aborting export.' % file_name) return False else: print_color( '[INFO] The file "%s" already exists, it will be overwritten.' % file_name, 'BLUE') with open(file_name, 'w') as file: file.write('%s\n' % '[settings]') file.write( '%s\n' % '# output path, current directory will be used if missing') file.write('%s = %s\n' % ('OUTPUT_PATH', self.__settings['OUTPUT_PATH'])) file.write( '%s\n' % '# directory relative to output path above to store MC generated files' ) file.write('%s = %s\n' % ('MCGEN_DATA', self.__settings['MCGEN_DATA'])) file.write('%s\n' % '# relative path to output path above ' 'where the Geant4 files should be stored') file.write('%s = %s\n' % ('GEANT_DATA', self.__settings['GEANT_DATA'])) file.write('%s\n' % '# log directory relative to output path above, ' 'log will be used if empty') file.write('%s = %s\n' % ('LOG_DATA', self.__settings['LOG_DATA'])) file.write( '%s\n' % '# path to the a2geant binaries, $PATH is used if empty') file.write('%s = %s\n' % ('A2_GEANT_PATH', self.__settings['A2_GEANT_PATH'])) file.write( '%s\n' % '# the A2 Geant binary which should be used, default "A2" if empty' ) file.write( '%s\n' % '# IMPORTANT: if you\'re using the new A2Geant4 version, set this parameter' ) file.write( '%s\n' % '# to "A2Geant4" and specify additional "GeantFlags" further down' ) file.write('%s = %s\n\n' % ('GEANT_BINARY', self.__settings['GEANT_BINARY'])) file.write('%s\n' % '# some default settings for jobs') file.write('%s = %s\n' % ('QSUB_BIN', self.__settings['QSUB_BIN'])) file.write( '%s\n' % '# mail when job aborts (a), begins (b), ends (e), no mails (n)' ) file.write('%s = %s\n' % ('QSUB_MAIL', self.__settings['QSUB_MAIL'])) file.write('%s = %s\n' % ('QUEUE', self.__settings['QUEUE'])) file.write('%s = %s\n' % ('WALLTIME', self.__settings['WALLTIME'])) file.write('%s = %s\n' % ('PRIORITY', self.__settings['PRIORITY'])) file.write('%s\n' % '# any additional flags needed for submission') file.write( '%s\n' % '# e.g. to restrict execution hosts to sl7: -l nodes=1:sl7') file.write('%s = %s\n\n' % ('QSUB_EXTRA', self.__settings['QSUB_EXTRA'])) file.write('%s\n' % '# simulation specific settings') file.write('%s\n' % '# the MC generator which should be used, i.e.') file.write('%s\n' % '# Ant-pluto, Ant-cocktail, or Ant-mcgun') file.write( '%s\n' % '# Note: If you specify a different MC generator, please') file.write( '%s\n' % '# make sure that its output is readable by Geant!') file.write('%s\n' % '# Flags can be passed via AddFlags option.') file.write('%s = %s\n' % ('GENERATOR', self.__settings['GENERATOR'])) file.write( '%s\n' % '# leave the generator path blank to use the $PATH variable') file.write('%s = %s\n' % ('GENERATOR_PATH', self.__settings['GENERATOR_PATH'])) file.write('%s\n' % '# minimum energy of the photon beam') file.write('%s: %s\n' % ('Emin', self.__settings['Emin'])) file.write('%s\n' % '# maximum energy of the photon beam') file.write('%s: %s\n' % ('Emax', self.__settings['Emax'])) file.write('%s\n' % '# Ant-cocktail specific settings') file.write('%s\n' % '# Define only ONE of the following two settings!') file.write( '%s\n' % '# Setup which should be used, e.g. "Setup_2014_07_EPT_Prod"') file.write('%s: %s\n' % ('COCKTAIL_SETUP', self.__settings['COCKTAIL_SETUP'])) file.write( '%s\n' % '# Binning for the beam energy, min and max energy as defined above' ) file.write( '%s: %s\n' % ('COCKTAIL_BINNING', self.__settings['COCKTAIL_BINNING'])) file.write( '%s\n' % '# Ant-mcgun specific settings, min and max energy used as defined above' ) file.write( '%s\n' % '# covered theta range in degree in the format min_theta max_theta' ) file.write('%s: %s\n' % ('GUN_THETA', self.__settings['GUN_THETA'])) file.write('%s\n' % '# opening angle between particles in degree') file.write('%s: %s\n' % ('GUN_OPENING', self.__settings['GUN_OPENING'])) file.write( '%s\n' % '# additional flags passed to runGeant (which calls a2geant), ' 'for example regex to replace information in detector macro setup' ) file.write( '%s\n' % "# like 's~^(/A2/det/setTargetLength).*~$1 5 cm~' (using the Ant branch of a2geant4)" ) file.write( '%s\n' % '# in case you\'re using the new A2Geant4 additional flags like the macros you want to use go here' ) file.write('%s: %s\n\n' % ('GeantFlags', self.__settings['GeantFlags'])) file.write( '%s\n' % '# additional flags passed to the generator, for example --flatEbeam (cocktail, pluto) or --flatTheta (mcgun)' ) file.write('%s: %s\n\n' % ('AddFlags', self.__settings['AddFlags'])) file.write('%s\n' % '[channels]') file.write( '%s\n' % '# channels which should be simulated with Ant-pluto, line has to start with ";", ' 'given in the syntax used in Pluto (do not forget the recoil proton!), ' 'the amount of files and the number of events per file') file.write('%s\n' % '#;"p pi0 [g g]" 10 100000') file.write( '%s\n' % '# in case of Ant-cocktail, start the string with "Cocktail", ' 'followed by the amount of files and the number of events per file' ) file.write('%s\n' % '#;"Cocktail" 100 10000') file.write( '%s\n' % '# for the particle gun Ant-mcgun, start the string with "Gun: <particle_list>", ' 'particle list separated by spaces, followed by the amount of files ' 'and the number of events per file') file.write('%s\n' % '#;"Gun: g g" 100 10000') return True
def main(): """Main function: process all information, prepare simulation process, submit jobs""" parser = argparse.ArgumentParser(description='Submit simulations on blaster, configuration ' 'is done via a config file "sim_settings" in ~ or . ' 'and/or a list of command line arguments') parser.add_argument('-c', '--config', nargs=1, metavar='config_file', dest='config',# required=True, type=lambda x: is_valid_file(parser, x), help='Optional: Specify a custom config file') parser.add_argument('-o', '--output', nargs=1, metavar='output_directory', type=lambda x: is_valid_dir(parser, x), help='Optional: Custom output directory') parser.add_argument('-g', '--generator', nargs=1, type=str, metavar='MC generator binary', help='Optional: If the generator binary is not specified in the config ' 'or should be overwritten, use this option to specify the binary manually') parser.add_argument('-p', '--path-generator', nargs=1, type=str, metavar='MC generator binary path', help='Optional: If the generator binary (e.g. Ant-pluto) can\'t be found ' 'in your $PATH variable, use this option to specify the path manually') parser.add_argument('-l', '--list', nargs='?', const=True, help='List the amount of existing files per channel ' 'in the output directory and exit; if "all" is specified ' 'as an optional argument the amount of events will be listed') parser.add_argument('-e', '--example-config', nargs='?', const='example_settings', help='Export an example of default settings and exit. ' 'If no file is specified, the output will be written to "example_settings"') parser.add_argument('-a', '--add-flags', nargs=1, type=str, metavar='"Additional flags"', help='Optional: Define additional flags which will be passed to the ' 'MC generator. Flags defined in the settings file will be overwritten. ' 'All additional flags must be given as one quoted string!') parser.add_argument('-t', '--tag', nargs=1, type=str, metavar='Job Tag', help='Optional: Specify a job tag, default is Sim') parser.add_argument('-w', '--walltime', type=int, nargs=1, metavar='walltime', help='Walltime for jobs, time in hours') parser.add_argument('-q', '--queue', type=str, nargs=1, metavar='queue', help='Queue which should be used for the jobs') parser.add_argument('-f', '--force', action='store_true', help='Force creation of directories if they do not exist ' 'or overwrite existing files') parser.add_argument('-v', '--verbose', action='store_true', help='Print additional output') args = parser.parse_args() verbose = args.verbose force = args.force settings = None if args.example_config: print_color('[INFO] Write example settings to file "%s"' % args.example_config, 'BLUE') settings = Settings() if not settings.export(args.example_config, force): print_error('[ERROR] Creating example settings "%s" failed.' % args.example_config) sys.exit(0) config = None channels = [] if args.config: config = args.config[0] if verbose: print('Use config file %s' % config) else: if verbose: print('Try to find a default config file') config = find_config(verbose=verbose) if not config: print_color('No config file found, use default values.', 'YELLOW') settings = Settings() if config: print_color('The config file "%s" will be used.' % config, 'BLUE') if verbose: print('Read config file %s' % config) with open(config, 'r') as conf: settings, channels = read_config(conf) if args.list: if verbose and args.list == 'all': print('Trying to determine the total amount of simulated events') if args.output: settings.set('OUTPUT_PATH', get_path(args.output[0])) list_file_amount(settings, args.list == 'all') sys.exit(0) if verbose: print('The following default settings were determined:') settings.print() if channels: print('The following channels have been found:') print(channels) if args.output: if not check_path(args.output[0], force): sys.exit('The output directory %s cannot be used' % args.output[0]) output = get_path(args.output[0]) print_color('Setting custom output directory: %s' % output, 'GREEN') settings.set('OUTPUT_PATH', output) if args.path_generator: path_generator = get_path(args.path_generator[0]) print_color('Setting custom path for MC generator %s' % path_generator, 'GREEN') settings.set('GENERATOR_PATH', path_generator) if not check_directories(settings, force, verbose): sys.exit(1) settings.set('MCGEN_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('MCGEN_DATA'))) settings.set('GEANT_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('GEANT_DATA'))) settings.set('LOG_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('LOG_DATA'))) if args.generator: generator = args.generator[0] print_color('Use custom MC generator %s' % generator, 'GREEN') settings.set('GENERATOR', generator) if args.add_flags: print_color('Set custom flags to pass to the MC generator: %s' % args.add_flags[0], 'GREEN') settings.set('AddFlags', args.add_flags[0]) mc_generator, geant = check_binaries(settings, verbose) if not mc_generator or not geant: sys.exit(1) if not channels: print_color('[Warning] No channels specified in the config file', 'YELLOW') print_color(' Use -e to export example settings or -h for help', 'YELLOW') channels = simulation_dialogue() if not channels: sys.exit('No channels entered for simulation, exiting.') if args.walltime: print_color('Setting custom walltime to %d hours' % args.walltime[0], 'GREEN') settings.set('WALLTIME', args.walltime[0]) if args.queue: print_color('Setting custom queue to %s' % args.queue[0], 'GREEN') settings.set('QUEUE', args.queue[0]) tag = args.tag[0] if args.tag else 'Sim' if verbose: print('Use the tag "%s" for the submitted jobs' % tag) if verbose: print('These are the updated settings:') settings.print() if 'Ant-cocktail' in mc_generator: if not sanity_check_cocktail(settings): sys.exit(1) if 'Ant-mcgun' in mc_generator: if not sanity_check_mcgun(settings): sys.exit(1) if not sanity_check_channels(mc_generator, channels): sys.exit(1) if verbose: print('Determining the existing amount of files...') simulation = [] for channel in channels: decay_string = get_decay_string(channel[0]) max_number = check_simulation_files(settings, decay_string) simulation.append([decay_string, channel[0], channel[1], channel[2], max_number]) print_color(str(len(simulation)) + ' channels configured. ' 'The following simulation will take place:', 'BLUE') total_files, total_events = 0, 0 for channel, _, files, events, _ in simulation: total = files*events chnl = channel if 'pluto' in mc_generator.lower(): chnl = format_channel(channel) print("{0:<20s} {1:>4d} files per {2:>4s} events (total {3:>4s} events)" .format(chnl, files, unit_prefix(events), unit_prefix(total))) total_files += files total_events += total print(" Total %s events in %d files" % (unit_prefix(total_events), total_files)) print(" Files will be stored in " + settings.get('OUTPUT_PATH')) # run a test job for the first command to be submitted and check the output print_color('Test provided commands', 'BLUE') print(' Running first test job locally . . .') if not run_test_job(settings, simulation, mc_generator, geant): print_error('[ERROR] Test job failed, aborting job submission') sys.exit(1) print_color('Test job successful', 'BLUE') # start the job submission print('Start submitting jobs, total', total_files) submit_jobs(settings, simulation, mc_generator, geant, tag, total_files, verbose) print_color('Done!', 'GREEN')