def read_config(config_file):
"""Read the given config file and return the parsed settings and channels"""
config = configparser.ConfigParser()
config.optionxform = str # preserve case of the options
config.read(config_file.name)
settings = list(config.items('settings'))
settings = Settings(settings)
if not config.has_section('channels'):
print_color('[WARNING] No channels specified in %s' % config_file.name, 'YELLOW')
return settings, []
lines = [line.rstrip('\n') for line in config_file.readlines() if not line.startswith('#') and line.split()] # last part excludes empty lines
channels = lines[lines.index('[channels]')+1:]
if not channels:
print_color('[WARNING] No channels specified in %s' % config_file.name, 'YELLOW')
return settings, []
delimiter = ' '
pattern = re.compile(r'''((?:[^%s"']|"[^"]*"|'[^']*')+)''' % delimiter) # split using delimiter outside of quotations [http://stackoverflow.com/a/2787064]
chnl = []
for channel in channels:
option = pattern.split(channel.lstrip(';'))[1::2]
if len(option) != 3:
print_error('[ERROR] Wrong number of arguments for channel %s' % option[0])
print(' This channel will be skipped')
continue
if option[1] is '0' or option[2] is '0':
#print('Skip channel', format_channel(option[0], False))
print('Skip channel', option[0])
continue
if not option[1].isdigit() or not option[2].isdigit():
print_error('[ERROR] The amount of files and events to simulate have to be integers!')
print(' Check your settings for channel %s' % option[0])
continue
chnl.append((option[0], int(option[1]), int(option[2])))
return settings, chnl
def read_config(config_file):
"""Read the given config file and return the parsed settings and channels"""
config = configparser.ConfigParser()
config.optionxform = str # preserve case of the options
config.read(config_file.name)
settings = list(config.items('settings'))
settings = Settings(settings)
if not config.has_section('channels'):
print_color('[WARNING] No channels specified in %s' % config_file.name, 'YELLOW')
return settings, []
lines = [line.rstrip('\n') for line in config_file.readlines() if not line.startswith('#') and line.split()] # last part excludes empty lines
channels = lines[lines.index('[channels]')+1:]
if not channels:
print_color('[WARNING] No channels specified in %s' % config_file.name, 'YELLOW')
return settings, []
delimiter = ' '
pattern = re.compile(r'''((?:[^%s"']|"[^"]*"|'[^']*')+)''' % delimiter) # split using delimiter outside of quotations [http://stackoverflow.com/a/2787064]
chnl = []
for channel in channels:
option = pattern.split(channel.lstrip(';'))[1::2]
if len(option) != 3:
print_error('[ERROR] Wrong number of arguments for channel %s' % option[0])
print(' This channel will be skipped')
continue
if option[1] is '0' or option[2] is '0':
#print('Skip channel', format_channel(option[0], False))
print('Skip channel', option[0])
continue
if not option[1].isdigit() or not option[2].isdigit():
print_error('[ERROR] The amount of files and events to simulate have to be integers!')
print(' Check your settings for channel %s' % option[0])
continue
chnl.append((option[0], int(option[1]), int(option[2])))
return settings, chnl
def __init__(self, settings=None):
self.__settings = {
'OUTPUT_PATH': '.',
'MCGEN_DATA': 'mcgen',
'GEANT_DATA': 'geant',
'MCGEN_PREFIX': 'mcgen',
'GEANT_PREFIX': 'g4sim',
'LOG_DATA': 'log',
'A2_GEANT_PATH': '',
'MCGEN_ONLY': False,
'GEANT_ONLY': False,
'QSUB_BIN': 'qsub',
'QSUB_MAIL': 'a',
'QSUB_EXTRA': '',
'QUEUE': 'dflt',
'WALLTIME': '12:00:00',
'PRIORITY': 0,
'GENERATOR': 'Ant-pluto',
'GENERATOR_PATH': '',
'Emin': 1420,
'Emax': 1580,
'COCKTAIL_SETUP': '',
'COCKTAIL_BINNING': 0,
'GUN_THETA': '0 180',
'GUN_OPENING': '',
'GeantFlags': '',
'AddFlags': '',
}
if settings and len(settings[0]) == 2:
for setting in settings:
if setting[0] not in self.__settings:
print_error('[ERROR] Setting "%s" not valid! Will be skipped.' % setting[0])
continue
if not setting[1] and self.__settings[setting[0]]:
print_color('[INFO] No value given for "{0}". Default value "{1}" will '
'be used'.format(setting[0], self.__settings[setting[0]]), 'BLUE')
continue
if setting[0] in ('PRIORITY', 'COCKTAIL_BINNING'):
if not setting[1].isdigit():
print_error('[ERROR] The given value for "%s" is not an integer, use '
'default value %d instead.' % (setting[1], self.__settings[setting[1]]))
continue
self.__settings[setting[0]] = int(setting[1])
continue
if setting[0] in ('Emin', 'Emax'):
if not isfloat(setting[1]):
print_error('[ERROR] The given energy "%s" is not a float! Use default '
'value %.1f instead.' % (setting[1], self.__settings[setting[0]]))
continue
self.__settings[setting[0]] = float(setting[1])
continue
self.__settings[setting[0]] = setting[1]
elif settings and len(settings[0]) != 2:
print_error('[ERROR] Given settings in the wrong format, '
'list with tuples (setting, value) expected.')
def upload_all_videos(directory, video_dict):
uploaded_video_ids = []
failed_videos = []
for video_file, action_upload in video_dict.items():
if action_upload:
response, _ = upload_video(os.path.join(directory, video_file))
if response is not None:
uploaded_video_ids.append(response["id"])
else:
failed_videos.append(video_file)
if failed_videos != []:
color.print_color("Those uploads have failed:", color="YELLOW")
color.print_color(', '.join(failed_videos), color="RED")
else:
print("Every video was uploaded successfully.")
return uploaded_video_ids, failed_videos
def test_process(cmd, time=None):
"""Try to run a process and check its return code
if something went wrong, print command output
if time is given, kill process after time expired"""
# use shell=True, otherwise the command passed to Popen to execute,
# including cmd.split(' ', 1), produces errors (probably due to reaction string)
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Python 3.3 needed for timeout, most recent version on blaster is 3.1
# if time:
# if time < 0:
# print_error('given time is negative!')
# return False
# try:
# outs, errs = proc.communicate(timeout=time)
# if proc.returncode:
# print_error('non-zero returncode for command: %s' % cmd)
# print_color('error output:', 'YELLOW')
# print(errs.decode('utf-8'))
# print_color('standard output:', 'YELLOW')
# print(outs.decode('utf-8'))
# return False
# else:
# return True
# except TimeoutExpired:
# proc.kill()
# outs, errs = proc.communicate()
# print_error('[ERROR] given timeout expired, process killed')
# print_color('error output:', 'YELLOW')
# print(errs.decode('utf-8'))
# print_color('standard output:', 'YELLOW')
# print(outs.decode('utf-8'))
# return False
outs, errs = proc.communicate()
if proc.returncode:
print_error('[ERROR] non-zero returncode for command: %s' % cmd)
print_color('error output:', 'YELLOW')
print(errs.decode('utf-8'))
print_color('standard output:', 'YELLOW')
print(outs.decode('utf-8'))
return False
return True
def sanity_check_cocktail(settings):
"""Check if the given settings for Ant-cocktail seem okay"""
setup = settings.get('COCKTAIL_SETUP')
binning = int(settings.get('COCKTAIL_BINNING'))
if setup and not setup.startswith('Setup_'):
print_color("[WARNING] The specified detector setup doesn't start with 'Setup_'", 'YELLOW')
if not setup and not binning:
print_error('[ERROR] No Setup and no binning for the beam energy specified!')
print_error(' Please make sure to provide one option')
return False
if setup and binning:
print_error('[ERROR] You provided both a setup and energy binning for the Cocktail')
print_error(' Please make sure to provide only one of these options')
return False
return True
def __init__(self, settings=None):
self.__settings = {
'OUTPUT_PATH': '.',
'PLUTO_DATA': 'pluto',
'GEANT_DATA': 'geant',
'LOG_DATA': 'log',
'A2_GEANT_PATH': '/home/neiser/opt/a2geant',
'QSUB_BIN': 'qsub',
'QSUB_MAIL': 'a',
'QUEUE': 'dflt',
'WALLTIME': '12:00:00',
'PRIORITY': 0,
'Emin': 1420,
'Emax': 1580
}
if settings and len(settings[0]) == 2:
for setting in settings:
if setting[0] not in self.__settings:
print_error('[ERROR] Setting "%s" not valid! Will be skipped.' % setting[0])
continue
if not setting[1]:
print_color('[INFO] No value given for "{0}". Default value "{1}" will '
'be used'.format(setting[0], self.__settings[setting[0]]), 'BLUE')
continue
if setting[0] == 'PRIORITY':
if not setting[1].isdigit():
print_error('[ERROR] The given priority is not an integer, use '
'default value %d instead.' % self.__settings[setting[1]])
continue
self.__settings[setting[0]] = int(setting[1])
continue
if setting[0] in ('Emin', 'Emax'):
if not isfloat(setting[1]):
print_error('[ERROR] The given energy "%s" is not a float! Use default '
'value %.1f instead.' % (setting[1], self.__settings[setting[0]]))
continue
self.__settings[setting[0]] = float(setting[1])
continue
self.__settings[setting[0]] = setting[1]
elif settings and len(settings[0]) != 2:
print_error('[ERROR] Given settings in the wrong format, '
'list with tuples (setting, value) expected.')
def simulation_dialogue():
"""Show a simulation dialogue in which the user can enter the channels to be simulated
if they're not specified in the config file"""
channels = []
positive_responses = ['y', 'Y', 'j', 'J', 'yes', 'Yes']
negative_responses = ['n', 'N', 'no', 'No']
allowed_responses = positive_responses + negative_responses
opt = input("\nDo you want to enter channels which should be simulated? [y/n]: ")
while opt not in allowed_responses:
opt = input("You've entered an invalid response! Please try again: ")
if opt in negative_responses:
return channels
while opt in positive_responses:
if not channels:
print('Please enter a channel which should be simulated:')
print('Note: The syntax has to be exactly the Pluto syntax for the reaction,\n'
' e.g. "pi0 [g g]" for the decay of a pi0 into two photons.\n'
' The recoil proton is taken into account, type only the desired reaction')
else:
opt = input("Do you want to enter another channel? [y/n]: ")
while opt not in allowed_responses:
opt = input("You've entered an invalid response! Please try again: ")
if opt in negative_responses:
continue
chan = 'p '
chan += input(chan)
n_files = input_digit("How much files should be generated for this channel?", max_retries=3)
if not n_files:
print("This channel will be skipped")
continue
n_events = input_digit("How mich events should be generated per file?")
if not n_events:
print("This channel will be skipped")
continue
channels.append([chan, n_files, n_events])
print_color("You've entered %d channels for the simulation process" % len(channels), 'BLUE')
return channels
def check_simulation_files(settings, channel):
"""Do some sanity checks if the existing simulation files seem to be okay
and return the maximum file number"""
pluto_files = [f for f in os.listdir(settings.get('PLUTO_DATA'))
if f.startswith(PLUTO_PREFIX) and channel in f]
geant_files = [f for f in os.listdir(settings.get('GEANT_DATA'))
if f.startswith(GEANT_PREFIX) and channel in f]
max_pluto = max_file_number(pluto_files)
max_geant = max_file_number(geant_files)
if max_geant > max_pluto:
print_color(" [Warning]", 'YELLOW')
print("There are more Geant4 simulation files than Pluto generated\nfiles for channel %s"
% format_channel(channel, False))
input("Will continue by pressing any key ")
elif max_geant < max_pluto:
print_color(" [Warning]", 'YELLOW')
print("There are more Pluto generated files than Geant4 simulated\nfiles for channel %s"
% format_channel(channel, False))
input("Will continue by pressing any key ")
return max(max_pluto, max_geant)
def get_decay_string(channel, level=1):
"""Get a decay string for a certain channel, print warning if proton is missing"""
channel = channel.strip('"')
# Ant-cocktail
if channel.lower().startswith('cocktail'):
return 'cocktail'
# Ant-mcgun
if channel.lower().startswith('gun:'):
channel = channel.split(':')[-1].strip()
channel = parse_pluto_string.get_initial_state(channel)
channel = parse_pluto_string.particle_list_to_string(channel)
return channel + '-gun'
# default: assume Ant-pluto reaction channel syntax
if channel.startswith('p '):
channel = channel[2:]
else:
print_color('[WARNING] recoil proton missing in decay string: %s' % channel, 'WARNING')
channel = parse_pluto_string.get_decay_string(channel, level)
return channel
def insert_videos_playlist(uploaded_video_ids, playlist_id):
playlist_insert_failed = []
print("\nuploaded_video_ids", uploaded_video_ids, end='\n\n')
for video_id in uploaded_video_ids:
try:
response = add_to_playlist(playlist_id, video_id)
if not 'id' in response:
playlist_insert_failed.append(video_id)
except Exception:
playlist_insert_failed.append(video_id)
print("http error when trying to insert video to playlist")
print(
"This video may be a duplicate or the playlist_id may be wrong"
)
if playlist_insert_failed != []:
color.print_color("Those playlist insertion have failed:", "YELLOW")
color.print_color(", ".join(playlist_insert_failed), "RED")
else:
color.print_color(
"Every video was successfully inserted into the playlist.",
color="CYAN")
def check_binaries(settings, ant_pluto='', verbose=False):
"""Check if the needed binaries exist, return the absolute paths to them"""
pluto, tid, geant = None, None, None
# first of all check if the specified qsub binary exists
if not find_executable(settings.get('QSUB_BIN')):
print_error('[ERROR] The binary %s could not be found!' % settings.get('QSUB_BIN'))
sys.exit(1)
if ant_pluto:
if verbose:
print('Searching for Ant-pluto and Ant-addTID in %s' % ant_pluto)
pluto = check_bin(ant_pluto, 'Ant-pluto')
if pluto and verbose:
print('Found Ant-pluto')
tid = check_bin(ant_pluto, 'Ant-addTID')
if tid and verbose:
print('Found Ant-addTID')
if not pluto:
print_error('[ERROR] Ant-pluto not found in %s!' % ant_pluto)
sys.exit(1)
if not tid:
print_error('[ERROR] Ant-addTID not found in %s!' % ant_pluto)
sys.exit(1)
else:
pluto = find_executable('Ant-pluto')
if not pluto:
print_error('[ERROR] Ant-pluto not found!')
if verbose:
print("Ant-pluto couldn't be found within your $PATH variable")
sys.exit(1)
tid = find_executable('Ant-addTID')
if not tid:
print_error('[ERROR] Ant-addTID not found!')
if verbose:
print("Ant-addTID couldn't be found within your $PATH variable")
sys.exit(1)
else:
pluto = abspath(pluto)
tid = abspath(tid)
if verbose:
print('Ant-pluto found:', pluto)
print('Ant-addTID found:', tid)
geant_path = settings.get('A2_GEANT_PATH')
if verbose:
print('Searching for the A2 binary in %s' % geant_path)
if not check_bin(geant_path, 'A2'):
print_error('[ERROR] A2 Geant executable not found!')
sys.exit(1)
elif verbose:
print('A2 executable found in %s' % geant_path)
geant = check_bin(geant_path, 'runGeant.sh')
if not geant:
print_error('[ERROR] The runGeant.sh script could not be found or used!')
sys.exit(1)
# check if Geant version is used which can read in Pluto files (without pluto2mkin converter)
if os.path.exists(get_path(geant_path, 'pluto2mkin')):
print_error('[ERROR] pluto2mkin converter found in %s' % geant_path)
print(" It's highly recommended to use the PlutoGen branch of the a2geant repository.")
sys.exit(1)
# check target length in A2 Geant4 DetectorSetup.mac
geant_macros = get_path(geant_path, 'macros')
if not check_file(geant_macros, 'DetectorSetup.mac'):
print(" No 'DetectorSetup.mac' macro found in the Geant macros directory.")
target_length = ''
with open(get_path(geant_macros, 'DetectorSetup.mac'), 'r') as mac:
for line in mac:
if '/A2/det/setTargetLength' in line:
target_length = line.split()[1]
if float(target_length) < 10.:
print_color("[WARNING] The target length specified in the 'DetectorSetup.mac' macro", 'YELLOW')
print_color(' in the Geant macros directory is smaller than the usual lH2 target', 'YELLOW')
print_color(' size: %s cm. If you consider to use a smeared z vertex, make sure' % target_length, 'YELLOW')
print_color(' the specified target length is correctly set.', 'YELLOW')
print()
return pluto, tid, geant
def export(self, file_name, force=False):
"""Export the current settings to the given file"""
file_name = os.path.expanduser(file_name)
path = os.path.dirname(file_name)
if not path:
path = os.getcwd()
file_name = os.path.join(path, file_name)
if not os.path.exists(path) and not force:
print_error('[ERROR] The path "%s" for exporting the settings does not exist!' % path)
return False
elif not os.path.exists(path):
print_color('[INFO] The path "%s" does not exist, it will be created now.' % path, 'BLUE')
try:
os.makedirs(path)
except OSError as exception:
if exception.errno == errno.EACCES:
print_error('[ERROR] You do not have the permission to create directories in "%s"'
% os.path.dirname(path))
return False
elif exception.errno != errno.EEXIST:
raise
if not os.access(path, os.W_OK):
print_error('[ERROR] The given path "%s" os not writable!' % path)
return False
if os.path.isfile(file_name):
if not force:
print_error('[ERROR] The specified file "%s" already exists, aborting export.' % file_name)
return False
else:
print_color('[INFO] The file "%s" already exists, it will be overwritten.' % file_name, 'BLUE')
with open(file_name, 'w') as file:
file.write('%s\n' % '[settings]')
file.write('%s\n' % '# output path, current directory will be used if missing')
file.write('%s = %s\n' % ('OUTPUT_PATH', self.__settings['OUTPUT_PATH']))
file.write('%s\n' % '# directory relative to output path above to store Pluto files')
file.write('%s = %s\n' % ('PLUTO_DATA', self.__settings['PLUTO_DATA']))
file.write('%s\n' % '# relative path to output path above '
'where the Geant4 files should be stored')
file.write('%s = %s\n' % ('GEANT_DATA', self.__settings['GEANT_DATA']))
file.write('%s\n' % '# log directory relative to output path above, '
'log will be used if empty')
file.write('%s = %s\n' % ('LOG_DATA', self.__settings['LOG_DATA']))
file.write('%s = %s\n\n' % ('A2_GEANT_PATH', self.__settings['A2_GEANT_PATH']))
file.write('%s\n' % '# some default settings for jobs')
file.write('%s = %s\n' % ('QSUB_BIN', self.__settings['QSUB_BIN']))
file.write('%s\n' % '# mail when job aborts (a), begins (b), ends (e), no mails (n)')
file.write('%s = %s\n' % ('QSUB_MAIL', self.__settings['QSUB_MAIL']))
file.write('%s = %s\n' % ('QUEUE', self.__settings['QUEUE']))
file.write('%s = %s\n' % ('WALLTIME', self.__settings['WALLTIME']))
file.write('%s = %s\n\n' % ('PRIORITY', self.__settings['PRIORITY']))
file.write('%s\n' % '# simulation specific settings')
file.write('%s\n' % '# minimum energy of the photon beam')
file.write('%s: %s\n' % ('Emin', self.__settings['Emin']))
file.write('%s\n' % '# maximum energy of the photon beam')
file.write('%s: %s\n\n' % ('Emax', self.__settings['Emax']))
file.write('%s\n' % '[channels]')
file.write('%s\n' % '# channels which should be simulated, line has to start with ";", '
'given in the syntax used in Pluto (do not forget the recoil proton!), '
'the amount of files and the number of events per file')
file.write('%s\n' % ';"p pi0 [g g]" 10 100000')
return True
def print_commands(commands):
for cmd in commands:
print_color(" | ".join(cmd.get_aliases()), bcolors.OKGREEN, end='')
print_color(" " + cmd.get_params_help(), bcolors.OKBLUE)
print("\t" + cmd.get_description())
#!/usr/bin/python
import color
color.print_color("apple")
#invalid
#print_color("apple");
from color import print_color
print_color("apple")
#invalid
#print_name("orange");
from color import *
print_color("apple")
print_name("orange")
def user_want_to_continue(directory, video_dict):
color.print_color(WARN_MSG.format(directory), color="RED")
color.print_color(', '.join(video_dict.keys()), color="YELLOW")
return input(color.format_color(WARN_INPUT, color="CYAN")) in 'yY'
def main():
"""Main function: process all information, prepare simulation process, submit jobs"""
parser = argparse.ArgumentParser(description='Submit simulations on blaster, configuration '
'is done via a config file "sim_settings" in ~ or . '
'and/or a list of command line arguments')
parser.add_argument('-c', '--config', nargs=1, metavar='config_file',
dest='config',# required=True,
type=lambda x: is_valid_file(parser, x),
help='Optional: Specify a custom config file')
parser.add_argument('-o', '--output', nargs=1, metavar='output_directory',
type=lambda x: is_valid_dir(parser, x),
help='Optional: Custom output directory')
parser.add_argument('-a', '--ant-pluto', nargs=1, type=str, metavar='Ant-pluto binary path',
help='Optional: If the Ant-pluto binary can\'t be found in your'
'$PATH variable, use this option to specify the path manually')
parser.add_argument('-l', '--list', nargs='?', const=True,
help='List the amount of existing files per channel'
'in the output directory and exit; if "all" is specified'
'as an optional argument the amount of events will be listed')
parser.add_argument('-e', '--example-config', nargs='?', const='example_settings',
help='Export an example of default settings and exit. '
'If no file is specified, the output will be written to "example_settings"')
parser.add_argument('-w', '--walltime', type=int, nargs=1, metavar='walltime',
help='Walltime for jobs, time in hours')
parser.add_argument('-q', '--queue', type=str, nargs=1, metavar='queue',
help='Queue which should be used for the jobs')
parser.add_argument('-f', '--force', action='store_true',
help='Force creation of directories if they do not exist'
'or overwrite existing files')
parser.add_argument('-v', '--verbose', action='store_true',
help='Print additional output')
args = parser.parse_args()
verbose = args.verbose
force = args.force
settings = None
if args.example_config:
print_color('[INFO] Write example settings to file "%s"' % args.example_config, 'BLUE')
settings = Settings()
if not settings.export(args.example_config, force):
print_error('[ERROR] Creating example settings "%s" failed.' % args.example_config)
sys.exit(0)
config = None
channels = []
if args.config:
config = args.config[0]
if verbose:
print('Use config file %s' % config)
else:
if verbose:
print('Try to find a default config file')
config = find_config(verbose=verbose)
if not config:
print_color('No config file found, use default values.', 'YELLOW')
settings = Settings()
if config:
print_color('The config file "%s" will be used.' % config, 'BLUE')
if verbose:
print('Read config file %s' % config)
with open(config, 'r') as conf:
settings, channels = read_config(conf)
if args.list:
if verbose and args.list == 'all':
print('Trying to determine the total amount of simulated events')
if args.output:
settings.set('OUTPUT_PATH', get_path(args.output[0]))
list_file_amount(settings, args.list == 'all')
sys.exit(0)
if verbose:
print('The following settings will be used:')
settings.print()
if channels:
print('The following channels have been found:')
print(channels)
if args.output:
if not check_path(args.output[0], force):
sys.exit('The output directory %s cannot be used' % args.output[0])
output = get_path(args.output[0])
print_color('Setting custom output directory: %s' % output, 'GREEN')
settings.set('OUTPUT_PATH', output)
ant_pluto = ''
if args.ant_pluto:
ant_pluto = args.ant_pluto[0]
print_color('Use custom Ant-pluto path %s' % ant_pluto, 'GREEN')
if not check_directories(settings, ant_pluto, force, verbose):
sys.exit(1)
pluto, tid, geant = check_binaries(settings, ant_pluto, verbose)
if not pluto or not tid or not geant:
sys.exit(1)
if not channels:
print_color('[Warning] No channels specified in the config file', 'YELLOW')
channels = simulation_dialogue()
if not channels:
sys.exit('No channels entered for simulation, exiting.')
if args.walltime:
print_color('Setting custom walltime to %d hours' % args.walltime[0], 'GREEN')
settings.set('WALLTIME', args.walltime[0])
if args.queue:
print_color('Setting custom queue to %s' % args.queue[0], 'GREEN')
settings.set('QUEUE', args.queue[0])
if verbose:
print('This are the updated settings:')
settings.print()
if verbose:
print('Determining the existing amount of files...')
simulation = []
for channel in channels:
decay_string = get_decay_string(channel[0])
max_number = check_simulation_files(settings, decay_string)
simulation.append([decay_string, channel[0], channel[1], channel[2], max_number])
print_color(str(len(simulation)) + ' channels configured. '
'The following simulation will take place:', 'BLUE')
total_files, total_events = 0, 0
for channel, _, files, events, _ in simulation:
total = files*events
print("{0:<20s} {1:>4d} files per {2:>4s} events (total {3:>4s} events)"
.format(format_channel(channel), files, unit_prefix(events), unit_prefix(total)))
total_files += files
total_events += total
print(" Total %s events in %d files" % (unit_prefix(total_events), total_files))
print(" Files will be stored in " + settings.get('OUTPUT_PATH'))
# start the job submission
print('Start submitting jobs, total', total_files)
submit_jobs(settings, simulation, pluto, tid, geant, total_files)
print_color('Done!', 'GREEN')
def check_binaries(settings, generator_path='', verbose=False):
"""Check if the needed binaries exist, return the absolute paths to them"""
generator, geant = None, None
# first of all check if the specified qsub binary exists
if not find_executable(settings.get('QSUB_BIN')):
print_error('[ERROR] The binary %s could not be found!' % settings.get('QSUB_BIN'))
sys.exit(1)
generator = settings.get('GENERATOR')
generator_path = settings.get('GENERATOR_PATH')
if generator_path:
if verbose:
print('Searching for MC generator in %s' % generator_path)
generator = check_bin(generator_path, generator)
if generator and verbose:
print('Found %s' % settings.get('GENERATOR'))
if not generator:
print_error('[ERROR] %s not found in %s!' % (settings.get('GENERATOR'), generator_path))
sys.exit(1)
else:
generator = find_executable(generator)
if not generator:
print_error('[ERROR] %s not found!' % settings.get('GENERATOR'))
if verbose:
print("%s couldn't be found within your $PATH variable" % settings.get('GENERATOR'))
sys.exit(1)
else:
generator = abspath(generator)
if verbose:
print('%s found: %s', (settings.get('GENERATOR'), generator))
geant_path = settings.get('A2_GEANT_PATH')
if geant_path:
if verbose:
print('Searching for the A2 binary in %s' % geant_path)
if not check_bin(geant_path, 'A2'):
print_error('[ERROR] A2 Geant executable not found!')
sys.exit(1)
elif verbose:
print('A2 executable found in %s' % geant_path)
if check_file(geant_path, 'runA2Geant'):
geant = check_bin(geant_path, 'runA2Geant')
else:
geant = check_bin(geant_path, 'runGeant.sh')
if not geant:
print_error('[ERROR] The runA2Geant or runGeant.sh script could not be found or used!')
sys.exit(1)
else:
geant = find_executable('A2')
# fallback solution if not in $PATH
if not geant:
print_color("[WARNING] The A2 Geant binary couldn't be found within your $PATH variable", 'YELLOW')
print(' try to use fallback solution')
fallback = '/home/neiser/opt/a2geant/build/bin'
if check_file(fallback, 'A2'):
geant = check_bin(fallback, 'A2')
if not geant:
print_error('[ERROR] A2 executable not found!')
if verbose:
print("The A2 Geant binary couldn't be found neither in your $PATH variable nor in the fallback solution")
sys.exit(1)
else:
if verbose:
print('A2 executable found:', geant)
geant_path = dirname(abspath(geant))
if check_file(geant_path, 'runA2Geant'):
geant = check_bin(geant_path, 'runA2Geant')
else:
geant = check_bin(geant_path, 'runGeant.sh')
if not geant:
print_error('[ERROR] The runA2Geant or runGeant.sh script could not be found or used!')
sys.exit(1)
# check if Geant version is used which can read in Pluto files (without pluto2mkin converter)
if os.path.exists(get_path(geant_path, 'pluto2mkin')):
print_error('[ERROR] pluto2mkin converter found in %s' % geant_path)
print(" It's highly recommended to use the Ant branch of the a2geant repository.")
sys.exit(1)
# check target length in A2 Geant4 DetectorSetup.mac
geant_base = geant_path.replace('/build/bin', '')
geant_macros = get_path(geant_base, 'macros')
if not check_file(geant_macros, 'DetectorSetup.mac'):
print(" No 'DetectorSetup.mac' macro found in the Geant macros directory.")
target_length = ''
with open(get_path(geant_macros, 'DetectorSetup.mac'), 'r') as mac:
for line in mac:
if '/A2/det/setTargetLength' in line:
target_length = line.split()[1]
if float(target_length) < 10.:
print_color("[WARNING] The target length specified in the 'DetectorSetup.mac' macro", 'YELLOW')
print_color(' in the Geant macros directory is smaller than the usual lH2 target', 'YELLOW')
print_color(' size: %s cm. If you consider to use a smeared z vertex, make sure' % target_length, 'YELLOW')
print_color(' the specified target length is correctly set.', 'YELLOW')
print()
return generator, geant
def check_playlist_valid(playlist_id):
if playlist_id is None:
color.print_color("Couldn't create the playlist. Aborting",
color="RED")
exit(-1)
def check_user_want_to_continue(directory, video_dict):
if not user_want_to_continue(directory, video_dict):
color.print_color("aborting... see you next time", color="RED")
exit(0)
def export(self, file_name, force=False):
"""Export the current settings to the given file"""
file_name = os.path.expanduser(file_name)
path = os.path.dirname(file_name)
if not path:
path = os.getcwd()
file_name = os.path.join(path, file_name)
if not os.path.exists(path) and not force:
print_error(
'[ERROR] The path "%s" for exporting the settings does not exist!'
% path)
return False
elif not os.path.exists(path):
print_color(
'[INFO] The path "%s" does not exist, it will be created now.'
% path, 'BLUE')
try:
os.makedirs(path)
except OSError as exception:
if exception.errno == errno.EACCES:
print_error(
'[ERROR] You do not have the permission to create directories in "%s"'
% os.path.dirname(path))
return False
elif exception.errno != errno.EEXIST:
raise
if not os.access(path, os.W_OK):
print_error('[ERROR] The given path "%s" os not writable!' % path)
return False
if os.path.isfile(file_name):
if not force:
print_error(
'[ERROR] The specified file "%s" already exists, aborting export.'
% file_name)
return False
else:
print_color(
'[INFO] The file "%s" already exists, it will be overwritten.'
% file_name, 'BLUE')
with open(file_name, 'w') as file:
file.write('%s\n' % '[settings]')
file.write(
'%s\n' %
'# output path, current directory will be used if missing')
file.write('%s = %s\n' %
('OUTPUT_PATH', self.__settings['OUTPUT_PATH']))
file.write(
'%s\n' %
'# directory relative to output path above to store MC generated files'
)
file.write('%s = %s\n' %
('MCGEN_DATA', self.__settings['MCGEN_DATA']))
file.write('%s\n' % '# relative path to output path above '
'where the Geant4 files should be stored')
file.write('%s = %s\n' %
('GEANT_DATA', self.__settings['GEANT_DATA']))
file.write('%s\n' %
'# log directory relative to output path above, '
'log will be used if empty')
file.write('%s = %s\n' % ('LOG_DATA', self.__settings['LOG_DATA']))
file.write(
'%s\n' %
'# path to the a2geant binaries, $PATH is used if empty')
file.write('%s = %s\n' %
('A2_GEANT_PATH', self.__settings['A2_GEANT_PATH']))
file.write(
'%s\n' %
'# the A2 Geant binary which should be used, default "A2" if empty'
)
file.write(
'%s\n' %
'# IMPORTANT: if you\'re using the new A2Geant4 version, set this parameter'
)
file.write(
'%s\n' %
'# to "A2Geant4" and specify additional "GeantFlags" further down'
)
file.write('%s = %s\n\n' %
('GEANT_BINARY', self.__settings['GEANT_BINARY']))
file.write('%s\n' % '# some default settings for jobs')
file.write('%s = %s\n' % ('QSUB_BIN', self.__settings['QSUB_BIN']))
file.write(
'%s\n' %
'# mail when job aborts (a), begins (b), ends (e), no mails (n)'
)
file.write('%s = %s\n' %
('QSUB_MAIL', self.__settings['QSUB_MAIL']))
file.write('%s = %s\n' % ('QUEUE', self.__settings['QUEUE']))
file.write('%s = %s\n' % ('WALLTIME', self.__settings['WALLTIME']))
file.write('%s = %s\n' % ('PRIORITY', self.__settings['PRIORITY']))
file.write('%s\n' % '# any additional flags needed for submission')
file.write(
'%s\n' %
'# e.g. to restrict execution hosts to sl7: -l nodes=1:sl7')
file.write('%s = %s\n\n' %
('QSUB_EXTRA', self.__settings['QSUB_EXTRA']))
file.write('%s\n' % '# simulation specific settings')
file.write('%s\n' %
'# the MC generator which should be used, i.e.')
file.write('%s\n' % '# Ant-pluto, Ant-cocktail, or Ant-mcgun')
file.write(
'%s\n' %
'# Note: If you specify a different MC generator, please')
file.write(
'%s\n' %
'# make sure that its output is readable by Geant!')
file.write('%s\n' %
'# Flags can be passed via AddFlags option.')
file.write('%s = %s\n' %
('GENERATOR', self.__settings['GENERATOR']))
file.write(
'%s\n' %
'# leave the generator path blank to use the $PATH variable')
file.write('%s = %s\n' %
('GENERATOR_PATH', self.__settings['GENERATOR_PATH']))
file.write('%s\n' % '# minimum energy of the photon beam')
file.write('%s: %s\n' % ('Emin', self.__settings['Emin']))
file.write('%s\n' % '# maximum energy of the photon beam')
file.write('%s: %s\n' % ('Emax', self.__settings['Emax']))
file.write('%s\n' % '# Ant-cocktail specific settings')
file.write('%s\n' %
'# Define only ONE of the following two settings!')
file.write(
'%s\n' %
'# Setup which should be used, e.g. "Setup_2014_07_EPT_Prod"')
file.write('%s: %s\n' %
('COCKTAIL_SETUP', self.__settings['COCKTAIL_SETUP']))
file.write(
'%s\n' %
'# Binning for the beam energy, min and max energy as defined above'
)
file.write(
'%s: %s\n' %
('COCKTAIL_BINNING', self.__settings['COCKTAIL_BINNING']))
file.write(
'%s\n' %
'# Ant-mcgun specific settings, min and max energy used as defined above'
)
file.write(
'%s\n' %
'# covered theta range in degree in the format min_theta max_theta'
)
file.write('%s: %s\n' %
('GUN_THETA', self.__settings['GUN_THETA']))
file.write('%s\n' % '# opening angle between particles in degree')
file.write('%s: %s\n' %
('GUN_OPENING', self.__settings['GUN_OPENING']))
file.write(
'%s\n' %
'# additional flags passed to runGeant (which calls a2geant), '
'for example regex to replace information in detector macro setup'
)
file.write(
'%s\n' %
"# like 's~^(/A2/det/setTargetLength).*~$1 5 cm~' (using the Ant branch of a2geant4)"
)
file.write(
'%s\n' %
'# in case you\'re using the new A2Geant4 additional flags like the macros you want to use go here'
)
file.write('%s: %s\n\n' %
('GeantFlags', self.__settings['GeantFlags']))
file.write(
'%s\n' %
'# additional flags passed to the generator, for example --flatEbeam (cocktail, pluto) or --flatTheta (mcgun)'
)
file.write('%s: %s\n\n' %
('AddFlags', self.__settings['AddFlags']))
file.write('%s\n' % '[channels]')
file.write(
'%s\n' %
'# channels which should be simulated with Ant-pluto, line has to start with ";", '
'given in the syntax used in Pluto (do not forget the recoil proton!), '
'the amount of files and the number of events per file')
file.write('%s\n' % '#;"p pi0 [g g]" 10 100000')
file.write(
'%s\n' %
'# in case of Ant-cocktail, start the string with "Cocktail", '
'followed by the amount of files and the number of events per file'
)
file.write('%s\n' % '#;"Cocktail" 100 10000')
file.write(
'%s\n' %
'# for the particle gun Ant-mcgun, start the string with "Gun: <particle_list>", '
'particle list separated by spaces, followed by the amount of files '
'and the number of events per file')
file.write('%s\n' % '#;"Gun: g g" 100 10000')
return True
def main():
"""Main function: process all information, prepare simulation process, submit jobs"""
parser = argparse.ArgumentParser(description='Submit simulations on blaster, configuration '
'is done via a config file "sim_settings" in ~ or . '
'and/or a list of command line arguments')
parser.add_argument('-c', '--config', nargs=1, metavar='config_file',
dest='config',# required=True,
type=lambda x: is_valid_file(parser, x),
help='Optional: Specify a custom config file')
parser.add_argument('-o', '--output', nargs=1, metavar='output_directory',
type=lambda x: is_valid_dir(parser, x),
help='Optional: Custom output directory')
parser.add_argument('-g', '--generator', nargs=1, type=str, metavar='MC generator binary',
help='Optional: If the generator binary is not specified in the config '
'or should be overwritten, use this option to specify the binary manually')
parser.add_argument('-p', '--path-generator', nargs=1, type=str, metavar='MC generator binary path',
help='Optional: If the generator binary (e.g. Ant-pluto) can\'t be found '
'in your $PATH variable, use this option to specify the path manually')
parser.add_argument('-l', '--list', nargs='?', const=True,
help='List the amount of existing files per channel '
'in the output directory and exit; if "all" is specified '
'as an optional argument the amount of events will be listed')
parser.add_argument('-e', '--example-config', nargs='?', const='example_settings',
help='Export an example of default settings and exit. '
'If no file is specified, the output will be written to "example_settings"')
parser.add_argument('-a', '--add-flags', nargs=1, type=str, metavar='"Additional flags"',
help='Optional: Define additional flags which will be passed to the '
'MC generator. Flags defined in the settings file will be overwritten. '
'All additional flags must be given as one quoted string!')
parser.add_argument('-t', '--tag', nargs=1, type=str, metavar='Job Tag',
help='Optional: Specify a job tag, default is Sim')
parser.add_argument('-w', '--walltime', type=int, nargs=1, metavar='walltime',
help='Walltime for jobs, time in hours')
parser.add_argument('-q', '--queue', type=str, nargs=1, metavar='queue',
help='Queue which should be used for the jobs')
parser.add_argument('-f', '--force', action='store_true',
help='Force creation of directories if they do not exist '
'or overwrite existing files')
parser.add_argument('-v', '--verbose', action='store_true',
help='Print additional output')
args = parser.parse_args()
verbose = args.verbose
force = args.force
settings = None
if args.example_config:
print_color('[INFO] Write example settings to file "%s"' % args.example_config, 'BLUE')
settings = Settings()
if not settings.export(args.example_config, force):
print_error('[ERROR] Creating example settings "%s" failed.' % args.example_config)
sys.exit(0)
config = None
channels = []
if args.config:
config = args.config[0]
if verbose:
print('Use config file %s' % config)
else:
if verbose:
print('Try to find a default config file')
config = find_config(verbose=verbose)
if not config:
print_color('No config file found, use default values.', 'YELLOW')
settings = Settings()
if config:
print_color('The config file "%s" will be used.' % config, 'BLUE')
if verbose:
print('Read config file %s' % config)
with open(config, 'r') as conf:
settings, channels = read_config(conf)
if args.list:
if verbose and args.list == 'all':
print('Trying to determine the total amount of simulated events')
if args.output:
settings.set('OUTPUT_PATH', get_path(args.output[0]))
list_file_amount(settings, args.list == 'all')
sys.exit(0)
if verbose:
print('The following default settings were determined:')
settings.print()
if channels:
print('The following channels have been found:')
print(channels)
if args.output:
if not check_path(args.output[0], force):
sys.exit('The output directory %s cannot be used' % args.output[0])
output = get_path(args.output[0])
print_color('Setting custom output directory: %s' % output, 'GREEN')
settings.set('OUTPUT_PATH', output)
if args.path_generator:
path_generator = get_path(args.path_generator[0])
print_color('Setting custom path for MC generator %s' % path_generator, 'GREEN')
settings.set('GENERATOR_PATH', path_generator)
if not check_directories(settings, force, verbose):
sys.exit(1)
settings.set('MCGEN_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('MCGEN_DATA')))
settings.set('GEANT_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('GEANT_DATA')))
settings.set('LOG_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('LOG_DATA')))
if args.generator:
generator = args.generator[0]
print_color('Use custom MC generator %s' % generator, 'GREEN')
settings.set('GENERATOR', generator)
if args.add_flags:
print_color('Set custom flags to pass to the MC generator: %s' % args.add_flags[0], 'GREEN')
settings.set('AddFlags', args.add_flags[0])
mc_generator, geant = check_binaries(settings, verbose)
if not mc_generator or not geant:
sys.exit(1)
if not channels:
print_color('[Warning] No channels specified in the config file', 'YELLOW')
print_color(' Use -e to export example settings or -h for help', 'YELLOW')
channels = simulation_dialogue()
if not channels:
sys.exit('No channels entered for simulation, exiting.')
if args.walltime:
print_color('Setting custom walltime to %d hours' % args.walltime[0], 'GREEN')
settings.set('WALLTIME', args.walltime[0])
if args.queue:
print_color('Setting custom queue to %s' % args.queue[0], 'GREEN')
settings.set('QUEUE', args.queue[0])
tag = args.tag[0] if args.tag else 'Sim'
if verbose:
print('Use the tag "%s" for the submitted jobs' % tag)
if verbose:
print('These are the updated settings:')
settings.print()
if 'Ant-cocktail' in mc_generator:
if not sanity_check_cocktail(settings):
sys.exit(1)
if 'Ant-mcgun' in mc_generator:
if not sanity_check_mcgun(settings):
sys.exit(1)
if not sanity_check_channels(mc_generator, channels):
sys.exit(1)
if verbose:
print('Determining the existing amount of files...')
simulation = []
for channel in channels:
decay_string = get_decay_string(channel[0])
max_number = check_simulation_files(settings, decay_string)
simulation.append([decay_string, channel[0], channel[1], channel[2], max_number])
print_color(str(len(simulation)) + ' channels configured. '
'The following simulation will take place:', 'BLUE')
total_files, total_events = 0, 0
for channel, _, files, events, _ in simulation:
total = files*events
chnl = channel
if 'pluto' in mc_generator.lower():
chnl = format_channel(channel)
print("{0:<20s} {1:>4d} files per {2:>4s} events (total {3:>4s} events)"
.format(chnl, files, unit_prefix(events), unit_prefix(total)))
total_files += files
total_events += total
print(" Total %s events in %d files" % (unit_prefix(total_events), total_files))
print(" Files will be stored in " + settings.get('OUTPUT_PATH'))
# run a test job for the first command to be submitted and check the output
print_color('Test provided commands', 'BLUE')
print(' Running first test job locally . . .')
if not run_test_job(settings, simulation, mc_generator, geant):
print_error('[ERROR] Test job failed, aborting job submission')
sys.exit(1)
print_color('Test job successful', 'BLUE')
# start the job submission
print('Start submitting jobs, total', total_files)
submit_jobs(settings, simulation, mc_generator, geant, tag, total_files, verbose)
print_color('Done!', 'GREEN')
