def test_changing_config_values(self):
settings = Settings()
test_val = 'OUTPUT_PATH'
path = settings.get(test_val)
new_path = pjoin(path, 'subdir')
settings.set(test_val, new_path)
self.assertEqual(settings.get(test_val), new_path)
# test exporting value and reading in the change value, too
self.assertTrue(settings.export(self.settings2))
config = None
with open(self.settings2, 'r') as conf:
config, _ = read_config(conf)
self.assertIsNotNone(config)
self.assertEqual(config.get(test_val), new_path)
def test_changing_config_values(self):
settings = Settings()
test_val = 'OUTPUT_PATH'
path = settings.get(test_val)
new_path = pjoin(path, 'subdir')
settings.set(test_val, new_path)
self.assertEqual(settings.get(test_val), new_path)
# test exporting value and reading in the change value, too
self.assertTrue(settings.export(self.settings2))
config = None
with open(self.settings2, 'r') as conf:
config, _ = read_config(conf)
self.assertIsNotNone(config)
self.assertEqual(config.get(test_val), new_path)
def main():
"""Main function: process all information, prepare simulation process, submit jobs"""
parser = argparse.ArgumentParser(description='Submit simulations on blaster, configuration '
'is done via a config file "sim_settings" in ~ or . '
'and/or a list of command line arguments')
parser.add_argument('-c', '--config', nargs=1, metavar='config_file',
dest='config',# required=True,
type=lambda x: is_valid_file(parser, x),
help='Optional: Specify a custom config file')
parser.add_argument('-o', '--output', nargs=1, metavar='output_directory',
type=lambda x: is_valid_dir(parser, x),
help='Optional: Custom output directory')
parser.add_argument('-a', '--ant-pluto', nargs=1, type=str, metavar='Ant-pluto binary path',
help='Optional: If the Ant-pluto binary can\'t be found in your'
'$PATH variable, use this option to specify the path manually')
parser.add_argument('-l', '--list', nargs='?', const=True,
help='List the amount of existing files per channel'
'in the output directory and exit; if "all" is specified'
'as an optional argument the amount of events will be listed')
parser.add_argument('-e', '--example-config', nargs='?', const='example_settings',
help='Export an example of default settings and exit. '
'If no file is specified, the output will be written to "example_settings"')
parser.add_argument('-w', '--walltime', type=int, nargs=1, metavar='walltime',
help='Walltime for jobs, time in hours')
parser.add_argument('-q', '--queue', type=str, nargs=1, metavar='queue',
help='Queue which should be used for the jobs')
parser.add_argument('-f', '--force', action='store_true',
help='Force creation of directories if they do not exist'
'or overwrite existing files')
parser.add_argument('-v', '--verbose', action='store_true',
help='Print additional output')
args = parser.parse_args()
verbose = args.verbose
force = args.force
settings = None
if args.example_config:
print_color('[INFO] Write example settings to file "%s"' % args.example_config, 'BLUE')
settings = Settings()
if not settings.export(args.example_config, force):
print_error('[ERROR] Creating example settings "%s" failed.' % args.example_config)
sys.exit(0)
config = None
channels = []
if args.config:
config = args.config[0]
if verbose:
print('Use config file %s' % config)
else:
if verbose:
print('Try to find a default config file')
config = find_config(verbose=verbose)
if not config:
print_color('No config file found, use default values.', 'YELLOW')
settings = Settings()
if config:
print_color('The config file "%s" will be used.' % config, 'BLUE')
if verbose:
print('Read config file %s' % config)
with open(config, 'r') as conf:
settings, channels = read_config(conf)
if args.list:
if verbose and args.list == 'all':
print('Trying to determine the total amount of simulated events')
if args.output:
settings.set('OUTPUT_PATH', get_path(args.output[0]))
list_file_amount(settings, args.list == 'all')
sys.exit(0)
if verbose:
print('The following settings will be used:')
settings.print()
if channels:
print('The following channels have been found:')
print(channels)
if args.output:
if not check_path(args.output[0], force):
sys.exit('The output directory %s cannot be used' % args.output[0])
output = get_path(args.output[0])
print_color('Setting custom output directory: %s' % output, 'GREEN')
settings.set('OUTPUT_PATH', output)
ant_pluto = ''
if args.ant_pluto:
ant_pluto = args.ant_pluto[0]
print_color('Use custom Ant-pluto path %s' % ant_pluto, 'GREEN')
if not check_directories(settings, ant_pluto, force, verbose):
sys.exit(1)
pluto, tid, geant = check_binaries(settings, ant_pluto, verbose)
if not pluto or not tid or not geant:
sys.exit(1)
if not channels:
print_color('[Warning] No channels specified in the config file', 'YELLOW')
channels = simulation_dialogue()
if not channels:
sys.exit('No channels entered for simulation, exiting.')
if args.walltime:
print_color('Setting custom walltime to %d hours' % args.walltime[0], 'GREEN')
settings.set('WALLTIME', args.walltime[0])
if args.queue:
print_color('Setting custom queue to %s' % args.queue[0], 'GREEN')
settings.set('QUEUE', args.queue[0])
if verbose:
print('This are the updated settings:')
settings.print()
if verbose:
print('Determining the existing amount of files...')
simulation = []
for channel in channels:
decay_string = get_decay_string(channel[0])
max_number = check_simulation_files(settings, decay_string)
simulation.append([decay_string, channel[0], channel[1], channel[2], max_number])
print_color(str(len(simulation)) + ' channels configured. '
'The following simulation will take place:', 'BLUE')
total_files, total_events = 0, 0
for channel, _, files, events, _ in simulation:
total = files*events
print("{0:<20s} {1:>4d} files per {2:>4s} events (total {3:>4s} events)"
.format(format_channel(channel), files, unit_prefix(events), unit_prefix(total)))
total_files += files
total_events += total
print(" Total %s events in %d files" % (unit_prefix(total_events), total_files))
print(" Files will be stored in " + settings.get('OUTPUT_PATH'))
# start the job submission
print('Start submitting jobs, total', total_files)
submit_jobs(settings, simulation, pluto, tid, geant, total_files)
print_color('Done!', 'GREEN')
text_key: str,
out_file: Path,
aug_config: Dict,
original_key: str = 'original'):
with find_free_file(out_file).open("x") as ostream:
# Dump config so it will be easy to know how data was augmented
ostream.write(f"{json.dumps(aug_config)}\n")
for entry in tqdm(data):
# Save original example:
ostream.write(f"{json.dumps(entry)}\n")
example = entry[text_key]
variants = augmenter.augment(example)
if isinstance(variants, str):
variants = [variants]
for var in variants:
new_entry = copy(entry)
new_entry.update({text_key: var, original_key: False})
ostream.write(f"{json.dumps(new_entry)}\n")
if __name__ == '__main__':
args = parse_args()
pipeline = build_augmentation_pipeline(args.config)
config = read_config(args.config)
data = read_original_data(args.input_json)
augment_dataset(augmenter=pipeline,
data=data,
text_key=args.text_key,
out_file=args.output_json,
aug_config=config)
def test_read_config(self):
config = None
with open(self.settings, 'r') as conf:
config, _ = read_config(conf)
self.assertIsNotNone(config)
def test_read_config(self):
config = None
with open(self.settings, 'r') as conf:
config, _ = read_config(conf)
self.assertIsNotNone(config)
return set_clean_up
else:
print ("remove_interval exceeds expected value - stopping now")
exit(1)
if __name__ == "__main__":
config_file = get_argv.get_argv(argv)
list_of_sources = []
news_list = []
try:
setdefaulttimeout(5)
config = parse_config.read_config(config_file)
connection = database.prepare_db(config)
cursor = connection.cursor()
database.check_table(cursor)
for key in config["sources"]:
feed = get_feed(config["sources"][key])
new_list = extract_news(feed, key, config)
news_list.append(check_for_dubs(cursor, new_list, key))
list_of_sources.append(key)
sorted_news = sort_news_list(news_list)
if sorted_news:
append_to_buffer(cursor, sorted_news)
def main():
"""Main function: process all information, prepare simulation process, submit jobs"""
parser = argparse.ArgumentParser(description='Submit simulations on blaster, configuration '
'is done via a config file "sim_settings" in ~ or . '
'and/or a list of command line arguments')
parser.add_argument('-c', '--config', nargs=1, metavar='config_file',
dest='config',# required=True,
type=lambda x: is_valid_file(parser, x),
help='Optional: Specify a custom config file')
parser.add_argument('-o', '--output', nargs=1, metavar='output_directory',
type=lambda x: is_valid_dir(parser, x),
help='Optional: Custom output directory')
parser.add_argument('-g', '--generator', nargs=1, type=str, metavar='MC generator binary',
help='Optional: If the generator binary is not specified in the config '
'or should be overwritten, use this option to specify the binary manually')
parser.add_argument('-p', '--path-generator', nargs=1, type=str, metavar='MC generator binary path',
help='Optional: If the generator binary (e.g. Ant-pluto) can\'t be found '
'in your $PATH variable, use this option to specify the path manually')
parser.add_argument('-l', '--list', nargs='?', const=True,
help='List the amount of existing files per channel '
'in the output directory and exit; if "all" is specified '
'as an optional argument the amount of events will be listed')
parser.add_argument('-e', '--example-config', nargs='?', const='example_settings',
help='Export an example of default settings and exit. '
'If no file is specified, the output will be written to "example_settings"')
parser.add_argument('-a', '--add-flags', nargs=1, type=str, metavar='"Additional flags"',
help='Optional: Define additional flags which will be passed to the '
'MC generator. Flags defined in the settings file will be overwritten. '
'All additional flags must be given as one quoted string!')
parser.add_argument('-t', '--tag', nargs=1, type=str, metavar='Job Tag',
help='Optional: Specify a job tag, default is Sim')
parser.add_argument('-w', '--walltime', type=int, nargs=1, metavar='walltime',
help='Walltime for jobs, time in hours')
parser.add_argument('-q', '--queue', type=str, nargs=1, metavar='queue',
help='Queue which should be used for the jobs')
parser.add_argument('-f', '--force', action='store_true',
help='Force creation of directories if they do not exist '
'or overwrite existing files')
parser.add_argument('-v', '--verbose', action='store_true',
help='Print additional output')
args = parser.parse_args()
verbose = args.verbose
force = args.force
settings = None
if args.example_config:
print_color('[INFO] Write example settings to file "%s"' % args.example_config, 'BLUE')
settings = Settings()
if not settings.export(args.example_config, force):
print_error('[ERROR] Creating example settings "%s" failed.' % args.example_config)
sys.exit(0)
config = None
channels = []
if args.config:
config = args.config[0]
if verbose:
print('Use config file %s' % config)
else:
if verbose:
print('Try to find a default config file')
config = find_config(verbose=verbose)
if not config:
print_color('No config file found, use default values.', 'YELLOW')
settings = Settings()
if config:
print_color('The config file "%s" will be used.' % config, 'BLUE')
if verbose:
print('Read config file %s' % config)
with open(config, 'r') as conf:
settings, channels = read_config(conf)
if args.list:
if verbose and args.list == 'all':
print('Trying to determine the total amount of simulated events')
if args.output:
settings.set('OUTPUT_PATH', get_path(args.output[0]))
list_file_amount(settings, args.list == 'all')
sys.exit(0)
if verbose:
print('The following default settings were determined:')
settings.print()
if channels:
print('The following channels have been found:')
print(channels)
if args.output:
if not check_path(args.output[0], force):
sys.exit('The output directory %s cannot be used' % args.output[0])
output = get_path(args.output[0])
print_color('Setting custom output directory: %s' % output, 'GREEN')
settings.set('OUTPUT_PATH', output)
if args.path_generator:
path_generator = get_path(args.path_generator[0])
print_color('Setting custom path for MC generator %s' % path_generator, 'GREEN')
settings.set('GENERATOR_PATH', path_generator)
if not check_directories(settings, force, verbose):
sys.exit(1)
settings.set('MCGEN_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('MCGEN_DATA')))
settings.set('GEANT_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('GEANT_DATA')))
settings.set('LOG_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('LOG_DATA')))
if args.generator:
generator = args.generator[0]
print_color('Use custom MC generator %s' % generator, 'GREEN')
settings.set('GENERATOR', generator)
if args.add_flags:
print_color('Set custom flags to pass to the MC generator: %s' % args.add_flags[0], 'GREEN')
settings.set('AddFlags', args.add_flags[0])
mc_generator, geant = check_binaries(settings, verbose)
if not mc_generator or not geant:
sys.exit(1)
if not channels:
print_color('[Warning] No channels specified in the config file', 'YELLOW')
print_color(' Use -e to export example settings or -h for help', 'YELLOW')
channels = simulation_dialogue()
if not channels:
sys.exit('No channels entered for simulation, exiting.')
if args.walltime:
print_color('Setting custom walltime to %d hours' % args.walltime[0], 'GREEN')
settings.set('WALLTIME', args.walltime[0])
if args.queue:
print_color('Setting custom queue to %s' % args.queue[0], 'GREEN')
settings.set('QUEUE', args.queue[0])
tag = args.tag[0] if args.tag else 'Sim'
if verbose:
print('Use the tag "%s" for the submitted jobs' % tag)
if verbose:
print('These are the updated settings:')
settings.print()
if 'Ant-cocktail' in mc_generator:
if not sanity_check_cocktail(settings):
sys.exit(1)
if 'Ant-mcgun' in mc_generator:
if not sanity_check_mcgun(settings):
sys.exit(1)
if not sanity_check_channels(mc_generator, channels):
sys.exit(1)
if verbose:
print('Determining the existing amount of files...')
simulation = []
for channel in channels:
decay_string = get_decay_string(channel[0])
max_number = check_simulation_files(settings, decay_string)
simulation.append([decay_string, channel[0], channel[1], channel[2], max_number])
print_color(str(len(simulation)) + ' channels configured. '
'The following simulation will take place:', 'BLUE')
total_files, total_events = 0, 0
for channel, _, files, events, _ in simulation:
total = files*events
chnl = channel
if 'pluto' in mc_generator.lower():
chnl = format_channel(channel)
print("{0:<20s} {1:>4d} files per {2:>4s} events (total {3:>4s} events)"
.format(chnl, files, unit_prefix(events), unit_prefix(total)))
total_files += files
total_events += total
print(" Total %s events in %d files" % (unit_prefix(total_events), total_files))
print(" Files will be stored in " + settings.get('OUTPUT_PATH'))
# run a test job for the first command to be submitted and check the output
print_color('Test provided commands', 'BLUE')
print(' Running first test job locally . . .')
if not run_test_job(settings, simulation, mc_generator, geant):
print_error('[ERROR] Test job failed, aborting job submission')
sys.exit(1)
print_color('Test job successful', 'BLUE')
# start the job submission
print('Start submitting jobs, total', total_files)
submit_jobs(settings, simulation, mc_generator, geant, tag, total_files, verbose)
print_color('Done!', 'GREEN')
