def test_changing_config_values(self): settings = Settings() test_val = 'OUTPUT_PATH' path = settings.get(test_val) new_path = pjoin(path, 'subdir') settings.set(test_val, new_path) self.assertEqual(settings.get(test_val), new_path) # test exporting value and reading in the change value, too self.assertTrue(settings.export(self.settings2)) config = None with open(self.settings2, 'r') as conf: config, _ = read_config(conf) self.assertIsNotNone(config) self.assertEqual(config.get(test_val), new_path)
def main(): """Main function: process all information, prepare simulation process, submit jobs""" parser = argparse.ArgumentParser(description='Submit simulations on blaster, configuration ' 'is done via a config file "sim_settings" in ~ or . ' 'and/or a list of command line arguments') parser.add_argument('-c', '--config', nargs=1, metavar='config_file', dest='config',# required=True, type=lambda x: is_valid_file(parser, x), help='Optional: Specify a custom config file') parser.add_argument('-o', '--output', nargs=1, metavar='output_directory', type=lambda x: is_valid_dir(parser, x), help='Optional: Custom output directory') parser.add_argument('-a', '--ant-pluto', nargs=1, type=str, metavar='Ant-pluto binary path', help='Optional: If the Ant-pluto binary can\'t be found in your' '$PATH variable, use this option to specify the path manually') parser.add_argument('-l', '--list', nargs='?', const=True, help='List the amount of existing files per channel' 'in the output directory and exit; if "all" is specified' 'as an optional argument the amount of events will be listed') parser.add_argument('-e', '--example-config', nargs='?', const='example_settings', help='Export an example of default settings and exit. ' 'If no file is specified, the output will be written to "example_settings"') parser.add_argument('-w', '--walltime', type=int, nargs=1, metavar='walltime', help='Walltime for jobs, time in hours') parser.add_argument('-q', '--queue', type=str, nargs=1, metavar='queue', help='Queue which should be used for the jobs') parser.add_argument('-f', '--force', action='store_true', help='Force creation of directories if they do not exist' 'or overwrite existing files') parser.add_argument('-v', '--verbose', action='store_true', help='Print additional output') args = parser.parse_args() verbose = args.verbose force = args.force settings = None if args.example_config: print_color('[INFO] Write example settings to file "%s"' % args.example_config, 'BLUE') settings = Settings() if not settings.export(args.example_config, force): print_error('[ERROR] Creating example settings "%s" failed.' % args.example_config) sys.exit(0) config = None channels = [] if args.config: config = args.config[0] if verbose: print('Use config file %s' % config) else: if verbose: print('Try to find a default config file') config = find_config(verbose=verbose) if not config: print_color('No config file found, use default values.', 'YELLOW') settings = Settings() if config: print_color('The config file "%s" will be used.' % config, 'BLUE') if verbose: print('Read config file %s' % config) with open(config, 'r') as conf: settings, channels = read_config(conf) if args.list: if verbose and args.list == 'all': print('Trying to determine the total amount of simulated events') if args.output: settings.set('OUTPUT_PATH', get_path(args.output[0])) list_file_amount(settings, args.list == 'all') sys.exit(0) if verbose: print('The following settings will be used:') settings.print() if channels: print('The following channels have been found:') print(channels) if args.output: if not check_path(args.output[0], force): sys.exit('The output directory %s cannot be used' % args.output[0]) output = get_path(args.output[0]) print_color('Setting custom output directory: %s' % output, 'GREEN') settings.set('OUTPUT_PATH', output) ant_pluto = '' if args.ant_pluto: ant_pluto = args.ant_pluto[0] print_color('Use custom Ant-pluto path %s' % ant_pluto, 'GREEN') if not check_directories(settings, ant_pluto, force, verbose): sys.exit(1) pluto, tid, geant = check_binaries(settings, ant_pluto, verbose) if not pluto or not tid or not geant: sys.exit(1) if not channels: print_color('[Warning] No channels specified in the config file', 'YELLOW') channels = simulation_dialogue() if not channels: sys.exit('No channels entered for simulation, exiting.') if args.walltime: print_color('Setting custom walltime to %d hours' % args.walltime[0], 'GREEN') settings.set('WALLTIME', args.walltime[0]) if args.queue: print_color('Setting custom queue to %s' % args.queue[0], 'GREEN') settings.set('QUEUE', args.queue[0]) if verbose: print('This are the updated settings:') settings.print() if verbose: print('Determining the existing amount of files...') simulation = [] for channel in channels: decay_string = get_decay_string(channel[0]) max_number = check_simulation_files(settings, decay_string) simulation.append([decay_string, channel[0], channel[1], channel[2], max_number]) print_color(str(len(simulation)) + ' channels configured. ' 'The following simulation will take place:', 'BLUE') total_files, total_events = 0, 0 for channel, _, files, events, _ in simulation: total = files*events print("{0:<20s} {1:>4d} files per {2:>4s} events (total {3:>4s} events)" .format(format_channel(channel), files, unit_prefix(events), unit_prefix(total))) total_files += files total_events += total print(" Total %s events in %d files" % (unit_prefix(total_events), total_files)) print(" Files will be stored in " + settings.get('OUTPUT_PATH')) # start the job submission print('Start submitting jobs, total', total_files) submit_jobs(settings, simulation, pluto, tid, geant, total_files) print_color('Done!', 'GREEN')
text_key: str, out_file: Path, aug_config: Dict, original_key: str = 'original'): with find_free_file(out_file).open("x") as ostream: # Dump config so it will be easy to know how data was augmented ostream.write(f"{json.dumps(aug_config)}\n") for entry in tqdm(data): # Save original example: ostream.write(f"{json.dumps(entry)}\n") example = entry[text_key] variants = augmenter.augment(example) if isinstance(variants, str): variants = [variants] for var in variants: new_entry = copy(entry) new_entry.update({text_key: var, original_key: False}) ostream.write(f"{json.dumps(new_entry)}\n") if __name__ == '__main__': args = parse_args() pipeline = build_augmentation_pipeline(args.config) config = read_config(args.config) data = read_original_data(args.input_json) augment_dataset(augmenter=pipeline, data=data, text_key=args.text_key, out_file=args.output_json, aug_config=config)
def test_read_config(self): config = None with open(self.settings, 'r') as conf: config, _ = read_config(conf) self.assertIsNotNone(config)
return set_clean_up else: print ("remove_interval exceeds expected value - stopping now") exit(1) if __name__ == "__main__": config_file = get_argv.get_argv(argv) list_of_sources = [] news_list = [] try: setdefaulttimeout(5) config = parse_config.read_config(config_file) connection = database.prepare_db(config) cursor = connection.cursor() database.check_table(cursor) for key in config["sources"]: feed = get_feed(config["sources"][key]) new_list = extract_news(feed, key, config) news_list.append(check_for_dubs(cursor, new_list, key)) list_of_sources.append(key) sorted_news = sort_news_list(news_list) if sorted_news: append_to_buffer(cursor, sorted_news)
def main(): """Main function: process all information, prepare simulation process, submit jobs""" parser = argparse.ArgumentParser(description='Submit simulations on blaster, configuration ' 'is done via a config file "sim_settings" in ~ or . ' 'and/or a list of command line arguments') parser.add_argument('-c', '--config', nargs=1, metavar='config_file', dest='config',# required=True, type=lambda x: is_valid_file(parser, x), help='Optional: Specify a custom config file') parser.add_argument('-o', '--output', nargs=1, metavar='output_directory', type=lambda x: is_valid_dir(parser, x), help='Optional: Custom output directory') parser.add_argument('-g', '--generator', nargs=1, type=str, metavar='MC generator binary', help='Optional: If the generator binary is not specified in the config ' 'or should be overwritten, use this option to specify the binary manually') parser.add_argument('-p', '--path-generator', nargs=1, type=str, metavar='MC generator binary path', help='Optional: If the generator binary (e.g. Ant-pluto) can\'t be found ' 'in your $PATH variable, use this option to specify the path manually') parser.add_argument('-l', '--list', nargs='?', const=True, help='List the amount of existing files per channel ' 'in the output directory and exit; if "all" is specified ' 'as an optional argument the amount of events will be listed') parser.add_argument('-e', '--example-config', nargs='?', const='example_settings', help='Export an example of default settings and exit. ' 'If no file is specified, the output will be written to "example_settings"') parser.add_argument('-a', '--add-flags', nargs=1, type=str, metavar='"Additional flags"', help='Optional: Define additional flags which will be passed to the ' 'MC generator. Flags defined in the settings file will be overwritten. ' 'All additional flags must be given as one quoted string!') parser.add_argument('-t', '--tag', nargs=1, type=str, metavar='Job Tag', help='Optional: Specify a job tag, default is Sim') parser.add_argument('-w', '--walltime', type=int, nargs=1, metavar='walltime', help='Walltime for jobs, time in hours') parser.add_argument('-q', '--queue', type=str, nargs=1, metavar='queue', help='Queue which should be used for the jobs') parser.add_argument('-f', '--force', action='store_true', help='Force creation of directories if they do not exist ' 'or overwrite existing files') parser.add_argument('-v', '--verbose', action='store_true', help='Print additional output') args = parser.parse_args() verbose = args.verbose force = args.force settings = None if args.example_config: print_color('[INFO] Write example settings to file "%s"' % args.example_config, 'BLUE') settings = Settings() if not settings.export(args.example_config, force): print_error('[ERROR] Creating example settings "%s" failed.' % args.example_config) sys.exit(0) config = None channels = [] if args.config: config = args.config[0] if verbose: print('Use config file %s' % config) else: if verbose: print('Try to find a default config file') config = find_config(verbose=verbose) if not config: print_color('No config file found, use default values.', 'YELLOW') settings = Settings() if config: print_color('The config file "%s" will be used.' % config, 'BLUE') if verbose: print('Read config file %s' % config) with open(config, 'r') as conf: settings, channels = read_config(conf) if args.list: if verbose and args.list == 'all': print('Trying to determine the total amount of simulated events') if args.output: settings.set('OUTPUT_PATH', get_path(args.output[0])) list_file_amount(settings, args.list == 'all') sys.exit(0) if verbose: print('The following default settings were determined:') settings.print() if channels: print('The following channels have been found:') print(channels) if args.output: if not check_path(args.output[0], force): sys.exit('The output directory %s cannot be used' % args.output[0]) output = get_path(args.output[0]) print_color('Setting custom output directory: %s' % output, 'GREEN') settings.set('OUTPUT_PATH', output) if args.path_generator: path_generator = get_path(args.path_generator[0]) print_color('Setting custom path for MC generator %s' % path_generator, 'GREEN') settings.set('GENERATOR_PATH', path_generator) if not check_directories(settings, force, verbose): sys.exit(1) settings.set('MCGEN_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('MCGEN_DATA'))) settings.set('GEANT_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('GEANT_DATA'))) settings.set('LOG_DATA', get_path(settings.get('OUTPUT_PATH'), settings.get('LOG_DATA'))) if args.generator: generator = args.generator[0] print_color('Use custom MC generator %s' % generator, 'GREEN') settings.set('GENERATOR', generator) if args.add_flags: print_color('Set custom flags to pass to the MC generator: %s' % args.add_flags[0], 'GREEN') settings.set('AddFlags', args.add_flags[0]) mc_generator, geant = check_binaries(settings, verbose) if not mc_generator or not geant: sys.exit(1) if not channels: print_color('[Warning] No channels specified in the config file', 'YELLOW') print_color(' Use -e to export example settings or -h for help', 'YELLOW') channels = simulation_dialogue() if not channels: sys.exit('No channels entered for simulation, exiting.') if args.walltime: print_color('Setting custom walltime to %d hours' % args.walltime[0], 'GREEN') settings.set('WALLTIME', args.walltime[0]) if args.queue: print_color('Setting custom queue to %s' % args.queue[0], 'GREEN') settings.set('QUEUE', args.queue[0]) tag = args.tag[0] if args.tag else 'Sim' if verbose: print('Use the tag "%s" for the submitted jobs' % tag) if verbose: print('These are the updated settings:') settings.print() if 'Ant-cocktail' in mc_generator: if not sanity_check_cocktail(settings): sys.exit(1) if 'Ant-mcgun' in mc_generator: if not sanity_check_mcgun(settings): sys.exit(1) if not sanity_check_channels(mc_generator, channels): sys.exit(1) if verbose: print('Determining the existing amount of files...') simulation = [] for channel in channels: decay_string = get_decay_string(channel[0]) max_number = check_simulation_files(settings, decay_string) simulation.append([decay_string, channel[0], channel[1], channel[2], max_number]) print_color(str(len(simulation)) + ' channels configured. ' 'The following simulation will take place:', 'BLUE') total_files, total_events = 0, 0 for channel, _, files, events, _ in simulation: total = files*events chnl = channel if 'pluto' in mc_generator.lower(): chnl = format_channel(channel) print("{0:<20s} {1:>4d} files per {2:>4s} events (total {3:>4s} events)" .format(chnl, files, unit_prefix(events), unit_prefix(total))) total_files += files total_events += total print(" Total %s events in %d files" % (unit_prefix(total_events), total_files)) print(" Files will be stored in " + settings.get('OUTPUT_PATH')) # run a test job for the first command to be submitted and check the output print_color('Test provided commands', 'BLUE') print(' Running first test job locally . . .') if not run_test_job(settings, simulation, mc_generator, geant): print_error('[ERROR] Test job failed, aborting job submission') sys.exit(1) print_color('Test job successful', 'BLUE') # start the job submission print('Start submitting jobs, total', total_files) submit_jobs(settings, simulation, mc_generator, geant, tag, total_files, verbose) print_color('Done!', 'GREEN')