def main(): """ The main entry point, as specified in the ``setup.py`` file. Adds commands from other subsidiary entry points (specified in the ``commands`` variable above,) and then uses ``arch.dispatch()`` to start the process. The ``RuntimeStateConfig()`` object is created here and handed to the parser as the object that will recive all command line data, rather than using a standard argparse namespace object. This allows all runtime argument parsing to happen inside of these config objects rather than spread among all of the entry points. This function catches and recovers from :exc:`KeyboardInterupt` which means that doesn't dump a stack trace following a Control-C. """ parser = get_base_parser() for namespace, entry_points in commands.items(): if namespace == 'main': argh.add_commands(parser, entry_points) else: argh.add_commands(parser, entry_points, namespace=namespace) args = RuntimeStateConfig() try: argh.dispatch(parser, namespace=args) except KeyboardInterrupt: logger.error('operation interrupted by user.')
def main(): signal.signal(signal.SIGINT, exit_gracefully) parser = argh.ArghParser() parser.add_commands(one_level_commands) for command in two_level_commands: argh.add_commands(parser, command.subcommands(), namespace=command.namespace(), help=command.help_message()) parser.dispatch()
def main(): parser = argh.ArghParser() parser.add_commands(one_level_commands) for command in two_level_commands: argh.add_commands(parser, command.subcommands(), namespace=command.namespace(), help=command.help_message()) parser.dispatch()
def main(): log.debug(' '.join(sys.argv)) parser = argh.helpers.ArghParser() argh.add_commands(parser, [print_installation_commands, install, install_full, list_data_sources, list_output, log_info, copy_files, version]) argh.dispatch(parser)
def add_commands(arg_parser, functions, config, section='DEFAULT', prefix=False, **kwargs): """ Adds given functions as commands to given parser and also sets default values from given config. Parameters ---------- arg_parser : ArghParser or ArgumentParser argument parser functions : list list of functions config : ConfigParser configuration, as read from config file(s) section : str section in configuration prefix: bool prefix argument with function name to create unique namespace Notes ----- Current version fails with variable number of paramters (*args) or keyword arguments (**kwargs) """ _inject_defaults(functions, config, section, prefix) argh.add_commands(arg_parser, functions, **kwargs)
def main(): parser = argparse.ArgumentParser( description='Tools for %s.' % ssh.HOSTNAME, ) argh.add_commands(parser, [update, proxy]) git.namespace.add_subcommands(parser) argh.dispatch(parser)
def add_subcommands(self, parser): argh.add_commands( parser, self.commands, namespace=self.name, namespace_kwargs=self.parser_kwargs )
def main(): """ Set up the context and connectors """ try: init() except custom_exceptions.NotConfigured: configure() init() # Adding this in case users are trying to run without adding a jira url. # I would like to take this out in a release or two. # TODO: REMOVE except (AttributeError, ConfigParser.NoOptionError): logging.error('It appears that your configuration is invalid, please reconfigure the app and try again.') configure() init() parser = argparse.ArgumentParser() # Now simply auto-discovering the methods listed in this module current_module = sys.modules[__name__] module_methods = [getattr(current_module, a, None) for a in dir(current_module) if isinstance(getattr(current_module, a, None), types.FunctionType) and a != 'main'] argh.add_commands(parser, module_methods) # Putting the error logging after the app is initialized because # we want to adhere to the user's preferences try: argh.dispatch(parser) # We don't want to report keyboard interrupts to rollbar except (KeyboardInterrupt, SystemExit): raise except Exception as e: if isinstance(e, jira.exceptions.JIRAError) and "HTTP 400" in e: logging.warning('It appears that your authentication with {0} is invalid. Please re-configure jtime: `jtime configure` with the correct credentials'.format(configuration.load_config['jira'].get('url'))) elif configured.get('jira').get('error_reporting', True): # Configure rollbar so that we report errors import rollbar from . import __version__ as version root_path = os.path.dirname(os.path.realpath(__file__)) rollbar.init('7541b8e188044831b6728fa8475eab9f', 'v%s' % version, root=root_path) logging.error('Sorry. It appears that there was an error when handling your command. ' 'This error has been reported to our error tracking system. To disable ' 'this reporting, please re-configure the app: `jtime config`.') extra_data = { # grab the command that we're running 'cmd': sys.argv[1], # we really don't want to see jtime in the args 'args': sys.argv[2:], # lets grab anything useful, python version? 'python': str(sys.version), } # We really shouldn't thit this line of code when running tests, so let's not cover it. rollbar.report_exc_info(extra_data=extra_data) # pragma: no cover else: logging.error('It appears that there was an error when handling your command.') raise
def main(): parser = get_base_parser() commands = [ mongo_to_po, po_to_mongo, verifier, ] argh.add_commands(parser, commands) args = RuntimeStateConfig() argh.dispatch(parser, namespace=args)
def add_to(cls, manager): ''' Add all public instance methods of this class to the given Manager as commands. ''' commands = cls(manager) argh.add_commands(manager.parser, [ getattr(commands, name) for name in dir(commands) if not name.startswith('_') and is_instance_method(getattr(commands, name)) ])
def main(): """ The main entry point, as specified in the ``setup.py`` file. Adds commands from other subsidiary entry points (specified in the ``commands`` variable above,) and then uses ``arch.dispatch()`` to start the process. The "DaggerConfig" object allows the application to delegate all argument parsing validation using setters and getters in the object where arg(h)parse stores the configuration data. This function catches and recovers from :exc:`KeyboardInterupt` which means that doesn't dump a stack trace following a Control-C. """ parser = get_base_parser() for namespace, entry_points in commands.items(): if namespace == 'main': argh.add_commands(parser, entry_points) else: argh.add_commands(parser, entry_points, namespace=namespace) args = dagger.config.cli.DaggerCliConfig() # set the logging level early to ensure logging is configured during # argument parsing. args.level = "info" # run the command, catching user-interruption and common error types directly. try: argh.dispatch(parser, namespace=args) except KeyboardInterrupt: logger.error('operation interrupted by user.') exit(1) except RuntimeError: logger.error("exiting due to a runtime error") exit(1) except (ValueError, TypeError, AttributeError) as e: # catch data access and validation errors, and, in common operation, # suppress the traceback, unless logging at debug level. logger.info("error: {0}, type: {1}".format(e, type(e))) tb = traceback.format_exc() err = libgiza.error.Error( message=("encountered data validation or access " "error during normal operation."), fatal=True, include_trace=True) err.payload = {"type": type(e), "error": e, "trace": tb} logger.debug(err.render_output()) logger.debug("exception traceback: \n" + tb) exit(1)
def main(): parser = get_base_parser() commands = [mine, stats, actions, setup] argh.add_commands(parser, commands) args = GithubRuntimeConfig() if args.level == 'info': args.level = 'warning' argh.dispatch(parser, namespace=args)
def main(): parser = get_base_parser() commands = [setup, setup_credential_file, config, progress, triage, make_versions, mirror_version, release] argh.add_commands(parser, commands) args = JeerahRuntimeStateConfig() if args.level == "info": args.level = "warning" argh.dispatch(parser, namespace=args)
def main(): """ The main entry point, as specified in the ``setup.py`` file. Adds commands from other subsidiary entry points (specified in the ``commands`` variable above,) and then uses ``arch.dispatch()`` to start the process. The "DaggerConfig" object allows the application to delegate all argument parsing validation using setters and getters in the object where arg(h)parse stores the configuration data. This function catches and recovers from :exc:`KeyboardInterupt` which means that doesn't dump a stack trace following a Control-C. """ parser = get_base_parser() for namespace, entry_points in commands.items(): if namespace == 'main': argh.add_commands(parser, entry_points) else: argh.add_commands(parser, entry_points, namespace=namespace) args = dagger.config.cli.DaggerCliConfig() # set the logging level early to ensure logging is configured during # argument parsing. args.level = "info" # run the command, catching user-interruption and common error types directly. try: argh.dispatch(parser, namespace=args) except KeyboardInterrupt: logger.error('operation interrupted by user.') exit(1) except RuntimeError: logger.error("exiting due to a runtime error") exit(1) except (ValueError, TypeError, AttributeError) as e: # catch data access and validation errors, and, in common operation, # suppress the traceback, unless logging at debug level. logger.info("error: {0}, type: {1}".format(e, type(e))) tb = traceback.format_exc() err = libgiza.error.Error(message=("encountered data validation or access " "error during normal operation."), fatal=True, include_trace=True) err.payload = {"type": type(e), "error": e, "trace": tb} logger.debug(err.render_output()) logger.debug("exception traceback: \n" + tb) exit(1)
def main(): parser = get_base_parser() for namespace, entry_points in commands.items(): if namespace == 'main': argh.add_commands(parser, entry_points) else: argh.add_commands(parser, entry_points, namespace=namespace) args = RuntimeStateConfig() try: argh.dispatch(parser, namespace=args) except KeyboardInterrupt: logger.error('operation interrupted by user.')
def main(): parser = get_base_parser() commands = [ setup, setup_credential_file, config, progress, triage, make_versions, mirror_version, release ] argh.add_commands(parser, commands) args = JeerahRuntimeStateConfig() if args.level == 'info': args.level = 'warning' argh.dispatch(parser, namespace=args)
def main(): parser = get_base_parser() commands = [mine, stats, actions, setup] argh.add_commands(parser, commands) args = GithubRuntimeConfig() if args.level == 'info': args.level = 'warning' if args.runner == 'process': logger.warning('this operation does not support multiprocessing, falling back to threads') args.runner = 'thread' argh.dispatch(parser, namespace=args)
def main(): parser = get_base_parser() commands = [mine, stats, actions, setup] argh.add_commands(parser, commands) args = GithubRuntimeConfig() if args.level == 'info': args.level = 'warning' if args.runner == 'process': logger.warning( 'this operation does not support multiprocessing, falling back to threads' ) args.runner = 'thread' argh.dispatch(parser, namespace=args)
def main(root_pkg, argv=None): """Invokes module functions in :mod:`pykern.pkcli` Looks in ``<root_pkg>.pkcli`` for the ``argv[1]`` module. It then invokes the ``argv[2]`` method of that module. Args: root_pkg (str): top level package name argv (list of str): Defaults to `sys.argv`. Only used for testing. Returns: int: 0 if ok. 1 if error (missing command, etc.) """ pkconfig.append_load_path(root_pkg) if not argv: argv = list(sys.argv) prog = os.path.basename(argv.pop(0)) if _is_help(argv): return _list_all(root_pkg, prog) module_name = argv.pop(0) cli = _module(root_pkg, module_name) if not cli: return 1 prog = prog + ' ' + module_name parser = argparse.ArgumentParser( prog=prog, formatter_class=argh.PARSER_FORMATTER) cmds = _commands(cli) dc = _default_command(cmds, argv) if dc: argh.set_default_command(parser, dc) else: argh.add_commands(parser, cmds) if len(argv) < 1: # Python 3: parser doesn't exit if not enough commands parser.error('too few arguments') if argv[0][0] != '-': argv[0] = _module_to_cmd(argv[0]) from pykern.pkdebug import pkdp try: res = argh.dispatch(parser, argv=argv) except CommandError as e: sys.stderr.write('error: {}\n'.format(e)) return 1 return 0
def main(): errors = StringIO() parser = argh.ArghParser() argh.add_commands(parser, [ source, run, run_piped, list_registered, source_registered, source_named, source_def, source_inline, update_env ], ) argh.add_commands(parser, functions=DaemonCommands.commands(), namespace=DaemonCommands.namespace, title=DaemonCommands.__doc__) argh.dispatch(parser, completion=False, errors_file=errors) if errors.len > 0: sys.exit(errors.getvalue().strip())
def main(): github_subcommands = [ migration.create_repositories, migration.delete_repositories, migration.edit_repositories, migration.import_repositories, migration.import_attachments, migration.import_issues, migration.import_issues_for_project ] ow2_subcommands = [ migration.clone_repositories, migration.gc_repositories, migration.prune_repositories ] parser = argparse.ArgumentParser() argh.add_commands(parser, github_subcommands, namespace='github') argh.add_commands(parser, ow2_subcommands, namespace='ow2') argh.dispatch(parser)
def main(root_pkg, argv=None): """Invokes module functions in :mod:`pykern.pykern_cli` Looks in ``<root_pkg>.pykern_cli`` for the ``argv[1]`` module. It then invokes the ``argv[2]`` method of that module. Args: root_pkg (str): top level package name argv (list of str): Defaults to `sys.argv`. Only used for testing. Returns: int: 0 if ok. 1 if error (missing command, etc.) """ pkconfig.append_load_path(root_pkg) if not argv: argv = list(sys.argv) prog = os.path.basename(argv.pop(0)) if _is_help(argv): return _list_all(root_pkg, prog) module_name = argv.pop(0) cli = _module(root_pkg, module_name) if not cli: return 1 prog = prog + ' ' + module_name parser = argparse.ArgumentParser( prog=prog, formatter_class=argh.PARSER_FORMATTER) cmds = _commands(cli) dc = _default_command(cmds, argv) if dc: argh.set_default_command(parser, dc) else: argh.add_commands(parser, cmds) if len(argv) < 1: # Python 3: parser doesn't exit if not enough commands parser.error('too few arguments') if argv[0][0] != '-': argv[0] = argv[0].replace('_', '-') argh.dispatch(parser, argv=argv) return 0
capacity=batch_size * 64, min_after_dequeue=batch_size * 32, allow_smaller_final_batch=False) logger.debug('x shape: {}, y shape: {}'.format(x.get_shape(), y.get_shape())) # 初始化所有的op init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(tf.local_variables_initializer()) sess.run(init) # 启动队列 coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) for i in range(2): val, l = sess.run([x, y]) # l = to_categorical(l, 12) print(val, l) coord.join() logger.debug('Test read tf record Succeed') parser = argparse.ArgumentParser() argh.add_commands(parser, [tfrecord, test]) if __name__ == "__main__": argh.dispatch(parser)
ei = np.array(ei) - e_min #e_min = min(ei_) #ei_ = np.array(ei_) - e_min plt.figure() plt.ylabel(r'$Total$ $Energy$ ($eV$)') plt.xlabel(r'$MD$ $Step$') # plt.xlim(0,i) # plt.ylim(0,np.max(hist)+0.01) plt.plot(ei,alpha=0.9, linestyle='-',# marker='o',markerfacecolor='k',markersize=5, color='k',label='ReaxFF-MPNN') # plt.plot(v,ei,alpha=0.9, # linestyle='-',marker='^',markerfacecolor='none', # markeredgewidth=1,markeredgecolor='blue',markersize=5, # color='blue',label='IRFF') # plt.text( 0.0, e_max, '%.3f' %e_min, fontdict={'size':10.5, 'color': 'k'}) plt.legend(loc='best',edgecolor='yellowgreen') # lower left upper right plt.savefig('Energy.pdf',transparent=True) plt.close() if __name__ == '__main__': ''' use commond like ./cp.py scale-md --T=2800 to run it''' parser = argparse.ArgumentParser() argh.add_commands(parser, [e]) argh.dispatch(parser)
batch_size = 8 x, y = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=batch_size * 64, min_after_dequeue=batch_size * 32, allow_smaller_final_batch=False) logger.debug('x shape: {}, y shape: {}'.format(x.get_shape(), y.get_shape())) # 初始化所有的op init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(tf.local_variables_initializer()) sess.run(init) # 启动队列 coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) for i in range(2): val, l = sess.run([x, y]) # l = to_categorical(l, 12) print(val, l) coord.join() logger.debug('Test read tf record Succeed') parser = argparse.ArgumentParser() argh.add_commands(parser, [tfrecord, test]) if __name__ == "__main__": argh.dispatch(parser)
buf = ExampleBuffer(positions, samples_per_game=samples_per_game) files = [] for _, model in sorted(models, reverse=True): local_model_dir = os.path.join(local_dir, model) if not tf.gfile.Exists(local_model_dir): print("Rsyncing", model) _rsync_dir(os.path.join(game_dir, model), local_model_dir) files.extend(tf.gfile.Glob(os.path.join(local_model_dir, '*.zz'))) print("{}: {} games".format(model, len(files))) if len(files) * samples_per_game > positions: break print("Filling from {} files".format(len(files))) buf.parallel_fill(files, threads=threads) print(buf) output = os.path.join(output_dir, str(model_num) + '.tfrecord.zz') print("Writing to", output) buf.flush(output) parser = argparse.ArgumentParser() argh.add_commands( parser, [fill_and_wait_models, fill_and_wait_time, smart_rsync, make_chunk_for]) if __name__ == "__main__": import sys remaining_argv = flags.FLAGS(sys.argv, known_only=True) argh.dispatch(parser, argv=remaining_argv[1:])
if TRAINING_CHUNK_RE.match(fname)] if read_file is not None: read_file = os.path.join(os.getcwd(), save_file) n = PolicyNetwork() n.initialize_variables(read_file) if logdir is not None: n.initialize_logging(logdir) last_save_checkpoint = 0 for i in range(epochs): random.shuffle(train_chunk_files) for file in train_chunk_files: print("Using %s" % file) with timer("load dataset"): train_dataset = DataSet.read(file) with timer("training"): n.train(train_dataset) with timer("save model"): n.save_variables(save_file) if n.get_global_step() > last_save_checkpoint + checkpoint_freq: with timer("test set evaluation"): n.check_accuracy(test_dataset) last_save_checkpoint = n.get_global_step() parser = argparse.ArgumentParser() argh.add_commands(parser, [gtp, preprocess, train]) if __name__ == '__main__': argh.dispatch(parser)
already_processed = set() num_already_processed = len(already_processed) for model_name, record_files in sorted(model_gamedata.items()): if set(record_files) <= already_processed: continue print("Gathering files for %s:" % model_name) for i, example_batch in enumerate( tqdm(preprocessing.shuffle_tf_examples(examples_per_record, record_files))): output_record = os.path.join(output_directory, '{}-{}.tfrecord.zz'.format(model_name, str(i))) preprocessing.write_tf_examples( output_record, example_batch, serialize=False) already_processed.update(record_files) print("Processed %s new files" % (len(already_processed) - num_already_processed)) with gfile.GFile(meta_file, 'w') as f: f.write('\n'.join(sorted(already_processed))) qmeas.stop_time('gather') parser = argparse.ArgumentParser() argh.add_commands(parser, [gtp, bootstrap, train, selfplay, gather, evaluate, validate]) if __name__ == '__main__': cloud_logging.configure() argh.dispatch(parser)
def main(): parser = get_base_parser() commands = [ render_config, clean, sphinx, deploy, push ] argh.add_commands(parser, commands) git_commands = [ giza.operations.git.apply_patch, giza.operations.git.pull_rebase, giza.operations.git.cherry_pick, ] argh.add_commands(parser, git_commands, namespace='git') generate_commands = [ giza.operations.generate.api, giza.operations.generate.assets, giza.operations.generate.images, giza.operations.generate.intersphinx, giza.operations.generate.options, giza.operations.generate.primer, giza.operations.generate.steps, giza.operations.generate.tables, giza.operations.generate.toc, ] argh.add_commands(parser, generate_commands, namespace='generate') include_commands = [ giza.operations.includes.recursive, giza.operations.includes.changed, giza.operations.includes.once, giza.operations.includes.unused, giza.operations.includes.list, giza.operations.includes.graph, giza.operations.includes.clean, ] argh.add_commands(parser, include_commands, namespace='includes') packaging_commands = [ giza.operations.packaging.fetch, giza.operations.packaging.unwind, giza.operations.packaging.create, giza.operations.packaging.deploy, ] argh.add_commands(parser, packaging_commands, namespace='package') translation_commands = [ giza.operations.tx.check_orphaned, giza.operations.tx.update_translations, giza.operations.tx.pull_translations, giza.operations.tx.push_translations, ] argh.add_commands(parser, translation_commands, namespace='tx') args = RuntimeStateConfig() argh.dispatch(parser, namespace=args)
write_gulp_in(A, runword='md qiterative conv', T=T, time_step=time_step, tot_step=tot_step, lib='reax') print('\n- running gulp nvt ...') system('gulp<inp-gulp>gulp.out') xyztotraj('his.xyz', mode=mode) def x(mode='w'): xyztotraj('his.xyz', mode=mode) def pm(gen='md.traj', index=-1): ''' pressMol ''' A = read(gen, index=index) cell = A.get_cell() print(cell) A = press_mol(A) A.write('poscar.gen') del A if __name__ == '__main__': ''' use commond like ./cp.py scale-md --T=2800 to run it''' parser = argparse.ArgumentParser() argh.add_commands(parser, [md, mmd, cmd, opt, traj, nvt, pm, x]) argh.dispatch(parser)
site = make_site( filters=filters, outpath=outputpath, contexts=[ (r'.*.html', loadAcademyData), (r'.*.custom', loadAcademyData), ], rules=[ (r'coaching-detail-pages.custom', render_coaching_detail_pages), (r'project-detail-pages.custom', render_project_detail_pages), ], searchpath=searchpath, staticpaths=['static', '../data'], ) manager = Manager( site_name='govlabacademy.org', site=site, sass_src_path=path.join(ROOT_DIR, 'sass', 'styles.scss'), sass_dest_path=path.join(searchpath, 'static', 'styles', 'styles.css') ) argh.add_commands(manager.parser, [deploy]) if __name__ == '__main__': manager.run()
commands.append(pip) def alembic(*args): prepare() if len(args) == 1: args = shlex.split(args[0]) binargs = ['alembic'] + list(args) os.execvp(binargs[0], binargs) commands.append(alembic) def runserver(*args): prepare() if len(args) == 1: args = shlex.split(args[0]) os.chdir(os.path.join(ROOT_DIR, 'server')) binargs = ['python', 'main.py'] + list(args) os.execvp(binargs[0], binargs) commands.append(runserver) if __name__ == '__main__': parser = ArgumentParser() add_commands(parser, commands) dispatch(parser)
rpcsqa.build_swift_rings(False, management_node, proxy_nodes, storage_nodes, 3) print '#' * 60 print "## Then run chef-client on all nodes in the following order: " print "## Management Node: {0}".format(rpcsqa.print_server_info(swift_management)) for proxy in swift_proxy: print "## Swift Proxy Server: {0} ##".format(rpcsqa.print_server_info(proxy)) for storage in swift_storage: print "## Swift Storage Server: {0} ##".format(rpcsqa.print_server_info(storage)) ##################################################################### # Successful Setup, exit ##################################################################### print '#' * 60 print "############# Swift Cluster Build Successful ###############" print '#' * 60 def test(name='autotest'): raise NotImplementedError def teardown(name='autotest'): raise NotImplementedError if __name__ == "__main__": parser = argh.ArghParser() argh.add_commands([build, test, teardown]) parser.dispatch()
for i in range(start, end): atoms = images[i] his.write(atoms=atoms) his.close() def collect(traj='siesta.traj', start=0, end=20): newt = traj[:-5] + '_.traj' # images = Trajectory(traj) his = TrajectoryWriter(newt, mode='w') cdir = getcwd() trajs = listdir(cdir) for traj in trajs: if traj.find('.traj') > 0 and traj != 'siesta_.traj': print('- reading file %s ...' % traj) images = Trajectory(traj) for i in range(start, end): atoms = images[i] his.write(atoms=atoms) his.close() if __name__ == '__main__': parser = argparse.ArgumentParser() argh.add_commands(parser, [merge, collect, col]) argh.dispatch(parser)
) if __name__ == '__main__': context = ReloadingContext() site = staticjinja.make_site( filters=filters(), outpath=_OUTPUTPATH, contexts=[ (r'.*.html', context.get), (r'project-detail-pages.custom', context.get), ], rules=[ (r'project-detail-pages.custom', render_project_detail_pages) ], searchpath=_SEARCHPATH, staticpaths=['static'] ) manager = Manager( sass_src_path=path.join(_SASSPATH, 'styles.scss'), sass_dest_path=path.join(_SEARCHPATH, 'static', 'styles', 'styles.css'), site=site, site_name='www.thegovlab.org', ) context.add_to(manager) argh.add_commands(manager.parser, [deploy, clean]) manager.run()
return resize( filename, dimensions=dimensions, format=format, quality=quality, fill=fill, bgcolor=bgcolor, upscale=upscale, progressive=progressive, placeholder=placeholder, ) parser = argh.ArghParser() argh.add_commands(parser, [generate]) argh.add_commands( parser, [list_cache, list_images], namespace='list', title="Commands for listing images and cache", ) argh.add_commands( parser, [sync_cache], namespace='sync', title="Commands for syncing data", ) argh.add_commands( parser,
subprocess.check_call('git subtree push --prefix site origin gh-pages', shell=True) if __name__ == '__main__': context = ReloadingContext() site = staticjinja.make_site(filters=filters(), outpath=_OUTPUTPATH, contexts=[ (r'.*.html', context.get), (r'project-detail-pages.custom', context.get), ], rules=[(r'project-detail-pages.custom', render_project_detail_pages)], searchpath=_SEARCHPATH, staticpaths=['static']) manager = Manager( sass_src_path=path.join(_SASSPATH, 'styles.scss'), sass_dest_path=path.join(_SEARCHPATH, 'static', 'styles', 'styles.css'), site=site, site_name='www.thegovlab.org', ) context.add_to(manager) argh.add_commands(manager.parser, [deploy, clean]) manager.run()
# restore all saved weights, except global_step meta_graph_def = meta_graph.read_meta_graph_file(load_file + '.meta') stored_var_names = set([ n.name for n in meta_graph_def.graph_def.node if n.op == 'VariableV2' ]) stored_var_names.remove('global_step') var_list = [ v for v in tf.global_variables() if v.op.name in stored_var_names ] tf.train.Saver(var_list=var_list).restore(sess, load_file) # manually set the global step global_step_tensor = tf.train.get_or_create_global_step() assign_op = tf.assign(global_step_tensor, global_step_value) sess.run(assign_op) # export a new savedmodel that has the right global step type tf.train.Saver().save(sess, dest_file) sess.close() tf.reset_default_graph() parser = argparse.ArgumentParser() argh.add_commands( parser, [gtp, bootstrap, train, selfplay, gather, evaluate, validate, convert]) if __name__ == '__main__': cloud_logging.configure() argh.dispatch(parser)
# Get module for reference version refVersMod = importlib.import_module( f'.v{referenceVersion}', 'CureCompanion.mariadb') # Make sure it has a trim function if not hasattr(refVersMod, 'trim'): raise CommandError('Version ' + referenceVersion + ' does not ' + 'support the trim operation.') # Set default trim version if trimVersion is None: trimVersion = refVersMod.previousVersion # Confirm with user response = askYesOrNoQuestion('Trim is a destructive and ' + 'irreversible operation. Are you sure you want to proceed?!') if response == 'y': # Delegate to appropriate trim function refVersMod.trim(trimVersion) else: _logger.info('Trim not performed (whew!)') if __name__ == '__main__': parser = argparse.ArgumentParser(description='MariaDB database operations') argh.add_commands(parser, [execute, installSchema, importData, getSchemaVersion, upgrade, overlay, trim]) argh.dispatch(parser)
assign_op = tf.assign(global_step_tensor, global_step_value) sess.run(assign_op) # export a new savedmodel that has the right global step type tf.train.Saver().save(sess, dest_file) sess.close() tf.reset_default_graph() def freeze_graph(load_file): """ Loads a network and serializes just the inference parts for use by e.g. the C++ binary """ n = dual_net.DualNetwork(load_file) out_graph = tf.graph_util.convert_variables_to_constants( n.sess, n.sess.graph.as_graph_def(), ["policy_output", "value_output"]) with gfile.GFile(os.path.join(load_file + '.pb'), 'wb') as f: f.write(out_graph.SerializeToString()) parser = argparse.ArgumentParser() argh.add_commands(parser, [ gtp, bootstrap, train, train_dir, freeze_graph, selfplay, evaluate, validate, convert ]) if __name__ == '__main__': cloud_logging.configure() # Let absl.flags parse known flags from argv, then pass the remaining flags # into argh for dispatching. remaining_argv = flags.FLAGS(sys.argv, known_only=True) argh.dispatch(parser, argv=remaining_argv[1:])
ensure_dir_exists(output_dir) models = [model for model in fsdb.get_models() if model[0] < model_num] buf = ExampleBuffer(positions, samples_per_game=samples_per_game) files = [] for _, model in sorted(models, reverse=True): local_model_dir = os.path.join(local_dir, model) if not tf.gfile.Exists(local_model_dir): print("Rsyncing", model) _rsync_dir(os.path.join(game_dir, model), local_model_dir) files.extend(tf.gfile.Glob(os.path.join(local_model_dir, '*.zz'))) print("{}: {} games".format(model, len(files))) if len(files) * samples_per_game > positions: break print("Filling from {} files".format(len(files))) buf.parallel_fill(files, threads=threads) print(buf) output = os.path.join(output_dir, str(model_num) + '.tfrecord.zz') print("Writing to", output) buf.flush(output) parser = argparse.ArgumentParser() argh.add_commands(parser, [fill_and_wait, smart_rsync, make_chunk_for]) if __name__ == "__main__": import sys remaining_argv = flags.FLAGS(sys.argv, known_only=True) argh.dispatch(parser, argv=remaining_argv[1:])
def main(): parser = get_base_parser() commands = [ make_project, render_config, clean, sphinx, deploy, push ] argh.add_commands(parser, commands) git_commands = [ giza.operations.git.apply_patch, giza.operations.git.pull_rebase, giza.operations.git.cherry_pick, giza.operations.git.merge, ] argh.add_commands(parser, git_commands, namespace='git') generate_commands = [ giza.operations.generate.api, giza.operations.generate.assets, giza.operations.generate.images, giza.operations.generate.intersphinx, giza.operations.generate.options, giza.operations.generate.primer, giza.operations.generate.steps, giza.operations.generate.tables, giza.operations.generate.toc, giza.operations.generate.examples, giza.operations.generate.redirects ] argh.add_commands(parser, generate_commands, namespace='generate') include_commands = [ giza.operations.includes.recursive, giza.operations.includes.changed, giza.operations.includes.once, giza.operations.includes.unused, giza.operations.includes.list, giza.operations.includes.graph, giza.operations.includes.clean, ] argh.add_commands(parser, include_commands, namespace='includes') packaging_commands = [ giza.operations.packaging.fetch, giza.operations.packaging.unwind, giza.operations.packaging.create, giza.operations.packaging.deploy, ] argh.add_commands(parser, packaging_commands, namespace='package') translate_commands = [ giza.operations.translate.create_corpora, giza.operations.translate.build_translation_model, giza.operations.translate.model_results, giza.operations.translate.merge_translations, giza.operations.translate.po_to_corpus, giza.operations.translate.dict_to_corpus, giza.operations.translate.translate_po, giza.operations.translate.translate_text_doc, giza.operations.translate.flip_text, giza.operations.translate.auto_approve_obvious_po, ] argh.add_commands(parser, translate_commands, namespace='translate') translation_commands = [ giza.operations.tx.check_orphaned, giza.operations.tx.update_translations, giza.operations.tx.pull_translations, giza.operations.tx.push_translations, ] argh.add_commands(parser, translation_commands, namespace='tx') args = RuntimeStateConfig() argh.dispatch(parser, namespace=args)
features, labels = dual_net.get_inference_input() dual_net.model_fn(features, labels, tf.estimator.ModeKeys.PREDICT, dual_net.get_default_hyperparams()) for model_name in tqdm(models): if model_name.endswith('-upgrade'): continue try: load_file = os.path.join(MODELS_DIR, model_name) dest_file = os.path.join(MODELS_DIR, model_name) main.convert(load_file, dest_file) except: print('failed on', model_name) continue def echo(): pass # Flags are echo'd in the ifmain block below. parser = argparse.ArgumentParser() argh.add_commands(parser, [train, selfplay, gather, echo, backfill, bootstrap, game_counts, validate]) if __name__ == '__main__': print_flags() cloud_logging.configure() remaining_argv = flags.FLAGS(sys.argv, known_only=True) argh.dispatch(parser, argv=remaining_argv[1:])
def gather(): print("Gathering game output...") main.gather(input_directory=SELFPLAY_DIR, output_directory=TRAINING_CHUNK_DIR) def train(logdir=None): model_num, model_name = get_latest_model() print("Training on gathered game data, initializing from {}".format(model_name)) new_model_name = shipname.generate(model_num + 1) print("New model will be {}".format(new_model_name)) load_file = os.path.join(MODELS_DIR, model_name) save_file = os.path.join(MODELS_DIR, new_model_name) try: main.train(TRAINING_CHUNK_DIR, save_file=save_file, load_file=load_file, generation_num=model_num, logdir=logdir) except: print("Got an error training, muddling on...") logging.exception("Train error") parser = argparse.ArgumentParser() argh.add_commands(parser, [train, selfplay, gather, bootstrap, convert_all]) if __name__ == '__main__': print_flags() cloud_logging.configure() argh.dispatch(parser)
# will get called within a graph context containing our model graph. self.summary_writer = SummaryWriterCache.get(self.working_dir) self.weight_tensors = tf.trainable_variables() self.global_step = tf.train.get_or_create_global_step() def before_run(self, run_context): global_step = run_context.session.run(self.global_step) if global_step % self.every_n_steps == 0: self.before_weights = run_context.session.run(self.weight_tensors) def after_run(self, run_context, run_values): global_step = run_context.session.run(self.global_step) if self.before_weights is not None: after_weights = run_context.session.run(self.weight_tensors) weight_update_summaries = compute_update_ratio( self.weight_tensors, self.before_weights, after_weights) self.summary_writer.add_summary(weight_update_summaries, global_step) self.before_weights = None parser = argparse.ArgumentParser() argh.add_commands(parser, [train, export_model, validate]) if __name__ == '__main__': # Let absl.flags parse known flags from argv, then pass the remaining flags # into argh for dispatching. remaining_argv = flags.FLAGS(sys.argv, known_only=True) argh.dispatch(parser, argv=remaining_argv[1:])
for model_name, record_files in sorted(model_gamedata.items()): with timer("Processing %s" % model_name): if set(record_files) <= already_processed: print("%s is already fully processed" % model_name) continue for i, example_batch in enumerate( tqdm( preprocessing.shuffle_tf_examples( examples_per_record, record_files))): output_record = os.path.join( output_directory, '{}-{}.tfrecord.zz'.format(model_name, str(i))) preprocessing.write_tf_examples(output_record, example_batch, serialize=False) already_processed.update(record_files) print("Processed %s new files" % (len(already_processed) - num_already_processed)) with gfile.GFile(meta_file, 'w') as f: f.write('\n'.join(sorted(already_processed))) parser = argparse.ArgumentParser() argh.add_commands(parser, [gtp, bootstrap, train, selfplay, gather, evaluate]) if __name__ == '__main__': cloud_logging.configure() argh.dispatch(parser)
js.dump(j, fj, sort_keys=True, indent=2) fj.close() def init_bonds(p_): spec, bonds, offd, angs, torp, hbs = [], [], [], [], [], [] for key in p_: k = key.split('_') if k[0] == 'bo1': bonds.append(k[1]) elif k[0] == 'rosi': kk = k[1].split('-') if len(kk) == 2: offd.append(k[1]) elif len(kk) == 1: spec.append(k[1]) elif k[0] == 'theta0': angs.append(k[1]) elif k[0] == 'tor1': torp.append(k[1]) elif k[0] == 'rohb': hbs.append(k[1]) return spec, bonds, offd, angs, torp, hbs if __name__ == '__main__': ''' use commond like ./gmd.py nvt --T=2800 to run it''' parser = argparse.ArgumentParser() argh.add_commands(parser, [q, i, ii, j, jj]) argh.dispatch(parser)
models = [m[1] for m in fsdb.get_models()] import dual_net import tensorflow as tf from tqdm import tqdm features, labels = dual_net.get_inference_input() dual_net.model_fn(features, labels, tf.estimator.ModeKeys.PREDICT, dual_net.get_default_hyperparams()) for model_name in tqdm(models): if model_name.endswith('-upgrade'): continue try: load_file = os.path.join(fsdb.models_dir(), model_name) dest_file = os.path.join(fsdb.models_dir(), model_name) main.convert(load_file, dest_file) except: print('failed on', model_name) continue parser = argparse.ArgumentParser() argh.add_commands( parser, [train, selfplay, backfill, bootstrap, fsdb.game_counts, validate]) if __name__ == '__main__': cloud_logging.configure() remaining_argv = flags.FLAGS(sys.argv, known_only=True) argh.dispatch(parser, argv=remaining_argv[1:])
if model_num is None: model_num, model_name = get_latest_model() else: model_num = int(model_num) model_name = get_model(model_num) # Model N was trained on games up through model N-2, so the validation set # should only be for models through N-2 as well, thus the (model_num - 1) # term. models = list( filter(lambda num_name: num_name[0] < (model_num - 1), get_models())) # Run on the most recent 50 generations, # TODO(brianklee): make this hyperparameter dependency explicit/not hardcoded holdout_dirs = [os.path.join(HOLDOUT_DIR, pair[1]) for pair in models[-50:]] main.validate(ESTIMATOR_WORKING_DIR, *holdout_dirs, checkpoint_name=os.path.join(MODELS_DIR, model_name), validate_name=validate_name) parser = argparse.ArgumentParser() argh.add_commands(parser, [train, selfplay, gather, bootstrap, game_counts, validate]) if __name__ == '__main__': print_flags() cloud_logging.configure() argh.dispatch(parser)
] save_file = os.path.join(os.getcwd(), save_file) n = PolicyNetwork() try: n.initialize_variables(save_file) except: n.initialize_variables(None) if logdir is not None: n.initialize_logging(logdir) last_save_checkpoint = 0 for i in range(epochs): random.shuffle(train_chunk_files) for file in train_chunk_files: print("Using %s" % file) train_dataset = DataSet.read(file) train_dataset.shuffle() with timer("training"): n.train(train_dataset) n.save_variables(save_file) if n.get_global_step() > last_save_checkpoint + checkpoint_freq: with timer("test set evaluation"): n.check_accuracy(test_dataset) last_save_checkpoint = n.get_global_step() parser = argparse.ArgumentParser() argh.add_commands(parser, [gtp, preprocess, train]) if __name__ == '__main__': argh.dispatch(parser)
def upload(vaultname, passphrase=None, key=AWS_ACCESS_KEY_ID, secret=AWS_SECRET_ACCESS_KEY, shelf_name=os.path.expanduser("~/glaciervault.db"), *filename): glacier = GlacierStore(key, secret, vaultname, shelf_name) glacier.upload(passphrase, *filename) def delete(vaultname, key=AWS_ACCESS_KEY_ID, secret=AWS_SECRET_ACCESS_KEY, shelf_name=os.path.expanduser("~/glaciervault.db"), *filename): glacier = GlacierStore(key, secret, vaultname, shelf_name) glacier.delete(*filename) def retrieve(vaultname, key=AWS_ACCESS_KEY_ID, secret=AWS_SECRET_ACCESS_KEY, topic='glacier_alerts', shelf_name=os.path.expanduser("~/glaciervault.db"), *filename): glacier = GlacierStore(key, secret, vaultname, shelf_name) glacier.retrieve(topic, *filename) def monitor(vaultname, key=AWS_ACCESS_KEY_ID, secret=AWS_SECRET_ACCESS_KEY, passphrase=None, shelf_name=os.path.expanduser("~/glaciervault.db"), *dirs): glacier = GlacierStore(key, secret, vaultname, shelf_name) glacier.monitor(passphrase, *dirs) def download(vaultname, key=AWS_ACCESS_KEY_ID, secret=AWS_SECRET_ACCESS_KEY, passphrase=None, shelf_name=os.path.expanduser("~/glaciervault.db")): glacier = GlacierStore(key, secret, vaultname, shelf_name) glacier.download(passphrase) parser = argparse.ArgumentParser() argh.add_commands( parser, [create, upload, delete, retrieve, monitor, download]) if __name__ == '__main__': argh.dispatch(parser)