def main(): args = get_parser() os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu args.device = torch.device("cuda:0") torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) if not os.path.exists(args.save_path): os.makedirs(args.save_path) args.model_path = os.path.join(args.save_path, args.model_name, 'model') if not os.path.exists(args.model_path): os.makedirs(args.model_path) if args.model_name == 'generator': g = Generator(args) if args.test: args.model_path = os.path.join(args.save_path, 'predictor', 'model') hs = args.hs args.hs = 512 p = Predictor(args) args.model_path = os.path.join(args.save_path, args.model_name, 'model') args.hs = hs g.meta_test(p) else: g.meta_train() elif args.model_name == 'predictor': p = Predictor(args) p.meta_train() else: raise ValueError('You should select generator|predictor|train_arch')
def roll(dice_spec: str) -> Tuple[int, Sequence[int]]: tracer = execution_context.get_opencensus_tracer() try: with tracer.span('initial_parse'): tracer.add_attribute_to_current_span("dice_spec", dice_spec) tree = get_parser().parse(dice_spec) except LarkError as e: raise RecognitionError( "Sorry, I couldn't understand your request") from e logging.debug("Initial parse tree:\n%s", pprint(tree)) try: tree = NumberTransformer().transform(tree) with tracer.span('dnd_knowledge'): tree = DnD5eKnowledge().transform(tree) tree = SimplifyTransformer().transform(tree) with tracer.span('crit_transform'): tree = CritTransformer().transform(tree) logging.debug("DnD transformed parse tree:\n%s", pprint(tree)) with tracer.span('final_eval'): transformer = EvalDice() tree = transformer.transform(tree) tree = SimplifyTransformer().transform(tree) except VisitError as e: # Get our nice exception out of lark's wrapper raise e.orig_exc return (tree.children[0], transformer.dice_results)
def main(): args = get_parser() if args.gpu == 'all': device_list = range(torch.cuda.device_count()) args.gpu = ','.join(str(_) for _ in device_list) else: device_list = [int(_) for _ in args.gpu.split(',')] os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu args.device = torch.device("cuda:0") args.batch_size = args.batch_size * max(len(device_list), 1) torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) args.model_path = os.path.join(args.save_path, args.model_name, 'model') if args.model_name == 'generator': graph_config = load_graph_config( args.graph_data_name, args.nvt, args.data_path) model = PredictorModel(args, graph_config) d = DatabaseOFA(args, model) else: d = DatabaseOFA(args) if args.collect: d.collect_db() else: assert args.index is not None assert args.imgnet is not None d.make_db()
def main(): ''' Initialize everything and train ''' options = get_parser().parse_args() if not os.path.exists(options.experiment_root): os.makedirs(options.experiment_root) if torch.cuda.is_available() and not options.cuda: print("WARNING: You have a CUDA device, so you should probably run with --cuda") init_seed(options) tr_dataloader, val_dataloader, trainval_dataloader, test_dataloader = init_dataset( options) model = init_protonet(options) optim = init_optim(options, model) lr_scheduler = init_lr_scheduler(options, optim) res = train(opt=options, tr_dataloader=tr_dataloader, val_dataloader=val_dataloader, model=model, optim=optim, lr_scheduler=lr_scheduler) best_state, best_acc, train_loss, train_acc, val_loss, val_acc = res print('Testing with last model..') test(opt=options, test_dataloader=test_dataloader, model=model) model.load_state_dict(best_state) print('Testing with best model..') test(opt=options, test_dataloader=test_dataloader, model=model)
def main(): options = get_parser().parse_args() if torch.cuda.is_available() and not options.cuda: print( "WARNING: You have a CUDA device, so you should probably run with --cuda" ) init_experiment(options) init_seed(options) healthy_dataloader, anomaly_dataloader = init_dataset(options) net_g, net_d = init_model(options) net_g.apply(weights_init) net_d.apply(weights_init) optim_g, optim_d = init_optimizer(options, net_g=net_g, net_d=net_d) if options.cuda: net_g = net_g.cuda() net_d = net_d.cuda() train(options, healthy_dataloader, anomaly_dataloader, net_g=net_g, net_d=net_d, optim_g=optim_g, optim_d=optim_d)
def download(url): print('open connection') try: _parser = parser.get_parser(url) except exceptions.NotBoorusPrefixError as e: logger.exception("invalid prefix in {}".format(e.url)) return except exceptions.SiteNotSupported as e: logger.exception("Site not supported {}".format(e.url)) return try: data = _parser.get_data() except IndexError: return parsed_tags = _parser.tagIndex() print("parsed tags", parsed_tags) outdir = tagResponse.find_folder(parsed_tags) print("outdir", outdir) if not os.path.isdir(outdir): os.makedirs(outdir) _parser.save_image_old_interface(outdir, data, parsed_tags)
def main(): parser = get_parser() args = parser.parse_args() print(args) sys.exit() if "preprocess" in args.operation_mode: print("prep") dataset = WavFilesDataset(args.input) dataset.save_as_hd5(args.h5_path) splitter = Splitter(args) #TODO if "train" in args.operation_mode: print("train") if args.world_size > 1: torch.cuda.set_device(args.rank % torch.cuda.device_count()) distributed.init_process_group(backend=args.dist_backend, init_method="tcp://" + args.master, rank=args.rank, world_size=args.world_size) if(args.train_step == 1): print("1") AutoencoderTrainer(args).train() else: print("2") MusicStarTrainer(args).train() if "translate" in args.operation_mode: print("translate") #TODO if "analyze" in args.operation_mode: print("analyze")
def main(): options = get_parser().parse_args() if not os.path.exists(os.path.join(options.model_out, options.dataset_type)): os.makedirs(os.path.join(options.model_out, options.dataset_type)) if torch.cuda.is_available() and not options.cuda: print( "WARNING: You have a CUDA device, so you should probably run with --cuda" ) init_seed(options) tr_dataloader = init_dataloader(options, "train") val_dataloader = init_dataloader(options, "val") test_dataloader = init_dataloader(options, "test") model = init_model() optim, scheduler = init_optim_scheduler(options, model) best_state, best_acc, train_loss, train_acc, val_loss, val_acc = train( options, tr_dataloader, model, optim, scheduler, val_dataloader) print('Testing with last model..') test(options, test_dataloader, model) print('Testing with best model..') model.load_state_dict(best_state) test(options, test_dataloader, model)
def _parse_expr(self, e, for_obj): from interpreter.builder import Builder from parser import get_parser pars = get_parser("%s;" % e) e = Builder().create_expression(pars.expression().tree, for_obj) e = self.type.normalize(e) self.type.validate(e) return e
def parse_program(input: str, forbid_rebuild: bool = False, filename: Optional[str] = None) -> Program: l = parser.get_lexer() p = parser.get_parser(forbid_rebuild=forbid_rebuild) prog: Program = p.parse(input=input, lexer=l, filename=filename) prog.input = input return prog
def main(): # loads a lot of default parser values from the 'parser' file parser = file_parser.get_parser() # get args from parser as an object # MOST IMPORTANT PART, INTERMS OF CONTROLLING THE EXPERIMENTS args = parser.parse_args() # initialize seeds misc_utils.init_seed(args.seed) # set up loader # 2 options: class_incremental and task_incremental # experiments in the paper only use task_incremental Loader = importlib.import_module('dataloaders.' + args.loader) # args.loader='task_incremental_loader' # print('loader stuff', args) loader = Loader.IncrementalLoader(args, seed=args.seed) # print('loader stuff after after', args) n_inputs, n_outputs, n_tasks = loader.get_dataset_info() # setup logging # logging is from 'misc_utils.py' from 'utils' folder timestamp = misc_utils.get_date_time( ) # this line is redundant bcz log_dir already takes care of it args.log_dir, args.tf_dir = misc_utils.log_dir( args, timestamp) # stores args into "training_parameters.json" # load model from the 'model' folder Model = importlib.import_module('model.' + args.model) # create the model neural net model = Model.Net(n_inputs, n_outputs, n_tasks, args) # make model cuda-ized if possible if args.cuda: try: model.net.cuda() except: pass # wandb.watch(model) # run model on loader if args.model == "iid2": # oracle baseline with all task data shown at same time result_val_t, result_val_a, result_test_t, result_test_a, spent_time = life_experience_iid( model, loader, args) else: # for all the CL baselines result_val_t, result_val_a, result_test_t, result_test_a, spent_time = life_experience( model, loader, args) # save results in files or print on terminal save_results(args, result_val_t, result_val_a, result_test_t, result_test_a, model, spent_time)
def WEAPON(self, name) -> Tree: tracer = execution_context.get_opencensus_tracer() weapon = self.find_named_object(name, WEAPONS) dice_spec = weapon["damage_dice"] with tracer.span('parse_weapon'): tracer.add_attribute_to_current_span("name", name) tracer.add_attribute_to_current_span("dice_spec", dice_spec) tree = get_parser().parse(dice_spec, start="sum") logging.debug("weapon %s has damage dice %s parsed as:\n%s", name, dice_spec, pprint(tree)) return tree
def spell_default(self, spell: Mapping[str, Any]) -> Tree: tracer = execution_context.get_opencensus_tracer() m = re.search(r"\d+d\d+( \+ \d+)?", spell["desc"]) if not m: raise ImpossibleSpellError( f"Sorry, I couldn't find the damage dice for %s" % spell["name"]) with tracer.span('parse_spell'): tracer.add_attribute_to_current_span("name", spell["name"]) tracer.add_attribute_to_current_span("dice_spec", m.group(0)) tree = get_parser().parse(m.group(0), start="sum") logging.debug("spell %s has base damage dice %s parsed as:\n%s", spell["name"], m.group(0), pprint(tree)) return tree
def main(): ''' Initialize everything and train ''' options = get_parser().parse_args() if torch.cuda.is_available() and not options.cuda: print("WARNING: You have a CUDA device, so you should probably run with --cuda") init_seed(options) tr_dataloader, val_dataloader, trainval_dataloader, test_dataloader = init_dataset( options) model = init_protonet(options) optim = init_optim(options, model) lr_scheduler = init_lr_scheduler(options, optim) best_state = train(opt=options, tr_dataloader=tr_dataloader, val_dataloader=val_dataloader, model=model, optim=optim, lr_scheduler=lr_scheduler) print('Testing with last model..') test(opt=options, test_dataloader=test_dataloader, model=model) model.load_state_dict(best_state) print('Testing with best model..') test(opt=options, test_dataloader=test_dataloader, model=model) # optim = init_optim(options, model) # lr_scheduler = init_lr_scheduler(options, optim) print('Training on train+val set..') train(opt=options, tr_dataloader=trainval_dataloader, val_dataloader=None, model=model, optim=optim, lr_scheduler=lr_scheduler) print('Testing final model..') test(opt=options, test_dataloader=test_dataloader, model=model)
def eval(opt): ''' Initialize everything and train ''' options = get_parser().parse_args() if torch.cuda.is_available() and not options.cuda: print( "WARNING: You have a CUDA device, so you should probably run with --cuda" ) init_seed(options) test_dataloader = init_dataset(options)[-1] model = init_protonet(options) model_path = os.path.join(opt.experiment_root, 'best_model.pth') model.load_state_dict(torch.load(model_path)) test(opt=options, test_dataloader=test_dataloader, model=model)
def get_args(): """ Preprocess dataset usage: preproess.py [options] <wav-dir> options: --output-dir=<dir> Directory where processed outputs are saved. [default: data_dir]. -h, --help Show help message. """ parser = get_parser(description="Preprocess dataset") parser.add_argument('wav_dir', type=str) parser.add_argument('--output-dir', type=str, help='config file', default='data_dir') parser.add_argument('--start-id', type=int, help='preprocessing checkpoint', default=0) #parser.add_argument('--dsp_config', type=str, help='dsp default config file', # default='./config/dsp.yaml') return parser.parse_args()
def main(): ''' Initialize everything and train ''' options = get_parser().parse_args() if torch.cuda.is_available() and not options.cuda: print("WARNING: You have a CUDA device, so you should probably run with --cuda") init_seed(options) tr_dataloader, val_dataloader = init_dataset(options) model = init_model(options) optim = init_optim(options, model) lr_scheduler = init_lr_scheduler(options, optim) train(opt=options, tr_dataloader=tr_dataloader, val_dataloader=val_dataloader, model=model, optim=optim, lr_scheduler=lr_scheduler)
def __init__(self, predicate, debug=0): """ Initializes the Predicate object with the string predicate. Arguments: predicate : String predicate debug : optional, defaults to 0. Controls the debug behavior of the underlying parser and lexer. Returns: Predicate object """ # Validate the predicate if not isinstance(predicate, str): raise TypeError("Predicate must be a string!") # Initialize the literal resolver LiteralResolver.__init__(self) # Store the predicate self.predicate = predicate # Setup the lexer lexer = get_lexer() self.lexer_errors = lexer.errors # Setup the parser p = get_parser(lexer=lexer, debug=debug) self.parser_errors = p.errors # Try to get the AST tree try: self.ast = p.parse(self.predicate, lexer=lexer) self.ast_validated = False self.ast_valid = False self.ast_errors = None except Exception, e: self.ast = None self.ast_validated = True self.ast_valid = False self.ast_errors = {"errors": [str(e)], "regex": {}}
def command_line_runner(): """Consumes commands to trichome.""" command_parser = parser.get_parser() args = vars(command_parser.parse_args()) if args['discover'] == 'discover': common_words = [] if args['common_words']: common_words = read_file(args['common_words'][0]) result = discover(args['URL'][-1], common_words) else: vector_input = None sensitive_input = None if args['vectors']: vector_input = read_file(args['vectors'][0]) if args['sensitive']: sensitive_input = read_file(args['sensitive'][0]) tested = test(args['URL'], vector_input, sensitive_input, args['random'], args['slow'])
def download(self): self._dl_btn['state'] = DISABLED pipe = multiprocessing.Pipe() try: while self._list.size() > 0: id_list = self._list.get(0, tkinter.END) self._list.delete(0, tkinter.END) map_list = list() for raw_id in id_list: _parser = None try: _parser = parser.get_parser(raw_id) except exceptions.NotBoorusPrefixError as e: logger.exception("invalid prefix in {}".format(e.url)) continue except exceptions.SiteNotSupported as e: logger.exception("Site not supported {}".format(e.url)) continue try: data = _parser.get_data() except IndexError: continue try: parsed_tags = _parser.tagIndex() except KeyError: continue outdir = tagResponse.find_folder(parsed_tags) if not os.path.isdir(outdir): os.makedirs(outdir) map_list.append((_parser, outdir, data, parsed_tags)) dl_pool = multiprocessing.Pool(processes=config.workers) results = dl_pool.map(parser.save_call, map_list, chunksize=1) pyimglib.transcoding.statistics.update_stats(results) except Exception as e: self._add_btn['state'] = DISABLED messagebox.showerror(e.__class__.__name__, str(e)) raise e self._current_item = None self._dl_btn['state'] = NORMAL
def main(): parser = file_parser.get_parser() args = parser.parse_args() # initialize seeds misc_utils.init_seed(args.seed) # set up loader # 2 options: class_incremental and task_incremental # experiments in the paper only use task_incremental Loader = importlib.import_module('dataloaders.' + args.loader) loader = Loader.IncrementalLoader(args, seed=args.seed) n_inputs, n_outputs, n_tasks = loader.get_dataset_info() # setup logging timestamp = misc_utils.get_date_time() args.log_dir, args.tf_dir = misc_utils.log_dir(args, timestamp) # load model Model = importlib.import_module('model.' + args.model) model = Model.Net(n_inputs, n_outputs, n_tasks, args) if args.cuda: try: model.net.cuda() except: pass # run model on loader if args.model == "iid2": # oracle baseline with all task data shown at same time result_val_t, result_val_a, result_test_t, result_test_a, spent_time = life_experience_iid( model, loader, args) else: # for all the CL baselines result_val_t, result_val_a, result_test_t, result_test_a, spent_time = life_experience( model, loader, args) # save results in files or print on terminal save_results(args, result_val_t, result_val_a, result_test_t, result_test_a, model, spent_time)
def get_args(): """Training WaveRNN Model. usage: train.py [options] <data-root> options: --checkpoint-dir=<dir> Directory where to save model checkpoints [default: checkpoints]. --checkpoint=<path> Restore model from checkpoint path if given. -h, --help Show this help message and exit """ parser = get_parser(description="Training WaveRNN Model") parser.add_argument('data_root', type=str, help='Preprocessed data dir.') parser.add_argument('--checkpoint-dir', type=str, default='checkpoints', help='Directory where to save model checkpoints') parser.add_argument('--checkpoint', type=str, default=None, help='Restore model from checkpoint path if given.') return parser.parse_args()
def main(): val, data = get_parser() print(val, data) if val == 'qt5': gs_text, series, s3_text, output_path, email, local_files_only, s3_files_only = read_qt5_input( data) elif val == 'arg': split_num, process_num, gs_text, series, s3_text, output_path, email, local_files_only, s3_files_only = read_args_input( data) print(local_files_only, s3_files_only) py3 = sys.version_info[ 0] > 2 #creates boolean value for test that Python major version > 2 try: os.mkdir(output_path) response = 'N' except OSError: if py3: response = input( "Directory already exists. Do you still want to continue (Y/N): " ) else: response = raw_input( "Directory already exists. Do you still want to continue (Y/N): " ) if response != 'Y': sys.exit( "Canceled, please rerun with different Directory or path.") if gs_text != '': if s3_files_only or local_files_only: make_manifest(gs_text, series, s3_text, output_path, email, response, local_files_only, s3_files_only) else: sra_series_fetch(split_num, process_num, gs_text, series, s3_text, output_path, email, response, local_files_only, s3_files_only)
def main(): args = parser.get_parser().parse_args() if args.epochs is None: args.epochs = 90 args.epochs = 90 start_epoch = 0 ending_epoch = args.epochs #Todo set device # Create the model model_helper = models.create_model_helper(args.framework, args.dataset, args.arch) compression_scheduler = None if args.deprecated_resume: print('The "--resume" flag is deprecated. Please use "--resume-from=YOUR_PATH" instead.') if not args.reset_optimizer: print('If you wish to also reset the optimizer, call with: --reset-optimizer') args.reset_optimizer = True args.resumed_checkpoint_path = args.deprecated_resume if args.compress:
def spell(self, spell: Mapping[str, Any], level: int) -> Tree: tracer = execution_context.get_opencensus_tracer() spell_tree = self.spell_default(spell) if level < spell["level_int"]: raise ImpossibleSpellError( "Sorry, %s is level %d, so I can't cast it at level %d" % (spell["name"], spell["level_int"], level)) m = re.search(r"\d+d\d+( + \d+)?", spell["higher_level"]) if not m: raise ImpossibleSpellError( "Sorry, I could't determine the additional damage dice for %s" % spell["name"]) with tracer.span('parse_spell_additional'): tracer.add_attribute_to_current_span("name", spell["name"]) tracer.add_attribute_to_current_span("dice_spec", m.group(0)) higher_level_tree = get_parser().parse(m.group(0), start="sum") logging.debug( "spell %s has damage dice %s per extra level parsed as:\n%s", spell["name"], m.group(0), pprint(higher_level_tree)) for level in range(level - spell["level_int"]): spell_tree = Tree('add', [spell_tree, higher_level_tree]) logging.debug("spell %s has complete parsed as:\n%s", spell["name"], pprint(spell_tree)) return spell_tree
def main(argv=[]): args = parser.get_parser() verbose = 1 if args.verbose: verbose = 2 if args.debug: verbose = 3 if verbose>2: log_level=logging.DEBUG elif verbose==2: log_level=logging.INFO elif verbose==1: log_level=logging.WARNING elif verbose<1: log_level=logging.ERROR logging.basicConfig(level=log_level, format="%(asctime)s %(levelname)s: %(message)s") proxy = args.proxy proxy_host = args.proxy_host proxy_port = args.proxy_port proxy_type = args.proxy_type config.first_run() if args.command == 'version': print torsurvey.get_version() return True elif args.command == 'update': return updater.check_update() ac = torapi.TorAPI(proxy_host=proxy_host, proxy_port=proxy_port, proxy_type=proxy_type, timeout=args.timeout) dbi = db.DbManager(args.dbpath) dbi.init() tc = controller.TorController(ac, dbi) # cx = CexMethods(ac, dbi) if args.command == 'checkip': print "Checking IP address" print ac.get_ip() return True elif args.command == 'flushdb': dbi.flush() print "Cleared db" elif args.command == 'fetch': if not args.url: logging.error("URL to fetch from required") return False print "Fetching from %s " % args.url f = tc.fetch_sitelist(args.url, args.cache, (not args.noinsert)) return True elif args.command == 'read': if not args.filepath: logging.error("Need file to read onion addresses from") f = tc.read_sitelist(args.filepath, (not args.noinsert)) elif args.command == 'survey': tc.survey(args.deadonly) elif args.command == 'list': dbi.list()
def main(): script_dir = os.path.dirname(__file__) module_path = os.path.abspath(os.path.join(script_dir, '..', '..')) global msglogger # Parse arguments args = parser.get_parser().parse_args() if args.epochs is None: args.epochs = 90 if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) msglogger = apputils.config_pylogger(os.path.join(script_dir, 'logging.conf'), args.name, args.output_dir) # Log various details about the execution environment. It is sometimes useful # to refer to past experiment executions and this information may be useful. apputils.log_execution_env_state(args.compress, msglogger.logdir, gitroot=module_path) msglogger.debug("Distiller: %s", distiller.__version__) start_epoch = 0 ending_epoch = args.epochs perf_scores_history = [] if args.evaluate: args.deterministic = True if args.deterministic: # Experiment reproducibility is sometimes important. Pete Warden expounded about this # in his blog: https://petewarden.com/2018/03/19/the-machine-learning-reproducibility-crisis/ distiller.set_deterministic() # Use a well-known seed, for repeatability of experiments else: # Turn on CUDNN benchmark mode for best performance. This is usually "safe" for image # classification models, as the input sizes don't change during the run # See here: https://discuss.pytorch.org/t/what-does-torch-backends-cudnn-benchmark-do/5936/3 cudnn.benchmark = True if args.cpu or not torch.cuda.is_available(): # Set GPU index to -1 if using CPU args.device = 'cpu' args.gpus = -1 else: args.device = 'cuda' if args.gpus is not None: try: args.gpus = [int(s) for s in args.gpus.split(',')] except ValueError: raise ValueError('ERROR: Argument --gpus must be a comma-separated list of integers only') available_gpus = torch.cuda.device_count() for dev_id in args.gpus: if dev_id >= available_gpus: raise ValueError('ERROR: GPU device ID {0} requested, but only {1} devices available' .format(dev_id, available_gpus)) # Set default device in case the first one on the list != 0 torch.cuda.set_device(args.gpus[0]) # Infer the dataset from the model name args.dataset = 'cifar10' if 'cifar' in args.arch else 'imagenet' args.num_classes = 10 if args.dataset == 'cifar10' else 1000 if args.earlyexit_thresholds: args.num_exits = len(args.earlyexit_thresholds) + 1 args.loss_exits = [0] * args.num_exits args.losses_exits = [] args.exiterrors = [] # Create the model model = create_model(args.pretrained, args.dataset, args.arch, parallel=not args.load_serialized, device_ids=args.gpus) compression_scheduler = None # Create a couple of logging backends. TensorBoardLogger writes log files in a format # that can be read by Google's Tensor Board. PythonLogger writes to the Python logger. tflogger = TensorBoardLogger(msglogger.logdir) pylogger = PythonLogger(msglogger) # capture thresholds for early-exit training if args.earlyexit_thresholds: msglogger.info('=> using early-exit threshold values of %s', args.earlyexit_thresholds) # TODO(barrh): args.deprecated_resume is deprecated since v0.3.1 if args.deprecated_resume: msglogger.warning('The "--resume" flag is deprecated. Please use "--resume-from=YOUR_PATH" instead.') if not args.reset_optimizer: msglogger.warning('If you wish to also reset the optimizer, call with: --reset-optimizer') args.reset_optimizer = True args.resumed_checkpoint_path = args.deprecated_resume # We can optionally resume from a checkpoint optimizer = None if args.resumed_checkpoint_path: model, compression_scheduler, optimizer, start_epoch = apputils.load_checkpoint( model, args.resumed_checkpoint_path, model_device=args.device) elif args.load_model_path: model = apputils.load_lean_checkpoint(model, args.load_model_path, model_device=args.device) if args.reset_optimizer: start_epoch = 0 if optimizer is not None: optimizer = None msglogger.info('\nreset_optimizer flag set: Overriding resumed optimizer and resetting epoch count to 0') # Define loss function (criterion) criterion = nn.CrossEntropyLoss().to(args.device) if optimizer is None: optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) msglogger.info('Optimizer Type: %s', type(optimizer)) msglogger.info('Optimizer Args: %s', optimizer.defaults) if args.AMC: return automated_deep_compression(model, criterion, optimizer, pylogger, args) if args.greedy: return greedy(model, criterion, optimizer, pylogger, args) # This sample application can be invoked to produce various summary reports. if args.summary: return summarize_model(model, args.dataset, which_summary=args.summary) activations_collectors = create_activation_stats_collectors(model, *args.activation_stats) if args.qe_calibration: msglogger.info('Quantization calibration stats collection enabled:') msglogger.info('\tStats will be collected for {:.1%} of test dataset'.format(args.qe_calibration)) msglogger.info('\tSetting constant seeds and converting model to serialized execution') distiller.set_deterministic() model = distiller.make_non_parallel_copy(model) activations_collectors.update(create_quantization_stats_collector(model)) args.evaluate = True args.effective_test_size = args.qe_calibration # Load the datasets: the dataset to load is inferred from the model name passed # in args.arch. The default dataset is ImageNet, but if args.arch contains the # substring "_cifar", then cifar10 is used. train_loader, val_loader, test_loader, _ = apputils.load_data( args.dataset, os.path.expanduser(args.data), args.batch_size, args.workers, args.validation_split, args.deterministic, args.effective_train_size, args.effective_valid_size, args.effective_test_size) msglogger.info('Dataset sizes:\n\ttraining=%d\n\tvalidation=%d\n\ttest=%d', len(train_loader.sampler), len(val_loader.sampler), len(test_loader.sampler)) if args.sensitivity is not None: sensitivities = np.arange(args.sensitivity_range[0], args.sensitivity_range[1], args.sensitivity_range[2]) return sensitivity_analysis(model, criterion, test_loader, pylogger, args, sensitivities) if args.evaluate: return evaluate_model(model, criterion, test_loader, pylogger, activations_collectors, args, compression_scheduler) if args.compress: # The main use-case for this sample application is CNN compression. Compression # requires a compression schedule configuration file in YAML. compression_scheduler = distiller.file_config(model, optimizer, args.compress, compression_scheduler, (start_epoch-1) if args.resumed_checkpoint_path else None) # Model is re-transferred to GPU in case parameters were added (e.g. PACTQuantizer) model.to(args.device) elif compression_scheduler is None: compression_scheduler = distiller.CompressionScheduler(model) if args.thinnify: #zeros_mask_dict = distiller.create_model_masks_dict(model) assert args.resumed_checkpoint_path is not None, \ "You must use --resume-from to provide a checkpoint file to thinnify" distiller.remove_filters(model, compression_scheduler.zeros_mask_dict, args.arch, args.dataset, optimizer=None) apputils.save_checkpoint(0, args.arch, model, optimizer=None, scheduler=compression_scheduler, name="{}_thinned".format(args.resumed_checkpoint_path.replace(".pth.tar", "")), dir=msglogger.logdir) print("Note: your model may have collapsed to random inference, so you may want to fine-tune") return args.kd_policy = None if args.kd_teacher: teacher = create_model(args.kd_pretrained, args.dataset, args.kd_teacher, device_ids=args.gpus) if args.kd_resume: teacher = apputils.load_lean_checkpoint(teacher, args.kd_resume) dlw = distiller.DistillationLossWeights(args.kd_distill_wt, args.kd_student_wt, args.kd_teacher_wt) args.kd_policy = distiller.KnowledgeDistillationPolicy(model, teacher, args.kd_temp, dlw) compression_scheduler.add_policy(args.kd_policy, starting_epoch=args.kd_start_epoch, ending_epoch=args.epochs, frequency=1) msglogger.info('\nStudent-Teacher knowledge distillation enabled:') msglogger.info('\tTeacher Model: %s', args.kd_teacher) msglogger.info('\tTemperature: %s', args.kd_temp) msglogger.info('\tLoss Weights (distillation | student | teacher): %s', ' | '.join(['{:.2f}'.format(val) for val in dlw])) msglogger.info('\tStarting from Epoch: %s', args.kd_start_epoch) if start_epoch >= ending_epoch: msglogger.error( 'epoch count is too low, starting epoch is {} but total epochs set to {}'.format( start_epoch, ending_epoch)) raise ValueError('Epochs parameter is too low. Nothing to do.') for epoch in range(start_epoch, ending_epoch): # This is the main training loop. msglogger.info('\n') if compression_scheduler: compression_scheduler.on_epoch_begin(epoch, metrics=(vloss if (epoch != start_epoch) else 10**6)) # Train for one epoch with collectors_context(activations_collectors["train"]) as collectors: train(train_loader, model, criterion, optimizer, epoch, compression_scheduler, loggers=[tflogger, pylogger], args=args) distiller.log_weights_sparsity(model, epoch, loggers=[tflogger, pylogger]) distiller.log_activation_statsitics(epoch, "train", loggers=[tflogger], collector=collectors["sparsity"]) if args.masks_sparsity: msglogger.info(distiller.masks_sparsity_tbl_summary(model, compression_scheduler)) # evaluate on validation set with collectors_context(activations_collectors["valid"]) as collectors: top1, top5, vloss = validate(val_loader, model, criterion, [pylogger], args, epoch) distiller.log_activation_statsitics(epoch, "valid", loggers=[tflogger], collector=collectors["sparsity"]) save_collectors_data(collectors, msglogger.logdir) stats = ('Performance/Validation/', OrderedDict([('Loss', vloss), ('Top1', top1), ('Top5', top5)])) distiller.log_training_progress(stats, None, epoch, steps_completed=0, total_steps=1, log_freq=1, loggers=[tflogger]) if compression_scheduler: compression_scheduler.on_epoch_end(epoch, optimizer) # Update the list of top scores achieved so far, and save the checkpoint update_training_scores_history(perf_scores_history, model, top1, top5, epoch, args.num_best_scores) is_best = epoch == perf_scores_history[0].epoch checkpoint_extras = {'current_top1': top1, 'best_top1': perf_scores_history[0].top1, 'best_epoch': perf_scores_history[0].epoch} apputils.save_checkpoint(epoch, args.arch, model, optimizer=optimizer, scheduler=compression_scheduler, extras=checkpoint_extras, is_best=is_best, name=args.name, dir=msglogger.logdir) # Finally run results on the test set test(test_loader, model, criterion, [pylogger], activations_collectors, args=args)
def main(argv=[]): args = parser.get_parser() verbose = 1 if args.verbose: verbose = 2 if args.debug: verbose = 3 if verbose>2: log_level=logging.DEBUG elif verbose==2: log_level=logging.INFO elif verbose==1: log_level=logging.WARNING elif verbose<1: log_level=logging.ERROR logging.basicConfig(level=log_level, format="%(asctime)s %(levelname)s: %(message)s") if args.command == 'version': print cexbot.get_version() return True # make sure this is always above command parsing # print config config.first_run() if verbose == 3: print args if args.command == 'config': if args.list: return config.list() elif args.edit: return config.edit_config() elif args.testauth: return config.test_auth() elif args.name and args.value: v = config.set(args.name, args.value) return config.cprint(args.name) elif args.name: return config.cprint(args.name) logging.error('Invalid config option') return 1 elif args.command == 'update': return updater.check_update() # not implemented elif args.command == 'cleardata': return config.clear_userdata() ac = cexapi.CexAPI(config.get('cex.username'), config.get('cex.apikey'), config.get('cex.secret')) dbi = db.DbManager() cx = CexMethods(ac, dbi) if args.command == 'balance': print "Balance: %s BTC" % ac.get_balance() return True elif args.command == 'initdb': return dbi.initdb() elif args.command == 'getmarket': return ac.get_market() elif args.command == 'getprice': return ac.get_market_quote() elif args.command == 'order': amount = args.amount price = args.price r = ac.place_order(amount, price) logging.info("Ordered: %s" % r) elif args.command == 'updatequotes': logging.info('Running updatequotes') ticker_timer = timer.ReqTimer(2, cx.update_ticker) ticker_timer.start() elif args.command == 'buybalance': logging.info('Running buybalance') balance_timer = timer.ReqTimer(5, ac.buy_balance) balance_timer.start()
import numpy as np from tqdm import tqdm from parser import get_parser from environments import EnvHub from agents import AgentHub from lib import plotting if __name__ == '__main__': parser = get_parser() args = parser.parse_args() env = EnvHub.get_env(env_id=args.env_id, save=args.save) get_agent = AgentHub[args.agent_id] agent = get_agent( action_space=env.action_space, discount_factor=args.discount_factor, ) stats = plotting.EpisodeStats( episode_lengths=np.zeros(args.num_episodes), episode_rewards=np.zeros(args.num_episodes), ) p_bar = tqdm(range(args.num_episodes)) for i_episode in p_bar: state = env.reset() done = False t = 0 while not done:
import time import numpy as np import torch torch.multiprocessing.set_start_method('spawn', force=True) import torch.nn.functional as F import torch.optim as optim import torch.distributed as dist from torch.multiprocessing import Process import parser import dataloader import hlsgd_logging args = parser.get_parser().parse_args() """ ########################################################################################## # # Get Arguments From Parser. # ########################################################################################## """ debug_mode_enabled = args.debug world_size = args.world_size batch_size = args.batch_size lr = args.learning_rate epoch_size = args.epoch_size gpu = args.gpu training_model = args.model
print "Server name: ", server.name print "Server id: ", server.id print "Networks: ", server.networks def delete(self): """ Deletes the server if the file already exists. """ server = self.get_server() if server: server.delete() print "Deleting server %s" % server.name self.wait_for_status('deleted', fail_on_not_exist=False) os.remove(self._config['id_file']) else: print "Nothing to delete." if __name__ == '__main__': parser = get_parser() args = parser.parse_args() conf = get_config(args) d = Devstork(conf) # Run the subcommand - either create or delete if args.action == 'create': d.create(wait_for_userdata=args.wait_for_userdata) elif args.action == 'delete': d.delete()
logstr += 'Epoch {}, used time: {}, {}'.format( n_epoch, time.time() - epoch_tick, result) logger.log(logstr) manager.stop() except Exception as e: print(e) if not manager.stop(): manager.stop() raise return sess if __name__ == '__main__': args = get_parser(sys.argv[1:]) tick = time.time() os.environ['CUDA_VISIBLE_DEVICES'] = args.device #load all necessary data local_data_path = 'data/{}_byuser.pk'.format(args.CITY) train_set, test_set = load_local_dataset(local_data_path) data_path = os.path.join( os.path.join( args.ROOT, 'data', '{}_INTV_processed_voc5_len2_setting_WITH_GPS_WITH_TIME_WITH_USERID.pk' .format(args.CITY))) _, dicts = load_data(data_path) args = complete_args(args, dicts) del _
elements = process_elements(grammar, rulename, elements) grammar.add_rule(rulename, *elements) while tail: item, tail = tail if item is None: continue rulename, alternation = item for elements in alternation: elements = process_elements(grammar, rulename, elements) grammar.add_rule(rulename, *elements) return grammar # (1) Get the tokenizer (2) Compile the grammar (3) Build the parser abnf_grammar.get_tokenizer() abnf_grammar.compile_grammar(Context) abnf_parser = get_parser(abnf_grammar, 'rulelist') ########################################################################### # The Public API ########################################################################### def build_grammar(data, context_class=None): context = Context(data) grammar = abnf_parser.run(data, context) grammar.get_tokenizer() grammar.compile_grammar(context_class) return grammar