def __init__(self, host: str = 'localhost', port: int = 6379) -> None: self.__client = StrictRedis(host, port=port, decode_responses=True) self.__client.config_set('maxmemory', '600mb') self.__client.config_set('maxmemory-policy', 'allkeys-lru') self.__cache = RedisCache(redis_client=self.__client) self.__logger = Logger().getLogger(__file__) self.__logger.info("Initialize Cache.")
def main(argv) -> None: del argv # Process the configuration from flags. config = process_config() if config.mode != "evaluate": # Define the datasets. train_dataset = MatchingDataset( batch_size=config.batch_size, folder="datasets/matching_aracati/train", x_shape=config.input_shape, y_shape=config.output_shape, is_evaluating=False) valid_dataset = MatchingDataset( batch_size=config.batch_size, folder="datasets/matching_aracati/validation", x_shape=config.input_shape, y_shape=config.output_shape, is_evaluating=False) # Define the model. loss = tf.keras.losses.BinaryCrossentropy() ranger = Lookahead(RectifiedAdam(learning_rate=config.learning_rate), sync_period=6, slow_step_size=0.5) model = DizygoticNet(filters=config.filters, loss=loss, optimizer=ranger) # Define the logger. logger = Logger() # Define the trainer. trainer = MatchingTrainer(model=model, logger=logger, train_dataset=train_dataset, valid_dataset=valid_dataset) if config.mode == "restore": trainer.load_checkpoint() trainer.train() else: # Define the test dataset. test_dataset = MatchingDataset(batch_size=1, folder="datasets/matching_aracati/test", x_shape=config.input_shape, y_shape=config.output_shape, is_evaluating=True) # Define the model. model = DizygoticNet(filters=config.filters, loss=None, optimizer=None) # Define the evaluator. evaluator = MatchingEvaluator(model=model, dataset=test_dataset) evaluator.load_checkpoint() evaluator.evaluate()
def main(argv) -> None: del argv # Process the configuration from flags. config = process_config() if config.mode != "evaluate": # Define the datasets. train_dataset = TripletDataset(batch_size=config.batch_size, folder="datasets/triplet_aracati/train", x_shape=config.input_shape, y_shape=config.output_shape, is_validation=False) valid_dataset = TripletDataset( batch_size=config.batch_size, folder="datasets/triplet_aracati/validation", x_shape=config.input_shape, y_shape=config.output_shape, is_validation=True) # Define the sonar model. son_loss = TripletLoss() son_ranger = Lookahead( RectifiedAdam(learning_rate=config.learning_rate), sync_period=6, slow_step_size=0.5) son_model = EncodeNet(filters=config.filters, loss=son_loss, optimizer=son_ranger) # Define the satellite model. sat_loss = TripletLoss() sat_ranger = Lookahead( RectifiedAdam(learning_rate=config.learning_rate), sync_period=6, slow_step_size=0.5) sat_model = EncodeNet(filters=config.filters, loss=sat_loss, optimizer=sat_ranger) # Define the logger. logger = Logger() # Define the trainer. trainer = TripletTrainer(son_model=son_model, sat_model=sat_model, logger=logger, train_dataset=train_dataset, valid_dataset=valid_dataset) if config.mode == "restore": trainer.load_checkpoint() trainer.train() else: logging.fatal("Evaluation not implemented yet.")
def train(): model = CTPN(cfg) data = DataLoader(cfg) # imdb = data.load_imdb('voc_2007_trainval') # roidb = data.get_training_roidb(imdb) sess = get_sess() logger = Logger(sess, cfg) trainer = CTPNTrainer(sess, model, data, logger) print('Solving...') trainer.train(cfg.TRAIN.MAX_ITERS, restore=False) print('done solving')
def main(argv) -> None: del argv # Process the configuration from flags. config = process_config() if config.mode != "evaluate": # Define the datasets. train_dataset = GeneralDataset(batch_size=config.batch_size, folder="datasets/general_aracati/train", x_shape=config.input_shape, y_shape=config.satellite_shape, z_shape=config.output_shape) valid_dataset = GeneralDataset( batch_size=config.batch_size, folder="datasets/general_aracati/validation", x_shape=config.input_shape, y_shape=config.satellite_shape, z_shape=config.output_shape) # Define the model. loss = tf.keras.losses.MeanAbsoluteError() ranger = Lookahead(RectifiedAdam(learning_rate=config.learning_rate), sync_period=6, slow_step_size=0.5) model = WNet(filters=config.filters, loss=loss, optimizer=ranger) # Define the logger. logger = Logger() # Define the trainer. trainer = GeneralTrainer(model=model, logger=logger, train_dataset=train_dataset, valid_dataset=valid_dataset) if config.mode == "restore": trainer.load_checkpoint() trainer.train() else: # Define the test dataset. test_dataset = GeneralDataset(batch_size=1, folder="datasets/general_aracati/test", x_shape=config.input_shape, y_shape=config.satellite_shape, z_shape=config.output_shape) # Define the model. model = WNet(filters=config.filters, loss=None, optimizer=None) # Define the evaluator. evaluator = GeneralEvaluator(model=model, dataset=test_dataset) evaluator.load_checkpoint() evaluator.evaluate()
class Cache(): def __init__(self, host: str = 'localhost', port: int = 6379) -> None: self.__client = StrictRedis(host, port=port, decode_responses=True) self.__client.config_set('maxmemory', '600mb') self.__client.config_set('maxmemory-policy', 'allkeys-lru') self.__cache = RedisCache(redis_client=self.__client) self.__logger = Logger().getLogger(__file__) self.__logger.info("Initialize Cache.") def key_exists(self, *args): serialized_data = dumps([args[1:], {}]) key = f'rc:{args[0]}:{serialized_data}' return self.__client.exists(key) >= 1 def get_all_keys(self): return self.__client.keys() def get_key_count(self): return len(self.get_all_keys()) def __call__(self, ttl=60 * 60 * 24 * 7, limit=5000, namespace=None): return self.__cache.cache(ttl, limit, namespace)
def main(argv) -> None: del argv # Process the configuration from flags. config = process_config() if config.mode != "evaluate": # Define the datasets. train_dataset = MatchingDataset(batch_size=config.batch_size, folder="datasets/aracati/train", x_shape=config.input_shape, y_shape=config.output_shape) valid_dataset = MatchingDataset(batch_size=config.batch_size, folder="datasets/aracati/validation", x_shape=config.input_shape, y_shape=config.output_shape) # Define the model. loss = tf.keras.losses.BinaryCrossentropy() optimizer = tf.keras.optimizers.Adam( learning_rate=config.learning_rate) model = DizygoticNet(filters=config.filters, loss=loss, optimizer=optimizer) if config.mode == "restore": model.load_checkpoint() # Define the logger. logger = Logger() # Define the trainer. trainer = MatchingTrainer(model=model, logger=logger, train_dataset=train_dataset, valid_dataset=valid_dataset) trainer.train() else: logging.fatal("Evaluation mode is not yet implemented.") exit(1)
def main(argv) -> None: del argv # Process the configuration from flags. config = process_config() if config.mode != "evaluate": # Define the datasets. train_dataset = SegmentationDataset( batch_size=config.batch_size, folder="datasets/segmentation_aracati/train", x_shape=config.input_shape, y_shape=config.output_shape) valid_dataset = SegmentationDataset( batch_size=config.batch_size, folder="datasets/segmentation_aracati/validation", x_shape=config.input_shape, y_shape=config.output_shape) # Define the model. loss = tf.keras.losses.CategoricalCrossentropy() ranger = Lookahead(RectifiedAdam(learning_rate=config.learning_rate), sync_period=6, slow_step_size=0.5) model = UNet(filters=config.filters, loss=loss, optimizer=ranger) # Define the logger. logger = Logger() # Define the trainer. trainer = SegmentationTrainer(model=model, logger=logger, train_dataset=train_dataset, valid_dataset=valid_dataset) if config.mode == "restore": trainer.load_checkpoint() trainer.train() else: logging.fatal("Evaluation not implemented yet.")
cfg.dist_fn_opt, cfg.img_backbone_opt, cfg.cap_backbone_opt, cfg.embed_size, cfg.batch_size, cfg.lr, cfg.step_size, cfg.cap_embed_type, cfg.image_melt_layer, cfg.np) if cfg.note: cfg.exp_name += "_" + cfg.note cfg.model_path = os.path.join("/shared/rsaas/aiyucui2/wider_person", cfg.model_path, cfg.exp_name) cfg.output_path = os.path.join("/shared/rsaas/aiyucui2/wider_person", cfg.output_path, cfg.exp_name) if not os.path.exists(cfg.model_path): os.mkdir(cfg.model_path) if not os.path.exists(cfg.output_path): os.mkdir(cfg.output_path) logger = Logger(os.path.join(cfg.output_path, "log.txt")) logger.log(str(cfg)) #------------------ ## agents #------------------ # Data loaders train_loader = build_wider_dataloader(cfg) cfg.num_ids = len(train_loader.dataset.person2label.values()) test_text_loader = build_text_test_loader(cfg) test_image_loader = build_image_test_loader(cfg) # Evaluators Evaluator = NPEvaluator if cfg.np else GlobalEvaluator evaluator = Evaluator(img_loader=test_image_loader,
def main(): """Main function""" args = parse_args() print('Called with args:') print(args) if not torch.cuda.is_available(): sys.exit("Need a CUDA device to run the code.") if args.cuda or cfg.NUM_GPUS > 0: cfg.CUDA = True else: raise ValueError("Need Cuda device to run !") if args.dataset == "fis_cell": cfg.TRAIN.DATASETS = ('fis_cell_train_val',) cfg.MODEL.NUM_CLASSES = 2 elif args.dataset == "coco2017": cfg.TRAIN.DATASETS = ('coco_2017_train',) cfg.MODEL.NUM_CLASSES = 81 elif args.dataset == "keypoints_coco2017": cfg.TRAIN.DATASETS = ('keypoints_coco_2017_train',) cfg.MODEL.NUM_CLASSES = 2 else: raise ValueError("Unexpected args.dataset: {}".format(args.dataset)) if args.close_fpn: args.cfg_file = "configs/few_shot/e2e_mask_rcnn_R-50-C4_1x_{}.yaml".format(args.group) cfg.OUTPUT_DIR = 'Outputs_wo_fpn' else: args.cfg_file = "configs/few_shot/e2e_mask_rcnn_R-50-FPN_1x_{}.yaml".format(args.group) cfg_from_file(args.cfg_file) if args.set_cfgs is not None: cfg_from_list(args.set_cfgs) cfg.RNG_SEED = args.random_seed if cfg.RNG_SEED is None: torch.backends.cudnn.deterministic = False torch.backends.cudnn.benchmark = True else: print('Make the experiment results deterministic.') random.seed(cfg.RNG_SEED) np.random.seed(cfg.RNG_SEED) torch.manual_seed(cfg.RNG_SEED) torch.cuda.manual_seed_all(cfg.RNG_SEED) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False cfg.SEEN = args.seen if args.close_co_atten: cfg.CO_ATTEN = False if not args.close_fpn: cfg.OUTPUT_DIR = 'Outputs_wo_co_atten' if args.close_relation_rcnn: cfg.RELATION_RCNN = False if not args.close_fpn: cfg.FAST_RCNN.ROI_BOX_HEAD = 'fast_rcnn_heads.roi_2mlp_head' cfg.MRCNN.ROI_MASK_HEAD = 'mask_rcnn_heads.mask_rcnn_fcn_head_v1up4convs' else: cfg.FAST_RCNN.ROI_BOX_HEAD = 'torchResNet.ResNet_roi_conv5_head' cfg.MRCNN.ROI_MASK_HEAD = 'mask_rcnn_heads.mask_rcnn_fcn_head_v0upshare' if not args.close_co_atten and not args.close_fpn: cfg.OUTPUT_DIR = 'Outputs_wo_relation_rcnn' if args.output_dir is not None: cfg.OUTPUT_DIR = args.output_dir if args.deform_conv: cfg.MODEL.USE_DEFORM = True ### Adaptively adjust some configs ### original_batch_size = cfg.NUM_GPUS * cfg.TRAIN.IMS_PER_BATCH original_ims_per_batch = cfg.TRAIN.IMS_PER_BATCH original_num_gpus = cfg.NUM_GPUS if args.batch_size is None: args.batch_size = original_batch_size cfg.NUM_GPUS = torch.cuda.device_count() assert (args.batch_size % cfg.NUM_GPUS) == 0, \ 'batch_size: %d, NUM_GPUS: %d' % (args.batch_size, cfg.NUM_GPUS) cfg.TRAIN.IMS_PER_BATCH = args.batch_size // cfg.NUM_GPUS effective_batch_size = args.iter_size * args.batch_size print('effective_batch_size = batch_size * iter_size = %d * %d' % (args.batch_size, args.iter_size)) print('Adaptive config changes:') print(' effective_batch_size: %d --> %d' % (original_batch_size, effective_batch_size)) print(' NUM_GPUS: %d --> %d' % (original_num_gpus, cfg.NUM_GPUS)) print(' IMS_PER_BATCH: %d --> %d' % (original_ims_per_batch, cfg.TRAIN.IMS_PER_BATCH)) ### Adjust learning based on batch size change linearly # For iter_size > 1, gradients are `accumulated`, so lr is scaled based # on batch_size instead of effective_batch_size old_base_lr = cfg.SOLVER.BASE_LR cfg.SOLVER.BASE_LR *= args.batch_size / original_batch_size print('Adjust BASE_LR linearly according to batch_size change:\n' ' BASE_LR: {} --> {}'.format(old_base_lr, cfg.SOLVER.BASE_LR)) ### Adjust solver steps step_scale = original_batch_size / effective_batch_size old_solver_steps = cfg.SOLVER.STEPS old_max_iter = cfg.SOLVER.MAX_ITER cfg.SOLVER.STEPS = list(map(lambda x: int(x * step_scale + 0.5), cfg.SOLVER.STEPS)) cfg.SOLVER.MAX_ITER = int(cfg.SOLVER.MAX_ITER * step_scale + 0.5) print('Adjust SOLVER.STEPS and SOLVER.MAX_ITER linearly based on effective_batch_size change:\n' ' SOLVER.STEPS: {} --> {}\n' ' SOLVER.MAX_ITER: {} --> {}'.format(old_solver_steps, cfg.SOLVER.STEPS, old_max_iter, cfg.SOLVER.MAX_ITER)) # Scale FPN rpn_proposals collect size (post_nms_topN) in `collect` function # of `collect_and_distribute_fpn_rpn_proposals.py` # # post_nms_topN = int(cfg[cfg_key].RPN_POST_NMS_TOP_N * cfg.FPN.RPN_COLLECT_SCALE + 0.5) if cfg.FPN.FPN_ON and cfg.MODEL.FASTER_RCNN: cfg.FPN.RPN_COLLECT_SCALE = cfg.TRAIN.IMS_PER_BATCH / original_ims_per_batch print('Scale FPN rpn_proposals collect size directly propotional to the change of IMS_PER_BATCH:\n' ' cfg.FPN.RPN_COLLECT_SCALE: {}'.format(cfg.FPN.RPN_COLLECT_SCALE)) if args.num_workers is not None: cfg.DATA_LOADER.NUM_THREADS = args.num_workers print('Number of data loading threads: %d' % cfg.DATA_LOADER.NUM_THREADS) ### Overwrite some solver settings from command line arguments if args.optimizer is not None: cfg.SOLVER.TYPE = args.optimizer if args.lr is not None: cfg.SOLVER.BASE_LR = args.lr if args.lr_decay_gamma is not None: cfg.SOLVER.GAMMA = args.lr_decay_gamma assert_and_infer_cfg() timers = defaultdict(Timer) ### Dataset ### timers['roidb'].tic() imdb, roidb, ratio_list, ratio_index, query, cat_list = combined_roidb( cfg.TRAIN.DATASETS, True) timers['roidb'].toc() roidb_size = len(roidb) logger.info('{:d} roidb entries'.format(roidb_size)) logger.info('Takes %.2f sec(s) to construct roidb', timers['roidb'].average_time) # Effective training sample size for one epoch train_size = roidb_size // args.batch_size * args.batch_size batchSampler = BatchSampler( sampler=MinibatchSampler(ratio_list, ratio_index), batch_size=args.batch_size, drop_last=True ) dataset = RoiDataLoader( roidb, ratio_list, ratio_index, query, cfg.MODEL.NUM_CLASSES, training=True, cat_list=cat_list, shot=args.shot) dataloader = torch.utils.data.DataLoader( dataset, batch_sampler=batchSampler, num_workers=cfg.DATA_LOADER.NUM_THREADS, collate_fn=collate_minibatch) dataiterator = iter(dataloader) ### Model ### maskRCNN = Generalized_RCNN() if cfg.CUDA: maskRCNN.cuda() ### Optimizer ### gn_param_nameset = set() for name, module in maskRCNN.named_modules(): if isinstance(module, nn.GroupNorm): gn_param_nameset.add(name+'.weight') gn_param_nameset.add(name+'.bias') gn_params = [] gn_param_names = [] bias_params = [] bias_param_names = [] nonbias_params = [] nonbias_param_names = [] nograd_param_names = [] for key, value in maskRCNN.named_parameters(): if value.requires_grad: if 'bias' in key: bias_params.append(value) bias_param_names.append(key) elif key in gn_param_nameset: gn_params.append(value) gn_param_names.append(key) else: nonbias_params.append(value) nonbias_param_names.append(key) else: nograd_param_names.append(key) assert (gn_param_nameset - set(nograd_param_names) - set(bias_param_names)) == set(gn_param_names) # Learning rate of 0 is a dummy value to be set properly at the start of training params = [ {'params': nonbias_params, 'lr': 0, 'weight_decay': cfg.SOLVER.WEIGHT_DECAY}, {'params': bias_params, 'lr': 0 * (cfg.SOLVER.BIAS_DOUBLE_LR + 1), 'weight_decay': cfg.SOLVER.WEIGHT_DECAY if cfg.SOLVER.BIAS_WEIGHT_DECAY else 0}, {'params': gn_params, 'lr': 0, 'weight_decay': cfg.SOLVER.WEIGHT_DECAY_GN} ] # names of paramerters for each paramter param_names = [nonbias_param_names, bias_param_names, gn_param_names] if cfg.SOLVER.TYPE == "SGD": optimizer = torch.optim.SGD(params, momentum=cfg.SOLVER.MOMENTUM) elif cfg.SOLVER.TYPE == "Adam": optimizer = torch.optim.Adam(params) ### Load checkpoint if args.load_ckpt: load_name = args.load_ckpt logging.info("loading checkpoint %s", load_name) checkpoint = torch.load(load_name, map_location=lambda storage, loc: storage) net_utils.load_ckpt(maskRCNN, checkpoint['model']) if args.resume: args.shot = checkpoint['shot'] args.start_step = checkpoint['step'] + 1 if 'train_size' in checkpoint: # For backward compatibility if checkpoint['train_size'] != train_size: print('train_size value: %d different from the one in checkpoint: %d' % (train_size, checkpoint['train_size'])) # reorder the params in optimizer checkpoint's params_groups if needed # misc_utils.ensure_optimizer_ckpt_params_order(param_names, checkpoint) # There is a bug in optimizer.load_state_dict on Pytorch 0.3.1. # However it's fixed on master. optimizer.load_state_dict(checkpoint['optimizer']) # misc_utils.load_optimizer_state_dict(optimizer, checkpoint['optimizer']) del checkpoint torch.cuda.empty_cache() if args.load_detectron: #TODO resume for detectron weights (load sgd momentum values) logging.info("loading Detectron weights %s", args.load_detectron) load_detectron_weight(maskRCNN, args.load_detectron) lr = optimizer.param_groups[0]['lr'] # lr of non-bias parameters, for commmand line outputs. maskRCNN = mynn.DataParallel(maskRCNN, cpu_keywords=['im_info', 'roidb'], minibatch=True) ### Training Setups ### args.run_name = misc_utils.get_run_name() + '_step' output_dir = misc_utils.get_output_dir(args, args.run_name) args.cfg_filename = os.path.basename(args.cfg_file) if not args.no_save: if not os.path.exists(output_dir): os.makedirs(output_dir) blob = {'cfg': yaml.dump(cfg), 'args': args} with open(os.path.join(output_dir, 'config_and_args.pkl'), 'wb') as f: pickle.dump(blob, f, pickle.HIGHEST_PROTOCOL) if args.use_tfboard: #from tensorboardX import SummaryWriter # Set the Tensorboard logger #tblogger = SummaryWriter(output_dir) from loggers.logger import Logger tblogger = Logger(output_dir) ### Training Loop ### maskRCNN.train() CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS) # Set index for decay steps decay_steps_ind = None for i in range(1, len(cfg.SOLVER.STEPS)): if cfg.SOLVER.STEPS[i] >= args.start_step: decay_steps_ind = i break if decay_steps_ind is None: decay_steps_ind = len(cfg.SOLVER.STEPS) training_stats = TrainingStats( args, args.disp_interval, tblogger if args.use_tfboard and not args.no_save else None) try: logger.info('Training starts !') step = args.start_step for step in range(args.start_step, cfg.SOLVER.MAX_ITER): # Warm up if step < cfg.SOLVER.WARM_UP_ITERS: #for p in maskRCNN.module.Conv_Body.parameters(): # p.requires_grad = False method = cfg.SOLVER.WARM_UP_METHOD if method == 'constant': warmup_factor = cfg.SOLVER.WARM_UP_FACTOR elif method == 'linear': alpha = step / cfg.SOLVER.WARM_UP_ITERS warmup_factor = cfg.SOLVER.WARM_UP_FACTOR * (1 - alpha) + alpha else: raise KeyError('Unknown SOLVER.WARM_UP_METHOD: {}'.format(method)) lr_new = cfg.SOLVER.BASE_LR * warmup_factor net_utils.update_learning_rate(optimizer, lr, lr_new) lr = optimizer.param_groups[0]['lr'] assert lr == lr_new elif step == cfg.SOLVER.WARM_UP_ITERS: #for p in maskRCNN.module.Conv_Body.parameters(): # p.requires_grad = True net_utils.update_learning_rate(optimizer, lr, cfg.SOLVER.BASE_LR) lr = optimizer.param_groups[0]['lr'] assert lr == cfg.SOLVER.BASE_LR # Learning rate decay if decay_steps_ind < len(cfg.SOLVER.STEPS) and \ step == cfg.SOLVER.STEPS[decay_steps_ind]: logger.info('Decay the learning on step %d', step) lr_new = lr * cfg.SOLVER.GAMMA net_utils.update_learning_rate(optimizer, lr, lr_new) lr = optimizer.param_groups[0]['lr'] assert lr == lr_new decay_steps_ind += 1 training_stats.IterTic() optimizer.zero_grad() for inner_iter in range(args.iter_size): try: input_data = next(dataiterator) except StopIteration: dataiterator = iter(dataloader) input_data = next(dataiterator) for key in input_data: if key != 'roidb' and key != 'query': # roidb is a list of ndarrays with inconsistent length input_data[key] = list(map(Variable, input_data[key])) if key == 'query': input_data[key] = [list(map(Variable, q)) for q in input_data[key]] with torch.autograd.detect_anomaly(): net_outputs = maskRCNN(**input_data) training_stats.UpdateIterStats(net_outputs, inner_iter) loss = net_outputs['total_loss'] loss.backward() torch.nn.utils.clip_grad_value_(maskRCNN.module.parameters(), clip_value=0.4) optimizer.step() training_stats.IterToc() training_stats.LogIterStats(step, lr, input_data, args.shot) if (step+1) % CHECKPOINT_PERIOD == 0: save_ckpt(output_dir, args, step, train_size, maskRCNN, optimizer) if (step+1) % args.disp_interval == 0: log_training_stats(training_stats, step, lr) # ---- Training ends ---- # Save last checkpoint save_ckpt(output_dir, args, step, train_size, maskRCNN, optimizer) except (RuntimeError, KeyboardInterrupt): del dataiterator logger.info('Save ckpt on exception ...') save_ckpt(output_dir, args, step, train_size, maskRCNN, optimizer) logger.info('Save ckpt done.') stack_trace = traceback.format_exc() print(stack_trace) finally: if args.use_tfboard and not args.no_save: tblogger.close()
class CompilerState: def __init__(self, print_table=False, table_logfile='log_symbol_table.txt', print_tokens=False, print_source_scanner=True, scanner_logfile='log_scanner_tokens.txt', print_productions=False, print_source_parser=False, print_info=False, parser_logfile=sys.stdout, print_warnings=False, **kwargs): # Initialize variables self.source_code = None self.source_lines = None # Initialize table self.symbol_table = SymbolTable() # Initialize symbol table logger if table_logfile in {sys.stdout, sys.stderr}: self.symbol_table_logger = Logger(table_logfile) else: self.symbol_table_logger = Logger(open(table_logfile, 'w')) if print_table: self.symbol_table_logger.add_switch(Logger.SYMBOL_TABLE) # Initialize token/lexer logger if scanner_logfile in {sys.stdout, sys.stderr}: self.token_logger = Logger(scanner_logfile) else: self.token_logger = Logger(open(scanner_logfile, 'w')) if print_source_scanner: self.token_logger.add_switch(Logger.SOURCE) if print_tokens: self.token_logger.add_switch(Logger.TOKEN) # Initialize parser logger if parser_logfile in {sys.stdout, sys.stderr}: self.parser_logger = Logger(parser_logfile) else: self.parser_logger = Logger(open(parser_logfile, 'w')) if print_source_parser: self.parser_logger.add_switch(Logger.SOURCE) if print_productions: self.parser_logger.add_switch(Logger.PRODUCTION) if print_info: self.parser_logger.add_switch(Logger.INFO) # Initialize warning logger self.warning_logger = Logger(sys.stdout) if print_warnings: self.warning_logger.add_switch(Logger.WARNING) # Other stuff self.function_scope_entered = False self.insert_mode = True # for debugging purposes self.clone_symbol_table_on_next_scope_exit = False self.cloned_tables = [] # Create JSTLexer and the lexer object self.jst_lexer = JSTLexer(self) self.lexer = lex.lex(module=self.jst_lexer) # Create JSTParser and the parser object self.jst_parser = JSTParser(self) self.parser = yacc.yacc(module=self.jst_parser, start='program') # we will need a reference to the symbol for the main function self.main_function = None def parse(self, source_code): # Lex uses 1 based indexing for line numbers. # We are using 0 based for source_code. if source_code is not None: self.source_code = source_code self.source_lines = source_code.split('\n') else: self.source_code = None self.source_lines = None # Parse using the parser object return self.parser.parse(input=self.source_code, lexer=self.lexer, tracking=True) def teardown(self): self.jst_lexer.teardown() self.jst_parser.teardown() def get_symbol_table_logger(self): return self.symbol_table_logger def get_token_logger(self): return self.token_logger def get_parser_logger(self): return self.parser_logger def get_warning_logger(self): return self.warning_logger def get_line_col(self, production, index): lexpos = production.lexpos(index) last_newline = self.source_code.rfind('\n', 0, lexpos) return production.lineno(index), max(0, lexpos - last_newline) def get_line_col_source(self, lineno, lexpos): last_newline = self.source_code.rfind('\n', 0, lexpos) return ( lineno, max(0, lexpos - last_newline), self.source_lines[lineno - 1] )
def main(): """Main function""" args = parse_args() print('Called with args:') print(args) if not torch.cuda.is_available(): sys.exit("Need a CUDA device to run the code.") if args.cuda or cfg.NUM_GPUS > 0: cfg.CUDA = True else: raise ValueError("Need Cuda device to run !") if args.dataset == "fis_cell": cfg.TRAIN.DATASETS = ('fis_cell_train_val',) cfg.MODEL.NUM_CLASSES = 2 elif args.dataset == "coco2017": cfg.TRAIN.DATASETS = ('coco_2017_train',) cfg.MODEL.NUM_CLASSES = 81 elif args.dataset == "keypoints_coco2017": cfg.TRAIN.DATASETS = ('keypoints_coco_2017_train',) cfg.MODEL.NUM_CLASSES = 2 else: raise ValueError("Unexpected args.dataset: {}".format(args.dataset)) args.cfg_file = "configs/few_shot/e2e_mask_rcnn_R-50-FPN_1x_{}.yaml".format(args.group) cfg_from_file(args.cfg_file) if args.set_cfgs is not None: cfg_from_list(args.set_cfgs) cfg.SEEN = args.seen ### Adaptively adjust some configs ### original_batch_size = cfg.NUM_GPUS * cfg.TRAIN.IMS_PER_BATCH original_ims_per_batch = cfg.TRAIN.IMS_PER_BATCH original_num_gpus = cfg.NUM_GPUS if args.batch_size is None: args.batch_size = original_batch_size cfg.NUM_GPUS = torch.cuda.device_count() assert (args.batch_size % cfg.NUM_GPUS) == 0, \ 'batch_size: %d, NUM_GPUS: %d' % (args.batch_size, cfg.NUM_GPUS) cfg.TRAIN.IMS_PER_BATCH = args.batch_size // cfg.NUM_GPUS print('Adaptive config changes:') print(' batch_size: %d --> %d' % (original_batch_size, args.batch_size)) print(' NUM_GPUS: %d --> %d' % (original_num_gpus, cfg.NUM_GPUS)) print(' IMS_PER_BATCH: %d --> %d' % (original_ims_per_batch, cfg.TRAIN.IMS_PER_BATCH)) if args.num_workers is not None: cfg.DATA_LOADER.NUM_THREADS = args.num_workers print('Number of data loading threads: %d' % cfg.DATA_LOADER.NUM_THREADS) ### Adjust learning based on batch size change linearly old_base_lr = cfg.SOLVER.BASE_LR cfg.SOLVER.BASE_LR *= args.batch_size / original_batch_size print('Adjust BASE_LR linearly according to batch size change: {} --> {}'.format( old_base_lr, cfg.SOLVER.BASE_LR)) # Scale FPN rpn_proposals collect size (post_nms_topN) in `collect` function # of `collect_and_distribute_fpn_rpn_proposals.py` # # post_nms_topN = int(cfg[cfg_key].RPN_POST_NMS_TOP_N * cfg.FPN.RPN_COLLECT_SCALE + 0.5) if cfg.FPN.FPN_ON and cfg.MODEL.FASTER_RCNN: cfg.FPN.RPN_COLLECT_SCALE = cfg.TRAIN.IMS_PER_BATCH / original_ims_per_batch print('Scale FPN rpn_proposals collect size directly propotional to the change of IMS_PER_BATCH:\n' ' cfg.FPN.RPN_COLLECT_SCALE: {}'.format(cfg.FPN.RPN_COLLECT_SCALE)) ### Overwrite some solver settings from command line arguments if args.optimizer is not None: cfg.SOLVER.TYPE = args.optimizer if args.lr is not None: cfg.SOLVER.BASE_LR = args.lr if args.lr_decay_gamma is not None: cfg.SOLVER.GAMMA = args.lr_decay_gamma timers = defaultdict(Timer) ### Dataset ### timers['roidb'].tic() imdb, roidb, ratio_list, ratio_index, query, cat_list = combined_roidb( cfg.TRAIN.DATASETS, True) timers['roidb'].toc() train_size = len(roidb) logger.info('{:d} roidb entries'.format(train_size)) logger.info('Takes %.2f sec(s) to construct roidb', timers['roidb'].average_time) sampler = MinibatchSampler(ratio_list, ratio_index) dataset = RoiDataLoader( roidb, ratio_list, ratio_index, query, cfg.MODEL.NUM_CLASSES, training=True, cat_list=cat_list, shot=args.shot) dataloader = torch.utils.data.DataLoader( dataset, batch_size=args.batch_size, sampler=sampler, num_workers=cfg.DATA_LOADER.NUM_THREADS, collate_fn=collate_minibatch) assert_and_infer_cfg() ### Model ### maskRCNN = Generalized_RCNN() if cfg.CUDA: maskRCNN.cuda() ### Optimizer ### bias_params = [] nonbias_params = [] for key, value in dict(maskRCNN.named_parameters()).items(): if value.requires_grad: if 'bias' in key: bias_params.append(value) else: nonbias_params.append(value) params = [ {'params': nonbias_params, 'lr': cfg.SOLVER.BASE_LR, 'weight_decay': cfg.SOLVER.WEIGHT_DECAY}, {'params': bias_params, 'lr': cfg.SOLVER.BASE_LR * (cfg.SOLVER.BIAS_DOUBLE_LR + 1), 'weight_decay': cfg.SOLVER.WEIGHT_DECAY if cfg.SOLVER.BIAS_WEIGHT_DECAY else 0} ] if cfg.SOLVER.TYPE == "SGD": optimizer = torch.optim.SGD(params, momentum=cfg.SOLVER.MOMENTUM) elif cfg.SOLVER.TYPE == "Adam": optimizer = torch.optim.Adam(params) ### Load checkpoint if args.load_ckpt: load_name = args.load_ckpt logging.info("loading checkpoint %s", load_name) checkpoint = torch.load(load_name, map_location=lambda storage, loc: storage) net_utils.load_ckpt(maskRCNN, checkpoint['model']) if args.resume: assert checkpoint['iters_per_epoch'] == train_size // args.batch_size, \ "iters_per_epoch should match for resume" # There is a bug in optimizer.load_state_dict on Pytorch 0.3.1. # However it's fixed on master. # optimizer.load_state_dict(checkpoint['optimizer']) misc_utils.load_optimizer_state_dict(optimizer, checkpoint['optimizer']) args.shot = checkpoint['shot'] if checkpoint['step'] == (checkpoint['iters_per_epoch'] - 1): # Resume from end of an epoch args.start_epoch = checkpoint['epoch'] + 1 args.start_iter = 0 else: # Resume from the middle of an epoch. # NOTE: dataloader is not synced with previous state args.start_epoch = checkpoint['epoch'] args.start_iter = checkpoint['step'] + 1 del checkpoint torch.cuda.empty_cache() if args.load_detectron: #TODO resume for detectron weights (load sgd momentum values) logging.info("loading Detectron weights %s", args.load_detectron) load_detectron_weight(maskRCNN, args.load_detectron) lr = optimizer.param_groups[0]['lr'] # lr of non-bias parameters, for commmand line outputs. maskRCNN = mynn.DataParallel(maskRCNN, cpu_keywords=['im_info', 'roidb'], minibatch=True) ### Training Setups ### args.run_name = misc_utils.get_run_name() output_dir = misc_utils.get_output_dir(args, args.run_name) args.cfg_filename = os.path.basename(args.cfg_file) if not args.no_save: if not os.path.exists(output_dir): os.makedirs(output_dir) blob = {'cfg': yaml.dump(cfg), 'args': args} with open(os.path.join(output_dir, 'config_and_args.pkl'), 'wb') as f: pickle.dump(blob, f, pickle.HIGHEST_PROTOCOL) if args.use_tfboard: #from tensorboardX import SummaryWriter # Set the Tensorboard logger #tblogger = SummaryWriter(output_dir) from loggers.logger import Logger tblogger = Logger(output_dir) ### Training Loop ### maskRCNN.train() training_stats = TrainingStats( args, args.disp_interval, tblogger if args.use_tfboard and not args.no_save else None) iters_per_epoch = int(train_size / args.batch_size) # drop last args.iters_per_epoch = iters_per_epoch ckpt_interval_per_epoch = iters_per_epoch // args.ckpt_num_per_epoch try: logger.info('Training starts !') args.step = args.start_iter global_step = iters_per_epoch * args.start_epoch + args.step for args.epoch in range(args.start_epoch, args.start_epoch + args.num_epochs): # ---- Start of epoch ---- # adjust learning rate if args.lr_decay_epochs and args.epoch == args.lr_decay_epochs[0] and args.start_iter == 0: args.lr_decay_epochs.pop(0) net_utils.decay_learning_rate(optimizer, lr, cfg.SOLVER.GAMMA) lr *= cfg.SOLVER.GAMMA for args.step, input_data in zip(range(args.start_iter, iters_per_epoch), dataloader): for key in input_data: if key != 'roidb' and key != 'query': # roidb is a list of ndarrays with inconsistent length input_data[key] = list(map(Variable, input_data[key])) if key == 'query': input_data[key] = [list(map(Variable, q)) for q in input_data[key]] training_stats.IterTic() net_outputs = maskRCNN(**input_data) training_stats.UpdateIterStats(net_outputs) loss = net_outputs['total_loss'] optimizer.zero_grad() loss.backward() optimizer.step() training_stats.IterToc() #if (args.step+1) % ckpt_interval_per_epoch == 0: if global_step % 500 == 0: net_utils.save_ckpt(output_dir, args, maskRCNN, optimizer) if args.step % args.disp_interval == 0: log_training_stats(training_stats, global_step, lr, input_data, args.shot) global_step += 1 # ---- End of epoch ---- # save checkpoint #net_utils.save_ckpt(output_dir, args, maskRCNN, optimizer) # reset starting iter number after first epoch args.start_iter = 0 # ---- Training ends ---- net_utils.save_ckpt(output_dir, args, maskRCNN, optimizer) if iters_per_epoch % args.disp_interval != 0: # log last stats at the end log_training_stats(training_stats, global_step, lr, input_data, args.shot) except (RuntimeError, KeyboardInterrupt): logger.info('Save ckpt on exception ...') net_utils.save_ckpt(output_dir, args, maskRCNN, optimizer) logger.info('Save ckpt done.') stack_trace = traceback.format_exc() print(stack_trace) finally: if args.use_tfboard and not args.no_save: tblogger.close()
load_dotenv(find_dotenv()) # Global variables with open(r'configs/zone_settings.yaml') as file: zones = yaml.load(file, Loader=yaml.FullLoader) with open(r'configs/class_settings.yaml') as file: class_settings = yaml.load(file, Loader=yaml.FullLoader) # Initialize the app app = dash.Dash(__name__) server = app.server # needed to launch gunicorn app.config.suppress_callback_exceptions = True # Initialize logger logger = Logger().getLogger(__file__) # Initialize WCL client client = WCLClient() # Get users try: with open(r'configs/users.yaml') as file: USERS = yaml.load(file, Loader=yaml.FullLoader) except FileNotFoundError: logger.error("User list not found.") raise # Initialize basic auth auth = dash_auth.BasicAuth(app, parse_users(USERS))
from datetime import datetime import pandas as pd from loggers.logger import Logger # Initialize logger logger = Logger().getLogger(__file__) THRESHOLD_PERCENTAGE = 0.10 def create_normalized_column(df, col_name): df[f'norm_{col_name}'] = 100 * df[col_name] / df[col_name].sum() return df def drop_rows_where_col_has_value(df, col_name, value): df = df[df[col_name] != value] return df def average_logs(logs): return pd.concat( [ pd.DataFrame(log["entries"])[['name', 'type', 'total']] .pipe(drop_rows_where_col_has_value, col_name='type', value='Pet') .pipe(create_normalized_column, col_name='total') for log in logs ] ).groupby(['name']).agg( _std=('norm_total', lambda x: x.unique().std()), # just 'std' will use ddof = 1
def __init__(self, *args, **kwargs): super(Test, self).__init__(*args, **kwargs) self.logger = Logger('logs/', 'screenshots/') self.getChromeOptions()
def __init__(self, print_table=False, table_logfile='log_symbol_table.txt', print_tokens=False, print_source_scanner=True, scanner_logfile='log_scanner_tokens.txt', print_productions=False, print_source_parser=False, print_info=False, parser_logfile=sys.stdout, print_warnings=False, **kwargs): # Initialize variables self.source_code = None self.source_lines = None # Initialize table self.symbol_table = SymbolTable() # Initialize symbol table logger if table_logfile in {sys.stdout, sys.stderr}: self.symbol_table_logger = Logger(table_logfile) else: self.symbol_table_logger = Logger(open(table_logfile, 'w')) if print_table: self.symbol_table_logger.add_switch(Logger.SYMBOL_TABLE) # Initialize token/lexer logger if scanner_logfile in {sys.stdout, sys.stderr}: self.token_logger = Logger(scanner_logfile) else: self.token_logger = Logger(open(scanner_logfile, 'w')) if print_source_scanner: self.token_logger.add_switch(Logger.SOURCE) if print_tokens: self.token_logger.add_switch(Logger.TOKEN) # Initialize parser logger if parser_logfile in {sys.stdout, sys.stderr}: self.parser_logger = Logger(parser_logfile) else: self.parser_logger = Logger(open(parser_logfile, 'w')) if print_source_parser: self.parser_logger.add_switch(Logger.SOURCE) if print_productions: self.parser_logger.add_switch(Logger.PRODUCTION) if print_info: self.parser_logger.add_switch(Logger.INFO) # Initialize warning logger self.warning_logger = Logger(sys.stdout) if print_warnings: self.warning_logger.add_switch(Logger.WARNING) # Other stuff self.function_scope_entered = False self.insert_mode = True # for debugging purposes self.clone_symbol_table_on_next_scope_exit = False self.cloned_tables = [] # Create JSTLexer and the lexer object self.jst_lexer = JSTLexer(self) self.lexer = lex.lex(module=self.jst_lexer) # Create JSTParser and the parser object self.jst_parser = JSTParser(self) self.parser = yacc.yacc(module=self.jst_parser, start='program') # we will need a reference to the symbol for the main function self.main_function = None
cfg.batch_size, cfg.lr, cfg.cap_embed_type, cfg.image_melt_layer, cfg.cos_margin, cfg.np) cfg.exp_name += "_id_only" cfg.model_path = os.path.join("/shared/rsaas/aiyucui2/wider_person", cfg.model_path, cfg.exp_name) cfg.output_path = os.path.join("/shared/rsaas/aiyucui2/wider_person", cfg.output_path, cfg.exp_name) if not os.path.exists(cfg.model_path): os.mkdir(cfg.model_path) if not os.path.exists(cfg.output_path): os.mkdir(cfg.output_path) logger = Logger(os.path.join(cfg.output_path, "log.txt")) logger.log(str(cfg)) #------------------ ## agents #------------------ # Data loaders train_loader = build_wider_dataloader(cfg) cfg.num_ids = len(train_loader.dataset.person2label.values()) test_text_loader = build_text_test_loader(cfg) test_image_loader = build_image_test_loader(cfg) # Evaluators Evaluator = NPEvaluator if cfg.np else GlobalEvaluator