def __init__(self, host: str = 'localhost', port: int = 6379) -> None:
self.__client = StrictRedis(host, port=port, decode_responses=True)
self.__client.config_set('maxmemory', '600mb')
self.__client.config_set('maxmemory-policy', 'allkeys-lru')
self.__cache = RedisCache(redis_client=self.__client)
self.__logger = Logger().getLogger(__file__)
self.__logger.info("Initialize Cache.")
def main(argv) -> None:
del argv
# Process the configuration from flags.
config = process_config()
if config.mode != "evaluate":
# Define the datasets.
train_dataset = MatchingDataset(
batch_size=config.batch_size,
folder="datasets/matching_aracati/train",
x_shape=config.input_shape,
y_shape=config.output_shape,
is_evaluating=False)
valid_dataset = MatchingDataset(
batch_size=config.batch_size,
folder="datasets/matching_aracati/validation",
x_shape=config.input_shape,
y_shape=config.output_shape,
is_evaluating=False)
# Define the model.
loss = tf.keras.losses.BinaryCrossentropy()
ranger = Lookahead(RectifiedAdam(learning_rate=config.learning_rate),
sync_period=6,
slow_step_size=0.5)
model = DizygoticNet(filters=config.filters,
loss=loss,
optimizer=ranger)
# Define the logger.
logger = Logger()
# Define the trainer.
trainer = MatchingTrainer(model=model,
logger=logger,
train_dataset=train_dataset,
valid_dataset=valid_dataset)
if config.mode == "restore":
trainer.load_checkpoint()
trainer.train()
else:
# Define the test dataset.
test_dataset = MatchingDataset(batch_size=1,
folder="datasets/matching_aracati/test",
x_shape=config.input_shape,
y_shape=config.output_shape,
is_evaluating=True)
# Define the model.
model = DizygoticNet(filters=config.filters, loss=None, optimizer=None)
# Define the evaluator.
evaluator = MatchingEvaluator(model=model, dataset=test_dataset)
evaluator.load_checkpoint()
evaluator.evaluate()
def main(argv) -> None:
del argv
# Process the configuration from flags.
config = process_config()
if config.mode != "evaluate":
# Define the datasets.
train_dataset = TripletDataset(batch_size=config.batch_size,
folder="datasets/triplet_aracati/train",
x_shape=config.input_shape,
y_shape=config.output_shape,
is_validation=False)
valid_dataset = TripletDataset(
batch_size=config.batch_size,
folder="datasets/triplet_aracati/validation",
x_shape=config.input_shape,
y_shape=config.output_shape,
is_validation=True)
# Define the sonar model.
son_loss = TripletLoss()
son_ranger = Lookahead(
RectifiedAdam(learning_rate=config.learning_rate),
sync_period=6,
slow_step_size=0.5)
son_model = EncodeNet(filters=config.filters,
loss=son_loss,
optimizer=son_ranger)
# Define the satellite model.
sat_loss = TripletLoss()
sat_ranger = Lookahead(
RectifiedAdam(learning_rate=config.learning_rate),
sync_period=6,
slow_step_size=0.5)
sat_model = EncodeNet(filters=config.filters,
loss=sat_loss,
optimizer=sat_ranger)
# Define the logger.
logger = Logger()
# Define the trainer.
trainer = TripletTrainer(son_model=son_model,
sat_model=sat_model,
logger=logger,
train_dataset=train_dataset,
valid_dataset=valid_dataset)
if config.mode == "restore":
trainer.load_checkpoint()
trainer.train()
else:
logging.fatal("Evaluation not implemented yet.")
def train():
model = CTPN(cfg)
data = DataLoader(cfg)
# imdb = data.load_imdb('voc_2007_trainval')
# roidb = data.get_training_roidb(imdb)
sess = get_sess()
logger = Logger(sess, cfg)
trainer = CTPNTrainer(sess, model, data, logger)
print('Solving...')
trainer.train(cfg.TRAIN.MAX_ITERS, restore=False)
print('done solving')
def main(argv) -> None:
del argv
# Process the configuration from flags.
config = process_config()
if config.mode != "evaluate":
# Define the datasets.
train_dataset = GeneralDataset(batch_size=config.batch_size,
folder="datasets/general_aracati/train",
x_shape=config.input_shape,
y_shape=config.satellite_shape,
z_shape=config.output_shape)
valid_dataset = GeneralDataset(
batch_size=config.batch_size,
folder="datasets/general_aracati/validation",
x_shape=config.input_shape,
y_shape=config.satellite_shape,
z_shape=config.output_shape)
# Define the model.
loss = tf.keras.losses.MeanAbsoluteError()
ranger = Lookahead(RectifiedAdam(learning_rate=config.learning_rate),
sync_period=6,
slow_step_size=0.5)
model = WNet(filters=config.filters, loss=loss, optimizer=ranger)
# Define the logger.
logger = Logger()
# Define the trainer.
trainer = GeneralTrainer(model=model,
logger=logger,
train_dataset=train_dataset,
valid_dataset=valid_dataset)
if config.mode == "restore":
trainer.load_checkpoint()
trainer.train()
else:
# Define the test dataset.
test_dataset = GeneralDataset(batch_size=1,
folder="datasets/general_aracati/test",
x_shape=config.input_shape,
y_shape=config.satellite_shape,
z_shape=config.output_shape)
# Define the model.
model = WNet(filters=config.filters, loss=None, optimizer=None)
# Define the evaluator.
evaluator = GeneralEvaluator(model=model, dataset=test_dataset)
evaluator.load_checkpoint()
evaluator.evaluate()
class Cache():
def __init__(self, host: str = 'localhost', port: int = 6379) -> None:
self.__client = StrictRedis(host, port=port, decode_responses=True)
self.__client.config_set('maxmemory', '600mb')
self.__client.config_set('maxmemory-policy', 'allkeys-lru')
self.__cache = RedisCache(redis_client=self.__client)
self.__logger = Logger().getLogger(__file__)
self.__logger.info("Initialize Cache.")
def key_exists(self, *args):
serialized_data = dumps([args[1:], {}])
key = f'rc:{args[0]}:{serialized_data}'
return self.__client.exists(key) >= 1
def get_all_keys(self):
return self.__client.keys()
def get_key_count(self):
return len(self.get_all_keys())
def __call__(self, ttl=60 * 60 * 24 * 7, limit=5000, namespace=None):
return self.__cache.cache(ttl, limit, namespace)
def main(argv) -> None:
del argv
# Process the configuration from flags.
config = process_config()
if config.mode != "evaluate":
# Define the datasets.
train_dataset = MatchingDataset(batch_size=config.batch_size,
folder="datasets/aracati/train",
x_shape=config.input_shape,
y_shape=config.output_shape)
valid_dataset = MatchingDataset(batch_size=config.batch_size,
folder="datasets/aracati/validation",
x_shape=config.input_shape,
y_shape=config.output_shape)
# Define the model.
loss = tf.keras.losses.BinaryCrossentropy()
optimizer = tf.keras.optimizers.Adam(
learning_rate=config.learning_rate)
model = DizygoticNet(filters=config.filters,
loss=loss,
optimizer=optimizer)
if config.mode == "restore":
model.load_checkpoint()
# Define the logger.
logger = Logger()
# Define the trainer.
trainer = MatchingTrainer(model=model,
logger=logger,
train_dataset=train_dataset,
valid_dataset=valid_dataset)
trainer.train()
else:
logging.fatal("Evaluation mode is not yet implemented.")
exit(1)
def main(argv) -> None:
del argv
# Process the configuration from flags.
config = process_config()
if config.mode != "evaluate":
# Define the datasets.
train_dataset = SegmentationDataset(
batch_size=config.batch_size,
folder="datasets/segmentation_aracati/train",
x_shape=config.input_shape,
y_shape=config.output_shape)
valid_dataset = SegmentationDataset(
batch_size=config.batch_size,
folder="datasets/segmentation_aracati/validation",
x_shape=config.input_shape,
y_shape=config.output_shape)
# Define the model.
loss = tf.keras.losses.CategoricalCrossentropy()
ranger = Lookahead(RectifiedAdam(learning_rate=config.learning_rate),
sync_period=6,
slow_step_size=0.5)
model = UNet(filters=config.filters, loss=loss, optimizer=ranger)
# Define the logger.
logger = Logger()
# Define the trainer.
trainer = SegmentationTrainer(model=model,
logger=logger,
train_dataset=train_dataset,
valid_dataset=valid_dataset)
if config.mode == "restore":
trainer.load_checkpoint()
trainer.train()
else:
logging.fatal("Evaluation not implemented yet.")
cfg.dist_fn_opt, cfg.img_backbone_opt, cfg.cap_backbone_opt,
cfg.embed_size, cfg.batch_size, cfg.lr, cfg.step_size, cfg.cap_embed_type,
cfg.image_melt_layer, cfg.np)
if cfg.note:
cfg.exp_name += "_" + cfg.note
cfg.model_path = os.path.join("/shared/rsaas/aiyucui2/wider_person",
cfg.model_path, cfg.exp_name)
cfg.output_path = os.path.join("/shared/rsaas/aiyucui2/wider_person",
cfg.output_path, cfg.exp_name)
if not os.path.exists(cfg.model_path):
os.mkdir(cfg.model_path)
if not os.path.exists(cfg.output_path):
os.mkdir(cfg.output_path)
logger = Logger(os.path.join(cfg.output_path, "log.txt"))
logger.log(str(cfg))
#------------------
## agents
#------------------
# Data loaders
train_loader = build_wider_dataloader(cfg)
cfg.num_ids = len(train_loader.dataset.person2label.values())
test_text_loader = build_text_test_loader(cfg)
test_image_loader = build_image_test_loader(cfg)
# Evaluators
Evaluator = NPEvaluator if cfg.np else GlobalEvaluator
evaluator = Evaluator(img_loader=test_image_loader,
def main():
"""Main function"""
args = parse_args()
print('Called with args:')
print(args)
if not torch.cuda.is_available():
sys.exit("Need a CUDA device to run the code.")
if args.cuda or cfg.NUM_GPUS > 0:
cfg.CUDA = True
else:
raise ValueError("Need Cuda device to run !")
if args.dataset == "fis_cell":
cfg.TRAIN.DATASETS = ('fis_cell_train_val',)
cfg.MODEL.NUM_CLASSES = 2
elif args.dataset == "coco2017":
cfg.TRAIN.DATASETS = ('coco_2017_train',)
cfg.MODEL.NUM_CLASSES = 81
elif args.dataset == "keypoints_coco2017":
cfg.TRAIN.DATASETS = ('keypoints_coco_2017_train',)
cfg.MODEL.NUM_CLASSES = 2
else:
raise ValueError("Unexpected args.dataset: {}".format(args.dataset))
if args.close_fpn:
args.cfg_file = "configs/few_shot/e2e_mask_rcnn_R-50-C4_1x_{}.yaml".format(args.group)
cfg.OUTPUT_DIR = 'Outputs_wo_fpn'
else:
args.cfg_file = "configs/few_shot/e2e_mask_rcnn_R-50-FPN_1x_{}.yaml".format(args.group)
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
cfg.RNG_SEED = args.random_seed
if cfg.RNG_SEED is None:
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
else:
print('Make the experiment results deterministic.')
random.seed(cfg.RNG_SEED)
np.random.seed(cfg.RNG_SEED)
torch.manual_seed(cfg.RNG_SEED)
torch.cuda.manual_seed_all(cfg.RNG_SEED)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
cfg.SEEN = args.seen
if args.close_co_atten:
cfg.CO_ATTEN = False
if not args.close_fpn:
cfg.OUTPUT_DIR = 'Outputs_wo_co_atten'
if args.close_relation_rcnn:
cfg.RELATION_RCNN = False
if not args.close_fpn:
cfg.FAST_RCNN.ROI_BOX_HEAD = 'fast_rcnn_heads.roi_2mlp_head'
cfg.MRCNN.ROI_MASK_HEAD = 'mask_rcnn_heads.mask_rcnn_fcn_head_v1up4convs'
else:
cfg.FAST_RCNN.ROI_BOX_HEAD = 'torchResNet.ResNet_roi_conv5_head'
cfg.MRCNN.ROI_MASK_HEAD = 'mask_rcnn_heads.mask_rcnn_fcn_head_v0upshare'
if not args.close_co_atten and not args.close_fpn:
cfg.OUTPUT_DIR = 'Outputs_wo_relation_rcnn'
if args.output_dir is not None:
cfg.OUTPUT_DIR = args.output_dir
if args.deform_conv:
cfg.MODEL.USE_DEFORM = True
### Adaptively adjust some configs ###
original_batch_size = cfg.NUM_GPUS * cfg.TRAIN.IMS_PER_BATCH
original_ims_per_batch = cfg.TRAIN.IMS_PER_BATCH
original_num_gpus = cfg.NUM_GPUS
if args.batch_size is None:
args.batch_size = original_batch_size
cfg.NUM_GPUS = torch.cuda.device_count()
assert (args.batch_size % cfg.NUM_GPUS) == 0, \
'batch_size: %d, NUM_GPUS: %d' % (args.batch_size, cfg.NUM_GPUS)
cfg.TRAIN.IMS_PER_BATCH = args.batch_size // cfg.NUM_GPUS
effective_batch_size = args.iter_size * args.batch_size
print('effective_batch_size = batch_size * iter_size = %d * %d' % (args.batch_size, args.iter_size))
print('Adaptive config changes:')
print(' effective_batch_size: %d --> %d' % (original_batch_size, effective_batch_size))
print(' NUM_GPUS: %d --> %d' % (original_num_gpus, cfg.NUM_GPUS))
print(' IMS_PER_BATCH: %d --> %d' % (original_ims_per_batch, cfg.TRAIN.IMS_PER_BATCH))
### Adjust learning based on batch size change linearly
# For iter_size > 1, gradients are `accumulated`, so lr is scaled based
# on batch_size instead of effective_batch_size
old_base_lr = cfg.SOLVER.BASE_LR
cfg.SOLVER.BASE_LR *= args.batch_size / original_batch_size
print('Adjust BASE_LR linearly according to batch_size change:\n'
' BASE_LR: {} --> {}'.format(old_base_lr, cfg.SOLVER.BASE_LR))
### Adjust solver steps
step_scale = original_batch_size / effective_batch_size
old_solver_steps = cfg.SOLVER.STEPS
old_max_iter = cfg.SOLVER.MAX_ITER
cfg.SOLVER.STEPS = list(map(lambda x: int(x * step_scale + 0.5), cfg.SOLVER.STEPS))
cfg.SOLVER.MAX_ITER = int(cfg.SOLVER.MAX_ITER * step_scale + 0.5)
print('Adjust SOLVER.STEPS and SOLVER.MAX_ITER linearly based on effective_batch_size change:\n'
' SOLVER.STEPS: {} --> {}\n'
' SOLVER.MAX_ITER: {} --> {}'.format(old_solver_steps, cfg.SOLVER.STEPS,
old_max_iter, cfg.SOLVER.MAX_ITER))
# Scale FPN rpn_proposals collect size (post_nms_topN) in `collect` function
# of `collect_and_distribute_fpn_rpn_proposals.py`
#
# post_nms_topN = int(cfg[cfg_key].RPN_POST_NMS_TOP_N * cfg.FPN.RPN_COLLECT_SCALE + 0.5)
if cfg.FPN.FPN_ON and cfg.MODEL.FASTER_RCNN:
cfg.FPN.RPN_COLLECT_SCALE = cfg.TRAIN.IMS_PER_BATCH / original_ims_per_batch
print('Scale FPN rpn_proposals collect size directly propotional to the change of IMS_PER_BATCH:\n'
' cfg.FPN.RPN_COLLECT_SCALE: {}'.format(cfg.FPN.RPN_COLLECT_SCALE))
if args.num_workers is not None:
cfg.DATA_LOADER.NUM_THREADS = args.num_workers
print('Number of data loading threads: %d' % cfg.DATA_LOADER.NUM_THREADS)
### Overwrite some solver settings from command line arguments
if args.optimizer is not None:
cfg.SOLVER.TYPE = args.optimizer
if args.lr is not None:
cfg.SOLVER.BASE_LR = args.lr
if args.lr_decay_gamma is not None:
cfg.SOLVER.GAMMA = args.lr_decay_gamma
assert_and_infer_cfg()
timers = defaultdict(Timer)
### Dataset ###
timers['roidb'].tic()
imdb, roidb, ratio_list, ratio_index, query, cat_list = combined_roidb(
cfg.TRAIN.DATASETS, True)
timers['roidb'].toc()
roidb_size = len(roidb)
logger.info('{:d} roidb entries'.format(roidb_size))
logger.info('Takes %.2f sec(s) to construct roidb', timers['roidb'].average_time)
# Effective training sample size for one epoch
train_size = roidb_size // args.batch_size * args.batch_size
batchSampler = BatchSampler(
sampler=MinibatchSampler(ratio_list, ratio_index),
batch_size=args.batch_size,
drop_last=True
)
dataset = RoiDataLoader(
roidb, ratio_list, ratio_index, query,
cfg.MODEL.NUM_CLASSES,
training=True, cat_list=cat_list, shot=args.shot)
dataloader = torch.utils.data.DataLoader(
dataset,
batch_sampler=batchSampler,
num_workers=cfg.DATA_LOADER.NUM_THREADS,
collate_fn=collate_minibatch)
dataiterator = iter(dataloader)
### Model ###
maskRCNN = Generalized_RCNN()
if cfg.CUDA:
maskRCNN.cuda()
### Optimizer ###
gn_param_nameset = set()
for name, module in maskRCNN.named_modules():
if isinstance(module, nn.GroupNorm):
gn_param_nameset.add(name+'.weight')
gn_param_nameset.add(name+'.bias')
gn_params = []
gn_param_names = []
bias_params = []
bias_param_names = []
nonbias_params = []
nonbias_param_names = []
nograd_param_names = []
for key, value in maskRCNN.named_parameters():
if value.requires_grad:
if 'bias' in key:
bias_params.append(value)
bias_param_names.append(key)
elif key in gn_param_nameset:
gn_params.append(value)
gn_param_names.append(key)
else:
nonbias_params.append(value)
nonbias_param_names.append(key)
else:
nograd_param_names.append(key)
assert (gn_param_nameset - set(nograd_param_names) - set(bias_param_names)) == set(gn_param_names)
# Learning rate of 0 is a dummy value to be set properly at the start of training
params = [
{'params': nonbias_params,
'lr': 0,
'weight_decay': cfg.SOLVER.WEIGHT_DECAY},
{'params': bias_params,
'lr': 0 * (cfg.SOLVER.BIAS_DOUBLE_LR + 1),
'weight_decay': cfg.SOLVER.WEIGHT_DECAY if cfg.SOLVER.BIAS_WEIGHT_DECAY else 0},
{'params': gn_params,
'lr': 0,
'weight_decay': cfg.SOLVER.WEIGHT_DECAY_GN}
]
# names of paramerters for each paramter
param_names = [nonbias_param_names, bias_param_names, gn_param_names]
if cfg.SOLVER.TYPE == "SGD":
optimizer = torch.optim.SGD(params, momentum=cfg.SOLVER.MOMENTUM)
elif cfg.SOLVER.TYPE == "Adam":
optimizer = torch.optim.Adam(params)
### Load checkpoint
if args.load_ckpt:
load_name = args.load_ckpt
logging.info("loading checkpoint %s", load_name)
checkpoint = torch.load(load_name, map_location=lambda storage, loc: storage)
net_utils.load_ckpt(maskRCNN, checkpoint['model'])
if args.resume:
args.shot = checkpoint['shot']
args.start_step = checkpoint['step'] + 1
if 'train_size' in checkpoint: # For backward compatibility
if checkpoint['train_size'] != train_size:
print('train_size value: %d different from the one in checkpoint: %d'
% (train_size, checkpoint['train_size']))
# reorder the params in optimizer checkpoint's params_groups if needed
# misc_utils.ensure_optimizer_ckpt_params_order(param_names, checkpoint)
# There is a bug in optimizer.load_state_dict on Pytorch 0.3.1.
# However it's fixed on master.
optimizer.load_state_dict(checkpoint['optimizer'])
# misc_utils.load_optimizer_state_dict(optimizer, checkpoint['optimizer'])
del checkpoint
torch.cuda.empty_cache()
if args.load_detectron: #TODO resume for detectron weights (load sgd momentum values)
logging.info("loading Detectron weights %s", args.load_detectron)
load_detectron_weight(maskRCNN, args.load_detectron)
lr = optimizer.param_groups[0]['lr'] # lr of non-bias parameters, for commmand line outputs.
maskRCNN = mynn.DataParallel(maskRCNN, cpu_keywords=['im_info', 'roidb'],
minibatch=True)
### Training Setups ###
args.run_name = misc_utils.get_run_name() + '_step'
output_dir = misc_utils.get_output_dir(args, args.run_name)
args.cfg_filename = os.path.basename(args.cfg_file)
if not args.no_save:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
blob = {'cfg': yaml.dump(cfg), 'args': args}
with open(os.path.join(output_dir, 'config_and_args.pkl'), 'wb') as f:
pickle.dump(blob, f, pickle.HIGHEST_PROTOCOL)
if args.use_tfboard:
#from tensorboardX import SummaryWriter
# Set the Tensorboard logger
#tblogger = SummaryWriter(output_dir)
from loggers.logger import Logger
tblogger = Logger(output_dir)
### Training Loop ###
maskRCNN.train()
CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS)
# Set index for decay steps
decay_steps_ind = None
for i in range(1, len(cfg.SOLVER.STEPS)):
if cfg.SOLVER.STEPS[i] >= args.start_step:
decay_steps_ind = i
break
if decay_steps_ind is None:
decay_steps_ind = len(cfg.SOLVER.STEPS)
training_stats = TrainingStats(
args,
args.disp_interval,
tblogger if args.use_tfboard and not args.no_save else None)
try:
logger.info('Training starts !')
step = args.start_step
for step in range(args.start_step, cfg.SOLVER.MAX_ITER):
# Warm up
if step < cfg.SOLVER.WARM_UP_ITERS:
#for p in maskRCNN.module.Conv_Body.parameters():
# p.requires_grad = False
method = cfg.SOLVER.WARM_UP_METHOD
if method == 'constant':
warmup_factor = cfg.SOLVER.WARM_UP_FACTOR
elif method == 'linear':
alpha = step / cfg.SOLVER.WARM_UP_ITERS
warmup_factor = cfg.SOLVER.WARM_UP_FACTOR * (1 - alpha) + alpha
else:
raise KeyError('Unknown SOLVER.WARM_UP_METHOD: {}'.format(method))
lr_new = cfg.SOLVER.BASE_LR * warmup_factor
net_utils.update_learning_rate(optimizer, lr, lr_new)
lr = optimizer.param_groups[0]['lr']
assert lr == lr_new
elif step == cfg.SOLVER.WARM_UP_ITERS:
#for p in maskRCNN.module.Conv_Body.parameters():
# p.requires_grad = True
net_utils.update_learning_rate(optimizer, lr, cfg.SOLVER.BASE_LR)
lr = optimizer.param_groups[0]['lr']
assert lr == cfg.SOLVER.BASE_LR
# Learning rate decay
if decay_steps_ind < len(cfg.SOLVER.STEPS) and \
step == cfg.SOLVER.STEPS[decay_steps_ind]:
logger.info('Decay the learning on step %d', step)
lr_new = lr * cfg.SOLVER.GAMMA
net_utils.update_learning_rate(optimizer, lr, lr_new)
lr = optimizer.param_groups[0]['lr']
assert lr == lr_new
decay_steps_ind += 1
training_stats.IterTic()
optimizer.zero_grad()
for inner_iter in range(args.iter_size):
try:
input_data = next(dataiterator)
except StopIteration:
dataiterator = iter(dataloader)
input_data = next(dataiterator)
for key in input_data:
if key != 'roidb' and key != 'query': # roidb is a list of ndarrays with inconsistent length
input_data[key] = list(map(Variable, input_data[key]))
if key == 'query':
input_data[key] = [list(map(Variable, q)) for q in input_data[key]]
with torch.autograd.detect_anomaly():
net_outputs = maskRCNN(**input_data)
training_stats.UpdateIterStats(net_outputs, inner_iter)
loss = net_outputs['total_loss']
loss.backward()
torch.nn.utils.clip_grad_value_(maskRCNN.module.parameters(), clip_value=0.4)
optimizer.step()
training_stats.IterToc()
training_stats.LogIterStats(step, lr, input_data, args.shot)
if (step+1) % CHECKPOINT_PERIOD == 0:
save_ckpt(output_dir, args, step, train_size, maskRCNN, optimizer)
if (step+1) % args.disp_interval == 0:
log_training_stats(training_stats, step, lr)
# ---- Training ends ----
# Save last checkpoint
save_ckpt(output_dir, args, step, train_size, maskRCNN, optimizer)
except (RuntimeError, KeyboardInterrupt):
del dataiterator
logger.info('Save ckpt on exception ...')
save_ckpt(output_dir, args, step, train_size, maskRCNN, optimizer)
logger.info('Save ckpt done.')
stack_trace = traceback.format_exc()
print(stack_trace)
finally:
if args.use_tfboard and not args.no_save:
tblogger.close()
class CompilerState:
def __init__(self,
print_table=False,
table_logfile='log_symbol_table.txt',
print_tokens=False, print_source_scanner=True,
scanner_logfile='log_scanner_tokens.txt',
print_productions=False, print_source_parser=False, print_info=False,
parser_logfile=sys.stdout,
print_warnings=False,
**kwargs):
# Initialize variables
self.source_code = None
self.source_lines = None
# Initialize table
self.symbol_table = SymbolTable()
# Initialize symbol table logger
if table_logfile in {sys.stdout, sys.stderr}:
self.symbol_table_logger = Logger(table_logfile)
else:
self.symbol_table_logger = Logger(open(table_logfile, 'w'))
if print_table:
self.symbol_table_logger.add_switch(Logger.SYMBOL_TABLE)
# Initialize token/lexer logger
if scanner_logfile in {sys.stdout, sys.stderr}:
self.token_logger = Logger(scanner_logfile)
else:
self.token_logger = Logger(open(scanner_logfile, 'w'))
if print_source_scanner:
self.token_logger.add_switch(Logger.SOURCE)
if print_tokens:
self.token_logger.add_switch(Logger.TOKEN)
# Initialize parser logger
if parser_logfile in {sys.stdout, sys.stderr}:
self.parser_logger = Logger(parser_logfile)
else:
self.parser_logger = Logger(open(parser_logfile, 'w'))
if print_source_parser:
self.parser_logger.add_switch(Logger.SOURCE)
if print_productions:
self.parser_logger.add_switch(Logger.PRODUCTION)
if print_info:
self.parser_logger.add_switch(Logger.INFO)
# Initialize warning logger
self.warning_logger = Logger(sys.stdout)
if print_warnings:
self.warning_logger.add_switch(Logger.WARNING)
# Other stuff
self.function_scope_entered = False
self.insert_mode = True
# for debugging purposes
self.clone_symbol_table_on_next_scope_exit = False
self.cloned_tables = []
# Create JSTLexer and the lexer object
self.jst_lexer = JSTLexer(self)
self.lexer = lex.lex(module=self.jst_lexer)
# Create JSTParser and the parser object
self.jst_parser = JSTParser(self)
self.parser = yacc.yacc(module=self.jst_parser, start='program')
# we will need a reference to the symbol for the main function
self.main_function = None
def parse(self, source_code):
# Lex uses 1 based indexing for line numbers.
# We are using 0 based for source_code.
if source_code is not None:
self.source_code = source_code
self.source_lines = source_code.split('\n')
else:
self.source_code = None
self.source_lines = None
# Parse using the parser object
return self.parser.parse(input=self.source_code, lexer=self.lexer, tracking=True)
def teardown(self):
self.jst_lexer.teardown()
self.jst_parser.teardown()
def get_symbol_table_logger(self):
return self.symbol_table_logger
def get_token_logger(self):
return self.token_logger
def get_parser_logger(self):
return self.parser_logger
def get_warning_logger(self):
return self.warning_logger
def get_line_col(self, production, index):
lexpos = production.lexpos(index)
last_newline = self.source_code.rfind('\n', 0, lexpos)
return production.lineno(index), max(0, lexpos - last_newline)
def get_line_col_source(self, lineno, lexpos):
last_newline = self.source_code.rfind('\n', 0, lexpos)
return (
lineno,
max(0, lexpos - last_newline),
self.source_lines[lineno - 1]
)
def main():
"""Main function"""
args = parse_args()
print('Called with args:')
print(args)
if not torch.cuda.is_available():
sys.exit("Need a CUDA device to run the code.")
if args.cuda or cfg.NUM_GPUS > 0:
cfg.CUDA = True
else:
raise ValueError("Need Cuda device to run !")
if args.dataset == "fis_cell":
cfg.TRAIN.DATASETS = ('fis_cell_train_val',)
cfg.MODEL.NUM_CLASSES = 2
elif args.dataset == "coco2017":
cfg.TRAIN.DATASETS = ('coco_2017_train',)
cfg.MODEL.NUM_CLASSES = 81
elif args.dataset == "keypoints_coco2017":
cfg.TRAIN.DATASETS = ('keypoints_coco_2017_train',)
cfg.MODEL.NUM_CLASSES = 2
else:
raise ValueError("Unexpected args.dataset: {}".format(args.dataset))
args.cfg_file = "configs/few_shot/e2e_mask_rcnn_R-50-FPN_1x_{}.yaml".format(args.group)
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
cfg.SEEN = args.seen
### Adaptively adjust some configs ###
original_batch_size = cfg.NUM_GPUS * cfg.TRAIN.IMS_PER_BATCH
original_ims_per_batch = cfg.TRAIN.IMS_PER_BATCH
original_num_gpus = cfg.NUM_GPUS
if args.batch_size is None:
args.batch_size = original_batch_size
cfg.NUM_GPUS = torch.cuda.device_count()
assert (args.batch_size % cfg.NUM_GPUS) == 0, \
'batch_size: %d, NUM_GPUS: %d' % (args.batch_size, cfg.NUM_GPUS)
cfg.TRAIN.IMS_PER_BATCH = args.batch_size // cfg.NUM_GPUS
print('Adaptive config changes:')
print(' batch_size: %d --> %d' % (original_batch_size, args.batch_size))
print(' NUM_GPUS: %d --> %d' % (original_num_gpus, cfg.NUM_GPUS))
print(' IMS_PER_BATCH: %d --> %d' % (original_ims_per_batch, cfg.TRAIN.IMS_PER_BATCH))
if args.num_workers is not None:
cfg.DATA_LOADER.NUM_THREADS = args.num_workers
print('Number of data loading threads: %d' % cfg.DATA_LOADER.NUM_THREADS)
### Adjust learning based on batch size change linearly
old_base_lr = cfg.SOLVER.BASE_LR
cfg.SOLVER.BASE_LR *= args.batch_size / original_batch_size
print('Adjust BASE_LR linearly according to batch size change: {} --> {}'.format(
old_base_lr, cfg.SOLVER.BASE_LR))
# Scale FPN rpn_proposals collect size (post_nms_topN) in `collect` function
# of `collect_and_distribute_fpn_rpn_proposals.py`
#
# post_nms_topN = int(cfg[cfg_key].RPN_POST_NMS_TOP_N * cfg.FPN.RPN_COLLECT_SCALE + 0.5)
if cfg.FPN.FPN_ON and cfg.MODEL.FASTER_RCNN:
cfg.FPN.RPN_COLLECT_SCALE = cfg.TRAIN.IMS_PER_BATCH / original_ims_per_batch
print('Scale FPN rpn_proposals collect size directly propotional to the change of IMS_PER_BATCH:\n'
' cfg.FPN.RPN_COLLECT_SCALE: {}'.format(cfg.FPN.RPN_COLLECT_SCALE))
### Overwrite some solver settings from command line arguments
if args.optimizer is not None:
cfg.SOLVER.TYPE = args.optimizer
if args.lr is not None:
cfg.SOLVER.BASE_LR = args.lr
if args.lr_decay_gamma is not None:
cfg.SOLVER.GAMMA = args.lr_decay_gamma
timers = defaultdict(Timer)
### Dataset ###
timers['roidb'].tic()
imdb, roidb, ratio_list, ratio_index, query, cat_list = combined_roidb(
cfg.TRAIN.DATASETS, True)
timers['roidb'].toc()
train_size = len(roidb)
logger.info('{:d} roidb entries'.format(train_size))
logger.info('Takes %.2f sec(s) to construct roidb', timers['roidb'].average_time)
sampler = MinibatchSampler(ratio_list, ratio_index)
dataset = RoiDataLoader(
roidb, ratio_list, ratio_index, query,
cfg.MODEL.NUM_CLASSES,
training=True, cat_list=cat_list, shot=args.shot)
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=args.batch_size,
sampler=sampler,
num_workers=cfg.DATA_LOADER.NUM_THREADS,
collate_fn=collate_minibatch)
assert_and_infer_cfg()
### Model ###
maskRCNN = Generalized_RCNN()
if cfg.CUDA:
maskRCNN.cuda()
### Optimizer ###
bias_params = []
nonbias_params = []
for key, value in dict(maskRCNN.named_parameters()).items():
if value.requires_grad:
if 'bias' in key:
bias_params.append(value)
else:
nonbias_params.append(value)
params = [
{'params': nonbias_params,
'lr': cfg.SOLVER.BASE_LR,
'weight_decay': cfg.SOLVER.WEIGHT_DECAY},
{'params': bias_params,
'lr': cfg.SOLVER.BASE_LR * (cfg.SOLVER.BIAS_DOUBLE_LR + 1),
'weight_decay': cfg.SOLVER.WEIGHT_DECAY if cfg.SOLVER.BIAS_WEIGHT_DECAY else 0}
]
if cfg.SOLVER.TYPE == "SGD":
optimizer = torch.optim.SGD(params, momentum=cfg.SOLVER.MOMENTUM)
elif cfg.SOLVER.TYPE == "Adam":
optimizer = torch.optim.Adam(params)
### Load checkpoint
if args.load_ckpt:
load_name = args.load_ckpt
logging.info("loading checkpoint %s", load_name)
checkpoint = torch.load(load_name, map_location=lambda storage, loc: storage)
net_utils.load_ckpt(maskRCNN, checkpoint['model'])
if args.resume:
assert checkpoint['iters_per_epoch'] == train_size // args.batch_size, \
"iters_per_epoch should match for resume"
# There is a bug in optimizer.load_state_dict on Pytorch 0.3.1.
# However it's fixed on master.
# optimizer.load_state_dict(checkpoint['optimizer'])
misc_utils.load_optimizer_state_dict(optimizer, checkpoint['optimizer'])
args.shot = checkpoint['shot']
if checkpoint['step'] == (checkpoint['iters_per_epoch'] - 1):
# Resume from end of an epoch
args.start_epoch = checkpoint['epoch'] + 1
args.start_iter = 0
else:
# Resume from the middle of an epoch.
# NOTE: dataloader is not synced with previous state
args.start_epoch = checkpoint['epoch']
args.start_iter = checkpoint['step'] + 1
del checkpoint
torch.cuda.empty_cache()
if args.load_detectron: #TODO resume for detectron weights (load sgd momentum values)
logging.info("loading Detectron weights %s", args.load_detectron)
load_detectron_weight(maskRCNN, args.load_detectron)
lr = optimizer.param_groups[0]['lr'] # lr of non-bias parameters, for commmand line outputs.
maskRCNN = mynn.DataParallel(maskRCNN, cpu_keywords=['im_info', 'roidb'],
minibatch=True)
### Training Setups ###
args.run_name = misc_utils.get_run_name()
output_dir = misc_utils.get_output_dir(args, args.run_name)
args.cfg_filename = os.path.basename(args.cfg_file)
if not args.no_save:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
blob = {'cfg': yaml.dump(cfg), 'args': args}
with open(os.path.join(output_dir, 'config_and_args.pkl'), 'wb') as f:
pickle.dump(blob, f, pickle.HIGHEST_PROTOCOL)
if args.use_tfboard:
#from tensorboardX import SummaryWriter
# Set the Tensorboard logger
#tblogger = SummaryWriter(output_dir)
from loggers.logger import Logger
tblogger = Logger(output_dir)
### Training Loop ###
maskRCNN.train()
training_stats = TrainingStats(
args,
args.disp_interval,
tblogger if args.use_tfboard and not args.no_save else None)
iters_per_epoch = int(train_size / args.batch_size) # drop last
args.iters_per_epoch = iters_per_epoch
ckpt_interval_per_epoch = iters_per_epoch // args.ckpt_num_per_epoch
try:
logger.info('Training starts !')
args.step = args.start_iter
global_step = iters_per_epoch * args.start_epoch + args.step
for args.epoch in range(args.start_epoch, args.start_epoch + args.num_epochs):
# ---- Start of epoch ----
# adjust learning rate
if args.lr_decay_epochs and args.epoch == args.lr_decay_epochs[0] and args.start_iter == 0:
args.lr_decay_epochs.pop(0)
net_utils.decay_learning_rate(optimizer, lr, cfg.SOLVER.GAMMA)
lr *= cfg.SOLVER.GAMMA
for args.step, input_data in zip(range(args.start_iter, iters_per_epoch), dataloader):
for key in input_data:
if key != 'roidb' and key != 'query': # roidb is a list of ndarrays with inconsistent length
input_data[key] = list(map(Variable, input_data[key]))
if key == 'query':
input_data[key] = [list(map(Variable, q)) for q in input_data[key]]
training_stats.IterTic()
net_outputs = maskRCNN(**input_data)
training_stats.UpdateIterStats(net_outputs)
loss = net_outputs['total_loss']
optimizer.zero_grad()
loss.backward()
optimizer.step()
training_stats.IterToc()
#if (args.step+1) % ckpt_interval_per_epoch == 0:
if global_step % 500 == 0:
net_utils.save_ckpt(output_dir, args, maskRCNN, optimizer)
if args.step % args.disp_interval == 0:
log_training_stats(training_stats, global_step, lr, input_data, args.shot)
global_step += 1
# ---- End of epoch ----
# save checkpoint
#net_utils.save_ckpt(output_dir, args, maskRCNN, optimizer)
# reset starting iter number after first epoch
args.start_iter = 0
# ---- Training ends ----
net_utils.save_ckpt(output_dir, args, maskRCNN, optimizer)
if iters_per_epoch % args.disp_interval != 0:
# log last stats at the end
log_training_stats(training_stats, global_step, lr, input_data, args.shot)
except (RuntimeError, KeyboardInterrupt):
logger.info('Save ckpt on exception ...')
net_utils.save_ckpt(output_dir, args, maskRCNN, optimizer)
logger.info('Save ckpt done.')
stack_trace = traceback.format_exc()
print(stack_trace)
finally:
if args.use_tfboard and not args.no_save:
tblogger.close()
load_dotenv(find_dotenv())
# Global variables
with open(r'configs/zone_settings.yaml') as file:
zones = yaml.load(file, Loader=yaml.FullLoader)
with open(r'configs/class_settings.yaml') as file:
class_settings = yaml.load(file, Loader=yaml.FullLoader)
# Initialize the app
app = dash.Dash(__name__)
server = app.server # needed to launch gunicorn
app.config.suppress_callback_exceptions = True
# Initialize logger
logger = Logger().getLogger(__file__)
# Initialize WCL client
client = WCLClient()
# Get users
try:
with open(r'configs/users.yaml') as file:
USERS = yaml.load(file, Loader=yaml.FullLoader)
except FileNotFoundError:
logger.error("User list not found.")
raise
# Initialize basic auth
auth = dash_auth.BasicAuth(app, parse_users(USERS))
from datetime import datetime
import pandas as pd
from loggers.logger import Logger
# Initialize logger
logger = Logger().getLogger(__file__)
THRESHOLD_PERCENTAGE = 0.10
def create_normalized_column(df, col_name):
df[f'norm_{col_name}'] = 100 * df[col_name] / df[col_name].sum()
return df
def drop_rows_where_col_has_value(df, col_name, value):
df = df[df[col_name] != value]
return df
def average_logs(logs):
return pd.concat(
[
pd.DataFrame(log["entries"])[['name', 'type', 'total']]
.pipe(drop_rows_where_col_has_value, col_name='type', value='Pet')
.pipe(create_normalized_column, col_name='total') for log in logs
]
).groupby(['name']).agg(
_std=('norm_total', lambda x: x.unique().std()), # just 'std' will use ddof = 1
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.logger = Logger('logs/', 'screenshots/')
self.getChromeOptions()
def __init__(self,
print_table=False,
table_logfile='log_symbol_table.txt',
print_tokens=False, print_source_scanner=True,
scanner_logfile='log_scanner_tokens.txt',
print_productions=False, print_source_parser=False, print_info=False,
parser_logfile=sys.stdout,
print_warnings=False,
**kwargs):
# Initialize variables
self.source_code = None
self.source_lines = None
# Initialize table
self.symbol_table = SymbolTable()
# Initialize symbol table logger
if table_logfile in {sys.stdout, sys.stderr}:
self.symbol_table_logger = Logger(table_logfile)
else:
self.symbol_table_logger = Logger(open(table_logfile, 'w'))
if print_table:
self.symbol_table_logger.add_switch(Logger.SYMBOL_TABLE)
# Initialize token/lexer logger
if scanner_logfile in {sys.stdout, sys.stderr}:
self.token_logger = Logger(scanner_logfile)
else:
self.token_logger = Logger(open(scanner_logfile, 'w'))
if print_source_scanner:
self.token_logger.add_switch(Logger.SOURCE)
if print_tokens:
self.token_logger.add_switch(Logger.TOKEN)
# Initialize parser logger
if parser_logfile in {sys.stdout, sys.stderr}:
self.parser_logger = Logger(parser_logfile)
else:
self.parser_logger = Logger(open(parser_logfile, 'w'))
if print_source_parser:
self.parser_logger.add_switch(Logger.SOURCE)
if print_productions:
self.parser_logger.add_switch(Logger.PRODUCTION)
if print_info:
self.parser_logger.add_switch(Logger.INFO)
# Initialize warning logger
self.warning_logger = Logger(sys.stdout)
if print_warnings:
self.warning_logger.add_switch(Logger.WARNING)
# Other stuff
self.function_scope_entered = False
self.insert_mode = True
# for debugging purposes
self.clone_symbol_table_on_next_scope_exit = False
self.cloned_tables = []
# Create JSTLexer and the lexer object
self.jst_lexer = JSTLexer(self)
self.lexer = lex.lex(module=self.jst_lexer)
# Create JSTParser and the parser object
self.jst_parser = JSTParser(self)
self.parser = yacc.yacc(module=self.jst_parser, start='program')
# we will need a reference to the symbol for the main function
self.main_function = None
cfg.batch_size,
cfg.lr,
cfg.cap_embed_type,
cfg.image_melt_layer,
cfg.cos_margin,
cfg.np)
cfg.exp_name += "_id_only"
cfg.model_path = os.path.join("/shared/rsaas/aiyucui2/wider_person", cfg.model_path, cfg.exp_name)
cfg.output_path = os.path.join("/shared/rsaas/aiyucui2/wider_person", cfg.output_path, cfg.exp_name)
if not os.path.exists(cfg.model_path):
os.mkdir(cfg.model_path)
if not os.path.exists(cfg.output_path):
os.mkdir(cfg.output_path)
logger = Logger(os.path.join(cfg.output_path, "log.txt"))
logger.log(str(cfg))
#------------------
## agents
#------------------
# Data loaders
train_loader = build_wider_dataloader(cfg)
cfg.num_ids = len(train_loader.dataset.person2label.values())
test_text_loader = build_text_test_loader(cfg)
test_image_loader = build_image_test_loader(cfg)
# Evaluators
Evaluator = NPEvaluator if cfg.np else GlobalEvaluator