def cli_download_segment(camid, timestart, dst_download_dir, tmp_download_dir,
logfile, loglevel, **kw):
"""
Скачать 15-минутный сегмент камеры camid, начинающийся со времени timestart во
временную папку, проверить на разрывы.
Если разрывов меньше чем в существующем файле, перезаписать его.
Если есть разрывы - повторять скачивание max-download-retries раз.
Сохранить отчет о разрывах в {segmentvideofile}.flv.gapreport.json
Если force == False то продолжаем считать кол-во попыток с прошлого запуска.
Если force == True то кол-во попыток с прошлого запуска обнуляется.
"""
setup_logging(loglevel.upper(), logfile)
tmpdir = Path(tmp_download_dir) / camid
dstdir = Path(dst_download_dir) / camid
if not exists(tmpdir): os.makedirs(tmpdir)
if not exists(dstdir): os.makedirs(dstdir)
result = asyncio.run(
process_segment(
camid,
timestart,
tmp=tmpdir / f'{camid}-{int(timestart.timestamp())}.flv',
dst=dstdir / f'{camid}-{int(timestart.timestamp())}.flv',
max_retries=kw['max_download_retries'],
force=kw['force'],
))
if result is False:
sys.exit(1) # Fail
sys.exit(0) # Success
def run_csfla(fitness_function,
n_try_runs,
results_file_path,
n,
m,
sn,
Gm=10,
Gs=10):
# Setup Logging
setup_logging()
logger = logging.getLogger(__name__)
open(results_file_path + 'testfitness.txt', 'w').close()
open(results_file_path + 'trainfitness.txt', 'w').close()
with open(results_file_path + 'testfitness.txt', 'a') as f:
f.write(
"run\twealth\treturn\tvalue\tprofit\tmdd\ttransactions\tshort transactions\n"
)
with open(results_file_path + 'trainfitness.txt', 'a') as f:
f.write(
"run\twealth\treturn\tvalue\tprofit\tmdd\ttransactions\tshort transactions\n"
)
# Main loop
n_runs = 0
tfitnesses = {}
for i in range(0, n_try_runs):
# Init algorithm
csfla = CSFLA(n=n, m=m, sn=sn, Gm=Gm, Gs=Gs)
frog = csfla.optimize(fitness_function)
# Test the frog and add it to list if it's valid
frog.test()
if (frog.tf.value < -100) or (frog.tf.mdd == 0):
logger.info(
"Run %d: frog not taken into account in average results: fitness is invalid"
% i)
else:
tfitnesses[i] = frog.tf
n_runs += 1
# Log and pickle results
frog.log(iteration=i, path=results_file_path)
pickle.dump(
frog.p,
open(results_file_path + "/pickles/frog_run_" + str(i) + ".pickle",
"wb"))
calculate_average_fitness(tfitnesses, results_file_path)
def predict_all(self, beam_size=3):
setup_logging(os.path.join(self.save_path, 'log', LOG_FILE))
all_refer = {}
l = len(self.test_loader)
for i in range(l):
pred_sent, dialog_list, sents, entities, dialog, ref_id, dial_id, atten_weight = self.predict(
i, beam_size)
all_refer[ref_id] = {
'speaker': pred_sent,
'dialog': dialog_list,
'REs': sents,
'attn_w': atten_weight
}
logging.info('%d/%d, ref id %s:' % (i + 1, l, ref_id))
logging.info('dial_id %s:' % (ref_id))
logging.info('Pred: %s' % (pred_sent))
logging.info('Dialog Pred: %s' % (dialog_list))
logging.info('Dialog: %s' % (dialog))
logging.info('REs: %s' % (sents))
logging.info('Entities: %s' % (entities))
return all_refer
def main(args):
utils.init_distributed_mode(args)
#case = "{}-{}-{}".format(args.model, args.data_set, args.batch_size)
args.output_dir = os.path.join(args.output_dir, datetime.today().strftime('%Y-%m-%d'))
verbose = print
if utils.get_rank() == 0:
log_suffix = "{}log".format("eval-" if args.eval else "")
tools.check_folder(args.output_dir)
tools.setup_logging(os.path.join(args.output_dir, log_suffix + '.txt'), resume=args.resume != '')
verbose = logging.info
verbose(args)
if args.distillation_type != 'none' and args.finetune and not args.eval:
raise NotImplementedError("Finetuning with distillation not yet supported")
device = torch.device(args.device)
# fix the seed for reproducibility
seed = args.seed + utils.get_rank()
torch.manual_seed(seed)
np.random.seed(seed)
# random.seed(seed)
cudnn.benchmark = True
dataset_train, args.nb_classes = build_dataset(is_train=True, args=args)
dataset_val, _ = build_dataset(is_train=False, args=args)
if True: # args.distributed:
num_tasks = utils.get_world_size()
global_rank = utils.get_rank()
if args.repeated_aug:
sampler_train = RASampler(
dataset_train, num_replicas=num_tasks, rank=global_rank, shuffle=True
)
else:
sampler_train = torch.utils.data.DistributedSampler(
dataset_train, num_replicas=num_tasks, rank=global_rank, shuffle=True
)
if args.dist_eval:
if len(dataset_val) % num_tasks != 0:
verbose('Warning: Enabling distributed evaluation with an eval dataset not divisible by process number. '
'This will slightly alter validation results as extra duplicate entries are added to achieve '
'equal num of samples per-process.')
sampler_val = torch.utils.data.DistributedSampler(
dataset_val, num_replicas=num_tasks, rank=global_rank, shuffle=False)
else:
sampler_val = torch.utils.data.SequentialSampler(dataset_val)
else:
sampler_train = torch.utils.data.RandomSampler(dataset_train)
sampler_val = torch.utils.data.SequentialSampler(dataset_val)
data_loader_train = torch.utils.data.DataLoader(
dataset_train, sampler=sampler_train,
batch_size=args.batch_size,
num_workers=args.num_workers,
pin_memory=args.pin_mem,
drop_last=True,
)
data_loader_val = torch.utils.data.DataLoader(
dataset_val, sampler=sampler_val,
batch_size=int(1.5 * args.batch_size),
num_workers=args.num_workers,
pin_memory=args.pin_mem,
drop_last=False
)
mixup_fn = None
mixup_active = args.mixup > 0 or args.cutmix > 0. or args.cutmix_minmax is not None
if mixup_active:
mixup_fn = Mixup(
mixup_alpha=args.mixup, cutmix_alpha=args.cutmix, cutmix_minmax=args.cutmix_minmax,
prob=args.mixup_prob, switch_prob=args.mixup_switch_prob, mode=args.mixup_mode,
label_smoothing=args.smoothing, num_classes=args.nb_classes)
verbose(f"Creating model: {args.model}")
model = create_model(
args.model,
pretrained=False,
num_classes=args.nb_classes,
drop_rate=args.drop,
drop_path_rate=args.drop_path,
drop_block_rate=None,
)
ms.policy.deploy_on_init(model, args.ms_policy, verbose=verbose, override_verbose=True)
verbose(f"verbose model: {model}")
if args.finetune:
if args.finetune.startswith('https'):
checkpoint = torch.hub.load_state_dict_from_url(
args.finetune, map_location='cpu', check_hash=True)
else:
checkpoint = torch.load(args.finetune, map_location='cpu')
checkpoint_model = checkpoint['model']
state_dict = model.state_dict()
for k in ['head.weight', 'head.bias', 'head_dist.weight', 'head_dist.bias']:
if k in checkpoint_model and checkpoint_model[k].shape != state_dict[k].shape:
verbose(f"Removing key {k} from pretrained checkpoint")
del checkpoint_model[k]
# interpolate position embedding
pos_embed_checkpoint = checkpoint_model['pos_embed']
embedding_size = pos_embed_checkpoint.shape[-1]
num_patches = model.patch_embed.num_patches
num_extra_tokens = model.pos_embed.shape[-2] - num_patches
# height (== width) for the checkpoint position embedding
orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5)
# height (== width) for the new position embedding
new_size = int(num_patches ** 0.5)
# class_token and dist_token are kept unchanged
extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
# only the position tokens are interpolated
pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
pos_tokens = torch.nn.functional.interpolate(
pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
checkpoint_model['pos_embed'] = new_pos_embed
model.load_state_dict(checkpoint_model, strict=False)
model.to(device)
model_ema = None
if args.model_ema:
# Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper
model_ema = ModelEma(
model,
decay=args.model_ema_decay,
device='cpu' if args.model_ema_force_cpu else '',
resume='')
model_without_ddp = model
if args.distributed:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True)
model_without_ddp = model.module
n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)
verbose(f'number of params: {n_parameters}')
linear_scaled_lr = args.lr * args.batch_size * utils.get_world_size() / 512.0
args.lr = linear_scaled_lr
optimizer = create_optimizer(args, model_without_ddp)
loss_scaler = NativeScaler()
lr_scheduler, _ = create_scheduler(args, optimizer)
criterion = LabelSmoothingCrossEntropy()
if args.mixup > 0.:
# smoothing is handled with mixup label transform
criterion = SoftTargetCrossEntropy()
elif args.smoothing:
criterion = LabelSmoothingCrossEntropy(smoothing=args.smoothing)
else:
criterion = torch.nn.CrossEntropyLoss()
teacher_model = None
if args.distillation_type != 'none':
assert args.teacher_path, 'need to specify teacher-path when using distillation'
verbose(f"Creating teacher model: {args.teacher_model}")
teacher_model = create_model(
args.teacher_model,
pretrained=False,
num_classes=args.nb_classes,
global_pool='avg',
)
if args.teacher_path.startswith('https'):
checkpoint = torch.hub.load_state_dict_from_url(
args.teacher_path, map_location='cpu', check_hash=True)
else:
checkpoint = torch.load(args.teacher_path, map_location='cpu')
teacher_model.load_state_dict(checkpoint['model'])
teacher_model.to(device)
teacher_model.eval()
# wrap the criterion in our custom DistillationLoss, which
# just dispatches to the original criterion if args.distillation_type is 'none'
criterion = DistillationLoss(
criterion, teacher_model, args.distillation_type, args.distillation_alpha, args.distillation_tau
)
output_dir = Path(args.output_dir)
if args.resume:
if args.resume.startswith('https'):
checkpoint = torch.hub.load_state_dict_from_url(
args.resume, map_location='cpu', check_hash=True)
else:
checkpoint = torch.load(args.resume, map_location='cpu')
#model_without_ddp.load_state_dict(checkpoint['model'])
setattr(args, 'pretrained', args.resume)
tools.load_pretrained(model_without_ddp, args)
if not args.eval and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint:
optimizer.load_state_dict(checkpoint['optimizer'])
lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])
args.start_epoch = checkpoint['epoch'] + 1
if args.model_ema:
utils._load_checkpoint_for_ema(model_ema, checkpoint['model_ema'])
if 'scaler' in checkpoint:
loss_scaler.load_state_dict(checkpoint['scaler'])
if args.eval:
test_stats = evaluate(data_loader_val, model, device, verbose=verbose)
verbose(f"Accuracy of the network on the {len(dataset_val)} test images: {test_stats['acc1']:.1f}%")
return
verbose(f"Start training for {args.epochs} epochs")
start_time = time.time()
max_accuracy = 0.0
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
data_loader_train.sampler.set_epoch(epoch)
train_stats = train_one_epoch(
model, criterion, data_loader_train,
optimizer, device, epoch, loss_scaler,
args.clip_grad, model_ema, mixup_fn,
set_training_mode=args.finetune == '', # keep in eval mode during finetuning
verbose = verbose,
)
lr_scheduler.step(epoch)
if args.output_dir:
checkpoint_paths = [output_dir / 'checkpoint.pth']
for checkpoint_path in checkpoint_paths:
utils.save_on_master({
'model': model_without_ddp.state_dict(),
'optimizer': optimizer.state_dict(),
'lr_scheduler': lr_scheduler.state_dict(),
'epoch': epoch,
'model_ema': get_state_dict(model_ema),
'scaler': loss_scaler.state_dict(),
'args': args,
}, checkpoint_path)
test_stats = evaluate(data_loader_val, model, device, verbose=verbose)
verbose(f"Accuracy of the network on the {len(dataset_val)} test images: {test_stats['acc1']:.1f}%")
max_accuracy = max(max_accuracy, test_stats["acc1"])
verbose(f'Max accuracy: {max_accuracy:.2f}%')
log_stats = {**{f'train_{k}': v for k, v in train_stats.items()},
**{f'test_{k}': v for k, v in test_stats.items()},
'epoch': epoch,
'n_parameters': n_parameters}
if args.output_dir and utils.is_main_process():
with (output_dir / "log.txt").open("a") as f:
f.write(json.dumps(log_stats) + "\n")
total_time = time.time() - start_time
total_time_str = str(datetime.timedelta(seconds=int(total_time)))
verbose('Training time {}'.format(total_time_str))
def run_pso(fitness_function,
results_file_path="",
swarm_size=40,
n_max_try_runs=10,
v_max: float = 10.0,
w_inertia: float = 0.55,
w_memory: float = 0.55,
w_neigh: float = 0.55,
k: int = 5,
vel_conv_threshold: float = 0.001,
neighbourhood: int = Neighbourhood.GLOBAL,
max_iter: int = 5):
# Setup Logging
setup_logging()
logger = logging.getLogger(__name__)
open(results_file_path + 'testfitness.txt', 'w').close()
open(results_file_path + 'trainfitness.txt', 'w').close()
with open(results_file_path + 'testfitness.txt', 'a') as f:
f.write(
"run\twealth\treturn\tvalue\tprofit\tmdd\ttransactions\tshort transactions\n"
)
with open(results_file_path + 'trainfitness.txt', 'a') as f:
f.write(
"run\twealth\treturn\tvalue\tprofit\tmdd\ttransactions\tshort transactions\n"
)
# Set the fitness function
fitness_function = fitness_function
# Init variables
n_runs = 0
tfitnesses = {}
particles = {}
# Main loop
for i in range(0, n_max_try_runs):
# Initialize the swarm
pso = PSO(swarm_size=swarm_size,
v_max=v_max,
w_inertia=w_inertia,
w_memory=w_memory,
w_neigh=w_neigh,
k=k,
vel_conv_threshold=vel_conv_threshold,
max_iter=max_iter)
# Optimize with swarm
particle = pso.optimize(fitness_function)
# Test the particle and add it to list if it's valid
particle.test()
if (particle.tf.value < -100) or (particle.tf.mdd == 0):
logger.info(
"Run %d: particle not taken into account in average results: fitness is invalid"
% i)
else:
tfitnesses[i] = particle.tf
particles[i] = particle
n_runs += 1
# Log results
particle.log(iteration=i, path=results_file_path)
pickle.dump(
particle.p,
open(
results_file_path + "/pickles/particle_run_" + str(i) +
".pickle", "wb"))
calculate_average_fitness(tfitnesses, results_file_path)
def train_and_eval(self, train_loader, val_loader):
setup_logging(os.path.join(self.log_root, 'log_train_and_eval.txt'))
if self.cfg.RL_Train:
max_score = 0.0
max_cider = 0.0
max_score_ckp = os.path.join(self.checkpoint_root,
'max_score_ckp') # eval loss min ckp
max_cider_ckp = os.path.join(self.checkpoint_root,
'max_cider_ckp') # eval loss min ckp
pretrain_model = os.path.join(
self.save_root, '..', self.cfg.TRAIN.RL.pretrain_model
) # train longer and eval loss min ckp
logging.info('\n resume training, load from ckp')
check_point = torch.load(pretrain_model)
self.model.load_state_dict(check_point['state_dict'])
else:
min_loss = 100
min_loss_file = os.path.join(self.checkpoint_root,
'min_loss_ckp') # eval loss min ckp
ckp_file = os.path.join(self.checkpoint_root,
'115_ckp') # eval loss min ckp
for i in range(1, self.cfg.TRAIN.MAX_EPOCH + 1):
logging.debug("run arguments: %s", self.cfg)
self.epoch = i
val1, val2 = self._epoch(train_loader, training=True)
eval_val1, eval_val2 = self._epoch(val_loader, training=False)
if self.cfg.RL_Train:
state_dict = {
'epoch': self.epoch,
'iter': self.iter,
'state_dict': self.model.state_dict(),
'mlp_dict': self.mlp.state_dict(),
'rl_optim': self.rl_optim.state_dict(),
'mlp_optim': self.mlp_optim.state_dict()
}
if eval_val1 >= max_score - 0.0005:
if eval_val1 >= max_score:
max_score = eval_val1
torch.save(state_dict, max_score_ckp)
if eval_val2 >= max_cider - 0.0005:
if eval_val2 >= max_cider:
max_cider = eval_val2
torch.save(state_dict, max_cider_ckp)
self.tbx_writer.add_scalar('_eval_model/socre', eval_val1,
self.epoch)
self.tbx_writer.add_scalar('_eval_model/cider', eval_val2,
self.epoch)
if self.cfg.RL_Train == False:
self.tbx_writer.add_scalar('_train_model/loss', val1,
self.epoch)
self.tbx_writer.add_scalar('_train_model/perp', val2,
self.epoch)
self.lr_decay(self.apd_optim)
else:
self.tbx_writer.add_scalar('_train_model/socre', val1,
self.epoch)
self.tbx_writer.add_scalar('_train_model/cider', val2,
self.epoch)
else:
# var1 = loss, var2 = perp
self.tbx_writer.add_scalar('_train_model/loss', val1,
self.epoch)
self.tbx_writer.add_scalar('_eval_model/loss', eval_val1,
self.epoch)
state_dict = {
'epoch': self.epoch,
'iter': self.iter,
'state_dict': self.model.state_dict(),
'optim': self.apd_optim.state_dict()
}
if eval_val1 <= min_loss + 0.005:
if eval_val1 <= min_loss:
min_loss = eval_val1
torch.save(state_dict, min_loss_file)
if self.epoch == 115:
torch.save(state_dict, ckp_file)
self.lr_decay(self.apd_optim)