def query_directory(self, first_time):
query_url = utils.gs.get_selector(self.server, utils.gs.QUERY)
payload = self.payload.create_query_payload(self.pending)
Logger.log(LogLevel.DEBUG, payload)
try:
if first_time:
resp = requests.post(query_url, data=payload,
headers=self.headers, verify=self.verify)
self.set_cookie(resp)
Logger.log(LogLevel.DEBUG, 'set cookie=', self.cookies)
else:
Logger.log(LogLevel.DEBUG, 'cookie=', self.cookies)
resp = requests.post(query_url, data=payload,
headers=self.headers,
cookies=self.cookies,
verify=self.verify)
if not self.handle_response(resp, first_time):
return False
except IOError as e:
Logger.log(LogLevel.ERROR, 'IO_ERROR', e)
return False
return True
def run(_run, _config, _log):
# check args sanity
_config = args_sanity_check(_config, _log)
#TODO why use SimpleNamespace
args = SimpleNamespace(**_config)
args.device = "cuda" if args.use_cuda else "cpu"
# setup loggers
logger = Logger(_log)
_log.info("Experiment Parameters:")
experiment_params = pprint.pformat(_config, indent=4, width=1)
_log.info("\n\n" + experiment_params + "\n")
# configure tensorboard logger
unique_token = "{}__{}".format(
args.name,
datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
args.unique_token = unique_token
if args.use_tensorboard:
tb_logs_direc = os.path.join(dirname(dirname(abspath(__file__))),
"results", "tb_logs")
tb_exp_direc = os.path.join(tb_logs_direc, "{}").format(unique_token)
logger.setup_tb(tb_exp_direc)
# sacred is on by default
logger.setup_sacred(_run)
# Run and train
run_sequential(args=args, logger=logger)
# Clean up after finishing
print("Exiting Main")
print("Stopping all threads")
for t in threading.enumerate():
if t.name != "MainThread":
print("Thread {} is alive! Is daemon: {}".format(t.name, t.daemon))
t.join(timeout=1)
print("Thread joined")
print("Exiting script")
# Making sure framework really exits
os._exit(os.EX_OK)
def main(args):
np.random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.benchmark = True
sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt'))
trainset = Feeder(args.feat_path, args.knn_graph_path, args.label_path,
args.seed, args.k_at_hop, args.active_connection)
trainloader = DataLoader(trainset,
batch_size=args.batch_size,
num_workers=args.workers,
shuffle=True,
pin_memory=True)
net = model.gcn().cuda()
opt = torch.optim.SGD(net.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
criterion = nn.CrossEntropyLoss().cuda()
save_checkpoint({
'state_dict': net.state_dict(),
'epoch': 0,
},
False,
fpath=osp.join(args.logs_dir, 'epoch_{}.ckpt'.format(0)))
for epoch in range(args.epochs):
adjust_lr(opt, epoch)
train(trainloader, net, criterion, opt, epoch)
save_checkpoint({
'state_dict': net.state_dict(),
'epoch': epoch + 1,
},
False,
fpath=osp.join(args.logs_dir,
'epoch_{}.ckpt'.format(epoch + 1)))
def my_main(_run, _config, _log):
global mongo_client
import datetime
# arglist = parse_args()
# unique_token = "{}__{}".format(arglist.name, datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
# run the framework
# run(_run, _config, _log, mongo_client, unique_token)
logger = Logger(_log)
# configure tensorboard logger
unique_token = "{}__{}".format(
_config["label"],
datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
use_tensorboard = False
if use_tensorboard:
tb_logs_direc = os.path.join(dirname(dirname(abspath(__file__))),
"results", "tb_logs")
tb_exp_direc = os.path.join(tb_logs_direc, "{}").format(unique_token)
logger.setup_tb(tb_exp_direc)
logger.setup_sacred(_run)
_log.info("Experiment Parameters:")
import pprint
experiment_params = pprint.pformat(_config, indent=4, width=1)
_log.info("\n\n" + experiment_params + "\n")
# START THE TRAINING PROCESS
runner = Runner(logger)
runner.load(_config)
runner.reset()
# args = vars(arglist)
runner.run(_config)
# runner.run(args)
# train(arglist, logger, _config)
# arglist = convert(_config)
# train(arglist)
# force exit
os._exit(0)
def run(_run, _config, _log, pymongo_client):
# check args sanity
_config = args_sanity_check(_config, _log)
args = SN(**_config)
args.device = "cuda" if args.use_cuda else "cpu"
# setup loggers
logger = Logger(_log)
_log.info("Experiment Parameters:")
experiment_params = pprint.pformat(_config, indent=4, width=1)
_log.info("\n\n" + experiment_params + "\n")
# configure tensorboard logger
unique_token = "{}__{}".format(
args.name,
datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
args.unique_token = unique_token
if args.use_tensorboard:
tb_logs_direc = os.path.join(dirname(dirname(abspath(__file__))),
"results", "tb_logs")
tb_exp_direc = os.path.join(tb_logs_direc, "{}").format(unique_token)
logger.setup_tb(tb_exp_direc)
# sacred is on by default
logger.setup_sacred(_run)
# Run and train
if args.cross_play and args.evaluate:
run_sequential_cross(args=args, logger=logger)
else:
run_sequential(args=args, logger=logger)
# Clean up after finishing
print("Exiting Main")
if pymongo_client is not None:
print("Attempting to close mongodb client")
pymongo_client.close()
print("Mongodb client closed")
def upload_directory(self):
# Use copy of the list for proper removal
res = True
for file_data in self.pending.values():
if not file_data.upload:
continue
try:
session = requests.Session()
json_request = self.payload.create_upload_payload(file_data)
Logger.log(LogLevel.DEBUG, json_request)
upload_url = utils.gs.get_selector(self.server,
utils.gs.UPLOAD)
with open(file_data.file_path, 'rb') as f:
form = MultipartEncoder({
"request": json_request,
"file": f,
})
headers = self.headers
headers["Content-Type"] = form.content_type
resp = session.post(upload_url,
headers=headers,
data=form,
cookies=self.cookies,
verify=self.verify)
Logger.log(LogLevel.DEBUG, resp)
if not self.handle_response(resp):
raise Exception('Failed to handle upload response')
except Exception as e:
Logger.log(LogLevel.ERROR, 'Uploading Error', e)
res = False
continue
return res
def evaluate(epoch, data, model, criterion):
"""Main evaluation procedure.
Arguments:
epoch -- current epoch
data -- DataLoader which can provide validation batches
model -- model to be evaluated
criterion -- instance of loss function to measure performance
"""
model.eval()
# initialize counters, etc.
mcd, mcd_count = 0, 0
cla, cla_count = 0, 0
eval_losses = {}
# loop through epoch batches
with torch.no_grad():
for i, batch in enumerate(data):
# parse batch
batch = list(map(to_gpu, batch))
src, src_len, trg_mel, trg_lin, trg_len, stop_trg, spkrs, langs = batch
# run the model (twice, with and without teacher forcing)
post_pred, pre_pred, stop_pred, alignment, spkrs_pred, enc_output = model(
src, src_len, trg_mel, trg_len, spkrs, langs, 1.0)
post_pred_0, _, stop_pred_0, alignment_0, _, _ = model(
src, src_len, trg_mel, trg_len, spkrs, langs, 0.0)
stop_pred_probs = torch.sigmoid(stop_pred_0)
# evaluate loss function
post_trg = trg_lin if hp.predict_linear else trg_mel
classifier = model._reversal_classifier if hp.reversal_classifier else None
loss, batch_losses = criterion(src_len, trg_len, pre_pred, trg_mel,
post_pred, post_trg, stop_pred,
stop_trg, alignment, spkrs,
spkrs_pred, enc_output, classifier)
# compute mel cepstral distorsion
for j, (gen, ref, stop) in enumerate(
zip(post_pred_0, trg_mel, stop_pred_probs)):
stop_idxes = np.where(stop.cpu().numpy() > 0.5)[0]
stop_idx = min(
np.min(stop_idxes) + hp.stop_frames,
gen.size()[1]) if len(stop_idxes) > 0 else gen.size()[1]
gen = gen[:, :stop_idx].data.cpu().numpy()
ref = ref[:, :trg_len[j]].data.cpu().numpy()
if hp.normalize_spectrogram:
gen = audio.denormalize_spectrogram(
gen, not hp.predict_linear)
ref = audio.denormalize_spectrogram(ref, True)
if hp.predict_linear: gen = audio.linear_to_mel(gen)
mcd = (mcd_count * mcd + audio.mel_cepstral_distorision(
gen, ref, 'dtw')) / (mcd_count + 1)
mcd_count += 1
# compute adversarial classifier accuracy
if hp.reversal_classifier:
input_mask = lengths_to_mask(src_len)
trg_spkrs = torch.zeros_like(input_mask, dtype=torch.int64)
for s in range(hp.speaker_number):
speaker_mask = (spkrs == s)
trg_spkrs[speaker_mask] = s
matches = (trg_spkrs == torch.argmax(
torch.nn.functional.softmax(spkrs_pred, dim=-1), dim=-1))
matches[~input_mask] = False
cla = (cla_count * cla + torch.sum(matches).item() /
torch.sum(input_mask).item()) / (cla_count + 1)
cla_count += 1
# add batch losses to epoch losses
for k, v in batch_losses.items():
eval_losses[k] = v + eval_losses[k] if k in eval_losses else v
# normalize loss per batch
for k in eval_losses.keys():
eval_losses[k] /= len(data)
# log evaluation
Logger.evaluation(epoch + 1, eval_losses, mcd, src_len, trg_len, src,
post_trg, post_pred, post_pred_0, stop_pred_probs,
stop_trg, alignment_0, cla)
return sum(eval_losses.values())
def main(args):
if not os.path.exists(args.logs_dir):
os.mkdir(args.logs_dir)
if not os.path.exists(args.tensorboard_dir):
os.mkdir(args.tensorboard_dir)
tensorboardWrite = SummaryWriter(log_dir = args.tensorboard_dir)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
cudnn.benchmark = True
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# log file
if args.evaluate == 1:
sys.stdout = Logger(osp.join(args.logs_dir, 'log_test.txt'))
else:
sys.stdout = Logger(osp.join(args.logs_dir, 'log_train.txt'))
print("==========\nArgs:{}\n==========".format(args))
print("Initializing dataset {}".format(args.dataset))
# from reid.data import get_data ,
dataset, num_classes, train_loader, query_loader, gallery_loader = \
get_data(args, args.dataset, args.split, args.data_dir,
args.batch_size, args.seq_len, args.seq_srd,
args.workers)
print('[len] train: {}, query: {}, gallery: {}'.format(*list(map(len, [train_loader, query_loader, gallery_loader]))))
# create CNN model
# cnn_model = models.create(args.a1, args.flow1, args.flow2, num_features=args.features, dropout=args.dropout)
cnn_model_flow = [models.create(args.a1, args.flow1, num_features=args.features, dropout=args.dropout)]
if any(args.flow2):
cnn_model_flow.append(models.create(args.a1, args.flow2, num_features=args.features, dropout=args.dropout))
# cnn_model_flow1 = cnn_model_flow1.cuda()
# cnn_model_flow2 = cnn_model_flow2.cuda()
# create ATT model
input_num = cnn_model_flow[0].feat.in_features # 2048
output_num = args.features # 128
att_model = models.create(args.a2, input_num, output_num)
# att_model.cuda()
# # ------peixian:tow attmodel------
# att_model_flow1 = models.create(args.a2, input_num, output_num)
# att_model_flow2 = models.create(args.a2, input_num, output_num)
# # --------------------------------
# create classifier model
class_num = 2
classifier_model = models.create(args.a3, output_num, class_num)
# classifier_model.cuda()
# CUDA acceleration model
# cnn_model = torch.nn.DataParallel(cnn_model).to(device)
# # ------peixian:tow attmodel------
# for att_model in [att_model_flow1, att_model_flow2]:
# att_model = att_model.to(device)
# # --------------------------------
att_model = att_model.cuda()
classifier_model = classifier_model.cuda()
# cnn_model = torch.nn.DataParallel(cnn_model).cuda()
# cnn_model_flow1 = torch.nn.DataParallel(cnn_model_flow1,device_ids=[0,1,2])
# cnn_model_flow2 = torch.nn.DataParallel(cnn_model_flow2,device_ids=[0,1,2])
#
cnn_model_flow[0].cuda()
cnn_model_flow[0] = torch.nn.DataParallel(cnn_model_flow[0],device_ids=[0])
if len(cnn_model_flow) > 1:
cnn_model_flow[1].cuda()
cnn_model_flow[1] = torch.nn.DataParallel(cnn_model_flow[1],device_ids=[0])
# att_model = torch.nn.DataParallel(att_model,device_ids=[1,2,3])
# classifier_model = torch.nn.DataParallel(classifier_model,device_ids=[1,2,3])
criterion_oim = OIMLoss(args.features, num_classes,
scalar=args.oim_scalar, momentum=args.oim_momentum)
criterion_veri = PairLoss(args.sampling_rate)
criterion_oim.cuda()
criterion_veri.cuda()
# criterion_oim.cuda()
# criterion_veri.cuda()
# Optimizer
optimizer1 = []
# cnn_model_flow = [cnn_model_flow1, cnn_model_flow2]
for cnn_model in range(len(cnn_model_flow)):
base_param_ids = set(map(id, cnn_model_flow[cnn_model].module.base.parameters()))
new_params = [p for p in cnn_model_flow[cnn_model].module.parameters() if
id(p) not in base_param_ids]
param_groups1 = [
{'params': cnn_model_flow[cnn_model].module.base.parameters(), 'lr_mult': 1},
{'params': new_params, 'lr_mult': 1}]
optimizer1.append(torch.optim.SGD(param_groups1, lr=args.lr1,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True))
param_groups2 = [
{'params': att_model.parameters(), 'lr_mult': 1},
{'params': classifier_model.parameters(), 'lr_mult': 1}]
optimizer2 = torch.optim.SGD(param_groups2, lr=args.lr2,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
# optimizer1 = torch.optim.Adam(param_groups1, lr=args.lr1, weight_decay=args.weight_decay)
#
# optimizer2 = torch.optim.Adam(param_groups2, lr=args.lr2, weight_decay=args.weight_decay)
# Schedule Learning rate
def adjust_lr1(epoch):
lr = args.lr1 * (0.1 ** (epoch/args.lr1step))
print(lr)
for o in optimizer1:
for g in o.param_groups:
g['lr'] = lr * g.get('lr_mult', 1)
def adjust_lr2(epoch):
lr = args.lr2 * (0.01 ** (epoch//args.lr2step))
print(lr)
for g in optimizer2.param_groups:
g['lr'] = lr * g.get('lr_mult', 1)
# # peixian: two attmodel:
# for o in optimizer2:
# for g in o.param_groups:
# g['lr'] = lr * g.get('lr_mult', 1)
# #
def adjust_lr3(epoch):
lr = args.lr3 * (0.000001 ** (epoch //args.lr3step))
print(lr)
return lr
# Trainer
trainer = SEQTrainer(cnn_model_flow, att_model, classifier_model, criterion_veri, criterion_oim, args.lr3, args.flow1rate)
# Evaluator
evaluator = ATTEvaluator(cnn_model_flow, att_model, classifier_model, args.flow1rate)
best_top1 = 0
if args.evaluate == 1 or args.pretrain == 1: # evaluate
for cnn_model in range(len(cnn_model_flow)):
checkpoint = load_checkpoint(osp.join(args.logs_dir, 'cnnmodel_best_flow' + str(cnn_model) + '.pth.tar'))
cnn_model_flow[cnn_model].module.load_state_dict(checkpoint['state_dict'])
checkpoint = load_checkpoint(osp.join(args.logs_dir, 'attmodel_best.pth.tar'))
att_model.load_state_dict(checkpoint['state_dict'])
checkpoint = load_checkpoint(osp.join(args.logs_dir, 'clsmodel_best.pth.tar'))
classifier_model.load_state_dict(checkpoint['state_dict'])
top1 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo)
# top1 = evaluator.evaluate(query_loader, gallery_loader,dataset.num_tracklet)
if args.evaluate == 0:
for epoch in range(args.start_epoch, args.epochs):
adjust_lr1(epoch)
adjust_lr2(epoch)
rate = adjust_lr3(epoch)
trainer.train(epoch, train_loader, optimizer1, optimizer2, rate,tensorboardWrite)
if (epoch+1) % 1 == 0 or (epoch+1) == args.epochs:
top1 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo)
is_best = top1 > best_top1
if is_best:
best_top1 = top1
for cnn_model in range(len(cnn_model_flow)):
save_cnn_checkpoint({
'state_dict': cnn_model_flow[cnn_model].module.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, index=cnn_model, fpath=osp.join(args.logs_dir, 'cnn_checkpoint_flow'+str(cnn_model)+'.pth.tar'))
save_att_checkpoint({
'state_dict': att_model.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, fpath=osp.join(args.logs_dir, 'att_checkpoint.pth.tar'))
save_cls_checkpoint({
'state_dict': classifier_model.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, fpath=osp.join(args.logs_dir, 'cls_checkpoint.pth.tar'))
def main(args):
np.random.seed(args.seed)
torch.manual_seed(args.seed)
# cudnn.benchmark = True
# Redirect print to both console and log file
if not args.evaluate:
sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt'))
# Create data loaders
if args.height is None or args.width is None:
args.height, args.width = (144, 56) if args.arch == 'inception' else \
(240, 240)
dataset, num_classes, train_loader, val_loader, test_loader = \
get_data(args.dataset, args.split, args.data_dir, args.height,
args.width, args.batch_size, args.workers, args.combine_trainval)
# Create model
img_branch = models.create(args.arch,
cut_layer=args.cut_layer,
num_classes=num_classes,
num_features=args.features)
diff_branch = models.create(args.arch,
cut_layer=args.cut_layer,
num_classes=num_classes,
num_features=args.features)
# Load from checkpoint
start_epoch = best_top1 = 0
if args.resume:
checkpoint = load_checkpoint(args.resume)
img_branch.load_state_dict(checkpoint['state_dict_img'])
diff_branch.load_state_dict(checkpoint['state_dict_diff'])
start_epoch = checkpoint['epoch']
best_top1 = checkpoint['best_top1']
print("=> Start epoch {} best top1 {:.1%}".format(
start_epoch, best_top1))
img_branch = nn.DataParallel(img_branch).cuda()
diff_branch = nn.DataParallel(diff_branch).cuda()
# img_branch = nn.DataParallel(img_branch)
# diff_branch = nn.DataParallel(diff_branch)
# Criterion
criterion = nn.CrossEntropyLoss().cuda()
# criterion = nn.CrossEntropyLoss()
# Evaluator
evaluator = Evaluator(img_branch, diff_branch, criterion)
if args.evaluate:
# print("Validation:")
# top1, _ = evaluator.evaluate(val_loader)
# print("Validation acc: {:.1%}".format(top1))
print("Test:")
top1, (gt, pred) = evaluator.evaluate(test_loader)
print("Test acc: {:.1%}".format(top1))
from confusion_matrix import plot_confusion_matrix
plot_confusion_matrix(gt, pred, dataset.classes, args.logs_dir)
return
img_param_groups = [
{
'params': img_branch.module.low_level_modules.parameters(),
'lr_mult': 0.1
},
{
'params': img_branch.module.high_level_modules.parameters(),
'lr_mult': 0.1
},
{
'params': img_branch.module.classifier.parameters(),
'lr_mult': 1
},
]
diff_param_groups = [
{
'params': diff_branch.module.low_level_modules.parameters(),
'lr_mult': 0.1
},
{
'params': diff_branch.module.high_level_modules.parameters(),
'lr_mult': 0.1
},
{
'params': diff_branch.module.classifier.parameters(),
'lr_mult': 1
},
]
img_optimizer = torch.optim.SGD(img_param_groups,
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
diff_optimizer = torch.optim.SGD(diff_param_groups,
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
# Trainer
trainer = Trainer(img_branch, diff_branch, criterion)
# Schedule learning rate
def adjust_lr(epoch):
step_size = args.step_size
lr = args.lr * (0.1**(epoch // step_size))
for g in img_optimizer.param_groups:
g['lr'] = lr * g.get('lr_mult', 1)
for g in diff_optimizer.param_groups:
g['lr'] = lr * g.get('lr_mult', 1)
# Start training
for epoch in range(start_epoch, args.epochs):
adjust_lr(epoch)
trainer.train(epoch, train_loader, img_optimizer, diff_optimizer)
if epoch < args.start_save:
continue
top1, _ = evaluator.evaluate(val_loader)
is_best = top1 > best_top1
best_top1 = max(top1, best_top1)
save_checkpoint(
{
'state_dict_img': img_branch.module.state_dict(),
'state_dict_diff': diff_branch.module.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
},
is_best,
fpath=osp.join(args.logs_dir, 'checkpoint.pth.tar'))
print('\n * Finished epoch {:3d} top1: {:5.1%} best: {:5.1%}{}\n'.
format(epoch, top1, best_top1, ' *' if is_best else ''))
# Final test
print('Test with best model:')
checkpoint = load_checkpoint(osp.join(args.logs_dir, 'model_best.pth.tar'))
img_branch.module.load_state_dict(checkpoint['state_dict_img'])
diff_branch.module.load_state_dict(checkpoint['state_dict_diff'])
top1, (gt, pred) = evaluator.evaluate(test_loader)
from confusion_matrix import plot_confusion_matrix
plot_confusion_matrix(gt, pred, dataset.classes, args.logs_dir)
print('\n * Test Accuarcy: {:5.1%}\n'.format(top1))
def main():
parser = argparse.ArgumentParser(
description='Threat Prevention API example')
files_argument_group = parser.add_mutually_exclusive_group(
required=not IS_ONLY_GENERATE_TOKEN)
files_argument_group.add_argument('-D',
'--directory',
help='The scanning directory')
files_argument_group.add_argument('-fp',
'--file_path',
help='Path to file')
parser.add_argument('-fn',
'--file_name',
help='File Name, relevant when file path supplied')
parser.add_argument(
'-R',
'--recursive',
action='store_true',
help=
'Emulate the files in the directory recursively, relevant when scanning directory supplied'
)
server_argument_group = parser.add_mutually_exclusive_group(
required=not IS_ONLY_GENERATE_TOKEN)
server_argument_group.add_argument('-k', '--key', help='API key')
server_argument_group.add_argument('-e',
'--sandblast_appliance',
help='Check Point SandBlast Appliance')
server_argument_group.add_argument(
'-ci',
'--client_id',
nargs=2,
metavar=('CLIENT_ID', 'ACCESS_KEY'),
help=
'Client ID and Access key, used for JWT token authenticated requests')
parser.add_argument(
'-gt',
'--generate_token',
action='store_true',
help='Only create the JWT token without sending a request')
parser.add_argument('-d',
'--debug',
action='store_true',
help='Add debugging')
blades_info = parser.add_argument_group('Blades info')
blades_info.add_argument('-t',
'--te',
action='store_true',
help='Activate Threat Emulation')
blades_info.add_argument(
'--tex',
action='store_true',
help='Activate Threat Extraction (supported only with cloud)')
blades_info.add_argument(
'--tex_folder',
help=
'A folder to download the Scrubbing attachments (required when TEX is active)'
)
blades_info.add_argument(
'-m',
'--tex_method',
choices=['convert', 'clean'],
default='convert',
help='Scrubbing method. Convert to PDF / CleanContent')
reports_section = parser.add_argument_group('Reports info',
'Download Reports')
reports_section.add_argument(
'-r',
'--reports',
help='A folder to download the reports to (required for cloud)',
required=False)
reports_section.add_argument(
'-p',
'--pdf',
action='store_true',
help='Download PDF reports',
)
reports_section.add_argument(
'-x',
'--xml',
action='store_true',
help='Download XML reports',
)
reports_section.add_argument(
'-s',
'--summary',
action='store_true',
help='Download summary reports',
)
args = parser.parse_args()
Logger.level = LogLevel.DEBUG if args.debug else LogLevel.INFO
# Asking the API to enable features and reports according
# to what was required by the user.
features = []
reports = []
server = ""
key = ""
client_id = ""
access_key = ""
file_path = ""
file_name = ""
directory = ""
args.te and features.append('te')
args.tex and features.append('extraction')
if (args.summary and args.pdf):
parser.error(
"Illegal request. Pdf reports are not available in the new Threat Emulation reports format. Requesting for pdf and summary reports simultaneously is not supported."
)
exit(-1)
args.xml and reports.append('xml')
args.pdf and reports.append('pdf')
args.summary and reports.append('summary')
# Verify the user values
if len(reports) and not args.reports:
parser.error("Please supply a reports directory")
exit(-1)
if args.key:
key = args.key
if not args.reports:
parser.error("API Key supplied, please supply a reports folder")
exit(-1)
elif args.client_id:
client_id = args.client_id[0]
access_key = args.client_id[1]
if not args.generate_token and not args.reports:
parser.error("API Token supplied, please supply a reports folder")
exit(-1)
elif args.sandblast_appliance:
if args.tex:
Logger.log(
LogLevel.ERROR,
'TEX is not supported with Check Point SandBlast Appliance')
features.remove('extraction')
server = args.sandblast_appliance
if args.tex:
if not args.tex_folder:
parser.error("TEX is active, please supply a tex folder")
exit(-1)
if not os.path.isdir(args.tex_folder):
Logger.log(LogLevel.ERROR, 'Invalid tex folder as input')
exit(-1)
if not args.generate_token:
if args.directory:
if not os.path.isdir(args.directory):
Logger.log(LogLevel.ERROR,
'Invalid scanning directory in input')
exit(-1)
directory = args.directory
else:
file_path = args.file_path.encode('utf-8')
if args.file_name and args.file_name != 0:
file_name = args.file_name.encode('utf-8')
else:
file_name = os.path.basename(file_path)
if not os.path.isfile(args.file_path):
Logger.log(LogLevel.ERROR,
'Invalid file path in input (%s)' % args.file_path)
exit(-1)
api = Run(directory, file_path, file_name, key, client_id, access_key,
args.generate_token, server, args.reports, args.tex_method,
args.tex_folder, features, reports, args.recursive)
if not api.is_pending_files():
Logger.log(LogLevel.INFO, 'The directory is empty')
exit(0)
if directory:
Logger.log(
LogLevel.INFO, 'Querying %d files from directory: %s' %
(len(api.pending), args.directory))
else:
Logger.log(LogLevel.INFO, 'Querying file: %s ' % (file_path))
api.query_directory(True)
api.print_arrays_status()
if api.is_pending_files():
Logger.log(LogLevel.INFO, 'UPLOADING'),
api.upload_directory()
api.print_arrays_status()
max_tries = MAX_TRIES
while api.is_pending_files() and max_tries > 0:
time.sleep(WAITING_SEC)
api.query_directory(False)
api.print_arrays_status()
max_tries -= 1
api.print_arrays()
ret = api.get_final_status()
print("return {}".format(ret))
exit(ret)
def create_meta_file(dataset_name, dataset_root_dir, output_metafile_name, audio_sample_rate, num_fft_freqs, spectrograms=True, phonemes=True):
"""Create the meta-file and spectrograms (mel and linear, optionally) or phonemized utterances (optionally).
Format details:
Every line of the metadata file contains info about one dataset item.
The line has following format
'id|speaker|language|audio_file_path|mel_spectrogram_path|linear_spectrogram_path|text|phonemized_text'
And the following must hold
'audio_file_path' can be empty if loading just spectrograms
'text' should be carefully normalized and should contain interpunction
'phonemized_text' can be empty if loading just raw text
Arguments:
dataset_name (string): Name of the dataset, loaders.py should contain a function for loading with a corresponding name.
dataset_root_dir (string): Root directory from which is the dataset build and to which are spectrograms and the meta-file saved..
output_metafile_name (string): Name of the output meta-file.
audio_sample_rate (int): Sample rate of audios, used if spectrograms is set True.
num_fft_freqs (int): Number of frequency bands used during spectrogram computation, used if spectrograms is set True.
Keyword arguments:
spectrograms (boolean, default True): If true, spetrograms (both mel and linear) are computed and saved.
phonemes (boolean, default True): If true, phonemized variants of utterances are computed and saved.
"""
# save current sample rate and fft freqs hyperparameters, as we may process dataset with different sample rate
if spectrograms:
old_sample_rate = hp.sample_rate
hp.sample_rate = audio_sample_rate
old_fft_freqs = hp.num_fft
hp.num_fft = num_fft_freqs
# load metafiles, an item is a list like: [text, audiopath, speaker_id, language_code]
items = loaders.get_loader_by_name(dataset_name)(dataset_root_dir)
# build dictionaries for translation to IPA from source languages, see utils.text for details
if phonemes:
text_lang_pairs = [(i[0], hp.languages[0] if i[3] == "" else i[3]) for i in items]
phoneme_dicts = text.build_phoneme_dicts(text_lang_pairs)
# prepare directories which will store spectrograms
if spectrograms:
spectrogram_dirs = [os.path.join(dataset_root_dir, 'spectrograms'),
os.path.join(dataset_root_dir, 'linear_spectrograms')]
for x in spectrogram_dirs:
if not os.path.exists(x): os.makedirs(x)
# iterate through items and build the meta-file
metafile_path = os.path.join(dataset_root_dir, output_metafile_name)
with open(metafile_path, 'w', encoding='utf-8') as f:
Logger.progress(0, prefix='Building metafile:')
for i in range(len(items)):
raw_text, audio_path, speaker, language = items[i]
if language == "": language = hp.languages[0]
phonemized_text = text.to_phoneme(raw_text, False, language, phoneme_dicts[language]) if phonemes else ""
spectrogram_paths = "|"
if spectrograms:
spec_name = f'{str(i).zfill(6)}.npy'
audio_data = audio.load(os.path.join(dataset_root_dir, audio_path))
np.save(os.path.join(spectrogram_dirs[0], spec_name), audio.spectrogram(audio_data, True))
np.save(os.path.join(spectrogram_dirs[1], spec_name), audio.spectrogram(audio_data, False))
spectrogram_paths = os.path.join('spectrograms', spec_name) + '|' + os.path.join('linear_spectrograms', spec_name)
print(f'{str(i).zfill(6)}|{speaker}|{language}|{audio_path}|{spectrogram_paths}|{raw_text}|{phonemized_text}', file=f)
Logger.progress((i + 1) / len(items), prefix='Building metafile:')
# restore the original sample rate and fft freq values
if spectrograms:
hp.sample_rate = old_sample_rate
hp.num_fft = old_fft_freqs
import os
from utils.logging import Logger
logger = Logger().logger
class AppConfig:
"""
Contains the application configurations to be used.
"""
def __init__(self):
self.MAX_CONSIDERED_SUBREDDITS = int()
self.MAX_SUBMISSIONS_PER_SUBREDDIT = int()
self.MAX_COMMENTS_PER_SUBMISSION = int()
self.updated = False
self.set_config_from_environment()
def set_config_from_environment(self):
"""
Sets configuration from environment and sets it globally. Defaults to some values on error
:return: Nothing
"""
self.MAX_CONSIDERED_SUBREDDITS = int(
os.environ.get('MAX_CONSIDERED_SUBREDDITS', 50))
self.MAX_SUBMISSIONS_PER_SUBREDDIT = int(
os.environ.get('MAX_SUBMISSIONS_PER_SUBREDDIT', 10))
self.MAX_COMMENTS_PER_SUBMISSION = int(
os.environ.get('MAX_COMMENTS_PER_SUBMISSION', 5))
from utils.logging import Logger
from utils.timing import Timer
from math import isinf
from multiprocessing import Process, Value
import gelpia
import gelpia_logging
import time
logger = Logger(level=Logger.HIGH, color=Logger.green)
timer = Timer()
class GelpiaInfError(Exception):
def __init__(self, query):
self.query = query
CACHE = dict()
class Result():
# Setup logging to avoid runtime error
gelpia_logging.set_log_filename(None)
# Silence gelpia
gelpia_logging.set_log_level(-10)
parser.add_argument('--features', type=int, default=128)
parser.add_argument('--dropout', type=float, default=0)
# ======================================================================
args = parser.parse_args()
np.random.seed(args.seed)
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
cudnn.benchmark = True
# Redirect print to both console and log file
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
# 建立网络
net = ResNet.resnet50(num_classes=8631, include_top=False)
weights_path = 'weight_file/resnet50_ft_weight.pkl'
with open(weights_path, 'rb') as f:
from utils.various import parse_float
from utils.logging import Logger
logger = Logger()
# todo: this should probably be arbitrary precision or rationals
class Interval():
def __init__(self, inf, sup):
self.inf = parse_float(inf)
self.sup = parse_float(sup)
assert (self.inf <= self.sup)
def __str__(self):
return "[{},{}]".format(self.inf, self.sup)
def __repr__(self):
return "Interval({}, {})".format(self.inf, self.sup)
def __abs__(self):
# 0
# <---------------+--------------->
# [********]
if self.inf >= 0.0:
return Interval(self.inf, self.sup)
# 0
# <---------------+--------------->
# [********]
if self.inf <= 0.0 and 0.0 <= self.sup:
return tr_nll
opt.step(closure)
tr_nll = crf.neg_log_likelihood(e, b)
return tr_nll
if config['optim']['type'] == 'rmsprop':
optim_step = rmsprop_step
if config['optim']['type'] == 'lbfgs':
optim_step = lbfgs_step
##################################################
# Train for several epochs; if Ctrl-C interrupts,
# print message and deallocate.
##################################################
logger = Logger(save_dir)
step = 0
try:
for _ in range(num_epochs):
dataset = NPZLoader(train_set, test_set, batch_size=batch_size)
for events, bases in dataset.training_batches:
# preprocess events and bases batches:
_events = Variable(events.transpose(0, 1).contiguous())
_bases = Variable(bases.transpose(0, 1).contiguous().long())
if cuda:
_events = _events.cuda()
_bases = _bases.cuda()
# compute loss, grads, updates:
tr_nll = optim_step(_events, _bases)
def run_simulation(**kwargs):
kp = KwargsParser(kwargs, DEFAULTS)
folder = Path(kp.folder).expanduser()
folder.mkdir(exist_ok=True, parents=True)
file_str = f'L_{kp.L}_g_{kp.g}_chi_{kp.chi}_dt_{kp.dt}_quench_{kp.quench}'
if kp.task_id:
file_str += f'_{kp.task_id}'
logger = Logger(folder.joinpath(file_str + '.log'), True)
opt_logger = Logger(folder.joinpath(file_str + '.opt.log'), True)
outfile = folder.joinpath(file_str + '.pkl')
kp.log(logger)
opt_opts = dict(display_fun=get_display_fun(opt_logger),
line_search_fn='strong_wolfe',
max_iter=kp.max_iter,
tolerance_grad=kp.tolerance_grad)
cont_opts = dict(contraction_method='brute')
model = TFIM(kp.g,
bc='obc',
lx=kp.L,
ly=kp.L,
dtype_hamiltonian=np.float64)
evolver = TimeEvolution(kp.g,
kp.dt,
'obc',
real_time=True,
lx=kp.L,
ly=kp.L,
pepo_dtype=np.complex128)
logger.log(f'Starting with groundstate of g={kp.g} TFIM')
# Prepare groundstate
gs = None
gs_energy = None
if kp.gs_file:
logger.log('GS file specified, loading GS from file')
try:
with open(kp.gs_file, 'rb') as f:
res = pickle.load(f)
gs_tensors = res['gs_tensors']
gs = Peps(gs_tensors, 'obc')
gs_energy = res['gs_energy']
assert np.allclose(kp.g, res['kwargs']['g'])
assert gs.lx == kp.L
assert gs.ly == kp.L
except Exception as e:
logger.log('Failed to load GS from file. Error: ' + str(e))
if (gs is None) or (gs_energy is None):
logger.log('No GS file specified, optimising gs...')
gs, gs_energy = model.groundstate(kp.chi, (kp.L, kp.L), 'ps', 0.05,
cont_opts, opt_opts)
logger.log('Saving GS to ' +
str(folder.joinpath(file_str + '.gs.pkl')))
results = dict(kwargs=kp.kwargs(), gs=gs, gs_energy=gs_energy, g=kp.g)
with open(folder.joinpath(file_str + '.gs.pkl'), 'wb') as f:
pickle.dump(results, f)
# Prepare quench
if kp.quench == 'X': # <Sx(r,t) Sx(center,0)>
quench_operator = sx
measure_operator = sx
elif kp.quench == 'Y': # <Sy(r,t) Sy(center,0)>
quench_operator = sy
measure_operator = sy
elif kp.quench == 'Z': # <Sz(r,t) Sz(center,0)>
quench_operator = sz
measure_operator = sz
elif kp.quench == '+': # <S+(r,t) S-(center,0)>
quench_operator = sm
measure_operator = sp
else:
raise ValueError(f'Illegal quench code {kp.quench}')
logger.log(f'Quench: Applying quench operator to center site')
quenched = SingleSiteOperator(quench_operator, kp.L // 2,
kp.L // 2).apply_to_peps(gs)
# Time evolution
x_snapshot_data = onp.zeros([kp.n_steps + 1, kp.L, kp.L])
y_snapshot_data = onp.zeros([kp.n_steps + 1, kp.L, kp.L])
z_snapshot_data = onp.zeros([kp.n_steps + 1, kp.L, kp.L])
correlator_data = onp.zeros([kp.n_steps + 1, kp.L, kp.L],
dtype=onp.complex)
t_data = onp.zeros([kp.n_steps + 1])
state = quenched
opt_opts['dtype'] = np.complex128
opt_opts['max_grad_evals_ls'] = 100
for n in range(kp.n_steps):
logger.log('Computing Observables')
t = n * kp.dt
x_snapshot_data[n, :, :] = x_snapshot(state, cont_opts)
y_snapshot_data[n, :, :] = y_snapshot(state, cont_opts)
z_snapshot_data[n, :, :] = z_snapshot(state, cont_opts)
correlator_data[n, :, :] = correlator_timeslice(
gs, state, measure_operator, gs_energy, t, **cont_opts)
t_data[n] = t
logger.log(f'Evolving to t={(n + 1) * kp.dt}')
state = evolver.evolve(state,
contraction_options=cont_opts,
optimisation_options=opt_opts,
random_dev=None,
initial=kp.initial)
# save results (will be overwritten), (in case process dies before it finishes)
results = dict(kwargs=kp.kwargs(),
quench=kp.quench,
x_snapshot=x_snapshot_data,
y_snapshot=y_snapshot_data,
z_snapshot=z_snapshot_data,
correlator=correlator_data,
t=t_data,
state_tensors=state.get_tensors())
with open(outfile, 'wb') as f:
pickle.dump(results, f)
if kp.save_all_peps:
results = dict(kwargs=kp.kwargs(),
t=t,
state_tensors=state.get_tensors())
with open(folder.joinpath(file_str + f'state_t_{t}.pkl'),
'wb') as f:
pickle.dump(results, f)
logger.log('Computing Observables')
t = kp.n_steps * kp.dt
x_snapshot_data[kp.n_steps, :, :] = x_snapshot(state, cont_opts)
y_snapshot_data[kp.n_steps, :, :] = y_snapshot(state, cont_opts)
z_snapshot_data[kp.n_steps, :, :] = z_snapshot(state, cont_opts)
correlator_data[kp.n_steps, :, :] = correlator_timeslice(
gs, state, measure_operator, gs_energy, t, **cont_opts)
t_data[kp.n_steps] = t
# save results
logger.log(f'saving results to {outfile}')
results = dict(kwargs=kp.kwargs(),
quench=kp.quench,
x_snapshot=x_snapshot_data,
y_snapshot=y_snapshot_data,
z_snapshot=z_snapshot_data,
correlator=correlator_data,
t=t_data,
state_tensors=state.get_tensors())
with open(outfile, 'wb') as f:
pickle.dump(results, f)
if kp.save_all_peps:
results = dict(kwargs=kp.kwargs(),
t=t,
state_tensors=state.get_tensors())
with open(folder.joinpath(file_str + f'state_t_{t}.pkl'), 'wb') as f:
pickle.dump(results, f)
from domain.ebay import EbayPhone
from domain.phone import PhoneDetails
import pandas as pd
import numpy as np
from utils.text_utils import match_word, tokenize, update_vocabulary
from utils.logging import Logger
from nlp.embeddings import WordEmbeddings
from nlp.training import DataSet
from config import Config
logger = Logger.of('DataPreparation')
def create_embeddings():
titles = pd.read_csv(Config.get_filepath('train-data'))['title'].tolist()
titles = ['EMP ' + title + ' EMP' for title in titles]
embeddings = WordEmbeddings.from_sentences(titles)
embeddings.save(Config.get_filepath('word2vec'))
def clean_phones():
logger.info('cleaning phones')
EbayPhone.objects().update(details=PhoneDetails())
for phone in EbayPhone.objects(title__contains='*'):
phone.update(title=phone.title.replace('*', ' '))
for phone in EbayPhone.objects(title__contains='/'):
phone.update(title=phone.title.replace('/', ' '))
for phone in EbayPhone.objects(title__contains='+'):
phone.update(title=phone.title.replace('+', ' plus '))
if torch.cuda.is_available():
if args.cuda:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
if not args.cuda:
print("WARNING: It looks like you have a CUDA device, but aren't " +
"using CUDA.\nRun with --cuda for optimal training speed.")
torch.set_default_tensor_type('torch.FloatTensor')
else:
torch.set_default_tensor_type('torch.FloatTensor')
if not os.path.exists(args.save_folder):
os.mkdir(args.save_folder)
sys.stdout = Logger(os.path.join(args.save_folder, 'log.txt'))
def train():
if args.dataset == 'COCO':
'''if args.dataset_root == VOC_ROOT:
if not os.path.exists(COCO_ROOT):
parser.error('Must specify dataset_root if specifying dataset')
print("WARNING: Using default COCO dataset_root because " +
"--dataset_root was not specified.")
args.dataset_root = COCO_ROOT
cfg = coco
dataset = COCODetection(root=args.dataset_root,
transform=SSDAugmentation(cfg['min_dim'],
MEANS))'''
elif args.dataset == 'VOC':
'''if args.dataset_root == COCO_ROOT:
parser.add_argument(
"--max_pool",
type=int,
default=None,
help="Use max pooling after the initial convolution layer.",
)
ARGS = parser.parse_args()
ARGS.cuda = torch.cuda.is_available() and ARGS.cuda
ARGS.device = torch.device("cuda") if ARGS.cuda else torch.device("cpu")
if ARGS.conv_kernel_size is not None and len(ARGS.conv_kernel_size):
ARGS.conv_kernel_size = ARGS.conv_kernel_size[0]
log = Logger("TRAIN", ARGS.debug, ARGS.log_dir)
"""
Get audio all audio files from the given data directory except they are broken.
"""
def get_audio_files():
audio_files = None
if input_data.can_load_from_csv():
log.info("Found csv files in {}".format(ARGS.data_dir))
else:
log.debug("Searching for audio files in {}".format(ARGS.data_dir))
if ARGS.filter_broken_audio:
data_dir_ = pathlib.Path(ARGS.data_dir)
audio_files = get_audio_files_from_dir(ARGS.data_dir)
log.debug("Moving possibly broken audio files to .bkp:")
def main():
global best_prec
global opt
if opt['id'] != '':
model_id = opt['id']
else:
model_id = time.strftime("%m_%d_%H-%M-%S")
sys.stdout = Logger(osp.join(opt['log_dir'], 'log.' + model_id + '.txt'))
# initialize
checkpoint_dir = osp.join(opt['checkpoint_dir'], model_id)
mkdir_if_missing(checkpoint_dir)
# check gpu
assert opt['gpus'] is not None
# set random seed
cudnn.benchmark = False
cudnn.deterministic = True
random.seed(opt['seed'])
np.random.seed(opt['seed'])
torch.manual_seed(opt['seed'])
torch.cuda.manual_seed_all(opt['seed'])
# load imdb
train_refdb = get_db('refvg_train_' + opt['model_method'])
vocab = train_refdb.load_dictionary()
opt['vocab_size'] = len(vocab)
val_refdb = get_db('refvg_val_' + opt['model_method'])
# model, criterion, optimizer
model = SGReason(opt)
model = torch.nn.DataParallel(model).cuda()
criterion = SoftmaxLoss().cuda()
optimizer = torch.optim.Adam(list(model.parameters()) +
list(criterion.parameters()),
lr=opt['learning_rate'],
betas=(opt['optim_alpha'], opt['optim_beta']),
eps=opt['optim_epsilon'])
scheduler = ReduceLROnPlateau(optimizer,
factor=0.1,
patience=3,
mode='max')
if opt['evaluate']:
if osp.isfile(opt['model']):
model, criterion = load_checkpoint(model, criterion, opt['model'])
test_refdb = get_db('refvg_test_' + opt['model_method'])
test_dataset = RefDataset(test_refdb, vocab, opt)
test_loader = torch.utils.data.DataLoader(
test_dataset,
batch_size=opt['batch_size'],
shuffle=False,
num_workers=opt['workers'],
pin_memory=True)
test_loss, test_prec = validate(test_loader, model, criterion)
print(test_loss, test_prec)
else:
print("=> no checkpoint found at '{}'".format(opt['model']))
return
# start training
epoch_cur = 0
train_dataset = RefDataset(train_refdb, vocab, opt)
val_dataset = RefDataset(val_refdb, vocab, opt)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=opt['batch_size'],
shuffle=True,
num_workers=opt['workers'],
pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=opt['batch_size'],
shuffle=False,
num_workers=opt['workers'],
pin_memory=True)
for epoch in range(epoch_cur, opt['max_epochs']):
train(train_loader, model, criterion, optimizer, epoch)
val_loss, prec = validate(val_loader, model, criterion, epoch)
scheduler.step(prec)
for i, param_group in enumerate(optimizer.param_groups):
print(float(param_group['lr']))
is_best = prec >= best_prec
best_prec = max(best_prec, prec)
save_checkpoint(
{
'model_state_dict': model.state_dict(),
'crit_state_dict': criterion.state_dict(),
'optimizer': optimizer.state_dict()
}, is_best, checkpoint_dir, str(epoch))
def main():
global args, best_prec1, t ,loss_weight, Identity_num
t = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
loss_weight = 0.001
Identity_num = 'Null'
args = parser.parse_args()
sys.stdout = Logger(os.path.join('.', 'logs', args.arch + t + '.txt'))
# create model
print("=> creating model '{}'".format(args.arch))
print("=> --learning-rate '{}'".format(args.lr))
print("=> --momentum '{}'".format(args.momentum))
print("=> --weight_decay '{}'".format(args.weight_decay))
print("=> --batch-size '{}'".format(args.batch_size))
print("=> --pretrained '{}'".format(args.pretrained))
print("=> --loss_weight '{}'".format(loss_weight))
print("=> --Identity_num '{}'".format(Identity_num))
print("=> --model_path '{}'".format(args.model_path))
print("=> --result_path '{}'".format(args.result_path))
model = XTmodelzoo.__dict__[args.arch](num_classes=20, pretrained=args.pretrained)
# optionally resume from a checkpoint
if os.path.isfile(args.model_path):
print("=> loading checkpoint '{}'".format(args.model_path))
checkpoint = torch.load(args.model_path)
best_prec1 = checkpoint['best_prec1']
new_dict = {k[7:] : v for k,v in checkpoint['state_dict'].items()}
model.load_state_dict(new_dict)
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.model_path, checkpoint['epoch']))
else:
print("=> no model found at '{}'".format(args.model_path))
cudnn.benchmark = True
# Data loading code
testdir = os.path.join(args.data)
scale = 0.875
print('Images transformed from size {} to {}'.format(
int(round(max(model.input_size) / scale)),
model.input_size))
test_tf = utils.TransformImage(model, scale=scale)
test_loader = torch.utils.data.DataLoader(
MyTestDataset(data_root=testdir,csv_path=os.path.join('..','DataProcess','testB','list.csv'), transform=test_tf),
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True)
nllloss = nn.NLLLoss().cuda()
# Center loss
loss_weight = 0.001
center_loss = CenterLoss(20, 20).cuda()
criterion = [nllloss, center_loss]
optimizer4nn = torch.optim.SGD(model.parameters(), args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimzer4center = torch.optim.SGD(center_loss.parameters(), lr=0.5)
optimizer = [optimizer4nn, optimzer4center]
model = torch.nn.DataParallel(model).cuda()
test(test_loader, model, criterion, args.result_path)
def main(args):
## fix random_seed
fixRandomSeed(1)
## cuda setting
cudnn.benchmark = True
cudnn.enabled = True
device = torch.device('cuda:' + str(args.gpuid))
torch.cuda.set_device(device)
## Logger setting
if not args.evaluate:
sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt'))
print('logs_dir=', args.logs_dir)
print('args : ', args)
## get dataset & dataloader:
dataset, source_num_classes, source_train_loader, \
target_train_loader, query_loader, gallery_loader = get_data(args.data_dir, args.source,args.target,
args.source_train_path, args.target_train_path,
args.source_extension,args.target_extension,
args.height, args.width,
args.batch_size, args.re, args.workers)
h, w = map(int, [args.height, args.width])
input_size_source = (h, w)
input_size_target = (h, w)
# cudnn.enabled = True
# Create Network
# model = Res_Deeplab(num_classes=args.num_classes)
model = Res_Deeplab(num_classes=source_num_classes)
if args.restore_from[:4] == 'http':
saved_state_dict = model_zoo.load_url(args.restore_from)
else:
saved_state_dict = torch.load(args.restore_from)
new_params = model.state_dict().copy()
## adapte new_params's layers / classes to saved_state_dict
for i in saved_state_dict:
i_parts = i.split('.')
if not args.num_classes == 19 or not i_parts[1] == 'layer5':
new_params['.'.join(i_parts[1:])] = saved_state_dict[i]
if args.restore_from[:4] == './mo':
model.load_state_dict(new_params)
else:
model.load_state_dict(saved_state_dict)
## set mode = train and moves the params of model to GPU
model.train()
model.cuda(args.gpu)
# cudnn.benchmark = True
# Init D
model_D = FCDiscriminator(num_classes=args.num_classes)
# =============================================================================
# #for retrain
# saved_state_dict_D = torch.load(RESTORE_FROM_D)
# model_D.load_state_dict(saved_state_dict_D)
# =============================================================================
model_D.train()
model_D.cuda(args.gpu)
# if not os.path.exists(args.snapshot_dir):
# os.makedirs(args.snapshot_dir)
if args.source == 'GTA5':
trainloader = data.DataLoader(GTA5DataSet(
args.data_dir,
args.data_list,
max_iters=args.num_steps * args.iter_size * args.batch_size,
crop_size=input_size_source,
scale=True,
mirror=True,
mean=IMG_MEAN),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
else:
trainloader = data.DataLoader(SYNTHIADataSet(
args.data_dir,
args.data_list,
max_iters=args.num_steps * args.iter_size * args.batch_size,
crop_size=input_size_source,
scale=True,
mirror=True,
mean=IMG_MEAN),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
trainloader_iter = enumerate(trainloader)
targetloader = data.DataLoader(cityscapesDataSet(
args.data_dir_target,
args.data_list_target,
max_iters=args.num_steps * args.iter_size * args.batch_size,
crop_size=input_size_target,
scale=True,
mirror=True,
mean=IMG_MEAN,
set=args.set),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
targetloader_iter = enumerate(targetloader)
optimizer = optim.SGD(model.optim_parameters(args),
lr=args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer.zero_grad()
optimizer_D = optim.Adam(model_D.parameters(),
lr=args.learning_rate_D,
betas=(0.9, 0.99))
optimizer_D.zero_grad()
bce_loss = torch.nn.BCEWithLogitsLoss()
weighted_bce_loss = WeightedBCEWithLogitsLoss()
interp_source = nn.Upsample(size=(input_size_source[1],
input_size_source[0]),
mode='bilinear',
align_corners=True)
interp_target = nn.Upsample(size=(input_size_target[1],
input_size_target[0]),
mode='bilinear',
align_corners=True)
# Labels for Adversarial Training
source_label = 0
target_label = 1
for i_iter in range(args.num_steps):
optimizer.zero_grad()
adjust_learning_rate(optimizer, i_iter)
optimizer_D.zero_grad()
adjust_learning_rate_D(optimizer_D, i_iter)
damping = (1 - i_iter / NUM_STEPS)
# ======================================================================================
# train G
# ======================================================================================
# Remove Grads in D
for param in model_D.parameters():
param.requires_grad = False
# Train with Source
_, batch = next(trainloader_iter)
images_s, labels_s, _, _, _ = batch
images_s = Variable(images_s).cuda(args.gpu)
pred_source1, pred_source2 = model(images_s)
pred_source1 = interp_source(pred_source1)
pred_source2 = interp_source(pred_source2)
# Segmentation Loss
loss_seg = (loss_calc(pred_source1, labels_s, args.gpu) +
loss_calc(pred_source2, labels_s, args.gpu))
loss_seg.backward()
# Train with Target
_, batch = next(targetloader_iter)
images_t, _, _, _ = batch
images_t = Variable(images_t).cuda(args.gpu)
pred_target1, pred_target2 = model(images_t)
pred_target1 = interp_target(pred_target1)
pred_target2 = interp_target(pred_target2)
weight_map = weightmap(F.softmax(pred_target1, dim=1),
F.softmax(pred_target2, dim=1))
D_out = interp_target(
model_D(F.softmax(pred_target1 + pred_target2, dim=1)))
# Adaptive Adversarial Loss
if (i_iter > PREHEAT_STEPS):
loss_adv = weighted_bce_loss(
D_out,
Variable(
torch.FloatTensor(
D_out.data.size()).fill_(source_label)).cuda(args.gpu),
weight_map, Epsilon, Lambda_local)
else:
loss_adv = bce_loss(
D_out,
Variable(
torch.FloatTensor(
D_out.data.size()).fill_(source_label)).cuda(args.gpu))
loss_adv = loss_adv * Lambda_adv * damping
loss_adv.backward()
# Weight Discrepancy Loss
W5 = None
W6 = None
if args.model == 'ResNet':
for (w5, w6) in zip(model.layer5.parameters(),
model.layer6.parameters()):
if W5 is None and W6 is None:
W5 = w5.view(-1)
W6 = w6.view(-1)
else:
W5 = torch.cat((W5, w5.view(-1)), 0)
W6 = torch.cat((W6, w6.view(-1)), 0)
loss_weight = (torch.matmul(W5, W6) /
(torch.norm(W5) * torch.norm(W6)) + 1
) # +1 is for a positive loss
loss_weight = loss_weight * Lambda_weight * damping * 2
loss_weight.backward()
# ======================================================================================
# train D
# ======================================================================================
# Bring back Grads in D
for param in model_D.parameters():
param.requires_grad = True
# Train with Source
pred_source1 = pred_source1.detach()
pred_source2 = pred_source2.detach()
D_out_s = interp_source(
model_D(F.softmax(pred_source1 + pred_source2, dim=1)))
loss_D_s = bce_loss(
D_out_s,
Variable(
torch.FloatTensor(
D_out_s.data.size()).fill_(source_label)).cuda(args.gpu))
loss_D_s.backward()
# Train with Target
pred_target1 = pred_target1.detach()
pred_target2 = pred_target2.detach()
weight_map = weight_map.detach()
D_out_t = interp_target(
model_D(F.softmax(pred_target1 + pred_target2, dim=1)))
# Adaptive Adversarial Loss
if (i_iter > PREHEAT_STEPS):
loss_D_t = weighted_bce_loss(
D_out_t,
Variable(
torch.FloatTensor(
D_out_t.data.size()).fill_(target_label)).cuda(
args.gpu), weight_map, Epsilon, Lambda_local)
else:
loss_D_t = bce_loss(
D_out_t,
Variable(
torch.FloatTensor(
D_out_t.data.size()).fill_(target_label)).cuda(
args.gpu))
loss_D_t.backward()
optimizer.step()
optimizer_D.step()
print('exp = {}'.format(args.snapshot_dir))
print(
'iter = {0:6d}/{1:6d}, loss_seg = {2:.4f} loss_adv = {3:.4f}, loss_weight = {4:.4f}, loss_D_s = {5:.4f} loss_D_t = {6:.4f}'
.format(i_iter, args.num_steps, loss_seg, loss_adv, loss_weight,
loss_D_s, loss_D_t))
f_loss = open(osp.join(args.snapshot_dir, 'loss.txt'), 'a')
f_loss.write('{0:.4f} {1:.4f} {2:.4f} {3:.4f} {4:.4f}\n'.format(
loss_seg, loss_adv, loss_weight, loss_D_s, loss_D_t))
f_loss.close()
if i_iter >= args.num_steps_stop - 1:
print('save model ...')
torch.save(
model.state_dict(),
osp.join(args.snapshot_dir,
'GTA5_' + str(args.num_steps) + '.pth'))
torch.save(
model_D.state_dict(),
osp.join(args.snapshot_dir,
'GTA5_' + str(args.num_steps) + '_D.pth'))
break
if i_iter % args.save_pred_every == 0 and i_iter != 0:
print('taking snapshot ...')
torch.save(
model.state_dict(),
osp.join(args.snapshot_dir, 'GTA5_' + str(i_iter) + '.pth'))
torch.save(
model_D.state_dict(),
osp.join(args.snapshot_dir, 'GTA5_' + str(i_iter) + '_D.pth'))
## create dataloader
dataset, source_num_classes, source_train_loader, target_train_loader, query_loader, gallery_loader = get_data(
args.data_dir, args.source, args.target, args.source_train_path,
args.target_train_path, args.source_extension, args.target_extension,
args.height, args.width, args.batch_size, args.re, args.workers)
h, w = map(int, args.input_size_source.split(','))
input_size_source = (h, w)
input_size_target = (h, w)
for epoch in range(start_epoch, 210):
pj_model.two_classifier(epoch, train_loader_s, train_loader_t, writer, logger, rand_src_1, rand_src_2)
if ((epoch+1) % 2 == 0):
evaluator.run(val_loader)
cmc, mAP = evaluator.state.metrics['r1_mAP']
logger.info("Validation Results - Epoch: {}".format(epoch))
logger.info("mAP: {:.1%}".format(mAP))
for r in [1, 5, 10, 20]:
logger.info("CMC curve, Rank-{:<3}:{:.1%}".format(r, cmc[r - 1]))
is_best = cmc[0] > best_top1
best_top1 = max(cmc[0], best_top1)
save_checkpoint({
'Content_encoder': pj_model.Content_Encoder.module.state_dict(),
'Content_optimizer': pj_model.Content_optimizer.state_dict(),
'Content_optimizer_fix': pj_model.Content_optimizer_fix.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, fpath=cfg.OUTPUT_DIR + 'checkpoint.pth.tar', info=ckpt+'.pth.tar')
logger.info('\n * Finished epoch {:3d} top1: {:5.1%} best: {:5.1%}{}\n'.
format(epoch, cmc[0], best_top1, ' *' if is_best else ''))
writer.close()
if __name__ == '__main__':
info = 'market_duke_stage1'
sys.stdout = Logger(osp.join(cfg.OUTPUT_DIR, info+'.txt'))
print(info)
print('--------------------------------------------------')
logger = setup_logger("reid_baseline", cfg.OUTPUT_DIR, 0, info+'.txt')
main('two', info, logger)
k: v
for k, v in checkpoint_state['model'].items() if k in model_dict
}
model_dict.update(pretrained_dict)
model.load_state_dict(model_dict)
# other states from checkpoint -- optimizer, scheduler, loss, epoch
initial_epoch = checkpoint_state['epoch'] + 1
optimizer.load_state_dict(checkpoint_state['optimizer'])
scheduler.load_state_dict(checkpoint_state['scheduler'])
criterion.load_state_dict(checkpoint_state['criterion'])
# initialize logger
log_dir = os.path.join(
args.base_directory, "logs",
f'{hp.version}-{datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")}')
Logger.initialize(log_dir, args.flush_seconds)
# training loop
best_eval = float('inf')
for epoch in range(initial_epoch, hp.epochs):
train(args.logging_start, epoch, train_data, model, criterion,
optimizer)
if hp.learning_rate_decay_start - hp.learning_rate_decay_each < epoch * len(
train_data):
scheduler.step()
eval_loss = evaluate(epoch, eval_data, model, criterion)
if (epoch + 1) % hp.checkpoint_each_epochs == 0:
# save checkpoint together with hyper-parameters, optimizer and scheduler states
checkpoint_file = f'{checkpoint_dir}/{hp.version}_loss-{epoch}-{eval_loss:2.3f}'
state_dict = {
'epoch': epoch,
def main(args):
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
cudnn.benchmark = True
# log file
sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt'))
dataset, num_classes, train_loader, query_loader, gallery_loader = \
get_data(args.dataset, args.split, args.data_dir,
args.batch_size, args.seq_len, args.seq_srd,
args.workers, args.train_mode)
# create CNN model
cnn_model = models.create(args.a1, num_features=args.features, dropout=args.dropout)
# create ATT model
input_num = cnn_model.feat.in_features
output_num = args.features
att_model = models.create(args.a2, input_num, output_num)
# create classifier model
class_num = 2
classifier_model = models.create(args.a3, output_num, class_num)
# CUDA acceleration model
cnn_model = torch.nn.DataParallel(cnn_model).cuda()
att_model = att_model.cuda()
classifier_model = classifier_model.cuda()
# Loss function
criterion_oim = OIMLoss(args.features, num_classes,
scalar=args.oim_scalar, momentum=args.oim_momentum)
criterion_veri = PairLoss(args.sampling_rate)
criterion_oim.cuda()
criterion_veri.cuda()
# Optimizer
base_param_ids = set(map(id, cnn_model.module.base.parameters()))
new_params = [p for p in cnn_model.parameters() if
id(p) not in base_param_ids]
param_groups1 = [
{'params': cnn_model.module.base.parameters(), 'lr_mult': 1},
{'params': new_params, 'lr_mult': 1}]
param_groups2 = [
{'params': att_model.parameters(), 'lr_mult': 1},
{'params': classifier_model.parameters(), 'lr_mult': 1}]
optimizer1 = torch.optim.SGD(param_groups1, lr=args.lr1,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
optimizer2 = torch.optim.SGD(param_groups2, lr=args.lr2,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=True)
# Schedule Learning rate
def adjust_lr1(epoch):
lr = args.lr1 * (0.1 ** (epoch/args.lr1step))
print(lr)
for g in optimizer1.param_groups:
g['lr'] = lr * g.get('lr_mult', 1)
def adjust_lr2(epoch):
lr = args.lr2 * (0.01 ** (epoch//args.lr2step))
print(lr)
for g in optimizer2.param_groups:
g['lr'] = lr * g.get('lr_mult', 1)
def adjust_lr3(epoch):
lr = args.lr3 * (0.000001 ** (epoch //args.lr3step))
print(lr)
return lr
best_top1 = 0
start_epoch = args.start_epoch
if args.evaluate == 1:
print('Evaluate:')
evaluator = ATTEvaluator(cnn_model, att_model, classifier_model, args.train_mode, criterion_veri)
top1, mAP = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo)
elif args.test == 1:
print('Test:')
checkpoint1 = load_checkpoint(osp.join(args.logs_dir, 'cnnmodel_best.pth.tar'))
cnn_model.load_state_dict(checkpoint1['state_dict'])
checkpoint2 = load_checkpoint(osp.join(args.logs_dir, 'attmodel_best.pth.tar'))
att_model.load_state_dict(checkpoint2['state_dict'])
checkpoint3 = load_checkpoint(osp.join(args.logs_dir, 'clsmodel_best.pth.tar'))
classifier_model.load_state_dict(checkpoint3['state_dict'])
evaluator = ATTEvaluator(cnn_model, att_model, classifier_model, args.train_mode, criterion_veri)
mAP, top1, top5, top10, top20 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo)
else:
tensorboard_test_logdir = osp.join(args.logs_dir, 'test_log')
writer = SummaryWriter(log_dir=tensorboard_test_logdir)
if args.resume == 1:
checkpoint1 = load_checkpoint(osp.join(args.logs_dir, 'cnn_checkpoint.pth.tar'))
cnn_model.load_state_dict(checkpoint1['state_dict'])
checkpoint2 = load_checkpoint(osp.join(args.logs_dir, 'att_checkpoint.pth.tar'))
att_model.load_state_dict(checkpoint2['state_dict'])
checkpoint3 = load_checkpoint(osp.join(args.logs_dir, 'cls_checkpoint.pth.tar'))
classifier_model.load_state_dict(checkpoint3['state_dict'])
start_epoch = checkpoint1['epoch']
best_top1 = checkpoint1['best_top1']
print("=> Start epoch {} best top1 {:.1%}"
.format(start_epoch, best_top1))
# Trainer
tensorboard_train_logdir = osp.join(args.logs_dir, 'train_log')
trainer = SEQTrainer(cnn_model, att_model, classifier_model, criterion_veri, criterion_oim, args.train_mode, args.lr3, tensorboard_train_logdir)
# Evaluator
if args.train_mode == 'cnn':
evaluator = CNNEvaluator(cnn_model, args.train_mode)
elif args.train_mode == 'cnn_rnn':
evaluator = ATTEvaluator(cnn_model, att_model, classifier_model, args.train_mode, criterion_veri)
else:
raise RuntimeError('Yes, Evaluator is necessary')
for epoch in range(start_epoch, args.epochs):
adjust_lr1(epoch)
adjust_lr2(epoch)
rate = adjust_lr3(epoch)
trainer.train(epoch, train_loader, optimizer1, optimizer2, rate)
if epoch % 1 == 0:
mAP, top1, top5, top10, top20 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo)
writer.add_scalar('test/mAP', mAP, epoch+1)
writer.add_scalar('test/top1', top1, epoch+1)
writer.add_scalar('test/top5', top5, epoch+1)
writer.add_scalar('test/top10', top10, epoch+1)
writer.add_scalar('test/top20', top20, epoch+1)
is_best = top1 > best_top1
if is_best:
best_top1 = top1
save_cnn_checkpoint({
'state_dict': cnn_model.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, fpath=osp.join(args.logs_dir, 'cnn_checkpoint.pth.tar'))
if args.train_mode == 'cnn_rnn':
save_att_checkpoint({
'state_dict': att_model.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, fpath=osp.join(args.logs_dir, 'att_checkpoint.pth.tar'))
save_cls_checkpoint({
'state_dict': classifier_model.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, fpath=osp.join(args.logs_dir, 'cls_checkpoint.pth.tar'))
print('Test: ')
checkpoint1 = load_checkpoint(osp.join(args.logs_dir, 'cnnmodel_best.pth.tar'))
cnn_model.load_state_dict(checkpoint1['state_dict'])
checkpoint2 = load_checkpoint(osp.join(args.logs_dir, 'attmodel_best.pth.tar'))
att_model.load_state_dict(checkpoint2['state_dict'])
checkpoint3 = load_checkpoint(osp.join(args.logs_dir, 'clsmodel_best.pth.tar'))
classifier_model.load_state_dict(checkpoint3['state_dict'])
evaluator = ATTEvaluator(cnn_model, att_model, classifier_model, args.train_mode, criterion_veri)
mAP, top1, top5, top10, top20 = evaluator.evaluate(query_loader, gallery_loader, dataset.queryinfo, dataset.galleryinfo)
def train(logging_start_epoch, epoch, data, model, criterion, optimizer):
"""Main training procedure.
Arguments:
logging_start_epoch -- number of the first epoch to be logged
epoch -- current epoch
data -- DataLoader which can provide batches for an epoch
model -- model to be trained
criterion -- instance of loss function to be optimized
optimizer -- instance of optimizer which will be used for parameter updates
"""
model.train()
# initialize counters, etc.
learning_rate = optimizer.param_groups[0]['lr']
cla = 0
done, start_time = 0, time.time()
# loop through epoch batches
for i, batch in enumerate(data):
global_step = done + epoch * len(data)
optimizer.zero_grad()
# parse batch
batch = list(map(to_gpu, batch))
src, src_len, trg_mel, trg_lin, trg_len, stop_trg, spkrs, langs = batch
# get teacher forcing ratio
if hp.constant_teacher_forcing: tf = hp.teacher_forcing
else:
tf = cos_decay(
max(global_step - hp.teacher_forcing_start_steps, 0),
hp.teacher_forcing_steps)
# run the model
post_pred, pre_pred, stop_pred, alignment, spkrs_pred, enc_output = model(
src, src_len, trg_mel, trg_len, spkrs, langs, tf)
# evaluate loss function
post_trg = trg_lin if hp.predict_linear else trg_mel
classifier = model._reversal_classifier if hp.reversal_classifier else None
loss, batch_losses = criterion(src_len, trg_len, pre_pred, trg_mel,
post_pred, post_trg, stop_pred,
stop_trg, alignment, spkrs, spkrs_pred,
enc_output, classifier)
# evaluate adversarial classifier accuracy, if present
if hp.reversal_classifier:
input_mask = lengths_to_mask(src_len)
trg_spkrs = torch.zeros_like(input_mask, dtype=torch.int64)
for s in range(hp.speaker_number):
speaker_mask = (spkrs == s)
trg_spkrs[speaker_mask] = s
matches = (trg_spkrs == torch.argmax(torch.nn.functional.softmax(
spkrs_pred, dim=-1),
dim=-1))
matches[~input_mask] = False
cla = torch.sum(matches).item() / torch.sum(input_mask).item()
# comptute gradients and make a step
loss.backward()
gradient = torch.nn.utils.clip_grad_norm_(model.parameters(),
hp.gradient_clipping)
optimizer.step()
# log training progress
if epoch >= logging_start_epoch:
Logger.training(global_step, batch_losses, gradient, learning_rate,
time.time() - start_time, cla)
# update criterion states (params and decay of the loss and so on ...)
criterion.update_states()
start_time = time.time()
done += 1
class CsvSplit(object):
def __init__(
self,
split_fracs: Dict[str, float],
working_dir: (str) = None,
seed: (int) = None,
split_per_dir=False,
):
if not np.isclose(np.sum([p for _, p in split_fracs.items()]), 1.):
raise ValueError("Split probabilities have to sum up to 1.")
self.split_fracs = split_fracs
self.working_dir = working_dir
self.seed = seed
self.split_per_dir = split_per_dir
self.splits = defaultdict(list)
self._logger = Logger("CSVSPLIT")
"""
Return split for given partition. If there is already an existing CSV split return this split if it is valid or
in case there exist not a split yet generate a new CSV split
"""
def load(self, split: str, files: List[Any] = None):
if split not in self.split_fracs:
raise ValueError(
"Provided split '{}' is not in `self.split_fracs`.".format(
split))
if self.splits[split]:
return self.splits[split]
if self.working_dir is None:
self.splits = self._split_with_seed(files)
return self.splits[split]
if self.can_load_from_csv():
if not self.split_per_dir:
csv_split_files = {
split_: (os.path.join(self.working_dir, split_ + ".csv"), )
for split_ in self.split_fracs.keys()
}
else:
csv_split_files = {}
for split_ in self.split_fracs.keys():
split_file = os.path.join(self.working_dir, split_)
csv_split_files[split_] = []
with open(split_file, "r") as f:
for line in f.readlines():
csv_split_files[split_].append(line.strip())
for split_ in self.split_fracs.keys():
for csv_file in csv_split_files[split_]:
if not csv_file or csv_file.startswith(r"#"):
continue
csv_file_path = os.path.join(self.working_dir, csv_file)
with open(csv_file_path, "r") as f:
reader = csv.reader(f)
for item in reader:
file_ = os.path.basename(item[0])
file_ = os.path.join(os.path.dirname(csv_file),
file_)
self.splits[split_].append(file_)
return self.splits[split]
if not self.split_per_dir:
working_dirs = (self.working_dir, )
else:
f_d_map = self._get_f_d_map(files)
working_dirs = [
os.path.join(self.working_dir, p) for p in f_d_map.keys()
]
for working_dir in working_dirs:
splits = self._split_with_seed(
files if not self.split_per_dir else f_d_map[working_dir])
for split_ in splits.keys():
csv_file = os.path.join(working_dir, split_ + ".csv")
self._logger.debug("Generating {}".format(csv_file))
if self.split_per_dir:
with open(os.path.join(self.working_dir, split_),
"a") as f:
p = pathlib.Path(csv_file).relative_to(
self.working_dir)
f.write(str(p) + "\n")
if len(splits[split_]) == 0:
raise ValueError(
"Error splitting dataset. Split '{}' has 0 entries".
format(split_))
with open(csv_file, "w", newline="") as fh:
writer = csv.writer(fh)
for item in splits[split_]:
writer.writerow([item])
self.splits[split_].extend(splits[split_])
return self.splits[split]
"""
Check whether it is possible to correctly load information from existing csv files
"""
def can_load_from_csv(self):
if not self.working_dir:
return False
if self.split_per_dir:
for split in self.split_fracs.keys():
split_file = os.path.join(self.working_dir, split)
if not os.path.isfile(split_file):
return False
self._logger.debug(
"Found dataset split file {}".format(split_file))
with open(split_file, "r") as f:
for line in f.readlines():
csv_file = line.strip()
if not csv_file or csv_file.startswith(r"#"):
continue
if not os.path.isfile(
os.path.join(self.working_dir, csv_file)):
self._logger.error(
"File not found: {}".format(csv_file))
raise ValueError(
"Split file found, but csv files are missing. "
"Aborting...")
else:
for split in self.split_fracs.keys():
csv_file = os.path.join(self.working_dir, split + ".csv")
if not os.path.isfile(csv_file):
return False
self._logger.debug("Found csv file {}".format(csv_file))
return True
"""
Create a mapping from directory to containing files.
"""
def _get_f_d_map(self, files: List[Any]):
f_d_map = defaultdict(list)
if self.working_dir is not None:
for f in files:
f_d_map[str(pathlib.Path(
self.working_dir).joinpath(f).parent)].append(f)
else:
for f in files:
f_d_map[str(
pathlib.Path(".").resolve().joinpath(f).parent)].append(f)
return f_d_map
"""
Randomly splits the dataset using given seed
"""
def _split_with_seed(self, files: List[Any]):
if not files:
raise ValueError("Provided list `files` is `None`.")
if self.seed:
random.seed(self.seed)
return self.split_fn(files)
"""
A generator function that returns all values for the given `split`.
"""
def split_fn(self, files: List[Any]):
_splits = np.split(
ary=random.sample(files, len(files)),
indices_or_sections=[
int(p * len(files)) for _, p in self.split_fracs.items()
],
)
splits = dict()
for i, key in enumerate(self.splits.keys()):
splits[key] = _splits[i]
return splits
dest="jit_load",
action="store_true",
help="Load model via torch jit (otherwise via torch load).",
)
parser.add_argument(
"--input_file",
type=str,
default=None,
help=
"Input file could either be a directory with multiple audio files or just one single audio file"
)
ARGS = parser.parse_args()
log = Logger("PREDICT", ARGS.debug, ARGS.log_dir)
"""
Main function to compute prediction by using a trained model together with the given input
"""
if __name__ == "__main__":
if ARGS.checkpoint_path is not None:
log.info("Restoring checkpoint from {} instead of using a model file.".
format(ARGS.checkpoint_path))
checkpoint = torch.load(ARGS.checkpoint_path)
model = UNet(1, 1, bilinear=False)
model.load_state_dict(checkpoint["modelState"])
log.warning(
"Using default preprocessing options. Provide Model file if they are changed"
)
dataOpts = DefaultSpecDatasetOps
def main():
args = cli.parse_args()
if not args.no_det:
np.random.seed(args.env_seed)
random.seed(args.env_seed)
torch.manual_seed(args.env_seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
### Logger setup ###
logger = Logger(args.fileloc, args.load_loc)
logger.init_config(args)
logger.load_results()
### If we're making a visualisation, branch here ###
if args.task == 'vis':
if args.ranking not in logger.data['scores'][0]:
print("\nNeed to compute ranking {} before doing visualisation!".
format(args.ranking))
exit()
run_and_save(logger)
exit()
### Get or update all results ###
counts_do = {
'redo': args.redo_all or not logger.is_done('counts'),
'update': args.more_count is not None
}
scores_do = {
'redo':
counts_do['redo'] or counts_do['update']
or not logger.is_done('scores'),
'update':
args.more_scoretypes is not None
}
interpol_do = {
'redo':
scores_do['redo'] or not logger.is_done('interpol')
or args.redo_interpol is not None,
'update':
scores_do['update']
}
### Counts
if counts_do['redo']:
print("\n----- Counting -----\n")
counts = count(logger)
logger.update_counts(counts)
if counts_do['update']:
N = args.more_count
print("\n----- Additional Counts ({} more runs) -----\n".format(N))
# If we're adding more counts without having run before, then we need to reset the
# env or we would be revisiting the same states because of the seed.
if not counts_do['redo']:
for _ in range(logger.config['n_runs']):
logger.config['env'].reset()
counts = count(logger, n_runs=N)
logger.update_counts(counts, addn=N)
if counts_do['redo'] or counts_do['update']:
logger.dump_results()
logger.dump_config()
### Scores
if scores_do['redo']:
print("\n----- Scoring -----\n")
scores = score(logger)
logger.update_scores(scores)
if scores_do['update']:
already_done = [
st for st in args.more_scoretypes
if st in logger.config['score_types']
]
if len(already_done) != 0:
raise Exception(
"Scoretypes", ",".join(already_done),
"already done! Remove them from --more_scoretypes")
print("\n----- Additional Scores ({}) -----\n".format(
args.more_scoretypes))
scores = score(logger, score_types=args.more_scoretypes)
logger.update_scores(scores)
if scores_do['redo'] or scores_do['update']:
logger.dump_results()
logger.dump_config()
### Interpolation
if interpol_do['redo']:
print("\n----- Interpolating -----\n")
if args.redo_interpol is not None:
i, t = args.redo_interpol
logger.config['n_inc'] = i if i >= 0 else logger.config['n_inc']
logger.config['n_test'] = t if t >= 0 else logger.config['n_test']
elif logger.config['n_inc'] == -1:
logger.config['n_inc'] = int(
logger.data['logs'][0]['counting_abs_states'] / 10)
interpol = interpolate(logger)
logger.update_interpolation(interpol)
if interpol_do['update']:
print("\n----- Additional Interpolations ({}) -----\n".format(
args.more_scoretypes))
interpol = interpolate(logger, score_types=args.more_scoretypes)
logger.update_interpolation(interpol)
if interpol_do['redo'] or interpol_do['update']:
logger.dump_results()
logger.dump_config()
### Display results ###
draw_interpol_results(logger,
logger.config['score_types'],
0, [1],
x_fracs=True,
y_fracs=True,
smooth=False,
x_name='States Restored (%)',
y_names=['Original Reward (%)'],
combine_sbfl=True)
draw_interpol_results(logger,
logger.config['score_types'],
4, [1],
y_fracs=True,
trans_x=lambda x: 1 - x,
x_name="Policy's Action Taken (% of Steps)",
y_names=['Original Reward (%)'],
smooth=False,
combine_sbfl=True)
