def main():
# Build model.
model = model_builder.build_model(cfg=cfg)
# Read checkpoint.
ckpt = torch.load(
cfg.MODEL.PATH2CKPT,
map_location=torch.device("cpu")) if cfg.GENERAL.RESUME else {}
if cfg.GENERAL.RESUME:
with utils.log_info(msg="Load pre-trained model.",
level="INFO",
state=True):
model.load_state_dict(ckpt["model"])
# Set device.
model, device = utils.set_device(model, cfg.GENERAL.GPU)
try:
test_data_loader = data_loader.build_data_loader(
cfg, cfg.DATA.DATASET, "test")
generate(cfg=cfg,
model=model,
data_loader=test_data_loader,
device=device)
except:
utils.notify("Can not build data loader for test set.", level="ERROR")
raise ValueError("")
def init_sslyze(hostname, port, starttls_smtp, options, sync=False):
global network_timeout
network_timeout = int(options.get("network_timeout", network_timeout))
tls_wrapped_protocol = TlsWrappedProtocolEnum.PLAIN_TLS
if starttls_smtp:
tls_wrapped_protocol = TlsWrappedProtocolEnum.STARTTLS_SMTP
try:
# logging.debug("\tTesting connectivity with timeout of %is." % network_timeout)
server_tester = ServerConnectivityTester(
hostname=hostname,
port=port,
tls_wrapped_protocol=tls_wrapped_protocol)
server_info = server_tester.perform(network_timeout=network_timeout)
except ServerConnectivityError as err:
logging.warn("\tServer connectivity not established during test.")
return None, None
except Exception as err:
utils.notify(err)
logging.warn(
"\tUnknown exception when performing server connectivity info.")
return None, None
if sync:
scanner = SynchronousScanner(network_timeout=network_timeout)
else:
scanner = ConcurrentScanner(network_timeout=network_timeout)
return server_info, scanner
def init_sslyze(hostname, port, starttls_smtp, options, sync=False):
global network_timeout, CA_FILE
network_timeout = int(options.get("network_timeout", network_timeout))
if options.get('ca_file'):
CA_FILE = options['ca_file']
tls_wrapped_protocol = TlsWrappedProtocolEnum.PLAIN_TLS
if starttls_smtp:
tls_wrapped_protocol = TlsWrappedProtocolEnum.STARTTLS_SMTP
try:
# logging.debug("\tTesting connectivity with timeout of %is." % network_timeout)
server_tester = ServerConnectivityTester(hostname=hostname, port=port, tls_wrapped_protocol=tls_wrapped_protocol)
server_info = server_tester.perform(network_timeout=network_timeout)
except ServerConnectivityError:
logging.warning("\tServer connectivity not established during test.")
return None, None
except Exception as err:
utils.notify(err)
logging.warning("\tUnknown exception when performing server connectivity info.")
return None, None
if sync:
scanner = SynchronousScanner(network_timeout=network_timeout)
else:
scanner = ConcurrentScanner(network_timeout=network_timeout)
return server_info, scanner
def generate(
cfg,
model: torch.nn.Module,
data_loader: torch.utils.data.DataLoader,
device: torch.device,
logger=None,
*args,
**kwargs,
):
model.eval()
total_loss = []
with utils.log_info(msg="Generate results", level="INFO", state=True, logger=logger):
pbar = tqdm(total=len(data_loader), dynamic_ncols=True)
for idx, data in enumerate(data_loader):
start_time = time.time()
output, *_ = utils.inference(model=model, data=data, device=device)
for i in range(output.shape[0]):
save_dir = os.path.join(cfg.SAVE.DIR, "results")
if not os.path.exists(save_dir):
os.makedirs(save_dir)
path2file = os.path.join(save_dir, data["img_idx"][i]+".png")
succeed = utils.save_image(output[i].detach().cpu().numpy(), cfg.DATA.MEAN, cfg.DATA.NORM, path2file)
if not succeed:
utils.notify("Cannot save image to {}".format(path2file))
pbar.update()
pbar.close()
def evaluate(
epoch: int,
cfg,
model: torch.nn.Module,
data_loader: torch.utils.data.DataLoader,
device: torch.device,
loss_fn,
metrics_handler,
phase="valid",
logger=None,
save=False,
*args,
**kwargs,
):
model.eval()
# Read data and evaluate and record info.
msg="{} at epoch: {}".format(phase.upper(), str(epoch).zfill(3))
with logger.log_info(msg=msg, level="INFO", state=True, logger=logger):
pbar = tqdm(total=len(data_loader), dynamic_ncols=True)
for idx, data in enumerate(data_loader):
outputs, targets, loss = utils.infer_and_calc_loss(
model=model, data=data, loss_fn=loss_fn, device=device, infer_version=cfg.gnrl.infer, *args, **kwargs
)
if save:
# Save results to directory.
for idx_batch in range(outputs.shape[0]):
out = (outputs[idx_batch].detach().cpu().numpy() * 255).astype(np.uint8)
dir_save = os.path.join(cfg.save.dir, data_loader.dataset.dataset, phase)
utils.try_make_path_exists(dir_save)
path2dest = os.path.join(dir_save, data["img_idx"][idx_batch]+".png")
succeed = cv2.imwrite(path2dest, out.transpose(1, 2, 0))
if not succeed:
utils.notify("Failed to save image to {}".format(path2dest))
cur_loss = loss.detach().cpu().item()
avg_loss = metrics_handler.update(data_loader.dataset.dataset, phase, epoch, "loss", cur_loss)
utils.calc_and_record_metrics(data_loader.dataset.dataset, phase, epoch, outputs, targets, metrics_handler, 1.0)
pbar.set_description("Epoch: {:>3} / {:<3}, avg loss: {:<5}, cur loss: {:<5}".format(
epoch, cfg.train.max_epoch, round(avg_loss, 6), round(cur_loss, 6)
))
pbar.update()
pbar.close()
metrics_handler.summarize(data_loader.dataset.dataset, phase, epoch, logger=logger)
return
def inference(
cfg,
model: torch.nn.Module,
data_loader: torch.utils.data.DataLoader,
device: torch.device,
phase,
logger=None,
*args,
**kwargs,
):
model.eval()
# Read data and evaluate and record info.
with logger.log_info(msg="Inference",
level="INFO",
state=True,
logger=logger):
pbar = tqdm(total=len(data_loader), dynamic_ncols=True)
for idx, data in enumerate(data_loader):
outputs, *_ = utils.infer(model=model,
data=data,
device=device,
infer_version=cfg.gnrl.infer,
infer_only=True,
*args,
**kwargs)
# Save results to directory.
for idx_batch in range(outputs.shape[0]):
out = (outputs[idx_batch].detach().cpu().numpy() * 255).astype(
np.uint8)
dir_save = os.path.join(cfg.save.dir,
data_loader.dataset.dataset, phase)
utils.try_make_path_exists(dir_save)
path2dest = os.path.join(dir_save,
data["img_idx"][idx_batch] + ".png")
succeed = cv2.imwrite(path2dest, out.transpose(1, 2, 0))
if not succeed:
utils.notify(
"Failed to save image to {}".format(path2dest))
pbar.update()
pbar.close()
def main():
# Set logger to record information.
utils.check_env(cfg)
logger = Logger(cfg)
logger.log_info(cfg)
metrics_handler = MetricsHandler(cfg.metrics)
# utils.pack_code(cfg, logger=logger)
# Build model.
model = model_builder.build_model(cfg=cfg, logger=logger)
optimizer = optimizer_helper.build_optimizer(cfg=cfg, model=model)
lr_scheduler = lr_scheduler_helper.build_scheduler(cfg=cfg,
optimizer=optimizer)
# Read checkpoint.
ckpt = torch.load(cfg.model.path2ckpt) if cfg.gnrl.resume else {}
if cfg.gnrl.resume:
with logger.log_info(msg="Load pre-trained model.",
level="INFO",
state=True,
logger=logger):
model.load_state_dict(ckpt["model"])
optimizer.load_state_dict(ckpt["optimizer"])
lr_scheduler.load_state_dict(ckpt["lr_scheduler"])
# Set device.
model, device = utils.set_pipline(
model, cfg) if cfg.gnrl.PIPLINE else utils.set_device(
model, cfg.gnrl.cuda)
resume_epoch = ckpt["epoch"] if cfg.gnrl.resume else 0
loss_fn = loss_fn_helper.build_loss_fn(cfg=cfg)
# Prepare dataset.
train_loaders, valid_loaders, test_loaders = dict(), dict(), dict()
for dataset in cfg.data.datasets:
if cfg.data[dataset].TRAIN:
try:
train_loaders[dataset] = data_loader.build_data_loader(
cfg, dataset, "train")
except:
utils.notify(msg="Failed to build train loader of %s" %
dataset)
if cfg.data[dataset].VALID:
try:
valid_loaders[dataset] = data_loader.build_data_loader(
cfg, dataset, "valid")
except:
utils.notify(msg="Failed to build valid loader of %s" %
dataset)
if cfg.data[dataset].TEST:
try:
test_loaders[dataset] = data_loader.build_data_loader(
cfg, dataset, "test")
except:
utils.notify(msg="Failed to build test loader of %s" % dataset)
# TODO Train, evaluate model and save checkpoint.
for epoch in range(cfg.train.max_epoch):
epoch += 1
if resume_epoch >= epoch:
continue
eval_kwargs = {
"epoch": epoch,
"cfg": cfg,
"model": model,
"loss_fn": loss_fn,
"device": device,
"metrics_handler": metrics_handler,
"logger": logger,
"save": cfg.save.save,
}
train_kwargs = {
"epoch": epoch,
"cfg": cfg,
"model": model,
"loss_fn": loss_fn,
"optimizer": optimizer,
"device": device,
"lr_scheduler": lr_scheduler,
"metrics_handler": metrics_handler,
"logger": logger,
}
ckpt_kwargs = {
"epoch": epoch,
"cfg": cfg,
"model": model.state_dict(),
"metrics_handler": metrics_handler,
"optimizer": optimizer.state_dict(),
"lr_scheduler": lr_scheduler.state_dict(),
}
for dataset in cfg.data.datasets:
if cfg.data[dataset].TRAIN:
utils.notify("Train on %s" % dataset)
train_one_epoch(data_loader=train_loaders[dataset],
**train_kwargs)
utils.save_ckpt(path2file=cfg.model.path2ckpt, **ckpt_kwargs)
if epoch in cfg.gnrl.ckphs:
utils.save_ckpt(path2file=os.path.join(
cfg.model.ckpts,
cfg.gnrl.id + "_" + str(epoch).zfill(5) + ".pth"),
**ckpt_kwargs)
for dataset in cfg.data.datasets:
utils.notify("Evaluating test set of %s" % dataset,
logger=logger)
if cfg.data[dataset].TEST:
evaluate(data_loader=test_loaders[dataset],
phase="test",
**eval_kwargs)
for dataset in cfg.data.datasets:
utils.notify("Evaluating valid set of %s" % dataset, logger=logger)
if cfg.data[dataset].VALID:
evaluate(data_loader=valid_loaders[dataset],
phase="valid",
**eval_kwargs)
# End of train-valid for loop.
eval_kwargs = {
"epoch": epoch,
"cfg": cfg,
"model": model,
"loss_fn": loss_fn,
"device": device,
"metrics_handler": metrics_handler,
"logger": logger,
"save": cfg.save.save,
}
for dataset in cfg.data.datasets:
if cfg.data[dataset].VALID:
utils.notify("Evaluating valid set of %s" % dataset, logger=logger)
evaluate(data_loader=valid_loaders[dataset],
phase="valid",
**eval_kwargs)
for dataset in cfg.data.datasets:
if cfg.data[dataset].TEST:
utils.notify("Evaluating test set of %s" % dataset, logger=logger)
evaluate(data_loader=test_loaders[dataset],
phase="test",
**eval_kwargs)
for dataset in cfg.data.datasets:
if "train" in cfg.data[dataset].INFER:
utils.notify("Inference on train set of %s" % dataset)
inference(data_loader=train_loaders[dataset],
phase="infer_train",
**eval_kwargs)
if "valid" in cfg.data[dataset].INFER:
utils.notify("Inference on valid set of %s" % dataset)
inference(data_loader=valid_loaders[dataset],
phase="infer_valid",
**eval_kwargs)
if "test" in cfg.data[dataset].INFER:
utils.notify("Inference on test set of %s" % dataset)
inference(data_loader=test_loaders[dataset],
phase="infer_test",
**eval_kwargs)
return None
def run_agent(config, sequences, align_table, name, best_score, updating,
total, individual, data_file):
"""
run the by config specified agent on the sequences and use the given align-table as util to speed up th alignment
:param config: configuration of the agent to train
:param sequences: sequences of the benchmark to run the agent on
:param align_table: table to use for alignment-speedup
:param name: name of the benchmark, needed for the reports after training
:param best_score: best_score ever reached on this benchmark
:param updating: flag indicating to update the json file
:param total: total number of trainings performed in this run
:param individual: flag to indicate the use of a fresh align-table
:param data_file: file to store all data about the computations
:return: - the message to print and to sent via Telegram,
- the score reached by the agent,
- the extended align_table with probably new alignments,
- the profile computed by this algorithm that leads to the result score and
- the permutation of the input sequences to get the resulting profile
"""
print(F"train {config.name} agent result on {name}")
global number
if config.id == MCTS_AGENT:
agent = MCTSAgent(sequences,
simulations=config.simulations,
rollouts=config.rollouts,
c=config.c,
score=config.score,
refinement=config.refinement,
console=config.print_console,
adjust=config.adjust)
if not individual:
agent.set_align_table(align_table)
# compute the alignment using UCT-MCTS to find the most promising sequence
env = RefinementWrapper(
sequences, agent,
config.score) if config.refinement else AlignmentWrapper(
sequences, agent)
try:
start = time.time()
_, permutation, profile, _ = env.run()
end = time.time()
runtime = end - start
except:
print("MCTS-Agent crashed! Return standard results.")
permutation = [] if config.refinement else list(
range(len(sequences)))
profile = sequences if config.refinement else align_progressive(
permutation, sequences)
runtime = 0
align_table = agent.get_align_table()
print("Align-Table-Stats:", align_table.stats)
# extract the score
score = profile.score() if profile is not None else (0, 0)
scoring = (score[0], score[1], "-", "-")
# create the results-message
mode = "Refinement" if config.refinement else "Progressive"
optimizing = "C" if config.score == C_SCORE else "SP"
message = F"MCTS algorithm result on {name}:\n" \
F"\tMode: {mode}, Optimizing: {optimizing}-Score\n" \
F"\tProgress: {number}/{total} ({round((number / total) * 100, 2)}%)\n" \
F"\tRuntime: {runtime} seconds\n" \
F"\tbest Alignment found: {score[0]}, {round(score[1], 2)}\n"
data_file.write(
F"MCTS\t{name}\t{mode}\t{optimizing}\t{runtime}\t{score[0]}\t{round(score[1], 2)}\t["
+ ";".join([str(p) for p in permutation]) + "]\t[" +
";".join([str(s) for s in align_table.stats]) + "]\n")
else:
# if not MCTS agent, create one of the other agents
if config.id == TABLE_AGENT:
agent, agent_trainer = create_table_agent(sequences, config)
elif config.id == VALUE_AGENT:
agent, agent_trainer = create_value_agent(sequences, config)
elif config.id == POLICY_AGENT:
agent, agent_trainer = create_policy_agent(sequences, config)
elif config.id == ACTOR_CRITIC_AGENT:
agent, agent_trainer = create_actor_critic_agent(sequences, config)
else:
agent, agent_trainer = create_alpha_zero_agent(sequences, config)
# train the agent ...
if not individual:
agent_trainer.set_align_table(align_table)
try:
_, _, _, runtime = agent_trainer.run(config.print_console,
config.print_graph)
align_table = agent_trainer.get_align_table()
print("Align-Table-Stats:", align_table.stats)
# ... and compute the final alignment
(profile, permutation), (reward, _, _,
_) = agent_trainer.evaluate_training()
except:
print("Error occurred in training of ")
permutation = list(range(len(sequences)))
profile = align_progressive(permutation, sequences, align_table)
reward = 0, 0
runtime = 0
# extract the score
score = profile.score() if profile is not None else (0, 0)
scoring = (score[0], score[1], reward[0], reward[1])
# create the results-message
mode = "Refinement" if config.refinement else "Progressive"
optimizing = "C" if config.score == C_SCORE else "SP"
message = F"{agent.name()} algorithm result on {name}:\n" \
F"\tMode: {mode}, Optimizing: {optimizing}-Score\n" \
F"\tProgress: {number}/{total} ({round((number / total) * 100, 2)}%)\n" \
F"\tRuntime: {runtime} seconds\n" \
F"\tbest Alignment found: {score[0]}, {round(score[1], 2)}\n" \
F"\tlast Alignment found: {reward[0]}, {round(reward[1], 2)}"
data_file.write(
F"{agent.name}\t{name}\t{mode}\t{optimizing}\t{runtime}\t{score[0]}\t{round(score[1], 2)}\t["
+ ";".join([str(p) for p in permutation]) + "]\t[" +
";".join([str(s) for s in align_table.stats]) + "]\n")
# increase counter for configs that have been tested
number += 1
# check, whether older alignment has been improved
if updating and (score[config.score], score[1 - config.score]) > \
(best_score[config.score], best_score[1 - config.score]):
message = "NEW BEST ALIGNMENT FOUND!\n" + message
# print the message of the results and notify via telegram
print(message)
if config.notification:
notify(message)
return message, scoring, align_table, profile, permutation
def init_sslyze(hostname, port, starttls_smtp, options, sync=False):
global network_timeout, CA_FILE
network_timeout = int(options.get("network_timeout", network_timeout))
if options.get('ca_file'):
CA_FILE = options['ca_file']
tls_wrapped_protocol = TlsWrappedProtocolEnum.PLAIN_TLS
if starttls_smtp:
tls_wrapped_protocol = TlsWrappedProtocolEnum.STARTTLS_SMTP
try:
# logging.debug("\tTesting connectivity with timeout of %is." % network_timeout)
server_tester = ServerConnectivityTester(
hostname=hostname,
port=port,
tls_wrapped_protocol=tls_wrapped_protocol)
server_info = server_tester.perform(network_timeout=network_timeout)
except ServerConnectivityError:
# Usually pshtt has already established that we can connect to the site, so let's try again a couple of times
try:
logging.debug(
"\t{}:{} Server connectivity check failed. Trying again...".
format(hostname, port))
time.sleep(10)
server_tester = ServerConnectivityTester(
hostname=hostname,
port=port,
tls_wrapped_protocol=tls_wrapped_protocol)
server_info = server_tester.perform(
network_timeout=(network_timeout * 2))
except Exception as err:
try:
logging.debug(
"\t{}:{} Server connectivity check failed. Trying again..."
.format(hostname, port))
time.sleep(30)
server_tester = ServerConnectivityTester(
hostname=hostname,
port=port,
tls_wrapped_protocol=tls_wrapped_protocol)
server_info = server_tester.perform(
network_timeout=(network_timeout * 2))
except Exception as err:
logging.warning(
"\t{}:{} Server connectivity not established during test.".
format(hostname, port))
return None, None
except dns.exception.DNSException as err:
logging.warning(
"\t{}:{} DNS exception when performing sslyze server connectivity info check."
.format(hostname, port))
logging.debug("\t:{}:{} DNS exception: {}".format(hostname, port, err))
return None, None
except Exception as err:
utils.notify(err)
logging.warning(
"\t{}:{} Unknown exception when performing server connectivity info."
.format(hostname, port))
return None, None
if sync:
scanner = SynchronousScanner(network_timeout=network_timeout)
else:
scanner = ConcurrentScanner(network_timeout=network_timeout)
return server_info, scanner
result.Optimize = C_SCORE
if result.Seed is not None:
torch.manual_seed(result.Seed)
print("parse configurations")
# process the arguments into
if result.Config is not None:
configurations, benchmarks = join_config_files([parse_config_file(config) for config in result.Config])
else:
configurations = [parse_arguments(result)]
benchmarks = {(benchs[b] if b in benchs else b): list(range(len(configurations))) for b in result.Benchmarks}
if result.Similarity:
print("compute identity matrices")
similarity_matrix(benchmarks)
exit(0)
print("start training")
# start the read part of the program
if result.BruteForce is not None and len(result.Benchmarks) != 0:
multiprocessing_brute_force(benchmarks, result)
else:
run_multiple_agents(benchmarks, configurations, result,
multithreading=(result.Multi != 1 and (result.Config is not None or len(benchmarks) > 1)))
time = time.time() - start
if result.Notify:
notify("Running finished within " + str(time) + " seconds.")
print("Used time:", time)
# button in case we would need selenium)
print(
colored('A) Getting latest web driver version...',
'magenta',
'on_grey',
attrs=['bold', 'underline']))
try:
driver = get_latest_webdriver()
except Exception as e:
print(
"\n\n\n🙁 Process aborted... The web driver could not be set. Please make sure your computer is connected to internet."
)
# Posting macOS X notification
notify(
title='dvid.py',
subtitle='Process aborted... ❌',
message=
'The web driver could not be set. Please make sure your computer is connected to internet.',
sound_path=project_path + "/Hero.wav")
# Terminating the program right now
sys.exit()
# Handling the input of the program
print(
colored('B) Handling the input of the program',
'magenta',
'on_grey',
attrs=['bold', 'underline']))
if argsDottextFilePath == 'clipboard': # in case we want to download only one video using the copied URL currently situated in the clipboard (this is only available when NOT in "debug mode")
textFileAsInput = False