def main(): # Build model. model = model_builder.build_model(cfg=cfg) # Read checkpoint. ckpt = torch.load( cfg.MODEL.PATH2CKPT, map_location=torch.device("cpu")) if cfg.GENERAL.RESUME else {} if cfg.GENERAL.RESUME: with utils.log_info(msg="Load pre-trained model.", level="INFO", state=True): model.load_state_dict(ckpt["model"]) # Set device. model, device = utils.set_device(model, cfg.GENERAL.GPU) try: test_data_loader = data_loader.build_data_loader( cfg, cfg.DATA.DATASET, "test") generate(cfg=cfg, model=model, data_loader=test_data_loader, device=device) except: utils.notify("Can not build data loader for test set.", level="ERROR") raise ValueError("")
def init_sslyze(hostname, port, starttls_smtp, options, sync=False): global network_timeout network_timeout = int(options.get("network_timeout", network_timeout)) tls_wrapped_protocol = TlsWrappedProtocolEnum.PLAIN_TLS if starttls_smtp: tls_wrapped_protocol = TlsWrappedProtocolEnum.STARTTLS_SMTP try: # logging.debug("\tTesting connectivity with timeout of %is." % network_timeout) server_tester = ServerConnectivityTester( hostname=hostname, port=port, tls_wrapped_protocol=tls_wrapped_protocol) server_info = server_tester.perform(network_timeout=network_timeout) except ServerConnectivityError as err: logging.warn("\tServer connectivity not established during test.") return None, None except Exception as err: utils.notify(err) logging.warn( "\tUnknown exception when performing server connectivity info.") return None, None if sync: scanner = SynchronousScanner(network_timeout=network_timeout) else: scanner = ConcurrentScanner(network_timeout=network_timeout) return server_info, scanner
def init_sslyze(hostname, port, starttls_smtp, options, sync=False): global network_timeout, CA_FILE network_timeout = int(options.get("network_timeout", network_timeout)) if options.get('ca_file'): CA_FILE = options['ca_file'] tls_wrapped_protocol = TlsWrappedProtocolEnum.PLAIN_TLS if starttls_smtp: tls_wrapped_protocol = TlsWrappedProtocolEnum.STARTTLS_SMTP try: # logging.debug("\tTesting connectivity with timeout of %is." % network_timeout) server_tester = ServerConnectivityTester(hostname=hostname, port=port, tls_wrapped_protocol=tls_wrapped_protocol) server_info = server_tester.perform(network_timeout=network_timeout) except ServerConnectivityError: logging.warning("\tServer connectivity not established during test.") return None, None except Exception as err: utils.notify(err) logging.warning("\tUnknown exception when performing server connectivity info.") return None, None if sync: scanner = SynchronousScanner(network_timeout=network_timeout) else: scanner = ConcurrentScanner(network_timeout=network_timeout) return server_info, scanner
def generate( cfg, model: torch.nn.Module, data_loader: torch.utils.data.DataLoader, device: torch.device, logger=None, *args, **kwargs, ): model.eval() total_loss = [] with utils.log_info(msg="Generate results", level="INFO", state=True, logger=logger): pbar = tqdm(total=len(data_loader), dynamic_ncols=True) for idx, data in enumerate(data_loader): start_time = time.time() output, *_ = utils.inference(model=model, data=data, device=device) for i in range(output.shape[0]): save_dir = os.path.join(cfg.SAVE.DIR, "results") if not os.path.exists(save_dir): os.makedirs(save_dir) path2file = os.path.join(save_dir, data["img_idx"][i]+".png") succeed = utils.save_image(output[i].detach().cpu().numpy(), cfg.DATA.MEAN, cfg.DATA.NORM, path2file) if not succeed: utils.notify("Cannot save image to {}".format(path2file)) pbar.update() pbar.close()
def evaluate( epoch: int, cfg, model: torch.nn.Module, data_loader: torch.utils.data.DataLoader, device: torch.device, loss_fn, metrics_handler, phase="valid", logger=None, save=False, *args, **kwargs, ): model.eval() # Read data and evaluate and record info. msg="{} at epoch: {}".format(phase.upper(), str(epoch).zfill(3)) with logger.log_info(msg=msg, level="INFO", state=True, logger=logger): pbar = tqdm(total=len(data_loader), dynamic_ncols=True) for idx, data in enumerate(data_loader): outputs, targets, loss = utils.infer_and_calc_loss( model=model, data=data, loss_fn=loss_fn, device=device, infer_version=cfg.gnrl.infer, *args, **kwargs ) if save: # Save results to directory. for idx_batch in range(outputs.shape[0]): out = (outputs[idx_batch].detach().cpu().numpy() * 255).astype(np.uint8) dir_save = os.path.join(cfg.save.dir, data_loader.dataset.dataset, phase) utils.try_make_path_exists(dir_save) path2dest = os.path.join(dir_save, data["img_idx"][idx_batch]+".png") succeed = cv2.imwrite(path2dest, out.transpose(1, 2, 0)) if not succeed: utils.notify("Failed to save image to {}".format(path2dest)) cur_loss = loss.detach().cpu().item() avg_loss = metrics_handler.update(data_loader.dataset.dataset, phase, epoch, "loss", cur_loss) utils.calc_and_record_metrics(data_loader.dataset.dataset, phase, epoch, outputs, targets, metrics_handler, 1.0) pbar.set_description("Epoch: {:>3} / {:<3}, avg loss: {:<5}, cur loss: {:<5}".format( epoch, cfg.train.max_epoch, round(avg_loss, 6), round(cur_loss, 6) )) pbar.update() pbar.close() metrics_handler.summarize(data_loader.dataset.dataset, phase, epoch, logger=logger) return
def inference( cfg, model: torch.nn.Module, data_loader: torch.utils.data.DataLoader, device: torch.device, phase, logger=None, *args, **kwargs, ): model.eval() # Read data and evaluate and record info. with logger.log_info(msg="Inference", level="INFO", state=True, logger=logger): pbar = tqdm(total=len(data_loader), dynamic_ncols=True) for idx, data in enumerate(data_loader): outputs, *_ = utils.infer(model=model, data=data, device=device, infer_version=cfg.gnrl.infer, infer_only=True, *args, **kwargs) # Save results to directory. for idx_batch in range(outputs.shape[0]): out = (outputs[idx_batch].detach().cpu().numpy() * 255).astype( np.uint8) dir_save = os.path.join(cfg.save.dir, data_loader.dataset.dataset, phase) utils.try_make_path_exists(dir_save) path2dest = os.path.join(dir_save, data["img_idx"][idx_batch] + ".png") succeed = cv2.imwrite(path2dest, out.transpose(1, 2, 0)) if not succeed: utils.notify( "Failed to save image to {}".format(path2dest)) pbar.update() pbar.close()
def main(): # Set logger to record information. utils.check_env(cfg) logger = Logger(cfg) logger.log_info(cfg) metrics_handler = MetricsHandler(cfg.metrics) # utils.pack_code(cfg, logger=logger) # Build model. model = model_builder.build_model(cfg=cfg, logger=logger) optimizer = optimizer_helper.build_optimizer(cfg=cfg, model=model) lr_scheduler = lr_scheduler_helper.build_scheduler(cfg=cfg, optimizer=optimizer) # Read checkpoint. ckpt = torch.load(cfg.model.path2ckpt) if cfg.gnrl.resume else {} if cfg.gnrl.resume: with logger.log_info(msg="Load pre-trained model.", level="INFO", state=True, logger=logger): model.load_state_dict(ckpt["model"]) optimizer.load_state_dict(ckpt["optimizer"]) lr_scheduler.load_state_dict(ckpt["lr_scheduler"]) # Set device. model, device = utils.set_pipline( model, cfg) if cfg.gnrl.PIPLINE else utils.set_device( model, cfg.gnrl.cuda) resume_epoch = ckpt["epoch"] if cfg.gnrl.resume else 0 loss_fn = loss_fn_helper.build_loss_fn(cfg=cfg) # Prepare dataset. train_loaders, valid_loaders, test_loaders = dict(), dict(), dict() for dataset in cfg.data.datasets: if cfg.data[dataset].TRAIN: try: train_loaders[dataset] = data_loader.build_data_loader( cfg, dataset, "train") except: utils.notify(msg="Failed to build train loader of %s" % dataset) if cfg.data[dataset].VALID: try: valid_loaders[dataset] = data_loader.build_data_loader( cfg, dataset, "valid") except: utils.notify(msg="Failed to build valid loader of %s" % dataset) if cfg.data[dataset].TEST: try: test_loaders[dataset] = data_loader.build_data_loader( cfg, dataset, "test") except: utils.notify(msg="Failed to build test loader of %s" % dataset) # TODO Train, evaluate model and save checkpoint. for epoch in range(cfg.train.max_epoch): epoch += 1 if resume_epoch >= epoch: continue eval_kwargs = { "epoch": epoch, "cfg": cfg, "model": model, "loss_fn": loss_fn, "device": device, "metrics_handler": metrics_handler, "logger": logger, "save": cfg.save.save, } train_kwargs = { "epoch": epoch, "cfg": cfg, "model": model, "loss_fn": loss_fn, "optimizer": optimizer, "device": device, "lr_scheduler": lr_scheduler, "metrics_handler": metrics_handler, "logger": logger, } ckpt_kwargs = { "epoch": epoch, "cfg": cfg, "model": model.state_dict(), "metrics_handler": metrics_handler, "optimizer": optimizer.state_dict(), "lr_scheduler": lr_scheduler.state_dict(), } for dataset in cfg.data.datasets: if cfg.data[dataset].TRAIN: utils.notify("Train on %s" % dataset) train_one_epoch(data_loader=train_loaders[dataset], **train_kwargs) utils.save_ckpt(path2file=cfg.model.path2ckpt, **ckpt_kwargs) if epoch in cfg.gnrl.ckphs: utils.save_ckpt(path2file=os.path.join( cfg.model.ckpts, cfg.gnrl.id + "_" + str(epoch).zfill(5) + ".pth"), **ckpt_kwargs) for dataset in cfg.data.datasets: utils.notify("Evaluating test set of %s" % dataset, logger=logger) if cfg.data[dataset].TEST: evaluate(data_loader=test_loaders[dataset], phase="test", **eval_kwargs) for dataset in cfg.data.datasets: utils.notify("Evaluating valid set of %s" % dataset, logger=logger) if cfg.data[dataset].VALID: evaluate(data_loader=valid_loaders[dataset], phase="valid", **eval_kwargs) # End of train-valid for loop. eval_kwargs = { "epoch": epoch, "cfg": cfg, "model": model, "loss_fn": loss_fn, "device": device, "metrics_handler": metrics_handler, "logger": logger, "save": cfg.save.save, } for dataset in cfg.data.datasets: if cfg.data[dataset].VALID: utils.notify("Evaluating valid set of %s" % dataset, logger=logger) evaluate(data_loader=valid_loaders[dataset], phase="valid", **eval_kwargs) for dataset in cfg.data.datasets: if cfg.data[dataset].TEST: utils.notify("Evaluating test set of %s" % dataset, logger=logger) evaluate(data_loader=test_loaders[dataset], phase="test", **eval_kwargs) for dataset in cfg.data.datasets: if "train" in cfg.data[dataset].INFER: utils.notify("Inference on train set of %s" % dataset) inference(data_loader=train_loaders[dataset], phase="infer_train", **eval_kwargs) if "valid" in cfg.data[dataset].INFER: utils.notify("Inference on valid set of %s" % dataset) inference(data_loader=valid_loaders[dataset], phase="infer_valid", **eval_kwargs) if "test" in cfg.data[dataset].INFER: utils.notify("Inference on test set of %s" % dataset) inference(data_loader=test_loaders[dataset], phase="infer_test", **eval_kwargs) return None
def run_agent(config, sequences, align_table, name, best_score, updating, total, individual, data_file): """ run the by config specified agent on the sequences and use the given align-table as util to speed up th alignment :param config: configuration of the agent to train :param sequences: sequences of the benchmark to run the agent on :param align_table: table to use for alignment-speedup :param name: name of the benchmark, needed for the reports after training :param best_score: best_score ever reached on this benchmark :param updating: flag indicating to update the json file :param total: total number of trainings performed in this run :param individual: flag to indicate the use of a fresh align-table :param data_file: file to store all data about the computations :return: - the message to print and to sent via Telegram, - the score reached by the agent, - the extended align_table with probably new alignments, - the profile computed by this algorithm that leads to the result score and - the permutation of the input sequences to get the resulting profile """ print(F"train {config.name} agent result on {name}") global number if config.id == MCTS_AGENT: agent = MCTSAgent(sequences, simulations=config.simulations, rollouts=config.rollouts, c=config.c, score=config.score, refinement=config.refinement, console=config.print_console, adjust=config.adjust) if not individual: agent.set_align_table(align_table) # compute the alignment using UCT-MCTS to find the most promising sequence env = RefinementWrapper( sequences, agent, config.score) if config.refinement else AlignmentWrapper( sequences, agent) try: start = time.time() _, permutation, profile, _ = env.run() end = time.time() runtime = end - start except: print("MCTS-Agent crashed! Return standard results.") permutation = [] if config.refinement else list( range(len(sequences))) profile = sequences if config.refinement else align_progressive( permutation, sequences) runtime = 0 align_table = agent.get_align_table() print("Align-Table-Stats:", align_table.stats) # extract the score score = profile.score() if profile is not None else (0, 0) scoring = (score[0], score[1], "-", "-") # create the results-message mode = "Refinement" if config.refinement else "Progressive" optimizing = "C" if config.score == C_SCORE else "SP" message = F"MCTS algorithm result on {name}:\n" \ F"\tMode: {mode}, Optimizing: {optimizing}-Score\n" \ F"\tProgress: {number}/{total} ({round((number / total) * 100, 2)}%)\n" \ F"\tRuntime: {runtime} seconds\n" \ F"\tbest Alignment found: {score[0]}, {round(score[1], 2)}\n" data_file.write( F"MCTS\t{name}\t{mode}\t{optimizing}\t{runtime}\t{score[0]}\t{round(score[1], 2)}\t[" + ";".join([str(p) for p in permutation]) + "]\t[" + ";".join([str(s) for s in align_table.stats]) + "]\n") else: # if not MCTS agent, create one of the other agents if config.id == TABLE_AGENT: agent, agent_trainer = create_table_agent(sequences, config) elif config.id == VALUE_AGENT: agent, agent_trainer = create_value_agent(sequences, config) elif config.id == POLICY_AGENT: agent, agent_trainer = create_policy_agent(sequences, config) elif config.id == ACTOR_CRITIC_AGENT: agent, agent_trainer = create_actor_critic_agent(sequences, config) else: agent, agent_trainer = create_alpha_zero_agent(sequences, config) # train the agent ... if not individual: agent_trainer.set_align_table(align_table) try: _, _, _, runtime = agent_trainer.run(config.print_console, config.print_graph) align_table = agent_trainer.get_align_table() print("Align-Table-Stats:", align_table.stats) # ... and compute the final alignment (profile, permutation), (reward, _, _, _) = agent_trainer.evaluate_training() except: print("Error occurred in training of ") permutation = list(range(len(sequences))) profile = align_progressive(permutation, sequences, align_table) reward = 0, 0 runtime = 0 # extract the score score = profile.score() if profile is not None else (0, 0) scoring = (score[0], score[1], reward[0], reward[1]) # create the results-message mode = "Refinement" if config.refinement else "Progressive" optimizing = "C" if config.score == C_SCORE else "SP" message = F"{agent.name()} algorithm result on {name}:\n" \ F"\tMode: {mode}, Optimizing: {optimizing}-Score\n" \ F"\tProgress: {number}/{total} ({round((number / total) * 100, 2)}%)\n" \ F"\tRuntime: {runtime} seconds\n" \ F"\tbest Alignment found: {score[0]}, {round(score[1], 2)}\n" \ F"\tlast Alignment found: {reward[0]}, {round(reward[1], 2)}" data_file.write( F"{agent.name}\t{name}\t{mode}\t{optimizing}\t{runtime}\t{score[0]}\t{round(score[1], 2)}\t[" + ";".join([str(p) for p in permutation]) + "]\t[" + ";".join([str(s) for s in align_table.stats]) + "]\n") # increase counter for configs that have been tested number += 1 # check, whether older alignment has been improved if updating and (score[config.score], score[1 - config.score]) > \ (best_score[config.score], best_score[1 - config.score]): message = "NEW BEST ALIGNMENT FOUND!\n" + message # print the message of the results and notify via telegram print(message) if config.notification: notify(message) return message, scoring, align_table, profile, permutation
def init_sslyze(hostname, port, starttls_smtp, options, sync=False): global network_timeout, CA_FILE network_timeout = int(options.get("network_timeout", network_timeout)) if options.get('ca_file'): CA_FILE = options['ca_file'] tls_wrapped_protocol = TlsWrappedProtocolEnum.PLAIN_TLS if starttls_smtp: tls_wrapped_protocol = TlsWrappedProtocolEnum.STARTTLS_SMTP try: # logging.debug("\tTesting connectivity with timeout of %is." % network_timeout) server_tester = ServerConnectivityTester( hostname=hostname, port=port, tls_wrapped_protocol=tls_wrapped_protocol) server_info = server_tester.perform(network_timeout=network_timeout) except ServerConnectivityError: # Usually pshtt has already established that we can connect to the site, so let's try again a couple of times try: logging.debug( "\t{}:{} Server connectivity check failed. Trying again...". format(hostname, port)) time.sleep(10) server_tester = ServerConnectivityTester( hostname=hostname, port=port, tls_wrapped_protocol=tls_wrapped_protocol) server_info = server_tester.perform( network_timeout=(network_timeout * 2)) except Exception as err: try: logging.debug( "\t{}:{} Server connectivity check failed. Trying again..." .format(hostname, port)) time.sleep(30) server_tester = ServerConnectivityTester( hostname=hostname, port=port, tls_wrapped_protocol=tls_wrapped_protocol) server_info = server_tester.perform( network_timeout=(network_timeout * 2)) except Exception as err: logging.warning( "\t{}:{} Server connectivity not established during test.". format(hostname, port)) return None, None except dns.exception.DNSException as err: logging.warning( "\t{}:{} DNS exception when performing sslyze server connectivity info check." .format(hostname, port)) logging.debug("\t:{}:{} DNS exception: {}".format(hostname, port, err)) return None, None except Exception as err: utils.notify(err) logging.warning( "\t{}:{} Unknown exception when performing server connectivity info." .format(hostname, port)) return None, None if sync: scanner = SynchronousScanner(network_timeout=network_timeout) else: scanner = ConcurrentScanner(network_timeout=network_timeout) return server_info, scanner
result.Optimize = C_SCORE if result.Seed is not None: torch.manual_seed(result.Seed) print("parse configurations") # process the arguments into if result.Config is not None: configurations, benchmarks = join_config_files([parse_config_file(config) for config in result.Config]) else: configurations = [parse_arguments(result)] benchmarks = {(benchs[b] if b in benchs else b): list(range(len(configurations))) for b in result.Benchmarks} if result.Similarity: print("compute identity matrices") similarity_matrix(benchmarks) exit(0) print("start training") # start the read part of the program if result.BruteForce is not None and len(result.Benchmarks) != 0: multiprocessing_brute_force(benchmarks, result) else: run_multiple_agents(benchmarks, configurations, result, multithreading=(result.Multi != 1 and (result.Config is not None or len(benchmarks) > 1))) time = time.time() - start if result.Notify: notify("Running finished within " + str(time) + " seconds.") print("Used time:", time)
# button in case we would need selenium) print( colored('A) Getting latest web driver version...', 'magenta', 'on_grey', attrs=['bold', 'underline'])) try: driver = get_latest_webdriver() except Exception as e: print( "\n\n\n🙁 Process aborted... The web driver could not be set. Please make sure your computer is connected to internet." ) # Posting macOS X notification notify( title='dvid.py', subtitle='Process aborted... ❌', message= 'The web driver could not be set. Please make sure your computer is connected to internet.', sound_path=project_path + "/Hero.wav") # Terminating the program right now sys.exit() # Handling the input of the program print( colored('B) Handling the input of the program', 'magenta', 'on_grey', attrs=['bold', 'underline'])) if argsDottextFilePath == 'clipboard': # in case we want to download only one video using the copied URL currently situated in the clipboard (this is only available when NOT in "debug mode") textFileAsInput = False