def main(dataset_name, net_name, xp_path, data_path, load_config, load_model, device, seed, tokenizer, clean_txt, embedding_size, pretrained_model, embedding_reduction, num_dimensions, flow_type, coupling_hidden_size, coupling_hidden_layers, coupling_num_flows, coupling_num_mixtures, coupling_dropout, coupling_input_dropout, max_seq_len, use_length_prior, use_time_embed, prior_dist_type, prior_dist_mu, prior_dist_sigma, prior_dist_start_x, prior_dist_stop_x, ad_score, n_attention_heads, attention_size, lambda_p, alpha_scheduler, optimizer_name, lr, n_epochs, lr_milestone, batch_size, weight_decay, n_jobs_dataloader, n_threads, normal_class): """ :arg DATASET_NAME: Name of the dataset to load. :arg NET_NAME: Name of the neural network to use. :arg XP_PATH: Export path for logging the experiment. :arg DATA_PATH: Root path of data. """ # Get configuration cfg = Config(locals().copy()) # Set up logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger() logger.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') xp_path += '/text_{}_{}'.format(dataset_name, net_name) if not os.path.exists(xp_path): os.makedirs(xp_path) log_file = xp_path + '/log.txt' file_handler = logging.FileHandler(log_file) file_handler.setLevel(logging.INFO) file_handler.setFormatter(formatter) logger.addHandler(file_handler) # Print paths logger.info('Log file is %s.' % log_file) logger.info('Data path is %s.' % data_path) logger.info('Export path is %s.' % xp_path) # Print experimental setup logger.info('Dataset: %s' % dataset_name) logger.info('Normal class: %d' % normal_class) logger.info('Network: %s' % net_name) logger.info('Tokenizer: %s' % cfg.settings['tokenizer']) logger.info('Clean text in pre-processing: %s' % cfg.settings['clean_txt']) if cfg.settings['embedding_size'] is not None: logger.info('Word vector embedding size: %d' % cfg.settings['embedding_size']) logger.info('Load pre-trained model: %s' % cfg.settings['pretrained_model']) # If specified, load experiment config from JSON-file if load_config: cfg.load_config(import_json=load_config) logger.info('Loaded configuration from %s.' % load_config) # Set seed for reproducibility if cfg.settings['seed'] != -1: random.seed(cfg.settings['seed']) np.random.seed(cfg.settings['seed']) torch.manual_seed(cfg.settings['seed']) torch.cuda.manual_seed(cfg.settings['seed']) torch.backends.cudnn.deterministic = True logger.info('Set seed to %d.' % cfg.settings['seed']) # Default device to 'cpu' if cuda is not available if not torch.cuda.is_available(): device = 'cpu' logger.info('Computation device: %s' % device) logger.info('Number of dataloader workers: %d' % n_jobs_dataloader) if n_threads > 0: torch.set_num_threads(n_threads) logger.info( 'Number of threads used for parallelizing CPU operations: %d' % n_threads) # Load data dataset = load_dataset(dataset_name, data_path, normal_class, cfg.settings['tokenizer'], clean_txt=cfg.settings['clean_txt'], max_seq_len=cfg.settings['max_seq_len']) if net_name == 'CNF': # Initialize CNF model cnf = CNF() encoding_params = { "num_flows": 0, "hidden_layers": 2, "hidden_size": 128 } cnf.set_network( net_name=net_name, dataset=dataset, pretrained_model=cfg.settings['pretrained_model'], embedding_size=cfg.settings['embedding_size'], num_dimensions=cfg.settings['num_dimensions'], encoding_params=encoding_params, coupling_hidden_size=cfg.settings['coupling_hidden_size'], coupling_hidden_layers=cfg.settings['coupling_hidden_layers'], coupling_num_flows=cfg.settings['coupling_num_flows'], coupling_num_mixtures=cfg.settings['coupling_num_mixtures'], coupling_dropout=cfg.settings['coupling_dropout'], coupling_input_dropout=cfg.settings['coupling_input_dropout'], max_seq_len=cfg.settings['max_seq_len'], use_time_embed=cfg.settings['use_time_embed']) # If specified, load model parameters from already trained model if load_model: cnf.load_model(import_path=load_model, device=device) logger.info('Loading model from %s.' % load_model) # Train model on dataset prior_dist_params = { "distribution_type": cfg.settings['prior_dist_type'], "mu": cfg.settings['prior_dist_mu'], "sigma": cfg.settings['prior_dist_sigma'], "start_x": cfg.settings['prior_dist_start_x'], "stop_x": cfg.settings['prior_dist_stop_x'] } cnf.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], use_length_prior=cfg.settings['use_length_prior'], prior_dist_params=prior_dist_params, weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model cnf.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader) elif net_name == 'EmbeddingNF': # Initialize EmbeddingNF model and set word embedding enf = EmbeddingNF() enf.set_network( net_name=net_name, dataset=dataset, pretrained_model=cfg.settings['pretrained_model'], embedding_size=cfg.settings['embedding_size'], embedding_reduction=cfg.settings['embedding_reduction'], flow_type=cfg.settings['flow_type'], coupling_hidden_size=cfg.settings['coupling_hidden_size'], coupling_num_flows=cfg.settings['coupling_num_flows'], use_length_prior=cfg.settings['use_length_prior'], device=cfg.settings['device']) # If specified, load model parameters from already trained model if load_model: enf.load_model(import_path=load_model, device=device) logger.info('Loading model from %s.' % load_model) # Train model on dataset enf.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model enf.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader) elif net_name == 'date_Net': os.environ["TOKENIZERS_PARALLELISM"] = 'false' masks_ = pkl.load(open(data_path + '/pseudo_labels128_p50.pkl', 'rb')) subset = [dataset.subset] tensorboard_dir = 'run' exp_prefix = 'tests' now = datetime.now() date_time = now.strftime("%m%d%Y_%H%M%S") run_name = f'{subset[0]}_{date_time}' train_args = { "fp16": False, "use_multiprocessing": False, "reprocess_input_data": False, "overwrite_output_dir": True, "num_train_epochs": 20, "save_eval_checkpoints": False, "save_model_every_epoch": False, "learning_rate": 1e-5, "warmup_steps": 1000, "train_batch_size": 16, #was 32 "eval_batch_size": 16, #was 32 "gradient_accumulation_steps": 1, "block_size": 128 + 2, "max_seq_length": 128 + 2, "dataset_type": "simple", "logging_steps": 500, "evaluate_during_training": True, "evaluate_during_training_steps": 500, #was 500 "evaluate_during_training_steps_anomaly": 500, #was 500 "anomaly_batch_size": 16, "evaluate_during_training_verbose": True, "use_cached_eval_features": True, "sliding_window": True, "vocab_size": 52000, "eval_anomalies": True, "random_generator": 1, "use_rtd_loss": True, "rtd_loss_weight": 50, "rmd_loss_weight": 100, "mlm_loss_weight": 1, "dump_histogram": 0, "eval_anomaly_after": 0, "train_just_generator": 0, "replace_tokens": 0, "extract_scores": 1, "subset_name": subset[0], "extract_repr": 0, # "vanilla_electra": { # "no_masks": masks, # }, # "vanilla_electra": False, "train_document": True, "tokenizer_name": "bert-base-uncased", "tensorboard_dir": f'{tensorboard_dir}/{exp_prefix}/{run_name}', "extract_reps": 0, "weight_decay": weight_decay, "optimizer": "AdamW", "scores_export_path": f"./token_scores/{run_name}/", "generator_config": { "embedding_size": 128, "hidden_size": 16, "num_hidden_layers": 1, }, "discriminator_config": { "hidden_dropout_prob": 0.5, "attention_probs_dropout_prob": 0.5, "embedding_size": 128, "hidden_size": 256, "num_hidden_layers": 4, }, "mlm_lr_ratio": 1, } train_file = f"{data_path}/{dataset_name}/train/{subset[0]}.txt" test_file = f"{data_path}/{dataset_name}/test/{subset[0]}.txt" outlier_file = f"{data_path}/{dataset_name}/test/{subset[0]}-outliers.txt" elif net_name == 'cvdd_Net': # Print CVDD configuration logger.info('Anomaly Score: %s' % cfg.settings['ad_score']) logger.info('Number of attention heads: %d' % cfg.settings['n_attention_heads']) logger.info('Attention size: %d' % cfg.settings['attention_size']) logger.info('Orthogonality regularization hyperparameter: %.3f' % cfg.settings['lambda_p']) logger.info('Temperature alpha annealing strategy: %s' % cfg.settings['alpha_scheduler']) # Initialize CVDD model and set word embedding cvdd = CVDD(cfg.settings['ad_score']) cvdd.set_network(net_name=net_name, dataset=dataset, pretrained_model=cfg.settings['pretrained_model'], embedding_size=cfg.settings['embedding_size'], attention_size=cfg.settings['attention_size'], n_attention_heads=cfg.settings['n_attention_heads']) # If specified, load model parameters from already trained model if load_model: cvdd.load_model(import_path=load_model, device=device) logger.info('Loading model from %s.' % load_model) # Train model on dataset cvdd.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], lambda_p=cfg.settings['lambda_p'], alpha_scheduler=cfg.settings['alpha_scheduler'], weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model cvdd.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader) elif net_name == 'cvdd_flow': # Initialize CVDD_Flow model and set word embedding cvdd_flow = CVDD_Flow(cfg.settings['ad_score']) cvdd_flow.set_network( net_name=net_name, dataset=dataset, pretrained_model=cfg.settings['pretrained_model'], embedding_size=cfg.settings['embedding_size'], attention_size=cfg.settings['attention_size'], n_attention_heads=cfg.settings['n_attention_heads']) # Train model on dataset cvdd_flow.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], lambda_p=cfg.settings['lambda_p'], alpha_scheduler=cfg.settings['alpha_scheduler'], weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model cvdd_flow.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
def test_config(self): from utils.config import Config status = Config('third_party_vulnerabilities', 'status').value self.assertTrue(int(status))
import discord import time from utils.logger import log from utils.config import Config config = Config() class Channel_Logger(): def __init__(self, bot): self.bot = bot async def log_to_channel(self, msg): if config.channel_logger_id: channel = self.bot.get_channel(config.channel_logger_id) if not channel: log.warning("Can't find logging master channel: {}".format(id)) else: try: await self.bot.send_message( channel, ":stopwatch: `{}` :mouse_three_button: `{}` {}".format( time.strftime(config.log_timeformat), channel.server.name, msg)) except discord.errors.Forbidden: log.warning( "Could not log to the channel log channel because I do not have permission to send messages in it!" ) async def mod_log(self, server, msg): log_channel = discord.utils.get(server.channels, name="mod-log")
# api-接口请求, ext-结果提取处理, ast-自定义断言 import json import pytest import allure from utils.config import Config from utils.log import logger from utils.sql import Sql from utils.support import encrypt from testAPI.common.pre_request import PRequest from testAPI.common.pre_sql_data import * # from testAPI.interface.mod_2_video.conftest import video_env # from testAPI.interface.mod_3_task.conftest import task_env __all__ = ('APIUser') config_user = Config('apiuser.yml') @pytest.fixture() def user_api(task_env): my_user = APIUser() def parse_api_user(user_conf): my_user.parse_conf_user(task_env, user_conf) return my_user yield parse_api_user my_user.delete_user() my_user.assert_user_attr('FAIL')
def test_caller(path, step_ind, on_val): ########################## # Initiate the environment ########################## # Choose which gpu to use GPU_ID = '1' # Set GPU visible device os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID # Disable warnings os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' ########################### # Load the model parameters ########################### # Load model parameters config = Config() config.load(path) ################################## # Change model parameters for test ################################## # Change parameters for the test here. For example, you can stop augmenting the input data. #config.augment_noise = 0.0001 #config.augment_color = 1.0 config.validation_size = 500 #config.batch_num = 10 ############## # Prepare Data ############## print() print('Dataset Preparation') print('*******************') # Initiate dataset configuration if config.dataset.startswith('ModelNet40'): dataset = ModelNet40Dataset(config.input_threads) elif config.dataset == 'S3DIS': dataset = S3DISDataset(config.input_threads) on_val = True elif config.dataset == 'Scannet': dataset = ScannetDataset(config.input_threads, load_test=(not on_val)) elif config.dataset.startswith('ShapeNetPart'): dataset = ShapeNetPartDataset( config.dataset.split('_')[1], config.input_threads) elif config.dataset == 'NPM3D': dataset = NPM3DDataset(config.input_threads, load_test=(not on_val)) elif config.dataset == 'Semantic3D': dataset = Semantic3DDataset(config.input_threads) else: raise ValueError('Unsupported dataset : ' + config.dataset) # Create subsample clouds of the models dl0 = config.first_subsampling_dl dataset.load_subsampled_clouds(dl0) # Initialize input pipelines if on_val: dataset.init_input_pipeline(config) else: dataset.init_test_input_pipeline(config) ############## # Define Model ############## print('Creating Model') print('**************\n') t1 = time.time() if config.dataset.startswith('ShapeNetPart'): model = KernelPointFCNN(dataset.flat_inputs, config) elif config.dataset.startswith('S3DIS'): model = KernelPointFCNN(dataset.flat_inputs, config) elif config.dataset.startswith('Scannet'): model = KernelPointFCNN(dataset.flat_inputs, config) elif config.dataset.startswith('NPM3D'): model = KernelPointFCNN(dataset.flat_inputs, config) elif config.dataset.startswith('ModelNet40'): model = KernelPointCNN(dataset.flat_inputs, config) elif config.dataset.startswith('Semantic3D'): model = KernelPointFCNN(dataset.flat_inputs, config) else: raise ValueError('Unsupported dataset : ' + config.dataset) # Find all snapshot in the chosen training folder snap_path = os.path.join(path, 'snapshots') snap_steps = [ int(f[:-5].split('-')[-1]) for f in os.listdir(snap_path) if f[-5:] == '.meta' ] # Find which snapshot to restore chosen_step = np.sort(snap_steps)[step_ind] chosen_snap = os.path.join(path, 'snapshots', 'snap-{:d}'.format(chosen_step)) # Create a tester class tester = ModelTester(model, restore_snap=chosen_snap) t2 = time.time() print('\n----------------') print('Done in {:.1f} s'.format(t2 - t1)) print('----------------\n') ############ # Start test ############ print('Start Test') print('**********\n') if config.dataset.startswith('ShapeNetPart'): if config.dataset.split('_')[1] == 'multi': tester.test_multi_segmentation(model, dataset) else: tester.test_segmentation(model, dataset) elif config.dataset.startswith('S3DIS'): tester.test_cloud_segmentation_on_val(model, dataset) elif config.dataset.startswith('Scannet'): if on_val: tester.test_cloud_segmentation_on_val(model, dataset) else: tester.test_cloud_segmentation(model, dataset) elif config.dataset.startswith('Semantic3D'): if on_val: tester.test_cloud_segmentation_on_val(model, dataset) else: tester.test_cloud_segmentation(model, dataset) elif config.dataset.startswith('NPM3D'): if on_val: tester.test_cloud_segmentation_on_val(model, dataset) else: tester.test_cloud_segmentation(model, dataset) elif config.dataset.startswith('ModelNet40'): tester.test_classification(model, dataset) else: raise ValueError('Unsupported dataset')
from torchtext.data import Field, BucketIterator, TabularDataset from model.seq2seq import Encoder, Decoder, Seq2Seq from utils.config import Config from utils import count_parameters, fix_seed, epoch_time from utils.postprocess import evaluate_results parser = argparse.ArgumentParser() parser.add_argument('--config', type=str, default='', help='Path to config file') args = parser.parse_args(args=['--config', 'config.txt']) if args.config: args = Config(args.config) if args.show: from utils import enable_tqdm as tqdm else: from utils import disable_tqdm as tqdm def training(model, iterator, optimizer, criterion, clip): model.train() epoch_loss = 0 total = 0 for i, batch in tqdm(enumerate(iterator), total=len(iterator)): src = batch.Question trg = batch.Answer total += len(batch)
from models.s3dis import PVCNN2 from utils.config import Config, configs # model configs.model = Config(PVCNN2) configs.model.num_classes = configs.data.num_classes configs.dataset.num_points = 8192
def main(model_log_dir, check_point, test_data_dir): tf.reset_default_graph() args = get_arguments() cfg = Config(dataset=args.dataset, is_training=False, INFER_SIZE=INFER_SIZE, filter_scale=args.filter_scale, eval_path_log=os.path.join(LOG_PATH, model_log_dir)) cfg.model_paths['others'] = os.path.join(LOG_PATH, model_log_dir, 'model.ckpt-%d' % check_point) model = model_config[args.model] net = model(cfg=cfg, mode='inference') weight = [0.4, 0.6] ensemble_input = tf.placeholder(dtype=net.logits_up.dtype, shape=[None]) ensemble_pred = tf.split(net.logits_up, 2, axis=len(net.logits_up.get_shape()) - 1)[1] * weight[0] ensemble_pred = tf.reshape(ensemble_pred, [-1, ]) ensemble_pred = ensemble_pred + tf.cast(ensemble_input, tf.float32) * tf.constant(weight[1]) ensemble_pred = tf.round(ensemble_pred) assert cfg.dataset == 'others' raw_data = {'img': [], 'rle_mask': []} net.create_session() net.restore(cfg.model_paths[args.model]) duration = 0 fig_list = glob.glob(test_data_dir + '*.jpg') t_model = get_model() for index, i in zip(range(len(fig_list)), fig_list): img = Image.open(i) start = time.time() input = np.squeeze(np.array(img, dtype=np.float32)) n_input = input.astype(np.uint8) res2, tnet_mask = ternauNet(n_input, t_model) feed_dict = {ensemble_input: np.reshape(res2, [-1, ]), net.img_placeholder: load_single_image(img_path=i, cfg=cfg)} ensemble_result, icnet = net.sess.run([ensemble_pred, net.output], feed_dict=feed_dict) end = time.time() duration += (end - start) # results = cv2.cvtColor((np.reshape(np.array(ensemble_result), cfg.param['eval_size']) * 255).astype(np.uint8), cv2.COLOR_GRAY2RGB) # tnet = Image.fromarray((tnet_mask * 255).astype(np.uint8)) # # icnet = np.array(np.reshape(icnet, cfg.param['eval_size']), dtype=np.uint8) * 255 # icnet = Image.fromarray(icnet.astype(np.uint8)) # # fig, ax1 = plt.subplots(figsize=(5, 3)) # # plot1 = plt.subplot(141) # plot1.set_title("Original", fontsize=10) # plt.imshow(img) # plt.axis('off') # # plot1 = plt.subplot(142) # plot1.set_title("TernauNet", fontsize=10) # plt.imshow(tnet) # plt.axis('off') # # plot1 = plt.subplot(143) # plot1.set_title("ICnet", fontsize=10) # plt.imshow(icnet) # plt.axis('off') # # plot2 = plt.subplot(144) # plot2.set_title("Ensemble Result", fontsize=10) # plt.imshow(results, cmap='gray') # plt.axis('off') # plt.show() ensemble_result = np.squeeze(ensemble_result) en = run_length_encode(ensemble_result) if i.find('.jpg') != -1: print('{}/ {} i is {}'.format(index, len(fig_list), i)) raw_data['img'].append(os.path.basename(i)) raw_data['rle_mask'].append(en) else: print('i is {}, not .jpg, exit now!'.format(i)) exit() mean_inference_time = duration / (len(fig_list) + 1) df = pandas.DataFrame(raw_data, columns=['img', 'rle_mask']) with open(os.path.join(LOG_PATH, model_log_dir, 'SUBMISSION.csv'), mode='w') as f: df.to_csv(f, index=False) Config.save_to_json( dict={'Total Inference Time': float(duration), "Mean Inference Time": mean_inference_time}, path=os.path.dirname(cfg.model_paths['others']), file_name='ensemble_inference.json', mode='eval') sess = tf.get_default_session() if sess: sess._exit__(None, None, None)
def compare_trainings(list_of_paths, list_of_labels=None): # Parameters # ********** steps_per_epoch = 0 smooth_epochs = 1 if list_of_labels is None: list_of_labels = [str(i) for i in range(len(list_of_paths))] # Read Training Logs # ****************** all_epochs = [] all_loss = [] all_lr = [] all_times = [] for path in list_of_paths: # Load parameters config = Config() config.load(path) # Compute number of steps per epoch if config.epoch_steps is None: if config.dataset == 'ModelNet40': steps_per_epoch = np.ceil(9843 / int(config.batch_num)) else: raise ValueError('Unsupported dataset') else: steps_per_epoch = config.epoch_steps smooth_n = int(steps_per_epoch * smooth_epochs) # Load results steps, L_out, L_reg, L_p, acc, t, memory = load_training_results(path) all_epochs += [np.array(steps) / steps_per_epoch] all_loss += [running_mean(L_out, smooth_n)] all_times += [t] # Learning rate lr_decay_v = np.array([lr_d for ep, lr_d in config.lr_decays.items()]) lr_decay_e = np.array([ep for ep, lr_d in config.lr_decays.items()]) max_e = max(np.max(all_epochs[-1]) + 1, np.max(lr_decay_e) + 1) lr_decays = np.ones(int(np.ceil(max_e)), dtype=np.float32) lr_decays[0] = float(config.learning_rate) lr_decays[lr_decay_e] = lr_decay_v lr = np.cumprod(lr_decays) all_lr += [lr[np.floor(all_epochs[-1]).astype(np.int32)]] # Plots learning rate # ******************* if False: # Figure fig = plt.figure('lr') for i, label in enumerate(list_of_labels): plt.plot(all_epochs[i], all_lr[i], linewidth=1, label=label) # Set names for axes plt.xlabel('epochs') plt.ylabel('lr') plt.yscale('log') # Display legends and title plt.legend(loc=1) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') # ax.set_yticks(np.arange(0.8, 1.02, 0.02)) # Plots loss # ********** # Figure fig = plt.figure('loss') for i, label in enumerate(list_of_labels): plt.plot(all_epochs[i], all_loss[i], linewidth=1, label=label) # Set names for axes plt.xlabel('epochs') plt.ylabel('loss') plt.yscale('log') # Display legends and title plt.legend(loc=1) plt.title('Losses compare') # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') # ax.set_yticks(np.arange(0.8, 1.02, 0.02)) # Plot Times # ********** # Figure fig = plt.figure('time') for i, label in enumerate(list_of_labels): plt.plot(all_epochs[i], np.array(all_times[i]) / 3600, linewidth=1, label=label) # Set names for axes plt.xlabel('epochs') plt.ylabel('time') # plt.yscale('log') # Display legends and title plt.legend(loc=0) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') # ax.set_yticks(np.arange(0.8, 1.02, 0.02)) # Show all plt.show()
class Test_Xjx_Dr_BorrowMoney(unittest.TestCase): base_url = Config().get('URL') bm_url = Config().get('before_borrowMoney_url') url = base_url + bm_url logger.info('请求的URL为:{0}'.format(url)) # 调用公共参数 data = JsonConfig().get_jsondata() # test_Xjx_Dr_borrowMoney1调用测试参数 test_data = JsonConfig( path='wholesale', jsonpath='borrowMoney.json').get_jsondata(element='before_borrowMoney') logger.info("测试数据为:{0}".format(test_data)) # test_Xjx_Dr_borrowMoney2调用测试参数 borrowMoney_test_data = JsonConfig( path='wholesale', jsonpath='borrowMoney.json').get_jsondata(element='borrowMoney') logger.info("测试数据为:{0}".format(borrowMoney_test_data)) def setUp(self): self.j = JMESPathExtractor() logger.info('开始执行测试前准备的数据,调用test_Xjx_login方法') self.xjx_login = Test_Xjx_login().test_Xjx_login() self.data['sessionid'] = self.xjx_login logger.info('123测试前准备的数据为:{0}'.format(self.data)) # 获取json测试数据 self.jsondata = self.test_data # 获取headers数据 self.j = JMESPathExtractor() self.test_headers = JsonConfig( path='wholesale', jsonpath='borrowMoney.json').get_jsondata(element='headers') # self.headers = self.j.addextract(query='headers', body=test_headers) self.test_headers['sessionid'] = self.xjx_login logger.info("123测试前准备的headers为:{0}".format(self.test_headers)) self.get_dbdata = database() select_userBankCardId = "SELECT id FROM user_card_info WHERE user_id = '768093098' ORDER BY id DESC LIMIT 1" logger.info('查询数据库数据的sql为:{0}'.format(select_userBankCardId)) db_userBankCardId_id = self.get_dbdata.fetch_one(select_userBankCardId) logger.info('查询数据库的数据,返回的order_id为:{0}'.format(db_userBankCardId_id)) json_str = json.dumps(db_userBankCardId_id) self.one_db_userBankCardId_id = self.j.extract(query='id', body=json_str) logger.info('one_db_userBankCardId_id的数据为:{0}'.format( self.one_db_userBankCardId_id)) self.jsondata['userBankCardId'] = self.one_db_userBankCardId_id logger.info('123data数据为:{0}'.format(self.jsondata)) def test_Xjx_Dr_borrowMoney1(self): ''' 大额借款前置方法(请求江西银行存管查询接口) :return: ''' before_url = self.url # 在测试参数前后追加双引号 str_test_data = "%s" % self.test_data client = HTTPClient(url=before_url, method='POST', headers=self.test_headers) res = client.send(data=str_test_data, params=self.data) logger.info('123client返回的参数:{0}'.format(res.text)) # 获取traceNo参数 self.traceNo = self.j.extract(query='data.traceNo', body=res.text) self.borrowMoney_test_data['traceNo'] = self.traceNo # 获取userBankCardId参数 select_orderid = "select id from asset_borrow_order where user_id = '768093098' ORDER BY id DESC LIMIT 1" logger.info('查询数据库数据的sql为:{0}'.format(select_orderid)) db_order_id = self.get_dbdata.fetch_one(select_orderid) self.userBankCardId = self.j.extract(query='data.traceNo', body=res.text) # 断言 # 获取borrowMoney.json文件中的assertlist数据 self.assertlist = JsonConfig( path='wholesale', jsonpath='borrowMoney.json').get_jsondata( element='before_borrowMoney_assertlist') logger.info('assertlist数据为:{0}'.format(self.assertlist)) # 断言rsq_code rsq_code = self.j.extract(query='code', body=res.text) self.assertEqual(rsq_code, self.j.addextract(query='code', body=self.assertlist)) # 断言rsq_message rsq_message = self.j.extract(query='message', body=res.text) self.assertEqual( rsq_message, self.j.addextract(query='message', body=self.assertlist)) # 断言rsq_orderAmount rsq_orderAmount = self.j.extract(query='data.orderAmount', body=res.text) self.assertEqual( rsq_orderAmount, self.j.addextract(query='orderAmount', body=self.assertlist)) # 断言rsq_bankName rsq_bankName = self.j.extract(query='data.bankName', body=res.text) self.assertEqual( rsq_bankName, self.j.addextract(query='bankName', body=self.assertlist)) # 断言rsq_cardNoLastFour rsq_cardNoLastFour = self.j.extract(query='data.cardNoLastFour', body=res.text) self.assertEqual( rsq_cardNoLastFour, self.j.addextract(query='cardNoLastFour', body=self.assertlist)) def test_Xjx_Dr_borrowMoney2(self): ''' 大额借款方法 :return: ''' borrowMoney_url = Config().get('borrowMoney_url') url = self.base_url + borrowMoney_url logger.info('大额借款的接口地址为:{0}'.format(url)) client = HTTPClient(url=url, method='POST', headers=self.test_headers) #set相关参数到测试参数中 self.borrowMoney_test_data[ 'userBankCardId'] = self.one_db_userBankCardId_id # 在测试参数前后追加双引号 # test_data = self.borrowMoney_test_data.encode("utf-8") str_test_data = "%s" % self.borrowMoney_test_data logger.info('321参数:{0}'.format(str_test_data)) res = client.send(data=str_test_data, params=self.data) logger.info('123456client返回的参数:{0}'.format(res.text))
Author : jccia date : 2021/5/23 ------------------------------------------------- """ import utils.config as globalVar import threading from resource.rizhiyiSearch import Rizhiyi from resource.updateVar import updateVar from resource.api import runDaemon as flaskApi from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.executors.pool import ThreadPoolExecutor from utils.config import Config rzy = Rizhiyi() updateVars = updateVar() conf = Config() executors = {'default': ThreadPoolExecutor(max_workers=2)} scheduler = BackgroundScheduler(executors=executors) def scheduleRunner(scheduleType): """ 定时任务器 :return: """ if scheduleType == "spl查询": scheduler.add_job(func=rzy.search, trigger="interval", seconds=conf.splSearchInterval) elif scheduleType == "凌晨更新变量":
for i in range(int(config.BOT_CLUSTERS)): self.instances.append(Instance(i + 1, loop=self.loop, main=self)) self.write_targets() scheduler = Scheduler(loop=self.loop, bot=self.bot) loop.create_task(scheduler.launch()) server = web.Server(self.handler) runner = web.ServerRunner(server) await runner.setup() site = web.TCPSite(runner, config.BOT_API_HOST, int(config.BOT_API_PORT)) await site.start() config = Config().load() loop = asyncio.get_event_loop() main = Main(loop=loop) loop.create_task(main.launch()) try: loop.run_forever() except KeyboardInterrupt: def shutdown_handler(_loop, context): if "exception" not in context or not isinstance( context["exception"], asyncio.CancelledError ): _loop.default_exception_handler(context)
def train(config_train): # load config config_encoder = Config(config_train.config_encoder) config_decoder = Config(config_train.config_decoder) # load training set and validation set (or development set) train_set = DataGenerator( path_formulas=config_train.train_path_formulas, dir_images=config_train.train_dir_images, path_matching=config_train.train_path_matching, ) val_set = DataGenerator(path_formulas=config_train.val_path_formulas, dir_images=config_train.val_dir_images, path_matching=config_train.val_path_matching) batch_size = config_train.batch_size clip_grad = config_train.clip_grad valid_steps = config_train.valid_steps log_steps = config_train.log_steps model_save_path = config_train.save_path # delele old archives delete_file(config_train.save_path + 'train_loss.txt') delete_file(config_train.save_path + 'val_loss.txt') encoder = ImageEncoder(config_encoder) vocab = Vocab.load(config_decoder.vocab_path) embedding = Embeddings(config_decoder, vocab) decoder = RNNDecoder(config_decoder, embedding) model = ImageToLatexModel(encoder, decoder) model.train() # initializing parameter with uniform distribution uniform_unit = config_train.uniform_unit if np.abs(uniform_unit) > 0: print('uniformly initializing parameters [-%f, +%f]' % (uniform_unit, uniform_unit), file=sys.stdout) for p in model.parameters(): p.data.uniform_(-uniform_unit, uniform_unit) device = torch.device('cuda:0' if config_train.cuda else 'cpu') print('use device: %s' % device, file=sys.stdout) model = model.to(device) optimizer = torch.optim.Adam(model.parameters(), lr=config_train.learning_rate) # number of trial to avoid overfitting, if reaching max, early stop! num_trial = 0 train_iter = patience = cum_loss = report_loss = cum_tgt_words = report_tgt_words = 0 cum_examples = report_examples = epoch = valid_num = 0 hist_valid_scores = [] train_time = begin_time = time.time() print('begin training ...') while True: epoch += 1 for batch in train_set.minibatch(batch_size): train_iter += 1 optimizer.zero_grad() loss = model(batch) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), clip_grad) optimizer.step() batch_loss = loss.item() * batch_size report_loss += batch_loss cum_loss += batch_loss target_words_num = sum(len(tgt[1:]) for tgt in batch[1]) report_tgt_words += target_words_num cum_tgt_words += target_words_num report_examples += batch_size cum_examples += batch_size if train_iter % log_steps == 0: print('epoch %d, iter %d, avg. loss %.2f, avg. ppl %.2f ' \ 'cum. examples %d, speed %.2f words/sec, time elapsed %.2f sec' % (epoch, train_iter, report_loss / report_examples, math.exp(report_loss / report_tgt_words), cum_examples, report_tgt_words / (time.time() - train_time), time.time() - begin_time), file=sys.stdout) train_time = time.time() report_loss = report_examples = report_tgt_words = 0. if train_iter % valid_steps == 0: print( 'epoch %d, iter %d, cum. loss %.2f, cum. ppl %.2f cum. examples %d' % (epoch, train_iter, cum_loss / cum_examples, np.exp(cum_loss / cum_tgt_words), cum_examples), file=sys.stdout) with open(config_train.save_path + 'train_loss.txt', 'a') as f: f.write(str(cum_loss / cum_examples) + '\n') cum_loss = cum_examples = 0. valid_num += 1 print('begin validation ...', file=sys.stdout) # compute val. ppl and belu val_ppl, val_loss = evaluate_ppl(model, val_set, batch_size=100) valid_metric = -val_ppl with open(config_train.save_path + 'val_loss.txt', 'a') as f: f.write(str(val_loss.data.item()) + '\n') print('validation: iter %d, val. ppl %f' % (train_iter, val_ppl), file=sys.stdout) is_better = len(hist_valid_scores ) == 0 or valid_metric > max(hist_valid_scores) hist_valid_scores.append(valid_metric) if is_better: patience = 0 print('save currently the best model to [%s]' % model_save_path, file=sys.stdout) model.save(model_save_path, 'best.model') torch.save(optimizer.state_dict(), model_save_path + '.optim') elif patience < config_train.patience: patience += 1 print('hit patience %d' % patience, file=sys.stdout) if patience == config_train.patience: num_trial += 1 print('hit #%d trial' % num_trial, file=sys.stdout) if num_trial == config_train.max_trial: print('eraly stop!', file=sys.stdout) sys.exit(0) # decay lr, and restore from previously best checkpoint # TODO: add more refined learning rate scheduler lr = optimizer.param_groups[0]['lr'] * float( config_train.lr_decay) print( 'load previously best model and decay learning rate to %f' % lr, file=sys.stdout) # load model params = torch.load( model_save_path + 'best.model', map_location=lambda storage, loc: storage) model.load_state_dict(params['state_dict']) model = model.to(device) print('restore parameters of the optimizers', file=sys.stdout) optimizer.load_state_dict( torch.load(model_save_path + '.optim')) # set new lr for param_group in optimizer.param_groups: param_group['lr'] = lr # reset patience patience = 0 if epoch == config_train.max_epoch: print('reach maximum number of epoches!', file=sys.stdout) sys.exit(0)
def __init__(self): self.sc = Config() self.imgs = []
def init(self): self.config = Config()
def compare_convergences_multisegment(list_of_paths, list_of_labels=None): # Parameters # ********** steps_per_epoch = 0 smooth_n = 10 if list_of_labels is None: list_of_labels = [str(i) for i in range(len(list_of_paths))] # Read Logs # ********* all_pred_epochs = [] all_instances_mIoUs = [] all_objs_mIoUs = [] all_objs_IoUs = [] all_parts = [] obj_list = [ 'Air', 'Bag', 'Cap', 'Car', 'Cha', 'Ear', 'Gui', 'Kni', 'Lam', 'Lap', 'Mot', 'Mug', 'Pis', 'Roc', 'Ska', 'Tab' ] print( 'Objs | Inst | Air Bag Cap Car Cha Ear Gui Kni Lam Lap Mot Mug Pis Roc Ska Tab' ) print( '-----|------|--------------------------------------------------------------------------------' ) for path in list_of_paths: # Load parameters config = Config() config.load(path) # Get the number of classes n_parts = [4, 2, 2, 4, 4, 3, 3, 2, 4, 2, 6, 2, 3, 3, 3, 3] part = config.dataset.split('_')[-1] # Get validation confusions file = join(path, 'val_IoUs.txt') val_IoUs = load_multi_IoU(file, n_parts) file = join(path, 'vote_IoUs.txt') vote_IoUs = load_multi_IoU(file, n_parts) #print(len(val_IoUs[0])) #print(val_IoUs[0][0].shape) # Get mean IoU #instances_mIoUs, objs_mIoUs = IoU_multi_metrics(val_IoUs, smooth_n) # Get mean IoU instances_mIoUs, objs_mIoUs = IoU_multi_metrics(vote_IoUs, smooth_n) # Aggregate results all_pred_epochs += [np.array([i for i in range(len(val_IoUs))])] all_instances_mIoUs += [instances_mIoUs] all_objs_IoUs += [objs_mIoUs] all_objs_mIoUs += [np.mean(objs_mIoUs, axis=1)] if part == 'multi': s = '{:4.1f} | {:4.1f} | '.format(100 * np.mean(objs_mIoUs[-1]), 100 * instances_mIoUs[-1]) for obj_mIoU in objs_mIoUs[-1]: s += '{:4.1f} '.format(100 * obj_mIoU) print(s) else: s = ' -- | -- | ' for obj_name in obj_list: if part.startswith(obj_name): s += '{:4.1f} '.format(100 * instances_mIoUs[-1]) else: s += ' -- '.format(100 * instances_mIoUs[-1]) print(s) all_parts += [part] # Plots # ***** if 'multi' in all_parts: # Figure fig = plt.figure('Instances mIoU') for i, label in enumerate(list_of_labels): if all_parts[i] == 'multi': plt.plot(all_pred_epochs[i], all_instances_mIoUs[i], linewidth=1, label=label) plt.xlabel('epochs') plt.ylabel('IoU') # Set limits for y axis #plt.ylim(0.55, 0.95) # Display legends and title plt.legend(loc=4) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) # Figure fig = plt.figure('mean of categories mIoU') for i, label in enumerate(list_of_labels): if all_parts[i] == 'multi': plt.plot(all_pred_epochs[i], all_objs_mIoUs[i], linewidth=1, label=label) plt.xlabel('epochs') plt.ylabel('IoU') # Set limits for y axis #plt.ylim(0.8, 1) # Display legends and title plt.legend(loc=4) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) for obj_i, obj_name in enumerate(obj_list): if np.any([part.startswith(obj_name) for part in all_parts]): # Figure fig = plt.figure(obj_name + ' mIoU') for i, label in enumerate(list_of_labels): if all_parts[i] == 'multi': plt.plot(all_pred_epochs[i], all_objs_IoUs[i][:, obj_i], linewidth=1, label=label) elif all_parts[i].startswith(obj_name): plt.plot(all_pred_epochs[i], all_objs_mIoUs[i], linewidth=1, label=label) plt.xlabel('epochs') plt.ylabel('IoU') # Set limits for y axis #plt.ylim(0.8, 1) # Display legends and title plt.legend(loc=4) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) # Show all plt.show()
import torch.optim as optim from models.shapenet import PointNet from utils.config import Config, configs # model configs.model = Config(PointNet) configs.model.num_classes = configs.data.num_classes configs.model.num_shapes = configs.data.num_shapes configs.model.extra_feature_channels = 0 configs.dataset.with_normal = False configs.train.scheduler = Config(optim.lr_scheduler.StepLR) configs.train.scheduler.step_size = 20 configs.train.scheduler.gamma = 0.5
def compare_convergences_segment(dataset, list_of_paths, list_of_names=None): # Parameters # ********** smooth_n = 10 if list_of_names is None: list_of_names = [str(i) for i in range(len(list_of_paths))] # Read Logs # ********* all_pred_epochs = [] all_mIoUs = [] all_class_IoUs = [] all_snap_epochs = [] all_snap_IoUs = [] # Load parameters config = Config() config.load(list_of_paths[0]) class_list = [ dataset.label_to_names[label] for label in dataset.label_values if label not in dataset.ignored_labels ] s = '{:^10}|'.format('mean') for c in class_list: s += '{:^10}'.format(c) print(s) print(10 * '-' + '|' + 10 * config.num_classes * '-') for path in list_of_paths: # Get validation IoUs file = join(path, 'val_IoUs.txt') val_IoUs = load_single_IoU(file, config.num_classes) # Get mean IoU class_IoUs, mIoUs = IoU_class_metrics(val_IoUs, smooth_n) # Aggregate results all_pred_epochs += [np.array([i for i in range(len(val_IoUs))])] all_mIoUs += [mIoUs] all_class_IoUs += [class_IoUs] s = '{:^10.1f}|'.format(100 * mIoUs[-1]) for IoU in class_IoUs[-1]: s += '{:^10.1f}'.format(100 * IoU) print(s) # Get optional full validation on clouds snap_epochs, snap_IoUs = load_snap_clouds(path, dataset) all_snap_epochs += [snap_epochs] all_snap_IoUs += [snap_IoUs] print(10 * '-' + '|' + 10 * config.num_classes * '-') for snap_IoUs in all_snap_IoUs: if len(snap_IoUs) > 0: s = '{:^10.1f}|'.format(100 * np.mean(snap_IoUs[-1])) for IoU in snap_IoUs[-1]: s += '{:^10.1f}'.format(100 * IoU) else: s = '{:^10s}'.format('-') for _ in range(config.num_classes): s += '{:^10s}'.format('-') print(s) # Plots # ***** # Figure fig = plt.figure('mIoUs') for i, name in enumerate(list_of_names): p = plt.plot(all_pred_epochs[i], all_mIoUs[i], '--', linewidth=1, label=name) plt.plot(all_snap_epochs[i], np.mean(all_snap_IoUs[i], axis=1), linewidth=1, color=p[-1].get_color()) plt.xlabel('epochs') plt.ylabel('IoU') # Set limits for y axis #plt.ylim(0.55, 0.95) # Display legends and title plt.legend(loc=4) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) displayed_classes = [0, 1, 2, 3, 4, 5, 6, 7] displayed_classes = [] for c_i, c_name in enumerate(class_list): if c_i in displayed_classes: # Figure fig = plt.figure(c_name + ' IoU') for i, name in enumerate(list_of_names): plt.plot(all_pred_epochs[i], all_class_IoUs[i][:, c_i], linewidth=1, label=name) plt.xlabel('epochs') plt.ylabel('IoU') # Set limits for y axis #plt.ylim(0.8, 1) # Display legends and title plt.legend(loc=4) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) # Show all plt.show()
configs.data.size_template_names = vkitti.class_names configs.data.num_size_templates = len(configs.data.size_template_names) configs.data.class_name_to_size_template_id = { cat: cls for cls, cat in enumerate(configs.data.size_template_names) } configs.data.size_template_id_to_class_name = { v: k for k, v in configs.data.class_name_to_size_template_id.items() } configs.data.size_templates = np.zeros((configs.data.num_size_templates, 3)) for i in range(configs.data.num_size_templates): configs.data.size_templates[i, :] = vkitti.class_name_to_size_template[ configs.data.size_template_id_to_class_name[i]] configs.data.size_templates = torch.from_numpy(configs.data.size_templates.astype(np.float32)) # dataset configs configs.source_dataset = Config(FrustumVkitti) configs.source_dataset.root = 'data/vkitti/frustum/frustum_data' configs.source_dataset.num_points = 1024 configs.source_dataset.classes = configs.data.classes configs.source_dataset.num_heading_angle_bins = configs.data.num_heading_angle_bins configs.source_dataset.class_name_to_size_template_id = configs.data.class_name_to_size_template_id configs.source_dataset.random_flip = True configs.source_dataset.random_shift = True configs.source_dataset.frustum_rotate = True configs.source_dataset.from_rgb_detection = False configs.target_dataset = Config(FrustumKitti) configs.target_dataset.root = 'data/kitti/frustum/frustum_data' configs.target_dataset.num_points = 1024 configs.target_dataset.classes = configs.data.classes configs.target_dataset.num_heading_angle_bins = configs.data.num_heading_angle_bins
def compare_convergences_classif(dataset, list_of_paths, list_of_labels=None): # Parameters # ********** steps_per_epoch = 0 smooth_n = 2 if list_of_labels is None: list_of_labels = [str(i) for i in range(len(list_of_paths))] # Read Logs # ********* all_pred_epochs = [] all_val_OA = [] all_train_OA = [] all_vote_OA = [] all_vote_confs = [] for path in list_of_paths: # Load parameters config = Config() config.load(list_of_paths[0]) # Get the number of classes n_class = config.num_classes # Get validation confusions file = join(path, 'val_confs.txt') val_C1 = load_confusions(file, n_class) val_PRE, val_REC, val_F1, val_IoU, val_ACC = smooth_metrics( val_C1, smooth_n=smooth_n) # Get vote confusions file = join(path, 'vote_confs.txt') if exists(file): vote_C2 = load_confusions(file, n_class) vote_PRE, vote_REC, vote_F1, vote_IoU, vote_ACC = smooth_metrics( vote_C2, smooth_n=2) else: vote_C2 = val_C1 vote_PRE, vote_REC, vote_F1, vote_IoU, vote_ACC = (val_PRE, val_REC, val_F1, val_IoU, val_ACC) # Get training confusions balanced file = join(path, 'training_confs.txt') train_C = load_confusions(file, n_class) train_PRE, train_REC, train_F1, train_IoU, train_ACC = smooth_metrics( train_C, smooth_n=smooth_n) # Aggregate results all_pred_epochs += [np.array([i for i in range(len(val_ACC))])] all_val_OA += [val_ACC] all_vote_OA += [vote_ACC] all_train_OA += [train_ACC] all_vote_confs += [vote_C2] #all_mean_IoU_scores += [running_mean(np.mean(val_IoU[:, 1:], axis=1), smooth_n)] # Best scores # *********** for i, label in enumerate(list_of_labels): print('\n' + label + '\n' + '*' * len(label) + '\n') best_epoch = np.argmax(all_vote_OA[i]) print('Best Accuracy : {:.1f} % (epoch {:d})'.format( 100 * all_vote_OA[i][best_epoch], best_epoch)) confs = all_vote_confs[i] TP_plus_FN = np.sum(confs, axis=-1, keepdims=True) class_avg_confs = confs.astype(np.float32) / TP_plus_FN.astype( np.float32) diags = np.diagonal(class_avg_confs, axis1=-2, axis2=-1) class_avg_ACC = np.sum(diags, axis=-1) / np.sum(class_avg_confs, axis=(-1, -2)) print('Corresponding mAcc : {:.1f} %'.format( 100 * class_avg_ACC[best_epoch])) # Plots # ***** # Figure fig = plt.figure('Validation') for i, label in enumerate(list_of_labels): plt.plot(all_pred_epochs[i], all_val_OA[i], linewidth=1, label=label) plt.xlabel('epochs') plt.ylabel('Validation Accuracy') # Set limits for y axis #plt.ylim(0.55, 0.95) # Display legends and title plt.legend(loc=4) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) # Figure fig = plt.figure('Vote Validation') for i, label in enumerate(list_of_labels): plt.plot(all_pred_epochs[i], all_vote_OA[i], linewidth=1, label=label) plt.xlabel('epochs') plt.ylabel('Validation Accuracy') # Set limits for y axis #plt.ylim(0.55, 0.95) # Display legends and title plt.legend(loc=4) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) # Figure fig = plt.figure('Training') for i, label in enumerate(list_of_labels): plt.plot(all_pred_epochs[i], all_train_OA[i], linewidth=1, label=label) plt.xlabel('epochs') plt.ylabel('Overall Accuracy') # Set limits for y axis #plt.ylim(0.8, 1) # Display legends and title plt.legend(loc=4) # Customize the graph ax = fig.gca() ax.grid(linestyle='-.', which='both') #ax.set_yticks(np.arange(0.8, 1.02, 0.02)) #for i, label in enumerate(list_of_labels): # print(label, np.max(all_train_OA[i]), np.max(all_val_OA[i])) # Show all plt.show()
def run(self): return self.scan() def scan(self): # 捺印录入 self.page.execute(' window.location="/abisweb/tp/scan/"') time.sleep(3) tpimgUload(self.image).operation_dialog() self.page.execute(' $(".ImportImg").click()') time.sleep(2) self.page.execute(' $("#ScanObjectTree_1_span").dblclick()') self.page.find_element(*(By.ID, 'AutoSplit')).click() self.page.find_element(*(By.ID, 'ShowTxt')).click() self.page.find_element(*(By.ID, 'NAME')).send_keys(generator.random_name()) time.sleep(1) personNum = self.page.find_element( *(By.ID, 'PERSON_NUM')).get_attribute("value") cardNum = self.page.find_element(*(By.ID, 'CARD_NUM')).get_attribute("value") logger.info("personNum:" + personNum) logger.info("cardNum:" + cardNum) self.page.find_element(*(By.ID, 'Save')).click() return cardNum if __name__ == '__main__': URL = Config().get('URL') page = Login().run() TpScan(page).run() page.quit()
parser = argparse.ArgumentParser() parser.add_argument('--config', type=str, help='all sets of configuration parameters', default='../config/dann.yml') parser.add_argument('--dataset', default='Office-31', type=str, help='which dataset') parser.add_argument('--src_address', default=None, type=str, help='address of image list of source dataset') parser.add_argument('--tgt_address', default=None, type=str, help='address of image list of target dataset') parser.add_argument('--src_test_address', default=None, type=str, help='address of image list of source test dataset') args = parser.parse_args() cfg = Config(args.config) source_file = args.src_address target_file = args.tgt_address source_file_test = args.src_test_address if args.dataset == 'Office-31': class_num = 31 width = 1024 srcweight = 4 is_cen = False elif args.dataset == 'image-clef': class_num = 12 width = 1024 srcweight = 4 is_cen = False
# Automatically retrieve the last trained model if chosen_log in handled_logs: # Dataset name test_dataset = '_'.join(chosen_log.split('_')[1:]) # List all training logs logs = np.sort([ os.path.join('results', f) for f in os.listdir('results') if f.startswith('Log') ]) # Find the last log of asked dataset for log in logs[::-1]: log_config = Config() log_config.load(log) if log_config.dataset.startswith(test_dataset): chosen_log = log break if chosen_log in handled_logs: raise ValueError('No log of the dataset "' + test_dataset + '" found') # Check if log exists if not os.path.exists(chosen_log): raise ValueError('The given log does not exists: ' + chosen_log) # Let's go test_caller(chosen_log, chosen_snapshot, on_val)
import selenium from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from utils.config import Config url = Config().get('JKURL') class BasePage(object): def get_url(self): return url
def main(dataset_name, net_name, xp_path, data_path, load_config, load_model, eta, ratio_known_normal, ratio_known_outlier, ratio_pollution, device, seed, optimizer_name, lr, n_epochs, lr_milestone, batch_size, weight_decay, pretrain, ae_optimizer_name, ae_lr, ae_n_epochs, ae_lr_milestone, ae_batch_size, ae_weight_decay, num_threads, n_jobs_dataloader, normal_class, known_outlier_class, n_known_outlier_classes): """ Deep SAD, a method for deep semi-supervised anomaly detection. :arg DATASET_NAME: Name of the dataset to load. :arg NET_NAME: Name of the neural network to use. :arg XP_PATH: Export path for logging the experiment. :arg DATA_PATH: Root path of data. """ # Get configuration cfg = Config(locals().copy()) # Set up logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger() logger.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') log_file = xp_path + '/log.txt' file_handler = logging.FileHandler(log_file) file_handler.setLevel(logging.INFO) file_handler.setFormatter(formatter) logger.addHandler(file_handler) # Print paths logger.info('Log file is %s' % log_file) logger.info('Data path is %s' % data_path) logger.info('Export path is %s' % xp_path) # Print experimental setup logger.info('Dataset: %s' % dataset_name) logger.info('Normal class: %d' % normal_class) logger.info('Ratio of labeled normal train samples: %.2f' % ratio_known_normal) logger.info('Ratio of labeled anomalous samples: %.2f' % ratio_known_outlier) logger.info('Pollution ratio of unlabeled train data: %.2f' % ratio_pollution) if n_known_outlier_classes == 1: logger.info('Known anomaly class: %d' % known_outlier_class) else: logger.info('Number of known anomaly classes: %d' % n_known_outlier_classes) logger.info('Network: %s' % net_name) # If specified, load experiment config from JSON-file if load_config: cfg.load_config(import_json=load_config) logger.info('Loaded configuration from %s.' % load_config) # Print model configuration logger.info('Eta-parameter: %.2f' % cfg.settings['eta']) # Set seed if cfg.settings['seed'] != -1: random.seed(cfg.settings['seed']) np.random.seed(cfg.settings['seed']) torch.manual_seed(cfg.settings['seed']) torch.cuda.manual_seed(cfg.settings['seed']) torch.backends.cudnn.deterministic = True logger.info('Set seed to %d.' % cfg.settings['seed']) # Default device to 'cpu' if cuda is not available if not torch.cuda.is_available(): device = 'cpu' # Set the number of threads used for parallelizing CPU operations if num_threads > 0: torch.set_num_threads(num_threads) logger.info('Computation device: %s' % device) logger.info('Number of threads: %d' % num_threads) logger.info('Number of dataloader workers: %d' % n_jobs_dataloader) # Load data dataset = load_dataset(dataset_name, data_path, normal_class, known_outlier_class, n_known_outlier_classes, ratio_known_normal, ratio_known_outlier, ratio_pollution, random_state=np.random.RandomState( cfg.settings['seed'])) # Log random sample of known anomaly classes if more than 1 class if n_known_outlier_classes > 1: logger.info('Known anomaly classes: %s' % (dataset.known_outlier_classes, )) # Initialize DeepSAD model and set neural network phi deepSAD = DeepSAD(cfg.settings['eta']) deepSAD.set_network(net_name) # If specified, load Deep SAD model (center c, network weights, and possibly autoencoder weights) if load_model: deepSAD.load_model(model_path=load_model, load_ae=True, map_location=device) logger.info('Loading model from %s.' % load_model) logger.info('Pretraining: %s' % pretrain) if pretrain: # Log pretraining details logger.info('Pretraining optimizer: %s' % cfg.settings['ae_optimizer_name']) logger.info('Pretraining learning rate: %g' % cfg.settings['ae_lr']) logger.info('Pretraining epochs: %d' % cfg.settings['ae_n_epochs']) logger.info('Pretraining learning rate scheduler milestones: %s' % (cfg.settings['ae_lr_milestone'], )) logger.info('Pretraining batch size: %d' % cfg.settings['ae_batch_size']) logger.info('Pretraining weight decay: %g' % cfg.settings['ae_weight_decay']) # Pretrain model on dataset (via autoencoder) deepSAD.pretrain(dataset, optimizer_name=cfg.settings['ae_optimizer_name'], lr=cfg.settings['ae_lr'], n_epochs=cfg.settings['ae_n_epochs'], lr_milestones=cfg.settings['ae_lr_milestone'], batch_size=cfg.settings['ae_batch_size'], weight_decay=cfg.settings['ae_weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Save pretraining results deepSAD.save_ae_results(export_json=xp_path + '/ae_results.json') # Log training details logger.info('Training optimizer: %s' % cfg.settings['optimizer_name']) logger.info('Training learning rate: %g' % cfg.settings['lr']) logger.info('Training epochs: %d' % cfg.settings['n_epochs']) logger.info('Training learning rate scheduler milestones: %s' % (cfg.settings['lr_milestone'], )) logger.info('Training batch size: %d' % cfg.settings['batch_size']) logger.info('Training weight decay: %g' % cfg.settings['weight_decay']) # Train model on dataset deepSAD.train(dataset, optimizer_name=cfg.settings['optimizer_name'], lr=cfg.settings['lr'], n_epochs=cfg.settings['n_epochs'], lr_milestones=cfg.settings['lr_milestone'], batch_size=cfg.settings['batch_size'], weight_decay=cfg.settings['weight_decay'], device=device, n_jobs_dataloader=n_jobs_dataloader) # Test model deepSAD.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader) # Save results, model, and configuration deepSAD.save_results(export_json=xp_path + '/results.json') deepSAD.save_model(export_model=xp_path + '/model.tar') cfg.save_config(export_json=xp_path + '/config.json') # Plot most anomalous and most normal test samples indices, labels, scores = zip(*deepSAD.results['test_scores']) indices, labels, scores = np.array(indices), np.array(labels), np.array( scores) idx_all_sorted = indices[np.argsort( scores)] # from lowest to highest score idx_normal_sorted = indices[labels == 0][np.argsort( scores[labels == 0])] # from lowest to highest score if dataset_name in ('mnist', 'fmnist', 'cifar10'): if dataset_name in ('mnist', 'fmnist'): X_all_low = dataset.test_set.data[idx_all_sorted[:32], ...].unsqueeze(1) X_all_high = dataset.test_set.data[idx_all_sorted[-32:], ...].unsqueeze(1) X_normal_low = dataset.test_set.data[idx_normal_sorted[:32], ...].unsqueeze(1) X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:], ...].unsqueeze(1) if dataset_name == 'cifar10': X_all_low = torch.tensor( np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...], (0, 3, 1, 2))) X_all_high = torch.tensor( np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...], (0, 3, 1, 2))) X_normal_low = torch.tensor( np.transpose( dataset.test_set.data[idx_normal_sorted[:32], ...], (0, 3, 1, 2))) X_normal_high = torch.tensor( np.transpose( dataset.test_set.data[idx_normal_sorted[-32:], ...], (0, 3, 1, 2))) plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2) plot_images_grid(X_all_high, export_img=xp_path + '/all_high', padding=2) plot_images_grid(X_normal_low, export_img=xp_path + '/normals_low', padding=2) plot_images_grid(X_normal_high, export_img=xp_path + '/normals_high', padding=2)
import torch.optim as optim from models.frustum_net import FrustumPointDanSimple from utils.config import Config, configs # model configs.model = Config(FrustumPointDanSimple) configs.model.num_classes = configs.data.num_classes configs.model.num_heading_angle_bins = configs.data.num_heading_angle_bins configs.model.num_size_templates = configs.data.num_size_templates configs.model.num_points_per_object = configs.data.num_points_per_object configs.model.size_templates = configs.data.size_templates configs.model.extra_feature_channels = 1 # train: scheduler configs.train.scheduler_g = Config(optim.lr_scheduler.CosineAnnealingLR) configs.train.scheduler_g.T_max = configs.train.num_epochs configs.train.scheduler_c = Config(optim.lr_scheduler.CosineAnnealingLR) configs.train.scheduler_c.T_max = configs.train.num_epochs
def __init__(self): self.__c = Config()
# %% from utils.config import Config from utils.data import Data import pandas as pd # %% c = Config() d = Data() # %% colnames = d.getDescriptorsColumnNames() filename = '1l_atomicPLMF_6138structures.csv' # %% desc_6k_df = pd.read_csv(c.get_datapath(filename), index_col='Monolayer') # %% desc_6k_df_filtered = desc_6k_df[colnames] # %% desc_6k_df_filtered.to_csv(c.get_descriptorspath('descriptors_master_6k.csv')) # %%
def __init__(self): self.log = logging.getLogger(__name__) self.config = Config() self.database = Database(self.config) self.init_args()
def main(): '''See _arr_slicing_speed.ipynb for better tests''' global logger1 import argparse import json parser = argparse.ArgumentParser() parser.add_argument('config_path') args = parser.parse_args() with open(args.config_path, 'r') as f: cfg = json.load(f) config = Config(**cfg) ########## # Variables t0 = time.time() percentile = 85 dx, dy = 4, 4 # nearest neighbors patch size min_z = 128 # min z value to consider a hit N_plots = 20 figsize = (12, 12) node_color, edge_color = 'OrangeRed', '#ff571a' node_color, edge_color = '#00e699', '#1affb2' # aquamarine node_color, edge_color = 'turquoise', '#65e6d9' # turqoise node_color, edge_color = 'deepskyblue', '#33ccff' node_color, edge_color = '#e68a00', '#ffa31a' # orange node_color, edge_color = '#33ff99', 'SpringGreen' node_color, edge_color = 'DarkViolet', 'BlueViolet' # '#b300ff' node_color, edge_color = '#a446d2', 'BlueViolet' # node_color='#0086CC' # cosmiq logo color (blue) # edge_color='#00a6ff', # node_color = '#33cccc' #turquise # edge_color = '#47d1d1' # node_color, edge_color = '#ffd100', '#e09900', # orange gold # node_color = '#29a329' # green # edge_color = '#33cc33' # node_color='#66ccff' # blue # edge_color='#999999' # best colors node_color, edge_color = '#cc9900', '#ffbf00' # gold # node_color, edge_color = 'l#4dff4d', '#00e600' # green default_node_size = 2 # 0.15 #4 plot_width_key, plot_width_mult = 'inferred_speed_mph', 0.085 # 0.08 # variable width # width_key, width_mult = 4, 1 # constant width if config.num_classes >7: use_totband = True else: use_totband = False save_shapefiles = True use_weighted_mean = True variable_edge_speed = False run_08a_plot_graph_plus_im = False verbose = False ########## # input dirs res_root_dir = os.path.join(config.path_results_root, config.test_results_dir) # path_images = os.path.join(config.path_data_root, config.test_data_refined_dir) graph_dir = os.path.join(res_root_dir, config.graph_dir) # get mask location, check if we are stitching together large images or not folds_dir = os.path.join(config.path_results_root, config.test_results_dir, config.folds_save_dir) mask_prefix = '' mask_dir = folds_dir # if os.path.exists(out_dir_mask_norm): # mask_dir = out_dir_mask_norm # else: # mask_dir = merge_dir log_file = os.path.join(res_root_dir, 'skeleton_speed.log') console, logger1 = make_logger.make_logger(log_file, logger_name='log') # ############################################################################### # # https://docs.python.org/3/howto/logging-cookbook.html#logging-to-multiple-destinations # # set up logging to file - see previous section for more details # logging.basicConfig(level=logging.DEBUG, # format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', # datefmt='%m-%d %H:%M', # filename=log_file, # filemode='w') # # define a Handler which writes INFO messages or higher to the sys.stderr # console = logging.StreamHandler() # console.setLevel(logging.INFO) # # set a format which is simpler for console use # formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') # #formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') # # tell the handler to use this format # console.setFormatter(formatter) # # add the handler to the root logger # logging.getLogger('').addHandler(console) # logger1 = logging.getLogger('log') # logger1.info("log file: {x}".format(x=log_file)) # ############################################################################### # output dirs graph_speed_dir = os.path.join(res_root_dir, config.graph_dir + '_speed') os.makedirs(graph_speed_dir, exist_ok=True) logger1.info("graph_speed_dir: " + graph_speed_dir) speed_conversion_file_binned = 'speed_conversion_binned10.csv' # load conversion file # get the conversion diction between pixel mask values and road speed (mph) conv_df, conv_dict \ = load_speed_conversion_dict_binned(speed_conversion_file_binned) logger1.info("speed conv_dict: " + str(conv_dict)) # Add travel time to entire dir add_travel_time_dir(graph_dir, mask_dir, conv_dict, graph_speed_dir, min_z=min_z, dx=dx, dy=dy, percentile=percentile, use_totband=use_totband, use_weighted_mean=use_weighted_mean, variable_edge_speed=variable_edge_speed, mask_prefix=mask_prefix, save_shapefiles=save_shapefiles, verbose=verbose) t1 = time.time() logger1.info("Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0)) # plot a few if N_plots > 0: logger1.info("\nPlot a few...") ## import apls_tools (or just copy plot_graph_on_in() func he) # local = False ## local # if local: # apls_dir = '/raid/cosmiq/apls/apls/src' ## dev box # else: # apls_dir = '/raid/local/src/apls/apls/src' # sys.path.append(apls_dir) # import apls_tools # define output dir graph_speed_plots_dir = os.path.join(res_root_dir, config.graph_dir + '_speed_plots') os.makedirs(graph_speed_plots_dir, exist_ok=True) # plot graph on image (with width proportional to speed) path_images = os.path.join(config.path_data_root, config.test_data_refined_dir) image_list = [z for z in os.listdir(path_images) if z.endswith('tif')] if len(image_list) > N_plots: image_names = np.random.choice(image_list, N_plots) else: image_names = sorted(image_list) # logger1.info("image_names: " + image_names) for i, image_name in enumerate(image_names): if i > 1000: break image_path = os.path.join(path_images, image_name) logger1.info("\n\nPlotting: " + image_name + " " + image_path) pkl_path = os.path.join(graph_speed_dir, image_name.split('.')[0] + '.gpickle') logger1.info(" pkl_path: " + pkl_path) if not os.path.exists(pkl_path): logger1.info(" missing pkl: " + pkl_path) continue G = nx.read_gpickle(pkl_path) # if not os.path.exists(image_path) figname = os.path.join(graph_speed_plots_dir, image_name) _ = plot_graph_on_im_yuge(G, image_path, figsize=figsize, show_endnodes=True, default_node_size=default_node_size, width_key=plot_width_key, width_mult=plot_width_mult, node_color=node_color, edge_color=edge_color, title=image_name, figname=figname, verbose=True, super_verbose=verbose) t2 = time.time() logger1.info("Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0)) logger1.info("Time to make plots: {x} seconds".format(x=t2 - t1)) logger1.info("Total time: {x} seconds".format(x=t2 - t0))