def run(self): #训练模型 # 1. 模型训练 self.option.signalInfo.emit(-1,"开始导入数据...") dataloader = load_data(self.option) model = Ganomaly(self.option, dataloader) self.option.signalInfo.emit(10, "导入数据完毕!") self.modelINFO = model.train() # 2. 对训练结果进行处理,生成一个字典 self.option.signalInfo.emit(100,"") self.modelINFO['opt'] = vars(self.option) self.modelINFO['opt'].pop('signalInfo') signal = self.modelINFO['opt'].pop('signal') self.modelINFO['modelName'] = self.modelINFO['opt'].pop('dataset') self.modelINFO['raw_path'] = self.modelINFO['opt'].pop('dataroot') self.modelINFO['desc'] = self.modelINFO['opt'].pop('desc') # 3. 将训练结果字典保存到json文件中 ## 默认保存路径./output/modelsData/models.json # filename = './output/modelsData/models.json' # data = {} # with open(filename,'r',encoding='utf-8') as f: # try: # data = json.load(f) # except json.decoder.JSONDecodeError: # 此处源文件没有数据,即尚未有模型被训练 # data = {} # with open(filename, 'w', encoding="utf-8") as f: # data[self.modelINFO['modelName']] = self.modelINFO # json.dump(data,f,sort_keys=True,indent=2) # 3. 将训练结果字典通过信号传递给主函数 signal.emit(copy.deepcopy(self.modelINFO))
def FinalTest(): dataset = 'cus_mnist' # dataroot = './data/cus_mnist' opt = Options().parse(dataset) opt.isTrain = False opt.load_weights = True ## # LOAD DATA dataloader = load_data(opt) print(opt) ## # LOAD MODEL model = Ganomaly(opt, dataloader) ## # TRAIN MODEL # model.train() minVal = None maxVal = None threshold = None with open(opt.dataroot + '/performance.txt', 'r+', encoding='utf-8') as f: res = f.readline() res = res.split('&') res = [float(i) for i in res] minVal = res[0] maxVal = res[1] threshold = res[2] model.FinalTest(minVal, maxVal, threshold)
def run(self): # self.option.isize = 128 self.dataloader = load_data(self.option) model = Ganomaly(self.option, self.dataloader) minVal = self.modelData[0] maxVal = self.modelData[1] threshold = self.modelData[2] resNor, resAbn = model.FinalTest(minVal, maxVal, threshold) opt = {} opt['path'] = self.option.dataroot opt['modelName'] = self.option.dataset self.option.signal.emit(copy.deepcopy(resNor), copy.deepcopy(resAbn), copy.deepcopy(opt))
def train(): """ Training """ dataset = 'cus_mnist' dataroot = 'E:/ProjectSet/Pycharm/WAIBAO/Code01/GAN/data/cus_mnist' opt = Options().parse(dataset) opt.load_weights = False dataloader = load_data(opt) print(opt) # LOAD MODEL model = Ganomaly(opt, dataloader) model.train()
def test(): ## # ARGUMENTS opt = Options().parse() ## # LOAD DATA dataloader = load_data(opt) ## # LOAD MODEL model = Ganomaly(opt, dataloader) ## # test model performance = model.test() print(performance)
def train(): """ Training """ ## # ARGUMENTS opt = Options().parse() ## # LOAD DATA dataloader = load_data(opt) ## # LOAD MODEL model = Ganomaly(opt, dataloader) ## # TRAIN MODEL model.train()
def test(): """ Testing """ dataset = 'cus_mnist' #dataroot = './data/cus_mnist' opt = Options().parse(dataset) opt.isTrain = False opt.load_weights = True ## # LOAD DATA dataloader = load_data(opt) print(opt) ## # LOAD MODEL model = Ganomaly(opt, dataloader) print(model.test())
def train(): """ Training """ ## # ARGUMENTS opt = Options().parse() ## # LOAD DATA dataloader = load_data(opt) ## # LOAD MODEL model = Ganomaly(opt, dataloader) ## # MODEL TEST res = model.test() model.z_train()
def train(self): """ Training """ self.textEdit.append('开始训练...') dataset = 'cus_mnist_2' # dataroot = './data/cus_mnist' dataroot = 'E:\ProjectSet\Pycharm\WAIBAO\cus_mnist2' opt = Options().parse(dataset,dataroot) opt.signal = self.Signal_TrainFinished opt.load_weights = False dataloader = load_data(opt) print(opt) # LOAD MODEL opt.showProcess = self.progressBar opt.showText = self.textEdit model = Ganomaly(opt, dataloader) model.train()
def test(): """ Training """ ## # ARGUMENTS opt = Options().parse() ## # LOAD DATA dataloader = load_data(opt) ## # LOAD MODEL model = Ganomaly(opt, dataloader) ## # MODEL TEST res = model.test() print('AUC:%f\n' % res['AUC'])
def train_dataloader(self, epoch: Optional[int] = None) -> DataLoader: epoch = epoch or self.current_epoch if "SYNTHETIC3" in self.conf.dataset_name and self.conf.model not in [ModelName.GNN, ModelName.RPGNN]: self.train_ds, _, _, id2graphlet_list = load_data(self.conf, epoch) self.train_id2graphlet = id2graphlet_list[0] seed_everything(self.conf.seed + epoch) np.random.seed(self.conf.seed + epoch) torch.manual_seed(self.conf.seed + epoch) common_file = None if self.conf.only_common and self.conf.dataset_name != "DD": common_file = self.conf.data_path_complete / "common.txt" assert common_file.exists() return DataLoader(self.train_ds, batch_size=self.conf.batch_size, shuffle=True, collate_fn=Batcher(self.train_id2graphlet, common_file=common_file), num_workers=0, pin_memory=True)
def train(): """ Training """ ## # ARGUMENTS opt = Options().parse() ## # LOAD DATA dataloader = load_data(opt) ## # LOAD MODEL model = Ganomaly(opt, dataloader) ## if opt.phase == 'train': # TRAIN MODEL model.train() elif opt.phase =='test': performance=model.test() print(performance)
def test(self): self.textEdit.append('开始测试,得出阈值...') """ Testing """ dataset = 'cus_mnist' # dataroot = './data/cus_mnist' opt = Options().parse(dataset) opt.isTrain = False opt.load_weights = True ## # LOAD DATA dataloader = load_data(opt) print(opt) ## # LOAD MODEL opt.showProcess = self.progressBar opt.showText = self.textEdit model = Ganomaly(opt, dataloader) print(model.test())
def FinalTest(self): self.Dialog01.textEdit.append('正在检测所有图片..') dataset = 'cus_mnist' dataroot = './data/cus_mnist' opt = Options().parse(dataset,dataroot) opt.isTrain = False opt.load_weights = True opt.signal = self.Signal_FinalTestFinished ## # LOAD DATA dataloader = load_data(opt) print(opt) ## # LOAD MODEL #opt.showProcess = self.progressBar #opt.showText = self.textEdit opt.showProcess = self.Dialog01.progressBar opt.showText = self.Dialog01.textEdit self.TESTCLASS = MyTest("测试模型", opt, dataloader) self.TESTCLASS.start() print('GOOD')
def train(): """ Training """ ## # ARGUMENTS opt = Options().parse() ## # LOAD DATA dataloader = load_data(opt) ## # LOAD MODEL model = Ganomaly(opt, dataloader) ## # TRAIN MODEL model.train() train_1 = model.train_final() train_1 = train_1.cpu().numpy() test_1, y_true, y_true_original, auroc_value, auprc_value = model.test_final( ) test_1 = test_1.cpu().numpy() y_true = y_true.cpu().numpy() y_true_original = y_true_original.cpu().numpy() test_path = os.path.join(opt.outf, opt.dataset, 'test', 'OCSVM', 'abnormal' + str(opt.abnormal_class), 'seed' + str(opt.manualseed)) if not os.path.isdir(test_path): os.makedirs(test_path) print("GANomaly AUROC: {}".format(auroc_value)) np.save(test_path + '/ganomaly_aucroc.npy', auroc_value) for i in range(len(y_true)): if y_true[i] == 1: y_true[i] = 0 else: y_true[i] = 1 ################################ cf = svm.OneClassSVM(gamma='scale', nu=0.1) train_ind = np.random.choice(train_1.shape[0], 10000, replace=False) cf.fit(train_1[train_ind, :]) y_scores = cf.score_samples(test_1) y_scores = (y_scores - min(y_scores)) / (max(y_scores) - min(y_scores)) auroc = metrics.roc_auc_score(y_true, y_scores) print("HybridGAN AUROC: {}".format(auroc)) np.save(test_path + '/svm_aucroc1.npy', auroc) np.save(test_path + '/svm_aucroc1_transduct_' + str(0) + '.npy', auroc) bandwidth = get_bandwidth(y_scores, test_1) for trans_iter in np.arange(0, 30, 1): optimal_threshold = find_optimal_threshold(y_scores=y_scores, train_1=train_1, test_1=test_1, y_true=y_true, train_ind=train_ind, test_path=test_path, bandwidth=bandwidth) abn_idx = np.where( y_scores < np.percentile(y_scores, optimal_threshold)) abn_tst_latent = test_1[abn_idx] kmeans = KMeans(n_clusters=1, random_state=0).fit(abn_tst_latent) train_1 = np.concatenate((train_1, kmeans.transform(train_1)), axis=1) test_1 = np.concatenate((test_1, kmeans.transform(test_1)), axis=1) cf = svm.OneClassSVM(gamma='scale', nu=0.1) cf.fit(train_1[train_ind, :]) y_scores = cf.score_samples(test_1) y_scores = (y_scores - min(y_scores)) / (max(y_scores) - min(y_scores)) auroc = metrics.roc_auc_score(y_true, y_scores) print("TransdeepOCSVM AUROC after {} iterations: {}".format( trans_iter + 1, auroc)) print("Optimal_threshold after {} iterations: {}".format( trans_iter + 1, optimal_threshold[0])) np.save( test_path + '/svm_aucroc1_transduct_' + str(trans_iter + 1) + '.npy', auroc) np.save( test_path + '/optimal_threshold_' + str(trans_iter + 1) + '.npy', optimal_threshold)
help='path to latest checkpoint (default: none)') parser.add_argument('--cuda', default=False, action='store_true', help='enable cuda') parser.add_argument('--colab', default=False, action='store_true', help='enable colab hosted runtime') args = parser.parse_args() device = torch.device('cuda' if args.cuda else 'cpu') try: from lib import data train_data = data.load_data(year=args.year) except ModuleNotFoundError: # workaround that Ta-lib cannot be installed on Colab train_data = (pd.read_csv('data/000001_prices_%d.csv' % args.year, index_col=0), pd.read_csv('data/000001_factors_%d.csv' % args.year, index_col=0)) env = environ.StockEnv(train_data, bars_count=BARS_COUNT, commission=0.0, reset_on_sell=False) # env = gym.wrappers.TimeLimit(env, max_episode_steps=1000) net = models.A2CConv1d(env.observation_space.shape, env.action_space.n).to(device)
def __init__(self, conf: Config, loss: Loss, **kwargs): super().__init__(**kwargs) self.loss = loss self.save_hyperparameters(OmegaConf.create(asdict(conf))) self.conf = conf self.train_ds, self.val_ds_list, self.test_ds_list, self.id2graphlet_list = load_data(self.conf) if conf.dataset_name == "brain-net": if self.conf.num_splits > 1: self.train_ds, val_ds = stratified_ksplit( self.train_ds, self.conf.num_splits, self.conf.split ) self.val_ds_list = [val_ds] self.test_ds_list = [] else: self.val_ds_list = [self.train_ds] if self.conf.model in [ModelName.RPGNN, ModelName.GNN]: self.train_id2graphlet = None self.id2graphlet_list = None else: self.train_id2graphlet = self.id2graphlet_list[0] seed_everything(self.conf.seed) np.random.seed(self.conf.seed) torch.manual_seed(self.conf.seed) model = self.build_model(conf, (self.train_ds, self.train_id2graphlet)) if not (conf.model == ModelName.GraphletCounting and conf.num_layers == 1): h_dim, act = None, None if conf.classifier_num_hidden > 0: h_dim = conf.classifier_h_dim act = nn.ReLU batch_norm = None if conf.batch_norm.presence and conf.model not in [ModelName.GNN, ModelName.RPGNN]: batch_norm = nn.BatchNorm1d(model.out_dim, affine=conf.batch_norm.affine) if conf.model is ModelName.GraphletCounting: assert conf.classifier_num_hidden == conf.num_layers - 2 model = FinalLayers( model, num_out=conf.num_out, h_dim=h_dim, act=act, n_hidden_layers=conf.classifier_num_hidden, batch_norm=batch_norm, dropout=conf.classifier_dropout, ) self.model = model loss.on_epoch_start(epoch=0, model=model) with_mcc = conf.dataset_name in ["NCI1", "NCI109", "PROTEINS", "DD"] self.train_acc = MatthewsCoef(num_classes=conf.num_out) if with_mcc else pl.metrics.Accuracy() self.val_acc = MatthewsCoef(num_classes=conf.num_out) if with_mcc else pl.metrics.Accuracy() self.test_acc = MatthewsCoef(num_classes=conf.num_out) if with_mcc else pl.metrics.Accuracy()
## # LIBRARIEStai from __future__ import print_function from option1 import Options from lib.data import load_data from ocgan.model import ocgan ## # def main(): """ Training """ ## # ARGUMENTS opt = Options().parse() print(1) ## # LOAD DATA dataloader = load_data(opt) print(2) ## # LOAD MODEL model = ocgan(opt, dataloader) print(3) ## # TRAIN MODEL model.train() print(4)