def run(self): #训练模型
# 1. 模型训练
self.option.signalInfo.emit(-1,"开始导入数据...")
dataloader = load_data(self.option)
model = Ganomaly(self.option, dataloader)
self.option.signalInfo.emit(10, "导入数据完毕！")
self.modelINFO = model.train()
# 2. 对训练结果进行处理，生成一个字典
self.option.signalInfo.emit(100,"")
self.modelINFO['opt'] = vars(self.option)
self.modelINFO['opt'].pop('signalInfo')
signal = self.modelINFO['opt'].pop('signal')
self.modelINFO['modelName'] = self.modelINFO['opt'].pop('dataset')
self.modelINFO['raw_path'] = self.modelINFO['opt'].pop('dataroot')
self.modelINFO['desc'] = self.modelINFO['opt'].pop('desc')
# 3. 将训练结果字典保存到json文件中
## 默认保存路径./output/modelsData/models.json
# filename = './output/modelsData/models.json'
# data = {}
# with open(filename,'r',encoding='utf-8') as f:
# try:
# data = json.load(f)
# except json.decoder.JSONDecodeError: # 此处源文件没有数据,即尚未有模型被训练
# data = {}
# with open(filename, 'w', encoding="utf-8") as f:
# data[self.modelINFO['modelName']] = self.modelINFO
# json.dump(data,f,sort_keys=True,indent=2)
# 3. 将训练结果字典通过信号传递给主函数
signal.emit(copy.deepcopy(self.modelINFO))
def FinalTest():
dataset = 'cus_mnist'
# dataroot = './data/cus_mnist'
opt = Options().parse(dataset)
opt.isTrain = False
opt.load_weights = True
##
# LOAD DATA
dataloader = load_data(opt)
print(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
##
# TRAIN MODEL
# model.train()
minVal = None
maxVal = None
threshold = None
with open(opt.dataroot + '/performance.txt', 'r+', encoding='utf-8') as f:
res = f.readline()
res = res.split('&')
res = [float(i) for i in res]
minVal = res[0]
maxVal = res[1]
threshold = res[2]
model.FinalTest(minVal, maxVal, threshold)
def run(self):
# self.option.isize = 128
self.dataloader = load_data(self.option)
model = Ganomaly(self.option, self.dataloader)
minVal = self.modelData[0]
maxVal = self.modelData[1]
threshold = self.modelData[2]
resNor, resAbn = model.FinalTest(minVal, maxVal, threshold)
opt = {}
opt['path'] = self.option.dataroot
opt['modelName'] = self.option.dataset
self.option.signal.emit(copy.deepcopy(resNor), copy.deepcopy(resAbn), copy.deepcopy(opt))
def train():
""" Training
"""
dataset = 'cus_mnist'
dataroot = 'E:/ProjectSet/Pycharm/WAIBAO/Code01/GAN/data/cus_mnist'
opt = Options().parse(dataset)
opt.load_weights = False
dataloader = load_data(opt)
print(opt)
# LOAD MODEL
model = Ganomaly(opt, dataloader)
model.train()
def test():
##
# ARGUMENTS
opt = Options().parse()
##
# LOAD DATA
dataloader = load_data(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
##
# test model
performance = model.test()
print(performance)
def train():
""" Training
"""
##
# ARGUMENTS
opt = Options().parse()
##
# LOAD DATA
dataloader = load_data(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
##
# TRAIN MODEL
model.train()
def test():
""" Testing
"""
dataset = 'cus_mnist'
#dataroot = './data/cus_mnist'
opt = Options().parse(dataset)
opt.isTrain = False
opt.load_weights = True
##
# LOAD DATA
dataloader = load_data(opt)
print(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
print(model.test())
def train():
""" Training
"""
##
# ARGUMENTS
opt = Options().parse()
##
# LOAD DATA
dataloader = load_data(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
##
# MODEL TEST
res = model.test()
model.z_train()
def train(self):
""" Training
"""
self.textEdit.append('开始训练...')
dataset = 'cus_mnist_2'
# dataroot = './data/cus_mnist'
dataroot = 'E:\ProjectSet\Pycharm\WAIBAO\cus_mnist2'
opt = Options().parse(dataset,dataroot)
opt.signal = self.Signal_TrainFinished
opt.load_weights = False
dataloader = load_data(opt)
print(opt)
# LOAD MODEL
opt.showProcess = self.progressBar
opt.showText = self.textEdit
model = Ganomaly(opt, dataloader)
model.train()
def test():
""" Training
"""
##
# ARGUMENTS
opt = Options().parse()
##
# LOAD DATA
dataloader = load_data(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
##
# MODEL TEST
res = model.test()
print('AUC:%f\n' % res['AUC'])
def train_dataloader(self, epoch: Optional[int] = None) -> DataLoader:
epoch = epoch or self.current_epoch
if "SYNTHETIC3" in self.conf.dataset_name and self.conf.model not in [ModelName.GNN, ModelName.RPGNN]:
self.train_ds, _, _, id2graphlet_list = load_data(self.conf, epoch)
self.train_id2graphlet = id2graphlet_list[0]
seed_everything(self.conf.seed + epoch)
np.random.seed(self.conf.seed + epoch)
torch.manual_seed(self.conf.seed + epoch)
common_file = None
if self.conf.only_common and self.conf.dataset_name != "DD":
common_file = self.conf.data_path_complete / "common.txt"
assert common_file.exists()
return DataLoader(self.train_ds, batch_size=self.conf.batch_size, shuffle=True,
collate_fn=Batcher(self.train_id2graphlet,
common_file=common_file),
num_workers=0, pin_memory=True)
def train():
""" Training
"""
##
# ARGUMENTS
opt = Options().parse()
##
# LOAD DATA
dataloader = load_data(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
##
if opt.phase == 'train':
# TRAIN MODEL
model.train()
elif opt.phase =='test':
performance=model.test()
print(performance)
def test(self):
self.textEdit.append('开始测试，得出阈值...')
""" Testing
"""
dataset = 'cus_mnist'
# dataroot = './data/cus_mnist'
opt = Options().parse(dataset)
opt.isTrain = False
opt.load_weights = True
##
# LOAD DATA
dataloader = load_data(opt)
print(opt)
##
# LOAD MODEL
opt.showProcess = self.progressBar
opt.showText = self.textEdit
model = Ganomaly(opt, dataloader)
print(model.test())
def FinalTest(self):
self.Dialog01.textEdit.append('正在检测所有图片..')
dataset = 'cus_mnist'
dataroot = './data/cus_mnist'
opt = Options().parse(dataset,dataroot)
opt.isTrain = False
opt.load_weights = True
opt.signal = self.Signal_FinalTestFinished
##
# LOAD DATA
dataloader = load_data(opt)
print(opt)
##
# LOAD MODEL
#opt.showProcess = self.progressBar
#opt.showText = self.textEdit
opt.showProcess = self.Dialog01.progressBar
opt.showText = self.Dialog01.textEdit
self.TESTCLASS = MyTest("测试模型", opt, dataloader)
self.TESTCLASS.start()
print('GOOD')
def train():
""" Training
"""
##
# ARGUMENTS
opt = Options().parse()
##
# LOAD DATA
dataloader = load_data(opt)
##
# LOAD MODEL
model = Ganomaly(opt, dataloader)
##
# TRAIN MODEL
model.train()
train_1 = model.train_final()
train_1 = train_1.cpu().numpy()
test_1, y_true, y_true_original, auroc_value, auprc_value = model.test_final(
)
test_1 = test_1.cpu().numpy()
y_true = y_true.cpu().numpy()
y_true_original = y_true_original.cpu().numpy()
test_path = os.path.join(opt.outf, opt.dataset, 'test', 'OCSVM',
'abnormal' + str(opt.abnormal_class),
'seed' + str(opt.manualseed))
if not os.path.isdir(test_path):
os.makedirs(test_path)
print("GANomaly AUROC: {}".format(auroc_value))
np.save(test_path + '/ganomaly_aucroc.npy', auroc_value)
for i in range(len(y_true)):
if y_true[i] == 1:
y_true[i] = 0
else:
y_true[i] = 1
################################
cf = svm.OneClassSVM(gamma='scale', nu=0.1)
train_ind = np.random.choice(train_1.shape[0], 10000, replace=False)
cf.fit(train_1[train_ind, :])
y_scores = cf.score_samples(test_1)
y_scores = (y_scores - min(y_scores)) / (max(y_scores) - min(y_scores))
auroc = metrics.roc_auc_score(y_true, y_scores)
print("HybridGAN AUROC: {}".format(auroc))
np.save(test_path + '/svm_aucroc1.npy', auroc)
np.save(test_path + '/svm_aucroc1_transduct_' + str(0) + '.npy', auroc)
bandwidth = get_bandwidth(y_scores, test_1)
for trans_iter in np.arange(0, 30, 1):
optimal_threshold = find_optimal_threshold(y_scores=y_scores,
train_1=train_1,
test_1=test_1,
y_true=y_true,
train_ind=train_ind,
test_path=test_path,
bandwidth=bandwidth)
abn_idx = np.where(
y_scores < np.percentile(y_scores, optimal_threshold))
abn_tst_latent = test_1[abn_idx]
kmeans = KMeans(n_clusters=1, random_state=0).fit(abn_tst_latent)
train_1 = np.concatenate((train_1, kmeans.transform(train_1)), axis=1)
test_1 = np.concatenate((test_1, kmeans.transform(test_1)), axis=1)
cf = svm.OneClassSVM(gamma='scale', nu=0.1)
cf.fit(train_1[train_ind, :])
y_scores = cf.score_samples(test_1)
y_scores = (y_scores - min(y_scores)) / (max(y_scores) - min(y_scores))
auroc = metrics.roc_auc_score(y_true, y_scores)
print("TransdeepOCSVM AUROC after {} iterations: {}".format(
trans_iter + 1, auroc))
print("Optimal_threshold after {} iterations: {}".format(
trans_iter + 1, optimal_threshold[0]))
np.save(
test_path + '/svm_aucroc1_transduct_' + str(trans_iter + 1) +
'.npy', auroc)
np.save(
test_path + '/optimal_threshold_' + str(trans_iter + 1) + '.npy',
optimal_threshold)
help='path to latest checkpoint (default: none)')
parser.add_argument('--cuda',
default=False,
action='store_true',
help='enable cuda')
parser.add_argument('--colab',
default=False,
action='store_true',
help='enable colab hosted runtime')
args = parser.parse_args()
device = torch.device('cuda' if args.cuda else 'cpu')
try:
from lib import data
train_data = data.load_data(year=args.year)
except ModuleNotFoundError:
# workaround that Ta-lib cannot be installed on Colab
train_data = (pd.read_csv('data/000001_prices_%d.csv' % args.year,
index_col=0),
pd.read_csv('data/000001_factors_%d.csv' % args.year,
index_col=0))
env = environ.StockEnv(train_data,
bars_count=BARS_COUNT,
commission=0.0,
reset_on_sell=False)
# env = gym.wrappers.TimeLimit(env, max_episode_steps=1000)
net = models.A2CConv1d(env.observation_space.shape,
env.action_space.n).to(device)
def __init__(self, conf: Config, loss: Loss, **kwargs):
super().__init__(**kwargs)
self.loss = loss
self.save_hyperparameters(OmegaConf.create(asdict(conf)))
self.conf = conf
self.train_ds, self.val_ds_list, self.test_ds_list, self.id2graphlet_list = load_data(self.conf)
if conf.dataset_name == "brain-net":
if self.conf.num_splits > 1:
self.train_ds, val_ds = stratified_ksplit(
self.train_ds, self.conf.num_splits, self.conf.split
)
self.val_ds_list = [val_ds]
self.test_ds_list = []
else:
self.val_ds_list = [self.train_ds]
if self.conf.model in [ModelName.RPGNN, ModelName.GNN]:
self.train_id2graphlet = None
self.id2graphlet_list = None
else:
self.train_id2graphlet = self.id2graphlet_list[0]
seed_everything(self.conf.seed)
np.random.seed(self.conf.seed)
torch.manual_seed(self.conf.seed)
model = self.build_model(conf, (self.train_ds, self.train_id2graphlet))
if not (conf.model == ModelName.GraphletCounting and conf.num_layers == 1):
h_dim, act = None, None
if conf.classifier_num_hidden > 0:
h_dim = conf.classifier_h_dim
act = nn.ReLU
batch_norm = None
if conf.batch_norm.presence and conf.model not in [ModelName.GNN, ModelName.RPGNN]:
batch_norm = nn.BatchNorm1d(model.out_dim, affine=conf.batch_norm.affine)
if conf.model is ModelName.GraphletCounting:
assert conf.classifier_num_hidden == conf.num_layers - 2
model = FinalLayers(
model,
num_out=conf.num_out,
h_dim=h_dim,
act=act,
n_hidden_layers=conf.classifier_num_hidden,
batch_norm=batch_norm,
dropout=conf.classifier_dropout,
)
self.model = model
loss.on_epoch_start(epoch=0, model=model)
with_mcc = conf.dataset_name in ["NCI1", "NCI109", "PROTEINS", "DD"]
self.train_acc = MatthewsCoef(num_classes=conf.num_out) if with_mcc else pl.metrics.Accuracy()
self.val_acc = MatthewsCoef(num_classes=conf.num_out) if with_mcc else pl.metrics.Accuracy()
self.test_acc = MatthewsCoef(num_classes=conf.num_out) if with_mcc else pl.metrics.Accuracy()
## # LIBRARIEStai from __future__ import print_function from option1 import Options from lib.data import load_data from ocgan.model import ocgan ## # def main(): """ Training """ ## # ARGUMENTS opt = Options().parse() print(1) ## # LOAD DATA dataloader = load_data(opt) print(2) ## # LOAD MODEL model = ocgan(opt, dataloader) print(3) ## # TRAIN MODEL model.train() print(4)
