def toplayer(args): from dataset import transform from data_process import name2index name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} utils.mkdirs(config.sub_dir) model = models.myecgnet() model.load_state_dict( torch.load(args.ckpt, map_location='cpu')['state_dict']) model = model.to(device) model.eval() sub_file = '%s.txt' % args.ex fout = open(sub_file, 'w', encoding='utf-8') with torch.no_grad(): for line in tqdm(open(config.train_label, encoding='utf-8')): fout.write(line.strip('\n')) id = line.split('\t')[0] file_path = os.path.join(config.train_dir, id) df = pd.read_csv(file_path, sep=' ').values x = transform(df).reshape((1, 1, 8, 2500)).to(device) output = torch.sigmoid(model(x)).squeeze().cpu().numpy() for i in output: fout.write("\t" + str(i)) fout.write('\n') fout.close()
def test_new(args): from dataset import transform from data_process import name2index, sex2Index, age2Index name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} sex2idx = sex2Index(config.test_label) age2idx = age2Index(config.test_label) utils.mkdirs(config.sub_dir) # model model = ResMlp(ResMlpParams) model.load_state_dict( torch.load(args.ckpt, map_location='cpu')['state_dict']) model = model.to(device) model.eval() sub_file = '%s/subA_%s.txt' % (config.sub_dir, time.strftime("%Y%m%d%H%M")) fout = open(sub_file, 'w', encoding='utf-8') with torch.no_grad(): for line in open(config.test_label, encoding='utf-8'): fout.write(line.strip('\n')) row = line.split('\t') id = row[0] sex, age = sex2idx[id], age2idx[id] file_path = os.path.join(config.test_dir, id) df = pd.read_csv(file_path, sep=' ').values x = transform(df).unsqueeze(0).to(device) # 因为conv1d需要三维的向量才能读进来 sex, age = sex.unsqueeze(0).to(device), age.unsqueeze(0).to(device) output = torch.sigmoid(model(x, sex, age)).squeeze().cpu().numpy() ixs = [i for i, out in enumerate(output) if out > 0.5] for i in ixs: fout.write("\t" + idx2name[i]) fout.write('\n') fout.close()
def test(args): from dataset import transform from data_process import name2index name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} utils.mkdirs(config.sub_dir) model = models.myecgnet() model.load_state_dict( torch.load(args.ckpt, map_location='cpu')['state_dict']) model = model.to(device) model.eval() sub_file = 'result.txt' fout = open(sub_file, 'w', encoding='utf-8') with torch.no_grad(): for line in open(config.test_label, encoding='utf-8'): fout.write(line.strip('\n')) id = line.split('\t')[0] file_path = os.path.join(config.test_dir, id) df = pd.read_csv(file_path, sep=' ').values x = transform(df).unsqueeze(0).to(device) output = torch.sigmoid(model(x)).squeeze().cpu().numpy() ixs = [i for i, out in enumerate(output) if out > 0.5] for i in ixs: fout.write("\t" + idx2name[i]) print(i, end=',') fout.write('\n') fout.close() print('\n', end='')
def test(args): from dataset import ECGDataset_test from data_process import name2index name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} sub_file = 'result.txt' fout = open(sub_file, 'w', encoding='utf-8') config.test_label = args.test_label config.test_dir = args.test_dir path_all = [] for line in open(config.test_label, encoding='utf-8'): # fout.write(line.strip('\n')) id = line.split('\t')[0] file_path = os.path.join(config.test_dir, id) path_all.append(file_path) test_dataset = ECGDataset_test(path_all) test_load = DataLoader(test_dataset, batch_size=64, num_workers=6, shuffle=False) out = np.zeros((len(test_dataset), 34)) print(len(test_dataset)) for fold in range(1): model_save_dir = '%s/%s' % (config.ckpt, config.model_name + '_' + str(fold)) print(model_save_dir) model = getattr(models, config.model_name)() model.load_state_dict( torch.load(os.path.join(model_save_dir, config.best_w), map_location='cpu')['state_dict']) model = model.to(device) model.eval() with torch.no_grad(): if torch.cuda.is_available(): pred_all = torch.Tensor().cuda() else: pred_all = torch.Tensor() for inputs, target in tqdm.tqdm(test_load): inputs = inputs.to(device) output, _ = model(inputs) output = torch.sigmoid(output) pred_all = torch.cat((pred_all, output), 0) out += pred_all.cpu().numpy() out = out #/ 5 num = 0 for line in open(config.test_label, encoding='utf-8'): fout.write(line.strip('\n')) output = out[num] ixs = [i for i, out in enumerate(output) if out > 0.5] for i in ixs: fout.write("\t" + idx2name[i]) fout.write('\n') num = num + 1 fout.close()
def test(args): if config.kind == 2 and config.top4_catboost: top4_catboost_test(args) # catboost else: from dataset import transform from data_process import name2index name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} utils.mkdirs(config.sub_dir) # model model = getattr(models, config.model_name)(num_classes=config.num_classes, channel_size=config.channel_size) model.load_state_dict( torch.load(args.ckpt, map_location='cpu')['state_dict']) model = model.to(device) model.eval() sub_file = '%s/subA_%s.txt' % (config.sub_dir, time.strftime("%Y%m%d%H%M")) fout = open(sub_file, 'w', encoding='utf-8') print(sub_file) with torch.no_grad(): for line in open(config.test_label, encoding='utf-8'): fout.write(line.strip('\n')) line = line.strip('\n') id = line.split('\t')[0] age = line.split('\t')[1] sex = line.split('\t')[2] if len(age) < 1: age = '-999' age = int(age) sex = {'FEMALE': 0, 'MALE': 1, '': -999}[sex] file_path = os.path.join(config.test_dir, id) df = utils.read_csv(file_path, sep=' ', channel_size=config.channel_size) x = transform(df.values).unsqueeze(0).to(device) fr = torch.tensor([age, sex], dtype=torch.float32).unsqueeze(0).to(device) if config.kind == 1: output = torch.sigmoid(model(x, fr)).squeeze().cpu().numpy() elif config.kind == 2: output, out2 = model(x) output = torch.sigmoid(output).squeeze().cpu().numpy() if config.top4_DeepNN: output = output[config.top4_tag_list] else: output = torch.sigmoid(model(x)).squeeze().cpu().numpy() ixs = [i for i, out in enumerate(output) if out > 0.5] for i in ixs: fout.write("\t" + idx2name[i]) fout.write('\n') fout.close()
def top4_make_dateset(model, dataloader, file_path, save=True): print('make dataset') if os.path.exists(file_path): val_df = pd.read_csv(file_path, index_col=None) return val_df values = [] if config.top4_DeepNN: model.eval() with torch.no_grad(): n = 0 for inputs, fr, target, other_f in dataloader: inputs = inputs.to(device) output, out1 = model(inputs) output = torch.sigmoid(output).cpu() out1 = out1.cpu() if config.top4_DeepNN_tag: output = output[:, config.top4_tag_list] # output, out1 = torch.zeros(64, 10), torch.ones(64, 20) vi = torch.cat([output, out1, other_f, fr, target], dim=1) values.append(vi) # n += 1 # if n == 100: # break else: n = 0 for inputs, fr, target, other_f in dataloader: output, out1 = torch.zeros(inputs.shape[0], 0), torch.ones(inputs.shape[0], 0) vi = torch.cat([output, out1, other_f, fr, target], dim=1) values.append(vi) # n += 1 # if n == 100: # break values = torch.cat(values, dim=0) columnslist = [] columnslist += ['dnn1_%d' % i for i in range(output.size(1))] columnslist += ['dnn2_%d' % i for i in range(out1.size(1))] print('len_dnn_feature', len(columnslist)) columnslist += ['other_f_%d' % i for i in range(other_f.size(1))] columnslist += ['sex', 'age'] name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} columnslist += [idx2name[i] for i in range(target.size(1))] df = pd.DataFrame(values.numpy(), columns=columnslist) df[columnslist[-2 - 55:]] = df[columnslist[-2 - 55:]].astype(int) if save: df.to_csv(file_path, index=None) return df
def test_ensemble(args): from dataset import transform from data_process import name2index, get_arrythmias, get_dict # arrythmias = get_arrythmias(config.arrythmia) # name2idx,idx2name = get_dict(arrythmias) name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} #utils.mkdirs(config.sub_dir) num_clases = 34 kfold = len(config.model_names) # model model = [] for fold in range(kfold): model.append(getattr(models, config.model_names)()) for fold in range(kfold): model[fold].load_state_dict( torch.load(os.path.join(args.ckpt, config.model_ckpts[fold], "best_weight.pth"), map_location='cpu')['state_dict']) model[fold] = model[fold].to(device) model[fold].eval() #sub_file = '%s/subB_%s.txt' % (config.sub_dir, time.strftime("%Y%m%d%H%M")) sub_file = './result.txt' fout = open(sub_file, 'w', encoding='utf-8') with torch.no_grad(): for line in tqdm(open(config.test_label, encoding='utf-8')): fout.write(line.strip('\n')) id = line.split('\t')[0] file_path = os.path.join(config.test_dir, id) df = pd.read_csv(file_path, sep=' ') df['III'] = df['II'] - df['I'] df['aVR'] = -(df['I'] + df['II']) / 2 df['aVL'] = df['I'] - df['II'] / 2 df['aVF'] = df['II'] - df['I'] / 2 x = transform(df.values).unsqueeze(0).to(device) output = 0 #np.zeros(num_clases) for fold in range(kfold): output += torch.sigmoid(model[fold](x)).squeeze().cpu().numpy() output = output / kfold ixs = [i for i, out in enumerate(output) if out > 0.5] for i in ixs: fout.write("\t" + idx2name[i]) fout.write('\n') fout.close()
def test(args): from dataset import transform from data_process import name2index import pickle from tqdm import tqdm test_age_sex = pickle.load(open(config.test_age_sex, 'rb')) name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} utils.mkdirs(config.sub_dir) # model model = getattr(models, config.model_name)() model.load_state_dict(torch.load(args.ckpt, map_location='cpu')['state_dict']) model = model.to(device) model.eval() sub_file = '%s/subA_%s.txt' % (config.sub_dir, time.strftime("%Y%m%d%H%M")) fout = open(sub_file, 'w', encoding='utf-8') with torch.no_grad(): for line in tqdm(open(config.test_label, encoding='utf-8')): fout.write(line.strip('\n')) id = line.split('\t')[0] file_path = os.path.join(config.test_dir, id) df = pd.read_csv(file_path, sep=' ').values df = add_4(df) age = test_age_sex[id.split('.')[0]]['age'] sex = test_age_sex[id.split('.')[0]]['sex'] age = torch.tensor(age.copy(), dtype=torch.float).unsqueeze(0).to(device) sex = torch.tensor(sex.copy(), dtype=torch.float).unsqueeze(0).to(device) x = transform(df).unsqueeze(0).to(device) output = torch.sigmoid(model(x, age, sex)).squeeze().cpu().numpy() ixs = [i for i, out in enumerate(output) if out > 0.5] for i in ixs: fout.write("\t" + idx2name[i]) fout.write('\n') fout.close()
def deep_predict(): from dataset import transform from data_process import name2index name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} # model model = getattr(models, config.model_name)() model.load_state_dict(torch.load(os.path.join(r'ckpt/resnet34', config.best_w), map_location='cpu')['state_dict']) # best_w = torch.load(os.path.join(r'ckpt/resnet34', config.best_w)) # model.load_state_dict(best_w['state_dict']) model = model.to(device) model.eval() test_data = [] for line in open(config.test_label, encoding='utf-8'): id = line.split('\t')[0] file_path = os.path.join(config.test_dir, id) test_data.append(file_path) print(len(test_data)) from dataset import ECGDataTest test_dataset = ECGDataTest(test_data) from multiprocessing import cpu_count print(cpu_count()) processes = cpu_count() - 1 if processes < 0: processes = 1 test_dataloader = DataLoader(test_dataset, batch_size=64, num_workers=processes) output_list = np.empty(shape=[0, config.num_classes]) with torch.no_grad(): for inputs in test_dataloader: inputs = inputs.to(device) output = torch.sigmoid(model(inputs)).cpu().numpy() output_list = np.vstack((output_list, output)) return np.array(output_list)
def test(self): name2idx = name2index('./hf_round2_arrythmia.txt') idx2name = {idx: name for name, idx in name2idx.items()} print(idx2name) # utils.mkdirs(self.opt.sub_dir) # model # model_save_dir = self.opt.ckpt # best_w = torch.load(os.path.join(model_save_dir, self.opt.best_w)) # best_w = torch.load('./best_w.pth') best_w = './best_w.pth' model = getattr(models, self.opt.model_name)() model.load_state_dict( torch.load(best_w, map_location='cpu')['state_dict']) model = model.to(self.device) model.eval() # sub_file = '%s/result.txt' % (self.opt.sub_dir) sub_file = './result.txt' fout = open(sub_file, 'w', encoding='utf-8') with torch.no_grad(): for line in open(self.opt.test_label, encoding='utf-8'): fout.write(line.strip('\n')) id = line.split('\t')[0] file_path = os.path.join(self.opt.test_dir, id) # df = pd.read_csv(file_path, sep=' ').values df = pd.read_csv(file_path, sep=' ') df = add_feature(df).values x = transform(df).unsqueeze(0).to(self.device) output = torch.sigmoid(model(x)).squeeze().cpu().numpy() ixs = [i for i, out in enumerate(output) if out > 0.5] print(id, ixs) for i in ixs: # print(i, idx2name[i]) if i < len(idx2name): fout.write("\t" + idx2name[i]) fout.write('\n') print("writing finish.") fout.close()
hh[:, i] = hh[:, i] > lim[i] ecoc = np.array(pd.read_csv('ecoc.csv', header=None)) ecout = np.zeros_like(hh) minhmd = np.zeros((h.shape[0], )) h = np.concatenate((h, hh), axis=1) for i in range(h.shape[0]): minhmd[i] = ecoc.shape[1] + 1 for j in range(ecoc.shape[0]): minhmd[i] = min(np.sum(h[i] != ecoc[j]), minhmd[i]) for j in range(ecoc.shape[0]): if (np.sum(h[i] != ecoc[j]) == minhmd[i]): ecout[i, j] = 1 if (np.sum(ecout[i]) > 5): ecout[i] = hh[i] print(np.sum(ecout, axis=0).astype('int')) np.savetxt('ecout.txt', ecout, '%d', '\t') cur = 0 name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} sub_file = './result.txt' fout = open(sub_file, 'w', encoding='utf-8') for line in open(config.test_label, encoding='utf-8'): fout.write(line.strip('\n')) output = ecout[cur] ixs = [i for i, out in enumerate(output) if out > 0.5] for i in ixs: fout.write("\t" + idx2name[i]) fout.write('\n') cur = cur + 1 fout.close()
from options import Options from dataset import load_data from model import Ecg from data_process import train, name2index ## # def main(): """ Training """ # torch.manual_seed(config.seed) # torch.cuda.manual_seed(config.seed) ## # ARGUMENTS opt = Options().parse() print(opt) name2idx = name2index(opt.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} train(name2idx, idx2name, config=opt) ## # LOAD DATA train_dataset, train_dataloader, val_dataset, val_dataloader = load_data(opt) ## # LOAD MODEL model = Ecg(opt, train_dataset, train_dataloader, val_dataset, val_dataloader) ## # TRAIN MODEL model.train()
def top4_catboost_test(args): print('top4_catboost_test') from dataset import transform from data_process import name2index name2idx = name2index(config.arrythmia) idx2name = {idx: name for name, idx in name2idx.items()} utils.mkdirs(config.sub_dir) if config.top4_DeepNN: # model model = getattr(models, config.model_name)(num_classes=config.num_classes, channel_size=config.channel_size) model.load_state_dict( torch.load(args.ckpt, map_location='cpu')['state_dict']) model = model.to(device) model.eval() print(config.model_name, args.ckpt) else: model = None print('no', config.model_name) sub_file = '%s/subA_%s.txt' % (config.sub_dir, time.strftime("%Y%m%d%H%M")) print(sub_file) fout = open(sub_file, 'w', encoding='utf-8') if config.kind == 2: model_list = load_model_list( os.path.join(config.ckpt, config.top4_catboost_model)) with torch.no_grad(): for line in open(config.test_label, encoding='utf-8'): fout.write(line.strip('\n')) line = line.strip('\n') id = line.split('\t')[0] age = line.split('\t')[1] sex = line.split('\t')[2] if len(age) < 1: age = '-999' age = int(age) sex = {'FEMALE': 0, 'MALE': 1, '': -999}[sex] file_path = os.path.join(config.test_dir, id) df = utils.read_csv(file_path, sep=' ', channel_size=config.channel_size) fr = torch.tensor([age, sex], dtype=torch.float32) if config.top4_DeepNN: x = transform(df.values).unsqueeze(0).to(device) output, out1 = model(x) output = torch.sigmoid(output).squeeze().cpu().numpy() out1 = out1.squeeze().cpu().numpy() if config.top4_DeepNN_tag: output = output[config.top4_tag_list] else: output, out1 = torch.zeros(0).numpy(), torch.ones(0).numpy() r_features_file = os.path.join(config.r_test_dir, id.replace('.txt', '.fea')) other_f = get_other_features(df, r_features_file) df_values = np.concatenate((output, out1, other_f, fr)) columnslist = [] columnslist += ['dnn1_%d' % i for i in range(len(output))] columnslist += ['dnn2_%d' % i for i in range(len(out1))] # print('len_dnn_feature', len(columnslist)) columnslist += ['other_f_%d' % i for i in range(len(other_f))] columnslist += ['sex', 'age'] df = pd.DataFrame(df_values.reshape(1, -1), columns=columnslist) df[df.columns[config.top4_cat_features]] = df[df.columns[ config.top4_cat_features]].astype(int) # for cindex in config.top4_cat_features: # df[df.columns[cindex]] = df[df.columns[cindex]].astype(int) output = model_list_predict(model_list, df).squeeze() # print(output) ixs = [i for i, out in enumerate(output) if out > 0.5] for i in ixs: fout.write("\t" + idx2name[i]) fout.write('\n') fout.close()