def _save_npz_data(self, data_list, data_path, max_num_per_file=10000): if not exists(data_path): os.makedirs(data_path) n = len(data_list) for i in range(int((n - 1) / max_num_per_file) + 1): file = 'part-%05d.npz' % i sub_data_list = self.data_list[i * max_num_per_file: (i + 1) * max_num_per_file] save_data_list_to_npz(join(data_path, file), sub_data_list)
def test_data_list_to_npz(self): data_list = [{"a": np.array([1, 23, 4])}, {"a": np.array([2, 34, 5])}] npz_file = 'tmp.npz' save_data_list_to_npz(data_list, npz_file) reload_data_list = load_npz_to_data_list(npz_file) self.assertEqual(len(data_list), len(reload_data_list)) for d1, d2 in zip(data_list, reload_data_list): self.assertEqual(len(d1), len(d2)) for key in d1: self.assertTrue((d1[key] == d2[key]).all())
def _save_npz_data(self, data_list, data_path, max_num_per_file=10000): if not exists(data_path): os.makedirs(data_path) sub_data_list = [] count = 0 for data in self.data_generator: sub_data_list.append(data) if len(sub_data_list) == 0: file = 'part-%05d.npz' % count save_data_list_to_npz(join(data_path, file), sub_data_list) sub_data_list = [] count += 1 if len(sub_data_list) > 0: file = 'part-%05d.npz' % count save_data_list_to_npz(join(data_path, file), sub_data_list)
def main(args): """Entry for data preprocessing.""" os.makedirs(os.path.join(data_dir, 'processed'), exist_ok=True) processed_dir = os.path.join(data_dir, 'processed') save_dir = '%s/GDSC/drug_graph_feat' % args.data_dir raw_drug_feature(Drug_smiles_file, save_dir) Drug_feature_file = '%s/GDSC/drug_graph_feat' % data_dir metadata = metadata_generate(Drug_info_file, Cell_line_info_file, Genomic_mutation_file, Drug_feature_file, Gene_expression_file, Methylation_file) drug_feature, mutation_feature, gexpr_feature, methylation_feature, data_idx = metadata['metadata'] drug_feature = gen_drug_feature(drug_feature, israndom=args.israndom) train_idx, test_idx = data_split(data_idx, args.split_ratio) print('==============================') print("train_set : test_set == %.2f" % (len(train_idx) / len(test_idx))) for split in ['train', 'test']: index = train_idx if split == 'train' else test_idx mutation_data, gexpr_data, methylation_data, target, cancer_type_list = gen_omics_feature(index, mutation_feature, gexpr_feature, methylation_feature) drug_list = gen_drug_graph(drug_feature, index) data_lst = [{'drug_list': drug_list, 'mutation_data': mutation_data, 'gexpr_data': gexpr_data, 'methylation_data': methylation_data, 'target': target, 'cancer_type_list': cancer_type_list}] npz = os.path.join(processed_dir, '{}_{}.npz'.format(split, args.split_ratio)) save_data_list_to_npz(data_lst, npz) print('==============================') print('{} training samples and {} testing samples have been generated and saved in {} '.format(len(train_idx), len(test_idx), processed_dir))
def preprocess_dataset(name): """ Preprocess raw datasets. Args: name (str): name of the dataset. """ data_dir = os.path.join('data', name, 'raw') if not os.path.exists(data_dir): print('Ignore MUTAG dataset. Cannot find the corresponding folder: %s.' % data_dir) return can, txt = Datasets[name] smiles_path = os.path.join(data_dir, can) labels_path = os.path.join(data_dir, txt) smiles_list = pd.read_csv(smiles_path, sep=' ', header=None)[0] labels = pd.read_csv(labels_path, header=None)[0].replace(-1, 0).values data_list, data_smiles_list = [], [] for i in range(len(smiles_list)): s = smiles_list[i] mol = AllChem.MolFromSmiles(s) if mol is not None: data = mol_to_graph_data(mol) data['label'] = labels[i].reshape([-1]) data_list.append(data) data_smiles_list.append(smiles_list[i]) processed_dir = os.path.join('data', name, 'processed') if not os.path.exists(processed_dir): os.makedirs(processed_dir) with open(os.path.join(processed_dir, 'smiles.txt'), 'w') as f: for smiles in smiles_list: f.write('%s\n' % smiles) save_data_list_to_npz( data_list, os.path.join(processed_dir, 'data.npz'))
def main(): """Entry for data preprocessing.""" tokenizer = ProteinTokenizer() for dataset in ['davis', 'kiba']: data_dir = os.path.join(args.dataset_root, dataset) if not os.path.exists(data_dir): print('Cannot find {}'.format(data_dir)) continue train_fold = json.load( open(os.path.join(data_dir, 'folds', 'train_fold_setting1.txt'))) train_fold = [ee for e in train_fold for ee in e] # flatten test_fold = json.load( open(os.path.join(data_dir, 'folds', 'test_fold_setting1.txt'))) ligands = json.load(open(os.path.join(data_dir, 'ligands_can.txt')), object_pairs_hook=OrderedDict) proteins = json.load(open(os.path.join(data_dir, 'proteins.txt')), object_pairs_hook=OrderedDict) # Use encoding 'latin1' to load py2 pkl from py3 # pylint: disable=E1123 affinity = pickle.load(open(os.path.join(data_dir, 'Y'), 'rb'), encoding='latin1') smiles_lst, protein_lst = [], [] for k in ligands.keys(): smiles = Chem.MolToSmiles(Chem.MolFromSmiles(ligands[k]), isomericSmiles=True) smiles_lst.append(smiles) for k in proteins.keys(): protein_lst.append(proteins[k]) if dataset == 'davis': # Kd data affinity = [-np.log10(y / 1e9) for y in affinity] affinity = np.asarray(affinity) # pylint: disable=E1123 os.makedirs(os.path.join(data_dir, 'processed'), exist_ok=True) for split in ['train', 'test']: print('processing {} set of {}'.format(split, dataset)) split_dir = os.path.join(data_dir, 'processed', split) # pylint: disable=E1123 os.makedirs(split_dir, exist_ok=True) fold = train_fold if split == 'train' else test_fold rows, cols = np.where(np.isnan(affinity) == False) rows, cols = rows[fold], cols[fold] data_lst = [] for idx in range(len(rows)): mol = AllChem.MolFromSmiles(smiles_lst[rows[idx]]) mol_graph = mol_to_graph_data(mol) data = {k: v for k, v in mol_graph.items()} seqs = [] for seq in protein_lst[cols[idx]].split('\x01'): seqs.extend(tokenizer.gen_token_ids(seq)) data['protein_token_ids'] = np.array(seqs) af = affinity[rows[idx], cols[idx]] if dataset == 'davis': data['Log10_Kd'] = np.array([af]) elif dataset == 'kiba': data['KIBA'] = np.array([af]) data_lst.append(data) random.shuffle(data_lst) npz = os.path.join(split_dir, '{}_{}.npz'.format(dataset, split)) save_data_list_to_npz(data_lst, npz) print('==============================') print('dataset:', dataset) print('train_fold:', len(train_fold)) print('test_fold:', len(test_fold)) print('unique drugs:', len(set(smiles_lst))) print('unique proteins:', len(set(protein_lst)))