def _encode_examples(self, dataset: RepertoireDataset, params: EncoderParams) -> Tuple[list, set, dict]: keys = set() example_count = dataset.get_example_count() arguments = [(repertoire, index, example_count) for index, repertoire in enumerate(dataset.repertoires)] with Pool(params.pool_size) as pool: chunksize = math.floor(dataset.get_example_count() / params.pool_size) + 1 examples = pool.starmap(self._process_repertoire_cached, arguments, chunksize=chunksize) for example in examples: keys.update(list(example.keys())) labels = dataset.get_metadata(params.label_config.get_labels_by_name()) if params.encode_labels else None return examples, keys, labels
def process_dataset(self, dataset: RepertoireDataset): extract_fn = self.build_matching_fn() repertoire_count = dataset.get_example_count() for index, repertoire in enumerate(dataset.get_data()): self.process_repertoire(repertoire, str(repertoire.identifier), extract_fn) logging.info("Repertoire {} ({}/{}) processed.".format( repertoire.identifier, index + 1, repertoire_count)) logging.info( f"Currently, there are {self.item_count} items in the comparison data matrix." ) self.merge_tmp_batches_to_matrix()
def _encode_repertoires(self, dataset: RepertoireDataset, params: EncoderParams): # Rows = repertoires, Columns = regex matches (one chain per column) encoded_repertoires = np.zeros( (dataset.get_example_count(), self.feature_count), dtype=int) labels = { label: [] for label in params.label_config.get_labels_by_name() } if params.encode_labels else None n_repertoires = dataset.get_example_count() for i, repertoire in enumerate(dataset.get_data()): print( f"{datetime.datetime.now()}: Encoding repertoire {i+1}/{n_repertoires}" ) encoded_repertoires[i] = self._match_repertoire_to_regexes( repertoire) if labels is not None: for label in params.label_config.get_labels_by_name(): labels[label].append(repertoire.metadata[label]) return encoded_repertoires, labels
def _encode_repertoires(self, dataset: RepertoireDataset, params): # Rows = repertoires, Columns = reference sequences encoded_repertories = np.zeros((dataset.get_example_count(), len(self.reference_sequences)), dtype=int) labels = {label: [] for label in params.label_config.get_labels_by_name()} if params.encode_labels else None for i, repertoire in enumerate(dataset.get_data()): encoded_repertories[i] = self._match_repertoire_to_reference(repertoire) for label in params.label_config.get_labels_by_name(): labels[label].append(repertoire.metadata[label]) return encoded_repertories, labels
def compare_repertoires(self, dataset: RepertoireDataset, comparison_fn): self.comparison_data = self.memo_by_params(dataset) repertoire_count = dataset.get_example_count() comparison_result = np.zeros([repertoire_count, repertoire_count]) repertoire_identifiers = dataset.get_repertoire_ids() for index1 in range(repertoire_count): repertoire_vector_1 = self.comparison_data.get_repertoire_vector( repertoire_identifiers[index1]) for index2 in range(index1, repertoire_count): repertoire_vector_2 = self.comparison_data.get_repertoire_vector( repertoire_identifiers[index2]) comparison_result[index1, index2] = comparison_fn( repertoire_vector_1, repertoire_vector_2) comparison_result[index2, index1] = comparison_result[index1, index2] comparison_df = pd.DataFrame(comparison_result, columns=repertoire_identifiers, index=repertoire_identifiers) return comparison_df
def test_run(self): dataset = RepertoireDataset(repertoires=[ Repertoire("0.npy", "", "0"), Repertoire("0.npy", "", "1"), Repertoire("0.npy", "", "2"), Repertoire("0.npy", "", "3"), Repertoire("0.npy", "", "4"), Repertoire("0.npy", "", "5"), Repertoire("0.npy", "", "6"), Repertoire("0.npy", "", "7") ]) paths = [ EnvironmentSettings.root_path + "test/tmp/datasplitter/split_{}".format(i) for i in range(5) ] for path in paths: PathBuilder.build(path) df = pd.DataFrame(data={ "key1": [0, 0, 1, 1, 1, 2, 2, 0], "filename": [0, 1, 2, 3, 4, 5, 6, 7] }) df.to_csv(EnvironmentSettings.root_path + "test/tmp/datasplitter/metadata.csv") dataset.metadata_file = EnvironmentSettings.root_path + "test/tmp/datasplitter/metadata.csv" training_percentage = 0.7 trains, tests = DataSplitter.run( DataSplitterParams(dataset=dataset, training_percentage=training_percentage, split_strategy=SplitType.RANDOM, split_count=5, paths=paths)) self.assertTrue(isinstance(trains[0], RepertoireDataset)) self.assertTrue(isinstance(tests[0], RepertoireDataset)) self.assertEqual(len(trains[0].get_data()), 5) self.assertEqual(len(tests[0].get_data()), 3) self.assertEqual(5, len(trains)) self.assertEqual(5, len(tests)) self.assertEqual(5, len(trains[0].repertoires)) trains2, tests2 = DataSplitter.run( DataSplitterParams(dataset=dataset, training_percentage=training_percentage, split_strategy=SplitType.RANDOM, split_count=5, paths=paths)) self.assertEqual(trains[0].get_repertoire_ids(), trains2[0].get_repertoire_ids()) paths = [ EnvironmentSettings.root_path + "test/tmp/datasplitter/split_{}".format(i) for i in range(dataset.get_example_count()) ] for path in paths: PathBuilder.build(path) trains, tests = DataSplitter.run( DataSplitterParams(dataset=dataset, split_strategy=SplitType.LOOCV, split_count=-1, training_percentage=-1, paths=paths)) self.assertTrue(isinstance(trains[0], RepertoireDataset)) self.assertTrue(isinstance(tests[0], RepertoireDataset)) self.assertEqual(len(trains[0].get_data()), 7) self.assertEqual(len(tests[0].get_data()), 1) self.assertEqual(8, len(trains)) self.assertEqual(8, len(tests)) paths = [ EnvironmentSettings.root_path + "test/tmp/datasplitter/split_{}".format(i) for i in range(5) ] for path in paths: PathBuilder.build(path) trains, tests = DataSplitter.run( DataSplitterParams(dataset=dataset, split_strategy=SplitType.K_FOLD, split_count=5, training_percentage=-1, paths=paths)) self.assertTrue(isinstance(trains[0], RepertoireDataset)) self.assertTrue(isinstance(tests[0], RepertoireDataset)) self.assertEqual(len(trains[0].get_data()), 6) self.assertEqual(len(tests[0].get_data()), 2) self.assertEqual(5, len(trains)) self.assertEqual(5, len(tests)) shutil.rmtree(EnvironmentSettings.root_path + "test/tmp/datasplitter/")