def _encode_data(self, dataset: SequenceDataset, params: EncoderParams): sequence_objs = [obj for obj in dataset.get_data(params.pool_size)] sequences = [obj.get_sequence() for obj in sequence_objs] example_ids = dataset.get_example_ids() max_seq_len = max([len(seq) for seq in sequences]) labels = self._get_labels(sequence_objs, params) if params.encode_labels else None examples = self._encode_sequence_list( sequences, pad_n_sequences=len(sequence_objs), pad_sequence_len=max_seq_len) feature_names = self._get_feature_names(max_seq_len) if self.flatten: examples = examples.reshape( (len(sequence_objs), max_seq_len * len(self.onehot_dimensions))) feature_names = [ item for sublist in feature_names for item in sublist ] encoded_data = EncodedData(examples=examples, labels=labels, example_ids=example_ids, feature_names=feature_names, encoding=OneHotEncoder.__name__) return encoded_data
def test_make_subset(self): sequences = [] for i in range(100): sequences.append( ReceptorSequence(amino_acid_sequence="AAA", identifier=str(i))) path = EnvironmentSettings.tmp_test_path / "element_generator_subset/" PathBuilder.build(path) for i in range(10): filepath = path / f"batch{i}.pkl" with filepath.open("wb") as file: sequences_to_pickle = sequences[i * 10:(i + 1) * 10] pickle.dump(sequences_to_pickle, file) d = SequenceDataset( filenames=[path / f"batch{i}.pkl" for i in range(10)], file_size=10) indices = [1, 20, 21, 22, 23, 24, 25, 50, 52, 60, 70, 77, 78, 90, 92] d2 = d.make_subset(indices, path, SequenceDataset.TRAIN) for batch in d2.get_batch(1000): for sequence in batch: self.assertTrue(int(sequence.identifier) in indices) self.assertEqual(15, d2.get_example_count()) shutil.rmtree(path)
def test_make_subset(self): sequences = [] for i in range(100): sequences.append(ReceptorSequence(amino_acid_sequence="AAA", identifier=str(i))) path = EnvironmentSettings.tmp_test_path / "element_generator_subset/" PathBuilder.build(path) for i in range(10): filepath = path / f"batch{i}.npy" sequences_to_pickle = sequences[i * 10:(i + 1) * 10] sequence_matrix = np.core.records.fromrecords([seq.get_record() for seq in sequences_to_pickle], names=ReceptorSequence.get_record_names()) np.save(str(filepath), sequence_matrix, allow_pickle=False) d = SequenceDataset(filenames=[path / f"batch{i}.npy" for i in range(10)], file_size=10) indices = [1, 20, 21, 22, 23, 24, 25, 50, 52, 60, 70, 77, 78, 90, 92] d2 = d.make_subset(indices, path, SequenceDataset.TRAIN) for batch in d2.get_batch(1000): for sequence in batch: self.assertTrue(int(sequence.identifier) in indices) self.assertEqual(15, d2.get_example_count()) shutil.rmtree(path)
def _encode_new_dataset(self, dataset: SequenceDataset, params: EncoderParams): encoded_data = self._encode_data(dataset, params) encoded_dataset = SequenceDataset(filenames=dataset.get_filenames(), encoded_data=encoded_data, labels=dataset.labels, file_size=dataset.file_size) return encoded_dataset
def create_dataset(self, path, dataset_size: int = 50): sequences = [] for i in range(dataset_size): if i % 2 == 0: sequences.append( ReceptorSequence( amino_acid_sequence="AAACCC", identifier=str(i), metadata=SequenceMetadata(custom_params={"l1": 1}))) else: sequences.append( ReceptorSequence( amino_acid_sequence="ACACAC", identifier=str(i), metadata=SequenceMetadata(custom_params={"l1": 2}))) PathBuilder.build(path) filename = path / "sequences.pkl" with open(filename, "wb") as file: pickle.dump(sequences, file) lc = LabelConfiguration() lc.add_label("l1", [1, 2]) dataset = SequenceDataset(labels={"l1": [1, 2]}, filenames=[filename], identifier="d1") return dataset
def _construct_test_dataset(self, path): sequences = [ ReceptorSequence(amino_acid_sequence="AAAA", identifier="1", metadata=SequenceMetadata(custom_params={ "l1": 1, "l2": 1 })), ReceptorSequence(amino_acid_sequence="ATA", identifier="2", metadata=SequenceMetadata(custom_params={ "l1": 2, "l2": 1 })), ReceptorSequence(amino_acid_sequence="ATT", identifier="3", metadata=SequenceMetadata(custom_params={ "l1": 1, "l2": 2 })) ] lc = LabelConfiguration() lc.add_label("l1", [1, 2]) lc.add_label("l2", [1, 2]) dataset = SequenceDataset.build(sequences=sequences, file_size=10, path=path) return dataset, lc
def import_sequence_dataset(import_class, params, dataset_name: str): PathBuilder.build(params.result_path) filenames = ImportHelper.get_sequence_filenames(params.path, dataset_name) file_index = 0 dataset_filenames = [] dataset_params = {} items = None for index, filename in enumerate(filenames): new_items = ImportHelper.import_items(import_class, filename, params) items = np.append(items, new_items) if items is not None else new_items dataset_params = ImportHelper.extract_sequence_dataset_params(items, params) while len(items) > params.sequence_file_size or (index == len(filenames) - 1 and len(items) > 0): dataset_filenames.append(params.result_path / "batch_{}.pickle".format(file_index)) ImportHelper.store_sequence_items(dataset_filenames, items, params.sequence_file_size) items = items[params.sequence_file_size:] file_index += 1 init_kwargs = {"filenames": dataset_filenames, "file_size": params.sequence_file_size, "name": dataset_name, "labels": dataset_params} dataset = ReceptorDataset(**init_kwargs) if params.paired else SequenceDataset(**init_kwargs) PickleExporter.export(dataset, params.result_path) return dataset
def _encode_data(self, dataset: SequenceDataset, params: EncoderParams): sequence_objs = [obj for obj in dataset.get_data(params.pool_size)] sequences = [ obj.get_sequence(self.sequence_type) for obj in sequence_objs ] if any(seq is None for seq in sequences): raise ValueError( f"{OneHotEncoder.__name__}: sequence dataset {dataset.name} (id: {dataset.identifier}) contains empty sequences for the specified " f"sequence type {self.sequence_type.name.lower()}. Please check that the dataset is imported correctly." ) example_ids = dataset.get_example_ids() max_seq_len = max([len(seq) for seq in sequences]) labels = self._get_labels(sequence_objs, params) if params.encode_labels else None examples = self._encode_sequence_list( sequences, pad_n_sequences=len(sequence_objs), pad_sequence_len=max_seq_len) feature_names = self._get_feature_names(max_seq_len) if self.flatten: examples = examples.reshape( (len(sequence_objs), max_seq_len * len(self.onehot_dimensions))) feature_names = [ item for sublist in feature_names for item in sublist ] encoded_data = EncodedData(examples=examples, labels=labels, example_ids=example_ids, feature_names=feature_names, encoding=OneHotEncoder.__name__) return encoded_data
def create_dummy_sequencedataset(self, path): sequences = [ReceptorSequence(amino_acid_sequence="AAATTT", identifier="1a", metadata=SequenceMetadata(v_gene="TRAV1", j_gene="TRAJ1", chain=Chain.ALPHA, frame_type="IN", custom_params={"d_call": "TRAD1", "custom1": "cust1"})), ReceptorSequence(amino_acid_sequence="ATATAT", identifier="1b", metadata=SequenceMetadata(v_gene="TRBV1", j_gene="TRBJ1", chain=Chain.BETA, frame_type="IN", custom_params={"d_call": "TRBD1", "custom2": "cust1"})), ReceptorSequence(amino_acid_sequence="ATATAT", identifier="2b", metadata=SequenceMetadata(v_gene="TRBV1", j_gene="TRBJ1", chain=Chain.BETA, frame_type="IN", custom_params={"d_call": "TRBD1", "custom2": "cust1"}))] sequences_path = path / "sequences" PathBuilder.build(sequences_path) return SequenceDataset.build_from_objects(sequences, 2, sequences_path)
def construct_test_flatten_dataset(self, path): sequences = [ ReceptorSequence( amino_acid_sequence="AAATTT", identifier="1", metadata=SequenceMetadata(custom_params={"l1": 1})), ReceptorSequence( amino_acid_sequence="ATATAT", identifier="2", metadata=SequenceMetadata(custom_params={"l1": 2})) ] PathBuilder.build(path) return SequenceDataset.build(sequences=sequences, file_size=10, path=path)
def load_sequence_dataset(params: dict, dataset_name: str) -> Dataset: iris_params = IRISImportParams.build_object(**params) filenames = ImportHelper.get_sequence_filenames(iris_params.path, dataset_name) file_index = 0 dataset_filenames = [] for index, filename in enumerate(filenames): items = IRISSequenceImport.import_items(filename, paired=iris_params.paired, all_dual_chains=iris_params.import_dual_chains, all_genes=iris_params.import_all_gene_combinations) while len(items) > iris_params.sequence_file_size or (index == len(filenames) - 1 and len(items) > 0): dataset_filenames.append(iris_params.result_path / "batch_{}.pickle".format(file_index)) ImportHelper.store_sequence_items(dataset_filenames, items, iris_params.sequence_file_size) items = items[iris_params.sequence_file_size:] file_index += 1 return ReceptorDataset(filenames=dataset_filenames, file_size=iris_params.sequence_file_size, name=dataset_name) if iris_params.paired \ else SequenceDataset(filenames=dataset_filenames, file_size=iris_params.sequence_file_size, name=dataset_name)
def generate_sequence_dataset(sequence_count: int, length_probabilities: dict, labels: dict, path: Path): """ Creates sequence_count receptor sequences (single chain) where the length of sequences in each chain is sampled independently for each sequence from length_probabilities distribution. The labels are also randomly assigned to sequences from the distribution given in labels. In this case, labels are multi-class, so each sequences will get one class from each label. This means that negative classes for the labels should be included as well in the specification. An example of input parameters is given below: sequence_count: 100 # generate 100 TRB ReceptorSequences length_probabilities: 14: 0.8 # 80% of all generated sequences for all receptors (for chain 1) will have length 14 15: 0.2 # 20% of all generated sequences across all receptors (for chain 1) will have length 15 labels: epitope1: # label name True: 0.5 # 50% of the receptors will have class True False: 0.5 # 50% of the receptors will have class False epitope2: # next label with classes that will be assigned to receptors independently of the previous label or other parameters 1: 0.3 # 30% of the generated receptors will have class 1 0: 0.7 # 70% of the generated receptors will have class 0 """ RandomDatasetGenerator._check_sequence_dataset_generation_params( sequence_count, length_probabilities, labels, path) alphabet = EnvironmentSettings.get_sequence_alphabet() PathBuilder.build(path) chain = "TRB" sequences = [ ReceptorSequence( "".join( random.choices(alphabet, k=random.choices( list(length_probabilities.keys()), length_probabilities.values())[0])), metadata=SequenceMetadata( count=1, v_subgroup=chain + "V1", v_gene=chain + "V1-1", v_allele=chain + "V1-1*01", j_subgroup=chain + "J1", j_gene=chain + "J1-1", j_allele=chain + "J1-1*01", chain=chain, custom_params={ **{ label: random.choices(list(label_dict.keys()), label_dict.values(), k=1)[0] for label, label_dict in labels.items() }, **{ "subject": f"subj_{i + 1}" } })) for i in range(sequence_count) ] filename = path / "batch01.npy" sequence_matrix = np.core.records.fromrecords( [seq.get_record() for seq in sequences], names=ReceptorSequence.get_record_names()) np.save(str(filename), sequence_matrix, allow_pickle=False) return SequenceDataset(labels={ label: list(label_dict.keys()) for label, label_dict in labels.items() }, filenames=[filename], file_size=sequence_count)
def test(self): sequences = [ ReceptorSequence( amino_acid_sequence="AAACCC", nucleotide_sequence="AAACCC", identifier="1", metadata=SequenceMetadata(custom_params={"l1": 1})), ReceptorSequence( amino_acid_sequence="ACACAC", nucleotide_sequence="ACACAC", identifier="2", metadata=SequenceMetadata(custom_params={"l1": 2})), ReceptorSequence( amino_acid_sequence="CCCAAA", nucleotide_sequence="CCCAAA", identifier="3", metadata=SequenceMetadata(custom_params={"l1": 1})), ReceptorSequence( amino_acid_sequence="AAACCC", nucleotide_sequence="AAACCC", identifier="4", metadata=SequenceMetadata(custom_params={"l1": 2})), ReceptorSequence( amino_acid_sequence="ACACAC", nucleotide_sequence="ACACAC", identifier="5", metadata=SequenceMetadata(custom_params={"l1": 1})), ReceptorSequence( amino_acid_sequence="CCCAAA", nucleotide_sequence="CCCAAA", identifier="6", metadata=SequenceMetadata(custom_params={"l1": 2})), ReceptorSequence( amino_acid_sequence="AAACCC", nucleotide_sequence="AAACCC", identifier="7", metadata=SequenceMetadata(custom_params={"l1": 1})), ReceptorSequence( amino_acid_sequence="ACACAC", nucleotide_sequence="ACACAC", identifier="8", metadata=SequenceMetadata(custom_params={"l1": 2})), ReceptorSequence( amino_acid_sequence="CCCAAA", nucleotide_sequence="CCCAAA", identifier="9", metadata=SequenceMetadata(custom_params={"l1": 1})) ] path = EnvironmentSettings.tmp_test_path / "kmrefreqseqfacencoder/" PathBuilder.build(path) dataset = SequenceDataset.build_from_objects( sequences, 100, PathBuilder.build(path / 'data'), 'd2') lc = LabelConfiguration() lc.add_label("l1", [1, 2]) encoder = KmerFreqSequenceEncoder.build_object( dataset, **{ "normalization_type": NormalizationType.RELATIVE_FREQUENCY.name, "reads": ReadsType.UNIQUE.name, "sequence_encoding": SequenceEncodingType.CONTINUOUS_KMER.name, "sequence_type": SequenceType.NUCLEOTIDE.name, "k": 3 }) encoded_dataset = encoder.encode( dataset, EncoderParams(result_path=path / "2/", label_config=lc, pool_size=2, learn_model=True, model={}, filename="dataset.csv")) self.assertEqual(9, encoded_dataset.encoded_data.examples.shape[0]) self.assertTrue( all(identifier in encoded_dataset.encoded_data.example_ids for identifier in ['1', '2', '3', '4', '5', '6', '7', '8', '9'])) self.assertTrue( numpy.array_equal(encoded_dataset.encoded_data.examples[0].A, encoded_dataset.encoded_data.examples[3].A)) shutil.rmtree(path)