def build_labels(self, dataset: RepertoireDataset,
params: EncoderParams) -> dict:
lbl = ["repertoire_identifier"]
lbl.extend(params.label_config.get_labels_by_name())
tmp_labels = dataset.get_metadata(lbl, return_df=True)
tmp_labels = tmp_labels.iloc[pd.Index(
tmp_labels['repertoire_identifier']).get_indexer(
dataset.get_repertoire_ids())]
tmp_labels = tmp_labels.to_dict("list")
del tmp_labels["repertoire_identifier"]
return tmp_labels
def _encode_data(self, dataset: RepertoireDataset, params: EncoderParams):
labels = params.label_config.get_labels_by_name()
assert len(labels) == 1, \
"SequenceAbundanceEncoder: this encoding works only for single label."
examples = self._calculate_sequence_abundance(dataset, self.comparison_data, labels[0], params)
encoded_data = EncodedData(examples, dataset.get_metadata([labels[0]]) if params.encode_labels else None, dataset.get_repertoire_ids(),
[SequenceAbundanceEncoder.RELEVANT_SEQUENCE_ABUNDANCE, SequenceAbundanceEncoder.TOTAL_SEQUENCE_ABUNDANCE],
encoding=SequenceAbundanceEncoder.__name__, info={'relevant_sequence_path': self.relevant_sequence_csv_path})
encoded_dataset = RepertoireDataset(params=dataset.params, encoded_data=encoded_data, repertoires=dataset.repertoires)
return encoded_dataset
def _encode_examples(self, dataset: RepertoireDataset, params: EncoderParams) -> Tuple[list, set, dict]:
keys = set()
example_count = dataset.get_example_count()
arguments = [(repertoire, index, example_count) for index, repertoire in enumerate(dataset.repertoires)]
with Pool(params.pool_size) as pool:
chunksize = math.floor(dataset.get_example_count() / params.pool_size) + 1
examples = pool.starmap(self._process_repertoire_cached, arguments, chunksize=chunksize)
for example in examples:
keys.update(list(example.keys()))
labels = dataset.get_metadata(params.label_config.get_labels_by_name()) if params.encode_labels else None
return examples, keys, labels
def _encode_sequence_count(self, dataset: RepertoireDataset, comparison_data: ComparisonData, label: str, params: EncoderParams) -> EncodedData:
sequence_p_values_indices, indices_path, relevant_sequences_path = SequenceFilterHelper.get_relevant_sequences(dataset, params, comparison_data, label, self.p_value_threshold,
self.comparison_attributes, self.relevant_indices_path)
if self.relevant_indices_path is None:
self.relevant_indices_path = indices_path
if self.relevant_sequence_csv_path is None:
self.relevant_sequence_csv_path = relevant_sequences_path
count_matrix = self._build_count_matrix(comparison_data, dataset.get_repertoire_ids(), sequence_p_values_indices)
feature_names = comparison_data.get_item_names()[sequence_p_values_indices]
encoded_data = EncodedData(count_matrix, dataset.get_metadata([label]) if params.encode_labels else None,
dataset.get_repertoire_ids(),
feature_names,
encoding=SequenceCountEncoder.__name__, info={'relevant_sequence_path': self.relevant_sequence_csv_path})
return encoded_data
def get_matching_indices(dataset: RepertoireDataset, criteria):
metadata = pd.DataFrame(dataset.get_metadata(None))
matches = CriteriaMatcher().match(criteria, metadata)
indices = np.where(matches)[0]
return indices