def _create(features: List) -> SequenceDataSource: return SequenceDataSource( metadata=GeneralSampleMetadata(id="foo"), channel_files=[], label=torch.tensor([]), categorical_non_image_features=torch.tensor([]), numerical_non_image_features=torch.tensor(features).float())
def _create(id: str, sequence_position: int, file: Optional[str], metadata: str) -> SequenceDataSource: return SequenceDataSource( channel_files=[file], numerical_non_image_features=torch.tensor([]), categorical_non_image_features=torch.tensor([]), label=torch.tensor([]), metadata=GeneralSampleMetadata(id=id, sequence_position=sequence_position, props={"M": metadata}))
def _create_item(id: str, sequence_position: int, metadata: str, label: Optional[float] = None) -> SequenceDataSource: return SequenceDataSource( channel_files=["foo"], numerical_non_image_features=torch.tensor([]), categorical_non_image_features=torch.tensor([]), label=(torch.tensor([label]) if label else torch.tensor([])), metadata=GeneralSampleMetadata(id=id, sequence_position=sequence_position, props={"M": metadata}))
def test_standardize_features_when_singleton(is_sequence: bool) -> None: """ Test how feature standardize copes with datasets that only have 1 entry. """ numerical_features = torch.ones((1, 3)) categorical_features = torch.tensor([[0, 1, 1], [1, 0, 0]]) item: Union[SequenceDataSource, ScalarDataSource] sources: Union[ListOfSequences, List[ScalarDataSource]] if is_sequence: item = SequenceDataSource( metadata=GeneralSampleMetadata(id="foo"), numerical_non_image_features=numerical_features, categorical_non_image_features=categorical_features, label=torch.tensor([]), channel_files=[]) sources = [ClassificationItemSequence(id="foo", items=[item])] mean_std = FeatureStatistics.from_data_sources(sources) else: item = ScalarDataSource( metadata=GeneralSampleMetadata(id="foo"), numerical_non_image_features=numerical_features, categorical_non_image_features=categorical_features, label=torch.tensor([]), channel_files=[]) sources = [item] mean_std = FeatureStatistics.from_data_sources(sources) assert_tensors_equal(mean_std.mean, numerical_features) # Standard deviation can't be computed because there is only one element, hence becomes nan. assert torch.all(torch.isnan(mean_std.std)) # When applying such a standardization to the sequences, they should not be changed (similar to features that # are constant) standardized_sources = mean_std.standardize(sources) if is_sequence: assert_tensors_equal( standardized_sources[0].items[0].numerical_non_image_features, numerical_features) assert_tensors_equal( standardized_sources[0].items[0].categorical_non_image_features, categorical_features) else: assert_tensors_equal( standardized_sources[0].numerical_non_image_features, numerical_features) assert_tensors_equal( standardized_sources[0].categorical_non_image_features, categorical_features)
def load_single_data_source(subject_rows: pd.DataFrame, subject_id: str, label_value_column: str, channel_column: str, image_channels: Optional[List[str]] = None, image_file_column: Optional[str] = None, label_channels: Optional[List[str]] = None, transform_labels: Union[Callable, List[Callable]] = LabelTransformation.identity, non_image_feature_channels: Optional[Dict] = None, numerical_columns: Optional[List[str]] = None, categorical_data_encoder: Optional[CategoricalToOneHotEncoder] = None, metadata_columns: Optional[Set[str]] = None, is_classification_dataset: bool = True, sequence_position_numeric: Optional[int] = None) -> T: """ Converts a set of dataset rows for a single subject to a ScalarDataSource instance, which contains the labels, the non-image features, and the paths to the image files. :param channel_column: The name of the column that contains the row identifier ("channels") :param metadata_columns: A list of columns that well be added to the item metadata as key/value pairs. :param subject_rows: All dataset rows that belong to the same subject. :param subject_id: The identifier of the subject that is being processed. :param image_channels: The names of all channels (stored in the CSV_CHANNEL_HEADER column of the dataframe) that are expected to be loaded from disk later because they are large images. :param image_file_column: The name of the column that contains the image file names. :param label_channels: The name of the channel where the label scalar or vector is read from. :param label_value_column: The column that contains the value for the label scalar or vector. :param non_image_feature_channels: non_image_feature_channels: A dictonary of the names of all channels where additional scalar values should be read from. THe keys should map each feature to its channels. :param numerical_columns: The names of all columns where additional scalar values should be read from. :param categorical_data_encoder: Encoding scheme for categorical data. :param is_classification_dataset: If the current dataset is classification or not. from. :param transform_labels: a label transformation or a list of label transformation to apply to the labels. If a list is provided, the transformations are applied in order from left to right. :param sequence_position_numeric: Numeric position of the data source in a data sequence. Assumed to be a non-sequential dataset item if None provided (default). :return: """ def _get_row_for_channel(channel: Optional[str]) -> Dict[str, str]: return _get_single_channel_row(subject_rows, channel, subject_id, channel_column) def _get_label_as_tensor(channel: Optional[str]) -> torch.Tensor: extract_fn = extract_label_classification if is_classification_dataset else extract_label_regression label_row = _get_row_for_channel(channel) label_string = label_row[label_value_column] return torch.tensor([extract_fn(label_string=label_string, sample_id=subject_id)], dtype=torch.float) def _apply_label_transforms(labels: Any) -> Any: """ Apply the transformations in order. """ if isinstance(transform_labels, List): for transform in transform_labels: labels = transform(labels) label = labels else: label = transform_labels(labels) return label def create_none_list(x: Optional[List]) -> List: return [None] if x is None or len(x) == 0 else x def get_none_list_from_dict(non_image_channels: Dict[str, List[str]], feature: str) -> Sequence[Optional[str]]: """ Return either the list of channels for a given column or if None was passed as numerical channels i.e. there are no channel to be specified return [None]. :param non_image_channels: Dict mapping features name to their channels :param feature: feature name for which to return the channels :return: List of channels for the given feature. """ if non_image_channels == {}: return [None] else: return non_image_channels[feature] def is_empty(x: Optional[List]) -> bool: return x is None or len(x) == 0 def none_if_missing_in_csv(x: Any) -> Optional[str]: # If the CSV contains missing values they turn into NaN here, but mark them as None rather. return None if isinstance(x, float) and np.isnan(x) else x subject_rows = subject_rows.fillna('') labels = [] if label_channels: for channel in label_channels: labels.append(_get_label_as_tensor(channel)) else: labels.append(_get_label_as_tensor(None)) label = _apply_label_transforms(labels) channel_for_metadata = label_channels[0] if label_channels else None label_row = _get_row_for_channel(channel_for_metadata) metadata = GeneralSampleMetadata(id=subject_id, props={key: none_if_missing_in_csv(label_row[key]) for key in metadata_columns or set()}) image_files = [] if image_file_column: for image_channel in create_none_list(image_channels): # Alternative: restrict rows to given channels first, then read out the relevant columns. file_path = _get_row_for_channel(image_channel)[image_file_column] image_files.append(none_if_missing_in_csv(file_path)) numerical_columns = numerical_columns or [] categorical_columns = categorical_data_encoder.get_supported_dataset_column_names() if categorical_data_encoder \ else [] _feature_columns = numerical_columns + categorical_columns if not non_image_feature_channels: non_image_feature_channels = {} numerical = [] categorical = {} if not is_empty(_feature_columns): for column in _feature_columns: list_channels: Sequence[Optional[str]] = [str(sequence_position_numeric)] \ if sequence_position_numeric is not None else get_none_list_from_dict(non_image_feature_channels, column) numerical_col, categorical_col = [], [] for channel in list_channels: # type: ignore row = _get_row_for_channel(channel) prefix = f"Channel {channel}, column {column}" if column in numerical_columns: numerical_col.append(_string_to_float(row[column], error_message_prefix=prefix)) else: categorical_col.append(row[column]) if column in numerical_columns: numerical.extend(numerical_col) else: categorical[column] = categorical_col categorical_non_image_features = categorical_data_encoder.encode(categorical) \ if categorical_data_encoder else torch.tensor(list(categorical.values())) datasource: Union[SequenceDataSource, ScalarDataSource] if sequence_position_numeric is not None: metadata.sequence_position = sequence_position_numeric datasource = SequenceDataSource( label=label, channel_files=image_files, numerical_non_image_features=torch.tensor(numerical).float(), categorical_non_image_features=categorical_non_image_features.float(), metadata=metadata ) return datasource # type: ignore datasource = ScalarDataSource( label=label, channel_files=image_files, numerical_non_image_features=torch.tensor(numerical).float(), categorical_non_image_features=categorical_non_image_features.float(), metadata=metadata ) return datasource # type: ignore