def __init__(self, samples, sa=None, fa=None, a=None):
"""
A Dataset might have an arbitrary number of attributes for samples,
features, or the dataset as a whole. However, only the data samples
themselves are required.
Parameters
----------
samples : ndarray
Data samples. This has to be a two-dimensional (samples x features)
array. If the samples are not in that format, please consider one of
the `AttrDataset.from_*` classmethods.
sa : SampleAttributesCollection
Samples attributes collection.
fa : FeatureAttributesCollection
Features attributes collection.
a : DatasetAttributesCollection
Dataset attributes collection.
"""
# conversions
if isinstance(samples, list):
samples = np.array(samples)
# Check all conditions we need to have for `samples` dtypes
if not hasattr(samples, 'dtype'):
raise ValueError(
"AttrDataset only supports dtypes as samples that have a "
"`dtype` attribute that behaves similar to the one of an "
"array-like.")
if not hasattr(samples, 'shape'):
raise ValueError(
"AttrDataset only supports dtypes as samples that have a "
"`shape` attribute that behaves similar to the one of an "
"array-like.")
if not len(samples.shape):
raise ValueError("Only `samples` with at least one axis are "
"supported (got: %i)" % len(samples.shape))
# handling of 1D-samples
# i.e. 1D is treated as multiple samples with a single feature
if len(samples.shape) == 1:
samples = np.atleast_2d(samples).T
# that's all -- accepted
self.samples = samples
# Everything in a dataset (except for samples) is organized in
# collections
# Number of samples is .shape[0] for sparse matrix support
self.sa = SampleAttributesCollection(length=len(self))
if not sa is None:
self.sa.update(sa)
self.fa = FeatureAttributesCollection(length=self.nfeatures)
if not fa is None:
self.fa.update(fa)
self.a = DatasetAttributesCollection()
if not a is None:
self.a.update(a)
def from_niml(dset, fa_labels=[], sa_labels=[], a_labels=[]):
'''Convert a NIML dataset to a Dataset
Parameters
----------
dset: dict
Dictionary with NIML key-value pairs, such as obtained from
mvpa2.support.nibabel.afni_niml_dset.read()
fa_labels: list
Keys in dset that are enforced to be feature attributes
sa_labels: list
Keys in dset that are enforced to be sample attributes
a_labels: list
Keys in dset that are enforced to be dataset attributes
Returns
-------
dataset: mvpa2.base.Dataset
a PyMVPA Dataset
'''
# check for singleton element
if type(dset) is list and len(dset) == 1:
# recursive call
return from_niml(dset[0])
if not type(dset) is dict:
raise ValueError("Expected a dict")
if not 'data' in dset:
raise ValueError("dset with no data?")
data = dset['data']
if len(data.shape) == 1:
nfeatures = data.shape[0]
nsamples = 1
else:
nfeatures, nsamples = data.shape
# some labels have predefined destinations
sa_labels_ = ['labels', 'stats', 'chunks', 'targets'] + sa_labels
fa_labels_ = ['node_indices', 'center_ids'] + fa_labels
a_labels_ = ['history'] + a_labels
ignore_labels = ('data', 'dset_type')
sa = SampleAttributesCollection(length=nsamples)
fa = FeatureAttributesCollection(length=nfeatures)
a = DatasetAttributesCollection()
labels_collections = [(sa_labels_, sa),
(fa_labels_, fa),
(a_labels_, a)]
infix2collection = {'sa': sa,
'fa': fa,
'a': a}
infix2length = {'sa': nsamples, 'fa': nfeatures}
for k, v in dset.iteritems():
if k in ignore_labels:
continue
if k.startswith(_PYMVPA_PREFIX + _PYMVPA_SEP):
# special PYVMPA field - do the proper conversion
k_split = k.split(_PYMVPA_SEP)
if len(k_split) > 2:
infix = k_split[1].lower()
collection = infix2collection.get(infix, None)
if not collection is None:
short_k = _PYMVPA_SEP.join(k_split[2:])
expected_length = infix2length.get(infix, None)
if expected_length:
if isinstance(v, np.ndarray) and np.dtype == np.str_:
v = str(v)
while isinstance(v, basestring):
# strings are seperated by ';'
# XXX what if this is part of the value
# intended by the user?
v = v.split(';')
if expected_length != len(v):
raise ValueError("Unexpected length: %d != %d" %
(expected_length, len(v)))
v = ArrayCollectable(v, length=expected_length)
collection[short_k] = v
continue
found_label = False
for label, collection in labels_collections:
if k in label:
collection[k] = v
found_label = True
break
if found_label:
continue
# try to be smart and deduce this from dimensions.
# this only works if nfeatures!=nsamples otherwise it would be
# ambiguous
# XXX is this ugly?
if nfeatures != nsamples:
try:
n = len(v)
if n == nfeatures:
fa[k] = v
continue
elif n == nsamples:
sa[k] = v
continue
except:
pass
# don't know what this is - make it a general attribute
a[k] = v
ds = Dataset(np.transpose(data), sa=sa, fa=fa, a=a)
return ds
run = [i+1 for _ in range(data.shape[0])]
runs.append(run)
ds_ = np.vstack(ds_list)
sa = SampleAttributesCollection({
'targets': np.hstack(runs),
'chunks': np.hstack(runs),
'runs': np.hstack(runs),
'subject': np.ones(ds_.shape[0]),
'file': ["Subj=1_connectivity_individualalpha.mat" for _ in range(ds_.shape[0])]
})
fa = FeatureAttributesCollection({'matrix_values':np.ones(ds_.shape[1])})
a = DatasetAttributesCollection({'data_path':'/media/robbis/DATA/meg/hcp/',
'experiment':'hcp',
})
ds = Dataset(ds_, sa=sa, a=a, fa=fa)
mat.close()
nan_mask = np.logical_not(np.isnan(ds.samples))
keep_idx = np.bool_(np.sum(nan_mask, axis=1))
ds = ds[keep_idx]
_default_options = {
'estimator': [
[('clf1', cluster.KMeans())],
[('clf1', cluster.AgglomerativeClustering())],
