def test_pipe_params(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
params = {
'combining__columns': ['DTI_FA', 'DTI_L1'],
'combining__column_name': 'combined',
}
pipe = Pipeline([
('combining', CombineMatrixTransformer()),
])
pipe.set_params(**params)
output = pipe.fit_transform(profiles.data_frame)
assert sorted(list(output.columns)) == ['DTI_FA', 'DTI_L1', 'combined']
assert (output.combined.FIS_007 == [[
6,
7,
8,
], [9, 10, 11], [12, 13, 14]]).all()
def test_spread_out_simple(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
profiles.load()
new_dataframe = profiles.spread_out_matrix(['DTI_L1'],
keep_matrix=False,
symmetric=False)
assert [
'DTI_FA',
'DTI_L1_0_0',
'DTI_L1_0_1',
'DTI_L1_0_2',
'DTI_L1_1_0',
'DTI_L1_1_1',
'DTI_L1_1_2',
'DTI_L1_2_0',
'DTI_L1_2_1',
'DTI_L1_2_2',
] == sorted(list(new_dataframe.columns))
assert new_dataframe.DTI_L1_0_0[0] == 11
assert new_dataframe.DTI_L1_0_1[0] == 12
assert new_dataframe.DTI_L1_0_2[0] == 13
assert new_dataframe.DTI_L1_1_0[0] == 14
assert new_dataframe.DTI_L1_1_1[0] == 15
assert new_dataframe.DTI_L1_1_2[0] == 16
assert new_dataframe.DTI_L1_2_0[0] == 17
assert new_dataframe.DTI_L1_2_1[0] == 18
assert new_dataframe.DTI_L1_2_2[0] == 19
def test_not_found(self, datafiles):
path = os.path.join(str(datafiles), 'valid')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
data_frame = profiles.data_frame
groups = {
'A': [1],
'B': [2],
'C': [3],
'A-B': [1, 2],
'A-B-C': [1, 2, 3],
'X': [],
}
with pytest.raises(KeyError) as error:
utils.get_multilabel(data_frame, 'profile', groups)
# W not found
assert 'W' in str(error.value)
def test_no_df_simple(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
pipe = Pipeline([
('combining',
CombineMatrixTransformer(columns=['0', '1'],
column_name='combined')),
])
output = pipe.fit_transform(profiles.data_frame.values)
assert (output[0][2] == np.asarray([[
6,
7,
8,
], [9, 10, 11], [12, 13, 14]])).all()
def test_pipe_spread_out_column(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
pipe = Pipeline([
('spread', SpreadOutMatrixTransformer(columns=['DTI_FA',
'DTI_L1'])),
('selecting', FeatureMatrixTransformer(
matrix_columns=['DTI_FA'], columns=None)),
])
output = pipe.fit_transform(profiles.data_frame)
assert sorted(list(output.columns)) == [
'DTI_FA_1_0', 'DTI_FA_2_0', 'DTI_FA_2_1']
assert output.DTI_FA_1_0.FIS_007 == 4
assert output.DTI_FA_2_0.FIS_007 == 7
assert output.DTI_FA_2_1.FIS_007 == 8
def test_fail_duplicates_data(self, datafiles):
path = os.path.join(str(datafiles), 'duplicated_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
with pytest.raises(ValueError) as error:
profiles.load()
assert str(error.value) == "Already existing value at [FIS_007, DTI_FA]: " \
"Value from FIS_007_FA_factor.csv cannot be loaded."
def test_fail_format(self, datafiles):
path = os.path.join(str(datafiles), 'duplicated_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path,
profiles_filename=filename,
format_file='mat')
with pytest.raises(NotImplementedError) as error:
profiles.load()
assert str(error.value) == 'Format \'mat\' not supported'
def test_warning(self, datafiles, caplog):
caplog.set_level(logging.INFO)
path = os.path.join(str(datafiles), 'index_not_found')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
profiles.load()
assert len(caplog.records) == 4
assert caplog.records[3].levelname == 'WARNING'
assert caplog.records[3].message == \
"Index FIS_007 couldn't be found in main file (total 1 index(es))."
def test_debug(self, datafiles, caplog):
caplog.set_level(logging.DEBUG)
path = os.path.join(str(datafiles), 'index_not_found')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
profiles.load()
assert len(caplog.records) == 5
assert caplog.records[2].levelname == 'DEBUG'
assert caplog.records[2].message == \
"Index (id) FIS_007 couldn't be found, skipped " \
"(file FIS_007_FA_factor.csv)"
def test_fail_duplicates_id(self, datafiles):
path = os.path.join(str(datafiles), 'duplicated_id')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
with pytest.raises(ValueError) as error:
profiles.load()
if six.PY2:
assert str(error.value) == "Index has duplicate keys: " \
"Index([u'FIS_007', u'TTO_06'], dtype='object', name=u'ID')"
else:
assert str(error.value) == "Index has duplicate keys: " \
"Index(['FIS_007', 'TTO_06'], dtype='object', name='ID')"
def test_default(self):
profiles = NeuProfiles('path', 'filename')
assert profiles.data_frame is None
assert profiles.profile_path == 'path'
assert profiles.profiles_filename == 'filename'
assert profiles.format_file == 'xls'
def test_pipe_symmetric(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
pipe = Pipeline([
('spread',
SpreadOutMatrixTransformer(columns=profiles.data_frame.columns,
symmetric=False)),
])
output = pipe.fit_transform(profiles.data_frame)
assert sorted(list(output.columns)) == [
'DTI_FA_0_0', 'DTI_FA_0_1', 'DTI_FA_0_2', 'DTI_FA_1_0',
'DTI_FA_1_1', 'DTI_FA_1_2', 'DTI_FA_2_0', 'DTI_FA_2_1',
'DTI_FA_2_2', 'DTI_L1_0_0', 'DTI_L1_0_1', 'DTI_L1_0_2',
'DTI_L1_1_0', 'DTI_L1_1_1', 'DTI_L1_1_2', 'DTI_L1_2_0',
'DTI_L1_2_1', 'DTI_L1_2_2'
]
assert output.DTI_FA_0_0.FIS_007 == 1
assert output.DTI_FA_0_1.FIS_007 == 2
assert output.DTI_FA_0_2.FIS_007 == 3
assert output.DTI_FA_1_0.FIS_007 == 4
assert output.DTI_FA_1_1.FIS_007 == 5
assert output.DTI_FA_1_2.FIS_007 == 6
assert output.DTI_FA_2_0.FIS_007 == 7
assert output.DTI_FA_2_1.FIS_007 == 8
assert output.DTI_FA_2_2.FIS_007 == 9
assert output.DTI_L1_0_0.FIS_007 == 11
assert output.DTI_L1_0_1.FIS_007 == 12
assert output.DTI_L1_0_2.FIS_007 == 13
assert output.DTI_L1_1_0.FIS_007 == 14
assert output.DTI_L1_1_1.FIS_007 == 15
assert output.DTI_L1_1_2.FIS_007 == 16
assert output.DTI_L1_2_0.FIS_007 == 17
assert output.DTI_L1_2_1.FIS_007 == 18
assert output.DTI_L1_2_2.FIS_007 == 19
def test_parameters(self):
profiles = NeuProfiles(profiles_path='path',
format_file='xls2',
profiles_filename='my_file.xls')
assert profiles.data_frame is None
assert profiles.profile_path == 'path'
assert profiles.profiles_filename == 'my_file.xls'
assert profiles.format_file == 'xls2'
def test_format_info(self, datafiles, caplog):
caplog.set_level(logging.INFO)
path = os.path.join(str(datafiles), 'valid')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert len(caplog.records) == 25
for record in caplog.records:
assert record.levelname == 'INFO'
assert 'Reading content in ' in record.message
assert path in record.message
assert 'directory with ' in record.message
def test_without_columns(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
pipe = Pipeline([
('combining', CombineMatrixTransformer(column_name='combined')),
])
output = pipe.fit_transform(profiles.data_frame)
assert sorted(list(output.columns)) == ['DTI_FA', 'DTI_L1']
def test_no_df_simple(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
pipe = Pipeline([
('spread', SpreadOutMatrixTransformer(columns=['0', '1'])),
])
output = pipe.fit_transform(profiles.data_frame.values)
assert output.tolist() == [[14., 17., 18., 4., 7., 8.]] or \
output.tolist() == [[4., 7., 8., 14., 17., 18.]]
def test_inplace_false(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
profiles.load()
old_dataframe = profiles.data_frame
new_dataframe = profiles.spread_out_matrix(['DTI_L1'],
keep_matrix=False,
inplace=False,
symmetric=False)
assert [
'DTI_FA',
'DTI_L1_0_0',
'DTI_L1_0_1',
'DTI_L1_0_2',
'DTI_L1_1_0',
'DTI_L1_1_1',
'DTI_L1_1_2',
'DTI_L1_2_0',
'DTI_L1_2_1',
'DTI_L1_2_2',
] == sorted(list(new_dataframe.columns))
for new_column in [
'DTI_L1_0_0',
'DTI_L1_0_1',
'DTI_L1_0_2',
'DTI_L1_1_0',
'DTI_L1_1_1',
'DTI_L1_1_2',
'DTI_L1_2_0',
'DTI_L1_2_1',
'DTI_L1_2_2',
]:
assert new_column not in profiles.data_frame.columns
assert (old_dataframe == profiles.data_frame).all().all()
def test_no_df_simple(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
pipe = Pipeline([
('spread', SpreadOutMatrixTransformer(columns=['0', '1'])),
('selecting', FeatureMatrixTransformer(
matrix_columns=['0'], columns=None)),
])
output = pipe.fit_transform(profiles.data_frame.values)
assert int(output[0]) in [4, 14]
def test_pipe_column(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
pipe = Pipeline([
('selecting', FeatureMatrixTransformer(
columns=['DTI_FA'], matrix_columns=None)
),
])
output = pipe.fit_transform(profiles.data_frame)
assert sorted(list(output.columns)) == ['DTI_FA', ]
assert (output.DTI_FA == profiles.data_frame.DTI_FA).all()
def test_small(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
assert isinstance(profiles.data_frame, pandas.DataFrame)
assert profiles.data_frame.shape == (1, 2)
assert profiles.data_frame.index.name == 'ID'
assert sorted(list(
profiles.data_frame.columns)) == ['DTI_FA', 'DTI_L1']
assert profiles.data_frame.DTI_FA.FIS_007[0][0] == 1
assert profiles.data_frame.DTI_FA.FIS_007[2][2] == 9
assert profiles.data_frame.DTI_L1.FIS_007[0][0] == 11
assert profiles.data_frame.DTI_L1.FIS_007[2][2] == 19
def test_simple(self, datafiles):
path = os.path.join(str(datafiles), 'valid')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
groups = {
'A': [1],
'B': [2],
'C': [3],
'A-B': [1, 2],
'A-B-C': [1, 2, 3],
'X': [],
'W': []
}
data_frame = profiles.data_frame
df = utils.get_multilabel(data_frame, 'profile', groups)
values = df.as_matrix()
# 3 columns, 63 items
assert values.shape == (63, 3)
assert (values[profiles.data_frame.profile == 'A'] == [1, 0, 0]).all()
assert (values[profiles.data_frame.profile == 'B'] == [0, 1, 0]).all()
assert (values[profiles.data_frame.profile == 'C'] == [0, 0, 1]).all()
assert (values[profiles.data_frame.profile == 'C'] == [0, 0, 1]).all()
assert (values[profiles.data_frame.profile == 'A-B'] == [1, 1, 0]).all()
assert (values[profiles.data_frame.profile == 'A-B-C'] == [1, 1, 1]).all()
assert (values[profiles.data_frame.profile == 'X'] == [0, 0, 0]).all()
assert (df.columns == [1, 2, 3]).all()
assert (df.index == profiles.data_frame.index).all()
def test_valid(self, datafiles):
path = os.path.join(str(datafiles), 'valid')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
assert isinstance(profiles.data_frame, pandas.DataFrame)
assert profiles.data_frame.shape == (63, 8)
assert profiles.data_frame.index.name == 'ID'
assert 'profile' == profiles.data_frame.columns[0]
for column_name in [
'RAW', 'LS', 'DTI_L1', 'DTI_MD', 'DTI_RX', 'DTI_FA', 'FUNC'
]:
assert column_name in profiles.data_frame.columns
first_i = profiles.data_frame[column_name].first_valid_index()
assert profiles.data_frame[column_name][first_i].shape == (76, 76)
def test_default(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
assert profiles.data_frame is None
profiles.load()
assert profiles.data_frame is not None
assert isinstance(profiles.data_frame, pandas.DataFrame)
assert profiles.data_frame.shape == (1, 2)
assert profiles.data_frame.index.name == 'ID'
assert sorted(list(
profiles.data_frame.columns)) == ['DTI_FA', 'DTI_L1']
assert profiles.data_frame.DTI_FA.FIS_007[0][0] == 1
assert profiles.data_frame.DTI_FA.FIS_007[2][2] == 9
assert profiles.data_frame.DTI_L1.FIS_007[0][0] == 11
assert profiles.data_frame.DTI_L1.FIS_007[2][2] == 19
assert profiles.data_frame.DTI_FA.FIS_007[0][0] == 1
assert profiles.data_frame.DTI_FA.FIS_007[2][2] == 9
assert profiles.data_frame.DTI_L1.FIS_007[0][0] == 11
assert profiles.data_frame.DTI_L1.FIS_007[2][2] == 19
profiles.combine_matrix(['DTI_FA', 'DTI_L1'], 'combined')
assert sorted(list(
profiles.data_frame.columns)) == ['DTI_FA', 'DTI_L1', 'combined']
assert (profiles.data_frame.combined.FIS_007 == [[
6,
7,
8,
], [9, 10, 11], [12, 13, 14]]).all()
def test_extract_multiple_keeping(self, datafiles):
path = os.path.join(str(datafiles), 'small_data')
filename = 'profiles.xlsx'
profiles = NeuProfiles(profiles_path=path, profiles_filename=filename)
profiles.load()
new_dataframe = profiles.spread_out_matrix([
'DTI_L1',
'DTI_FA',
],
keep_matrix=True,
symmetric=False)
assert [
'DTI_FA',
'DTI_FA_0_0',
'DTI_FA_0_1',
'DTI_FA_0_2',
'DTI_FA_1_0',
'DTI_FA_1_1',
'DTI_FA_1_2',
'DTI_FA_2_0',
'DTI_FA_2_1',
'DTI_FA_2_2',
'DTI_L1',
'DTI_L1_0_0',
'DTI_L1_0_1',
'DTI_L1_0_2',
'DTI_L1_1_0',
'DTI_L1_1_1',
'DTI_L1_1_2',
'DTI_L1_2_0',
'DTI_L1_2_1',
'DTI_L1_2_2',
] == sorted(list(new_dataframe.columns))
assert new_dataframe.DTI_L1_0_0[0] == 11
assert new_dataframe.DTI_L1_0_1[0] == 12
assert new_dataframe.DTI_L1_0_2[0] == 13
assert new_dataframe.DTI_L1_1_0[0] == 14
assert new_dataframe.DTI_L1_1_1[0] == 15
assert new_dataframe.DTI_L1_1_2[0] == 16
assert new_dataframe.DTI_L1_2_0[0] == 17
assert new_dataframe.DTI_L1_2_1[0] == 18
assert new_dataframe.DTI_L1_2_2[0] == 19
assert new_dataframe.DTI_FA_0_0[0] == 1
assert new_dataframe.DTI_FA_0_1[0] == 2
assert new_dataframe.DTI_FA_0_2[0] == 3
assert new_dataframe.DTI_FA_1_0[0] == 4
assert new_dataframe.DTI_FA_1_1[0] == 5
assert new_dataframe.DTI_FA_1_2[0] == 6
assert new_dataframe.DTI_FA_2_0[0] == 7
assert new_dataframe.DTI_FA_2_1[0] == 8
assert new_dataframe.DTI_FA_2_2[0] == 9
assert profiles.data_frame.DTI_FA.FIS_007[0][0] == 1
assert profiles.data_frame.DTI_FA.FIS_007[2][2] == 9
assert profiles.data_frame.DTI_L1.FIS_007[0][0] == 11
assert profiles.data_frame.DTI_L1.FIS_007[2][2] == 19