def test_fetch_cobre(tmp_path, request_mocker):
metadata, ids = _cobre_metadata()
request_mocker.url_mapping["*phenotypic_data.tsv.gz"] = _cobre_data(ids)
request_mocker.url_mapping["*articles/4197885"] = metadata
# All subjects
cobre_data = check_deprecation(func.fetch_cobre,
"'fetch_cobre' has been deprecated")(
n_subjects=None, data_dir=str(tmp_path))
phenotypic_names = [
'func', 'confounds', 'phenotypic', 'description', 'desc_con',
'desc_phenotypic'
]
# test length of functional filenames to max 146
assert len(cobre_data.func) == 146
# test length of corresponding confounds files of same length to max 146
assert len(cobre_data.confounds) == 146
# test return type variables
assert sorted(cobre_data) == sorted(phenotypic_names)
# test functional filenames in a list
assert isinstance(cobre_data.func, list)
# test confounds files in a list
assert isinstance(cobre_data.confounds, list)
assert isinstance(cobre_data.func[0], str)
# returned phenotypic data will be an array
assert isinstance(cobre_data.phenotypic, np.recarray)
# Fetch only 30 subjects
data_30_subjects = func.fetch_cobre(n_subjects=30, data_dir=str(tmp_path))
assert len(data_30_subjects.func) == 30
assert len(data_30_subjects.confounds) == 30
# Test more than maximum subjects
test_150_subjects = func.fetch_cobre(n_subjects=150,
data_dir=str(tmp_path))
assert len(test_150_subjects.func) == 146
def test_fetch_cobre():
ids_n = [40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008,
40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017,
40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026,
40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035,
40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044,
40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053,
40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062,
40063, 40064, 40065, 40066, 40067, 40068, 40069, 40071, 40072,
40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, 40081,
40082, 40084, 40085, 40086, 40087, 40088, 40089, 40090, 40091,
40092, 40093, 40094, 40095, 40096, 40097, 40098, 40099, 40100,
40101, 40102, 40103, 40104, 40105, 40106, 40107, 40108, 40109,
40110, 40111, 40112, 40113, 40114, 40115, 40116, 40117, 40118,
40119, 40120, 40121, 40122, 40123, 40124, 40125, 40126, 40127,
40128, 40129, 40130, 40131, 40132, 40133, 40134, 40135, 40136,
40137, 40138, 40139, 40140, 40141, 40142, 40143, 40144, 40145,
40146, 40147]
ids = np.asarray(ids_n, dtype='|U17')
current_age = np.ones(len(ids), dtype='<f8')
gender = np.ones(len(ids), dtype='<f8')
handedness = np.ones(len(ids), dtype='<f8')
subject_type = np.empty(len(ids), dtype="S10")
subject_type[0:74] = 'Control'
subject_type[74:146] = 'Patient'
diagnosis = np.ones(len(ids), dtype='<f8')
frames_ok = np.ones(len(ids), dtype='<f8')
fd = np.ones(len(ids), dtype='<f8')
fd_scrubbed = np.ones(len(ids), dtype='<f8')
csv = np.rec.array([ids, current_age, gender, handedness, subject_type,
diagnosis, frames_ok, fd, fd_scrubbed],
dtype=[('ID', '|U17'), ('Current Age', '<f8'),
('Gender', '<f8'), ('Handedness', '<f8'),
('Subject Type', '|U17'), ('Diagnosis', '<f8'),
('Frames OK', '<f8'), ('FD', '<f8'),
('FD Scrubbed', '<f8')])
# Create a dummy 'files'
cobre_dir = os.path.join(tst.tmpdir, 'cobre')
os.mkdir(cobre_dir)
# Create the tsv
name_f = os.path.join(cobre_dir, 'phenotypic_data.tsv')
with open(name_f, 'wb') as f:
header = '# {0}\n'.format('\t'.join(csv.dtype.names))
f.write(header.encode())
np.savetxt(f, csv, delimiter='\t', fmt='%s')
# create an empty gz file
f_in = open(name_f)
name_f_gz = os.path.join(cobre_dir, 'phenotypic_data.tsv.gz')
f_out = gzip.open(name_f_gz, 'wb')
f_out.close()
f_in.close()
dummy = os.path.join(cobre_dir, '4197885')
dummy_data = []
for i in np.hstack(ids_n):
# Func file
f = 'fmri_00' + str(i) + '.nii.gz'
m = 'fmri_00' + str(i) + '.tsv.gz'
dummy_data.append({'download_url': 'whatever', 'name': f})
dummy_data.append({'download_url': 'whatever', 'name': m})
# Add the TSV file
dummy_data.append({
'download_url': 'whatever', 'name': 'phenotypic_data.tsv.gz'})
# Add JSON files
dummy_data.append({
'download_url': 'whatever', 'name': 'keys_confounds.json'})
dummy_data.append({
'download_url': 'whatever', 'name': 'keys_phenotypic_data.json'})
dummy_data = {'files': dummy_data}
json.dump(dummy_data, open(dummy, 'w'))
local_url = "file://" + dummy
# All subjects
cobre_data = func.fetch_cobre(n_subjects=None, data_dir=tst.tmpdir,
url=local_url)
phenotypic_names = ['func', 'confounds', 'phenotypic', 'description',
'desc_con', 'desc_phenotypic']
# test length of functional filenames to max 146
assert_equal(len(cobre_data.func), 146)
# test length of corresponding confounds files of same length to max 146
assert_equal(len(cobre_data.confounds), 146)
# test return type variables
assert_equal(sorted(cobre_data), sorted(phenotypic_names))
# test functional filenames in a list
assert_true(isinstance(cobre_data.func, list))
# test confounds files in a list
assert_true(isinstance(cobre_data.confounds, list))
assert_true(isinstance(cobre_data.func[0], _basestring))
# returned phenotypic data will be an array
assert_true(isinstance(cobre_data.phenotypic, np.recarray))
# Fetch only 30 subjects
data_30_subjects = func.fetch_cobre(n_subjects=30, url=local_url,
data_dir=tst.tmpdir)
assert_equal(len(data_30_subjects.func), 30)
assert_equal(len(data_30_subjects.confounds), 30)
# Test more than maximum subjects
test_150_subjects = func.fetch_cobre(n_subjects=150, url=local_url,
data_dir=tst.tmpdir)
assert_equal(len(test_150_subjects.func), 146)
os.remove(dummy)
def test_fetch_cobre():
ids_sc = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 16, 21, 22, 25, 28, 29,
32, 34, 37, 39, 40, 41, 42, 44, 46, 47, 49, 59, 60, 64, 71, 72, 73, 75,
77, 78, 79, 80, 81, 82, 84, 85, 88, 89, 92, 94, 96, 97, 98, 99, 100,
101, 103, 105, 106, 108, 109, 110, 112, 117, 122, 126, 132, 133, 137,
142, 143, 145
]
ids_con = [
13, 14, 17, 18, 19, 20, 23, 24, 26, 27, 30, 31, 33, 35, 36, 38, 43, 45,
48, 50, 51, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 65, 66, 67, 68, 69,
74, 76, 86, 87, 90, 91, 93, 95, 102, 104, 107, 111, 113, 114, 115, 116,
118, 119, 120, 121, 123, 124, 125, 127, 128, 129, 130, 131, 134, 135,
136, 138, 139, 140, 141, 144, 146, 147
]
ids_sch = ['szxxx0040%03d' % i for i in ids_sc]
ids_cont = ['contxxx0040%03d' % i for i in ids_con]
ids = np.asarray(ids_sch + ids_cont, dtype='|U17')
sz = np.asarray([i.startswith('s') for i in ids], dtype='<f8')
age = np.ones(len(ids), dtype='<f8')
sex = np.ones(len(ids), dtype='<f8')
fd = np.ones(len(ids), dtype='<f8')
csv = np.rec.array([ids, sz, age, sex, fd],
dtype=[('id', '|U17'), ('sz', '<f8'), ('age', '<f8'),
('sex', '<f8'), ('fd', '<f8')])
tst.mock_fetch_files.add_csv('cobre_model_group.csv', csv)
# Create a dummy 'files'
cobre_dir = os.path.join(tst.tmpdir, 'cobre')
os.mkdir(cobre_dir)
dummy = os.path.join(cobre_dir, 'files')
dummy_data = []
for i in np.hstack([ids_sch, ids_cont]):
# Func file
f = 'fmri_' + i + '_session1_run1.nii.gz'
m = 'fmri_' + i + '_session1_run1_extra.mat'
dummy_data.append({'downloadUrl': 'whatever', 'name': f})
dummy_data.append({'downloadUrl': 'whatever', 'name': m})
# Add the CSV file
dummy_data.append({
'downloadUrl': 'whatever',
'name': 'cobre_model_group.csv'
})
json.dump(dummy_data, open(dummy, 'w'))
local_url = "file://" + dummy
# All subjects
cobre_data = func.fetch_cobre(n_subjects=None,
data_dir=tst.tmpdir,
url=local_url)
phenotypic_names = ['description', 'func', 'mat_files', 'phenotypic']
# test length of functional filenames to max 146
assert_equal(len(cobre_data.func), 146)
# test length of corresponding matlab files of same length to max 146
assert_equal(len(cobre_data.mat_files), 146)
# test return type variables
assert_equal(sorted(cobre_data), phenotypic_names)
# test functional filenames in a list
assert_true(isinstance(cobre_data.func, list))
# test matlab files in a list
assert_true(isinstance(cobre_data.mat_files, list))
assert_true(isinstance(cobre_data.func[0], _basestring))
# returned phenotypic data will be an array
assert_true(isinstance(cobre_data.phenotypic, np.recarray))
# data description should not be empty
assert_not_equal(cobre_data.description, '')
# Fetch only 30 subjects
data_30_subjects = func.fetch_cobre(n_subjects=30,
url=local_url,
data_dir=tst.tmpdir)
assert_equal(len(data_30_subjects.func), 30)
assert_equal(len(data_30_subjects.mat_files), 30)
# Test more than maximum subjects
test_150_subjects = func.fetch_cobre(n_subjects=150,
url=local_url,
data_dir=tst.tmpdir)
assert_equal(len(test_150_subjects.func), 146)
os.remove(dummy)
def test_fetch_cobre():
ids_n = [40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008,
40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017,
40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026,
40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035,
40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044,
40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053,
40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062,
40063, 40064, 40065, 40066, 40067, 40068, 40069, 40071, 40072,
40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, 40081,
40082, 40084, 40085, 40086, 40087, 40088, 40089, 40090, 40091,
40092, 40093, 40094, 40095, 40096, 40097, 40098, 40099, 40100,
40101, 40102, 40103, 40104, 40105, 40106, 40107, 40108, 40109,
40110, 40111, 40112, 40113, 40114, 40115, 40116, 40117, 40118,
40119, 40120, 40121, 40122, 40123, 40124, 40125, 40126, 40127,
40128, 40129, 40130, 40131, 40132, 40133, 40134, 40135, 40136,
40137, 40138, 40139, 40140, 40141, 40142, 40143, 40144, 40145,
40146, 40147]
ids = np.asarray(ids_n, dtype='|U17')
current_age = np.ones(len(ids), dtype='<f8')
gender = np.ones(len(ids), dtype='<f8')
handedness = np.ones(len(ids), dtype='<f8')
subject_type = np.empty(len(ids), dtype="S10")
subject_type[0:74] = 'Control'
subject_type[74:146] = 'Patient'
diagnosis = np.ones(len(ids), dtype='<f8')
frames_ok = np.ones(len(ids), dtype='<f8')
fd = np.ones(len(ids), dtype='<f8')
fd_scrubbed = np.ones(len(ids), dtype='<f8')
csv = np.rec.array([ids, current_age, gender, handedness, subject_type,
diagnosis, frames_ok, fd, fd_scrubbed],
dtype=[('ID', '|U17'), ('Current Age', '<f8'),
('Gender', '<f8'), ('Handedness', '<f8'),
('Subject Type', '|U17'), ('Diagnosis', '<f8'),
('Frames OK', '<f8'), ('FD', '<f8'),
('FD Scrubbed', '<f8')])
# Create a dummy 'files'
cobre_dir = os.path.join(tst.tmpdir, 'cobre')
os.mkdir(cobre_dir)
# Create the tsv
name_f = os.path.join(cobre_dir, 'phenotypic_data.tsv')
with open(name_f, 'wb') as f:
header = '# {0}\n'.format('\t'.join(csv.dtype.names))
f.write(header.encode())
np.savetxt(f, csv, delimiter='\t', fmt='%s')
# create an empty gz file
f_in = open(name_f)
name_f_gz = os.path.join(cobre_dir, 'phenotypic_data.tsv.gz')
f_out = gzip.open(name_f_gz, 'wb')
f_out.close()
f_in.close()
dummy = os.path.join(cobre_dir, '4197885')
dummy_data = []
for i in np.hstack(ids_n):
# Func file
f = 'fmri_00' + str(i) + '.nii.gz'
m = 'fmri_00' + str(i) + '.tsv.gz'
dummy_data.append({'download_url': 'whatever', 'name': f})
dummy_data.append({'download_url': 'whatever', 'name': m})
# Add the TSV file
dummy_data.append({
'download_url': 'whatever', 'name': 'phenotypic_data.tsv.gz'})
# Add JSON files
dummy_data.append({
'download_url': 'whatever', 'name': 'keys_confounds.json'})
dummy_data.append({
'download_url': 'whatever', 'name': 'keys_phenotypic_data.json'})
dummy_data = {'files': dummy_data}
json.dump(dummy_data, open(dummy, 'w'))
local_url = "file://" + dummy
# All subjects
cobre_data = func.fetch_cobre(n_subjects=None, data_dir=tst.tmpdir,
url=local_url)
phenotypic_names = ['func', 'confounds', 'phenotypic', 'description',
'desc_con', 'desc_phenotypic']
# test length of functional filenames to max 146
assert_equal(len(cobre_data.func), 146)
# test length of corresponding confounds files of same length to max 146
assert_equal(len(cobre_data.confounds), 146)
# test return type variables
assert_equal(sorted(cobre_data), sorted(phenotypic_names))
# test functional filenames in a list
assert_true(isinstance(cobre_data.func, list))
# test confounds files in a list
assert_true(isinstance(cobre_data.confounds, list))
assert_true(isinstance(cobre_data.func[0], _basestring))
# returned phenotypic data will be an array
assert_true(isinstance(cobre_data.phenotypic, np.recarray))
# Fetch only 30 subjects
data_30_subjects = func.fetch_cobre(n_subjects=30, url=local_url,
data_dir=tst.tmpdir)
assert_equal(len(data_30_subjects.func), 30)
assert_equal(len(data_30_subjects.confounds), 30)
# Test more than maximum subjects
test_150_subjects = func.fetch_cobre(n_subjects=150, url=local_url,
data_dir=tst.tmpdir)
assert_equal(len(test_150_subjects.func), 146)
os.remove(dummy)
def test_fetch_cobre():
ids_sc = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 16, 21, 22, 25,
28, 29, 32, 34, 37, 39, 40, 41, 42, 44, 46, 47, 49, 59, 60,
64, 71, 72, 73, 75, 77, 78, 79, 80, 81, 82, 84, 85, 88, 89,
92, 94, 96, 97, 98, 99, 100, 101, 103, 105, 106, 108, 109, 110,
112, 117, 122, 126, 132, 133, 137, 142, 143, 145]
ids_con = [13, 14, 17, 18, 19, 20, 23, 24, 26, 27, 30, 31, 33, 35, 36,
38, 43, 45, 48, 50, 51, 52, 53, 54, 55, 56, 57, 58, 61, 62,
63, 65, 66, 67, 68, 69, 74, 76, 86, 87, 90, 91, 93, 95, 102,
104, 107, 111, 113, 114, 115, 116, 118, 119, 120, 121, 123,
124, 125, 127, 128, 129, 130, 131, 134, 135, 136, 138, 139,
140, 141, 144, 146, 147]
ids_sch = ['szxxx0040%03d' % i for i in ids_sc]
ids_cont = ['contxxx0040%03d' % i for i in ids_con]
ids = np.asarray(ids_sch + ids_cont, dtype='|U17')
sz = np.asarray([i.startswith('s') for i in ids], dtype='<f8')
age = np.ones(len(ids), dtype='<f8')
sex = np.ones(len(ids), dtype='<f8')
fd = np.ones(len(ids), dtype='<f8')
csv = np.rec.array([ids, sz, age, sex, fd],
dtype=[('id', '|U17'), ('sz', '<f8'),
('age', '<f8'), ('sex', '<f8'),
('fd', '<f8')])
tst.mock_fetch_files.add_csv('cobre_model_group.csv', csv)
# Create a dummy 'files'
cobre_dir = os.path.join(tst.tmpdir, 'cobre')
os.mkdir(cobre_dir)
dummy = os.path.join(cobre_dir, 'files')
dummy_data = []
for i in np.hstack([ids_sch, ids_cont]):
# Func file
f = 'fmri_' + i + '_session1_run1.nii.gz'
m = 'fmri_' + i + '_session1_run1_extra.mat'
dummy_data.append({'downloadUrl': 'whatever', 'name': f})
dummy_data.append({'downloadUrl': 'whatever', 'name': m})
# Add the CSV file
dummy_data.append({
'downloadUrl': 'whatever', 'name': 'cobre_model_group.csv'})
json.dump(dummy_data, open(dummy, 'w'))
local_url = "file://" + dummy
# All subjects
cobre_data = func.fetch_cobre(n_subjects=None, data_dir=tst.tmpdir,
url=local_url)
phenotypic_names = ['description', 'func', 'mat_files', 'phenotypic']
# test length of functional filenames to max 146
assert_equal(len(cobre_data.func), 146)
# test length of corresponding matlab files of same length to max 146
assert_equal(len(cobre_data.mat_files), 146)
# test return type variables
assert_equal(sorted(cobre_data), phenotypic_names)
# test functional filenames in a list
assert_true(isinstance(cobre_data.func, list))
# test matlab files in a list
assert_true(isinstance(cobre_data.mat_files, list))
assert_true(isinstance(cobre_data.func[0], _basestring))
# returned phenotypic data will be an array
assert_true(isinstance(cobre_data.phenotypic, np.recarray))
# data description should not be empty
assert_not_equal(cobre_data.description, '')
# Fetch only 30 subjects
data_30_subjects = func.fetch_cobre(n_subjects=30, url=local_url,
data_dir=tst.tmpdir)
assert_equal(len(data_30_subjects.func), 30)
assert_equal(len(data_30_subjects.mat_files), 30)
# Test more than maximum subjects
test_150_subjects = func.fetch_cobre(n_subjects=150, url=local_url,
data_dir=tst.tmpdir)
assert_equal(len(test_150_subjects.func), 146)
os.remove(dummy)
