def setUpClass(cls):
# *** Once for all, run the export
for ttl_name in test_files:
ttl = TEST_DIR+ttl_name
test_dir = os.path.dirname(ttl)
# If test data is available (usually if the test is run locally)
# then compute a fresh export
with open(os.path.join(test_dir, 'config.json')) as data_file:
metadata = json.load(data_file)
data_dir = os.path.join(TEST_DATA_DIR, metadata["data_dir"])
version = metadata["version"]
# Turtle file obtained with FSL NI-DM export tool
provn = ttl.replace(".ttl", ".provn")
if os.path.isdir(data_dir):
logging.debug("Computing NIDM FSL export")
# Export to NIDM using FSL export tool
# fslnidm = FSL_NIDM(feat_dir=DATA_DIR_001);
fslnidm = FSLtoNIDMExporter(feat_dir=data_dir, version=version)
fslnidm.parse()
export_dir = fslnidm.export()
# Copy provn export to test directory
shutil.copy(os.path.join(export_dir, 'nidm.provn'),
os.path.join(provn))
shutil.copy(os.path.join(export_dir, 'nidm.ttl'),
os.path.join(ttl))
def setUpClass(cls):
# *** Once for all, run the export and convert provn to ttl
# Turtle file obtained with FSL NI-DM export tool
fsl_export_provn = os.path.join(TEST_FOLDER, 'FSL_example.provn');
# If test data is available (usually if the test is run locally) then
# compute a fresh export
if os.path.isdir(DATA_DIR):
logging.debug("Computing NIDM FSL export")
# Export to NIDM using FSL export tool
# fslnidm = FSL_NIDM(feat_dir=DATA_DIR);
fslnidm = FSLtoNIDMExporter(feat_dir=DATA_DIR, version="0.2.0")
fslnidm.parse()
export_dir = fslnidm.export()
# Copy provn export to test directory
shutil.copy(os.path.join(export_dir, 'nidm.provn'),
os.path.join(fsl_export_provn))
# Equivalent turtle file converted using the ProvStore API
# Local file to save the turtle export (and avoid multiple calls to ProvStore)
fsl_export_ttl = os.path.join(TEST_FOLDER, 'FSL_example.ttl');
fsl_export_ttl_url = get_turtle(fsl_export_provn)
ttl_fid = open(fsl_export_ttl, 'w')
ttl_fid.write(urllib2.urlopen(fsl_export_ttl_url).read())
ttl_fid.close()
def testFEAT_NIDM(self):
# test feat parsing
for fname, info in self.testfiles.items():
info['found_feat'] = False
for root, dirs, files in os.walk(info['dir']):
if detect_feat_directory(root):
print 'Found FEAT directory at {}.'.format(root)
info['found_feat'] = True
try:
fslnidm = FSLtoNIDMExporter(feat_dir=root, version="0.2.0")
fslnidm.parse()
export_dir = fslnidm.export()
ttl_file = os.path.join(export_dir,'nidm.ttl')
except:
print("Unable to parse the FEAT directory: \n{0}.".format(get_traceback()))
# confirm results path and existence
self.assertTrue(os.path.exists(export_dir))
self.assertEquals(os.path.join(info['dir'],info['export_dir']),export_dir)
# incomplete ttl = failure in processing
self.assertGreaterEqual(os.path.getsize(ttl_file),info['ttl_fsize'])
# test upload nidm
for fname, info in self.testfiles.items():
nidm_zpath = os.path.join(self.tmpdir,'{}.nidm.zip'.format(fname.replace('.zip','')))
nidm_zip = zipfile.ZipFile(nidm_zpath, 'w')
for root, dirs, files in os.walk(os.path.join(info['dir'],info['export_dir'])):
for nfile in files:
nidm_zip.write(os.path.join(root, nfile))
nidm_zip.close()
zname = os.path.basename(nidm_zip.filename)
post_dict = {
'name': zname,
'description':'{0} upload test'.format(zname),
'collection':self.coll.pk,
}
file_dict = {'zip_file': SimpleUploadedFile(zname, open(nidm_zpath,'r').read())}
form = NIDMResultsForm(post_dict, file_dict)
# validate NIDM Results
self.assertEqual(form.errors, {})
nidm = form.save()
statmaps = nidm.nidmresultstatisticmap_set.all()
self.assertEquals(len(statmaps),info['num_statmaps'])
map_types = [v.map_type for v in statmaps]
self.assertEquals(sorted(map_types), sorted(info['map_types']))
names = [v.name for v in statmaps]
self.assertEquals(sorted(names), sorted(info['names']))
def populate_feat_directory(request,collection,existing_dir=None):
from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter
tmp_dir = tempfile.mkdtemp() if existing_dir is None else existing_dir
exc = ValidationError
try:
if existing_dir is None:
zip = zipfile.ZipFile(request.FILES['file'])
zip.extractall(path=tmp_dir)
rootpaths = [v for v in os.listdir(tmp_dir)
if not v.startswith('.') and not v.startswith('__MACOSX')]
if not rootpaths:
raise exc("No contents found in the FEAT directory.")
subdir = os.path.join(tmp_dir,rootpaths[0])
feat_dir = subdir if len(rootpaths) is 1 and os.path.isdir(subdir) else tmp_dir
except:
raise exc("Unable to unzip the FEAT directory: \n{0}.".format(get_traceback()))
try:
fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0")
fslnidm.parse()
export_dir = fslnidm.export()
except:
raise exc("Unable to parse the FEAT directory: \n{0}.".format(get_traceback()))
if not os.path.exists(export_dir):
raise exc("Unable find nidm export of FEAT directory.")
try:
if existing_dir is not None:
dname = os.path.basename(feat_dir)
suffix = '.nidm.zip' if dname.endswith('feat') else '.feat.nidm.zip'
destname = dname + suffix
else:
destname = request.FILES['file'].name.replace('feat.zip','feat.nidm.zip')
destpath = os.path.join(tmp_dir,destname)
nidm_zip = zipfile.ZipFile(destpath, 'w', zipfile.ZIP_DEFLATED)
rootlen = len(feat_dir) + 1
for root, dirs, files in os.walk(export_dir):
for dfile in files:
filenm = os.path.join(root, dfile)
nidm_zip.write(filenm, filenm[rootlen:])
nidm_zip.close()
fh = open(destpath,'r')
request.FILES['file'] = InMemoryUploadedFile(
ContentFile(fh.read()), "file", fh.name.split('/')[-1],
"application/zip", os.path.getsize(destpath), "utf-8")
except:
raise exc("Unable to convert NIDM results for NeuroVault: \n{0}".format(get_traceback()))
else:
return populate_nidm_results(request,collection)
finally:
shutil.rmtree(tmp_dir)
def populate_feat_directory(request, collection, existing_dir=None):
from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter
tmp_dir = tempfile.mkdtemp() if existing_dir is None else existing_dir
exc = ValidationError
try:
if existing_dir is None:
zip = zipfile.ZipFile(request.FILES['file'])
zip.extractall(path=tmp_dir)
rootpaths = [
v for v in os.listdir(tmp_dir)
if not v.startswith('.') and not v.startswith('__MACOSX')
]
if not rootpaths:
raise exc("No contents found in the FEAT directory.")
subdir = os.path.join(tmp_dir, rootpaths[0])
feat_dir = subdir if len(rootpaths) is 1 and os.path.isdir(
subdir) else tmp_dir
except:
raise exc("Unable to unzip the FEAT directory: \n{0}.".format(
get_traceback()))
try:
fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="1.2.0")
fslnidm.parse()
nidm_file = fslnidm.export()
except:
raise exc("Unable to parse the FEAT directory: \n{0}.".format(
get_traceback()))
if not os.path.exists(nidm_file):
raise exc("Unable find nidm export of FEAT directory.")
try:
fh = open(nidm_file, 'r')
request.FILES['file'] = InMemoryUploadedFile(
ContentFile(fh.read()), "file",
fh.name.split('/')[-1], "application/zip",
os.path.getsize(nidm_file), "utf-8")
except:
raise exc(
"Unable to convert NIDM results for NeuroVault: \n{0}".format(
get_traceback()))
else:
return populate_nidm_results(request, collection)
finally:
shutil.rmtree(tmp_dir)
def populate_feat_directory(request, collection, existing_dir=None):
from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter
tmp_dir = tempfile.mkdtemp() if existing_dir is None else existing_dir
exc = ValidationError
try:
if existing_dir is None:
zip = zipfile.ZipFile(request.FILES["file"])
zip.extractall(path=tmp_dir)
rootpaths = [v for v in os.listdir(tmp_dir) if not v.startswith(".") and not v.startswith("__MACOSX")]
if not rootpaths:
raise exc("No contents found in the FEAT directory.")
subdir = os.path.join(tmp_dir, rootpaths[0])
feat_dir = subdir if len(rootpaths) is 1 and os.path.isdir(subdir) else tmp_dir
except:
raise exc("Unable to unzip the FEAT directory: \n{0}.".format(get_traceback()))
try:
fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="1.2.0")
fslnidm.parse()
nidm_file = fslnidm.export()
except:
raise exc("Unable to parse the FEAT directory: \n{0}.".format(get_traceback()))
if not os.path.exists(nidm_file):
raise exc("Unable find nidm export of FEAT directory.")
try:
fh = open(nidm_file, "r")
request.FILES["file"] = InMemoryUploadedFile(
ContentFile(fh.read()),
"file",
fh.name.split("/")[-1],
"application/zip",
os.path.getsize(nidm_file),
"utf-8",
)
except:
raise exc("Unable to convert NIDM results for NeuroVault: \n{0}".format(get_traceback()))
else:
return populate_nidm_results(request, collection)
finally:
shutil.rmtree(tmp_dir)
def testFEAT_NIDM(self):
# test feat parsing
for fname, info in self.testfiles.items():
info['found_feat'] = False
for root, dirs, files in os.walk(info['dir']):
if detect_feat_directory(root):
print 'Found FEAT directory at {}.'.format(root)
info['found_feat'] = True
fslnidm = FSLtoNIDMExporter(feat_dir=root, version="1.2.0")
fslnidm.parse()
info['nidm_file'] = fslnidm.export()
# confirm results path and existence
self.assertTrue(os.path.exists(info['nidm_file']))
# test upload nidm
for fname, info in self.testfiles.items():
zname = os.path.basename(info['nidm_file'])
post_dict = {
'name': zname,
'description': '{0} upload test'.format(zname),
'collection': self.coll.pk,
}
file_dict = {'zip_file': SimpleUploadedFile(zname, open(info['nidm_file'],'r').read())}
form = NIDMResultsForm(post_dict, file_dict)
# validate NIDM Results
self.assertEqual(form.errors, {})
nidm = form.save()
statmaps = nidm.nidmresultstatisticmap_set.all()
self.assertEquals(len(statmaps),info['num_statmaps'])
map_types = [v.map_type for v in statmaps]
self.assertEquals(sorted(map_types), sorted(info['map_types']))
names = [v.name for v in statmaps]
self.assertEquals(sorted(names), sorted(info['names']))
def testFEAT_NIDM(self):
# test feat parsing
for fname, info in self.testfiles.items():
info['found_feat'] = False
for root, dirs, files in os.walk(info['dir']):
if detect_feat_directory(root):
print 'Found FEAT directory at {}.'.format(root)
info['found_feat'] = True
try:
fslnidm = FSLtoNIDMExporter(feat_dir=root,
version="0.2.0")
fslnidm.parse()
export_dir = fslnidm.export()
ttl_file = os.path.join(export_dir, 'nidm.ttl')
except:
print("Unable to parse the FEAT directory: \n{0}.".
format(get_traceback()))
# confirm results path and existence
self.assertTrue(os.path.exists(export_dir))
self.assertEquals(
os.path.join(info['dir'], info['export_dir']),
export_dir)
# incomplete ttl = failure in processing
self.assertEquals(os.path.getsize(ttl_file),
info['ttl_fsize'])
# test upload nidm
for fname, info in self.testfiles.items():
nidm_zpath = os.path.join(
self.tmpdir, '{}.nidm.zip'.format(fname.replace('.zip', '')))
nidm_zip = zipfile.ZipFile(nidm_zpath, 'w')
for root, dirs, files in os.walk(
os.path.join(info['dir'], info['export_dir'])):
for nfile in files:
nidm_zip.write(os.path.join(root, nfile))
nidm_zip.close()
zname = os.path.basename(nidm_zip.filename)
post_dict = {
'name': zname,
'description': '{0} upload test'.format(zname),
'collection': self.coll.pk,
}
file_dict = {
'zip_file': SimpleUploadedFile(zname,
open(nidm_zpath, 'r').read())
}
form = NIDMResultsForm(post_dict, file_dict)
# validate NIDM Results
self.assertTrue(form.is_valid())
nidm = form.save()
statmaps = nidm.nidmresultstatisticmap_set.all()
self.assertEquals(len(statmaps), info['num_statmaps'])
map_types = [v.map_type for v in statmaps]
self.assertEquals(sorted(map_types), sorted(info['map_types']))
names = [v.name for v in statmaps]
self.assertEquals(sorted(names), sorted(info['names']))
data_dir = os.path.dirname(cfg)
if metadata["software"].lower() == "fsl":
test_name = os.path.basename(data_dir)
versions = metadata["versions"]
for version in versions:
version_str = version.replace(".", "")
if os.path.isdir(data_dir):
logging.debug("Computing NIDM FSL export")
# Export to NIDM using FSL export tool
# fslnidm = FSL_NIDM(feat_dir=DATA_DIR_001);
fslnidm = FSLtoNIDMExporter(
feat_dir=data_dir, version=version, zipped=True)
fslnidm.parse()
zipped_dir = fslnidm.export()
print 'NIDM export available at '+zipped_dir
# Copy provn export to test directory
test_export_dir = os.path.join(
TEST_DIR, 'ex_' + test_name + '_' + version_str)
if not os.path.exists(test_export_dir):
os.makedirs(test_export_dir)
with zipfile.ZipFile(zipped_dir) as z:
z.extract('nidm.ttl', test_export_dir)
z.extract('nidm.provn', test_export_dir)
def populate_feat_directory(request, collection, existing_dir=None):
from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter
tmp_dir = tempfile.mkdtemp() if existing_dir is None else existing_dir
exc = ValidationError
try:
if existing_dir is None:
zip = zipfile.ZipFile(request.FILES['file'])
zip.extractall(path=tmp_dir)
rootpaths = [
v for v in os.listdir(tmp_dir)
if not v.startswith('.') and not v.startswith('__MACOSX')
]
if not rootpaths:
raise exc("No contents found in the FEAT directory.")
subdir = os.path.join(tmp_dir, rootpaths[0])
feat_dir = subdir if len(rootpaths) is 1 and os.path.isdir(
subdir) else tmp_dir
except:
raise exc("Unable to unzip the FEAT directory: \n{0}.".format(
get_traceback()))
try:
fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0")
fslnidm.parse()
export_dir = fslnidm.export()
except:
raise exc("Unable to parse the FEAT directory: \n{0}.".format(
get_traceback()))
if not os.path.exists(export_dir):
raise exc("Unable find nidm export of FEAT directory.")
try:
if existing_dir is not None:
dname = os.path.basename(feat_dir)
suffix = '.nidm.zip' if dname.endswith(
'feat') else '.feat.nidm.zip'
destname = dname + suffix
else:
destname = request.FILES['file'].name.replace(
'feat.zip', 'feat.nidm.zip')
destpath = os.path.join(tmp_dir, destname)
nidm_zip = zipfile.ZipFile(destpath, 'w', zipfile.ZIP_DEFLATED)
rootlen = len(feat_dir) + 1
for root, dirs, files in os.walk(export_dir):
for dfile in files:
filenm = os.path.join(root, dfile)
nidm_zip.write(filenm, filenm[rootlen:])
nidm_zip.close()
fh = open(destpath, 'r')
request.FILES['file'] = InMemoryUploadedFile(ContentFile(fh.read()),
"file",
fh.name.split('/')[-1],
"application/zip",
os.path.getsize(destpath),
"utf-8")
except:
raise exc(
"Unable to convert NIDM results for NeuroVault: \n{0}".format(
get_traceback()))
else:
return populate_nidm_results(request, collection)
finally:
shutil.rmtree(tmp_dir)
if __name__ == '__main__':
feat_dirs_path = raw_input("Enter location of test data:") or '/vagrant/FEAT'
find = 0
parse = 0
fail = 0
for root, dirs, files in os.walk(feat_dirs_path, topdown=False):
if detect_feat_directory(root):
if '.files' in dirs:
call(["rm", "-rf",os.path.join(root,'.files')])
tmpdir = tempfile.mkdtemp()
feat_dir = os.path.join(tmpdir,os.path.basename(root))
shutil.copytree(root, feat_dir)
print 'found feat directory at {0}'.format(root)
print 'testing {0}'.format(feat_dir)
find += 1
try:
fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0")
fslnidm.parse()
export_dir = fslnidm.export()
parse += 1
except:
fail += 1
print("Unable to parse the FEAT directory: \n{0}.".format(get_traceback()))
finally:
print 'ttl length: {}'.format(os.path.getsize(os.path.join(export_dir,'nidm.ttl')))
shutil.rmtree(tmpdir)
print 'found {0} FEAT dirs, {1} successfully processed, {2} failures.'.format(find,parse,fail)
find = 0
parse = 0
fail = 0
for root, dirs, files in os.walk(feat_dirs_path, topdown=False):
if detect_feat_directory(root):
if '.files' in dirs:
call(["rm", "-rf", os.path.join(root, '.files')])
tmpdir = tempfile.mkdtemp()
feat_dir = os.path.join(tmpdir, os.path.basename(root))
shutil.copytree(root, feat_dir)
print 'found feat directory at {0}'.format(root)
print 'testing {0}'.format(feat_dir)
find += 1
try:
fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0")
fslnidm.parse()
export_dir = fslnidm.export()
parse += 1
except:
fail += 1
print("Unable to parse the FEAT directory: \n{0}.".format(
get_traceback()))
finally:
print 'ttl length: {}'.format(
os.path.getsize(os.path.join(export_dir, 'nidm.ttl')))
shutil.rmtree(tmpdir)
print 'found {0} FEAT dirs, {1} successfully processed, {2} failures.'.format(
find, parse, fail)
