def testFEAT_NIDM(self): # test feat parsing for fname, info in self.testfiles.items(): info['found_feat'] = False for root, dirs, files in os.walk(info['dir']): if detect_feat_directory(root): print 'Found FEAT directory at {}.'.format(root) info['found_feat'] = True try: fslnidm = FSLtoNIDMExporter(feat_dir=root, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() ttl_file = os.path.join(export_dir,'nidm.ttl') except: print("Unable to parse the FEAT directory: \n{0}.".format(get_traceback())) # confirm results path and existence self.assertTrue(os.path.exists(export_dir)) self.assertEquals(os.path.join(info['dir'],info['export_dir']),export_dir) # incomplete ttl = failure in processing self.assertGreaterEqual(os.path.getsize(ttl_file),info['ttl_fsize']) # test upload nidm for fname, info in self.testfiles.items(): nidm_zpath = os.path.join(self.tmpdir,'{}.nidm.zip'.format(fname.replace('.zip',''))) nidm_zip = zipfile.ZipFile(nidm_zpath, 'w') for root, dirs, files in os.walk(os.path.join(info['dir'],info['export_dir'])): for nfile in files: nidm_zip.write(os.path.join(root, nfile)) nidm_zip.close() zname = os.path.basename(nidm_zip.filename) post_dict = { 'name': zname, 'description':'{0} upload test'.format(zname), 'collection':self.coll.pk, } file_dict = {'zip_file': SimpleUploadedFile(zname, open(nidm_zpath,'r').read())} form = NIDMResultsForm(post_dict, file_dict) # validate NIDM Results self.assertEqual(form.errors, {}) nidm = form.save() statmaps = nidm.nidmresultstatisticmap_set.all() self.assertEquals(len(statmaps),info['num_statmaps']) map_types = [v.map_type for v in statmaps] self.assertEquals(sorted(map_types), sorted(info['map_types'])) names = [v.name for v in statmaps] self.assertEquals(sorted(names), sorted(info['names']))
def testFEAT_NIDM(self): # test feat parsing for fname, info in self.testfiles.items(): info['found_feat'] = False for root, dirs, files in os.walk(info['dir']): if detect_feat_directory(root): print 'Found FEAT directory at {}.'.format(root) info['found_feat'] = True fslnidm = FSLtoNIDMExporter(feat_dir=root, version="1.2.0") fslnidm.parse() info['nidm_file'] = fslnidm.export() # confirm results path and existence self.assertTrue(os.path.exists(info['nidm_file'])) # test upload nidm for fname, info in self.testfiles.items(): zname = os.path.basename(info['nidm_file']) post_dict = { 'name': zname, 'description': '{0} upload test'.format(zname), 'collection': self.coll.pk, } file_dict = {'zip_file': SimpleUploadedFile(zname, open(info['nidm_file'],'r').read())} form = NIDMResultsForm(post_dict, file_dict) # validate NIDM Results self.assertEqual(form.errors, {}) nidm = form.save() statmaps = nidm.nidmresultstatisticmap_set.all() self.assertEquals(len(statmaps),info['num_statmaps']) map_types = [v.map_type for v in statmaps] self.assertEquals(sorted(map_types), sorted(info['map_types'])) names = [v.name for v in statmaps] self.assertEquals(sorted(names), sorted(info['names']))
def testFEAT_NIDM(self): # test feat parsing for fname, info in self.testfiles.items(): info['found_feat'] = False for root, dirs, files in os.walk(info['dir']): if detect_feat_directory(root): print 'Found FEAT directory at {}.'.format(root) info['found_feat'] = True try: fslnidm = FSLtoNIDMExporter(feat_dir=root, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() ttl_file = os.path.join(export_dir, 'nidm.ttl') except: print("Unable to parse the FEAT directory: \n{0}.". format(get_traceback())) # confirm results path and existence self.assertTrue(os.path.exists(export_dir)) self.assertEquals( os.path.join(info['dir'], info['export_dir']), export_dir) # incomplete ttl = failure in processing self.assertEquals(os.path.getsize(ttl_file), info['ttl_fsize']) # test upload nidm for fname, info in self.testfiles.items(): nidm_zpath = os.path.join( self.tmpdir, '{}.nidm.zip'.format(fname.replace('.zip', ''))) nidm_zip = zipfile.ZipFile(nidm_zpath, 'w') for root, dirs, files in os.walk( os.path.join(info['dir'], info['export_dir'])): for nfile in files: nidm_zip.write(os.path.join(root, nfile)) nidm_zip.close() zname = os.path.basename(nidm_zip.filename) post_dict = { 'name': zname, 'description': '{0} upload test'.format(zname), 'collection': self.coll.pk, } file_dict = { 'zip_file': SimpleUploadedFile(zname, open(nidm_zpath, 'r').read()) } form = NIDMResultsForm(post_dict, file_dict) # validate NIDM Results self.assertTrue(form.is_valid()) nidm = form.save() statmaps = nidm.nidmresultstatisticmap_set.all() self.assertEquals(len(statmaps), info['num_statmaps']) map_types = [v.map_type for v in statmaps] self.assertEquals(sorted(map_types), sorted(info['map_types'])) names = [v.name for v in statmaps] self.assertEquals(sorted(names), sorted(info['names']))
def upload_folder(request, collection_cid): collection = get_collection(collection_cid, request) allowed_extensions = ['.nii', '.img', '.nii.gz'] niftiFiles = [] if request.method == 'POST': print request.POST print request.FILES form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): tmp_directory = tempfile.mkdtemp() print tmp_directory try: # Save archive (.zip or .tar.gz) to disk if "file" in request.FILES: archive_name = request.FILES['file'].name if fnmatch(archive_name, '*.nidm.zip'): populate_nidm_results(request, collection) return HttpResponseRedirect( collection.get_absolute_url()) _, archive_ext = os.path.splitext(archive_name) if archive_ext == '.zip': compressed = zipfile.ZipFile(request.FILES['file']) elif archive_ext == '.gz': django_file = request.FILES['file'] django_file.open() compressed = tarfile.TarFile(fileobj=gzip.GzipFile( fileobj=django_file.file, mode='r'), mode='r') else: raise Exception("Unsupported archive type %s." % archive_name) compressed.extractall(path=tmp_directory) elif "file_input[]" in request.FILES: for f, path in zip(request.FILES.getlist("file_input[]"), request.POST.getlist("paths[]")): if fnmatch(f.name, '*.nidm.zip'): request.FILES['file'] = f populate_nidm_results(request, collection) continue new_path, _ = os.path.split( os.path.join(tmp_directory, path)) mkdir_p(new_path) filename = os.path.join(new_path, f.name) tmp_file = open(filename, 'w') tmp_file.write(f.read()) tmp_file.close() else: raise Exception("Unable to find uploaded files.") atlases = {} for root, subdirs, filenames in os.walk(tmp_directory): if detect_feat_directory(root): populate_feat_directory(request, collection, root) del (subdirs) filenames = [] # .gfeat parent dir under cope*.feat should not be added as statmaps # this may be affected by future nidm-results_fsl parsing changes if root.endswith('.gfeat'): filenames = [] filenames = [f for f in filenames if not f[0] == '.'] for fname in sorted(filenames): name, ext = splitext_nii_gz(fname) nii_path = os.path.join(root, fname) if ext == '.xml': print "found xml" dom = minidom.parse(os.path.join(root, fname)) for atlas in dom.getElementsByTagName( "summaryimagefile"): print "found atlas" path, base = os.path.split( atlas.lastChild.nodeValue) nifti_name = os.path.join(path, base) atlases[str(os.path.join( root, nifti_name[1:]))] = os.path.join( root, fname) if ext in allowed_extensions: nii = nib.load(nii_path) if detect_afni4D(nii): niftiFiles.extend(split_afni4D_to_3D(nii)) else: niftiFiles.append((fname, nii_path)) for label, fpath in niftiFiles: # Read nifti file information nii = nib.load(fpath) if len(nii.get_shape()) > 3 and nii.get_shape()[3] > 1: print "skipping wrong size" continue hdr = nii.get_header() raw_hdr = hdr.structarr # SPM only !!! # Check if filename corresponds to a T-map Tregexp = re.compile('spmT.*') # Fregexp = re.compile('spmF.*') if Tregexp.search(fpath) is not None: map_type = StatisticMap.T else: # Check if filename corresponds to a F-map if Tregexp.search(fpath) is not None: map_type = StatisticMap.F else: map_type = StatisticMap.OTHER path, name, ext = split_filename(fpath) dname = name + ".nii.gz" spaced_name = name.replace('_', ' ').replace('-', ' ') if ext.lower() != ".nii.gz": new_file_tmp_dir = tempfile.mkdtemp() new_file_tmp = os.path.join(new_file_tmp_dir, name) + '.nii.gz' nib.save(nii, new_file_tmp) f = ContentFile(open(new_file_tmp).read(), name=dname) shutil.rmtree(new_file_tmp_dir) label += " (old ext: %s)" % ext else: f = ContentFile(open(fpath).read(), name=dname) collection = get_collection(collection_cid, request) if os.path.join(path, name) in atlases: new_image = Atlas(name=spaced_name, description=raw_hdr['descrip'], collection=collection) new_image.label_description_file = ContentFile( open(atlases[os.path.join(path, name)]).read(), name=name + ".xml") else: new_image = StatisticMap(name=spaced_name, description=raw_hdr['descrip'] or label, collection=collection) new_image.map_type = map_type new_image.file = f new_image.save() except: raise error = traceback.format_exc().splitlines()[-1] msg = "An error occurred with this upload: {}".format(error) messages.warning(request, msg) return HttpResponseRedirect(collection.get_absolute_url()) finally: shutil.rmtree(tmp_directory) return HttpResponseRedirect(collection.get_absolute_url()) else: form = UploadFileForm() return render_to_response("statmaps/upload_folder.html", {'form': form}, RequestContext(request))
def upload_folder(request, collection_cid): collection = get_collection(collection_cid,request) allowed_extensions = ['.nii', '.img', '.nii.gz'] niftiFiles = [] if request.method == 'POST': print request.POST print request.FILES form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): tmp_directory = tempfile.mkdtemp() print tmp_directory try: # Save archive (.zip or .tar.gz) to disk if "file" in request.FILES: archive_name = request.FILES['file'].name if fnmatch(archive_name,'*.nidm.zip'): form = populate_nidm_results(request,collection) if not form: messages.warning(request, "Invalid NIDM-Results file.") return HttpResponseRedirect(collection.get_absolute_url()) _, archive_ext = os.path.splitext(archive_name) if archive_ext == '.zip': compressed = zipfile.ZipFile(request.FILES['file']) elif archive_ext == '.gz': django_file = request.FILES['file'] django_file.open() compressed = tarfile.TarFile(fileobj=gzip.GzipFile(fileobj=django_file.file, mode='r'), mode='r') else: raise Exception("Unsupported archive type %s."%archive_name) compressed.extractall(path=tmp_directory) elif "file_input[]" in request.FILES: for f, path in zip(request.FILES.getlist( "file_input[]"), request.POST.getlist("paths[]")): if fnmatch(f.name,'*.nidm.zip'): request.FILES['file'] = f populate_nidm_results(request,collection) continue new_path, _ = os.path.split(os.path.join(tmp_directory, path)) mkdir_p(new_path) filename = os.path.join(new_path,f.name) tmp_file = open(filename, 'w') tmp_file.write(f.read()) tmp_file.close() else: raise Exception("Unable to find uploaded files.") atlases = {} for root, subdirs, filenames in os.walk(tmp_directory): if detect_feat_directory(root): populate_feat_directory(request,collection,root) del(subdirs) filenames = [] # .gfeat parent dir under cope*.feat should not be added as statmaps # this may be affected by future nidm-results_fsl parsing changes if root.endswith('.gfeat'): filenames = [] filenames = [f for f in filenames if not f[0] == '.'] for fname in sorted(filenames): name, ext = splitext_nii_gz(fname) nii_path = os.path.join(root, fname) if ext == '.xml': print "found xml" dom = minidom.parse(os.path.join(root, fname)) for atlas in dom.getElementsByTagName("summaryimagefile"): print "found atlas" path, base = os.path.split(atlas.lastChild.nodeValue) nifti_name = os.path.join(path, base) atlases[str(os.path.join(root, nifti_name[1:]))] = os.path.join(root, fname) if ext in allowed_extensions: nii = nib.load(nii_path) if detect_4D(nii): niftiFiles.extend(split_4D_to_3D(nii)) else: niftiFiles.append((fname,nii_path)) for label,fpath in niftiFiles: # Read nifti file information nii = nib.load(fpath) if len(nii.get_shape()) > 3 and nii.get_shape()[3] > 1: messages.warning(request, "Skipping %s - not a 3D file."%label) continue hdr = nii.get_header() raw_hdr = hdr.structarr # SPM only !!! # Check if filename corresponds to a T-map Tregexp = re.compile('spmT.*') # Fregexp = re.compile('spmF.*') if Tregexp.search(fpath) is not None: map_type = StatisticMap.T else: # Check if filename corresponds to a F-map if Tregexp.search(fpath) is not None: map_type = StatisticMap.F else: map_type = StatisticMap.OTHER path, name, ext = split_filename(fpath) dname = name + ".nii.gz" spaced_name = name.replace('_',' ').replace('-',' ') if ext.lower() != ".nii.gz": new_file_tmp_dir = tempfile.mkdtemp() new_file_tmp = os.path.join(new_file_tmp_dir, name) + '.nii.gz' nib.save(nii, new_file_tmp) f = ContentFile(open(new_file_tmp).read(), name=dname) shutil.rmtree(new_file_tmp_dir) label += " (old ext: %s)" % ext else: f = ContentFile(open(fpath).read(), name=dname) collection = get_collection(collection_cid,request) if os.path.join(path, name) in atlases: new_image = Atlas(name=spaced_name, description=raw_hdr['descrip'], collection=collection) new_image.label_description_file = ContentFile( open(atlases[os.path.join(path,name)]).read(), name=name + ".xml") else: new_image = StatisticMap(name=spaced_name, description=raw_hdr['descrip'] or label, collection=collection) new_image.map_type = map_type new_image.file = f new_image.save() except: error = traceback.format_exc().splitlines()[-1] msg = "An error occurred with this upload: {}".format(error) messages.warning(request, msg) return HttpResponseRedirect(collection.get_absolute_url()) finally: shutil.rmtree(tmp_directory) if not niftiFiles: messages.warning(request, "No NIFTI files (.nii, .nii.gz, .img/.hdr) found in the upload.") return HttpResponseRedirect(collection.get_absolute_url()) else: form = UploadFileForm() return render_to_response("statmaps/upload_folder.html", {'form': form}, RequestContext(request))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "neurovault.settings") django.setup() from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter from neurovault.apps.statmaps.utils import detect_feat_directory, get_traceback if __name__ == '__main__': feat_dirs_path = raw_input("Enter location of test data:") or '/vagrant/FEAT' find = 0 parse = 0 fail = 0 for root, dirs, files in os.walk(feat_dirs_path, topdown=False): if detect_feat_directory(root): if '.files' in dirs: call(["rm", "-rf",os.path.join(root,'.files')]) tmpdir = tempfile.mkdtemp() feat_dir = os.path.join(tmpdir,os.path.basename(root)) shutil.copytree(root, feat_dir) print 'found feat directory at {0}'.format(root) print 'testing {0}'.format(feat_dir) find += 1 try: fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() parse += 1 except: fail += 1
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "neurovault.settings") django.setup() from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter from neurovault.apps.statmaps.utils import detect_feat_directory, get_traceback if __name__ == '__main__': feat_dirs_path = raw_input( "Enter location of test data:") or '/vagrant/FEAT' find = 0 parse = 0 fail = 0 for root, dirs, files in os.walk(feat_dirs_path, topdown=False): if detect_feat_directory(root): if '.files' in dirs: call(["rm", "-rf", os.path.join(root, '.files')]) tmpdir = tempfile.mkdtemp() feat_dir = os.path.join(tmpdir, os.path.basename(root)) shutil.copytree(root, feat_dir) print 'found feat directory at {0}'.format(root) print 'testing {0}'.format(feat_dir) find += 1 try: fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() parse += 1 except: fail += 1