def setUpClass(cls): # *** Once for all, run the export and convert provn to ttl # Turtle file obtained with FSL NI-DM export tool fsl_export_provn = os.path.join(TEST_FOLDER, 'FSL_example.provn'); # If test data is available (usually if the test is run locally) then # compute a fresh export if os.path.isdir(DATA_DIR): logging.debug("Computing NIDM FSL export") # Export to NIDM using FSL export tool # fslnidm = FSL_NIDM(feat_dir=DATA_DIR); fslnidm = FSLtoNIDMExporter(feat_dir=DATA_DIR, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() # Copy provn export to test directory shutil.copy(os.path.join(export_dir, 'nidm.provn'), os.path.join(fsl_export_provn)) # Equivalent turtle file converted using the ProvStore API # Local file to save the turtle export (and avoid multiple calls to ProvStore) fsl_export_ttl = os.path.join(TEST_FOLDER, 'FSL_example.ttl'); fsl_export_ttl_url = get_turtle(fsl_export_provn) ttl_fid = open(fsl_export_ttl, 'w') ttl_fid.write(urllib2.urlopen(fsl_export_ttl_url).read()) ttl_fid.close()
def setUpClass(cls): # *** Once for all, run the export for ttl_name in test_files: ttl = TEST_DIR+ttl_name test_dir = os.path.dirname(ttl) # If test data is available (usually if the test is run locally) # then compute a fresh export with open(os.path.join(test_dir, 'config.json')) as data_file: metadata = json.load(data_file) data_dir = os.path.join(TEST_DATA_DIR, metadata["data_dir"]) version = metadata["version"] # Turtle file obtained with FSL NI-DM export tool provn = ttl.replace(".ttl", ".provn") if os.path.isdir(data_dir): logging.debug("Computing NIDM FSL export") # Export to NIDM using FSL export tool # fslnidm = FSL_NIDM(feat_dir=DATA_DIR_001); fslnidm = FSLtoNIDMExporter(feat_dir=data_dir, version=version) fslnidm.parse() export_dir = fslnidm.export() # Copy provn export to test directory shutil.copy(os.path.join(export_dir, 'nidm.provn'), os.path.join(provn)) shutil.copy(os.path.join(export_dir, 'nidm.ttl'), os.path.join(ttl))
def testFEAT_NIDM(self): # test feat parsing for fname, info in self.testfiles.items(): info['found_feat'] = False for root, dirs, files in os.walk(info['dir']): if detect_feat_directory(root): print 'Found FEAT directory at {}.'.format(root) info['found_feat'] = True try: fslnidm = FSLtoNIDMExporter(feat_dir=root, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() ttl_file = os.path.join(export_dir,'nidm.ttl') except: print("Unable to parse the FEAT directory: \n{0}.".format(get_traceback())) # confirm results path and existence self.assertTrue(os.path.exists(export_dir)) self.assertEquals(os.path.join(info['dir'],info['export_dir']),export_dir) # incomplete ttl = failure in processing self.assertGreaterEqual(os.path.getsize(ttl_file),info['ttl_fsize']) # test upload nidm for fname, info in self.testfiles.items(): nidm_zpath = os.path.join(self.tmpdir,'{}.nidm.zip'.format(fname.replace('.zip',''))) nidm_zip = zipfile.ZipFile(nidm_zpath, 'w') for root, dirs, files in os.walk(os.path.join(info['dir'],info['export_dir'])): for nfile in files: nidm_zip.write(os.path.join(root, nfile)) nidm_zip.close() zname = os.path.basename(nidm_zip.filename) post_dict = { 'name': zname, 'description':'{0} upload test'.format(zname), 'collection':self.coll.pk, } file_dict = {'zip_file': SimpleUploadedFile(zname, open(nidm_zpath,'r').read())} form = NIDMResultsForm(post_dict, file_dict) # validate NIDM Results self.assertEqual(form.errors, {}) nidm = form.save() statmaps = nidm.nidmresultstatisticmap_set.all() self.assertEquals(len(statmaps),info['num_statmaps']) map_types = [v.map_type for v in statmaps] self.assertEquals(sorted(map_types), sorted(info['map_types'])) names = [v.name for v in statmaps] self.assertEquals(sorted(names), sorted(info['names']))
def populate_feat_directory(request,collection,existing_dir=None): from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter tmp_dir = tempfile.mkdtemp() if existing_dir is None else existing_dir exc = ValidationError try: if existing_dir is None: zip = zipfile.ZipFile(request.FILES['file']) zip.extractall(path=tmp_dir) rootpaths = [v for v in os.listdir(tmp_dir) if not v.startswith('.') and not v.startswith('__MACOSX')] if not rootpaths: raise exc("No contents found in the FEAT directory.") subdir = os.path.join(tmp_dir,rootpaths[0]) feat_dir = subdir if len(rootpaths) is 1 and os.path.isdir(subdir) else tmp_dir except: raise exc("Unable to unzip the FEAT directory: \n{0}.".format(get_traceback())) try: fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() except: raise exc("Unable to parse the FEAT directory: \n{0}.".format(get_traceback())) if not os.path.exists(export_dir): raise exc("Unable find nidm export of FEAT directory.") try: if existing_dir is not None: dname = os.path.basename(feat_dir) suffix = '.nidm.zip' if dname.endswith('feat') else '.feat.nidm.zip' destname = dname + suffix else: destname = request.FILES['file'].name.replace('feat.zip','feat.nidm.zip') destpath = os.path.join(tmp_dir,destname) nidm_zip = zipfile.ZipFile(destpath, 'w', zipfile.ZIP_DEFLATED) rootlen = len(feat_dir) + 1 for root, dirs, files in os.walk(export_dir): for dfile in files: filenm = os.path.join(root, dfile) nidm_zip.write(filenm, filenm[rootlen:]) nidm_zip.close() fh = open(destpath,'r') request.FILES['file'] = InMemoryUploadedFile( ContentFile(fh.read()), "file", fh.name.split('/')[-1], "application/zip", os.path.getsize(destpath), "utf-8") except: raise exc("Unable to convert NIDM results for NeuroVault: \n{0}".format(get_traceback())) else: return populate_nidm_results(request,collection) finally: shutil.rmtree(tmp_dir)
def populate_feat_directory(request, collection, existing_dir=None): from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter tmp_dir = tempfile.mkdtemp() if existing_dir is None else existing_dir exc = ValidationError try: if existing_dir is None: zip = zipfile.ZipFile(request.FILES['file']) zip.extractall(path=tmp_dir) rootpaths = [ v for v in os.listdir(tmp_dir) if not v.startswith('.') and not v.startswith('__MACOSX') ] if not rootpaths: raise exc("No contents found in the FEAT directory.") subdir = os.path.join(tmp_dir, rootpaths[0]) feat_dir = subdir if len(rootpaths) is 1 and os.path.isdir( subdir) else tmp_dir except: raise exc("Unable to unzip the FEAT directory: \n{0}.".format( get_traceback())) try: fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="1.2.0") fslnidm.parse() nidm_file = fslnidm.export() except: raise exc("Unable to parse the FEAT directory: \n{0}.".format( get_traceback())) if not os.path.exists(nidm_file): raise exc("Unable find nidm export of FEAT directory.") try: fh = open(nidm_file, 'r') request.FILES['file'] = InMemoryUploadedFile( ContentFile(fh.read()), "file", fh.name.split('/')[-1], "application/zip", os.path.getsize(nidm_file), "utf-8") except: raise exc( "Unable to convert NIDM results for NeuroVault: \n{0}".format( get_traceback())) else: return populate_nidm_results(request, collection) finally: shutil.rmtree(tmp_dir)
def populate_feat_directory(request, collection, existing_dir=None): from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter tmp_dir = tempfile.mkdtemp() if existing_dir is None else existing_dir exc = ValidationError try: if existing_dir is None: zip = zipfile.ZipFile(request.FILES["file"]) zip.extractall(path=tmp_dir) rootpaths = [v for v in os.listdir(tmp_dir) if not v.startswith(".") and not v.startswith("__MACOSX")] if not rootpaths: raise exc("No contents found in the FEAT directory.") subdir = os.path.join(tmp_dir, rootpaths[0]) feat_dir = subdir if len(rootpaths) is 1 and os.path.isdir(subdir) else tmp_dir except: raise exc("Unable to unzip the FEAT directory: \n{0}.".format(get_traceback())) try: fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="1.2.0") fslnidm.parse() nidm_file = fslnidm.export() except: raise exc("Unable to parse the FEAT directory: \n{0}.".format(get_traceback())) if not os.path.exists(nidm_file): raise exc("Unable find nidm export of FEAT directory.") try: fh = open(nidm_file, "r") request.FILES["file"] = InMemoryUploadedFile( ContentFile(fh.read()), "file", fh.name.split("/")[-1], "application/zip", os.path.getsize(nidm_file), "utf-8", ) except: raise exc("Unable to convert NIDM results for NeuroVault: \n{0}".format(get_traceback())) else: return populate_nidm_results(request, collection) finally: shutil.rmtree(tmp_dir)
def testFEAT_NIDM(self): # test feat parsing for fname, info in self.testfiles.items(): info['found_feat'] = False for root, dirs, files in os.walk(info['dir']): if detect_feat_directory(root): print 'Found FEAT directory at {}.'.format(root) info['found_feat'] = True fslnidm = FSLtoNIDMExporter(feat_dir=root, version="1.2.0") fslnidm.parse() info['nidm_file'] = fslnidm.export() # confirm results path and existence self.assertTrue(os.path.exists(info['nidm_file'])) # test upload nidm for fname, info in self.testfiles.items(): zname = os.path.basename(info['nidm_file']) post_dict = { 'name': zname, 'description': '{0} upload test'.format(zname), 'collection': self.coll.pk, } file_dict = {'zip_file': SimpleUploadedFile(zname, open(info['nidm_file'],'r').read())} form = NIDMResultsForm(post_dict, file_dict) # validate NIDM Results self.assertEqual(form.errors, {}) nidm = form.save() statmaps = nidm.nidmresultstatisticmap_set.all() self.assertEquals(len(statmaps),info['num_statmaps']) map_types = [v.map_type for v in statmaps] self.assertEquals(sorted(map_types), sorted(info['map_types'])) names = [v.name for v in statmaps] self.assertEquals(sorted(names), sorted(info['names']))
def testFEAT_NIDM(self): # test feat parsing for fname, info in self.testfiles.items(): info['found_feat'] = False for root, dirs, files in os.walk(info['dir']): if detect_feat_directory(root): print 'Found FEAT directory at {}.'.format(root) info['found_feat'] = True try: fslnidm = FSLtoNIDMExporter(feat_dir=root, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() ttl_file = os.path.join(export_dir, 'nidm.ttl') except: print("Unable to parse the FEAT directory: \n{0}.". format(get_traceback())) # confirm results path and existence self.assertTrue(os.path.exists(export_dir)) self.assertEquals( os.path.join(info['dir'], info['export_dir']), export_dir) # incomplete ttl = failure in processing self.assertEquals(os.path.getsize(ttl_file), info['ttl_fsize']) # test upload nidm for fname, info in self.testfiles.items(): nidm_zpath = os.path.join( self.tmpdir, '{}.nidm.zip'.format(fname.replace('.zip', ''))) nidm_zip = zipfile.ZipFile(nidm_zpath, 'w') for root, dirs, files in os.walk( os.path.join(info['dir'], info['export_dir'])): for nfile in files: nidm_zip.write(os.path.join(root, nfile)) nidm_zip.close() zname = os.path.basename(nidm_zip.filename) post_dict = { 'name': zname, 'description': '{0} upload test'.format(zname), 'collection': self.coll.pk, } file_dict = { 'zip_file': SimpleUploadedFile(zname, open(nidm_zpath, 'r').read()) } form = NIDMResultsForm(post_dict, file_dict) # validate NIDM Results self.assertTrue(form.is_valid()) nidm = form.save() statmaps = nidm.nidmresultstatisticmap_set.all() self.assertEquals(len(statmaps), info['num_statmaps']) map_types = [v.map_type for v in statmaps] self.assertEquals(sorted(map_types), sorted(info['map_types'])) names = [v.name for v in statmaps] self.assertEquals(sorted(names), sorted(info['names']))
if metadata["software"].lower() == "fsl": test_name = os.path.basename(data_dir) versions = metadata["versions"] for version in versions: version_str = version.replace(".", "") if os.path.isdir(data_dir): logging.debug("Computing NIDM FSL export") # Export to NIDM using FSL export tool # fslnidm = FSL_NIDM(feat_dir=DATA_DIR_001); fslnidm = FSLtoNIDMExporter( feat_dir=data_dir, version=version, zipped=True) fslnidm.parse() zipped_dir = fslnidm.export() print 'NIDM export available at '+zipped_dir # Copy provn export to test directory test_export_dir = os.path.join( TEST_DIR, 'ex_' + test_name + '_' + version_str) if not os.path.exists(test_export_dir): os.makedirs(test_export_dir) with zipfile.ZipFile(zipped_dir) as z: z.extract('nidm.ttl', test_export_dir) z.extract('nidm.provn', test_export_dir) cfg_file = os.path.join(test_export_dir, 'config.json')
def populate_feat_directory(request, collection, existing_dir=None): from nidmfsl.fsl_exporter.fsl_exporter import FSLtoNIDMExporter tmp_dir = tempfile.mkdtemp() if existing_dir is None else existing_dir exc = ValidationError try: if existing_dir is None: zip = zipfile.ZipFile(request.FILES['file']) zip.extractall(path=tmp_dir) rootpaths = [ v for v in os.listdir(tmp_dir) if not v.startswith('.') and not v.startswith('__MACOSX') ] if not rootpaths: raise exc("No contents found in the FEAT directory.") subdir = os.path.join(tmp_dir, rootpaths[0]) feat_dir = subdir if len(rootpaths) is 1 and os.path.isdir( subdir) else tmp_dir except: raise exc("Unable to unzip the FEAT directory: \n{0}.".format( get_traceback())) try: fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() except: raise exc("Unable to parse the FEAT directory: \n{0}.".format( get_traceback())) if not os.path.exists(export_dir): raise exc("Unable find nidm export of FEAT directory.") try: if existing_dir is not None: dname = os.path.basename(feat_dir) suffix = '.nidm.zip' if dname.endswith( 'feat') else '.feat.nidm.zip' destname = dname + suffix else: destname = request.FILES['file'].name.replace( 'feat.zip', 'feat.nidm.zip') destpath = os.path.join(tmp_dir, destname) nidm_zip = zipfile.ZipFile(destpath, 'w', zipfile.ZIP_DEFLATED) rootlen = len(feat_dir) + 1 for root, dirs, files in os.walk(export_dir): for dfile in files: filenm = os.path.join(root, dfile) nidm_zip.write(filenm, filenm[rootlen:]) nidm_zip.close() fh = open(destpath, 'r') request.FILES['file'] = InMemoryUploadedFile(ContentFile(fh.read()), "file", fh.name.split('/')[-1], "application/zip", os.path.getsize(destpath), "utf-8") except: raise exc( "Unable to convert NIDM results for NeuroVault: \n{0}".format( get_traceback())) else: return populate_nidm_results(request, collection) finally: shutil.rmtree(tmp_dir)
find = 0 parse = 0 fail = 0 for root, dirs, files in os.walk(feat_dirs_path, topdown=False): if detect_feat_directory(root): if '.files' in dirs: call(["rm", "-rf", os.path.join(root, '.files')]) tmpdir = tempfile.mkdtemp() feat_dir = os.path.join(tmpdir, os.path.basename(root)) shutil.copytree(root, feat_dir) print 'found feat directory at {0}'.format(root) print 'testing {0}'.format(feat_dir) find += 1 try: fslnidm = FSLtoNIDMExporter(feat_dir=feat_dir, version="0.2.0") fslnidm.parse() export_dir = fslnidm.export() parse += 1 except: fail += 1 print("Unable to parse the FEAT directory: \n{0}.".format( get_traceback())) finally: print 'ttl length: {}'.format( os.path.getsize(os.path.join(export_dir, 'nidm.ttl'))) shutil.rmtree(tmpdir) print 'found {0} FEAT dirs, {1} successfully processed, {2} failures.'.format( find, parse, fail)