def test_statmaps_processing(self): # The counter is the count of the number of images with the field "transform" set to None # The field is populated with the file when image comparisons are done, meaning that if there is only one # image in the database (case below) we cannot calculate comparisons, and the "transform" field remains none # This is currently the only way that we can test the counter, which will be "1" in this case print "\nTesting Counter - added statistic maps ###" Image1 = StatisticMap(name='Image1', collection=self.Collection1, file='motor_lips.nii.gz', map_type="Z", analysis_level='G', number_of_subjects=10) Image1.file = SimpleUploadedFile('motor_lips.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/motor_lips.nii.gz')).read()) Image1.save() images_processing = count_processing_comparisons(Image1.pk) print "%s images processing [should be 0]" %(images_processing) self.assertEqual(images_processing,0) # When we add an image, the comparison will be calculated with image1, and both images transform fields will be populated # the counter will be set to 0. Celery runs in synchronous mode when testing (meaning that jobs are run locally, one # after the other, instead of being sent to worker nodes) so there is no way to test submitting a batch of async # jobs and watching the "images still processing" counter go from N to 0. There is also no way of arbitrarily # setting an image transform field to "None" because on save, all image comparisons are automatically re-calcualted Image2 = StatisticMap(name='Image2', collection=self.Collection2, file='beta_0001.nii.gz', map_type="Other", analysis_level='G', number_of_subjects=10) Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read()) Image2.save() images_processing = count_processing_comparisons(Image1.pk) print "%s images processing [should be 0]" %(images_processing) self.assertEqual(images_processing,0) # We should have 2 images total, so 1 comparison total_comparisons = count_existing_comparisons(Image1.pk) self.assertEqual(total_comparisons,1)
def test_adding_nidm(self): Image2 = StatisticMap(name='Image2', collection=self.Collection1, file='beta_0001.nii.gz', map_type="Other") Image2.file = SimpleUploadedFile( 'beta_0001.nii.gz', file( os.path.join(self.test_path, 'test_data/statmaps/beta_0001.nii.gz')).read()) Image2.save() zip_file = open( os.path.join(self.test_path, 'test_data/nidm/spm_example.nidm.zip'), 'rb') post_dict = { 'name': 'spm_nidm', 'description': '{0} upload test'.format('spm_example'), 'collection': self.Collection2.pk } fname = os.path.basename( os.path.join(self.test_path, 'test_data/nidm/spm_example.nidm.zip')) file_dict = {'zip_file': SimpleUploadedFile(fname, zip_file.read())} zip_file.close() form = NIDMResultsForm(post_dict, file_dict) # Transforms should be generated synchronously nidm = form.save() print "\nTesting Counter - added nidm result ###" # We should have 2 images total, so 1 comparison total_comparisons = count_existing_comparisons(Image2.pk) self.assertEqual(total_comparisons, 1) #Let's add a single subject map - this should not trigger a comparison Image2ss = StatisticMap(name='Image2 - single subject', collection=self.Collection3, file='beta_0001.nii.gz', map_type="Other", analysis_level='S') Image2ss.file = SimpleUploadedFile( 'beta_0001.nii.gz', file( os.path.join(self.test_path, 'test_data/statmaps/beta_0001.nii.gz')).read()) Image2ss.save() total_comparisons = count_existing_comparisons(Image2ss.pk) self.assertEqual(total_comparisons, 0) # Make sure comparisons were calculated number_comparisons = len(Comparison.objects.all()) print "\n %s comparisons exist after adding NIDM `[should not be 0]" % ( number_comparisons) self.assertEqual(number_comparisons > 0, True)
def add_image(request, collection_cid): collection = get_collection(collection_cid, request) image = StatisticMap(collection=collection) if request.method == "POST": form = StatisticMapForm(request.POST, request.FILES, instance=image) if form.is_valid(): image = form.save() return HttpResponseRedirect(image.get_absolute_url()) else: form = StatisticMapForm(instance=image) context = {"form": form} return render(request, "statmaps/add_image.html.haml", context)
def add_image(request, collection_cid): collection = get_collection(collection_cid,request) image = StatisticMap(collection=collection) if request.method == "POST": form = StatisticMapForm(request.POST, request.FILES, instance=image) if form.is_valid(): image = form.save() return HttpResponseRedirect(image.get_absolute_url()) else: form = StatisticMapForm(instance=image) context = {"form": form} return render(request, "statmaps/add_image.html.haml", context)
def test_adding_nidm(self): Image2 = StatisticMap(name='Image2', collection=self.Collection1, file='beta_0001.nii.gz', map_type="Other") Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read()) Image2.save() zip_file = open(os.path.join(self.test_path,'test_data/nidm/spm_example.nidm.zip'), 'rb') post_dict = { 'name': 'spm_nidm', 'description':'{0} upload test'.format('spm_example'), 'collection':self.Collection2.pk} fname = os.path.basename(os.path.join(self.test_path,'test_data/nidm/spm_example.nidm.zip')) file_dict = {'zip_file': SimpleUploadedFile(fname, zip_file.read())} zip_file.close() form = NIDMResultsForm(post_dict, file_dict) # Transforms should be generated synchronously nidm = form.save() print "\nTesting Counter - added nidm result ###" # We should have 2 images total, so 1 comparison total_comparisons = count_existing_comparisons(Image2.pk) self.assertEqual(total_comparisons,1) #Let's add a single subject map - this should not trigger a comparison Image2ss = StatisticMap(name='Image2 - single subject', collection=self.Collection3, file='beta_0001.nii.gz', map_type="Other", analysis_level='S') Image2ss.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read()) Image2ss.save() total_comparisons = count_existing_comparisons(Image2ss.pk) self.assertEqual(total_comparisons,0) # Make sure comparisons were calculated number_comparisons = len(Comparison.objects.all()) print "\n %s comparisons exist after adding NIDM `[should not be 0]" %(number_comparisons) self.assertEqual(number_comparisons>0,True)
def test_statmaps_processing(self): # The counter is the count of the number of images with the field "transform" set to None # The field is populated with the file when image comparisons are done, meaning that if there is only one # image in the database (case below) we cannot calculate comparisons, and the "transform" field remains none # This is currently the only way that we can test the counter, which will be "1" in this case print "\nTesting Counter - added statistic maps ###" Image1 = StatisticMap(name='Image1', collection=self.Collection1, file='motor_lips.nii.gz', map_type="Z") Image1.file = SimpleUploadedFile('motor_lips.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/motor_lips.nii.gz')).read()) Image1.save() images_processing = count_processing_comparisons(Image1.pk) print "%s images processing [should be 0]" %(images_processing) self.assertEqual(images_processing,0) # When we add an image, the comparison will be calculated with image1, and both images transform fields will be populated # the counter will be set to 0. Celery runs in synchronous mode when testing (meaning that jobs are run locally, one # after the other, instead of being sent to worker nodes) so there is no way to test submitting a batch of async # jobs and watching the "images still processing" counter go from N to 0. There is also no way of arbitrarily # setting an image transform field to "None" because on save, all image comparisons are automatically re-calcualted Image2 = StatisticMap(name='Image2', collection=self.Collection2, file='beta_0001.nii.gz', map_type="Other") Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read()) Image2.save() images_processing = count_processing_comparisons(Image1.pk) print "%s images processing [should be 0]" %(images_processing) self.assertEqual(images_processing,0) # We should have 2 images total, so 1 comparison total_comparisons = count_existing_comparisons(Image1.pk) self.assertEqual(total_comparisons,1)
def add_image_for_neurosynth(request): temp_collection_name = "%s's temporary collection" % request.user.username #this is a hack we need to make sure this collection can be only #owned by the same user try: temp_collection = Collection.objects.get(name=temp_collection_name) except Collection.DoesNotExist: priv_token = generate_url_token() temp_collection = Collection(name=temp_collection_name, owner=request.user, private=False, private_token=priv_token) temp_collection.save() image = StatisticMap(collection=temp_collection) if request.method == "POST": form = SimplifiedStatisticMapForm(request.POST, request.FILES, instance=image, user=request.user) if form.is_valid(): image = form.save() return HttpResponseRedirect( "http://neurosynth.org/decode/?neurovault=%s-%s" % (temp_collection.private_token, image.id)) else: form = SimplifiedStatisticMapForm(user=request.user, instance=image) context = {"form": form} return render(request, "statmaps/add_image_for_neurosynth.html.haml", context)
def add_image(request, collection_cid): collection = get_collection(collection_cid,request) if not owner_or_contrib(request,collection): return HttpResponseForbidden() image = StatisticMap(collection=collection) if request.method == "POST": form = AddStatisticMapForm(request.POST, request.FILES, instance=image) if form.is_valid(): image = form.save() return HttpResponseRedirect(image.get_absolute_url()) else: form = AddStatisticMapForm(instance=image) contrasts = get_contrast_lookup() context = {"form": form,"contrasts": json.dumps(contrasts)} return render(request, "statmaps/add_image.html", context)
def edit_images(request, collection_cid): collection = get_collection(collection_cid, request) if not owner_or_contrib(request, collection): return HttpResponseForbidden() if request.method == "POST": formset = CollectionFormSet(request.POST, request.FILES, instance=collection) for n, form in enumerate(formset): # hack: check fields to determine polymorphic type if form.instance.polymorphic_ctype is None: atlas_f = 'image_set-{0}-label_description_file'.format(n) has_atlas = [v for v in form.files if v == atlas_f] if has_atlas: use_model = Atlas use_form = AtlasForm else: use_model = StatisticMap use_form = StatisticMapForm form.instance = use_model(collection=collection) form.base_fields = use_form.base_fields form.fields = use_form.base_fields if formset.is_valid(): formset.save() return HttpResponseRedirect(collection.get_absolute_url()) else: formset = CollectionFormSet(instance=collection) blank_statmap = StatisticMapForm(instance=StatisticMap( collection=collection)) blank_atlas = AtlasForm(instance=Atlas(collection=collection)) upload_form = UploadFileForm() context = { "formset": formset, "blank_statmap": blank_statmap, "blank_atlas": blank_atlas, "upload_form": upload_form } return render(request, "statmaps/edit_images.html", context)
def upload_folder(request, collection_cid): collection = get_collection(collection_cid, request) allowed_extensions = ['.nii', '.img', '.nii.gz'] niftiFiles = [] if request.method == 'POST': print request.POST print request.FILES form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): tmp_directory = tempfile.mkdtemp() print tmp_directory try: # Save archive (.zip or .tar.gz) to disk if "file" in request.FILES: archive_name = request.FILES['file'].name if fnmatch(archive_name, '*.nidm.zip'): populate_nidm_results(request, collection) return HttpResponseRedirect( collection.get_absolute_url()) _, archive_ext = os.path.splitext(archive_name) if archive_ext == '.zip': compressed = zipfile.ZipFile(request.FILES['file']) elif archive_ext == '.gz': django_file = request.FILES['file'] django_file.open() compressed = tarfile.TarFile(fileobj=gzip.GzipFile( fileobj=django_file.file, mode='r'), mode='r') else: raise Exception("Unsupported archive type %s." % archive_name) compressed.extractall(path=tmp_directory) elif "file_input[]" in request.FILES: for f, path in zip(request.FILES.getlist("file_input[]"), request.POST.getlist("paths[]")): if fnmatch(f.name, '*.nidm.zip'): request.FILES['file'] = f populate_nidm_results(request, collection) continue new_path, _ = os.path.split( os.path.join(tmp_directory, path)) mkdir_p(new_path) filename = os.path.join(new_path, f.name) tmp_file = open(filename, 'w') tmp_file.write(f.read()) tmp_file.close() else: raise Exception("Unable to find uploaded files.") atlases = {} for root, subdirs, filenames in os.walk(tmp_directory): if detect_feat_directory(root): populate_feat_directory(request, collection, root) del (subdirs) filenames = [] # .gfeat parent dir under cope*.feat should not be added as statmaps # this may be affected by future nidm-results_fsl parsing changes if root.endswith('.gfeat'): filenames = [] filenames = [f for f in filenames if not f[0] == '.'] for fname in sorted(filenames): name, ext = splitext_nii_gz(fname) nii_path = os.path.join(root, fname) if ext == '.xml': print "found xml" dom = minidom.parse(os.path.join(root, fname)) for atlas in dom.getElementsByTagName( "summaryimagefile"): print "found atlas" path, base = os.path.split( atlas.lastChild.nodeValue) nifti_name = os.path.join(path, base) atlases[str(os.path.join( root, nifti_name[1:]))] = os.path.join( root, fname) if ext in allowed_extensions: nii = nib.load(nii_path) if detect_afni4D(nii): niftiFiles.extend(split_afni4D_to_3D(nii)) else: niftiFiles.append((fname, nii_path)) for label, fpath in niftiFiles: # Read nifti file information nii = nib.load(fpath) if len(nii.get_shape()) > 3 and nii.get_shape()[3] > 1: print "skipping wrong size" continue hdr = nii.get_header() raw_hdr = hdr.structarr # SPM only !!! # Check if filename corresponds to a T-map Tregexp = re.compile('spmT.*') # Fregexp = re.compile('spmF.*') if Tregexp.search(fpath) is not None: map_type = StatisticMap.T else: # Check if filename corresponds to a F-map if Tregexp.search(fpath) is not None: map_type = StatisticMap.F else: map_type = StatisticMap.OTHER path, name, ext = split_filename(fpath) dname = name + ".nii.gz" spaced_name = name.replace('_', ' ').replace('-', ' ') if ext.lower() != ".nii.gz": new_file_tmp_dir = tempfile.mkdtemp() new_file_tmp = os.path.join(new_file_tmp_dir, name) + '.nii.gz' nib.save(nii, new_file_tmp) f = ContentFile(open(new_file_tmp).read(), name=dname) shutil.rmtree(new_file_tmp_dir) label += " (old ext: %s)" % ext else: f = ContentFile(open(fpath).read(), name=dname) collection = get_collection(collection_cid, request) if os.path.join(path, name) in atlases: new_image = Atlas(name=spaced_name, description=raw_hdr['descrip'], collection=collection) new_image.label_description_file = ContentFile( open(atlases[os.path.join(path, name)]).read(), name=name + ".xml") else: new_image = StatisticMap(name=spaced_name, description=raw_hdr['descrip'] or label, collection=collection) new_image.map_type = map_type new_image.file = f new_image.save() except: raise error = traceback.format_exc().splitlines()[-1] msg = "An error occurred with this upload: {}".format(error) messages.warning(request, msg) return HttpResponseRedirect(collection.get_absolute_url()) finally: shutil.rmtree(tmp_directory) return HttpResponseRedirect(collection.get_absolute_url()) else: form = UploadFileForm() return render_to_response("statmaps/upload_folder.html", {'form': form}, RequestContext(request))