def test_statmaps_processing(self):
# The counter is the count of the number of images with the field "transform" set to None
# The field is populated with the file when image comparisons are done, meaning that if there is only one
# image in the database (case below) we cannot calculate comparisons, and the "transform" field remains none
# This is currently the only way that we can test the counter, which will be "1" in this case
print "\nTesting Counter - added statistic maps ###"
Image1 = StatisticMap(name='Image1', collection=self.Collection1, file='motor_lips.nii.gz',
map_type="Z", analysis_level='G', number_of_subjects=10)
Image1.file = SimpleUploadedFile('motor_lips.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/motor_lips.nii.gz')).read())
Image1.save()
images_processing = count_processing_comparisons(Image1.pk)
print "%s images processing [should be 0]" %(images_processing)
self.assertEqual(images_processing,0)
# When we add an image, the comparison will be calculated with image1, and both images transform fields will be populated
# the counter will be set to 0. Celery runs in synchronous mode when testing (meaning that jobs are run locally, one
# after the other, instead of being sent to worker nodes) so there is no way to test submitting a batch of async
# jobs and watching the "images still processing" counter go from N to 0. There is also no way of arbitrarily
# setting an image transform field to "None" because on save, all image comparisons are automatically re-calcualted
Image2 = StatisticMap(name='Image2', collection=self.Collection2, file='beta_0001.nii.gz',
map_type="Other", analysis_level='G', number_of_subjects=10)
Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read())
Image2.save()
images_processing = count_processing_comparisons(Image1.pk)
print "%s images processing [should be 0]" %(images_processing)
self.assertEqual(images_processing,0)
# We should have 2 images total, so 1 comparison
total_comparisons = count_existing_comparisons(Image1.pk)
self.assertEqual(total_comparisons,1)
def test_adding_nidm(self):
Image2 = StatisticMap(name='Image2',
collection=self.Collection1,
file='beta_0001.nii.gz',
map_type="Other")
Image2.file = SimpleUploadedFile(
'beta_0001.nii.gz',
file(
os.path.join(self.test_path,
'test_data/statmaps/beta_0001.nii.gz')).read())
Image2.save()
zip_file = open(
os.path.join(self.test_path,
'test_data/nidm/spm_example.nidm.zip'), 'rb')
post_dict = {
'name': 'spm_nidm',
'description': '{0} upload test'.format('spm_example'),
'collection': self.Collection2.pk
}
fname = os.path.basename(
os.path.join(self.test_path,
'test_data/nidm/spm_example.nidm.zip'))
file_dict = {'zip_file': SimpleUploadedFile(fname, zip_file.read())}
zip_file.close()
form = NIDMResultsForm(post_dict, file_dict)
# Transforms should be generated synchronously
nidm = form.save()
print "\nTesting Counter - added nidm result ###"
# We should have 2 images total, so 1 comparison
total_comparisons = count_existing_comparisons(Image2.pk)
self.assertEqual(total_comparisons, 1)
#Let's add a single subject map - this should not trigger a comparison
Image2ss = StatisticMap(name='Image2 - single subject',
collection=self.Collection3,
file='beta_0001.nii.gz',
map_type="Other",
analysis_level='S')
Image2ss.file = SimpleUploadedFile(
'beta_0001.nii.gz',
file(
os.path.join(self.test_path,
'test_data/statmaps/beta_0001.nii.gz')).read())
Image2ss.save()
total_comparisons = count_existing_comparisons(Image2ss.pk)
self.assertEqual(total_comparisons, 0)
# Make sure comparisons were calculated
number_comparisons = len(Comparison.objects.all())
print "\n %s comparisons exist after adding NIDM `[should not be 0]" % (
number_comparisons)
self.assertEqual(number_comparisons > 0, True)
def add_image(request, collection_cid):
collection = get_collection(collection_cid, request)
image = StatisticMap(collection=collection)
if request.method == "POST":
form = StatisticMapForm(request.POST, request.FILES, instance=image)
if form.is_valid():
image = form.save()
return HttpResponseRedirect(image.get_absolute_url())
else:
form = StatisticMapForm(instance=image)
context = {"form": form}
return render(request, "statmaps/add_image.html.haml", context)
def add_image(request, collection_cid):
collection = get_collection(collection_cid,request)
image = StatisticMap(collection=collection)
if request.method == "POST":
form = StatisticMapForm(request.POST, request.FILES, instance=image)
if form.is_valid():
image = form.save()
return HttpResponseRedirect(image.get_absolute_url())
else:
form = StatisticMapForm(instance=image)
context = {"form": form}
return render(request, "statmaps/add_image.html.haml", context)
def test_adding_nidm(self):
Image2 = StatisticMap(name='Image2', collection=self.Collection1, file='beta_0001.nii.gz', map_type="Other")
Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read())
Image2.save()
zip_file = open(os.path.join(self.test_path,'test_data/nidm/spm_example.nidm.zip'), 'rb')
post_dict = {
'name': 'spm_nidm',
'description':'{0} upload test'.format('spm_example'),
'collection':self.Collection2.pk}
fname = os.path.basename(os.path.join(self.test_path,'test_data/nidm/spm_example.nidm.zip'))
file_dict = {'zip_file': SimpleUploadedFile(fname, zip_file.read())}
zip_file.close()
form = NIDMResultsForm(post_dict, file_dict)
# Transforms should be generated synchronously
nidm = form.save()
print "\nTesting Counter - added nidm result ###"
# We should have 2 images total, so 1 comparison
total_comparisons = count_existing_comparisons(Image2.pk)
self.assertEqual(total_comparisons,1)
#Let's add a single subject map - this should not trigger a comparison
Image2ss = StatisticMap(name='Image2 - single subject', collection=self.Collection3, file='beta_0001.nii.gz', map_type="Other", analysis_level='S')
Image2ss.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read())
Image2ss.save()
total_comparisons = count_existing_comparisons(Image2ss.pk)
self.assertEqual(total_comparisons,0)
# Make sure comparisons were calculated
number_comparisons = len(Comparison.objects.all())
print "\n %s comparisons exist after adding NIDM `[should not be 0]" %(number_comparisons)
self.assertEqual(number_comparisons>0,True)
def test_statmaps_processing(self):
# The counter is the count of the number of images with the field "transform" set to None
# The field is populated with the file when image comparisons are done, meaning that if there is only one
# image in the database (case below) we cannot calculate comparisons, and the "transform" field remains none
# This is currently the only way that we can test the counter, which will be "1" in this case
print "\nTesting Counter - added statistic maps ###"
Image1 = StatisticMap(name='Image1', collection=self.Collection1, file='motor_lips.nii.gz', map_type="Z")
Image1.file = SimpleUploadedFile('motor_lips.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/motor_lips.nii.gz')).read())
Image1.save()
images_processing = count_processing_comparisons(Image1.pk)
print "%s images processing [should be 0]" %(images_processing)
self.assertEqual(images_processing,0)
# When we add an image, the comparison will be calculated with image1, and both images transform fields will be populated
# the counter will be set to 0. Celery runs in synchronous mode when testing (meaning that jobs are run locally, one
# after the other, instead of being sent to worker nodes) so there is no way to test submitting a batch of async
# jobs and watching the "images still processing" counter go from N to 0. There is also no way of arbitrarily
# setting an image transform field to "None" because on save, all image comparisons are automatically re-calcualted
Image2 = StatisticMap(name='Image2', collection=self.Collection2, file='beta_0001.nii.gz', map_type="Other")
Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read())
Image2.save()
images_processing = count_processing_comparisons(Image1.pk)
print "%s images processing [should be 0]" %(images_processing)
self.assertEqual(images_processing,0)
# We should have 2 images total, so 1 comparison
total_comparisons = count_existing_comparisons(Image1.pk)
self.assertEqual(total_comparisons,1)
def add_image_for_neurosynth(request):
temp_collection_name = "%s's temporary collection" % request.user.username
#this is a hack we need to make sure this collection can be only
#owned by the same user
try:
temp_collection = Collection.objects.get(name=temp_collection_name)
except Collection.DoesNotExist:
priv_token = generate_url_token()
temp_collection = Collection(name=temp_collection_name,
owner=request.user,
private=False,
private_token=priv_token)
temp_collection.save()
image = StatisticMap(collection=temp_collection)
if request.method == "POST":
form = SimplifiedStatisticMapForm(request.POST,
request.FILES,
instance=image,
user=request.user)
if form.is_valid():
image = form.save()
return HttpResponseRedirect(
"http://neurosynth.org/decode/?neurovault=%s-%s" %
(temp_collection.private_token, image.id))
else:
form = SimplifiedStatisticMapForm(user=request.user, instance=image)
context = {"form": form}
return render(request, "statmaps/add_image_for_neurosynth.html.haml",
context)
def add_image(request, collection_cid):
collection = get_collection(collection_cid,request)
if not owner_or_contrib(request,collection):
return HttpResponseForbidden()
image = StatisticMap(collection=collection)
if request.method == "POST":
form = AddStatisticMapForm(request.POST, request.FILES, instance=image)
if form.is_valid():
image = form.save()
return HttpResponseRedirect(image.get_absolute_url())
else:
form = AddStatisticMapForm(instance=image)
contrasts = get_contrast_lookup()
context = {"form": form,"contrasts": json.dumps(contrasts)}
return render(request, "statmaps/add_image.html", context)
def edit_images(request, collection_cid):
collection = get_collection(collection_cid, request)
if not owner_or_contrib(request, collection):
return HttpResponseForbidden()
if request.method == "POST":
formset = CollectionFormSet(request.POST,
request.FILES,
instance=collection)
for n, form in enumerate(formset):
# hack: check fields to determine polymorphic type
if form.instance.polymorphic_ctype is None:
atlas_f = 'image_set-{0}-label_description_file'.format(n)
has_atlas = [v for v in form.files if v == atlas_f]
if has_atlas:
use_model = Atlas
use_form = AtlasForm
else:
use_model = StatisticMap
use_form = StatisticMapForm
form.instance = use_model(collection=collection)
form.base_fields = use_form.base_fields
form.fields = use_form.base_fields
if formset.is_valid():
formset.save()
return HttpResponseRedirect(collection.get_absolute_url())
else:
formset = CollectionFormSet(instance=collection)
blank_statmap = StatisticMapForm(instance=StatisticMap(
collection=collection))
blank_atlas = AtlasForm(instance=Atlas(collection=collection))
upload_form = UploadFileForm()
context = {
"formset": formset,
"blank_statmap": blank_statmap,
"blank_atlas": blank_atlas,
"upload_form": upload_form
}
return render(request, "statmaps/edit_images.html", context)
def upload_folder(request, collection_cid):
collection = get_collection(collection_cid, request)
allowed_extensions = ['.nii', '.img', '.nii.gz']
niftiFiles = []
if request.method == 'POST':
print request.POST
print request.FILES
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
tmp_directory = tempfile.mkdtemp()
print tmp_directory
try:
# Save archive (.zip or .tar.gz) to disk
if "file" in request.FILES:
archive_name = request.FILES['file'].name
if fnmatch(archive_name, '*.nidm.zip'):
populate_nidm_results(request, collection)
return HttpResponseRedirect(
collection.get_absolute_url())
_, archive_ext = os.path.splitext(archive_name)
if archive_ext == '.zip':
compressed = zipfile.ZipFile(request.FILES['file'])
elif archive_ext == '.gz':
django_file = request.FILES['file']
django_file.open()
compressed = tarfile.TarFile(fileobj=gzip.GzipFile(
fileobj=django_file.file, mode='r'),
mode='r')
else:
raise Exception("Unsupported archive type %s." %
archive_name)
compressed.extractall(path=tmp_directory)
elif "file_input[]" in request.FILES:
for f, path in zip(request.FILES.getlist("file_input[]"),
request.POST.getlist("paths[]")):
if fnmatch(f.name, '*.nidm.zip'):
request.FILES['file'] = f
populate_nidm_results(request, collection)
continue
new_path, _ = os.path.split(
os.path.join(tmp_directory, path))
mkdir_p(new_path)
filename = os.path.join(new_path, f.name)
tmp_file = open(filename, 'w')
tmp_file.write(f.read())
tmp_file.close()
else:
raise Exception("Unable to find uploaded files.")
atlases = {}
for root, subdirs, filenames in os.walk(tmp_directory):
if detect_feat_directory(root):
populate_feat_directory(request, collection, root)
del (subdirs)
filenames = []
# .gfeat parent dir under cope*.feat should not be added as statmaps
# this may be affected by future nidm-results_fsl parsing changes
if root.endswith('.gfeat'):
filenames = []
filenames = [f for f in filenames if not f[0] == '.']
for fname in sorted(filenames):
name, ext = splitext_nii_gz(fname)
nii_path = os.path.join(root, fname)
if ext == '.xml':
print "found xml"
dom = minidom.parse(os.path.join(root, fname))
for atlas in dom.getElementsByTagName(
"summaryimagefile"):
print "found atlas"
path, base = os.path.split(
atlas.lastChild.nodeValue)
nifti_name = os.path.join(path, base)
atlases[str(os.path.join(
root, nifti_name[1:]))] = os.path.join(
root, fname)
if ext in allowed_extensions:
nii = nib.load(nii_path)
if detect_afni4D(nii):
niftiFiles.extend(split_afni4D_to_3D(nii))
else:
niftiFiles.append((fname, nii_path))
for label, fpath in niftiFiles:
# Read nifti file information
nii = nib.load(fpath)
if len(nii.get_shape()) > 3 and nii.get_shape()[3] > 1:
print "skipping wrong size"
continue
hdr = nii.get_header()
raw_hdr = hdr.structarr
# SPM only !!!
# Check if filename corresponds to a T-map
Tregexp = re.compile('spmT.*')
# Fregexp = re.compile('spmF.*')
if Tregexp.search(fpath) is not None:
map_type = StatisticMap.T
else:
# Check if filename corresponds to a F-map
if Tregexp.search(fpath) is not None:
map_type = StatisticMap.F
else:
map_type = StatisticMap.OTHER
path, name, ext = split_filename(fpath)
dname = name + ".nii.gz"
spaced_name = name.replace('_', ' ').replace('-', ' ')
if ext.lower() != ".nii.gz":
new_file_tmp_dir = tempfile.mkdtemp()
new_file_tmp = os.path.join(new_file_tmp_dir,
name) + '.nii.gz'
nib.save(nii, new_file_tmp)
f = ContentFile(open(new_file_tmp).read(), name=dname)
shutil.rmtree(new_file_tmp_dir)
label += " (old ext: %s)" % ext
else:
f = ContentFile(open(fpath).read(), name=dname)
collection = get_collection(collection_cid, request)
if os.path.join(path, name) in atlases:
new_image = Atlas(name=spaced_name,
description=raw_hdr['descrip'],
collection=collection)
new_image.label_description_file = ContentFile(
open(atlases[os.path.join(path, name)]).read(),
name=name + ".xml")
else:
new_image = StatisticMap(name=spaced_name,
description=raw_hdr['descrip']
or label,
collection=collection)
new_image.map_type = map_type
new_image.file = f
new_image.save()
except:
raise
error = traceback.format_exc().splitlines()[-1]
msg = "An error occurred with this upload: {}".format(error)
messages.warning(request, msg)
return HttpResponseRedirect(collection.get_absolute_url())
finally:
shutil.rmtree(tmp_directory)
return HttpResponseRedirect(collection.get_absolute_url())
else:
form = UploadFileForm()
return render_to_response("statmaps/upload_folder.html", {'form': form},
RequestContext(request))
