Ejemplo n.º 1
0
    def test_statmaps_processing(self):

        # The counter is the count of the number of images with the field "transform" set to None
        # The field is populated with the file when image comparisons are done, meaning that if there is only one
        # image in the database (case below) we cannot calculate comparisons, and the "transform" field remains none
        # This is currently the only way that we can test the counter, which will be "1" in this case
        print "\nTesting Counter - added statistic maps ###" 
        Image1 = StatisticMap(name='Image1', collection=self.Collection1, file='motor_lips.nii.gz',
                              map_type="Z", analysis_level='G', number_of_subjects=10)
        Image1.file = SimpleUploadedFile('motor_lips.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/motor_lips.nii.gz')).read())
        Image1.save()
        images_processing = count_processing_comparisons(Image1.pk)
        print "%s images processing [should be 0]" %(images_processing)
        self.assertEqual(images_processing,0)

        # When we add an image, the comparison will be calculated with image1, and both images transform fields will be populated
        # the counter will be set to 0.  Celery runs in synchronous mode when testing (meaning that jobs are run locally, one
        # after the other, instead of being sent to worker nodes) so there is no way to test submitting a batch of async
        # jobs and watching the "images still processing" counter go from N to 0. There is also no way of arbitrarily
        # setting an image transform field to "None" because on save, all image comparisons are automatically re-calcualted        
        Image2 = StatisticMap(name='Image2', collection=self.Collection2, file='beta_0001.nii.gz',
                              map_type="Other", analysis_level='G', number_of_subjects=10)
        Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read())
        Image2.save()
        images_processing = count_processing_comparisons(Image1.pk)
        print "%s images processing [should be 0]" %(images_processing)
        self.assertEqual(images_processing,0)

        # We should have 2 images total, so 1 comparison
        total_comparisons = count_existing_comparisons(Image1.pk)
        self.assertEqual(total_comparisons,1)
Ejemplo n.º 2
0
    def test_adding_nidm(self):
        Image2 = StatisticMap(name='Image2',
                              collection=self.Collection1,
                              file='beta_0001.nii.gz',
                              map_type="Other")
        Image2.file = SimpleUploadedFile(
            'beta_0001.nii.gz',
            file(
                os.path.join(self.test_path,
                             'test_data/statmaps/beta_0001.nii.gz')).read())
        Image2.save()

        zip_file = open(
            os.path.join(self.test_path,
                         'test_data/nidm/spm_example.nidm.zip'), 'rb')
        post_dict = {
            'name': 'spm_nidm',
            'description': '{0} upload test'.format('spm_example'),
            'collection': self.Collection2.pk
        }
        fname = os.path.basename(
            os.path.join(self.test_path,
                         'test_data/nidm/spm_example.nidm.zip'))
        file_dict = {'zip_file': SimpleUploadedFile(fname, zip_file.read())}
        zip_file.close()
        form = NIDMResultsForm(post_dict, file_dict)
        # Transforms should be generated synchronously
        nidm = form.save()
        print "\nTesting Counter - added nidm result ###"

        # We should have 2 images total, so 1 comparison
        total_comparisons = count_existing_comparisons(Image2.pk)
        self.assertEqual(total_comparisons, 1)

        #Let's add a single subject map - this should not trigger a comparison
        Image2ss = StatisticMap(name='Image2 - single subject',
                                collection=self.Collection3,
                                file='beta_0001.nii.gz',
                                map_type="Other",
                                analysis_level='S')
        Image2ss.file = SimpleUploadedFile(
            'beta_0001.nii.gz',
            file(
                os.path.join(self.test_path,
                             'test_data/statmaps/beta_0001.nii.gz')).read())
        Image2ss.save()
        total_comparisons = count_existing_comparisons(Image2ss.pk)
        self.assertEqual(total_comparisons, 0)

        # Make sure comparisons were calculated
        number_comparisons = len(Comparison.objects.all())
        print "\n %s comparisons exist after adding NIDM `[should not be 0]" % (
            number_comparisons)
        self.assertEqual(number_comparisons > 0, True)
Ejemplo n.º 3
0
def add_image(request, collection_cid):
    collection = get_collection(collection_cid, request)
    image = StatisticMap(collection=collection)
    if request.method == "POST":
        form = StatisticMapForm(request.POST, request.FILES, instance=image)
        if form.is_valid():
            image = form.save()
            return HttpResponseRedirect(image.get_absolute_url())
    else:
        form = StatisticMapForm(instance=image)

    context = {"form": form}
    return render(request, "statmaps/add_image.html.haml", context)
Ejemplo n.º 4
0
def add_image(request, collection_cid):
    collection = get_collection(collection_cid,request)
    image = StatisticMap(collection=collection)
    if request.method == "POST":
        form = StatisticMapForm(request.POST, request.FILES, instance=image)
        if form.is_valid():
            image = form.save()
            return HttpResponseRedirect(image.get_absolute_url())
    else:
        form = StatisticMapForm(instance=image)

    context = {"form": form}
    return render(request, "statmaps/add_image.html.haml", context)
Ejemplo n.º 5
0
    def test_adding_nidm(self):
        Image2 = StatisticMap(name='Image2', collection=self.Collection1, file='beta_0001.nii.gz', map_type="Other")
        Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read())
        Image2.save()
        
        zip_file = open(os.path.join(self.test_path,'test_data/nidm/spm_example.nidm.zip'), 'rb')
        post_dict = {
            'name': 'spm_nidm',
            'description':'{0} upload test'.format('spm_example'),
            'collection':self.Collection2.pk}
        fname = os.path.basename(os.path.join(self.test_path,'test_data/nidm/spm_example.nidm.zip'))
        file_dict = {'zip_file': SimpleUploadedFile(fname, zip_file.read())}
        zip_file.close()
        form = NIDMResultsForm(post_dict, file_dict)
        # Transforms should be generated synchronously
        nidm = form.save()
        print "\nTesting Counter - added nidm result ###"

        # We should have 2 images total, so 1 comparison
        total_comparisons = count_existing_comparisons(Image2.pk)
        self.assertEqual(total_comparisons,1)
        
        #Let's add a single subject map - this should not trigger a comparison
        Image2ss = StatisticMap(name='Image2 - single subject', collection=self.Collection3, file='beta_0001.nii.gz', map_type="Other", analysis_level='S')
        Image2ss.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read())
        Image2ss.save()
        total_comparisons = count_existing_comparisons(Image2ss.pk)
        self.assertEqual(total_comparisons,0)

        # Make sure comparisons were calculated
        number_comparisons = len(Comparison.objects.all())
        print "\n %s comparisons exist after adding NIDM `[should not be 0]" %(number_comparisons)
        self.assertEqual(number_comparisons>0,True)
Ejemplo n.º 6
0
    def test_statmaps_processing(self):

        # The counter is the count of the number of images with the field "transform" set to None
        # The field is populated with the file when image comparisons are done, meaning that if there is only one
        # image in the database (case below) we cannot calculate comparisons, and the "transform" field remains none
        # This is currently the only way that we can test the counter, which will be "1" in this case
        print "\nTesting Counter - added statistic maps ###" 
        Image1 = StatisticMap(name='Image1', collection=self.Collection1, file='motor_lips.nii.gz', map_type="Z")
        Image1.file = SimpleUploadedFile('motor_lips.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/motor_lips.nii.gz')).read())
        Image1.save()
        images_processing = count_processing_comparisons(Image1.pk)
        print "%s images processing [should be 0]" %(images_processing)
        self.assertEqual(images_processing,0)

        # When we add an image, the comparison will be calculated with image1, and both images transform fields will be populated
        # the counter will be set to 0.  Celery runs in synchronous mode when testing (meaning that jobs are run locally, one
        # after the other, instead of being sent to worker nodes) so there is no way to test submitting a batch of async
        # jobs and watching the "images still processing" counter go from N to 0. There is also no way of arbitrarily
        # setting an image transform field to "None" because on save, all image comparisons are automatically re-calcualted        
        Image2 = StatisticMap(name='Image2', collection=self.Collection2, file='beta_0001.nii.gz', map_type="Other")
        Image2.file = SimpleUploadedFile('beta_0001.nii.gz', file(os.path.join(self.test_path,'test_data/statmaps/beta_0001.nii.gz')).read())
        Image2.save()
        images_processing = count_processing_comparisons(Image1.pk)
        print "%s images processing [should be 0]" %(images_processing)
        self.assertEqual(images_processing,0)

        # We should have 2 images total, so 1 comparison
        total_comparisons = count_existing_comparisons(Image1.pk)
        self.assertEqual(total_comparisons,1)
Ejemplo n.º 7
0
def add_image_for_neurosynth(request):
    temp_collection_name = "%s's temporary collection" % request.user.username
    #this is a hack we need to make sure this collection can be only
    #owned by the same user
    try:
        temp_collection = Collection.objects.get(name=temp_collection_name)
    except Collection.DoesNotExist:
        priv_token = generate_url_token()
        temp_collection = Collection(name=temp_collection_name,
                                     owner=request.user,
                                     private=False,
                                     private_token=priv_token)
        temp_collection.save()
    image = StatisticMap(collection=temp_collection)
    if request.method == "POST":
        form = SimplifiedStatisticMapForm(request.POST,
                                          request.FILES,
                                          instance=image,
                                          user=request.user)
        if form.is_valid():
            image = form.save()
            return HttpResponseRedirect(
                "http://neurosynth.org/decode/?neurovault=%s-%s" %
                (temp_collection.private_token, image.id))
    else:
        form = SimplifiedStatisticMapForm(user=request.user, instance=image)

    context = {"form": form}
    return render(request, "statmaps/add_image_for_neurosynth.html.haml",
                  context)
Ejemplo n.º 8
0
def add_image(request, collection_cid):
    collection = get_collection(collection_cid,request)
    if not owner_or_contrib(request,collection):
        return HttpResponseForbidden()
    image = StatisticMap(collection=collection)
    if request.method == "POST":
        form = AddStatisticMapForm(request.POST, request.FILES, instance=image)
        if form.is_valid():
            image = form.save()
            return HttpResponseRedirect(image.get_absolute_url())
    else:
        form = AddStatisticMapForm(instance=image)

    contrasts = get_contrast_lookup()
    context = {"form": form,"contrasts": json.dumps(contrasts)}
    return render(request, "statmaps/add_image.html", context)
Ejemplo n.º 9
0
def edit_images(request, collection_cid):
    collection = get_collection(collection_cid, request)
    if not owner_or_contrib(request, collection):
        return HttpResponseForbidden()
    if request.method == "POST":
        formset = CollectionFormSet(request.POST,
                                    request.FILES,
                                    instance=collection)
        for n, form in enumerate(formset):
            # hack: check fields to determine polymorphic type
            if form.instance.polymorphic_ctype is None:
                atlas_f = 'image_set-{0}-label_description_file'.format(n)
                has_atlas = [v for v in form.files if v == atlas_f]
                if has_atlas:
                    use_model = Atlas
                    use_form = AtlasForm
                else:
                    use_model = StatisticMap
                    use_form = StatisticMapForm
                form.instance = use_model(collection=collection)
                form.base_fields = use_form.base_fields
                form.fields = use_form.base_fields
        if formset.is_valid():
            formset.save()
            return HttpResponseRedirect(collection.get_absolute_url())
    else:
        formset = CollectionFormSet(instance=collection)

    blank_statmap = StatisticMapForm(instance=StatisticMap(
        collection=collection))
    blank_atlas = AtlasForm(instance=Atlas(collection=collection))
    upload_form = UploadFileForm()
    context = {
        "formset": formset,
        "blank_statmap": blank_statmap,
        "blank_atlas": blank_atlas,
        "upload_form": upload_form
    }

    return render(request, "statmaps/edit_images.html", context)
Ejemplo n.º 10
0
def upload_folder(request, collection_cid):
    collection = get_collection(collection_cid, request)
    allowed_extensions = ['.nii', '.img', '.nii.gz']
    niftiFiles = []
    if request.method == 'POST':
        print request.POST
        print request.FILES
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            tmp_directory = tempfile.mkdtemp()
            print tmp_directory
            try:
                # Save archive (.zip or .tar.gz) to disk
                if "file" in request.FILES:
                    archive_name = request.FILES['file'].name
                    if fnmatch(archive_name, '*.nidm.zip'):
                        populate_nidm_results(request, collection)
                        return HttpResponseRedirect(
                            collection.get_absolute_url())

                    _, archive_ext = os.path.splitext(archive_name)
                    if archive_ext == '.zip':
                        compressed = zipfile.ZipFile(request.FILES['file'])
                    elif archive_ext == '.gz':
                        django_file = request.FILES['file']
                        django_file.open()
                        compressed = tarfile.TarFile(fileobj=gzip.GzipFile(
                            fileobj=django_file.file, mode='r'),
                                                     mode='r')
                    else:
                        raise Exception("Unsupported archive type %s." %
                                        archive_name)
                    compressed.extractall(path=tmp_directory)

                elif "file_input[]" in request.FILES:

                    for f, path in zip(request.FILES.getlist("file_input[]"),
                                       request.POST.getlist("paths[]")):
                        if fnmatch(f.name, '*.nidm.zip'):
                            request.FILES['file'] = f
                            populate_nidm_results(request, collection)
                            continue

                        new_path, _ = os.path.split(
                            os.path.join(tmp_directory, path))
                        mkdir_p(new_path)
                        filename = os.path.join(new_path, f.name)
                        tmp_file = open(filename, 'w')
                        tmp_file.write(f.read())
                        tmp_file.close()
                else:
                    raise Exception("Unable to find uploaded files.")

                atlases = {}
                for root, subdirs, filenames in os.walk(tmp_directory):
                    if detect_feat_directory(root):
                        populate_feat_directory(request, collection, root)
                        del (subdirs)
                        filenames = []

                    # .gfeat parent dir under cope*.feat should not be added as statmaps
                    # this may be affected by future nidm-results_fsl parsing changes
                    if root.endswith('.gfeat'):
                        filenames = []

                    filenames = [f for f in filenames if not f[0] == '.']
                    for fname in sorted(filenames):
                        name, ext = splitext_nii_gz(fname)
                        nii_path = os.path.join(root, fname)

                        if ext == '.xml':
                            print "found xml"
                            dom = minidom.parse(os.path.join(root, fname))
                            for atlas in dom.getElementsByTagName(
                                    "summaryimagefile"):
                                print "found atlas"
                                path, base = os.path.split(
                                    atlas.lastChild.nodeValue)
                                nifti_name = os.path.join(path, base)
                                atlases[str(os.path.join(
                                    root, nifti_name[1:]))] = os.path.join(
                                        root, fname)
                        if ext in allowed_extensions:
                            nii = nib.load(nii_path)
                            if detect_afni4D(nii):
                                niftiFiles.extend(split_afni4D_to_3D(nii))
                            else:
                                niftiFiles.append((fname, nii_path))

                for label, fpath in niftiFiles:
                    # Read nifti file information
                    nii = nib.load(fpath)
                    if len(nii.get_shape()) > 3 and nii.get_shape()[3] > 1:
                        print "skipping wrong size"
                        continue
                    hdr = nii.get_header()
                    raw_hdr = hdr.structarr

                    # SPM only !!!
                    # Check if filename corresponds to a T-map
                    Tregexp = re.compile('spmT.*')
                    # Fregexp = re.compile('spmF.*')

                    if Tregexp.search(fpath) is not None:
                        map_type = StatisticMap.T
                    else:
                        # Check if filename corresponds to a F-map
                        if Tregexp.search(fpath) is not None:
                            map_type = StatisticMap.F
                        else:
                            map_type = StatisticMap.OTHER

                    path, name, ext = split_filename(fpath)
                    dname = name + ".nii.gz"
                    spaced_name = name.replace('_', ' ').replace('-', ' ')

                    if ext.lower() != ".nii.gz":
                        new_file_tmp_dir = tempfile.mkdtemp()
                        new_file_tmp = os.path.join(new_file_tmp_dir,
                                                    name) + '.nii.gz'
                        nib.save(nii, new_file_tmp)
                        f = ContentFile(open(new_file_tmp).read(), name=dname)
                        shutil.rmtree(new_file_tmp_dir)
                        label += " (old ext: %s)" % ext
                    else:
                        f = ContentFile(open(fpath).read(), name=dname)

                    collection = get_collection(collection_cid, request)

                    if os.path.join(path, name) in atlases:

                        new_image = Atlas(name=spaced_name,
                                          description=raw_hdr['descrip'],
                                          collection=collection)

                        new_image.label_description_file = ContentFile(
                            open(atlases[os.path.join(path, name)]).read(),
                            name=name + ".xml")
                    else:
                        new_image = StatisticMap(name=spaced_name,
                                                 description=raw_hdr['descrip']
                                                 or label,
                                                 collection=collection)
                        new_image.map_type = map_type

                    new_image.file = f
                    new_image.save()

            except:
                raise
                error = traceback.format_exc().splitlines()[-1]
                msg = "An error occurred with this upload: {}".format(error)
                messages.warning(request, msg)
                return HttpResponseRedirect(collection.get_absolute_url())

            finally:
                shutil.rmtree(tmp_directory)

            return HttpResponseRedirect(collection.get_absolute_url())
    else:
        form = UploadFileForm()
    return render_to_response("statmaps/upload_folder.html", {'form': form},
                              RequestContext(request))