def edit_collection(request, cid=None): page_header = "Add new collection" if cid: collection = get_collection(cid, request) is_owner = True if collection.owner == request.user else False page_header = 'Edit collection' if not owner_or_contrib(request, collection): return HttpResponseForbidden() else: is_owner = True collection = Collection(owner=request.user) if request.method == "POST": if is_owner: form = OwnerCollectionForm(request.POST, request.FILES, instance=collection) else: form = CollectionForm(request.POST, request.FILES, instance=collection) if form.is_valid(): previous_contribs = set() if form.instance.pk is not None: previous_contribs = set(form.instance.contributors.all()) collection = form.save(commit=False) if collection.private and collection.private_token is None: collection.private_token = generate_url_token() collection.save() if is_owner: form.save_m2m() # save contributors current_contribs = set(collection.contributors.all()) new_contribs = list( current_contribs.difference(previous_contribs)) context = { 'owner': collection.owner.username, 'collection': collection.name, 'url': get_server_url(request) + collection.get_absolute_url(), } subj = '%s has added you to a NeuroVault collection' % context[ 'owner'] send_email_notification('new_contributor', subj, new_contribs, context) return HttpResponseRedirect(collection.get_absolute_url()) else: if is_owner: form = OwnerCollectionForm(instance=collection) else: form = CollectionForm(instance=collection) context = {"form": form, "page_header": page_header, "is_owner": is_owner} return render(request, "statmaps/edit_collection.html.haml", context)
def crawl_anima(): import neurovault.apps.statmaps.models as models from neurovault.apps.statmaps.forms import StatisticMapForm, CollectionForm username = "******" email = "*****@*****.**" try: anima_user = models.User.objects.create_user(username, email) anima_user.save() except IntegrityError: anima_user = models.User.objects.get(username=username, email=email) url = "http://anima.modelgui.org/api/studies" response = urllib.urlopen(url); datasets = json.loads(response.read()) # results = tarfile.open(mode="r:gz", fileobj=StringIO(response.content)) # for member in results.getmembers(): # f = results.extractfile(member) # if member.name.endswith(".study"): for url in datasets: response = requests.get(url) content = response.content.replace("PubMed ID", "PubMedID") xml_obj = e.fromstring(content) version = xml_obj.find(".").find(".//Element[@name='Version']").text.strip() study_description = xml_obj.find(".//Element[@name='Description']").text.strip() study_description += " This dataset was automatically imported from the ANIMA <http://anima.modelgui.org/> database. Version: %s"%version study_name = xml_obj.find(".").attrib['name'] tags = xml_obj.find(".//Element[@name='Keywords']").text.strip().split(";") tags.append("ANIMA") doi = xml_obj.find(".//Element[@name='DOI']") pubmedid = xml_obj.find(".//Element[@name='PubMedID']") post_dict = { 'name': study_name, 'description': study_description, 'full_dataset_url': "http://anima.modelgui.org/studies/" + os.path.split(url)[1].replace(".study", "") } if doi != None: post_dict['DOI'] = doi.text.strip() elif pubmedid != None: pubmedid = pubmedid.text.strip() url = "http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids=%s&format=json" % pubmedid response = urllib.urlopen(url); parsed = json.loads(response.read()) post_dict['DOI'] = parsed['records'][0]['doi'] try: col = models.Collection.objects.get(DOI=post_dict['DOI']) except models.Collection.DoesNotExist: col = None if col and not col.description.endswith(version): col.DOI = None old_version = re.search(r"Version: (?P<version>\w)", col.description).group("version") col.name = study_name + " (version %s - deprecated)"%old_version col.save() if not col or not col.description.endswith(version): collection = models.Collection(owner=anima_user) form = CollectionForm(post_dict, instance=collection) form.is_valid() collection = form.save() arch_response = requests.get(url.replace("library", "library/archives").replace(".study", ".tar.gz")) arch_results = tarfile.open(mode="r:gz", fileobj=StringIO(arch_response.content)) for study_element in xml_obj.findall(".//StudyElement[@type='VolumeFile']"): image_name = study_element.attrib['name'].strip() image_filename = study_element.attrib['file'] image_fileobject = arch_results.extractfile(xml_obj.find(".").attrib['directory'] + "/" + image_filename) map_type = models.BaseStatisticMap.OTHER quantity_dict = {"Mask": models.BaseStatisticMap.M, "F-statistic": models.BaseStatisticMap.F, "T-statistic": models.BaseStatisticMap.T, "Z-statistic": models.BaseStatisticMap.Z, "Beta": models.BaseStatisticMap.U} quantity = study_element.find("./Metadata/Element[@name='Quantity']") if quantity != None: quantity = quantity.text.strip() if quantity in quantity_dict.keys(): map_type = quantity_dict[quantity] post_dict = { 'name': image_name, 'modality': models.StatisticMap.fMRI_BOLD, 'map_type': map_type, 'analysis_level': models.BaseStatisticMap.M, 'collection': collection.pk, 'ignore_file_warning': True, 'cognitive_paradigm_cogatlas': 'None', 'tags': ", ".join(tags) } image_description = study_element.find("./Metadata/Element[@name='Caption']").text if image_description: post_dict["description"] = image_description.strip() file_dict = {'file': SimpleUploadedFile(image_filename, image_fileobject.read())} form = StatisticMapForm(post_dict, file_dict) form.is_valid() form.save()
def crawl_anima(): import neurovault.apps.statmaps.models as models from neurovault.apps.statmaps.forms import StatisticMapForm, CollectionForm username = "******" email = "*****@*****.**" try: anima_user = models.User.objects.create_user(username, email) anima_user.save() except IntegrityError: anima_user = models.User.objects.get(username=username, email=email) url = "http://anima.fz-juelich.de/api/studies" response = urllib.urlopen(url) datasets = json.loads(response.read()) # results = tarfile.open(mode="r:gz", fileobj=StringIO(response.content)) # for member in results.getmembers(): # f = results.extractfile(member) # if member.name.endswith(".study"): for url in datasets: response = requests.get(url) content = response.content.replace("PubMed ID", "PubMedID") xml_obj = e.fromstring(content) version = xml_obj.find(".").find( ".//Element[@name='Version']").text.strip() study_description = xml_obj.find( ".//Element[@name='Description']").text.strip() study_description += " This dataset was automatically imported from the ANIMA <http://anima.fz-juelich.de/> database. Version: %s" % version study_name = xml_obj.find(".").attrib['name'] tags = xml_obj.find(".//Element[@name='Keywords']").text.strip().split( ";") tags.append("ANIMA") doi = xml_obj.find(".//Element[@name='DOI']") pubmedid = xml_obj.find(".//Element[@name='PubMedID']") post_dict = { 'name': study_name, 'description': study_description, 'full_dataset_url': "http://anima.fz-juelich.de/studies/" + os.path.split(url)[1].replace(".study", "") } if doi != None: post_dict['DOI'] = doi.text.strip() elif pubmedid != None: pubmedid = pubmedid.text.strip() url = "http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids=%s&format=json" % pubmedid response = urllib.urlopen(url) parsed = json.loads(response.read()) post_dict['DOI'] = parsed['records'][0]['doi'] try: col = models.Collection.objects.get(DOI=post_dict['DOI']) except models.Collection.DoesNotExist: col = None if col and not col.description.endswith(version): col.DOI = None old_version = re.search(r"Version: (?P<version>\w)", col.description).group("version") col.name = study_name + " (version %s - deprecated)" % old_version col.save() if not col or not col.description.endswith(version): collection = models.Collection(owner=anima_user) form = CollectionForm(post_dict, instance=collection) form.is_valid() collection = form.save() arch_response = requests.get( url.replace("library", "library/archives").replace(".study", ".tar.gz")) arch_results = tarfile.open(mode="r:gz", fileobj=StringIO( arch_response.content)) for study_element in xml_obj.findall( ".//StudyElement[@type='VolumeFile']"): image_name = study_element.attrib['name'].strip() image_filename = study_element.attrib['file'] try: image_fileobject = arch_results.extractfile( xml_obj.find(".").attrib['directory'] + "/" + image_filename) except KeyError: image_fileobject = arch_results.extractfile( xml_obj.find(".").attrib['directory'] + "/" + xml_obj.find(".").attrib['directory'] + "/" + image_filename) map_type = models.BaseStatisticMap.OTHER quantity_dict = { "Mask": models.BaseStatisticMap.M, "F-statistic": models.BaseStatisticMap.F, "T-statistic": models.BaseStatisticMap.T, "Z-statistic": models.BaseStatisticMap.Z, "Beta": models.BaseStatisticMap.U } quantity = study_element.find( "./Metadata/Element[@name='Quantity']") if quantity != None: quantity = quantity.text.strip() if quantity in quantity_dict.keys(): map_type = quantity_dict[quantity] post_dict = { 'name': image_name, 'modality': models.StatisticMap.fMRI_BOLD, 'map_type': map_type, 'analysis_level': models.BaseStatisticMap.M, 'collection': collection.pk, 'ignore_file_warning': True, 'cognitive_paradigm_cogatlas': 'None', 'tags': ", ".join(tags) } image_description = study_element.find( "./Metadata/Element[@name='Caption']").text if image_description: post_dict["description"] = image_description.strip() file_dict = { 'file': SimpleUploadedFile(image_filename, image_fileobject.read()) } form = StatisticMapForm(post_dict, file_dict) form.is_valid() form.save()
post_dict['DOI'] = parsed['records'][0]['doi'] try: col = Collection.objects.get(DOI=post_dict['DOI']) except Collection.DoesNotExist: col = None if col and not col.description.endswith(version): col.DOI = None old_version = re.search(r"Version: (?P<version>\w)", col.description).group("version") col.name = study_name + " (version %s - deprecated)"%old_version col.save() if not col or not col.description.endswith(version): collection = Collection(owner=anima_user) form = CollectionForm(post_dict, instance=collection) form.is_valid() print form.errors collection = form.save() arch_response = requests.get(url.replace("library", "library/archives").replace(".study", ".tar.gz")) arch_results = tarfile.open(mode="r:gz", fileobj=StringIO(arch_response.content)) for study_element in xml_obj.findall(".//StudyElement[@type='VolumeFile']"): image_name = study_element.attrib['name'].strip() image_filename = study_element.attrib['file'] image_fileobject = arch_results.extractfile(xml_obj.find(".").attrib['directory'] + "/" + image_filename) print image_name, image_filename, image_fileobject.size, "\n"
try: col = Collection.objects.get(DOI=post_dict['DOI']) except Collection.DoesNotExist: col = None if col and not col.description.endswith(version): col.DOI = None old_version = re.search(r"Version: (?P<version>\w)", col.description).group("version") col.name = study_name + " (version %s - deprecated)" % old_version col.save() if not col or not col.description.endswith(version): collection = Collection(owner=anima_user) form = CollectionForm(post_dict, instance=collection) form.is_valid() print form.errors collection = form.save() arch_response = requests.get( url.replace("library", "library/archives").replace(".study", ".tar.gz")) arch_results = tarfile.open(mode="r:gz", fileobj=StringIO(arch_response.content)) for study_element in xml_obj.findall( ".//StudyElement[@type='VolumeFile']"): image_name = study_element.attrib['name'].strip() image_filename = study_element.attrib['file'] image_fileobject = arch_results.extractfile(