def edit_collection(request, cid=None):
page_header = "Add new collection"
if cid:
collection = get_collection(cid, request)
is_owner = True if collection.owner == request.user else False
page_header = 'Edit collection'
if not owner_or_contrib(request, collection):
return HttpResponseForbidden()
else:
is_owner = True
collection = Collection(owner=request.user)
if request.method == "POST":
if is_owner:
form = OwnerCollectionForm(request.POST,
request.FILES,
instance=collection)
else:
form = CollectionForm(request.POST,
request.FILES,
instance=collection)
if form.is_valid():
previous_contribs = set()
if form.instance.pk is not None:
previous_contribs = set(form.instance.contributors.all())
collection = form.save(commit=False)
if collection.private and collection.private_token is None:
collection.private_token = generate_url_token()
collection.save()
if is_owner:
form.save_m2m() # save contributors
current_contribs = set(collection.contributors.all())
new_contribs = list(
current_contribs.difference(previous_contribs))
context = {
'owner': collection.owner.username,
'collection': collection.name,
'url':
get_server_url(request) + collection.get_absolute_url(),
}
subj = '%s has added you to a NeuroVault collection' % context[
'owner']
send_email_notification('new_contributor', subj, new_contribs,
context)
return HttpResponseRedirect(collection.get_absolute_url())
else:
if is_owner:
form = OwnerCollectionForm(instance=collection)
else:
form = CollectionForm(instance=collection)
context = {"form": form, "page_header": page_header, "is_owner": is_owner}
return render(request, "statmaps/edit_collection.html.haml", context)
def crawl_anima():
import neurovault.apps.statmaps.models as models
from neurovault.apps.statmaps.forms import StatisticMapForm, CollectionForm
username = "******"
email = "*****@*****.**"
try:
anima_user = models.User.objects.create_user(username, email)
anima_user.save()
except IntegrityError:
anima_user = models.User.objects.get(username=username, email=email)
url = "http://anima.modelgui.org/api/studies"
response = urllib.urlopen(url);
datasets = json.loads(response.read())
# results = tarfile.open(mode="r:gz", fileobj=StringIO(response.content))
# for member in results.getmembers():
# f = results.extractfile(member)
# if member.name.endswith(".study"):
for url in datasets:
response = requests.get(url)
content = response.content.replace("PubMed ID", "PubMedID")
xml_obj = e.fromstring(content)
version = xml_obj.find(".").find(".//Element[@name='Version']").text.strip()
study_description = xml_obj.find(".//Element[@name='Description']").text.strip()
study_description += " This dataset was automatically imported from the ANIMA <http://anima.modelgui.org/> database. Version: %s"%version
study_name = xml_obj.find(".").attrib['name']
tags = xml_obj.find(".//Element[@name='Keywords']").text.strip().split(";")
tags.append("ANIMA")
doi = xml_obj.find(".//Element[@name='DOI']")
pubmedid = xml_obj.find(".//Element[@name='PubMedID']")
post_dict = {
'name': study_name,
'description': study_description,
'full_dataset_url': "http://anima.modelgui.org/studies/" + os.path.split(url)[1].replace(".study", "")
}
if doi != None:
post_dict['DOI'] = doi.text.strip()
elif pubmedid != None:
pubmedid = pubmedid.text.strip()
url = "http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids=%s&format=json" % pubmedid
response = urllib.urlopen(url);
parsed = json.loads(response.read())
post_dict['DOI'] = parsed['records'][0]['doi']
try:
col = models.Collection.objects.get(DOI=post_dict['DOI'])
except models.Collection.DoesNotExist:
col = None
if col and not col.description.endswith(version):
col.DOI = None
old_version = re.search(r"Version: (?P<version>\w)", col.description).group("version")
col.name = study_name + " (version %s - deprecated)"%old_version
col.save()
if not col or not col.description.endswith(version):
collection = models.Collection(owner=anima_user)
form = CollectionForm(post_dict, instance=collection)
form.is_valid()
collection = form.save()
arch_response = requests.get(url.replace("library", "library/archives").replace(".study", ".tar.gz"))
arch_results = tarfile.open(mode="r:gz", fileobj=StringIO(arch_response.content))
for study_element in xml_obj.findall(".//StudyElement[@type='VolumeFile']"):
image_name = study_element.attrib['name'].strip()
image_filename = study_element.attrib['file']
image_fileobject = arch_results.extractfile(xml_obj.find(".").attrib['directory'] + "/" + image_filename)
map_type = models.BaseStatisticMap.OTHER
quantity_dict = {"Mask": models.BaseStatisticMap.M,
"F-statistic": models.BaseStatisticMap.F,
"T-statistic": models.BaseStatisticMap.T,
"Z-statistic": models.BaseStatisticMap.Z,
"Beta": models.BaseStatisticMap.U}
quantity = study_element.find("./Metadata/Element[@name='Quantity']")
if quantity != None:
quantity = quantity.text.strip()
if quantity in quantity_dict.keys():
map_type = quantity_dict[quantity]
post_dict = {
'name': image_name,
'modality': models.StatisticMap.fMRI_BOLD,
'map_type': map_type,
'analysis_level': models.BaseStatisticMap.M,
'collection': collection.pk,
'ignore_file_warning': True,
'cognitive_paradigm_cogatlas': 'None',
'tags': ", ".join(tags)
}
image_description = study_element.find("./Metadata/Element[@name='Caption']").text
if image_description:
post_dict["description"] = image_description.strip()
file_dict = {'file': SimpleUploadedFile(image_filename, image_fileobject.read())}
form = StatisticMapForm(post_dict, file_dict)
form.is_valid()
form.save()
def crawl_anima():
import neurovault.apps.statmaps.models as models
from neurovault.apps.statmaps.forms import StatisticMapForm, CollectionForm
username = "******"
email = "*****@*****.**"
try:
anima_user = models.User.objects.create_user(username, email)
anima_user.save()
except IntegrityError:
anima_user = models.User.objects.get(username=username, email=email)
url = "http://anima.fz-juelich.de/api/studies"
response = urllib.urlopen(url)
datasets = json.loads(response.read())
# results = tarfile.open(mode="r:gz", fileobj=StringIO(response.content))
# for member in results.getmembers():
# f = results.extractfile(member)
# if member.name.endswith(".study"):
for url in datasets:
response = requests.get(url)
content = response.content.replace("PubMed ID", "PubMedID")
xml_obj = e.fromstring(content)
version = xml_obj.find(".").find(
".//Element[@name='Version']").text.strip()
study_description = xml_obj.find(
".//Element[@name='Description']").text.strip()
study_description += " This dataset was automatically imported from the ANIMA <http://anima.fz-juelich.de/> database. Version: %s" % version
study_name = xml_obj.find(".").attrib['name']
tags = xml_obj.find(".//Element[@name='Keywords']").text.strip().split(
";")
tags.append("ANIMA")
doi = xml_obj.find(".//Element[@name='DOI']")
pubmedid = xml_obj.find(".//Element[@name='PubMedID']")
post_dict = {
'name':
study_name,
'description':
study_description,
'full_dataset_url':
"http://anima.fz-juelich.de/studies/" +
os.path.split(url)[1].replace(".study", "")
}
if doi != None:
post_dict['DOI'] = doi.text.strip()
elif pubmedid != None:
pubmedid = pubmedid.text.strip()
url = "http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids=%s&format=json" % pubmedid
response = urllib.urlopen(url)
parsed = json.loads(response.read())
post_dict['DOI'] = parsed['records'][0]['doi']
try:
col = models.Collection.objects.get(DOI=post_dict['DOI'])
except models.Collection.DoesNotExist:
col = None
if col and not col.description.endswith(version):
col.DOI = None
old_version = re.search(r"Version: (?P<version>\w)",
col.description).group("version")
col.name = study_name + " (version %s - deprecated)" % old_version
col.save()
if not col or not col.description.endswith(version):
collection = models.Collection(owner=anima_user)
form = CollectionForm(post_dict, instance=collection)
form.is_valid()
collection = form.save()
arch_response = requests.get(
url.replace("library",
"library/archives").replace(".study", ".tar.gz"))
arch_results = tarfile.open(mode="r:gz",
fileobj=StringIO(
arch_response.content))
for study_element in xml_obj.findall(
".//StudyElement[@type='VolumeFile']"):
image_name = study_element.attrib['name'].strip()
image_filename = study_element.attrib['file']
try:
image_fileobject = arch_results.extractfile(
xml_obj.find(".").attrib['directory'] + "/" +
image_filename)
except KeyError:
image_fileobject = arch_results.extractfile(
xml_obj.find(".").attrib['directory'] + "/" +
xml_obj.find(".").attrib['directory'] + "/" +
image_filename)
map_type = models.BaseStatisticMap.OTHER
quantity_dict = {
"Mask": models.BaseStatisticMap.M,
"F-statistic": models.BaseStatisticMap.F,
"T-statistic": models.BaseStatisticMap.T,
"Z-statistic": models.BaseStatisticMap.Z,
"Beta": models.BaseStatisticMap.U
}
quantity = study_element.find(
"./Metadata/Element[@name='Quantity']")
if quantity != None:
quantity = quantity.text.strip()
if quantity in quantity_dict.keys():
map_type = quantity_dict[quantity]
post_dict = {
'name': image_name,
'modality': models.StatisticMap.fMRI_BOLD,
'map_type': map_type,
'analysis_level': models.BaseStatisticMap.M,
'collection': collection.pk,
'ignore_file_warning': True,
'cognitive_paradigm_cogatlas': 'None',
'tags': ", ".join(tags)
}
image_description = study_element.find(
"./Metadata/Element[@name='Caption']").text
if image_description:
post_dict["description"] = image_description.strip()
file_dict = {
'file':
SimpleUploadedFile(image_filename, image_fileobject.read())
}
form = StatisticMapForm(post_dict, file_dict)
form.is_valid()
form.save()
post_dict['DOI'] = parsed['records'][0]['doi']
try:
col = Collection.objects.get(DOI=post_dict['DOI'])
except Collection.DoesNotExist:
col = None
if col and not col.description.endswith(version):
col.DOI = None
old_version = re.search(r"Version: (?P<version>\w)", col.description).group("version")
col.name = study_name + " (version %s - deprecated)"%old_version
col.save()
if not col or not col.description.endswith(version):
collection = Collection(owner=anima_user)
form = CollectionForm(post_dict, instance=collection)
form.is_valid()
print form.errors
collection = form.save()
arch_response = requests.get(url.replace("library", "library/archives").replace(".study", ".tar.gz"))
arch_results = tarfile.open(mode="r:gz", fileobj=StringIO(arch_response.content))
for study_element in xml_obj.findall(".//StudyElement[@type='VolumeFile']"):
image_name = study_element.attrib['name'].strip()
image_filename = study_element.attrib['file']
image_fileobject = arch_results.extractfile(xml_obj.find(".").attrib['directory'] + "/" + image_filename)
print image_name, image_filename, image_fileobject.size, "\n"
try:
col = Collection.objects.get(DOI=post_dict['DOI'])
except Collection.DoesNotExist:
col = None
if col and not col.description.endswith(version):
col.DOI = None
old_version = re.search(r"Version: (?P<version>\w)",
col.description).group("version")
col.name = study_name + " (version %s - deprecated)" % old_version
col.save()
if not col or not col.description.endswith(version):
collection = Collection(owner=anima_user)
form = CollectionForm(post_dict, instance=collection)
form.is_valid()
print form.errors
collection = form.save()
arch_response = requests.get(
url.replace("library",
"library/archives").replace(".study", ".tar.gz"))
arch_results = tarfile.open(mode="r:gz",
fileobj=StringIO(arch_response.content))
for study_element in xml_obj.findall(
".//StudyElement[@type='VolumeFile']"):
image_name = study_element.attrib['name'].strip()
image_filename = study_element.attrib['file']
image_fileobject = arch_results.extractfile(
