def _get_metadata_formats_id(schemas, user_schemas, registries):
# We get all template versions for the given schemas
# First, we take care of user defined schema
templates_id_user = Template.objects(title__in=user_schemas).distinct(
field="id")
templates_id_user = [str(x) for x in templates_id_user]
# Take care of the rest, with versions
templates_versions = Template.objects(title__in=schemas).distinct(
field="templateVersion")
# We get all templates ID, for all versions
all_templates_id_common = TemplateVersion.objects(pk__in=templates_versions, isDeleted=False)\
.distinct(field="versions")
# We remove the removed version
all_templates_id_common_removed = TemplateVersion.objects(pk__in=templates_versions, isDeleted=False)\
.distinct( field="deletedVersions")
templates_id_common = list(
set(all_templates_id_common) - set(all_templates_id_common_removed))
templates_id = templates_id_user + templates_id_common
if len(registries) == 0:
# We retrieve deactivated registries so as not to get their metadata formats
deactivatedRegistries = [
str(x.id)
for x in OaiRegistry.objects(isDeactivated=True).order_by('id')
]
metadataFormatsID = OaiMetadataFormat.objects(
template__in=templates_id,
registry__not__in=deactivatedRegistries).distinct(field="id")
else:
# We retrieve registries from the refinement
metadataFormatsID = OaiMetadataFormat.objects(
template__in=templates_id,
registry__in=registries).distinct(field="id")
return metadataFormatsID
def test_delete_template_and_version(self):
numberTemplate = len(Template.objects())
numberTemplateVersion = len(TemplateVersion.objects())
template = self.createTemplate()
delete_template_and_version(str(template.id))
self.assertEquals(len(Template.objects()), numberTemplate)
self.assertEquals(len(TemplateVersion.objects()), numberTemplateVersion)
def createTemplateVersion(self):
countTemplateVersion = len(TemplateVersion.objects())
templateVersion = TemplateVersion(nbVersions=1,
isDeleted=False,
current=FAKE_ID).save()
self.assertEquals(len(TemplateVersion.objects()),
countTemplateVersion + 1)
return templateVersion
def test_delete_template_and_version(self):
numberTemplate = len(Template.objects())
numberTemplateVersion = len(TemplateVersion.objects())
template = self.createTemplate()
delete_template_and_version(str(template.id))
self.assertEquals(len(Template.objects()), numberTemplate)
self.assertEquals(len(TemplateVersion.objects()),
numberTemplateVersion)
def curate_from_schema(request):
try:
schema_name = request.GET['template']
templates = Template.objects(title=schema_name)
if 'curate_edit' in request.session and request.session['curate_edit'] == False:
# if the schemas are all versions of the same schema
if len(set(templates.values_list('templateVersion'))) == 1:
template_id = TemplateVersion.objects().get(pk=templates[0].templateVersion).current
request.session['currentTemplateID'] = template_id
#
# form_data = FormData(user=str(request.user.id), template=template_id, name=schema_name)
# form_data.save()
#
# request.session['curateFormData'] = str(form_data.pk)
# request.session['curate_edit'] = False
if 'useForm' in request.GET and request.GET['useForm'] == 'true':
pass
else:
if 'formString' in request.session:
del request.session['formString']
# if 'xmlDocTree' in request.session:
# del request.session['xmlDocTree']
else:
error_message = "The selection of template by name can't be used if the MDCS contain more than one "
error_message += "template with the same name."
raise MDCSError(error_message)
except:
raise MDCSError("The template you are looking for doesn't exist.")
def manage_schemas(request):
if request.user.is_authenticated() and request.user.is_staff:
template = loader.get_template('admin/manage_uploads.html')
currentTemplateVersions = []
for tpl_version in TemplateVersion.objects():
currentTemplateVersions.append(tpl_version.current)
currentTemplates = dict()
for tpl_version in currentTemplateVersions:
tpl = Template.objects.get(pk=tpl_version)
templateVersions = TemplateVersion.objects.get(
pk=tpl.templateVersion)
currentTemplates[tpl] = templateVersions.isDeleted
context = RequestContext(request, {
'objects': currentTemplates,
'objectType': "Template"
})
request.session['currentYear'] = currentYear()
return HttpResponse(template.render(context))
else:
if 'loggedOut' in request.session:
del request.session['loggedOut']
request.session['next'] = '/'
return redirect('/login')
def test_delete_object_template(self):
template = self.createTemplate()
url = '/dashboard/delete_object'
data = {'objectID': template.id, 'objectType': 'Template'}
r = self.doRequestPostAdminClientLogged(url=url, data=data)
self.assertEquals(0, len(Template.objects()))
self.assertEquals(0, len(TemplateVersion.objects()))
def index(request):
template = loader.get_template('curate.html')
request.session['currentYear'] = currentYear()
if request.user.is_authenticated():
currentTemplateVersions = []
for tpl_version in TemplateVersion.objects():
currentTemplateVersions.append(tpl_version.current)
currentTemplates = dict()
for tpl_version in currentTemplateVersions:
tpl = Template.objects.get(pk=tpl_version)
templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion)
currentTemplates[tpl] = templateVersions.isDeleted
context = RequestContext(request, {
'templates':currentTemplates,
'userTemplates': Template.objects(user=str(request.user.id)),
})
return HttpResponse(template.render(context))
else:
if 'loggedOut' in request.session:
del request.session['loggedOut']
request.session['next'] = '/curate'
return redirect('/login')
def index(request):
template = loader.get_template('explore.html')
request.session['currentYear'] = currentYear()
if request.user.is_authenticated():
currentTemplateVersions = []
for tpl_version in TemplateVersion.objects():
currentTemplateVersions.append(tpl_version.current)
currentTemplates = dict()
for tpl_version in currentTemplateVersions:
tpl = Template.objects.get(pk=tpl_version)
templateVersions = TemplateVersion.objects.get(
pk=tpl.templateVersion)
currentTemplates[tpl] = templateVersions.isDeleted
context = RequestContext(
request, {
'templates': currentTemplates,
'userTemplates': Template.objects(user=str(request.user.id)),
})
return HttpResponse(template.render(context))
else:
if 'loggedOut' in request.session:
del request.session['loggedOut']
request.session['next'] = '/explore'
return redirect('/login')
def manage_schemas(request):
if request.user.is_authenticated() and request.user.is_staff:
template = loader.get_template('admin/manage_uploads.html')
currentTemplateVersions = []
for tpl_version in TemplateVersion.objects():
currentTemplateVersions.append(tpl_version.current)
currentTemplates = dict()
for tpl_version in currentTemplateVersions:
tpl = Template.objects.get(pk=tpl_version)
templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion)
currentTemplates[tpl] = templateVersions.isDeleted
context = RequestContext(request, {
'objects':currentTemplates,
'objectType': "Template"
})
request.session['currentYear'] = currentYear()
return HttpResponse(template.render(context))
else:
if 'loggedOut' in request.session:
del request.session['loggedOut']
request.session['next'] = '/'
return redirect('/login')
def __init__(self, userId=""):
self.SCHEMAS_OPTIONS = []
self.SCHEMAS_USER_OPTIONS = []
#We retrieve all common template + user template
currentSchemas = TemplateVersion.objects(isDeleted=False).distinct(field="current")
schemas = Template.objects(pk__in=currentSchemas, user=None).distinct(field="title")
userSchemas = Template.objects(user=str(userId)).distinct(field="title")
for schema in schemas:
#We add them
self.SCHEMAS_OPTIONS.append((schema, schema))
for schema in userSchemas:
#We add them
self.SCHEMAS_USER_OPTIONS.append((schema, schema))
super(KeywordForm, self).__init__()
self.fields['my_schemas'].choices = []
self.fields['my_schemas'].choices = self.SCHEMAS_OPTIONS
self.fields['my_user_schemas'].choices = []
self.fields['my_user_schemas'].choices = self.SCHEMAS_USER_OPTIONS
self.my_schemas_nb = len(self.SCHEMAS_OPTIONS)
self.my_user_schemas_nb = len(self.SCHEMAS_USER_OPTIONS)
if self.my_schemas_nb + self.my_user_schemas_nb == 1:
self.fields['my_schemas'].widget.attrs['disabled'] = True
self.fields['my_user_schemas'].widget.attrs['disabled'] = True
def __init__(self, *args, **kwargs):
templatesVersionID = Template.objects.distinct(field="templateVersion")
templatesID = TemplateVersion.objects(pk__in=templatesVersionID, isDeleted=False).distinct(field="current")
templates = Template.objects(pk__in=templatesID).all()
super(MyTemplateMetadataFormatForm, self).__init__(*args, **kwargs)
self.fields['template'].queryset = []
self.fields['template'].queryset = templates
def test_delete_object_template(self):
template = self.createTemplate()
url = '/dashboard/delete_object'
data = {'objectID': template.id, 'objectType': 'Template'}
r = self.doRequestPostAdminClientLogged(url=url, data=data)
self.assertEquals(0, len(Template.objects()))
self.assertEquals(0, len(TemplateVersion.objects()))
def explore_all_versions_results(request):
template_id = request.GET['id']
template = Template.objects().get(pk=template_id)
version_id = template.templateVersion
template_version = TemplateVersion.objects().get(pk=version_id)
if len(template_version.versions) == 1:
query = {"schema": template_id}
else:
list_query = []
for version in template_version.versions:
list_query.append({'schema': version})
query = {"$or": list_query}
request.session['queryExplore'] = query
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
json_instances = [Instance(name="Local", protocol=protocol, address=request.META['REMOTE_ADDR'], port=request.META['SERVER_PORT'], access_token="token", refresh_token="token").to_json()]
request.session['instancesExplore'] = json_instances
template = loader.get_template('explore/explore_results.html')
context = RequestContext(request, {
'': '',
})
if 'exploreCurrentTemplateID' not in request.session:
return redirect('/explore/select-template')
else:
return HttpResponse(template.render(context))
def __init__(self, userId=""):
self.SCHEMAS_OPTIONS = []
self.SCHEMAS_USER_OPTIONS = []
#We retrieve all common template + user template
currentSchemas = TemplateVersion.objects(isDeleted=False).distinct(
field="current")
schemas = Template.objects(pk__in=currentSchemas,
user=None).distinct(field="title")
userSchemas = Template.objects(user=str(userId)).distinct(
field="title")
for schema in schemas:
#We add them
self.SCHEMAS_OPTIONS.append((schema, schema))
for schema in userSchemas:
#We add them
self.SCHEMAS_USER_OPTIONS.append((schema, schema))
super(KeywordForm, self).__init__()
#Init KeywordFormOAIPMH
self.form_oai_pmh = KeywordFormOAIPMH()
self.fields['my_schemas'].choices = []
self.fields['my_schemas'].choices = self.SCHEMAS_OPTIONS
self.fields['my_user_schemas'].choices = []
self.fields['my_user_schemas'].choices = self.SCHEMAS_USER_OPTIONS
self.my_schemas_nb = len(self.SCHEMAS_OPTIONS)
self.my_user_schemas_nb = len(self.SCHEMAS_USER_OPTIONS)
if self.my_schemas_nb + self.my_user_schemas_nb == 1:
self.fields['my_schemas'].widget.attrs['disabled'] = True
self.fields['my_user_schemas'].widget.attrs['disabled'] = True
def curate_from_schema(request):
try:
schema_name = request.GET['template']
templates = Template.objects(title=schema_name)
# if the schemas are all versions of the same schema
if len(set(templates.values_list('templateVersion'))) == 1:
template_id = TemplateVersion.objects().get(
pk=templates[0].templateVersion).current
request.session['currentTemplateID'] = template_id
form_data = FormData(user=str(request.user.id),
template=template_id,
name=schema_name)
form_data.save()
request.session['curateFormData'] = str(form_data.pk)
request.session['curate_edit'] = False
if 'xmlDocTree' in request.session:
del request.session['xmlDocTree']
else:
error_message = "The selection of template by name can't be used if the MDCS contain more than one "
error_message += "template with the same name."
raise MDCSError(error_message)
except:
raise MDCSError("The template you are looking for doesn't exist.")
def test_delete_template_with_dependencies(self):
template = self.createTemplate()
XMLdata(schemaID=str(template.id), title='testRecord', xml='<test>test xmldata</test>').save()
FormData(template=str(template.id), name='testFormData', xml_data='testest', user=str(1)).save()
listDependencies = delete_template(str(template.id))
self.assertEquals(len(Template.objects()), 1)
self.assertEquals(len(TemplateVersion.objects()), 1)
self.assertEquals(listDependencies, 'testFormData, testRecord')
def execute_test(self, file_id):
filename_inc = '{}_inc.xsd'.format(file_id)
# load XSD
xsd_file_path_inc = join(self.resources_path, filename_inc)
xsd_file_inc = open(xsd_file_path_inc, 'r')
xsd_file_content_inc = xsd_file_inc.read()
# create template
template_inc = create_template(xsd_file_content_inc, filename_inc,
filename_inc)
template_version = TemplateVersion.objects().get(
pk=template_inc.templateVersion)
# remove the dependency template
template_version.deletedVersions.append(str(template_inc.id))
template_version.isDeleted = True
template_version.save()
filename = '{}.xsd'.format(file_id)
# load XSD
xsd_file_path = join(self.resources_path, filename)
xsd_file = open(xsd_file_path, 'r')
xsd_file_content = xsd_file.read()
# create template
xsd_tree = etree.fromstring(xsd_file_content)
update_dependencies(xsd_tree, {filename_inc: str(template_inc.id)})
xsd_file_content = etree.tostring(xsd_tree)
template = create_template(xsd_file_content, filename, filename,
[str(template_inc.id)])
# load the form
load_new_form(self.driver, self.base_url, file_id)
time.sleep(TIMEOUT)
# download XML
self.driver.execute_script("downloadCurrentXML();")
# wait a bit more to let time to save the file
time.sleep(TIMEOUT * 5)
# load expected result
exp_result_path = join(self.resources_path, "{0}.xml".format(file_id))
exp_result_file = open(exp_result_path, 'r')
exp_result_content = exp_result_file.read()
# load result generated by the form
result_path = join(self.results_path, "{0}.xml".format(file_id))
result_file = open(result_path, 'r')
result_content = result_file.read()
expected = etree.fromstring(exp_result_content)
result = etree.fromstring(result_content)
if not are_equals(expected, result):
raise Exception('NOT EQUALS TO EXPECTED: {}'.format(file_id))
def _get_flatten_schema_and_namespaces(template_name):
schemas = Template.objects(title=template_name)
schema_id = TemplateVersion.objects().get(pk=schemas[0].templateVersion).current
schema = Template.objects().get(pk=schema_id)
flattener = XSDFlattenerURL(schema.content.encode('utf-8'))
ref_xml_schema_content = flattener.get_flat()
# find the namespaces
namespaces = common.get_namespaces(BytesIO(schema.content.encode('utf-8')))
return ref_xml_schema_content, namespaces
def test_delete_object_template_with_dependencie(self):
self.assertEquals(0, len(Template.objects()))
template = self.createTemplate()
self.assertEquals(1, len(Template.objects()))
self.createXMLData(schemaID=template.id)
url = '/dashboard/delete_object'
data = {'objectID': template.id, 'objectType': 'Template'}
r = self.doRequestPostAdminClientLogged(url=url, data=data)
self.assertEquals(1, len(Template.objects()))
self.assertEquals(1, len(TemplateVersion.objects()))
def test_delete_object_template_with_dependencie(self):
self.assertEquals(0, len(Template.objects()))
template = self.createTemplate()
self.assertEquals(1, len(Template.objects()))
self.createXMLData(schemaID=template.id)
url = '/dashboard/delete_object'
data = {'objectID': template.id, 'objectType': 'Template'}
r = self.doRequestPostAdminClientLogged(url=url, data=data)
self.assertEquals(1, len(Template.objects()))
self.assertEquals(1, len(TemplateVersion.objects()))
def execute_test(self, file_id):
filename_inc = '{}_inc.xsd'.format(file_id)
# load XSD
xsd_file_path_inc = join(self.resources_path, filename_inc)
xsd_file_inc = open(xsd_file_path_inc, 'r')
xsd_file_content_inc = xsd_file_inc.read()
# create template
template_inc = create_template(xsd_file_content_inc, filename_inc, filename_inc)
template_version = TemplateVersion.objects().get(pk=template_inc.templateVersion)
# remove the dependency template
template_version.deletedVersions.append(str(template_inc.id))
template_version.isDeleted = True
template_version.save()
filename = '{}.xsd'.format(file_id)
# load XSD
xsd_file_path = join(self.resources_path, filename)
xsd_file = open(xsd_file_path, 'r')
xsd_file_content = xsd_file.read()
# create template
xsd_tree = etree.fromstring(xsd_file_content)
update_dependencies(xsd_tree, {filename_inc: str(template_inc.id)})
xsd_file_content = etree.tostring(xsd_tree)
template = create_template(xsd_file_content, filename, filename, [str(template_inc.id)])
# load the form
load_new_form(self.driver, self.base_url, file_id)
time.sleep(TIMEOUT)
# download XML
self.driver.execute_script("downloadCurrentXML();")
# wait a bit more to let time to save the file
time.sleep(TIMEOUT * 5)
# load expected result
exp_result_path = join(self.resources_path, "{0}.xml".format(file_id))
exp_result_file = open(exp_result_path, 'r')
exp_result_content = exp_result_file.read()
# load result generated by the form
result_path = join(self.results_path, "{0}.xml".format(file_id))
result_file = open(result_path, 'r')
result_content = result_file.read()
expected = etree.fromstring(exp_result_content)
result = etree.fromstring(result_content)
if not are_equals(expected, result):
raise Exception('NOT EQUALS TO EXPECTED: {}'.format(file_id))
def test_delete_template_with_dependencies(self):
template = self.createTemplate()
XMLdata(schemaID=str(template.id),
title='testRecord',
xml='<test>test xmldata</test>').save()
FormData(template=str(template.id),
name='testFormData',
xml_data='testest',
user=str(1)).save()
listDependencies = delete_template(str(template.id))
self.assertEquals(len(Template.objects()), 1)
self.assertEquals(len(TemplateVersion.objects()), 1)
self.assertEquals(listDependencies, 'testFormData, testRecord')
def get_xsd(request, schema):
#TODO Available if publication ok and no user template
#We retrieve the schema filename in the schema attribute
#Get the templateVersion ID
try:
templatesVersionID = Template.objects(filename=schema).distinct(field="templateVersion")
templateID = TemplateVersion.objects(pk__in=templatesVersionID, isDeleted=False).distinct(field="current")
templates = Template.objects.get(pk__in=templateID)
#Get the XML schema
contentEncoded = templates.content.encode('utf-8')
fileObj = StringIO(contentEncoded)
return HttpResponse(fileObj, content_type='text/xml')
except MONGO_ERRORS.DoesNotExist, e:
return HttpResponseNotFound('Impossible to retrieve the schema with the given name.')
def get_xsd(request, schema):
#TODO Available if publication ok and no user template
#We retrieve the schema filename in the schema attribute
#Get the templateVersion ID
try:
templatesVersionID = Template.objects(filename=schema).distinct(field="templateVersion")
templateID = TemplateVersion.objects(pk__in=templatesVersionID, isDeleted=False).distinct(field="current")
templates = Template.objects.get(pk__in=templateID)
#Get the XML schema
contentEncoded = templates.content.encode('utf-8')
fileObj = StringIO(contentEncoded)
return HttpResponse(fileObj, content_type='text/xml')
except MONGO_ERRORS.DoesNotExist, e:
return HttpResponseNotFound('Impossible to retrieve the schema with the given name.')
def index(request):
template = loader.get_template('compose/compose.html')
currentTemplateVersions = []
for tpl_version in TemplateVersion.objects():
currentTemplateVersions.append(tpl_version.current)
currentTemplates = dict()
for tpl_version in currentTemplateVersions:
tpl = Template.objects.get(pk=tpl_version)
templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion)
currentTemplates[tpl] = templateVersions.isDeleted
context = RequestContext(request, {
'templates': currentTemplates,
'userTemplates': Template.objects(user=str(request.user.id)),
})
return HttpResponse(template.render(context))
def manage_schemas(request):
template = loader.get_template('admin/manage_uploads.html')
currentTemplateVersions = []
for tpl_version in TemplateVersion.objects():
currentTemplateVersions.append(tpl_version.current)
currentTemplates = dict()
for tpl_version in currentTemplateVersions:
tpl = Template.objects.get(pk=tpl_version)
templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion)
currentTemplates[tpl] = templateVersions.isDeleted
context = RequestContext(request, {
'objects':currentTemplates,
'objectType': "Template"
})
return HttpResponse(template.render(context))
def manage_schemas(request):
template = loader.get_template('admin/manage_uploads.html')
currentTemplateVersions = []
for tpl_version in TemplateVersion.objects():
currentTemplateVersions.append(tpl_version.current)
currentTemplates = dict()
for tpl_version in currentTemplateVersions:
tpl = Template.objects.get(pk=tpl_version)
templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion)
currentTemplates[tpl] = templateVersions.isDeleted
context = RequestContext(request, {
'objects': currentTemplates,
'objectType': "Template"
})
return HttpResponse(template.render(context))
def index(request):
currentTemplateVersions = []
for tpl_version in TemplateVersion.objects():
currentTemplateVersions.append(tpl_version.current)
currentTemplates = dict()
for tpl_version in currentTemplateVersions:
tpl = Template.objects.get(pk=tpl_version)
templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion)
currentTemplates[tpl] = templateVersions.isDeleted
template = loader.get_template('explore/explore.html')
context = RequestContext(request, {
'templates':currentTemplates,
'userTemplates': Template.objects(user=str(request.user.id)),
})
return HttpResponse(template.render(context))
def get_xsd(request, schema):
# TODO Available if publication ok and no user template
# We retrieve the schema filename in the schema attribute
# Get the templateVersion ID
try:
templatesVersionID = Template.objects(filename=schema).distinct(
field="templateVersion")
templateID = TemplateVersion.objects(
pk__in=templatesVersionID,
isDeleted=False).distinct(field="current")
template = Template.objects.get(pk=templateID[0])
flattener = XSDFlattenerDatabaseOrURL(template.content.encode('utf-8'))
content_encoded = flattener.get_flat()
file_obj = StringIO(content_encoded)
return HttpResponse(file_obj, content_type='text/xml')
except Exception, e:
return HttpResponseBadRequest(
'Impossible to retrieve the schema with the given name.')
def explore_all_versions_results(request):
template_id = request.GET['id']
template = Template.objects().get(pk=template_id)
version_id = template.templateVersion
template_version = TemplateVersion.objects().get(pk=version_id)
if len(template_version.versions) == 1:
query = {"schema": template_id}
else:
list_query = []
for version in template_version.versions:
list_query.append({'schema': version})
query = {"$or": list_query}
request.session['queryExplore'] = query
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
json_instances = [
Instance(name="Local",
protocol=protocol,
address=request.META['REMOTE_ADDR'],
port=request.META['SERVER_PORT'],
access_token="token",
refresh_token="token").to_json()
]
request.session['instancesExplore'] = json_instances
template = loader.get_template('explore/explore_results.html')
context = RequestContext(request, {
'': '',
})
if 'exploreCurrentTemplateID' not in request.session:
return redirect('/explore/select-template')
else:
return HttpResponse(template.render(context))
def get_results_by_instance_keyword(request):
print 'BEGIN def getResultsKeyword(request)'
resultsByKeyword = []
results = []
resultString = ""
#Instance
json_instances = []
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
instance = Instance(name="Local", protocol=protocol, address=request.META['REMOTE_ADDR'], port=request.META['SERVER_PORT'], access_token="token", refresh_token="token")
json_instances.append(instance.to_json())
request.session['instancesExplore'] = json_instances
sessionName = "resultsExploreOaiPMh" + instance['name']
try:
keyword = request.GET['keyword']
schemas = request.GET.getlist('schemas[]')
userSchemas = request.GET.getlist('userSchemas[]')
refinements = refinements_to_mongo(request.GET.getlist('refinements[]'))
if 'onlySuggestions' in request.GET:
onlySuggestions = json.loads(request.GET['onlySuggestions'])
else:
onlySuggestions = False
except:
keyword = ''
schemas = []
userSchemas = []
refinements = {}
onlySuggestions = True
#We get all template versions for the given schemas
#First, we take care of user defined schema
templatesIDUser = Template.objects(title__in=userSchemas).distinct(field="id")
templatesIDUser = [str(x) for x in templatesIDUser]
#Take care of the rest, with versions
templatesVersions = Template.objects(title__in=schemas).distinct(field="templateVersion")
#We get all templates ID, for all versions
allTemplatesIDCommon = TemplateVersion.objects(pk__in=templatesVersions, isDeleted=False).distinct(field="versions")
#We remove the removed version
allTemplatesIDCommonRemoved = TemplateVersion.objects(pk__in=templatesVersions, isDeleted=False).distinct(field="deletedVersions")
templatesIDCommon = list(set(allTemplatesIDCommon) - set(allTemplatesIDCommonRemoved))
templatesID = templatesIDUser + templatesIDCommon
#We retrieve deactivated registries so as not to get their metadata formats
deactivatedRegistries = [str(x.id) for x in OaiRegistry.objects(isDeactivated=True).order_by('id')]
metadataFormatsID = OaiMetadataFormat.objects(template__in=templatesID, registry__not__in=deactivatedRegistries).distinct(field="id")
instanceResults = OaiRecord.executeFullTextQuery(keyword, metadataFormatsID, refinements)
if len(instanceResults) > 0:
if not onlySuggestions:
xsltPath = os.path.join(settings.SITE_ROOT, 'static/resources/xsl/xml2html.xsl')
xslt = etree.parse(xsltPath)
transform = etree.XSLT(xslt)
template = loader.get_template('oai_pmh/explore/explore_result_keyword.html')
#Retrieve schema and registries. Avoid to retrieve the information for each result
registriesName = {}
objMetadataFormats = {}
listRegistriesID = set([x['registry'] for x in instanceResults])
for registryId in listRegistriesID:
obj = OaiRegistry.objects(pk=registryId).get()
registriesName[str(registryId)] = obj.name
listSchemaId = set([x['metadataformat'] for x in instanceResults])
for schemaId in listSchemaId:
obj = OaiMetadataFormat.objects(pk=schemaId).get()
objMetadataFormats[str(schemaId)] = obj
listItems = []
xmltodictunparse = xmltodict.unparse
appendResult = results.append
toXML = etree.XML
parse = etree.parse
XSLT = etree.XSLT
if not onlySuggestions:
for instanceResult in instanceResults:
custom_xslt = False
appendResult({'title':instanceResult['identifier'], 'content':xmltodictunparse(instanceResult['metadata']),'id':str(instanceResult['_id'])})
dom = toXML(str(xmltodictunparse(instanceResult['metadata']).encode('utf-8')))
#Check if a custom list result XSLT has to be used
try:
metadataFormat = objMetadataFormats[str(instanceResult['metadataformat'])]
if metadataFormat.template.ResultXsltList:
listXslt = parse(BytesIO(metadataFormat.template.ResultXsltList.content.encode('utf-8')))
listTransform = XSLT(listXslt)
newdom = listTransform(dom)
custom_xslt = True
else:
newdom = transform(dom)
except Exception, e:
#We use the default one
newdom = transform(dom)
custom_xslt = False
context = RequestContext(request, {'id':str(instanceResult['_id']),
'xml': str(newdom),
'title': instanceResult['identifier'],
'custom_xslt': custom_xslt,
'schema_name': metadataFormat.metadataPrefix,
'registry_name': registriesName[instanceResult['registry']],
'oai_pmh': True})
resultString+= template.render(context)
else:
for instanceResult in instanceResults[:20]:
wordList = re.sub("[^\w]", " ", keyword).split()
wordList = [x + "|" + x +"\w+" for x in wordList]
wordList = '|'.join(wordList)
listWholeKeywords = re.findall("\\b("+ wordList +")\\b", xmltodict.unparse(instanceResult['metadata']).encode('utf-8'), flags=re.IGNORECASE)
labels = list(set(listWholeKeywords))
for label in labels:
label = label.lower()
result_json = {}
result_json['label'] = label
result_json['value'] = label
if not result_json in resultsByKeyword:
resultsByKeyword.append(result_json)
def get_results_by_instance_keyword(request):
print 'BEGIN def getResultsKeyword(request)'
resultsByKeyword = []
results = []
resultString = ""
#Instance
json_instances = []
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
instance = Instance(name="Local",
protocol=protocol,
address=request.META['REMOTE_ADDR'],
port=request.META['SERVER_PORT'],
access_token="token",
refresh_token="token")
json_instances.append(instance.to_json())
request.session['instancesExplore'] = json_instances
sessionName = "resultsExploreOaiPMh" + instance['name']
try:
keyword = request.GET['keyword']
schemas = request.GET.getlist('schemas[]')
userSchemas = request.GET.getlist('userSchemas[]')
refinements = refinements_to_mongo(
request.GET.getlist('refinements[]'))
if 'onlySuggestions' in request.GET:
onlySuggestions = json.loads(request.GET['onlySuggestions'])
else:
onlySuggestions = False
except:
keyword = ''
schemas = []
userSchemas = []
refinements = {}
onlySuggestions = True
#We get all template versions for the given schemas
#First, we take care of user defined schema
templatesIDUser = Template.objects(title__in=userSchemas).distinct(
field="id")
templatesIDUser = [str(x) for x in templatesIDUser]
#Take care of the rest, with versions
templatesVersions = Template.objects(title__in=schemas).distinct(
field="templateVersion")
#We get all templates ID, for all versions
allTemplatesIDCommon = TemplateVersion.objects(
pk__in=templatesVersions, isDeleted=False).distinct(field="versions")
#We remove the removed version
allTemplatesIDCommonRemoved = TemplateVersion.objects(
pk__in=templatesVersions,
isDeleted=False).distinct(field="deletedVersions")
templatesIDCommon = list(
set(allTemplatesIDCommon) - set(allTemplatesIDCommonRemoved))
templatesID = templatesIDUser + templatesIDCommon
#We retrieve deactivated registries so as not to get their metadata formats
deactivatedRegistries = [
str(x.id)
for x in OaiRegistry.objects(isDeactivated=True).order_by('id')
]
metadataFormatsID = OaiMetadataFormat.objects(
template__in=templatesID,
registry__not__in=deactivatedRegistries).distinct(field="id")
instanceResults = OaiRecord.executeFullTextQuery(keyword,
metadataFormatsID,
refinements)
if len(instanceResults) > 0:
if not onlySuggestions:
xsltPath = os.path.join(settings.SITE_ROOT,
'static/resources/xsl/xml2html.xsl')
xslt = etree.parse(xsltPath)
transform = etree.XSLT(xslt)
template = loader.get_template(
'oai_pmh/explore/explore_result_keyword.html')
#Retrieve schema and registries. Avoid to retrieve the information for each result
registriesName = {}
objMetadataFormats = {}
listRegistriesID = set([x['registry'] for x in instanceResults])
for registryId in listRegistriesID:
obj = OaiRegistry.objects(pk=registryId).get()
registriesName[str(registryId)] = obj.name
listSchemaId = set([x['metadataformat'] for x in instanceResults])
for schemaId in listSchemaId:
obj = OaiMetadataFormat.objects(pk=schemaId).get()
objMetadataFormats[str(schemaId)] = obj
listItems = []
xmltodictunparse = xmltodict.unparse
appendResult = results.append
toXML = etree.XML
parse = etree.parse
XSLT = etree.XSLT
if not onlySuggestions:
for instanceResult in instanceResults:
custom_xslt = False
appendResult({
'title':
instanceResult['identifier'],
'content':
xmltodictunparse(instanceResult['metadata']),
'id':
str(instanceResult['_id'])
})
dom = toXML(
str(
xmltodictunparse(
instanceResult['metadata']).encode('utf-8')))
#Check if a custom list result XSLT has to be used
try:
metadataFormat = objMetadataFormats[str(
instanceResult['metadataformat'])]
if metadataFormat.template.ResultXsltList:
listXslt = parse(
BytesIO(
metadataFormat.template.ResultXsltList.content.
encode('utf-8')))
listTransform = XSLT(listXslt)
newdom = listTransform(dom)
custom_xslt = True
else:
newdom = transform(dom)
except Exception, e:
#We use the default one
newdom = transform(dom)
custom_xslt = False
context = RequestContext(
request, {
'id': str(instanceResult['_id']),
'xml': str(newdom),
'title': instanceResult['identifier'],
'custom_xslt': custom_xslt,
'schema_name': metadataFormat.metadataPrefix,
'registry_name':
registriesName[instanceResult['registry']],
'oai_pmh': True
})
resultString += template.render(context)
else:
for instanceResult in instanceResults[:20]:
wordList = re.sub("[^\w]", " ", keyword).split()
wordList = [x + "|" + x + "\w+" for x in wordList]
wordList = '|'.join(wordList)
listWholeKeywords = re.findall(
"\\b(" + wordList + ")\\b",
xmltodict.unparse(
instanceResult['metadata']).encode('utf-8'),
flags=re.IGNORECASE)
labels = list(set(listWholeKeywords))
for label in labels:
label = label.lower()
result_json = {}
result_json['label'] = label
result_json['value'] = label
if not result_json in resultsByKeyword:
resultsByKeyword.append(result_json)
def dump_template_version(self):
self.assertEquals(len(TemplateVersion.objects()), 0)
self.restoreDump(join(DUMP_TEST_PATH, 'template_version.bson'),
'template_version')
self.assertTrue(len(TemplateVersion.objects()) > 0)
def test_delete_template_no_dependencies(self):
template = self.createTemplate()
delete_template(str(template.id))
self.assertEquals(len(Template.objects()), 0)
self.assertEquals(len(TemplateVersion.objects()), 0)
def start_curate(request):
if 'currentTemplateID' not in request.session and 'template' not in request.GET:
return redirect('/curate/select-template')
else:
if request.method == 'POST':
# parameters to build FormData object in db
user = request.user.id
template_id = request.session['currentTemplateID']
form_data = None
selected_option = request.POST['curate_form']
if selected_option == "new":
request.session['curate_edit'] = False
new_form = NewForm(request.POST)
try:
form_data = FormData(user=str(user),
template=template_id,
name=new_form.data['document_name'])
form_data.save()
except:
return HttpResponseBadRequest(
'Unable to create the form. A form with the same name may already exist.'
)
elif selected_option == "open":
request.session['curate_edit'] = True
open_form = OpenForm(request.POST)
form_data = FormData.objects.get(
pk=ObjectId(open_form.data['forms']))
elif selected_option == "upload":
request.session['curate_edit'] = True
upload_form = UploadForm(request.POST, request.FILES)
xml_file = request.FILES['file']
# put the cursor at the beginning of the file
xml_file.seek(0)
# read the content of the file
xml_data = xml_file.read()
# check XML data or not?
try:
etree.fromstring(xml_data)
except XMLSyntaxError:
return HttpResponseBadRequest(
'Uploaded File is not well formed XML.')
try:
form_data = FormData(user=str(user),
template=template_id,
name=xml_file.name,
xml_data=xml_data).save()
except:
return HttpResponseBadRequest(
'Unable to create the form. A form with the same name may already exist.'
)
# parameters that will be used during curation
request.session['curateFormData'] = str(form_data.id)
return HttpResponse('ok')
else:
try:
ajaxCall = False
context_params = dict()
if 'template' in request.GET:
schema_name = request.GET['template']
context_params['template_name'] = schema_name
try:
templates = Template.objects(title=schema_name)
except:
raise MDCSError(
"The template you are looking for doesn't exist.")
# if the schemas are all versions of the same schema
if len(set(templates.values_list('templateVersion'))) == 1:
template_id = TemplateVersion.objects().get(
pk=templates[0].templateVersion).current
request.session['currentTemplateID'] = template_id
else:
raise MDCSError(
"The selection of template by name can't be used if the MDCS contain more than "
"one template with the same name.")
template = loader.get_template(
'curate/curate_full_start.html')
if 'xmlDocTree' in request.session:
del request.session['xmlDocTree']
else:
ajaxCall = True
template = loader.get_template('curate/curate_start.html')
open_form = OpenForm(forms=FormData.objects(
user=str(request.user.id),
template=request.session['currentTemplateID'],
xml_data_id__exists=False))
new_form = NewForm()
upload_form = UploadForm()
context_params['new_form'] = new_form
context_params['open_form'] = open_form
context_params['upload_form'] = upload_form
context = RequestContext(
request,
context_params) # , 'options_form': options_form})
if ajaxCall:
return HttpResponse(json.dumps(
{'template': template.render(context)}),
content_type='application/javascript')
else:
return HttpResponse(template.render(context))
except MDCSError, e:
template = loader.get_template('curate/errors.html')
context = RequestContext(request, {
'errors': e.message,
})
return HttpResponse(template.render(context))
def start_curate(request):
if 'currentTemplateID' not in request.session and 'template' not in request.GET:
return redirect('/curate/select-template')
else:
if request.method == 'POST':
# parameters to build FormData object in db
user = request.user.id
template_id = request.session['currentTemplateID']
form_data = None
selected_option = request.POST['curate_form']
if selected_option == "new":
request.session['curate_edit'] = False
new_form = NewForm(request.POST)
try:
form_data = FormData(user=str(user), template=template_id, name=new_form.data['document_name'],
xml_data=None).save()
except:
return HttpResponseBadRequest(
'Unable to create the form. A form with the same name may already exists.')
elif selected_option == "open":
request.session['curate_edit'] = True
open_form = OpenForm(request.POST)
form_data = FormData.objects.get(pk=ObjectId(open_form.data['forms']))
request.session['curate_edit_data'] = form_data.xml_data
elif selected_option == "upload":
request.session['curate_edit'] = True
upload_form = UploadForm(request.POST, request.FILES)
xml_file = request.FILES['file']
# put the cursor at the beginning of the file
xml_file.seek(0)
# read the content of the file
xml_data = xml_file.read()
# check XML data or not?
try:
etree.fromstring(xml_data)
except XMLSyntaxError:
return HttpResponseBadRequest('Uploaded File is not well formed XML.')
try:
form_data = FormData(user=str(user), template=template_id, name=xml_file.name,
xml_data=xml_data).save()
except:
return HttpResponseBadRequest(
'Unable to create the form. A form with the same name may already exist.')
# parameters that will be used during curation
request.session['curateFormData'] = str(form_data.id)
return HttpResponse('ok')
else:
try:
ajaxCall = False
context_params = dict()
if 'template' in request.GET:
schema_name = request.GET['template']
context_params['template_name'] = schema_name
try:
templates = Template.objects(title=schema_name)
except:
raise MDCSError("The template you are looking for doesn't exist.")
# if the schemas are all versions of the same schema
if len(set(templates.values_list('templateVersion'))) == 1:
template_id = TemplateVersion.objects().get(pk=templates[0].templateVersion).current
request.session['currentTemplateID'] = template_id
else:
raise MDCSError("The selection of template by name can't be used if the MDCS contain more than one template with the same name.")
template = loader.get_template('curate/curate_full_start.html')
if 'formString' in request.session:
del request.session['formString']
if 'xmlDocTree' in request.session:
del request.session['xmlDocTree']
else:
ajaxCall = True
template = loader.get_template('curate/curate_start.html')
open_form = OpenForm(
forms=FormData.objects(user=str(request.user.id), template=request.session['currentTemplateID'],
xml_data_id__exists=False))
new_form = NewForm()
upload_form = UploadForm()
context_params['new_form'] = new_form
context_params['open_form'] = open_form
context_params['upload_form'] = upload_form
context = RequestContext(request, context_params) # , 'options_form': options_form})
if ajaxCall:
return HttpResponse(json.dumps({'template': template.render(context)}),
content_type='application/javascript')
else:
return HttpResponse(template.render(context))
except MDCSError, e:
template = loader.get_template('curate/errors.html')
context = RequestContext(request, {
'errors': e.message,
})
return HttpResponse(template.render(context))
def dump_template_version(self):
self.assertEquals(len(TemplateVersion.objects()), 0)
self.restoreDump(join(DUMP_TEST_PATH, 'template_version.bson'), 'template_version')
self.assertTrue(len(TemplateVersion.objects()) > 0)
def test_delete_template_no_dependencies(self):
template = self.createTemplate()
delete_template(str(template.id))
self.assertEquals(len(Template.objects()), 0)
self.assertEquals(len(TemplateVersion.objects()), 0)
def test_delete_template_and_version(self):
template = self.createTemplate()
delete_template_and_version(str(template.id))
self.assertEquals(len(Template.objects()), 0)
self.assertEquals(len(TemplateVersion.objects()), 0)