def _get_metadata_formats_id(schemas, user_schemas, registries): # We get all template versions for the given schemas # First, we take care of user defined schema templates_id_user = Template.objects(title__in=user_schemas).distinct( field="id") templates_id_user = [str(x) for x in templates_id_user] # Take care of the rest, with versions templates_versions = Template.objects(title__in=schemas).distinct( field="templateVersion") # We get all templates ID, for all versions all_templates_id_common = TemplateVersion.objects(pk__in=templates_versions, isDeleted=False)\ .distinct(field="versions") # We remove the removed version all_templates_id_common_removed = TemplateVersion.objects(pk__in=templates_versions, isDeleted=False)\ .distinct( field="deletedVersions") templates_id_common = list( set(all_templates_id_common) - set(all_templates_id_common_removed)) templates_id = templates_id_user + templates_id_common if len(registries) == 0: # We retrieve deactivated registries so as not to get their metadata formats deactivatedRegistries = [ str(x.id) for x in OaiRegistry.objects(isDeactivated=True).order_by('id') ] metadataFormatsID = OaiMetadataFormat.objects( template__in=templates_id, registry__not__in=deactivatedRegistries).distinct(field="id") else: # We retrieve registries from the refinement metadataFormatsID = OaiMetadataFormat.objects( template__in=templates_id, registry__in=registries).distinct(field="id") return metadataFormatsID
def test_delete_template_and_version(self): numberTemplate = len(Template.objects()) numberTemplateVersion = len(TemplateVersion.objects()) template = self.createTemplate() delete_template_and_version(str(template.id)) self.assertEquals(len(Template.objects()), numberTemplate) self.assertEquals(len(TemplateVersion.objects()), numberTemplateVersion)
def createTemplateVersion(self): countTemplateVersion = len(TemplateVersion.objects()) templateVersion = TemplateVersion(nbVersions=1, isDeleted=False, current=FAKE_ID).save() self.assertEquals(len(TemplateVersion.objects()), countTemplateVersion + 1) return templateVersion
def index(request): template = loader.get_template('explore.html') request.session['currentYear'] = currentYear() if request.user.is_authenticated(): currentTemplateVersions = [] for tpl_version in TemplateVersion.objects(): currentTemplateVersions.append(tpl_version.current) currentTemplates = dict() for tpl_version in currentTemplateVersions: tpl = Template.objects.get(pk=tpl_version) templateVersions = TemplateVersion.objects.get( pk=tpl.templateVersion) currentTemplates[tpl] = templateVersions.isDeleted context = RequestContext( request, { 'templates': currentTemplates, 'userTemplates': Template.objects(user=str(request.user.id)), }) return HttpResponse(template.render(context)) else: if 'loggedOut' in request.session: del request.session['loggedOut'] request.session['next'] = '/explore' return redirect('/login')
def index(request): template = loader.get_template('curate.html') request.session['currentYear'] = currentYear() if request.user.is_authenticated(): currentTemplateVersions = [] for tpl_version in TemplateVersion.objects(): currentTemplateVersions.append(tpl_version.current) currentTemplates = dict() for tpl_version in currentTemplateVersions: tpl = Template.objects.get(pk=tpl_version) templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion) currentTemplates[tpl] = templateVersions.isDeleted context = RequestContext(request, { 'templates':currentTemplates, 'userTemplates': Template.objects(user=str(request.user.id)), }) return HttpResponse(template.render(context)) else: if 'loggedOut' in request.session: del request.session['loggedOut'] request.session['next'] = '/curate' return redirect('/login')
def manage_schemas(request): if request.user.is_authenticated() and request.user.is_staff: template = loader.get_template('admin/manage_uploads.html') currentTemplateVersions = [] for tpl_version in TemplateVersion.objects(): currentTemplateVersions.append(tpl_version.current) currentTemplates = dict() for tpl_version in currentTemplateVersions: tpl = Template.objects.get(pk=tpl_version) templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion) currentTemplates[tpl] = templateVersions.isDeleted context = RequestContext(request, { 'objects':currentTemplates, 'objectType': "Template" }) request.session['currentYear'] = currentYear() return HttpResponse(template.render(context)) else: if 'loggedOut' in request.session: del request.session['loggedOut'] request.session['next'] = '/' return redirect('/login')
def __init__(self, userId=""): self.SCHEMAS_OPTIONS = [] self.SCHEMAS_USER_OPTIONS = [] #We retrieve all common template + user template currentSchemas = TemplateVersion.objects(isDeleted=False).distinct(field="current") schemas = Template.objects(pk__in=currentSchemas, user=None).distinct(field="title") userSchemas = Template.objects(user=str(userId)).distinct(field="title") for schema in schemas: #We add them self.SCHEMAS_OPTIONS.append((schema, schema)) for schema in userSchemas: #We add them self.SCHEMAS_USER_OPTIONS.append((schema, schema)) super(KeywordForm, self).__init__() self.fields['my_schemas'].choices = [] self.fields['my_schemas'].choices = self.SCHEMAS_OPTIONS self.fields['my_user_schemas'].choices = [] self.fields['my_user_schemas'].choices = self.SCHEMAS_USER_OPTIONS self.my_schemas_nb = len(self.SCHEMAS_OPTIONS) self.my_user_schemas_nb = len(self.SCHEMAS_USER_OPTIONS) if self.my_schemas_nb + self.my_user_schemas_nb == 1: self.fields['my_schemas'].widget.attrs['disabled'] = True self.fields['my_user_schemas'].widget.attrs['disabled'] = True
def __init__(self, *args, **kwargs): templatesVersionID = Template.objects.distinct(field="templateVersion") templatesID = TemplateVersion.objects(pk__in=templatesVersionID, isDeleted=False).distinct(field="current") templates = Template.objects(pk__in=templatesID).all() super(MyTemplateMetadataFormatForm, self).__init__(*args, **kwargs) self.fields['template'].queryset = [] self.fields['template'].queryset = templates
def curate_from_schema(request): try: schema_name = request.GET['template'] templates = Template.objects(title=schema_name) if 'curate_edit' in request.session and request.session['curate_edit'] == False: # if the schemas are all versions of the same schema if len(set(templates.values_list('templateVersion'))) == 1: template_id = TemplateVersion.objects().get(pk=templates[0].templateVersion).current request.session['currentTemplateID'] = template_id # # form_data = FormData(user=str(request.user.id), template=template_id, name=schema_name) # form_data.save() # # request.session['curateFormData'] = str(form_data.pk) # request.session['curate_edit'] = False if 'useForm' in request.GET and request.GET['useForm'] == 'true': pass else: if 'formString' in request.session: del request.session['formString'] # if 'xmlDocTree' in request.session: # del request.session['xmlDocTree'] else: error_message = "The selection of template by name can't be used if the MDCS contain more than one " error_message += "template with the same name." raise MDCSError(error_message) except: raise MDCSError("The template you are looking for doesn't exist.")
def curate_from_schema(request): try: schema_name = request.GET['template'] templates = Template.objects(title=schema_name) # if the schemas are all versions of the same schema if len(set(templates.values_list('templateVersion'))) == 1: template_id = TemplateVersion.objects().get( pk=templates[0].templateVersion).current request.session['currentTemplateID'] = template_id form_data = FormData(user=str(request.user.id), template=template_id, name=schema_name) form_data.save() request.session['curateFormData'] = str(form_data.pk) request.session['curate_edit'] = False if 'xmlDocTree' in request.session: del request.session['xmlDocTree'] else: error_message = "The selection of template by name can't be used if the MDCS contain more than one " error_message += "template with the same name." raise MDCSError(error_message) except: raise MDCSError("The template you are looking for doesn't exist.")
def manage_schemas(request): if request.user.is_authenticated() and request.user.is_staff: template = loader.get_template('admin/manage_uploads.html') currentTemplateVersions = [] for tpl_version in TemplateVersion.objects(): currentTemplateVersions.append(tpl_version.current) currentTemplates = dict() for tpl_version in currentTemplateVersions: tpl = Template.objects.get(pk=tpl_version) templateVersions = TemplateVersion.objects.get( pk=tpl.templateVersion) currentTemplates[tpl] = templateVersions.isDeleted context = RequestContext(request, { 'objects': currentTemplates, 'objectType': "Template" }) request.session['currentYear'] = currentYear() return HttpResponse(template.render(context)) else: if 'loggedOut' in request.session: del request.session['loggedOut'] request.session['next'] = '/' return redirect('/login')
def test_delete_object_template(self): template = self.createTemplate() url = '/dashboard/delete_object' data = {'objectID': template.id, 'objectType': 'Template'} r = self.doRequestPostAdminClientLogged(url=url, data=data) self.assertEquals(0, len(Template.objects())) self.assertEquals(0, len(TemplateVersion.objects()))
def explore_all_versions_results(request): template_id = request.GET['id'] template = Template.objects().get(pk=template_id) version_id = template.templateVersion template_version = TemplateVersion.objects().get(pk=version_id) if len(template_version.versions) == 1: query = {"schema": template_id} else: list_query = [] for version in template_version.versions: list_query.append({'schema': version}) query = {"$or": list_query} request.session['queryExplore'] = query if 'HTTPS' in request.META['SERVER_PROTOCOL']: protocol = "https" else: protocol = "http" json_instances = [Instance(name="Local", protocol=protocol, address=request.META['REMOTE_ADDR'], port=request.META['SERVER_PORT'], access_token="token", refresh_token="token").to_json()] request.session['instancesExplore'] = json_instances template = loader.get_template('explore/explore_results.html') context = RequestContext(request, { '': '', }) if 'exploreCurrentTemplateID' not in request.session: return redirect('/explore/select-template') else: return HttpResponse(template.render(context))
def __init__(self, userId=""): self.SCHEMAS_OPTIONS = [] self.SCHEMAS_USER_OPTIONS = [] #We retrieve all common template + user template currentSchemas = TemplateVersion.objects(isDeleted=False).distinct( field="current") schemas = Template.objects(pk__in=currentSchemas, user=None).distinct(field="title") userSchemas = Template.objects(user=str(userId)).distinct( field="title") for schema in schemas: #We add them self.SCHEMAS_OPTIONS.append((schema, schema)) for schema in userSchemas: #We add them self.SCHEMAS_USER_OPTIONS.append((schema, schema)) super(KeywordForm, self).__init__() #Init KeywordFormOAIPMH self.form_oai_pmh = KeywordFormOAIPMH() self.fields['my_schemas'].choices = [] self.fields['my_schemas'].choices = self.SCHEMAS_OPTIONS self.fields['my_user_schemas'].choices = [] self.fields['my_user_schemas'].choices = self.SCHEMAS_USER_OPTIONS self.my_schemas_nb = len(self.SCHEMAS_OPTIONS) self.my_user_schemas_nb = len(self.SCHEMAS_USER_OPTIONS) if self.my_schemas_nb + self.my_user_schemas_nb == 1: self.fields['my_schemas'].widget.attrs['disabled'] = True self.fields['my_user_schemas'].widget.attrs['disabled'] = True
def test_delete_template_with_dependencies(self): template = self.createTemplate() XMLdata(schemaID=str(template.id), title='testRecord', xml='<test>test xmldata</test>').save() FormData(template=str(template.id), name='testFormData', xml_data='testest', user=str(1)).save() listDependencies = delete_template(str(template.id)) self.assertEquals(len(Template.objects()), 1) self.assertEquals(len(TemplateVersion.objects()), 1) self.assertEquals(listDependencies, 'testFormData, testRecord')
def execute_test(self, file_id): filename_inc = '{}_inc.xsd'.format(file_id) # load XSD xsd_file_path_inc = join(self.resources_path, filename_inc) xsd_file_inc = open(xsd_file_path_inc, 'r') xsd_file_content_inc = xsd_file_inc.read() # create template template_inc = create_template(xsd_file_content_inc, filename_inc, filename_inc) template_version = TemplateVersion.objects().get( pk=template_inc.templateVersion) # remove the dependency template template_version.deletedVersions.append(str(template_inc.id)) template_version.isDeleted = True template_version.save() filename = '{}.xsd'.format(file_id) # load XSD xsd_file_path = join(self.resources_path, filename) xsd_file = open(xsd_file_path, 'r') xsd_file_content = xsd_file.read() # create template xsd_tree = etree.fromstring(xsd_file_content) update_dependencies(xsd_tree, {filename_inc: str(template_inc.id)}) xsd_file_content = etree.tostring(xsd_tree) template = create_template(xsd_file_content, filename, filename, [str(template_inc.id)]) # load the form load_new_form(self.driver, self.base_url, file_id) time.sleep(TIMEOUT) # download XML self.driver.execute_script("downloadCurrentXML();") # wait a bit more to let time to save the file time.sleep(TIMEOUT * 5) # load expected result exp_result_path = join(self.resources_path, "{0}.xml".format(file_id)) exp_result_file = open(exp_result_path, 'r') exp_result_content = exp_result_file.read() # load result generated by the form result_path = join(self.results_path, "{0}.xml".format(file_id)) result_file = open(result_path, 'r') result_content = result_file.read() expected = etree.fromstring(exp_result_content) result = etree.fromstring(result_content) if not are_equals(expected, result): raise Exception('NOT EQUALS TO EXPECTED: {}'.format(file_id))
def test_delete_object_template_with_dependencie(self): self.assertEquals(0, len(Template.objects())) template = self.createTemplate() self.assertEquals(1, len(Template.objects())) self.createXMLData(schemaID=template.id) url = '/dashboard/delete_object' data = {'objectID': template.id, 'objectType': 'Template'} r = self.doRequestPostAdminClientLogged(url=url, data=data) self.assertEquals(1, len(Template.objects())) self.assertEquals(1, len(TemplateVersion.objects()))
def _get_flatten_schema_and_namespaces(template_name): schemas = Template.objects(title=template_name) schema_id = TemplateVersion.objects().get(pk=schemas[0].templateVersion).current schema = Template.objects().get(pk=schema_id) flattener = XSDFlattenerURL(schema.content.encode('utf-8')) ref_xml_schema_content = flattener.get_flat() # find the namespaces namespaces = common.get_namespaces(BytesIO(schema.content.encode('utf-8'))) return ref_xml_schema_content, namespaces
def execute_test(self, file_id): filename_inc = '{}_inc.xsd'.format(file_id) # load XSD xsd_file_path_inc = join(self.resources_path, filename_inc) xsd_file_inc = open(xsd_file_path_inc, 'r') xsd_file_content_inc = xsd_file_inc.read() # create template template_inc = create_template(xsd_file_content_inc, filename_inc, filename_inc) template_version = TemplateVersion.objects().get(pk=template_inc.templateVersion) # remove the dependency template template_version.deletedVersions.append(str(template_inc.id)) template_version.isDeleted = True template_version.save() filename = '{}.xsd'.format(file_id) # load XSD xsd_file_path = join(self.resources_path, filename) xsd_file = open(xsd_file_path, 'r') xsd_file_content = xsd_file.read() # create template xsd_tree = etree.fromstring(xsd_file_content) update_dependencies(xsd_tree, {filename_inc: str(template_inc.id)}) xsd_file_content = etree.tostring(xsd_tree) template = create_template(xsd_file_content, filename, filename, [str(template_inc.id)]) # load the form load_new_form(self.driver, self.base_url, file_id) time.sleep(TIMEOUT) # download XML self.driver.execute_script("downloadCurrentXML();") # wait a bit more to let time to save the file time.sleep(TIMEOUT * 5) # load expected result exp_result_path = join(self.resources_path, "{0}.xml".format(file_id)) exp_result_file = open(exp_result_path, 'r') exp_result_content = exp_result_file.read() # load result generated by the form result_path = join(self.results_path, "{0}.xml".format(file_id)) result_file = open(result_path, 'r') result_content = result_file.read() expected = etree.fromstring(exp_result_content) result = etree.fromstring(result_content) if not are_equals(expected, result): raise Exception('NOT EQUALS TO EXPECTED: {}'.format(file_id))
def get_xsd(request, schema): #TODO Available if publication ok and no user template #We retrieve the schema filename in the schema attribute #Get the templateVersion ID try: templatesVersionID = Template.objects(filename=schema).distinct(field="templateVersion") templateID = TemplateVersion.objects(pk__in=templatesVersionID, isDeleted=False).distinct(field="current") templates = Template.objects.get(pk__in=templateID) #Get the XML schema contentEncoded = templates.content.encode('utf-8') fileObj = StringIO(contentEncoded) return HttpResponse(fileObj, content_type='text/xml') except MONGO_ERRORS.DoesNotExist, e: return HttpResponseNotFound('Impossible to retrieve the schema with the given name.')
def index(request): template = loader.get_template('compose/compose.html') currentTemplateVersions = [] for tpl_version in TemplateVersion.objects(): currentTemplateVersions.append(tpl_version.current) currentTemplates = dict() for tpl_version in currentTemplateVersions: tpl = Template.objects.get(pk=tpl_version) templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion) currentTemplates[tpl] = templateVersions.isDeleted context = RequestContext(request, { 'templates': currentTemplates, 'userTemplates': Template.objects(user=str(request.user.id)), }) return HttpResponse(template.render(context))
def manage_schemas(request): template = loader.get_template('admin/manage_uploads.html') currentTemplateVersions = [] for tpl_version in TemplateVersion.objects(): currentTemplateVersions.append(tpl_version.current) currentTemplates = dict() for tpl_version in currentTemplateVersions: tpl = Template.objects.get(pk=tpl_version) templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion) currentTemplates[tpl] = templateVersions.isDeleted context = RequestContext(request, { 'objects': currentTemplates, 'objectType': "Template" }) return HttpResponse(template.render(context))
def manage_schemas(request): template = loader.get_template('admin/manage_uploads.html') currentTemplateVersions = [] for tpl_version in TemplateVersion.objects(): currentTemplateVersions.append(tpl_version.current) currentTemplates = dict() for tpl_version in currentTemplateVersions: tpl = Template.objects.get(pk=tpl_version) templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion) currentTemplates[tpl] = templateVersions.isDeleted context = RequestContext(request, { 'objects':currentTemplates, 'objectType': "Template" }) return HttpResponse(template.render(context))
def index(request): currentTemplateVersions = [] for tpl_version in TemplateVersion.objects(): currentTemplateVersions.append(tpl_version.current) currentTemplates = dict() for tpl_version in currentTemplateVersions: tpl = Template.objects.get(pk=tpl_version) templateVersions = TemplateVersion.objects.get(pk=tpl.templateVersion) currentTemplates[tpl] = templateVersions.isDeleted template = loader.get_template('explore/explore.html') context = RequestContext(request, { 'templates':currentTemplates, 'userTemplates': Template.objects(user=str(request.user.id)), }) return HttpResponse(template.render(context))
def get_xsd(request, schema): # TODO Available if publication ok and no user template # We retrieve the schema filename in the schema attribute # Get the templateVersion ID try: templatesVersionID = Template.objects(filename=schema).distinct( field="templateVersion") templateID = TemplateVersion.objects( pk__in=templatesVersionID, isDeleted=False).distinct(field="current") template = Template.objects.get(pk=templateID[0]) flattener = XSDFlattenerDatabaseOrURL(template.content.encode('utf-8')) content_encoded = flattener.get_flat() file_obj = StringIO(content_encoded) return HttpResponse(file_obj, content_type='text/xml') except Exception, e: return HttpResponseBadRequest( 'Impossible to retrieve the schema with the given name.')
def explore_all_versions_results(request): template_id = request.GET['id'] template = Template.objects().get(pk=template_id) version_id = template.templateVersion template_version = TemplateVersion.objects().get(pk=version_id) if len(template_version.versions) == 1: query = {"schema": template_id} else: list_query = [] for version in template_version.versions: list_query.append({'schema': version}) query = {"$or": list_query} request.session['queryExplore'] = query if 'HTTPS' in request.META['SERVER_PROTOCOL']: protocol = "https" else: protocol = "http" json_instances = [ Instance(name="Local", protocol=protocol, address=request.META['REMOTE_ADDR'], port=request.META['SERVER_PORT'], access_token="token", refresh_token="token").to_json() ] request.session['instancesExplore'] = json_instances template = loader.get_template('explore/explore_results.html') context = RequestContext(request, { '': '', }) if 'exploreCurrentTemplateID' not in request.session: return redirect('/explore/select-template') else: return HttpResponse(template.render(context))
def createTemplateVersionDeleted(self): return TemplateVersion(nbVersions=1, isDeleted=True).save()
def createTemplateVersion(self): return TemplateVersion(nbVersions=1, isDeleted=False, current=FAKE_ID).save()
def dump_template_version(self): self.assertEquals(len(TemplateVersion.objects()), 0) self.restoreDump(join(DUMP_TEST_PATH, 'template_version.bson'), 'template_version') self.assertTrue(len(TemplateVersion.objects()) > 0)
def test_delete_template_and_version(self): template = self.createTemplate() delete_template_and_version(str(template.id)) self.assertEquals(len(Template.objects()), 0) self.assertEquals(len(TemplateVersion.objects()), 0)
def save_object(request): print 'BEGIN def save_object(request)' objectName = None objectFilename = None objectContent = None objectType = None objectFlat = None objectApiurl = None if ('uploadObjectValid' in request.session and request.session['uploadObjectValid'] == True and 'uploadObjectName' in request.session and request.session['uploadObjectName'] is not None and 'uploadObjectFilename' in request.session and request.session['uploadObjectFilename'] is not None and 'uploadObjectContent' in request.session and request.session['uploadObjectContent'] is not None and 'uploadObjectType' in request.session and request.session['uploadObjectType'] is not None): objectName = request.session['uploadObjectName'] objectFilename = request.session['uploadObjectFilename'] objectContent = request.session['uploadObjectContent'] objectType = request.session['uploadObjectType'] if 'uploadObjectFlat' in request.session and request.session[ 'uploadObjectFlat'] is not None: objectFlat = request.session['uploadObjectFlat'] else: objectFlat = None if 'uploadObjectAPIurl' in request.session and request.session[ 'uploadObjectAPIurl'] is not None: objectApiurl = request.session['uploadObjectAPIurl'] else: objectApiurl = None if 'uploadDependencies' in request.session and request.session[ 'uploadDependencies'] is not None: dependencies = request.session['uploadDependencies'] else: dependencies = None hash = XSDhash.get_hash(objectContent) # save the object if objectType == "Template": objectVersions = TemplateVersion(nbVersions=1, isDeleted=False).save() object = Template(title=objectName, filename=objectFilename, content=objectContent, version=1, templateVersion=str(objectVersions.id), hash=hash).save() #We add default exporters try: exporters = Exporter.objects.filter(available_for_all=True) object.exporters = exporters except: pass elif objectType == "Type": objectVersions = TypeVersion(nbVersions=1, isDeleted=False).save() object = Type(title=objectName, filename=objectFilename, content=objectContent, version=1, typeVersion=str(objectVersions.id), hash=hash).save() buckets = request.POST.getlist('buckets[]') for bucket_id in buckets: bucket = Bucket.objects.get(pk=bucket_id) bucket.types.append(str(objectVersions.id)) bucket.save() objectVersions.versions = [str(object.id)] objectVersions.current = str(object.id) objectVersions.save() object.save() if objectFlat is not None and objectApiurl is not None and dependencies is not None: MetaSchema(schemaId=str(object.id), flat_content=objectFlat, api_content=objectApiurl).save() object.dependencies = dependencies object.save() clear_object(request) else: response_dict = {'errors': 'True'} return HttpResponse(json.dumps(response_dict), content_type='application/javascript') return HttpResponse(json.dumps({}), content_type='application/javascript')
def load_templates(): """ Loads templates/xslt for NMRR the first time """ # if templates are already present, initialization already happened existing_templates = Template.objects() if len(existing_templates) == 0: templates = { 'all': 'AllResources.xsd', 'organization': 'Organization.xsd', 'datacollection': 'DataCollection.xsd', 'repository': 'Repository.xsd', 'projectarchive': 'ProjectArchive.xsd', 'database': 'Database.xsd', 'dataset': 'Dataset.xsd', 'document': 'Document.xsd', 'service': 'Service.xsd', 'informational': 'Informational.xsd', 'software': 'Software.xsd', } template_ids = [] template_results = { 'full': 'nmrr-full.xsl', 'detail': 'nmrr-detail.xsl', } template_results_id = { 'full': None, 'detail': None, } # connect to mongo client = MongoClient(MONGODB_URI) # connect to the db 'mgi' db = client[MGI_DB] # Add the templates for template_name, template_path in templates.iteritems(): file = open( os.path.join(SITE_ROOT, 'static', 'resources', 'xsd', template_path), 'r') templateContent = file.read() hash = XSDhash.get_hash(templateContent) #create template/ template version objectVersions = TemplateVersion(nbVersions=1, isDeleted=False).save() object = Template(title=template_name, filename=template_path, content=templateContent, version=1, templateVersion=str(objectVersions.id), hash=hash).save() objectVersions.versions = [str(object.id)] objectVersions.current = str(object.id) objectVersions.save() object.save() # save template id template_ids.append(str(object.id)) # Add xslt xsl_col = db['result_xslt'] for xsl_name, xsl_path in template_results.iteritems(): file = open( os.path.join(SITE_ROOT, 'static', 'resources', 'xsl', xsl_path), 'r') fileContent = file.read() xsl = {} xsl['name'] = xsl_name xsl['filename'] = xsl_path xsl['content'] = fileContent xsl_id = xsl_col.insert(xsl) template_results_id[xsl_name] = str(xsl_id) templates = db['template'] results_xslt = { 'ResultXsltList': template_results_id['full'], 'ResultXsltDetailed': template_results_id['detail'] } templates.update({}, {"$set": results_xslt}, upsert=False, multi=True)
def load_templates(): """ Loads templates/xslt for NMRR the first time """ # if templates are already present, initialization already happened existing_templates = Template.objects() if len(existing_templates) == 0: templates = { 'all':'AllResources.xsd', 'organization': 'Organization.xsd', 'datacollection': 'DataCollection.xsd', 'repository': 'Repository.xsd', 'projectarchive': 'ProjectArchive.xsd', 'database': 'Database.xsd', 'dataset': 'Dataset.xsd', 'service': 'Service.xsd', 'informational': 'Informational.xsd', 'software': 'Software.xsd', } template_ids = [] template_results = { 'full': 'nmrr-full.xsl', 'detail': 'nmrr-detail.xsl', } template_results_id = { 'full': None, 'detail': None, } # connect to mongo client = MongoClient(MONGODB_URI) # connect to the db 'mgi' db = client[MGI_DB] # Add the templates for template_name, template_path in templates.iteritems(): file = open(os.path.join(SITE_ROOT, 'static', 'resources', 'xsd', template_path),'r') templateContent = file.read() hash = XSDhash.get_hash(templateContent) #create template/ template version objectVersions = TemplateVersion(nbVersions=1, isDeleted=False).save() object = Template(title=template_name, filename=template_path, content=templateContent, version=1, templateVersion=str(objectVersions.id), hash=hash).save() objectVersions.versions = [str(object.id)] objectVersions.current = str(object.id) objectVersions.save() object.save() # save template id template_ids.append(str(object.id)) # Add xslt xsl_col = db['result_xslt'] for xsl_name, xsl_path in template_results.iteritems(): file = open(os.path.join(SITE_ROOT, 'static', 'resources', 'xsl', xsl_path),'r') fileContent = file.read() xsl = {} xsl['name'] = xsl_name xsl['filename'] = xsl_path xsl['content'] = fileContent xsl_id = xsl_col.insert(xsl) template_results_id[xsl_name] = str(xsl_id) templates = db['template'] results_xslt = {'ResultXsltList': template_results_id['full'], 'ResultXsltDetailed': template_results_id['detail']} templates.update({}, {"$set":results_xslt}, upsert=False, multi=True)
def test_delete_template_no_dependencies(self): template = self.createTemplate() delete_template(str(template.id)) self.assertEquals(len(Template.objects()), 0) self.assertEquals(len(TemplateVersion.objects()), 0)
def get_results_by_instance_keyword(request): print 'BEGIN def getResultsKeyword(request)' resultsByKeyword = [] results = [] resultString = "" #Instance json_instances = [] if 'HTTPS' in request.META['SERVER_PROTOCOL']: protocol = "https" else: protocol = "http" instance = Instance(name="Local", protocol=protocol, address=request.META['REMOTE_ADDR'], port=request.META['SERVER_PORT'], access_token="token", refresh_token="token") json_instances.append(instance.to_json()) request.session['instancesExplore'] = json_instances sessionName = "resultsExploreOaiPMh" + instance['name'] try: keyword = request.GET['keyword'] schemas = request.GET.getlist('schemas[]') userSchemas = request.GET.getlist('userSchemas[]') refinements = refinements_to_mongo( request.GET.getlist('refinements[]')) if 'onlySuggestions' in request.GET: onlySuggestions = json.loads(request.GET['onlySuggestions']) else: onlySuggestions = False except: keyword = '' schemas = [] userSchemas = [] refinements = {} onlySuggestions = True #We get all template versions for the given schemas #First, we take care of user defined schema templatesIDUser = Template.objects(title__in=userSchemas).distinct( field="id") templatesIDUser = [str(x) for x in templatesIDUser] #Take care of the rest, with versions templatesVersions = Template.objects(title__in=schemas).distinct( field="templateVersion") #We get all templates ID, for all versions allTemplatesIDCommon = TemplateVersion.objects( pk__in=templatesVersions, isDeleted=False).distinct(field="versions") #We remove the removed version allTemplatesIDCommonRemoved = TemplateVersion.objects( pk__in=templatesVersions, isDeleted=False).distinct(field="deletedVersions") templatesIDCommon = list( set(allTemplatesIDCommon) - set(allTemplatesIDCommonRemoved)) templatesID = templatesIDUser + templatesIDCommon #We retrieve deactivated registries so as not to get their metadata formats deactivatedRegistries = [ str(x.id) for x in OaiRegistry.objects(isDeactivated=True).order_by('id') ] metadataFormatsID = OaiMetadataFormat.objects( template__in=templatesID, registry__not__in=deactivatedRegistries).distinct(field="id") instanceResults = OaiRecord.executeFullTextQuery(keyword, metadataFormatsID, refinements) if len(instanceResults) > 0: if not onlySuggestions: xsltPath = os.path.join(settings.SITE_ROOT, 'static/resources/xsl/xml2html.xsl') xslt = etree.parse(xsltPath) transform = etree.XSLT(xslt) template = loader.get_template( 'oai_pmh/explore/explore_result_keyword.html') #Retrieve schema and registries. Avoid to retrieve the information for each result registriesName = {} objMetadataFormats = {} listRegistriesID = set([x['registry'] for x in instanceResults]) for registryId in listRegistriesID: obj = OaiRegistry.objects(pk=registryId).get() registriesName[str(registryId)] = obj.name listSchemaId = set([x['metadataformat'] for x in instanceResults]) for schemaId in listSchemaId: obj = OaiMetadataFormat.objects(pk=schemaId).get() objMetadataFormats[str(schemaId)] = obj listItems = [] xmltodictunparse = xmltodict.unparse appendResult = results.append toXML = etree.XML parse = etree.parse XSLT = etree.XSLT if not onlySuggestions: for instanceResult in instanceResults: custom_xslt = False appendResult({ 'title': instanceResult['identifier'], 'content': xmltodictunparse(instanceResult['metadata']), 'id': str(instanceResult['_id']) }) dom = toXML( str( xmltodictunparse( instanceResult['metadata']).encode('utf-8'))) #Check if a custom list result XSLT has to be used try: metadataFormat = objMetadataFormats[str( instanceResult['metadataformat'])] if metadataFormat.template.ResultXsltList: listXslt = parse( BytesIO( metadataFormat.template.ResultXsltList.content. encode('utf-8'))) listTransform = XSLT(listXslt) newdom = listTransform(dom) custom_xslt = True else: newdom = transform(dom) except Exception, e: #We use the default one newdom = transform(dom) custom_xslt = False context = RequestContext( request, { 'id': str(instanceResult['_id']), 'xml': str(newdom), 'title': instanceResult['identifier'], 'custom_xslt': custom_xslt, 'schema_name': metadataFormat.metadataPrefix, 'registry_name': registriesName[instanceResult['registry']], 'oai_pmh': True }) resultString += template.render(context) else: for instanceResult in instanceResults[:20]: wordList = re.sub("[^\w]", " ", keyword).split() wordList = [x + "|" + x + "\w+" for x in wordList] wordList = '|'.join(wordList) listWholeKeywords = re.findall( "\\b(" + wordList + ")\\b", xmltodict.unparse( instanceResult['metadata']).encode('utf-8'), flags=re.IGNORECASE) labels = list(set(listWholeKeywords)) for label in labels: label = label.lower() result_json = {} result_json['label'] = label result_json['value'] = label if not result_json in resultsByKeyword: resultsByKeyword.append(result_json)
def get_results_by_instance_keyword(request): print 'BEGIN def getResultsKeyword(request)' resultsByKeyword = [] results = [] resultString = "" #Instance json_instances = [] if 'HTTPS' in request.META['SERVER_PROTOCOL']: protocol = "https" else: protocol = "http" instance = Instance(name="Local", protocol=protocol, address=request.META['REMOTE_ADDR'], port=request.META['SERVER_PORT'], access_token="token", refresh_token="token") json_instances.append(instance.to_json()) request.session['instancesExplore'] = json_instances sessionName = "resultsExploreOaiPMh" + instance['name'] try: keyword = request.GET['keyword'] schemas = request.GET.getlist('schemas[]') userSchemas = request.GET.getlist('userSchemas[]') refinements = refinements_to_mongo(request.GET.getlist('refinements[]')) if 'onlySuggestions' in request.GET: onlySuggestions = json.loads(request.GET['onlySuggestions']) else: onlySuggestions = False except: keyword = '' schemas = [] userSchemas = [] refinements = {} onlySuggestions = True #We get all template versions for the given schemas #First, we take care of user defined schema templatesIDUser = Template.objects(title__in=userSchemas).distinct(field="id") templatesIDUser = [str(x) for x in templatesIDUser] #Take care of the rest, with versions templatesVersions = Template.objects(title__in=schemas).distinct(field="templateVersion") #We get all templates ID, for all versions allTemplatesIDCommon = TemplateVersion.objects(pk__in=templatesVersions, isDeleted=False).distinct(field="versions") #We remove the removed version allTemplatesIDCommonRemoved = TemplateVersion.objects(pk__in=templatesVersions, isDeleted=False).distinct(field="deletedVersions") templatesIDCommon = list(set(allTemplatesIDCommon) - set(allTemplatesIDCommonRemoved)) templatesID = templatesIDUser + templatesIDCommon #We retrieve deactivated registries so as not to get their metadata formats deactivatedRegistries = [str(x.id) for x in OaiRegistry.objects(isDeactivated=True).order_by('id')] metadataFormatsID = OaiMetadataFormat.objects(template__in=templatesID, registry__not__in=deactivatedRegistries).distinct(field="id") instanceResults = OaiRecord.executeFullTextQuery(keyword, metadataFormatsID, refinements) if len(instanceResults) > 0: if not onlySuggestions: xsltPath = os.path.join(settings.SITE_ROOT, 'static/resources/xsl/xml2html.xsl') xslt = etree.parse(xsltPath) transform = etree.XSLT(xslt) template = loader.get_template('oai_pmh/explore/explore_result_keyword.html') #Retrieve schema and registries. Avoid to retrieve the information for each result registriesName = {} objMetadataFormats = {} listRegistriesID = set([x['registry'] for x in instanceResults]) for registryId in listRegistriesID: obj = OaiRegistry.objects(pk=registryId).get() registriesName[str(registryId)] = obj.name listSchemaId = set([x['metadataformat'] for x in instanceResults]) for schemaId in listSchemaId: obj = OaiMetadataFormat.objects(pk=schemaId).get() objMetadataFormats[str(schemaId)] = obj listItems = [] xmltodictunparse = xmltodict.unparse appendResult = results.append toXML = etree.XML parse = etree.parse XSLT = etree.XSLT if not onlySuggestions: for instanceResult in instanceResults: custom_xslt = False appendResult({'title':instanceResult['identifier'], 'content':xmltodictunparse(instanceResult['metadata']),'id':str(instanceResult['_id'])}) dom = toXML(str(xmltodictunparse(instanceResult['metadata']).encode('utf-8'))) #Check if a custom list result XSLT has to be used try: metadataFormat = objMetadataFormats[str(instanceResult['metadataformat'])] if metadataFormat.template.ResultXsltList: listXslt = parse(BytesIO(metadataFormat.template.ResultXsltList.content.encode('utf-8'))) listTransform = XSLT(listXslt) newdom = listTransform(dom) custom_xslt = True else: newdom = transform(dom) except Exception, e: #We use the default one newdom = transform(dom) custom_xslt = False context = RequestContext(request, {'id':str(instanceResult['_id']), 'xml': str(newdom), 'title': instanceResult['identifier'], 'custom_xslt': custom_xslt, 'schema_name': metadataFormat.metadataPrefix, 'registry_name': registriesName[instanceResult['registry']], 'oai_pmh': True}) resultString+= template.render(context) else: for instanceResult in instanceResults[:20]: wordList = re.sub("[^\w]", " ", keyword).split() wordList = [x + "|" + x +"\w+" for x in wordList] wordList = '|'.join(wordList) listWholeKeywords = re.findall("\\b("+ wordList +")\\b", xmltodict.unparse(instanceResult['metadata']).encode('utf-8'), flags=re.IGNORECASE) labels = list(set(listWholeKeywords)) for label in labels: label = label.lower() result_json = {} result_json['label'] = label result_json['value'] = label if not result_json in resultsByKeyword: resultsByKeyword.append(result_json)
def start_curate(request): if 'currentTemplateID' not in request.session and 'template' not in request.GET: return redirect('/curate/select-template') else: if request.method == 'POST': # parameters to build FormData object in db user = request.user.id template_id = request.session['currentTemplateID'] form_data = None selected_option = request.POST['curate_form'] if selected_option == "new": request.session['curate_edit'] = False new_form = NewForm(request.POST) try: form_data = FormData(user=str(user), template=template_id, name=new_form.data['document_name'], xml_data=None).save() except: return HttpResponseBadRequest( 'Unable to create the form. A form with the same name may already exists.') elif selected_option == "open": request.session['curate_edit'] = True open_form = OpenForm(request.POST) form_data = FormData.objects.get(pk=ObjectId(open_form.data['forms'])) request.session['curate_edit_data'] = form_data.xml_data elif selected_option == "upload": request.session['curate_edit'] = True upload_form = UploadForm(request.POST, request.FILES) xml_file = request.FILES['file'] # put the cursor at the beginning of the file xml_file.seek(0) # read the content of the file xml_data = xml_file.read() # check XML data or not? try: etree.fromstring(xml_data) except XMLSyntaxError: return HttpResponseBadRequest('Uploaded File is not well formed XML.') try: form_data = FormData(user=str(user), template=template_id, name=xml_file.name, xml_data=xml_data).save() except: return HttpResponseBadRequest( 'Unable to create the form. A form with the same name may already exist.') # parameters that will be used during curation request.session['curateFormData'] = str(form_data.id) return HttpResponse('ok') else: try: ajaxCall = False context_params = dict() if 'template' in request.GET: schema_name = request.GET['template'] context_params['template_name'] = schema_name try: templates = Template.objects(title=schema_name) except: raise MDCSError("The template you are looking for doesn't exist.") # if the schemas are all versions of the same schema if len(set(templates.values_list('templateVersion'))) == 1: template_id = TemplateVersion.objects().get(pk=templates[0].templateVersion).current request.session['currentTemplateID'] = template_id else: raise MDCSError("The selection of template by name can't be used if the MDCS contain more than one template with the same name.") template = loader.get_template('curate/curate_full_start.html') if 'formString' in request.session: del request.session['formString'] if 'xmlDocTree' in request.session: del request.session['xmlDocTree'] else: ajaxCall = True template = loader.get_template('curate/curate_start.html') open_form = OpenForm( forms=FormData.objects(user=str(request.user.id), template=request.session['currentTemplateID'], xml_data_id__exists=False)) new_form = NewForm() upload_form = UploadForm() context_params['new_form'] = new_form context_params['open_form'] = open_form context_params['upload_form'] = upload_form context = RequestContext(request, context_params) # , 'options_form': options_form}) if ajaxCall: return HttpResponse(json.dumps({'template': template.render(context)}), content_type='application/javascript') else: return HttpResponse(template.render(context)) except MDCSError, e: template = loader.get_template('curate/errors.html') context = RequestContext(request, { 'errors': e.message, }) return HttpResponse(template.render(context))
def save_object(request): print 'BEGIN def save_object(request)' objectName = None objectFilename = None objectContent = None objectType = None objectFlat = None objectApiurl = None if ('uploadObjectValid' in request.session and request.session['uploadObjectValid'] == True and 'uploadObjectName' in request.session and request.session['uploadObjectName'] is not None and 'uploadObjectFilename' in request.session and request.session['uploadObjectFilename'] is not None and 'uploadObjectContent' in request.session and request.session['uploadObjectContent'] is not None and 'uploadObjectType' in request.session and request.session['uploadObjectType'] is not None): objectName = request.session['uploadObjectName'] objectFilename = request.session['uploadObjectFilename'] objectContent = request.session['uploadObjectContent'] objectType = request.session['uploadObjectType'] if 'uploadObjectFlat' in request.session and request.session['uploadObjectFlat'] is not None: objectFlat = request.session['uploadObjectFlat'] else: objectFlat = None if 'uploadObjectAPIurl' in request.session and request.session['uploadObjectAPIurl'] is not None: objectApiurl = request.session['uploadObjectAPIurl'] else: objectApiurl = None if 'uploadDependencies' in request.session and request.session['uploadDependencies'] is not None: dependencies = request.session['uploadDependencies'] else: dependencies = None hash = XSDhash.get_hash(objectContent) # save the object if objectType == "Template": objectVersions = TemplateVersion(nbVersions=1, isDeleted=False).save() object = Template(title=objectName, filename=objectFilename, content=objectContent, version=1, templateVersion=str(objectVersions.id), hash=hash).save() #We add default exporters try: exporters = Exporter.objects.filter(available_for_all=True) object.exporters = exporters except: pass elif objectType == "Type": objectVersions = TypeVersion(nbVersions=1, isDeleted=False).save() object = Type(title=objectName, filename=objectFilename, content=objectContent, version=1, typeVersion=str(objectVersions.id), hash=hash).save() buckets = request.POST.getlist('buckets[]') for bucket_id in buckets: bucket = Bucket.objects.get(pk=bucket_id) bucket.types.append(str(objectVersions.id)) bucket.save() objectVersions.versions = [str(object.id)] objectVersions.current = str(object.id) objectVersions.save() object.save() if objectFlat is not None and objectApiurl is not None and dependencies is not None: MetaSchema(schemaId=str(object.id), flat_content=objectFlat, api_content=objectApiurl).save() object.dependencies = dependencies object.save() clear_object(request) else: response_dict = {'errors': 'True'} return HttpResponse(json.dumps(response_dict), content_type='application/javascript') return HttpResponse(json.dumps({}), content_type='application/javascript')
def start_curate(request): if 'currentTemplateID' not in request.session and 'template' not in request.GET: return redirect('/curate/select-template') else: if request.method == 'POST': # parameters to build FormData object in db user = request.user.id template_id = request.session['currentTemplateID'] form_data = None selected_option = request.POST['curate_form'] if selected_option == "new": request.session['curate_edit'] = False new_form = NewForm(request.POST) try: form_data = FormData(user=str(user), template=template_id, name=new_form.data['document_name']) form_data.save() except: return HttpResponseBadRequest( 'Unable to create the form. A form with the same name may already exist.' ) elif selected_option == "open": request.session['curate_edit'] = True open_form = OpenForm(request.POST) form_data = FormData.objects.get( pk=ObjectId(open_form.data['forms'])) elif selected_option == "upload": request.session['curate_edit'] = True upload_form = UploadForm(request.POST, request.FILES) xml_file = request.FILES['file'] # put the cursor at the beginning of the file xml_file.seek(0) # read the content of the file xml_data = xml_file.read() # check XML data or not? try: etree.fromstring(xml_data) except XMLSyntaxError: return HttpResponseBadRequest( 'Uploaded File is not well formed XML.') try: form_data = FormData(user=str(user), template=template_id, name=xml_file.name, xml_data=xml_data).save() except: return HttpResponseBadRequest( 'Unable to create the form. A form with the same name may already exist.' ) # parameters that will be used during curation request.session['curateFormData'] = str(form_data.id) return HttpResponse('ok') else: try: ajaxCall = False context_params = dict() if 'template' in request.GET: schema_name = request.GET['template'] context_params['template_name'] = schema_name try: templates = Template.objects(title=schema_name) except: raise MDCSError( "The template you are looking for doesn't exist.") # if the schemas are all versions of the same schema if len(set(templates.values_list('templateVersion'))) == 1: template_id = TemplateVersion.objects().get( pk=templates[0].templateVersion).current request.session['currentTemplateID'] = template_id else: raise MDCSError( "The selection of template by name can't be used if the MDCS contain more than " "one template with the same name.") template = loader.get_template( 'curate/curate_full_start.html') if 'xmlDocTree' in request.session: del request.session['xmlDocTree'] else: ajaxCall = True template = loader.get_template('curate/curate_start.html') open_form = OpenForm(forms=FormData.objects( user=str(request.user.id), template=request.session['currentTemplateID'], xml_data_id__exists=False)) new_form = NewForm() upload_form = UploadForm() context_params['new_form'] = new_form context_params['open_form'] = open_form context_params['upload_form'] = upload_form context = RequestContext( request, context_params) # , 'options_form': options_form}) if ajaxCall: return HttpResponse(json.dumps( {'template': template.render(context)}), content_type='application/javascript') else: return HttpResponse(template.render(context)) except MDCSError, e: template = loader.get_template('curate/errors.html') context = RequestContext(request, { 'errors': e.message, }) return HttpResponse(template.render(context))