def test_delete_result(self): id = self.createXMLData() self.assertIsNotNone(XMLdata.get(id)) url = '/dashboard/delete_result' data = {'result_id': str(id)} r = self.doRequestGetAdminClientLogged(url=url, data=data) self.assertIsNone(XMLdata.get(id))
def test_update_publish_draft(self): status = Status.ACTIVE new_xml = "<Resource localid='' status='"+status+"'><identity>" \ "<title>My new software</title></identity><curation><publisher>PF</publisher><contact><name></name>" \ "</contact></curation><content><description>This is a new record</description><subject></subject>" \ "<referenceURL></referenceURL></content></Resource>" id = self.createXMLData(ispublished=True) xmlData = XMLdata.get(id) self.assertNotEquals(new_xml, XMLdata.unparse(xmlData['content'])) adminId = self.getAdmin().id template = self.createTemplate() elements = SchemaElement.objects().all() self.assertEqual(len(elements), 0) elementsForm = FormData.objects().all() self.assertEqual(len(elementsForm), 0) formData = self.createFormData(user=adminId, name='name', template=str(template.id), xml_data=new_xml, xml_data_id=str(id)) url = '/dashboard/update_publish_draft' data = {'draft_id': str(formData.id)} r = self.doRequestGetAdminClientLogged(url=url, data=data) xmlDataInDatabase = XMLdata.get(id) elements = SchemaElement.objects().all() self.assertEqual(len(elements), 0) elementsForm = FormData.objects().all() self.assertEqual(len(elementsForm), 0) self.assertEquals(etree.XML(new_xml).text, etree.XML(str(XMLdata.unparse(xmlDataInDatabase['content']))).text) self.assertEquals(True, xmlDataInDatabase.get('ispublished')) self.assertEquals(str(adminId), xmlDataInDatabase.get('iduser')) self.assertNotEquals(xmlData.get('lastmodificationdate'), xmlDataInDatabase.get('lastmodificationdate')) self.assertNotEquals(xmlData.get('publicationdate'), xmlDataInDatabase.get('publicationdate')) self.assertEquals(status, xmlDataInDatabase.get('status'))
def my_profile_resources(request): template = loader.get_template('profile/my_profile_resources.html') if 'template' in request.GET: template_name = request.GET['template'] if template_name == 'all': context = RequestContext(request, { 'XMLdatas': XMLdata.find({'iduser' : str(request.user.id)}), }) else : if template_name == 'datacollection': templateNamesQuery = list(chain(Template.objects.filter(title=template_name).values_list('id'), Template.objects.filter(title='repository').values_list('id'), Template.objects.filter(title='database').values_list('id'), Template.objects.filter(title='projectarchive').values_list('id'))) else : templateNamesQuery = Template.objects.filter(title=template_name).values_list('id') templateNames = [] for templateQuery in templateNamesQuery: templateNames.append(str(templateQuery)) context = RequestContext(request, { 'XMLdatas': XMLdata.find({'iduser' : str(request.user.id), 'schema':{"$in" : templateNames}}), 'template': template_name }) else : context = RequestContext(request, { 'XMLdatas': XMLdata.find({'iduser' : str(request.user.id)}), }) return HttpResponse(template.render(context))
def explore_detail_result_process(request): result_id = request.GET['id'] xmlString = XMLdata.get(result_id) schemaId = xmlString['schema'] if 'title' in request.GET: title = request.GET['title'] else: title = xmlString['title'] xmlString = XMLdata.unparse(xmlString['content']).encode('utf-8') xsltPath = os.path.join(settings.SITE_ROOT, 'static', 'resources', 'xsl', 'xml2html.xsl') xslt = etree.parse(xsltPath) transform = etree.XSLT(xslt) #Check if a custom detailed result XSLT has to be used schema = Template.objects.get(pk=schemaId) try: if (xmlString != ""): dom = etree.fromstring(str(xmlString)) if schema.ResultXsltDetailed: shortXslt = etree.parse(BytesIO(schema.ResultXsltDetailed.content.encode('utf-8'))) shortTransform = etree.XSLT(shortXslt) newdom = shortTransform(dom) else: newdom = transform(dom) except Exception, e: #We use the default one newdom = transform(dom)
def test_invert_numeric(self): criteria = build_criteria("content.root.integer", "=", 1, "xs:int", "xs") results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 1) inverted = invertQuery(criteria) inverted_results = XMLdata.executeQueryFullResult(inverted) self.assertTrue(len(inverted_results) == 2)
def test_update_unpublish(self): id = self.createXMLData(ispublished=True) self.assertEquals(True, XMLdata.get(id)['ispublished']) url = '/dashboard/update_unpublish' data = {'result_id': str(id)} r = self.doRequestGetAdminClientLogged(url=url, data=data) self.assertEquals(False, XMLdata.get(id)['ispublished'])
def test_invert_str(self): criteria = build_criteria("content.root.str", "is", "test1", "xs:string", "xs") results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 1) inverted = invertQuery(criteria) inverted_results = XMLdata.executeQueryFullResult(inverted) self.assertTrue(len(inverted_results) == 2)
def change_owner_record(request): if 'recordID' in request.POST and 'userID' in request.POST: xml_data_id = request.POST['recordID'] user_id = request.POST['userID'] try: XMLdata.update_user(xml_data_id, user=user_id) messages.add_message(request, messages.INFO, 'Record Owner changed with success.') except Exception, e: return HttpResponseServerError({"Something wrong occurred during the change of owner."}, status=500)
def test_invert_and_numeric(self): criteria1 = build_criteria("content.root.integer", "gt", 1, "xs:int", "xs") criteria2 = build_criteria("content.root.integer", "lte", 3, "xs:int", "xs") criteria = ANDCriteria(criteria1, criteria2) results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 2) inverted = invertQuery(criteria) inverted_results = XMLdata.executeQueryFullResult(inverted) self.assertTrue(len(inverted_results) == 3)
def change_status_case_inactive(self, ispublished): id = self.createXMLData(ispublished=ispublished) XMLdata.change_status(id, Status.INACTIVE, ispublished) list_xmldata = XMLdata.find({'_id': ObjectId(id)}) self.assertEquals(Status.INACTIVE, list_xmldata[0]['status']) self.assertEquals(Status.INACTIVE, list_xmldata[0]['content']['Resource']['@status']) if ispublished: self.assertNotEquals(None, list_xmldata[0].get('oai_datestamp', None)) else: self.assertEquals(None, list_xmldata[0].get('oai_datestamp', None))
def delete_result(request): result_id = request.GET['result_id'] try: XMLdata.delete(result_id) except: # XML can't be found pass return HttpResponse(json.dumps({}), content_type='application/javascript')
def dashboard_resources(request): template = loader.get_template('dashboard/my_dashboard_my_records.html') query = {} context = RequestContext(request, {}) ispublished = request.GET.get('ispublished', None) template_name = request.GET.get('template', None) query['iduser'] = str(request.user.id) #If ispublished not None, check if we want publish or unpublish records if ispublished: ispublished = ispublished == 'true' query['ispublished'] = ispublished if template_name: context.update({'template': template_name}) if template_name == 'datacollection': templateNamesQuery = list(chain(Template.objects.filter(title=template_name).values_list('id'), Template.objects.filter(title='repository').values_list('id'), Template.objects.filter(title='database').values_list('id'), Template.objects.filter(title='projectarchive').values_list('id'))) else : templateNamesQuery = Template.objects.filter(title=template_name).values_list('id') templateNames = [] for templateQuery in templateNamesQuery: templateNames.append(str(templateQuery)) query['schema'] = {"$in" : templateNames} userXmlData = sorted(XMLdata.find(query), key=lambda data: data['lastmodificationdate'], reverse=True) #Add user_form for change owner user_form = UserForm(request.user) context.update({'XMLdatas': userXmlData, 'ispublished': ispublished, 'user_form': user_form}) #If the user is an admin, we get records for other users if request.user.is_staff: #Get user name for admin usernames = dict((str(x.id), x.username) for x in User.objects.all()) query['iduser'] = {"$ne": str(request.user.id)} otherUsersXmlData = sorted(XMLdata.find(query), key=lambda data: data['lastmodificationdate'], reverse=True) context.update({'OtherUsersXMLdatas': otherUsersXmlData, 'usernames': usernames}) #Get new version of records listIds = [str(x['_id']) for x in userXmlData] if request.user.is_staff: listIdsOtherUsers = [str(x['_id']) for x in otherUsersXmlData] listIds = list(set(listIds).union(set(listIdsOtherUsers))) drafts = FormData.objects(xml_data_id__in=listIds, isNewVersionOfRecord=True).all() XMLdatasDrafts = dict() for draft in drafts: XMLdatasDrafts[draft.xml_data_id] = draft.id context.update({'XMLdatasDrafts': XMLdatasDrafts}) #Add Status enum context.update({'Status': Status}) return HttpResponse(template.render(context))
def test_change_owner_record(self): userId = self.getUser().id adminId = self.getAdmin().id template = self.createTemplate() xmldataid = self.createXMLData(iduser=userId, schemaID=template.id) self.assertEquals(str(userId), str(XMLdata.get(xmldataid)['iduser'])) url = '/dashboard/change-owner-record' data = {'recordID': str(xmldataid), 'userID': str(adminId)} r = self.doRequestPostUserClientLogged(url=url, data=data) self.assertEquals(str(adminId), str(XMLdata.get(xmldataid)['iduser'])) self.isStatusOK(r.status_code)
def change_status_case_deleted(self, ispublished): id = self.createXMLData(ispublished=ispublished) XMLdata.change_status(id, Status.DELETED, ispublished) list_xmldata = XMLdata.find({'_id': ObjectId(id)}) self.assertEquals(Status.DELETED, list_xmldata[0]['status']) self.assertEquals(Status.DELETED, list_xmldata[0]['content']['Resource']['@status']) if ispublished: self.assertNotEquals(None, list_xmldata[0].get('oai_datestamp', None)) else: self.assertEquals(None, list_xmldata[0].get('oai_datestamp', None))
def change_owner_record(request): if 'recordID' in request.POST and 'userID' in request.POST: xml_data_id = request.POST['recordID'] user_id = request.POST['userID'] try: XMLdata.update_user(xml_data_id, user=user_id) messages.add_message(request, messages.INFO, 'Record Owner changed with success.') except Exception, e: return HttpResponseServerError( {"Something wrong occurred during the change of owner."}, status=500)
def dashboard_detail_record(request): template = loader.get_template('dashboard/my_dashboard_detail_record.html') record_id = request.GET['id'] record_type = request.GET['type'] if record_type == 'form': form_data = FormData.objects.get(pk=ObjectId(record_id)) xml_string = form_data.xml_data.encode(encoding='UTF-8') title = form_data.name schema_id = form_data.template elif record_type == 'record': xml_string = XMLdata.get(record_id) title = xml_string['title'] schema_id = xml_string['schema'] xml_string = XMLdata.unparse(xml_string['content']).encode('utf-8') else: raise Exception("Unknow record type: " + str(record_type)) xslt_path = os.path.join(settings.SITE_ROOT, 'static', 'resources', 'xsl', 'xml2html.xsl') xslt = etree.parse(xslt_path) transform = etree.XSLT(xslt) dom = '' # Check if a custom detailed result XSLT has to be used try: if xml_string != "": dom = etree.fromstring(xml_string) schema = Template.objects.get(pk=schema_id) if schema.ResultXsltDetailed: short_xslt = etree.parse( BytesIO(schema.ResultXsltDetailed.content.encode('utf-8'))) short_transform = etree.XSLT(short_xslt) newdom = short_transform(dom) else: newdom = transform(dom) else: newdom = 'No data has been saved to this form yet.' except Exception as e: # We use the default one newdom = transform(dom) result = str(newdom) context = RequestContext(request, { 'XMLHolder': result, 'title': title, 'type': record_type }) return HttpResponse(template.render(context))
def retrieve_xml(docID): """ Get the xml assiociated to the id :param request: :return: """ xml_data = XMLdata.getXMLdata(docID) xml_dta = json.dumps(xml_data.items()[3][1],sort_keys=False) xml_d = json.loads(xml_dta, object_pairs_hook=OrderedDict) # Convert the ordered dict in dict xsdDocData = XMLdata.unparse(xml_d) xsdEncoded = xsdDocData.encode('utf-8') return xsdEncoded
def update_publish(request): XMLdata.update_publish(request.GET['result_id']) resource = XMLdata.get(request.GET['result_id']) # Send mail to the user and the admin context = {'URI': MDCS_URI, 'title': resource['title'], 'publicationdate': resource['publicationdate'], 'user': request.user.username} send_mail_to_managers(subject='Resource Published', pathToTemplate='dashboard/email/resource_published.html', context=context) return HttpResponse(json.dumps({}), content_type='application/javascript')
def save_xml_data_to_db(request): form_data_id = request.session['curateFormData'] form_data = FormData.objects.get(pk=form_data_id) form_id = request.session['form_id'] root_element = SchemaElement.objects.get(pk=form_id) xml_renderer = XmlRenderer(root_element) xml_string = xml_renderer.render() # xmlString = request.session['xmlString'] # template_id = request.session['currentTemplateID'] template_id = form_data.template # Parse data from form form = SaveDataForm(request.POST) if not form.data['title'].lower().endswith('.xml'): form.data['title'] += ".xml" if not form.is_valid(): return HttpResponseBadRequest('Invalid form name') if xml_string == "" or xml_string is None: return HttpResponseBadRequest('No XML data found') try: # update data if id is present if form_data.xml_data_id is not None: XMLdata.update_content( form_data.xml_data_id, xml_string, title=form.data['title'] ) else: # create new data otherwise xml_data = XMLdata( schemaID=template_id, xml=xml_string, title=form.data['title'], iduser=str(request.user.id) ) xml_data.save() form_data.delete() return HttpResponse('ok') except Exception, e: message = e.message.replace('"', '\'') return HttpResponseBadRequest(message)
def save_xml_data_to_db(request): form_data_id = request.session['curateFormData'] form_data = FormData.objects.get(pk=form_data_id) form_id = request.session['form_id'] root_element = SchemaElement.objects.get(pk=form_id) xml_renderer = XmlRenderer(root_element) xml_string = xml_renderer.render() template_id = form_data.template # Parse data from form form = SaveDataForm(request.POST) if not form.data['title'].lower().endswith('.xml'): form.data['title'] += ".xml" if not form.is_valid(): return HttpResponseBadRequest('Invalid form name') if xml_string == "" or xml_string is None: return HttpResponseBadRequest('No XML data found') try: # update data if id is present if form_data.xml_data_id is not None: XMLdata.update_content(form_data.xml_data_id, content=xml_string, title=form.data['title']) #Specific MDCS XMLdata.update_publish(form_data.xml_data_id) else: # create new data otherwise xml_data = XMLdata(schemaID=template_id, xml=xml_string, title=form.data['title'], iduser=str(request.user.id)) xml_data_id = xml_data.save() XMLdata.update_publish(xml_data_id) print 'SchemaID====', template_id print 'xml_data_id====', xml_data_id request.session['xml_data_id'] = str(xml_data_id) print 'request.session-xml_data_id', request.session['xml_data_id'] if form_data.schema_element_root is not None: delete_branch_from_db(form_data.schema_element_root.pk) form_data.delete() return HttpResponse('ok') except Exception, e: message = e.message.replace('"', '\'') return HttpResponseBadRequest(message)
def list_identifiers(self): try: #Template name self.template_name = 'oai_pmh/xml/list_identifiers.xml' query = dict() items = [] #Handle FROM and UNTIL query = self.check_dates() try: #Get the metadata format thanks to the prefix myMetadataFormat = OaiMyMetadataFormat.objects.get( metadataPrefix=self.metadataPrefix) #Get all template using it (activated True) templates = OaiTemplMfXslt.objects( myMetadataFormat=myMetadataFormat, activated=True).distinct(field="template") #Ids templatesID = [str(x.id) for x in templates] #If myMetadataFormat is linked to a template, we add the template id if myMetadataFormat.isTemplate: templatesID.append(str(myMetadataFormat.template.id)) except: #The metadata format doesn't exist raise cannotDisseminateFormat(self.metadataPrefix) if self.set: try: setsTemplates = OaiMySet.objects( setSpec=self.set).only('templates').get() templatesID = set(templatesID).intersection( [str(x.id) for x in setsTemplates.templates]) except Exception, e: raise noRecordsMatch for template in templatesID: #Retrieve sets for this template sets = OaiMySet.objects(templates=template).all() query['schema'] = template #The record has to be published query['ispublished'] = True #Get all records for this template data = XMLdata.executeQueryFullResult(query) #IF no records, go to the next template if len(data) == 0: continue for i in data: #Fill the response identifier = '%s:%s:id/%s' % (settings.OAI_SCHEME, settings.OAI_REPO_IDENTIFIER, str(i['_id'])) item_info = { 'identifier': identifier, 'last_modified': self.get_last_modified_date(i), 'sets': sets, 'deleted': i.get('status', '') == Status.DELETED } items.append(item_info) #If there is no records if len(items) == 0: raise noRecordsMatch return self.render_to_response({'items': items})
def curate_edit_data(request): try: if 'useForm' in request.GET and request.GET['useForm'] == 'true': pass else: xml_data_id = request.GET['id'] xml_data = XMLdata.get(xml_data_id) json_content = xml_data['content'] xml_content = xmltodict.unparse(json_content) request.session['curate_edit_data'] = xml_content request.session['curate_edit'] = True request.session['currentTemplateID'] = xml_data['schema'] # remove previously created forms when editing a new one previous_forms = FormData.objects(user=str(request.user.id), xml_data_id__exists=True) for previous_form in previous_forms: # TODO: check if need to delete all SchemaElements previous_form.delete() form_data = FormData( user=str(request.user.id), template=xml_data['schema'], name=xml_data['title'], xml_data=xml_content, xml_data_id=xml_data_id ) form_data.save() request.session['curateFormData'] = str(form_data.id) if 'form_id' in request.session: del request.session['form_id'] if 'formString' in request.session: del request.session['formString'] if 'xmlDocTree' in request.session: del request.session['xmlDocTree'] except: raise MDCSError("The document you are looking for doesn't exist.")
def curate_edit_data(request): try: xml_data_id = request.GET['id'] xml_data = XMLdata.get(xml_data_id) xml_content = xml_data['xml_file'] request.session['curate_edit'] = True request.session['currentTemplateID'] = xml_data['schema'] # remove previously created forms when editing a new one previous_forms = FormData.objects(user=str(request.user.id), xml_data_id__exists=True) for previous_form in previous_forms: if previous_form.schema_element_root is not None: delete_branch_from_db(previous_form.schema_element_root.pk) previous_form.delete() form_data = FormData(user=str(request.user.id), template=xml_data['schema'], name=xml_data['title'], xml_data=xml_content, xml_data_id=xml_data_id) form_data.save() request.session['curateFormData'] = str(form_data.pk) if 'form_id' in request.session: del request.session['form_id'] if 'xmlDocTree' in request.session: del request.session['xmlDocTree'] except: raise MDCSError("The document you are looking for doesn't exist.")
def curate_enter_data(request): print "BEGIN curate_enter_data(request)" try: context = RequestContext(request, {}) if 'id' in request.GET: xml_data_id = request.GET['id'] xml_data = XMLdata.get(xml_data_id) template = Template.objects().get(pk=ObjectId(xml_data['schema'])) context = RequestContext(request, { 'edit': True, 'template_name': template.title }) curate_edit_data(request) elif 'template' in request.GET: context = RequestContext( request, {'template_name': request.GET['template']}) curate_from_schema(request) elif 'templateid' in request.GET: pass template = loader.get_template('curate/curate_enter_data.html') return HttpResponse(template.render(context)) except MDCSError, e: template = loader.get_template('curate/errors.html') context = RequestContext(request, { 'errors': e.message, }) return HttpResponse(template.render(context))
def explore_detail_result_keyword(request): template = loader.get_template("oai_pmh/explore/explore_detail_results_keyword.html") result_id = request.GET["id"] record = OaiRecord.objects.get(pk=result_id) # schemaId = xmlString['schema'] if "title" in request.GET: title = request.GET["title"] else: title = record.identifier xmlString = XMLdata.unparse(record.getMetadataOrdered()).encode("utf-8") xsltPath = os.path.join(settings.SITE_ROOT, "static", "resources", "xsl", "xml2html.xsl") xslt = etree.parse(xsltPath) transform = etree.XSLT(xslt) dom = etree.fromstring(str(xmlString)) # Check if a custom list result XSLT has to be used try: metadataFormat = record.metadataformat if metadataFormat.template.ResultXsltDetailed: listXslt = etree.parse(BytesIO(metadataFormat.template.ResultXsltDetailed.content.encode("utf-8"))) transform = etree.XSLT(listXslt) newdom = transform(dom) else: newdom = transform(dom) except Exception, e: # We use the default one newdom = transform(dom)
def curate_enter_data(request): print "BEGIN curate_enter_data(request)" try: context = RequestContext(request, {}) if 'id' in request.GET: xml_data_id = request.GET['id'] xml_data = XMLdata.get(xml_data_id) template = Template.objects().get(pk=ObjectId(xml_data['schema'])) context = RequestContext(request, {'edit': True, 'template_name': template.title}) curate_edit_data(request) elif 'template' in request.GET: context = RequestContext(request, {'template_name': request.GET['template']}) curate_from_schema(request) elif 'templateid' in request.GET: pass template = loader.get_template('curate/curate_enter_data.html') return HttpResponse(template.render(context)) except MDCSError, e: template = loader.get_template('curate/errors.html') context = RequestContext(request, { 'errors': e.message, }) return HttpResponse(template.render(context))
def update_publish_draft(request): form_id = request.GET['draft_id'] form_data = FormData.objects.get(pk=form_id) xml_string = form_data.xml_data title = form_data.name if xml_string == "" or xml_string is None: return HttpResponseBadRequest('No XML data found') try: # update form data if id is present if form_data.xml_data_id is not None: publicationdate = XMLdata.update_publish_draft(form_data.xml_data_id, xml_string, str(request.user.id)) #Delete the form if form_data.schema_element_root is not None: delete_branch_from_db(form_data.schema_element_root.pk) form_data.delete() # Send mail to the managers context = {'URI': MDCS_URI, 'title': title, 'publicationdate': publicationdate, 'user': request.user.username} send_mail_to_managers(subject='Resource Published', pathToTemplate='dashboard/email/resource_published.html', context=context) messages.add_message(request, messages.INFO, 'New version published with success.') return HttpResponse(json.dumps({}), content_type='application/javascript') else: return HttpResponseBadRequest('Related record not found. Impossible to publish the draft') except Exception, e: message = e.message.replace('"', '\'') return HttpResponseBadRequest(message)
def explore_detail_result_process(request): result_id = request.GET['id'] xmlString = XMLdata.get(result_id) schemaId = xmlString['schema'] if 'title' in request.GET: title = request.GET['title'] else: title = xmlString['title'] xmlString = xmltodict.unparse(xmlString['content']).encode('utf-8') xsltPath = os.path.join(settings.SITE_ROOT, 'static', 'resources', 'xsl', 'xml2html.xsl') xslt = etree.parse(xsltPath) transform = etree.XSLT(xslt) #Check if a custom detailed result XSLT has to be used try: if (xmlString != ""): dom = etree.fromstring(str(xmlString)) schema = Template.objects.get(pk=schemaId) if schema.ResultXsltDetailed: shortXslt = etree.parse( BytesIO(schema.ResultXsltDetailed.content.encode('utf-8'))) shortTransform = etree.XSLT(shortXslt) newdom = shortTransform(dom) else: newdom = transform(dom) except Exception, e: #We use the default one newdom = transform(dom)
def explore_detail_result_keyword(request): template = loader.get_template( 'oai_pmh/explore/explore_detail_results_keyword.html') result_id = request.GET['id'] record = OaiRecord.objects.get(pk=result_id) # schemaId = xmlString['schema'] if 'title' in request.GET: title = request.GET['title'] else: title = record.identifier xmlString = XMLdata.unparse(record.getMetadataOrdered()).encode('utf-8') xsltPath = os.path.join(settings.SITE_ROOT, 'static', 'resources', 'xsl', 'xml2html.xsl') xslt = etree.parse(xsltPath) transform = etree.XSLT(xslt) dom = etree.fromstring(str(xmlString)) #Check if a custom list result XSLT has to be used try: metadataFormat = record.metadataformat if metadataFormat.template.ResultXsltDetailed: listXslt = etree.parse( BytesIO( metadataFormat.template.ResultXsltDetailed.content.encode( 'utf-8'))) transform = etree.XSLT(listXslt) newdom = transform(dom) else: newdom = transform(dom) except Exception, e: #We use the default one newdom = transform(dom)
def execute_query(filters=list(), projection=None): """ :param filters: :param projection: :return: """ # query = { # "$and": [] # } results_id = {xml_data["_id"] for xml_data in XMLdata.objects()} results = [] # Parsing filters if present for _filter in filters: if _is_advanced_filter(_filter): json_filter = json.loads(_filter) # Get matching document # list possible values of the right hand side # match resulted documents documents_field = json_filter["documents"].values()[0] values = get_filter_values(documents_field) matching_documents = get_matching_document( json_filter["documents"].keys()[0], values, json_filter["query"]) # Extract correct documents filter_result = [] for doc in matching_documents: doc_cross_query = { json_filter["documents"].values()[0]: get_projection(doc) } filter_result += XMLdata.executeQueryFullResult( doc_cross_query, json.loads(projection)) else: filter_result = XMLdata.executeQueryFullResult( json.loads(_filter), json.loads(projection)) filter_id = {document["_id"] for document in filter_result} results_id = results_id.intersection(filter_id) results = [doc for doc in filter_result if doc["_id"] in results_id] return results
def get_record(self): try: #Bool if we need to transform the XML via XSLT hasToBeTransformed = False #Check if the identifier pattern is OK id = self.check_identifier() #Template name self.template_name = 'oai_pmh/xml/get_record.xml' query = dict() #Convert id to ObjectId try: query['_id'] = ObjectId(id) #The record has to be published query['ispublished'] = True except Exception: raise idDoesNotExist(self.identifier) data = XMLdata.executeQueryFullResult(query) #This id doesn't exist if len(data) == 0: raise idDoesNotExist(self.identifier) data = data[0] #Get the template for the identifier template = data['schema'] #Retrieve sets for this template sets = OaiMySet.objects(templates=template).all() #Retrieve the XSLT for the transformation try: #Get the metadataformat for the provided prefix myMetadataFormat = OaiMyMetadataFormat.objects.get(metadataPrefix=self.metadataPrefix) #If this metadata prefix is not associated to a template, we need to retrieve the XSLT to do the transformation if not myMetadataFormat.isTemplate: hasToBeTransformed = True #Get information about the XSLT for the MF and the template objTempMfXslt = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, template=template, activated=True).get() #If no information or desactivated if not objTempMfXslt.xslt: raise cannotDisseminateFormat(self.metadataPrefix) else: #Get the XSLT for the transformation xslt = objTempMfXslt.xslt except: raise cannotDisseminateFormat(self.metadataPrefix) #Transform XML data dataToTransform = [{'title': data['_id'], 'content': self.cleanXML(xmltodict.unparse(data['content']))}] if hasToBeTransformed: dataXML = self.getXMLTranformXSLT(dataToTransform, xslt) else: dataXML = dataToTransform #Fill the response record_info = { 'identifier': self.identifier, 'last_modified': datestamp.datetime_to_datestamp(data['publicationdate']) if 'publicationdate' in data else datestamp.datetime_to_datestamp(datetime.datetime.min), 'sets': sets, 'XML': dataXML[0]['content'] } return self.render_to_response(record_info) except OAIExceptions, e: return self.errors(e.errors)
def my_profile_resources(request): template = loader.get_template('profile/my_profile_resources.html') if 'template' in request.GET: template_name = request.GET['template'] if template_name == 'all': context = RequestContext( request, { 'XMLdatas': XMLdata.find({'iduser': str(request.user.id)}), }) else: if template_name == 'datacollection': templateNamesQuery = list( chain( Template.objects.filter( title=template_name).values_list('id'), Template.objects.filter( title='repository').values_list('id'), Template.objects.filter( title='database').values_list('id'), Template.objects.filter( title='projectarchive').values_list('id'))) else: templateNamesQuery = Template.objects.filter( title=template_name).values_list('id') templateNames = [] for templateQuery in templateNamesQuery: templateNames.append(str(templateQuery)) context = RequestContext( request, { 'XMLdatas': XMLdata.find({ 'iduser': str(request.user.id), 'schema': { "$in": templateNames } }), 'template': template_name }) else: context = RequestContext( request, { 'XMLdatas': XMLdata.find({'iduser': str(request.user.id)}), }) return HttpResponse(template.render(context))
def test_or_numeric_3(self): criteria1 = build_criteria("content.root.integer", "=", 1, "xs:int", "xs") criteria2 = build_criteria("content.root.integer", "gte", 2, "xs:int", "xs") criteria = ORCriteria(criteria1, criteria2) results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 3)
def save_xml_data_to_db(request): xmlString = request.session['xmlString'] templateID = request.session['currentTemplateID'] form = SaveDataForm(request.POST) if form.is_valid(): if xmlString != "": try: # get form data from the database form_data_id = request.session['curateFormData'] form_data = FormData.objects.get(pk=ObjectId(form_data_id)) if not form.data['title'].lower().endswith('.xml'): form.data['title'] += ".xml" # update data if id is present if form_data.xml_data_id is not None: XMLdata.update_content(form_data.xml_data_id, xmlString, title=form.data['title']) else: #create new data otherwise newJSONData = XMLdata(schemaID=templateID, xml=xmlString, title=form.data['title'], iduser=str(request.user.id)) newJSONData.save() # delete form data try: form_data = FormData.objects().get(pk=form_data_id) # cascade delete references for form_element_id in form_data.elements.values(): try: form_element = FormElement.objects().get(pk=form_element_id) if form_element.xml_element is not None: try: xml_element = XMLElement.objects().get(pk=str(form_element.xml_element.id)) xml_element.delete() except: # raise an exception when element not found pass form_element.delete() except: # raise an exception when element not found pass form_data.delete() except Exception, e: return HttpResponseBadRequest('Unable to save data.') return HttpResponse('ok') except Exception, e: message = e.message.replace('"', '\'') return HttpResponseBadRequest(message)
def update_publish(request): XMLdata.update_publish(request.GET['result_id']) resource = XMLdata.get(request.GET['result_id']) # Send mail to the user and the admin context = { 'URI': MDCS_URI, 'title': resource['title'], 'publicationdate': resource['publicationdate'], 'user': request.user.username } send_mail_to_managers( subject='Resource Published', pathToTemplate='dashboard/email/resource_published.html', context=context) return HttpResponse(json.dumps({}), content_type='application/javascript')
def explore_detail_result_process(request, result_id): # result_id = request.GET['id'] xmlString = XMLdata.get(result_id) if 'title' in request.GET: title = request.GET['title'] else: title = None return get_detail_result(request, xmlString, title)
def test_parse_unparse_test(self): with open(join(RESOURCES_PATH, 'test.xml'), 'r') as data_file: data_content = data_file.read() # test parsing xml_data = XMLdata(xml=data_content) # test unparsing xml_string = unparse(xml_data.content['content']) self.assertEquals(_strip(data_content), _strip(xml_string))
def test_and_numeric_1(self): criteria1 = build_criteria("content.root.integer", "=", 1, "xs:int", "xs") criteria2 = build_criteria("content.root.integer", "lte", 3, "xs:int", "xs") criteria = ANDCriteria(criteria1, criteria2) results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 1)
def test_parse_unparse_test(self): with open(join(RESOURCES_PATH, 'test.xml'), 'r') as data_file: data_content = data_file.read() # test parsing xml_data = XMLdata(xml=data_content) # test unparsing xml_string = XMLdata.unparse(xml_data.content['content']) self.assertEquals(_strip(data_content), _strip(xml_string))
def test_decimal_not(self): criteria = build_criteria("content.root.float", "=", 1, "xs:float", "xs", isNot=True) results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 2)
def test_numeric_not(self): criteria = build_criteria("content.root.integer", "=", 1, "xs:int", "xs", isNot=True) results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 2)
def test_str_encoding(self): criteria = build_criteria("content.root.str", "is", "test", "xs:string", "xs", isNot=True) results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 1)
def test_enum_not(self): criteria = build_criteria("content.root.enum", "is", "a", "enum", "xs", isNot=True) results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 2)
def explore_detail_result_process(request): result_id = request.GET['id'] xml_data = XMLdata.get(result_id) schema_id = xml_data['schema'] if 'title' in request.GET: xml_data['title'] = request.GET['title'] schema = Template.objects.get(pk=schema_id) return _create_context_detail_view(request, xml_data, schema)
def test_get_record_no_templ_xslt(self): self.dump_oai_my_set() self.dump_xmldata() self.dump_oai_my_metadata_format() idXMLdata = str(XMLdata.find({'title': 'MGI Code Catalog.xml'})[0]['_id']) identifier = '%s:%s:id/%s' % (OAI_SCHEME, OAI_REPO_IDENTIFIER, idXMLdata) data = {'verb': 'GetRecord', 'identifier': identifier, 'metadataPrefix': 'oai_dc'} r = self.doRequestServer(data=data) self.isStatusOK(r.status_code) self.checkTagErrorCode(r.text, DISSEMINATE_FORMAT)
def test_regex_not(self): criteria = build_criteria("content.root.str", "like", "test", "xs:string", "xs", isNot=True) manageRegexBeforeExe(criteria) results = XMLdata.executeQueryFullResult(criteria) self.assertTrue(len(results) == 0)
def list_test_deleted(self, verb): self.dump_oai_templ_mf_xslt() self.dump_oai_my_metadata_format() self.dump_oai_my_set() self.dump_xmldata() data = {'verb': verb, 'metadataPrefix': 'oai_soft'} r = self.doRequestServer(data=data) self.isStatusOK(r.status_code) #Check attribute status='deleted' of header doesn't exist self.checkTagExist(r.text, verb) #Delete one record template = Template.objects(filename='Software.xsd').get() dataSoft = XMLdata.find({'schema': str(template.id), 'status': {'$ne': Status.DELETED}}) if len(dataSoft) > 0: XMLdata.update(dataSoft[0]['_id'], {'status': Status.DELETED}) r = self.doRequestServer(data=data) self.isStatusOK(r.status_code) self.checkTagExist(r.text, verb) #Check attribute status='deleted' of header does exist self.checkTagWithParamExist(r.text, 'header', 'status="deleted"')
def get_earliest_date(self): try: #Get the earliest publication date for the identify request response data = XMLdata.getMinValue('oai_datestamp') #If we have a date if data != None: return datestamp.datetime_to_datestamp(data) else: return '' except Exception: return ''
def get_filter_values(field): """ Get matching values in MongoDB for a given field :param field: :return: """ query = {field: {"$exists": True}} projection = {field: 1} documents = XMLdata.executeQueryFullResult(query, projection) return {get_projection(doc) for doc in documents}
def test_list_metadataformat_with_identifier(self): self.dump_xmldata() self.dump_oai_templ_mf_xslt() self.dump_oai_my_metadata_format() idXMLdata = str(XMLdata.find({'title':'MPInterfaces.xml'})[0]['_id']) identifier = '%s:%s:id/%s' % (OAI_SCHEME, OAI_REPO_IDENTIFIER, idXMLdata) data = {'verb': 'ListMetadataFormats', 'identifier': identifier} r = self.doRequestServer(data=data) self.isStatusOK(r.status_code) self.checkTagExist(r.text, 'ListMetadataFormats') self.checkTagExist(r.text, 'metadataFormat')
def get_earliest_date(self): try: #Get the earliest publication date for the identify request response data = XMLdata.getMinValue('publicationdate') #If we have a date if data != None: return datestamp.datetime_to_datestamp(data) else: return '' except Exception: return ''
def list_identifiers(self): try: #Template name self.template_name = 'oai_pmh/xml/list_identifiers.xml' query = dict() items=[] #Handle FROM and UNTIL query = self.check_dates() try: #Get the metadata format thanks to the prefix myMetadataFormat = OaiMyMetadataFormat.objects.get(metadataPrefix=self.metadataPrefix) #Get all template using it (activated True) templates = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, activated=True).distinct(field="template") #Ids templatesID = [str(x.id) for x in templates] #If myMetadataFormat is linked to a template, we add the template id if myMetadataFormat.isTemplate: templatesID.append(str(myMetadataFormat.template.id)) except: #The metadata format doesn't exist raise cannotDisseminateFormat(self.metadataPrefix) if self.set: try: setsTemplates = OaiMySet.objects(setSpec=self.set).only('templates').get() templatesID = set(templatesID).intersection([str(x.id) for x in setsTemplates.templates]) except Exception, e: raise noRecordsMatch for template in templatesID: #Retrieve sets for this template sets = OaiMySet.objects(templates=template).all() query['schema'] = template #The record has to be published query['ispublished'] = True #Get all records for this template data = XMLdata.executeQueryFullResult(query) #IF no records, go to the next template if len(data) == 0: continue for i in data: #Fill the response identifier = '%s:%s:id/%s' % (settings.OAI_SCHEME, settings.OAI_REPO_IDENTIFIER, str(i['_id'])) item_info = { 'identifier': identifier, 'last_modified': self.get_last_modified_date(i), 'sets': sets, 'deleted': i.get('status', '') == Status.DELETED } items.append(item_info) #If there is no records if len(items) == 0: raise noRecordsMatch return self.render_to_response({'items': items})
def dashboard_resources(request): template = loader.get_template('dashboard/my_dashboard_my_resources.html') if 'template' in request.GET: template_name = request.GET['template'] if template_name == 'datacollection': templateNamesQuery = list(chain(Template.objects.filter(title=template_name).values_list('id'), Template.objects.filter(title='repository').values_list('id'), Template.objects.filter(title='database').values_list('id'), Template.objects.filter(title='projectarchive').values_list('id'))) else : templateNamesQuery = Template.objects.filter(title=template_name).values_list('id') templateNames = [] for templateQuery in templateNamesQuery: templateNames.append(str(templateQuery)) if 'ispublished' in request.GET: ispublished = request.GET['ispublished'] context = RequestContext(request, { 'XMLdatas': sorted(XMLdata.find({'iduser' : str(request.user.id), 'schema':{"$in" : templateNames}, 'ispublished': ispublished=='true'}), key=lambda data: data['lastmodificationdate'], reverse=True), 'template': template_name, 'ispublished': ispublished, }) else: context = RequestContext(request, { 'XMLdatas': sorted(XMLdata.find({'iduser' : str(request.user.id), 'schema':{"$in" : templateNames}}), key=lambda data: data['lastmodificationdate'], reverse=True), 'template': template_name, }) else: if 'ispublished' in request.GET: ispublished = request.GET['ispublished'] context = RequestContext(request, { 'XMLdatas': sorted(XMLdata.find({'iduser' : str(request.user.id), 'ispublished': ispublished=='true'}), key=lambda data: data['lastmodificationdate'], reverse=True), 'ispublished': ispublished, }) else: context = RequestContext(request, { 'XMLdatas': sorted(XMLdata.find({'iduser' : str(request.user.id)}), key=lambda data: data['lastmodificationdate'], reverse=True), }) return HttpResponse(template.render(context))
def dashboard_detail_resource(request) : template = loader.get_template('dashboard/my_dashboard_detail_resource.html') result_id = request.GET['id'] type = request.GET['type'] if type=='form': form_data = FormData.objects.get(pk=ObjectId(result_id)) xmlString = form_data.xml_data.encode('utf-8') title = form_data.name schemaId = form_data.template elif type=='record': xmlString = XMLdata.get(result_id) title = xmlString['title'] schemaId = xmlString['schema'] xmlString = XMLdata.unparse(xmlString['content']).encode('utf-8') xsltPath = os.path.join(settings.SITE_ROOT, 'static', 'resources', 'xsl', 'xml2html.xsl') xslt = etree.parse(xsltPath) transform = etree.XSLT(xslt) #Check if a custom detailed result XSLT has to be used try: if (xmlString != ""): dom = etree.fromstring(str(xmlString)) schema = Template.objects.get(pk=schemaId) if schema.ResultXsltDetailed: shortXslt = etree.parse(BytesIO(schema.ResultXsltDetailed.content.encode('utf-8'))) shortTransform = etree.XSLT(shortXslt) newdom = shortTransform(dom) else: newdom = transform(dom) else: newdom = "No data to display" except Exception, e: #We use the default one newdom = transform(dom)
def test_get_record_deleted(self): self.dump_oai_templ_mf_xslt() self.dump_oai_my_metadata_format() self.dump_oai_my_set() self.dump_xmldata() template = Template.objects(filename='Software.xsd').get() dataSoft = XMLdata.find({'schema': str(template.id), 'status': {'$ne': Status.DELETED}}) if len(dataSoft) > 0: xmlDataId = dataSoft[0]['_id'] identifier = '%s:%s:id/%s' % (OAI_SCHEME, OAI_REPO_IDENTIFIER, xmlDataId) data = {'verb': 'GetRecord', 'identifier': identifier, 'metadataPrefix': 'oai_soft'} r = self.doRequestServer(data=data) self.isStatusOK(r.status_code) #Check attribute status='deleted' of header doesn't exist self.checkTagExist(r.text, 'GetRecord') self.checkTagExist(r.text, 'record') #Delete one record XMLdata.update(xmlDataId, {'status': Status.DELETED}) r = self.doRequestServer(data=data) self.isStatusOK(r.status_code) #Check attribute status='deleted' of header does exist self.checkTagExist(r.text, 'GetRecord') # Only for NMRR self.checkTagWithParamExist(r.text, 'header', 'status="deleted"')