def test_invalid_conversion(self): # Arrange date_to_convert = "20166-11-18T08:30:00Z" # Act # Assert with self.assertRaises(Exception): UTCdatetime.utc_datetime_iso8601_to_datetime(date_to_convert)
def render_to_response(self, context, **response_kwargs): # All OAI responses should be XML if "content_type" not in response_kwargs: response_kwargs["content_type"] = self.content_type # Add common context data needed for all responses context.update({ "now": UTCdatetime.datetime_to_utc_datetime_iso8601(datetime.now()), "verb": self.oai_verb, "identifier": self.identifier, "metadataPrefix": self.metadata_prefix, "url": self.request.build_absolute_uri(self.request.path), "from": self.from_date, "until": self.until_date, "set": self.set, }) # Render the template with the context information return super().render_to_response(context, **response_kwargs)
def render_to_response(self, context, **response_kwargs): # All OAI responses should be XML if 'content_type' not in response_kwargs: response_kwargs['content_type'] = self.content_type # Add common context data needed for all responses context.update({ 'now': UTCdatetime.datetime_to_utc_datetime_iso8601(datetime.now()), 'verb': self.oai_verb, 'identifier': self.identifier if hasattr(self, 'identifier') else None, 'metadataPrefix': self.metadata_prefix if hasattr(self, 'metadata_prefix') else None, 'url': self.request.build_absolute_uri(self.request.path), 'from': self.From if hasattr(self, 'From') else None, 'until': self.until if hasattr(self, 'until') else None, 'set': self.set if hasattr(self, 'set') else None, }) # Render the template with the context information return super(TemplateView, self) \ .render_to_response(context, **response_kwargs)
def transform_dict_record_to_oai_record(data, registry_all_sets=[]): """Transforms a dict to a list of OaiRecord object. Args: data: Data to transform. registry_all_sets: List of all sets. Returns: List of OaiRecord instances. """ list_records = [] for obj in data: oai_record = OaiRecord() oai_record.identifier = obj["identifier"] oai_record.last_modification_date = ( UTCdatetime.utc_datetime_iso8601_to_datetime(obj["datestamp"]) ) oai_record.deleted = obj["deleted"] oai_record.harvester_sets = [ x for x in registry_all_sets if x.set_spec in obj["sets"] ] oai_record.xml_content = ( str(obj["metadata"]) if obj["metadata"] is not None else None ) list_records.append(oai_record) return list_records
def test_valid_conversion(self): # Arrange date_to_convert = "2016-11-18T08:30:00Z" # Act result = UTCdatetime.utc_datetime_iso8601_to_datetime(date_to_convert) # Assert self.assertEquals(result, datetime.datetime(2016, 11, 18, 8, 30))
def get_record(self): """ Response to GetRecord request. Returns: XML type response. """ try: self.template_name = 'core_oaipmh_provider_app/user/xml/get_record.html' # Check if the identifier pattern is OK. record_id = CheckOaiPmhRequest.check_identifier(self.identifier) try: oai_data = oai_data_api.get_by_data(record_id) except: raise oai_provider_exceptions.IdDoesNotExist(self.identifier) try: metadata_format = oai_provider_metadata_format_api.get_by_metadata_prefix( self.metadata_prefix) # Check if the record and the given metadata prefix use the same template. use_raw = metadata_format.is_template and ( oai_data.template == metadata_format.template) if oai_data.status != oai_status.DELETED: if use_raw: xml = oai_data.data.xml_content else: xslt = oai_xsl_template_api.get_by_template_id_and_metadata_format_id( oai_data.template.id, metadata_format.id).xslt xml = xsl_transformation_api.xsl_transform( oai_data.data.xml_content, xslt.name) else: xml = None except: raise oai_provider_exceptions.CannotDisseminateFormat( self.metadata_prefix) record_info = { 'identifier': self.identifier, 'last_modified': UTCdatetime.datetime_to_utc_datetime_iso8601( oai_data.oai_date_stamp), 'sets': oai_provider_set_api.get_all_by_template_ids( [oai_data.template.id]), 'XML': xml, 'deleted': oai_data.status == oai_status.DELETED } return self.render_to_response(record_info) except oai_provider_exceptions.OAIExceptions, e: return self.errors(e.errors)
def get_last_update_by_metadata_format_and_set(oai_harvester_metadata_format, oai_harvester_set): """ Get the last update by OaiHarvesterMetadataFormat and OaiHarvesterSet. Args: oai_harvester_metadata_format: oai_harvester_set: Returns: OaiHarvesterMetadataFormatSet last update (string). """ return UTCdatetime.datetime_to_utc_datetime_iso8601(get_by_metadata_format_and_set(oai_harvester_metadata_format, oai_harvester_set).last_update)
def _get_items(templates_id, from_date, until_date, metadata_format, include_metadata=False, use_raw=True): items = [] for template in templates_id: try: xslt = None if not use_raw: xslt = oai_xsl_template_api.get_by_template_id_and_metadata_format_id( template, metadata_format).xslt oai_data = oai_data_api.get_all_by_template( template, from_date=from_date, until_date=until_date) for elt in oai_data: # Only data from public workspace. if elt.data.workspace is not None and \ workspace_api.is_workspace_public(elt.data.workspace): identifier = '%s:%s:id/%s' % ( settings.OAI_SCHEME, settings.OAI_REPO_IDENTIFIER, str(elt.data_id)) item_info = { 'identifier': identifier, 'last_modified': UTCdatetime.datetime_to_utc_datetime_iso8601( elt.oai_date_stamp), 'sets': oai_provider_set_api.get_all_by_template_ids( [template]), 'deleted': elt.status == oai_status.DELETED } if include_metadata and elt.status == oai_status.ACTIVE: if use_raw: item_info.update({'XML': elt.data.xml_content}) else: xml = xsl_transformation_api.xsl_transform( elt.data.xml_content, xslt.name) item_info.update({'XML': xml}) items.append(item_info) except (exceptions.DoesNotExist, exceptions.XMLError, Exception): pass # If there is no records if len(items) == 0: raise oai_provider_exceptions.NoRecordsMatch return items
def check_resumption_token(resumption_token): """Check resumption token and return associated OAIRequestPage object Args: resumption_token: Raises: BadResumptionToken: Returns: """ try: oai_request_page_object = oai_request_page_api.get_by_resumption_token( resumption_token) # Check if the resumption token is not expired if UTCdatetime.datetime_to_utc_datetime_iso8601( oai_request_page_object.expiration_date ) < UTCdatetime.datetime_to_utc_datetime_iso8601(datetime.now()): raise Exception("Token expired") return oai_request_page_object except Exception: raise oai_provider_exceptions.BadResumptionToken(resumption_token)
def _check_dates(date): """Check date in parameter. Args: date: Date. Returns: Date. Raises: Exception: """ try: return UTCdatetime.utc_datetime_iso8601_to_datetime(date) except Exception as e: raise e
def _harvest_by_metadata_formats(registry, metadata_formats, registry_all_sets): """ Harvests data by metadata formats. Args: registry: Registry. metadata_formats: List of metadata formats to harvest. registry_all_sets: List of all sets. Returns: List of potential errors. """ all_errors = [] for metadata_format in metadata_formats: try: # Retrieve the last update for this metadata format last_update = UTCdatetime.datetime_to_utc_datetime_iso8601( metadata_format.last_update) except: last_update = None # Update the new date for the metadataFormat current_update_mf = datetime.datetime.now() errors = _harvest_records(registry, metadata_format, last_update, registry_all_sets) # If no exceptions was thrown and no errors occurred, we can update the last_update date if len(errors) == 0: # Update the update date for all sets # Would be useful if we do a _harvest_by_metadata_formats_and_sets in the future: won't retrieve everything if len(registry_all_sets) != 0: for set_ in registry_all_sets: oai_harvester_metadata_format_set_api\ .upsert_last_update_by_metadata_format_and_set(metadata_format, set_, current_update_mf) # Update the update date metadata_format.last_update = current_update_mf oai_harvester_metadata_format_api.upsert(metadata_format) else: all_errors.append(errors) return all_errors
def _get_earliest_date(): try: return UTCdatetime.datetime_to_utc_datetime_iso8601( oai_data_api.get_earliest_data_date()) except (exceptions.ModelError, Exception): return UTCdatetime.datetime_to_utc_datetime_iso8601(datetime.min)
def get_record(self): """Response to GetRecord request. Returns: XML type response. """ try: self.template_name = "core_oaipmh_provider_app/user/xml/get_record.html" # Check if the identifier pattern is OK. record_id = request_checker.check_identifier(self.identifier) try: oai_data = oai_data_api.get_by_data(record_id) except Exception: raise oai_provider_exceptions.IdDoesNotExist(self.identifier) try: metadata_format = ( oai_provider_metadata_format_api.get_by_metadata_prefix( self.metadata_prefix)) # Check if the record and the given metadata prefix use the same template. use_raw = (metadata_format.is_template and oai_data.template == metadata_format.template) if oai_data.status != oai_status.DELETED: xml = re.sub(r"<\?xml[^?]+\?>", "", oai_data.data.xml_content) if not use_raw: xslt = oai_xsl_template_api.get_by_template_id_and_metadata_format_id( oai_data.template.id, metadata_format.id).xslt xml = xsl_transformation_api.xsl_transform( xml, xslt.name) else: xml = None except Exception: raise oai_provider_exceptions.CannotDisseminateFormat( self.metadata_prefix) record_info = { "identifier": self.identifier, "last_modified": UTCdatetime.datetime_to_utc_datetime_iso8601( oai_data.oai_date_stamp), "sets": oai_provider_set_api.get_all_by_template_ids( [oai_data.template.id], request=self.request), "xml": xml, "deleted": oai_data.status == oai_status.DELETED, } return self.render_to_response(record_info) except oai_provider_exceptions.OAIExceptions as e: return self.errors(e.errors) except oai_provider_exceptions.OAIException as e: return self.error(e) except Exception as e: return HttpResponse( {"content": escape(str(e))}, status=status.HTTP_500_INTERNAL_SERVER_ERROR, )
def _get_items( template_id_list, metadata_format, oai_set=None, from_date=None, until_date=None, include_metadata=False, use_raw=True, page_nb=1, request=None, ): items = [] output_resumption_token = None try: # FIXME filtering public data is needed since all data has # a OAI data associated with it public_workspace_list = workspace_api.get_all_public_workspaces() data_list = system_api.get_all_data_in_workspaces_for_templates( public_workspace_list, template_id_list) oai_data = oai_data_api.get_all_by_data_list(data_list, from_date=from_date, until_date=until_date) oai_data_paginator = Paginator(oai_data, RESULTS_PER_PAGE) # If there are more pages to display if oai_data_paginator.num_pages > page_nb: exp_date = UTCdatetime.datetime_to_utc_datetime_iso8601( datetime.now() + timedelta(days=7)) oai_request_page_object = oai_request_page_api.upsert( OaiRequestPage( template_id_list=template_id_list, metadata_format=metadata_format.metadata_prefix, oai_set=oai_set, from_date=from_date, until_date=until_date, expiration_date=exp_date, page_number=page_nb + 1, )) output_resumption_token = { "token": oai_request_page_object.resumption_token, "expiration_date": exp_date, "list_size": oai_data.count(), "cursor": RESULTS_PER_PAGE * (page_nb - 1), } elif page_nb != 1: # If on the last page, send empty token output_resumption_token = { "token": "", "list_size": oai_data.count(), "cursor": RESULTS_PER_PAGE * (page_nb - 1), } for elt in oai_data_paginator.page(page_nb): identifier = "%s:%s:id/%s" % ( settings.OAI_SCHEME, settings.OAI_REPO_IDENTIFIER, str(elt.data_id), ) item_info = { "identifier": identifier, "last_modified": UTCdatetime.datetime_to_utc_datetime_iso8601( elt.oai_date_stamp), "sets": oai_provider_set_api.get_all_by_template_ids( [elt.data.template], request=request), "deleted": elt.status == oai_status.DELETED, } # Add data information if needed if include_metadata and elt.status == oai_status.ACTIVE: xml = re.sub(r"<\?xml[^?]+\?>", "", elt.data.xml_content) if not use_raw: # FIXME gets recomputed for every element xslt = oai_xsl_template_api.get_by_template_id_and_metadata_format_id( elt.data.template, metadata_format).xslt xml = xsl_transformation_api.xsl_transform( xml, xslt.name) item_info.update({"xml": xml}) items.append(item_info) except Exception as e: logger.warning("_get_items threw an exception: {0}".format(str(e))) if len(items) == 0: # No records retrieved raise oai_provider_exceptions.NoRecordsMatch return items, output_resumption_token