def testDeleteWithoutOaiEnvelope(self): recordFile = join(self.tempdir, 'id.record') os.system('touch ' + recordFile) self.assertTrue(isfile(recordFile)) self.uploader._filenameFor = lambda *args: recordFile self.target.oaiEnvelope = False repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.id = 'repositoryId' upload = Upload(repository=repository) upload.id = 'id' self.uploader.delete(upload) DELETED_RECORDS = join(self.tempdir, 'deleted_records') self.assertTrue(isfile(DELETED_RECORDS)) with open(DELETED_RECORDS) as fp: self.assertEqual(['id\n'], fp.readlines()) self.assertFalse(isfile(recordFile)) upload.id = 'second:id' self.uploader.delete(upload) with open(DELETED_RECORDS) as fp: self.assertEqual(['id\n', 'second:id\n'], fp.readlines())
def testDeleteWithoutOaiEnvelope(self): recordFile = join(self.tempdir, 'id.record') os.system('touch ' + recordFile) self.assertTrue(isfile(recordFile)) self.uploader._filenameFor = lambda *args: recordFile self.target.oaiEnvelope = False repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.id = 'repositoryId' upload = Upload(repository=repository) upload.id = 'id' self.uploader.delete(upload) DELETED_RECORDS = join(self.tempdir, 'deleted_records') self.assertTrue(isfile(DELETED_RECORDS)) self.assertEquals(['id\n'], open(DELETED_RECORDS).readlines()) self.assertFalse(isfile(recordFile)) upload.id = 'second:id' self.uploader.delete(upload) self.assertEquals(['id\n', 'second:id\n'], open(DELETED_RECORDS).readlines())
def createUpload(about=None): repository = CallTrace('repository') repository.id = 'repoId' upload = Upload(repository=repository, oaiResponse=oaiResponse(about=about)) upload.id = 'id' return upload
def getFilename(anId): repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.id = 'repositoryId' upload = Upload(repository=repository) upload.id = anId return self.uploader._filenameFor(upload)
def testHarvesterIgnoringInvalidDataErrors(self): observer = CallTrace('observer') upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid')) upload.id = 'mockid' observer.returnValues['createUpload'] = upload observer.returnValues['totalInvalidIds'] = 0 observer.exceptions['send'] = InvalidDataException(upload.id, "message") repository=CallTrace("repository", returnValues={'maxIgnore': 100}) harvester = Harvester(repository) harvester.addObserver(observer) harvester.upload(oaiResponse()) self.assertEquals(['createUpload', "notifyHarvestedRecord", "send", 'logInvalidData', "totalInvalidIds", 'logIgnoredIdentifierWarning'], [m.name for m in observer.calledMethods])
def testHarvesterStopsIgnoringAfter100records(self): observer = CallTrace('observer') upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid')) upload.id = 'mockid' observer.returnValues['createUpload'] = upload observer.returnValues['totalInvalidIds'] = 101 observer.exceptions['send'] = InvalidDataException(upload.id, "message") repository=CallTrace("repository", returnValues={'maxIgnore': 100}) harvester = Harvester(repository) harvester.addObserver(observer) self.assertRaises(TooMuchInvalidDataException, lambda: harvester.upload(oaiResponse(identifier='mockid'))) self.assertEquals(['createUpload', "notifyHarvestedRecord", "send", "logInvalidData", "totalInvalidIds"], [m.name for m in observer.calledMethods])
def testDeleteWithOaiEnvelope(self): RECORD_FILENAME = join(self.tempdir, 'id.record') self.uploader._filenameFor = lambda *args: RECORD_FILENAME self.uploader.tznow = lambda: "VANDAAG_EN_NU" self.target.oaiEnvelope = True repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.metadataPrefix = 'oai_dc' repository.baseurl = "http://repository" repository.id = 'repositoryId' upload = Upload(repository=repository, oaiResponse=oaiResponse(identifier='id.record', deleted=True)) self.assertFalse(isfile(RECORD_FILENAME)) self.uploader.delete(upload) self.assertTrue(isfile(RECORD_FILENAME)) with open(RECORD_FILENAME) as f: self.assertEqualsWS( """<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"> <responseDate>VANDAAG_EN_NU</responseDate> <request verb="GetRecord" metadataPrefix="oai_dc" identifier="id.record">http://repository</request> <GetRecord> <record> <header status="deleted"> <identifier>id.record</identifier> <datestamp>2005-08-29T07:08:09Z</datestamp> </header> </record> </GetRecord> </OAI-PMH>""", f.read())
def testHarvesterIgnoringInvalidDataErrors(self): observer = CallTrace('observer') upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid')) upload.id = 'mockid' observer.returnValues['createUpload'] = upload observer.returnValues['totalInvalidIds'] = 0 observer.exceptions['send'] = InvalidDataException( upload.id, "message") repository = CallTrace("repository", returnValues={'maxIgnore': 100}) harvester = Harvester(repository) harvester.addObserver(observer) harvester.upload(oaiResponse()) self.assertEqual([ 'createUpload', "notifyHarvestedRecord", "send", 'logInvalidData', "totalInvalidIds", 'logIgnoredIdentifierWarning' ], [m.name for m in observer.calledMethods])
def testHarvesterStopsIgnoringAfter100records(self): observer = CallTrace('observer') upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid')) upload.id = 'mockid' observer.returnValues['createUpload'] = upload observer.returnValues['totalInvalidIds'] = 101 observer.exceptions['send'] = InvalidDataException( upload.id, "message") repository = CallTrace("repository", returnValues={'maxIgnore': 100}) harvester = Harvester(repository) harvester.addObserver(observer) self.assertRaises( TooMuchInvalidDataException, lambda: harvester.upload(oaiResponse(identifier='mockid'))) self.assertEqual([ 'createUpload', "notifyHarvestedRecord", "send", "logInvalidData", "totalInvalidIds" ], [m.name for m in observer.calledMethods])
def createUpload(repository, oaiResponse): upload = Upload(repository=repository, oaiResponse=oaiResponse) upload.id = "tud:mockid" upload.skip = True return upload