def testDeleteWithoutOaiEnvelope(self):
recordFile = join(self.tempdir, 'id.record')
os.system('touch ' + recordFile)
self.assertTrue(isfile(recordFile))
self.uploader._filenameFor = lambda *args: recordFile
self.target.oaiEnvelope = False
repository = CallTrace('Repository')
repository.repositoryGroupId = 'groupId'
repository.id = 'repositoryId'
upload = Upload(repository=repository)
upload.id = 'id'
self.uploader.delete(upload)
DELETED_RECORDS = join(self.tempdir, 'deleted_records')
self.assertTrue(isfile(DELETED_RECORDS))
with open(DELETED_RECORDS) as fp:
self.assertEqual(['id\n'], fp.readlines())
self.assertFalse(isfile(recordFile))
upload.id = 'second:id'
self.uploader.delete(upload)
with open(DELETED_RECORDS) as fp:
self.assertEqual(['id\n', 'second:id\n'], fp.readlines())
def testDeleteWithoutOaiEnvelope(self):
recordFile = join(self.tempdir, 'id.record')
os.system('touch ' + recordFile)
self.assertTrue(isfile(recordFile))
self.uploader._filenameFor = lambda *args: recordFile
self.target.oaiEnvelope = False
repository = CallTrace('Repository')
repository.repositoryGroupId = 'groupId'
repository.id = 'repositoryId'
upload = Upload(repository=repository)
upload.id = 'id'
self.uploader.delete(upload)
DELETED_RECORDS = join(self.tempdir, 'deleted_records')
self.assertTrue(isfile(DELETED_RECORDS))
self.assertEquals(['id\n'], open(DELETED_RECORDS).readlines())
self.assertFalse(isfile(recordFile))
upload.id = 'second:id'
self.uploader.delete(upload)
self.assertEquals(['id\n', 'second:id\n'], open(DELETED_RECORDS).readlines())
def createUpload(about=None):
repository = CallTrace('repository')
repository.id = 'repoId'
upload = Upload(repository=repository, oaiResponse=oaiResponse(about=about))
upload.id = 'id'
return upload
def getFilename(anId):
repository = CallTrace('Repository')
repository.repositoryGroupId = 'groupId'
repository.id = 'repositoryId'
upload = Upload(repository=repository)
upload.id = anId
return self.uploader._filenameFor(upload)
def getFilename(anId):
repository = CallTrace('Repository')
repository.repositoryGroupId = 'groupId'
repository.id = 'repositoryId'
upload = Upload(repository=repository)
upload.id = anId
return self.uploader._filenameFor(upload)
def createUpload(about=None):
repository = CallTrace('repository')
repository.id = 'repoId'
upload = Upload(repository=repository,
oaiResponse=oaiResponse(about=about))
upload.id = 'id'
return upload
def testHarvesterIgnoringInvalidDataErrors(self):
observer = CallTrace('observer')
upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid'))
upload.id = 'mockid'
observer.returnValues['createUpload'] = upload
observer.returnValues['totalInvalidIds'] = 0
observer.exceptions['send'] = InvalidDataException(upload.id, "message")
repository=CallTrace("repository", returnValues={'maxIgnore': 100})
harvester = Harvester(repository)
harvester.addObserver(observer)
harvester.upload(oaiResponse())
self.assertEquals(['createUpload', "notifyHarvestedRecord", "send", 'logInvalidData', "totalInvalidIds", 'logIgnoredIdentifierWarning'], [m.name for m in observer.calledMethods])
def testHarvesterStopsIgnoringAfter100records(self):
observer = CallTrace('observer')
upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid'))
upload.id = 'mockid'
observer.returnValues['createUpload'] = upload
observer.returnValues['totalInvalidIds'] = 101
observer.exceptions['send'] = InvalidDataException(upload.id, "message")
repository=CallTrace("repository", returnValues={'maxIgnore': 100})
harvester = Harvester(repository)
harvester.addObserver(observer)
self.assertRaises(TooMuchInvalidDataException, lambda: harvester.upload(oaiResponse(identifier='mockid')))
self.assertEquals(['createUpload', "notifyHarvestedRecord", "send", "logInvalidData", "totalInvalidIds"], [m.name for m in observer.calledMethods])
def testDeleteWithOaiEnvelope(self):
RECORD_FILENAME = join(self.tempdir, 'id.record')
self.uploader._filenameFor = lambda *args: RECORD_FILENAME
self.uploader.tznow = lambda: "VANDAAG_EN_NU"
self.target.oaiEnvelope = True
repository = CallTrace('Repository')
repository.repositoryGroupId = 'groupId'
repository.metadataPrefix = 'oai_dc'
repository.baseurl = "http://repository"
repository.id = 'repositoryId'
upload = Upload(repository=repository,
oaiResponse=oaiResponse(identifier='id.record',
deleted=True))
self.assertFalse(isfile(RECORD_FILENAME))
self.uploader.delete(upload)
self.assertTrue(isfile(RECORD_FILENAME))
with open(RECORD_FILENAME) as f:
self.assertEqualsWS(
"""<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
<responseDate>VANDAAG_EN_NU</responseDate>
<request verb="GetRecord" metadataPrefix="oai_dc" identifier="id.record">http://repository</request>
<GetRecord>
<record>
<header status="deleted">
<identifier>id.record</identifier>
<datestamp>2005-08-29T07:08:09Z</datestamp>
</header>
</record>
</GetRecord>
</OAI-PMH>""", f.read())
def testHarvesterIgnoringInvalidDataErrors(self):
observer = CallTrace('observer')
upload = Upload(repository=None,
oaiResponse=oaiResponse(identifier='mockid'))
upload.id = 'mockid'
observer.returnValues['createUpload'] = upload
observer.returnValues['totalInvalidIds'] = 0
observer.exceptions['send'] = InvalidDataException(
upload.id, "message")
repository = CallTrace("repository", returnValues={'maxIgnore': 100})
harvester = Harvester(repository)
harvester.addObserver(observer)
harvester.upload(oaiResponse())
self.assertEqual([
'createUpload', "notifyHarvestedRecord", "send", 'logInvalidData',
"totalInvalidIds", 'logIgnoredIdentifierWarning'
], [m.name for m in observer.calledMethods])
def testHarvesterStopsIgnoringAfter100records(self):
observer = CallTrace('observer')
upload = Upload(repository=None,
oaiResponse=oaiResponse(identifier='mockid'))
upload.id = 'mockid'
observer.returnValues['createUpload'] = upload
observer.returnValues['totalInvalidIds'] = 101
observer.exceptions['send'] = InvalidDataException(
upload.id, "message")
repository = CallTrace("repository", returnValues={'maxIgnore': 100})
harvester = Harvester(repository)
harvester.addObserver(observer)
self.assertRaises(
TooMuchInvalidDataException,
lambda: harvester.upload(oaiResponse(identifier='mockid')))
self.assertEqual([
'createUpload', "notifyHarvestedRecord", "send", "logInvalidData",
"totalInvalidIds"
], [m.name for m in observer.calledMethods])
def createUpload(repository, oaiResponse):
upload = Upload(repository=repository, oaiResponse=oaiResponse)
upload.id = "tud:mockid"
upload.skip = True
return upload
def createUpload(repository, oaiResponse):
upload = Upload(repository=repository, oaiResponse=oaiResponse)
upload.id = "tud:mockid"
upload.skip = True
return upload
