def testAssertion(self):
datamap = Mapping('mappingId')
datamap.code = """
doAssert(1==1)
doAssert(1==2, "1 not equal 2")
upload.parts['record']="<somexml/>"
"""
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.addObserver(logger)
try:
datamap.createUpload(TestRepository(),
oaiResponse(),
doAsserts=True)
self.fail()
except DataMapAssertionException as ex:
self.assertEqual(
'ERROR\t[repository.id:oai:ident:321]\tAssertion: 1 not equal 2\n',
stream.getvalue()[26:])
self.assertEqual('1 not equal 2', str(ex))
try:
datamap.createUpload(TestRepository(),
oaiResponse(),
doAsserts=True)
self.fail()
except DataMapAssertionException as ex:
self.assertEqual('1 not equal 2', str(ex))
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=False)
self.assertEqual('', stream.getvalue())
def testHarvesterIgnoringInvalidDataErrors(self):
observer = CallTrace('observer')
upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid'))
upload.id = 'mockid'
observer.returnValues['createUpload'] = upload
observer.returnValues['totalInvalidIds'] = 0
observer.exceptions['send'] = InvalidDataException(upload.id, "message")
repository=CallTrace("repository", returnValues={'maxIgnore': 100})
harvester = Harvester(repository)
harvester.addObserver(observer)
harvester.upload(oaiResponse())
self.assertEquals(['createUpload', "notifyHarvestedRecord", "send", 'logInvalidData', "totalInvalidIds", 'logIgnoredIdentifierWarning'], [m.name for m in observer.calledMethods])
def testHarvesterStopsIgnoringAfter100records(self):
observer = CallTrace('observer')
upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid'))
upload.id = 'mockid'
observer.returnValues['createUpload'] = upload
observer.returnValues['totalInvalidIds'] = 101
observer.exceptions['send'] = InvalidDataException(upload.id, "message")
repository=CallTrace("repository", returnValues={'maxIgnore': 100})
harvester = Harvester(repository)
harvester.addObserver(observer)
self.assertRaises(TooMuchInvalidDataException, lambda: harvester.upload(oaiResponse(identifier='mockid')))
self.assertEquals(['createUpload', "notifyHarvestedRecord", "send", "logInvalidData", "totalInvalidIds"], [m.name for m in observer.calledMethods])
def testDeleteWithOaiEnvelope(self):
RECORD_FILENAME = join(self.tempdir, 'id.record')
self.uploader._filenameFor = lambda *args: RECORD_FILENAME
self.uploader.tznow = lambda: "VANDAAG_EN_NU"
self.target.oaiEnvelope = True
repository = CallTrace('Repository')
repository.repositoryGroupId = 'groupId'
repository.metadataPrefix = 'oai_dc'
repository.baseurl = "http://repository"
repository.id = 'repositoryId'
upload = Upload(repository=repository,
oaiResponse=oaiResponse(identifier='id.record',
deleted=True))
self.assertFalse(isfile(RECORD_FILENAME))
self.uploader.delete(upload)
self.assertTrue(isfile(RECORD_FILENAME))
with open(RECORD_FILENAME) as f:
self.assertEqualsWS(
"""<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
<responseDate>VANDAAG_EN_NU</responseDate>
<request verb="GetRecord" metadataPrefix="oai_dc" identifier="id.record">http://repository</request>
<GetRecord>
<record>
<header status="deleted">
<identifier>id.record</identifier>
<datestamp>2005-08-29T07:08:09Z</datestamp>
</header>
</record>
</GetRecord>
</OAI-PMH>""", f.read())
def testDeleteWithOaiEnvelope(self):
RECORD_FILENAME = join(self.tempdir, 'id.record')
self.uploader._filenameFor = lambda *args: RECORD_FILENAME
self.uploader.tznow = lambda: "VANDAAG_EN_NU"
self.target.oaiEnvelope = True
repository = CallTrace('Repository')
repository.repositoryGroupId = 'groupId'
repository.metadataPrefix = 'oai_dc'
repository.baseurl = "http://repository"
repository.id = 'repositoryId'
upload = Upload(repository=repository, oaiResponse=oaiResponse(identifier='id.record', deleted=True))
self.assertFalse(isfile(RECORD_FILENAME))
self.uploader.delete(upload)
self.assertTrue(isfile(RECORD_FILENAME))
self.assertEqualsWS("""<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
<responseDate>VANDAAG_EN_NU</responseDate>
<request verb="GetRecord" metadataPrefix="oai_dc" identifier="id.record">http://repository</request>
<GetRecord>
<record>
<header status="deleted">
<identifier>id.record</identifier>
<datestamp>2005-08-29T07:08:09Z</datestamp>
</header>
</record>
</GetRecord>
</OAI-PMH>""", open(RECORD_FILENAME).read())
def createUpload(about=None):
repository = CallTrace('repository')
repository.id = 'repoId'
upload = Upload(repository=repository, oaiResponse=oaiResponse(about=about))
upload.id = 'id'
return upload
def testHarvesterIgnoringInvalidDataErrors(self):
observer = CallTrace('observer')
upload = Upload(repository=None,
oaiResponse=oaiResponse(identifier='mockid'))
upload.id = 'mockid'
observer.returnValues['createUpload'] = upload
observer.returnValues['totalInvalidIds'] = 0
observer.exceptions['send'] = InvalidDataException(
upload.id, "message")
repository = CallTrace("repository", returnValues={'maxIgnore': 100})
harvester = Harvester(repository)
harvester.addObserver(observer)
harvester.upload(oaiResponse())
self.assertEqual([
'createUpload', "notifyHarvestedRecord", "send", 'logInvalidData',
"totalInvalidIds", 'logIgnoredIdentifierWarning'
], [m.name for m in observer.calledMethods])
def testNoLogging(self):
datamap = Mapping('mappingId')
datamap.code = """
upload.parts['record']="<somexml/>"
logger.logError('Iets om te zeuren')
"""
upload = datamap.createUpload(TestRepository(), oaiResponse())
self.assertEquals('<somexml/>',upload.parts['record'])
def testNoLogging(self):
datamap = Mapping('mappingId')
datamap.code = """
upload.parts['record']="<somexml/>"
logger.logError('Iets om te zeuren')
"""
upload = datamap.createUpload(TestRepository(), oaiResponse())
self.assertEqual('<somexml/>', upload.parts['record'])
def createUpload(about=None):
repository = CallTrace('repository')
repository.id = 'repoId'
upload = Upload(repository=repository,
oaiResponse=oaiResponse(about=about))
upload.id = 'id'
return upload
def testSend(self):
recordFile = self.tempdir + '/group/repo/id.record'
self.uploader._filenameFor = lambda *args: recordFile
upload = createUpload()
self.uploader.send(upload)
self.assertTrue(isfile(recordFile))
self.assertEqualsLxml(oaiResponse().record, parse(open(recordFile)))
def testSend(self):
recordFile = self.tempdir + '/group/repo/id.record'
self.uploader._filenameFor = lambda *args: recordFile
upload = createUpload()
self.uploader.send(upload)
self.assertTrue(isfile(recordFile))
with open(recordFile) as fp:
self.assertEqualsLxml(oaiResponse().record, parse(fp))
def testHarvesterStopsIgnoringAfter100records(self):
observer = CallTrace('observer')
upload = Upload(repository=None,
oaiResponse=oaiResponse(identifier='mockid'))
upload.id = 'mockid'
observer.returnValues['createUpload'] = upload
observer.returnValues['totalInvalidIds'] = 101
observer.exceptions['send'] = InvalidDataException(
upload.id, "message")
repository = CallTrace("repository", returnValues={'maxIgnore': 100})
harvester = Harvester(repository)
harvester.addObserver(observer)
self.assertRaises(
TooMuchInvalidDataException,
lambda: harvester.upload(oaiResponse(identifier='mockid')))
self.assertEqual([
'createUpload', "notifyHarvestedRecord", "send", "logInvalidData",
"totalInvalidIds"
], [m.name for m in observer.calledMethods])
def testSkippedRecord(self):
harvester = self.createHarvesterWithMockUploader('tud')
def createUpload(repository, oaiResponse):
upload = Upload(repository=repository, oaiResponse=oaiResponse)
upload.id = "tud:mockid"
upload.skip = True
return upload
self.mapper.createUpload = createUpload
harvester.upload(oaiResponse(identifier='mockid'))
self.assertEquals([], self.sendId)
self.assertFalse(hasattr(self, 'delete_id'))
def testLogging(self):
datamap = Mapping('mappingId')
datamap.code = """
upload.parts['record']="<somexml/>"
logger.logError('Iets om te zeuren')
"""
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.addObserver(logger)
datamap.createUpload(TestRepository(), oaiResponse=oaiResponse())
self.assertEquals('ERROR\t[]\tIets om te zeuren\n',stream.getvalue()[26:])
def testSkip(self):
datamap = Mapping('mappingId')
datamap.code = """
skipRecord("Don't like it here.")
"""
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.addObserver(logger)
upload = datamap.createUpload(TestRepository(), oaiResponse())
self.assertTrue(upload.skip)
self.assertEquals("SKIP\t[repository.id:oai:ident:321]\tDon't like it here.\n", stream.getvalue()[26:])
def testLogging(self):
datamap = Mapping('mappingId')
datamap.code = """
upload.parts['record']="<somexml/>"
logger.logError('Iets om te zeuren')
"""
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.addObserver(logger)
datamap.createUpload(TestRepository(), oaiResponse=oaiResponse())
self.assertEqual('ERROR\t[]\tIets om te zeuren\n',
stream.getvalue()[26:])
def testSkip(self):
datamap = Mapping('mappingId')
datamap.code = """
skipRecord("Don't like it here.")
"""
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.addObserver(logger)
upload = datamap.createUpload(TestRepository(), oaiResponse())
self.assertTrue(upload.skip)
self.assertEqual(
"SKIP\t[repository.id:oai:ident:321]\tDon't like it here.\n",
stream.getvalue()[26:])
def testSkippedRecord(self):
harvester = self.createHarvesterWithMockUploader('tud')
def createUpload(repository, oaiResponse):
upload = Upload(repository=repository, oaiResponse=oaiResponse)
upload.id = "tud:mockid"
upload.skip = True
return upload
self.mapper.createUpload = createUpload
harvester.upload(oaiResponse(identifier='mockid'))
self.assertEqual([], self.sendId)
self.assertFalse(hasattr(self, 'delete_id'))
def testAssertion(self):
datamap = Mapping('mappingId')
datamap.code = """
doAssert(1==1)
doAssert(1==2, "1 not equal 2")
upload.parts['record']="<somexml/>"
"""
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.addObserver(logger)
try:
datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=True)
self.fail()
except DataMapAssertionException, ex:
self.assertEquals('ERROR\t[repository.id:oai:ident:321]\tAssertion: 1 not equal 2\n',stream.getvalue()[26:])
self.assertEquals('1 not equal 2', str(ex))
def assertPart(self, expected, partname, code):
datamap = Mapping('mappingId')
datamap.code = code
upload = datamap.createUpload(TestRepository(), oaiResponse())
self.assertEquals(expected,upload.parts[partname])
doAssert(1==2, "1 not equal 2")
upload.parts['record']="<somexml/>"
"""
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.addObserver(logger)
try:
datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=True)
self.fail()
except DataMapAssertionException, ex:
self.assertEquals('ERROR\t[repository.id:oai:ident:321]\tAssertion: 1 not equal 2\n',stream.getvalue()[26:])
self.assertEquals('1 not equal 2', str(ex))
try:
datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=True)
self.fail()
except DataMapAssertionException, ex:
self.assertEquals('1 not equal 2', str(ex))
stream = StringIO()
logger = StreamEventLogger(stream)
datamap.createUpload(TestRepository(), oaiResponse() , doAsserts=False)
self.assertEquals('',stream.getvalue())
def assertPart(self, expected, partname, code):
datamap = Mapping('mappingId')
datamap.code = code
upload = datamap.createUpload(TestRepository(), oaiResponse())
self.assertEquals(expected,upload.parts[partname])
def testUploadRecord(self):
harvester = self.createHarvesterWithMockUploader('tud')
harvester.upload(oaiResponse(identifier='mockid'))
self.assertEquals(['tud:mockid'], self.sendId)
self.assertFalse(hasattr(self, 'delete_id'))
def testDelete(self):
harvester = self.createHarvesterWithMockUploader('tud')
harvester.upload(oaiResponse(identifier='mockid', deleted=True))
self.assertEquals([], self.sendId)
self.assertEquals('tud:mockid', self.delete_id)
def testDelete(self):
harvester = self.createHarvesterWithMockUploader('tud')
harvester.upload(oaiResponse(identifier='mockid', deleted=True))
self.assertEqual([], self.sendId)
self.assertEqual('tud:mockid', self.delete_id)
def testUploadRecord(self):
harvester = self.createHarvesterWithMockUploader('tud')
harvester.upload(oaiResponse(identifier='mockid'))
self.assertEqual(['tud:mockid'], self.sendId)
self.assertFalse(hasattr(self, 'delete_id'))
def assertPart(self, expected, partname, code):
datamap = Mapping('mappingId')
datamap.code = code
upload = datamap.createUpload(TestRepository(), oaiResponse())
self.assertEqual(expected, upload.parts[partname])
