def testAssertion(self): datamap = Mapping('mappingId') datamap.code = """ doAssert(1==1) doAssert(1==2, "1 not equal 2") upload.parts['record']="<somexml/>" """ stream = StringIO() logger = StreamEventLogger(stream) datamap.addObserver(logger) try: datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=True) self.fail() except DataMapAssertionException as ex: self.assertEqual( 'ERROR\t[repository.id:oai:ident:321]\tAssertion: 1 not equal 2\n', stream.getvalue()[26:]) self.assertEqual('1 not equal 2', str(ex)) try: datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=True) self.fail() except DataMapAssertionException as ex: self.assertEqual('1 not equal 2', str(ex)) stream = StringIO() logger = StreamEventLogger(stream) datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=False) self.assertEqual('', stream.getvalue())
def testHarvesterIgnoringInvalidDataErrors(self): observer = CallTrace('observer') upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid')) upload.id = 'mockid' observer.returnValues['createUpload'] = upload observer.returnValues['totalInvalidIds'] = 0 observer.exceptions['send'] = InvalidDataException(upload.id, "message") repository=CallTrace("repository", returnValues={'maxIgnore': 100}) harvester = Harvester(repository) harvester.addObserver(observer) harvester.upload(oaiResponse()) self.assertEquals(['createUpload', "notifyHarvestedRecord", "send", 'logInvalidData', "totalInvalidIds", 'logIgnoredIdentifierWarning'], [m.name for m in observer.calledMethods])
def testHarvesterStopsIgnoringAfter100records(self): observer = CallTrace('observer') upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid')) upload.id = 'mockid' observer.returnValues['createUpload'] = upload observer.returnValues['totalInvalidIds'] = 101 observer.exceptions['send'] = InvalidDataException(upload.id, "message") repository=CallTrace("repository", returnValues={'maxIgnore': 100}) harvester = Harvester(repository) harvester.addObserver(observer) self.assertRaises(TooMuchInvalidDataException, lambda: harvester.upload(oaiResponse(identifier='mockid'))) self.assertEquals(['createUpload', "notifyHarvestedRecord", "send", "logInvalidData", "totalInvalidIds"], [m.name for m in observer.calledMethods])
def testDeleteWithOaiEnvelope(self): RECORD_FILENAME = join(self.tempdir, 'id.record') self.uploader._filenameFor = lambda *args: RECORD_FILENAME self.uploader.tznow = lambda: "VANDAAG_EN_NU" self.target.oaiEnvelope = True repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.metadataPrefix = 'oai_dc' repository.baseurl = "http://repository" repository.id = 'repositoryId' upload = Upload(repository=repository, oaiResponse=oaiResponse(identifier='id.record', deleted=True)) self.assertFalse(isfile(RECORD_FILENAME)) self.uploader.delete(upload) self.assertTrue(isfile(RECORD_FILENAME)) with open(RECORD_FILENAME) as f: self.assertEqualsWS( """<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"> <responseDate>VANDAAG_EN_NU</responseDate> <request verb="GetRecord" metadataPrefix="oai_dc" identifier="id.record">http://repository</request> <GetRecord> <record> <header status="deleted"> <identifier>id.record</identifier> <datestamp>2005-08-29T07:08:09Z</datestamp> </header> </record> </GetRecord> </OAI-PMH>""", f.read())
def testDeleteWithOaiEnvelope(self): RECORD_FILENAME = join(self.tempdir, 'id.record') self.uploader._filenameFor = lambda *args: RECORD_FILENAME self.uploader.tznow = lambda: "VANDAAG_EN_NU" self.target.oaiEnvelope = True repository = CallTrace('Repository') repository.repositoryGroupId = 'groupId' repository.metadataPrefix = 'oai_dc' repository.baseurl = "http://repository" repository.id = 'repositoryId' upload = Upload(repository=repository, oaiResponse=oaiResponse(identifier='id.record', deleted=True)) self.assertFalse(isfile(RECORD_FILENAME)) self.uploader.delete(upload) self.assertTrue(isfile(RECORD_FILENAME)) self.assertEqualsWS("""<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"> <responseDate>VANDAAG_EN_NU</responseDate> <request verb="GetRecord" metadataPrefix="oai_dc" identifier="id.record">http://repository</request> <GetRecord> <record> <header status="deleted"> <identifier>id.record</identifier> <datestamp>2005-08-29T07:08:09Z</datestamp> </header> </record> </GetRecord> </OAI-PMH>""", open(RECORD_FILENAME).read())
def createUpload(about=None): repository = CallTrace('repository') repository.id = 'repoId' upload = Upload(repository=repository, oaiResponse=oaiResponse(about=about)) upload.id = 'id' return upload
def testHarvesterIgnoringInvalidDataErrors(self): observer = CallTrace('observer') upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid')) upload.id = 'mockid' observer.returnValues['createUpload'] = upload observer.returnValues['totalInvalidIds'] = 0 observer.exceptions['send'] = InvalidDataException( upload.id, "message") repository = CallTrace("repository", returnValues={'maxIgnore': 100}) harvester = Harvester(repository) harvester.addObserver(observer) harvester.upload(oaiResponse()) self.assertEqual([ 'createUpload', "notifyHarvestedRecord", "send", 'logInvalidData', "totalInvalidIds", 'logIgnoredIdentifierWarning' ], [m.name for m in observer.calledMethods])
def testNoLogging(self): datamap = Mapping('mappingId') datamap.code = """ upload.parts['record']="<somexml/>" logger.logError('Iets om te zeuren') """ upload = datamap.createUpload(TestRepository(), oaiResponse()) self.assertEquals('<somexml/>',upload.parts['record'])
def testNoLogging(self): datamap = Mapping('mappingId') datamap.code = """ upload.parts['record']="<somexml/>" logger.logError('Iets om te zeuren') """ upload = datamap.createUpload(TestRepository(), oaiResponse()) self.assertEqual('<somexml/>', upload.parts['record'])
def testSend(self): recordFile = self.tempdir + '/group/repo/id.record' self.uploader._filenameFor = lambda *args: recordFile upload = createUpload() self.uploader.send(upload) self.assertTrue(isfile(recordFile)) self.assertEqualsLxml(oaiResponse().record, parse(open(recordFile)))
def testSend(self): recordFile = self.tempdir + '/group/repo/id.record' self.uploader._filenameFor = lambda *args: recordFile upload = createUpload() self.uploader.send(upload) self.assertTrue(isfile(recordFile)) with open(recordFile) as fp: self.assertEqualsLxml(oaiResponse().record, parse(fp))
def testHarvesterStopsIgnoringAfter100records(self): observer = CallTrace('observer') upload = Upload(repository=None, oaiResponse=oaiResponse(identifier='mockid')) upload.id = 'mockid' observer.returnValues['createUpload'] = upload observer.returnValues['totalInvalidIds'] = 101 observer.exceptions['send'] = InvalidDataException( upload.id, "message") repository = CallTrace("repository", returnValues={'maxIgnore': 100}) harvester = Harvester(repository) harvester.addObserver(observer) self.assertRaises( TooMuchInvalidDataException, lambda: harvester.upload(oaiResponse(identifier='mockid'))) self.assertEqual([ 'createUpload', "notifyHarvestedRecord", "send", "logInvalidData", "totalInvalidIds" ], [m.name for m in observer.calledMethods])
def testSkippedRecord(self): harvester = self.createHarvesterWithMockUploader('tud') def createUpload(repository, oaiResponse): upload = Upload(repository=repository, oaiResponse=oaiResponse) upload.id = "tud:mockid" upload.skip = True return upload self.mapper.createUpload = createUpload harvester.upload(oaiResponse(identifier='mockid')) self.assertEquals([], self.sendId) self.assertFalse(hasattr(self, 'delete_id'))
def testLogging(self): datamap = Mapping('mappingId') datamap.code = """ upload.parts['record']="<somexml/>" logger.logError('Iets om te zeuren') """ stream = StringIO() logger = StreamEventLogger(stream) datamap.addObserver(logger) datamap.createUpload(TestRepository(), oaiResponse=oaiResponse()) self.assertEquals('ERROR\t[]\tIets om te zeuren\n',stream.getvalue()[26:])
def testSkip(self): datamap = Mapping('mappingId') datamap.code = """ skipRecord("Don't like it here.") """ stream = StringIO() logger = StreamEventLogger(stream) datamap.addObserver(logger) upload = datamap.createUpload(TestRepository(), oaiResponse()) self.assertTrue(upload.skip) self.assertEquals("SKIP\t[repository.id:oai:ident:321]\tDon't like it here.\n", stream.getvalue()[26:])
def testLogging(self): datamap = Mapping('mappingId') datamap.code = """ upload.parts['record']="<somexml/>" logger.logError('Iets om te zeuren') """ stream = StringIO() logger = StreamEventLogger(stream) datamap.addObserver(logger) datamap.createUpload(TestRepository(), oaiResponse=oaiResponse()) self.assertEqual('ERROR\t[]\tIets om te zeuren\n', stream.getvalue()[26:])
def testSkip(self): datamap = Mapping('mappingId') datamap.code = """ skipRecord("Don't like it here.") """ stream = StringIO() logger = StreamEventLogger(stream) datamap.addObserver(logger) upload = datamap.createUpload(TestRepository(), oaiResponse()) self.assertTrue(upload.skip) self.assertEqual( "SKIP\t[repository.id:oai:ident:321]\tDon't like it here.\n", stream.getvalue()[26:])
def testSkippedRecord(self): harvester = self.createHarvesterWithMockUploader('tud') def createUpload(repository, oaiResponse): upload = Upload(repository=repository, oaiResponse=oaiResponse) upload.id = "tud:mockid" upload.skip = True return upload self.mapper.createUpload = createUpload harvester.upload(oaiResponse(identifier='mockid')) self.assertEqual([], self.sendId) self.assertFalse(hasattr(self, 'delete_id'))
def testAssertion(self): datamap = Mapping('mappingId') datamap.code = """ doAssert(1==1) doAssert(1==2, "1 not equal 2") upload.parts['record']="<somexml/>" """ stream = StringIO() logger = StreamEventLogger(stream) datamap.addObserver(logger) try: datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=True) self.fail() except DataMapAssertionException, ex: self.assertEquals('ERROR\t[repository.id:oai:ident:321]\tAssertion: 1 not equal 2\n',stream.getvalue()[26:]) self.assertEquals('1 not equal 2', str(ex))
def assertPart(self, expected, partname, code): datamap = Mapping('mappingId') datamap.code = code upload = datamap.createUpload(TestRepository(), oaiResponse()) self.assertEquals(expected,upload.parts[partname])
doAssert(1==2, "1 not equal 2") upload.parts['record']="<somexml/>" """ stream = StringIO() logger = StreamEventLogger(stream) datamap.addObserver(logger) try: datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=True) self.fail() except DataMapAssertionException, ex: self.assertEquals('ERROR\t[repository.id:oai:ident:321]\tAssertion: 1 not equal 2\n',stream.getvalue()[26:]) self.assertEquals('1 not equal 2', str(ex)) try: datamap.createUpload(TestRepository(), oaiResponse(), doAsserts=True) self.fail() except DataMapAssertionException, ex: self.assertEquals('1 not equal 2', str(ex)) stream = StringIO() logger = StreamEventLogger(stream) datamap.createUpload(TestRepository(), oaiResponse() , doAsserts=False) self.assertEquals('',stream.getvalue()) def assertPart(self, expected, partname, code): datamap = Mapping('mappingId') datamap.code = code upload = datamap.createUpload(TestRepository(), oaiResponse()) self.assertEquals(expected,upload.parts[partname])
def testUploadRecord(self): harvester = self.createHarvesterWithMockUploader('tud') harvester.upload(oaiResponse(identifier='mockid')) self.assertEquals(['tud:mockid'], self.sendId) self.assertFalse(hasattr(self, 'delete_id'))
def testDelete(self): harvester = self.createHarvesterWithMockUploader('tud') harvester.upload(oaiResponse(identifier='mockid', deleted=True)) self.assertEquals([], self.sendId) self.assertEquals('tud:mockid', self.delete_id)
def testDelete(self): harvester = self.createHarvesterWithMockUploader('tud') harvester.upload(oaiResponse(identifier='mockid', deleted=True)) self.assertEqual([], self.sendId) self.assertEqual('tud:mockid', self.delete_id)
def testUploadRecord(self): harvester = self.createHarvesterWithMockUploader('tud') harvester.upload(oaiResponse(identifier='mockid')) self.assertEqual(['tud:mockid'], self.sendId) self.assertFalse(hasattr(self, 'delete_id'))
def assertPart(self, expected, partname, code): datamap = Mapping('mappingId') datamap.code = code upload = datamap.createUpload(TestRepository(), oaiResponse()) self.assertEqual(expected, upload.parts[partname])