def testMarkHarvesterAfterExceptionChange(self):
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:15:00Z')
state.markStarted()
try:
raise ValueError("whatever")
except:
exType, exValue, exTraceback = exc_info()
state.markException(exType, exValue, (100, 80, 20, 93))
self.assertEqual([100, 80, 20],
event_counts(state, 'records_harvested',
'records_uploaded',
'records_deleted'))
self.assertEqual(
{
"changedate": "2012-08-13 12:15:00",
"status": "Error",
"message": "whatever"
}, JsonDict.load(join(self.statePath, 'repo.running')))
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:17:00Z')
state.markStarted()
state.markHarvested((42, 31, 11, 135), "resumptionToken",
"2012-08-13T12:14:00")
self.assertEqual([142, 111, 31],
event_counts(state, 'records_harvested',
'records_uploaded',
'records_deleted'))
self.assertEqual(
{
"changedate": "2012-08-13 12:17:00",
"status": "Ok",
"message": ""
}, JsonDict.load(join(self.statePath, 'repo.running')))
def testMarkDeletedAfterExceptionChange(self):
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:15:00Z')
state.markStarted()
try:
raise ValueError("whatever")
except:
exType, exValue, exTraceback = exc_info()
state.markException(exType, exValue, (9999, 9999, 9999, 9999))
self.assertEqual(
{
"changedate": "2012-08-13 12:15:00",
"status": "Error",
"message": "whatever"
}, JsonDict.load(join(self.statePath, 'repo.running')))
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:17:00Z')
state.markStarted()
state.markDeleted()
self.assertEqual(
{
"changedate": "2012-08-13 12:17:00",
"status": "Ok",
"message": ""
}, JsonDict.load(join(self.statePath, 'repo.running')))
def getLastSuccessfulHarvestTime(self):
if self.lastSuccessfulHarvest:
return ZuluTime(self.lastSuccessfulHarvest)
if self.from_:
if 'T' not in self.from_:
return ZuluTime(self.from_ + "T00:00:00Z")
return ZuluTime(self.from_)
return None
def __call__(self, value):
try:
result = ZuluTime(value)
if len(value) <= len('YEAR'):
result = result.add(months=6)
if len(value) <= len('YEAR-MM-DDT'):
result = result.add(hours=12)
return result
except TimeError as e:
raise ValueError(e)
def __call__(self, value):
try:
result = ZuluTime(value)
if len(value) <= len('YEAR'):
result = result.add(months=6)
if len(value) <= len('YEAR-MM-DDT'):
result = result.add(hours=12)
return result
except TimeError,e:
raise ValueError(e)
def handleRequest(self, path, Headers, **kwargs):
if path in self._mapping:
item = self._mapping[path]
defaultHeaders = [
"Date: {}\r\n".format(ZuluTime().rfc1123()),
'etag: {}\r\n'.format(item['etag']),
'Last-Modified: {}\r\n'.format(item['lastModified'])
]
for key in Headers:
if key.lower() == 'etag' and Headers[key] == item['etag']:
yield "HTTP/1.0 304 Not Modified\r\n"
for each in defaultHeaders:
yield each
yield "\r\n"
return
filename = self._mapping[path]['filename']
_, ext = splitext(filename)
yield 'HTTP/1.0 200 OK\r\n'
yield 'Content-type: {}\r\n'.format(
self._magic.from_file(filename))
for each in defaultHeaders:
yield each
yield '\r\n'
with open(filename, "rb") as fp:
yield fp.read()
def testLastSuccessfulHarvestTime(self):
with open(join(self.statePath, 'repository.next'), 'w') as f:
f.write('{"from": "2020-12-21T01:42:24.403578+01:00"}')
with self._State('repository') as s:
self.assertEqual(
ZuluTime('2020-12-21T00:42:24Z').zulu(),
s.getLastSuccessfulHarvestTime().zulu())
def testMarkException(self):
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:15:00Z')
state.markStarted()
state.markHarvested((9999, 9999, 9999, 9999), "resumptionToken",
"2012-08-13T12:14:00")
self.assertEqual([1, 0, 1],
event_counts(state, 'started', 'errors',
'harvested'))
self.assertRepoStats(
'Started: 2012-08-13 12:15:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-13 12:15:00, ResumptionToken: resumptionToken\n'
)
self.assertEqual(
{
"from": "2012-08-13T12:14:00",
"resumptionToken": "resumptionToken",
'lastSuccessfulHarvest': '2012-08-13T12:15:00Z'
}, JsonDict.load(join(self.statePath, 'repo.next')))
self.assertEqual(
{
"changedate": "2012-08-13 12:15:00",
"status": "Ok",
"message": ""
}, JsonDict.load(join(self.statePath, 'repo.running')))
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:17:00Z')
state.markStarted()
try:
raise ValueError("whatever")
except:
exType, exValue, exTraceback = exc_info()
state.markException(exType, exValue, (9999, 9999, 9999, 9999))
self.assertEqual([2, 1, 1],
event_counts(state, 'started', 'errors',
'harvested'))
self.assertRepoStats(
"""Started: 2012-08-13 12:15:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-13 12:15:00, ResumptionToken: resumptionToken
Started: 2012-08-13 12:17:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Error: <class 'ValueError'>: whatever
""")
self.assertEqual(
{
"changedate": "2012-08-13 12:17:00",
"status": "Error",
"message": "whatever"
}, JsonDict.load(join(self.statePath, 'repo.running')))
def _updateFile(self, shortname, filename):
with open(filename, 'rb') as fp:
etag = md5(fp.read()).hexdigest()
self._mapping[shortname] = dict(
filename=filename,
etag=etag,
lastModified=ZuluTime(stat(filename).st_mtime).rfc1123(),
)
def testMarkDeleted(self):
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:15:00Z')
state.markStarted()
state.markHarvested((9999, 9999, 9999, 9999), "resumptionToken",
"2012-08-13T12:14:00")
self.assertEqual([1, 0], event_counts(state, 'harvested',
'deleted'))
self.assertRepoStats(
'Started: 2012-08-13 12:15:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-13 12:15:00, ResumptionToken: resumptionToken\n'
)
self.assertEqual(
{
"from": "2012-08-13T12:14:00",
"resumptionToken": "resumptionToken",
'lastSuccessfulHarvest': '2012-08-13T12:15:00Z'
}, JsonDict.load(join(self.statePath, 'repo.next')))
self.assertEqual(
{
"changedate": "2012-08-13 12:15:00",
"status": "Ok",
"message": ""
}, JsonDict.load(join(self.statePath, 'repo.running')))
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:17:00Z')
state.markDeleted()
self.assertRepoStats(
"""Started: 2012-08-13 12:15:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-13 12:15:00, ResumptionToken: resumptionToken
Started: 2012-08-13 12:17:00, Harvested/Uploaded/Deleted/Total: 0/0/0/0, Done: Deleted all ids.
""")
self.assertEqual(
{
"from": "",
"resumptionToken": "",
'lastSuccessfulHarvest': None
}, JsonDict.load(join(self.statePath, 'repo.next')))
self.assertEqual(
{
"changedate": "2012-08-13 12:15:00",
"status": "Ok",
"message": ""
}, JsonDict.load(join(self.statePath, 'repo.running')))
def dictToLogline(dataDict):
return ' '.join([
ZuluTime.parseEpoch(int(dataDict['timestamp'])).zulu(),
dataDict['ipAddress'],
'-' if dataDict['size'] is None else ('%.1fK' % (dataDict['size'] / 1024.0)),
'-' if dataDict['duration'] is None else ('%.3fs' % (dataDict['duration'] / 1000.0)),
'-' if dataDict['hits'] is None else '%shits' % dataDict['hits'],
dataDict['path'],
dataDict['arguments'],
])
def testGetLastSuccessfulHarvestTime(self):
with self._State('repo') as state:
self.assertEqual(None, state.from_)
self.assertEqual(None, state.lastSuccessfulHarvest)
self.assertEqual(None, state.getLastSuccessfulHarvestTime())
state.from_ = "2019-01-01"
self.assertEqual(ZuluTime("2019-01-01T00:00:00Z"),
state.getLastSuccessfulHarvestTime())
def loglineToDict(line):
parts = line.split(' ') + ['']
zulutime, ipAddress, size, duration, hits, path, arguments = parts[:7]
return {
'timestamp': long(ZuluTime(zulutime).epoch if 'Z' in zulutime else long(zulutime)),
'ipAddress': ipAddress,
'size': int(float(size[:-1]) * 1024) if size != '-' else None,
'duration': int(float(duration.split('s')[0])*1000) if duration != '-' else None,
'hits': int(hits.split('hits')[0]) if hits != '-' else None,
'path': path,
'arguments': arguments,
}
def testHasWorkBeforeAndAfterDoingWorkContinuous(self):
with harvesterLog(stateDir=self.stateDir, logDir=self.logDir, name= 'name') as logger:
self.assertTrue(logger.hasWork(continuousInterval=60))
logger.startRepository()
logger.endRepository(None, logger._state.getZTime().zulu())
with harvesterLog(stateDir=self.stateDir, logDir=self.logDir, name= 'name') as logger:
# mock current Time
logger._state.getZTime = lambda: ZuluTime().add(seconds=-61)
self.assertFalse(logger.hasWork(continuousInterval=60))
logger.startRepository()
logger.endRepository(None, "2000-01-02T03:04:05Z") # ignore what repository says as responseDate
with harvesterLog(stateDir=self.stateDir, logDir=self.logDir, name= 'name') as logger:
self.assertFalse(logger.hasWork(continuousInterval=65))
self.assertTrue(logger.hasWork(continuousInterval=60))
def testSetToLastCleanState(self):
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:15:00Z')
state.markStarted()
state.markHarvested((9999, 9999, 9999, 9999), "",
"2012-08-13T12:14:00")
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-14T12:17:00Z')
state.markStarted()
state.markHarvested((9999, 9999, 9999, 9999), "resumptionToken",
"2012-08-14T12:16:00")
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-15T12:19:00Z')
state.setToLastCleanState()
self.assertRepoStats(
"""Started: 2012-08-13 12:15:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-13 12:15:00, ResumptionToken: \nStarted: 2012-08-14 12:17:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-14 12:17:00, ResumptionToken: resumptionToken
Started: 2012-08-15 12:19:00, Done: Reset to last clean state. ResumptionToken: \n"""
)
self.assertEqual(
{
"from": "2012-08-14T12:16:00",
"resumptionToken": "",
'lastSuccessfulHarvest': '2012-08-15T12:19:00Z'
}, JsonDict.load(join(self.statePath, 'repo.next')))
def writeLogLine(self, year, month, day, token=None, exception=None):
with self.newHarvesterLog() as h:
h._state.getZTime = lambda: ZuluTime('{}-{}-{}T12:15:00Z'.format(
year, month, day))
h.startRepository()
h.notifyHarvestedRecord("repo:uploadId1")
h.uploadIdentifier("repo:uploadId1")
if exception != None:
try:
raise Exception(exception)
except:
exType, exValue, exTb = exc_info()
h.endWithException(exType, exValue, exTb)
else:
h.endRepository(
token,
h._state.getZTime().display("%Y-%m-%dT%H:%M:%SZ"))
def testHalfwayStrategyEpoch(self):
normalize = DateFill.toHalfway().normalizeToEpoch
self.assertEqual(1540998692, normalize('2018-10-31T15:11:32Z'))
self.assertEqual(1540998692, normalize('2018-10-31T16:11:32+01:00'))
self.assertEqual(1529943092, normalize('2018-06-25T16:11:32Z'))
self.assertEqual(1529943092, normalize('2018-06-25T18:11:32+02:00'))
self.assertEqual(1529943092, normalize('2018-06-25T16:11:32.6Z'))
self.assertEqual(ZuluTime('2018-06-25T16:11:32Z').epoch, normalize('2018-06-25T16:11:32'))
self.assertEqual(ZuluTime('2018-06-25T16:11:00Z').epoch, normalize('2018-06-25T16:11'))
self.assertEqual(ZuluTime('2018-06-25T16:00:00Z').epoch, normalize('2018-06-25T16'))
self.assertEqual(ZuluTime('2018-06-25T12:00:00Z').epoch, normalize('2018-06-25'))
self.assertEqual(ZuluTime('2018-06-01T12:00:00Z').epoch, normalize('2018-06'))
self.assertEqual(ZuluTime('2018-07-01T12:00:00Z').epoch, normalize('2018'))
self.assertRaises(ValueError, lambda: normalize('ooit'))
def __call__(self, value):
try:
return ZuluTime(value)
except TimeError as e:
raise ValueError(e)
def writeMarkDeleted(self, year, month, day):
with self.newHarvesterLog() as h:
h._state.getZTime = lambda: ZuluTime('{}-{}-{}T12:15:00Z'.format(
year, month, day))
h.markDeleted()
def setUp(self):
SeecrTestCase.setUp(self)
self.handler = SessionHandler()
self.cookiestore = CookieMemoryStore(name='session', timeout=10)
self.handler.addObserver(self.cookiestore)
self.handler._zulutime = lambda: ZuluTime('2015-01-27T13:34:45Z')
def errorMessage(self):
return 'Data considered incomplete after crash [last safe point: %s].' % ZuluTime.parseEpoch(self.lastSafePoint()).zulu() if self.didCrash else None
def _zulu():
return ZuluTime().zulu()
def testMarkHarvested(self):
with self._State('repo') as state:
state.getZTime = lambda: ZuluTime('2012-08-13T12:15:00Z')
self.assertEqual([0, 0], event_counts(state, 'started',
'harvested'))
state.markStarted()
self.assertEqual([1, 0], event_counts(state, 'started',
'harvested'))
state.markHarvested((9999, 9999, 9999, 9999), "resumptionToken",
"2012-08-13T12:14:00")
self.assertEqual([1, 1], event_counts(state, 'started',
'harvested'))
self.assertRepoStats(
'Started: 2012-08-13 12:15:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-13 12:15:00, ResumptionToken: resumptionToken\n'
)
self.assertEqual(
{
"from": "2012-08-13T12:14:00",
"resumptionToken": "resumptionToken",
'lastSuccessfulHarvest': '2012-08-13T12:15:00Z'
}, JsonDict.load(self.statePath / 'repo.next'))
self.assertEqual(
{
"changedate": "2012-08-13 12:15:00",
"status": "Ok",
"message": ""
}, JsonDict.load(self.statePath / 'repo.running'))
with self._State('repo') as state:
self.assertEqual('2012-08-13T12:14:00', state.from_)
self.assertEqual('resumptionToken', state.token)
state.getZTime = lambda: ZuluTime('2012-08-13T12:17:00Z')
self.assertEqual([1, 1], event_counts(state, 'started',
'harvested'))
state.markStarted()
self.assertEqual([2, 1], event_counts(state, 'started',
'harvested'))
state.markHarvested((9999, 9999, 9999, 9999), "newToken",
"2012-08-13T12:16:00Z")
self.assertEqual([2, 2], event_counts(state, 'started',
'harvested'))
self.assertRepoStats(
"""Started: 2012-08-13 12:15:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-13 12:15:00, ResumptionToken: resumptionToken
Started: 2012-08-13 12:17:00, Harvested/Uploaded/Deleted/Total: 9999/9999/9999/9999, Done: 2012-08-13 12:17:00, ResumptionToken: newToken
""")
self.assertEqual(
{
"from": "2012-08-13T12:14:00",
"resumptionToken": "newToken",
'lastSuccessfulHarvest': '2012-08-13T12:17:00Z'
}, JsonDict.load(join(self.statePath, 'repo.next')))
self.assertEqual(
{
"changedate": "2012-08-13 12:15:00",
"status": "Ok",
"message": ""
}, JsonDict.load(join(self.statePath, 'repo.running')))
with self._State('repo') as state:
self.assertEqual('2012-08-13T12:14:00', state.from_)
self.assertEqual('newToken', state.token)
state.getZTime = lambda: ZuluTime('2012-08-13T12:20:00Z')
state.verbose = True
state.markStarted()
state.markHarvested((9999, 9999, 9999, 9999),
token=None,
responseDate="2012-08-13T12:19:00Z")
self.assertEqual([3, 3], event_counts(state, 'started',
'harvested'))
self.assertEqual(
{
"from": "2012-08-13T12:14:00",
"resumptionToken": "",
'lastSuccessfulHarvest': '2012-08-13T12:20:00Z'
}, JsonDict.load(join(self.statePath, 'repo.next')))
self.assertEqual(
{
"changedate": "2012-08-13 12:15:00",
"status": "Ok",
"message": ""
}, JsonDict.load(join(self.statePath, 'repo.running')))
def getZTime():
return ZuluTime()
