def test_date_range(self):
""" Test date ranges. """
start_date = datetime.date(1970, 1, 1)
end_date = datetime.date(1970, 10, 1)
dates = date_range(start_date, end_date, 2, 'months')
self.assertEquals(5, len(dates))
start_date = datetime.date(1970, 1, 1)
end_date = datetime.date(1970, 1, 3)
dates = date_range(start_date, end_date, 1, 'days')
self.assertEquals(3, len(dates))
self.assertEquals(dates, [datetime.date(1970, 1, 1),
datetime.date(1970, 1, 2),
datetime.date(1970, 1, 3)])
def test_date_range(self):
""" Test date ranges. """
start_date = datetime.date(1970, 1, 1)
end_date = datetime.date(1970, 10, 1)
dates = date_range(start_date, end_date, 2, 'months')
self.assertEquals(5, len(dates))
start_date = datetime.date(1970, 1, 1)
end_date = datetime.date(1970, 1, 3)
dates = date_range(start_date, end_date, 1, 'days')
self.assertEquals(3, len(dates))
self.assertEquals(dates, [
datetime.date(1970, 1, 1),
datetime.date(1970, 1, 2),
datetime.date(1970, 1, 3)
])
def requires(self):
if self.update not in ('days', 'weeks', 'months'):
raise RuntimeError('update can only be: days, weeks or months')
dates = date_range(self.begin, self.end, 1, self.update)
tasks = [CrossrefHarvestChunk(begin=dates[i], end=dates[i + 1])
for i in range(len(dates) - 1)]
return sorted(tasks)
def requires(self):
""" Only require up to the last full month. """
date = self.closest()
if date < self.begin:
raise RuntimeError("invalid range: %s - %s" % (self.begin, self.date))
dates = date_range(self.begin, date, 1, "months")
for i, _ in enumerate(dates[:-1]):
yield QucosaMarcChunk(begin=dates[i], end=dates[i + 1])
def requires(self):
""" Harvest in 1-month chunks. """
if self.end < self.begin:
raise RuntimeError('Invalid range: %s - %s' % (self.begin, self.end))
dates = date_range(self.begin, self.end, 1, 'months')
for begin, end in zip(dates[:-1], dates[1:]):
yield OAIHarvestChunk(begin=begin, end=end, prefix=self.prefix,
url=self.url, collection=self.collection)
def requires(self):
if self.update not in ('days', 'weeks', 'months'):
raise RuntimeError('update can only be: days, weeks or months')
dates = [dt for dt in date_range(self.begin, self.end, 1, self.update)]
tasks = [
CrossrefHarvestChunkWithCursor(begin=dates[i], end=dates[i + 1])
for i in range(len(dates) - 1)
]
return sorted(tasks)
def requires(self):
""" Only require up to the last full month. """
begin = datetime.date(self.begin.year, self.begin.month, 1)
end = datetime.date(self.end.year, self.end.month, 1)
if end < self.begin:
raise RuntimeError('Invalid range: %s - %s' % (begin, end))
dates = date_range(begin, end, 7, 'days')
for i, _ in enumerate(dates[:-1]):
yield HathiHarvestChunk(begin=dates[i], end=dates[i + 1], url=self.url,
prefix=self.prefix, collection=self.collection, delay=self.delay)
def requires(self):
""" Only require up to the last full month. """
begin = datetime.date(self.begin.year, self.begin.month, 1)
end = datetime.date(self.end.year, self.end.month, 1)
if end < self.begin:
raise RuntimeError('Invalid range: %s - %s' % (begin, end))
dates = date_range(begin, end, 1, 'months')
for i, _ in enumerate(dates[:-1]):
yield BNFMarcChunk(begin=dates[i], end=dates[i + 1],
prefix=self.prefix, collection=self.collection)
def requires(self):
if self.update not in ('days', 'weeks', 'months'):
raise RuntimeError('update can only be: days, weeks or months')
dates = [
dt for dt in date_range(self.begin, self.date, 1, self.update)
]
tasks = [
CrossrefChunkItems(begin=dates[i - 1], end=dates[i])
for i in range(1, len(dates))
]
return tasks
def requires(self):
if self.update not in ('days', 'weeks', 'months'):
raise RuntimeError('update can only be: days, weeks or months')
dates = [dt for dt in date_range(self.begin, self.date, 1, self.update)]
tasks = [CrossrefChunkItems(begin=dates[i - 1], end=dates[i]) for i in range(1, len(dates))]
return tasks