Ejemplo n.º 1
0
    def test_date_range(self):
        """ Test date ranges. """
        start_date = datetime.date(1970, 1, 1)
        end_date = datetime.date(1970, 10, 1)
        dates = date_range(start_date, end_date, 2, 'months')
        self.assertEquals(5, len(dates))

        start_date = datetime.date(1970, 1, 1)
        end_date = datetime.date(1970, 1, 3)
        dates = date_range(start_date, end_date, 1, 'days')
        self.assertEquals(3, len(dates))
        self.assertEquals(dates, [datetime.date(1970, 1, 1),
                                  datetime.date(1970, 1, 2),
                                  datetime.date(1970, 1, 3)])
Ejemplo n.º 2
0
    def test_date_range(self):
        """ Test date ranges. """
        start_date = datetime.date(1970, 1, 1)
        end_date = datetime.date(1970, 10, 1)
        dates = date_range(start_date, end_date, 2, 'months')
        self.assertEquals(5, len(dates))

        start_date = datetime.date(1970, 1, 1)
        end_date = datetime.date(1970, 1, 3)
        dates = date_range(start_date, end_date, 1, 'days')
        self.assertEquals(3, len(dates))
        self.assertEquals(dates, [
            datetime.date(1970, 1, 1),
            datetime.date(1970, 1, 2),
            datetime.date(1970, 1, 3)
        ])
Ejemplo n.º 3
0
 def requires(self):
     if self.update not in ('days', 'weeks', 'months'):
         raise RuntimeError('update can only be: days, weeks or months')
     dates = date_range(self.begin, self.end, 1, self.update)
     tasks = [CrossrefHarvestChunk(begin=dates[i], end=dates[i + 1])
              for i in range(len(dates) - 1)]
     return sorted(tasks)
Ejemplo n.º 4
0
 def requires(self):
     """ Only require up to the last full month. """
     date = self.closest()
     if date < self.begin:
         raise RuntimeError("invalid range: %s - %s" % (self.begin, self.date))
     dates = date_range(self.begin, date, 1, "months")
     for i, _ in enumerate(dates[:-1]):
         yield QucosaMarcChunk(begin=dates[i], end=dates[i + 1])
Ejemplo n.º 5
0
 def requires(self):
     """ Harvest in 1-month chunks. """
     if self.end < self.begin:
         raise RuntimeError('Invalid range: %s - %s' % (self.begin, self.end))
     dates = date_range(self.begin, self.end, 1, 'months')
     for begin, end in zip(dates[:-1], dates[1:]):
         yield OAIHarvestChunk(begin=begin, end=end, prefix=self.prefix,
                               url=self.url, collection=self.collection)
Ejemplo n.º 6
0
 def requires(self):
     if self.update not in ('days', 'weeks', 'months'):
         raise RuntimeError('update can only be: days, weeks or months')
     dates = [dt for dt in date_range(self.begin, self.end, 1, self.update)]
     tasks = [
         CrossrefHarvestChunkWithCursor(begin=dates[i], end=dates[i + 1])
         for i in range(len(dates) - 1)
     ]
     return sorted(tasks)
Ejemplo n.º 7
0
 def requires(self):
     """ Only require up to the last full month. """
     begin = datetime.date(self.begin.year, self.begin.month, 1)
     end = datetime.date(self.end.year, self.end.month, 1)
     if end < self.begin:
         raise RuntimeError('Invalid range: %s - %s' % (begin, end))
     dates = date_range(begin, end, 7, 'days')
     for i, _ in enumerate(dates[:-1]):
         yield HathiHarvestChunk(begin=dates[i], end=dates[i + 1], url=self.url,
                                 prefix=self.prefix, collection=self.collection, delay=self.delay)
Ejemplo n.º 8
0
 def requires(self):
     """ Only require up to the last full month. """
     begin = datetime.date(self.begin.year, self.begin.month, 1)
     end = datetime.date(self.end.year, self.end.month, 1)
     if end < self.begin:
         raise RuntimeError('Invalid range: %s - %s' % (begin, end))
     dates = date_range(begin, end, 1, 'months')
     for i, _ in enumerate(dates[:-1]):
         yield BNFMarcChunk(begin=dates[i], end=dates[i + 1],
                            prefix=self.prefix, collection=self.collection)
Ejemplo n.º 9
0
 def requires(self):
     if self.update not in ('days', 'weeks', 'months'):
         raise RuntimeError('update can only be: days, weeks or months')
     dates = [
         dt for dt in date_range(self.begin, self.date, 1, self.update)
     ]
     tasks = [
         CrossrefChunkItems(begin=dates[i - 1], end=dates[i])
         for i in range(1, len(dates))
     ]
     return tasks
Ejemplo n.º 10
0
 def requires(self):
     if self.update not in ('days', 'weeks', 'months'):
         raise RuntimeError('update can only be: days, weeks or months')
     dates = [dt for dt in date_range(self.begin, self.date, 1, self.update)]
     tasks = [CrossrefChunkItems(begin=dates[i - 1], end=dates[i]) for i in range(1, len(dates))]
     return tasks