def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum) -> Iterator[Tuple[str, Union[str, dict, bytes]]]: url = furl(self.config.base_url) url.args['dateStart'] = start_date.date().strftime('%m/%d/%Y') url.args['dateEnd'] = end_date.date().strftime('%m/%d/%Y') url.args['offset'] = 0 return self.fetch_records(url)
def do_harvest( self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum ) -> Iterator[Tuple[str, Union[str, dict, bytes]]]: url = furl(self.url) url.args['dateStart'] = start_date.date().strftime('%m/%d/%Y') url.args['dateEnd'] = end_date.date().strftime('%m/%d/%Y') url.args['offset'] = 0 return self.fetch_records(url)
def _do_fetch(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum) -> Iterator[Tuple[str, Union[str, dict, bytes]]]: url = furl(self.config.base_url) url.args['dateStart'] = start_date.date().strftime('%m/%d/%Y') url.args['dateEnd'] = end_date.date().strftime('%m/%d/%Y') url.args['offset'] = 0 url.args['printFields'] = ','.join(NSF_FIELDS) url.args['rpp'] = PAGE_SIZE return self.fetch_records(url)
def do_harvest( self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum ) -> Iterator[Tuple[str, Union[str, dict, bytes]]]: url = furl(self.config.base_url) url.args['dateStart'] = start_date.date().strftime('%m/%d/%Y') url.args['dateEnd'] = end_date.date().strftime('%m/%d/%Y') url.args['offset'] = 0 url.args['printFields'] = ','.join(NSF_FIELDS) return self.fetch_records(url)
def analyse_prices(execution_date: Pendulum) -> pd.DataFrame: listings = pd.DataFrame(db.select_all('Listings')) listings.drop('id', axis=1, inplace=True) listings.set_index('listing_id', inplace=True) listings = listings.astype({ 'num_bedrooms': 'float64', 'listed_on': 'datetime64', 'last_reduced_on': 'datetime64' }) listings['date'] = listings.last_reduced_on.combine_first( listings.listed_on) listings['year'] = listings.date.apply(lambda date: date.year) listings['week'] = listings.date.apply(lambda date: date.week) listings['week_commencing'] = listings.date.apply( lambda date: date - timedelta(days=date.weekday())) grouped = listings.groupby('week_commencing') prices_by_week = grouped.mean().join(grouped.count(), rsuffix='_count') prices_by_week = prices_by_week.loc[prices_by_week.index.year >= 2020] prices_by_week = prices_by_week.loc[ prices_by_week.index.date <= execution_date.date()] prices_by_week = prices_by_week.loc[:][[ 'listing_price', 'listing_price_count' ]] prices_by_week.rename({'listing_price': 'average_price'}, axis=1, inplace=True) prices_by_week.rename({'listing_price_count': 'num_listings'}, axis=1, inplace=True) return prices_by_week
def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum, set_spec=None) -> list: url = furl(self.url) if set_spec: url.args['set'] = set_spec url.args['verb'] = 'ListRecords' url.args['metadataPrefix'] = self.metadata_prefix if self.time_granularity: url.args[self.from_param] = start_date.format('YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z' url.args[self.until_param] = end_date.format('YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z' else: url.args[self.from_param] = start_date.date().isoformat() url.args[self.until_param] = end_date.date().isoformat() return self.fetch_records(url)
def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum, set_spec=None) -> list: url = furl(self.url) if set_spec: url.args['set'] = set_spec url.args['verb'] = 'ListRecords' url.args['metadataPrefix'] = self.metadata_prefix if self.time_granularity: url.args[self.from_param] = start_date.format( 'YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z' url.args[self.until_param] = end_date.format( 'YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z' else: url.args[self.from_param] = start_date.date().isoformat() url.args[self.until_param] = end_date.date().isoformat() return self.fetch_records(url)
def test_date(self): dt = Pendulum(2016, 10, 20, 10, 40, 34, 123456) d = dt.date() self.assertIsInstanceOfDate(d) self.assertDate(d, 2016, 10, 20)
class BehaviorTest(AbstractTestCase): def setUp(self): super(BehaviorTest, self).setUp() self.p = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris') self.p1 = self.p.in_tz('America/New_York') self.tz = timezone('Europe/Paris') self.d = self.tz.convert(datetime(2016, 8, 27, 12, 34, 56, 123456)) def test_timetuple(self): self.assertEqual(self.d.timetuple(), self.p.timetuple()) def test_utctimetuple(self): self.assertEqual(self.d.utctimetuple(), self.p.utctimetuple()) def test_date(self): self.assertEqual(self.d.date(), self.p.date()) def test_time(self): self.assertEqual(self.p.time(), self.d.time()) def test_timetz(self): self.assertEqual(self.d.timetz(), self.p.timetz()) def test_astimezone(self): self.assertEqual(self.d.astimezone(self.p1.tzinfo), self.p.astimezone(self.p1.tzinfo)) def test_ctime(self): self.assertEqual(self.d.ctime(), self.p.ctime()) def test_isoformat(self): self.assertEqual(self.d.isoformat(), self.p.isoformat()) def test_utcoffset(self): self.assertEqual(self.d.utcoffset(), self.p.utcoffset()) def test_tzname(self): self.assertEqual(self.d.tzname(), self.p.tzname()) def test_dst(self): self.assertEqual(self.d.dst(), self.p.dst()) def test_toordinal(self): self.assertEqual(self.d.toordinal(), self.p.toordinal()) def test_weekday(self): self.assertEqual(self.d.weekday(), self.p.weekday()) def test_isoweekday(self): self.assertEqual(self.d.isoweekday(), self.p.isoweekday()) def test_isocalendar(self): self.assertEqual(self.d.isocalendar(), self.p.isocalendar()) def test_fromtimestamp(self): self.assertEqual(datetime.fromtimestamp(0, pendulum.UTC), pendulum.fromtimestamp(0, pendulum.UTC)) def test_utcfromtimestamp(self): self.assertEqual(datetime.utcfromtimestamp(0), pendulum.utcfromtimestamp(0)) def test_fromordinal(self): self.assertEqual(datetime.fromordinal(730120), pendulum.fromordinal(730120)) def test_combine(self): self.assertEqual( datetime.combine(date(2016, 1, 1), time(1, 2, 3, 123456)), pendulum.combine(date(2016, 1, 1), time(1, 2, 3, 123456))) def test_hash(self): dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris') dt2 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris') dt3 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'America/Toronto') self.assertEqual(hash(dt1), hash(dt2)) self.assertNotEqual(hash(dt1), hash(dt3)) def test_pickle(self): dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris') s = pickle.dumps(dt1) dt2 = pickle.loads(s) self.assertEqual(dt1, dt2) def test_pickle_with_integer_tzinfo(self): dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 0) s = pickle.dumps(dt1) dt2 = pickle.loads(s) self.assertEqual(dt1, dt2) def test_proper_dst(self): dt = pendulum.create(1941, 7, 1, tz='Europe/Amsterdam') self.assertEqual(timedelta(0, 6000), dt.dst()) def test_deepcopy(self): dt = pendulum.create(1941, 7, 1, tz='Europe/Amsterdam') self.assertEqual(dt, deepcopy(dt)) def test_deepcopy_datetime(self): dt = pendulum.create(1941, 7, 1, tz='Europe/Amsterdam') self.assertEqual(dt._datetime, deepcopy(dt._datetime)) def test_pickle_timezone(self): dt1 = pendulum.timezone('Europe/Amsterdam') s = pickle.dumps(dt1) dt2 = pickle.loads(s) self.assertTrue(isinstance(dt2, Timezone)) dt1 = pendulum.timezone('UTC') s = pickle.dumps(dt1) dt2 = pickle.loads(s) self.assertTrue(isinstance(dt2, Timezone))
class BehaviorTest(AbstractTestCase): def setUp(self): super(BehaviorTest, self).setUp() self.p = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris') self.p1 = self.p.in_tz('America/New_York') self.tz = timezone('Europe/Paris') self.d = self.tz.convert(datetime(2016, 8, 27, 12, 34, 56, 123456)) def test_timetuple(self): self.assertEqual(self.d.timetuple(), self.p.timetuple()) def test_utctimetuple(self): self.assertEqual(self.d.utctimetuple(), self.p.utctimetuple()) def test_date(self): self.assertEqual(self.d.date(), self.p.date()) def test_time(self): self.assertEqual(self.d.time(), self.p.time()) def test_timetz(self): self.assertEqual(self.d.timetz(), self.p.timetz()) def test_astimezone(self): self.assertEqual(self.d.astimezone(self.p1.tzinfo), self.p.astimezone(self.p1.tzinfo)) def test_ctime(self): self.assertEqual(self.d.ctime(), self.p.ctime()) def test_isoformat(self): self.assertEqual(self.d.isoformat(), self.p.isoformat()) def test_utcoffset(self): self.assertEqual(self.d.utcoffset(), self.p.utcoffset()) def test_tzname(self): self.assertEqual(self.d.tzname(), self.p.tzname()) def test_dst(self): self.assertEqual(self.d.dst(), self.p.dst()) def test_toordinal(self): self.assertEqual(self.d.toordinal(), self.p.toordinal()) def test_weekday(self): self.assertEqual(self.d.weekday(), self.p.weekday()) def test_isoweekday(self): self.assertEqual(self.d.isoweekday(), self.p.isoweekday()) def test_isocalendar(self): self.assertEqual(self.d.isocalendar(), self.p.isocalendar()) def test_fromtimestamp(self): self.assertEqual(datetime.fromtimestamp(0, pendulum.UTC), pendulum.fromtimestamp(0, pendulum.UTC)) def test_utcfromtimestamp(self): self.assertEqual(datetime.utcfromtimestamp(0), pendulum.utcfromtimestamp(0)) def test_fromordinal(self): self.assertEqual(datetime.fromordinal(730120), pendulum.fromordinal(730120)) def test_combine(self): self.assertEqual( datetime.combine(date(2016, 1, 1), time(1, 2, 3, 123456)), pendulum.combine(date(2016, 1, 1), time(1, 2, 3, 123456))) def test_hash(self): dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris') dt2 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris') dt3 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'America/Toronto') self.assertEqual(hash(dt1), hash(dt2)) self.assertNotEqual(hash(dt1), hash(dt3)) def test_pickle(self): dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris') s = pickle.dumps(dt1) dt2 = pickle.loads(s) self.assertEqual(dt1, dt2)