def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum) -> Iterator[Tuple[str, Union[str, dict, bytes]]]:
url = furl(self.config.base_url)
url.args['dateStart'] = start_date.date().strftime('%m/%d/%Y')
url.args['dateEnd'] = end_date.date().strftime('%m/%d/%Y')
url.args['offset'] = 0
return self.fetch_records(url)
def do_harvest(
self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum
) -> Iterator[Tuple[str, Union[str, dict, bytes]]]:
url = furl(self.url)
url.args['dateStart'] = start_date.date().strftime('%m/%d/%Y')
url.args['dateEnd'] = end_date.date().strftime('%m/%d/%Y')
url.args['offset'] = 0
return self.fetch_records(url)
def _do_fetch(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum) -> Iterator[Tuple[str, Union[str, dict, bytes]]]:
url = furl(self.config.base_url)
url.args['dateStart'] = start_date.date().strftime('%m/%d/%Y')
url.args['dateEnd'] = end_date.date().strftime('%m/%d/%Y')
url.args['offset'] = 0
url.args['printFields'] = ','.join(NSF_FIELDS)
url.args['rpp'] = PAGE_SIZE
return self.fetch_records(url)
def do_harvest(
self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum
) -> Iterator[Tuple[str, Union[str, dict, bytes]]]:
url = furl(self.config.base_url)
url.args['dateStart'] = start_date.date().strftime('%m/%d/%Y')
url.args['dateEnd'] = end_date.date().strftime('%m/%d/%Y')
url.args['offset'] = 0
url.args['printFields'] = ','.join(NSF_FIELDS)
return self.fetch_records(url)
def analyse_prices(execution_date: Pendulum) -> pd.DataFrame:
listings = pd.DataFrame(db.select_all('Listings'))
listings.drop('id', axis=1, inplace=True)
listings.set_index('listing_id', inplace=True)
listings = listings.astype({
'num_bedrooms': 'float64',
'listed_on': 'datetime64',
'last_reduced_on': 'datetime64'
})
listings['date'] = listings.last_reduced_on.combine_first(
listings.listed_on)
listings['year'] = listings.date.apply(lambda date: date.year)
listings['week'] = listings.date.apply(lambda date: date.week)
listings['week_commencing'] = listings.date.apply(
lambda date: date - timedelta(days=date.weekday()))
grouped = listings.groupby('week_commencing')
prices_by_week = grouped.mean().join(grouped.count(), rsuffix='_count')
prices_by_week = prices_by_week.loc[prices_by_week.index.year >= 2020]
prices_by_week = prices_by_week.loc[
prices_by_week.index.date <= execution_date.date()]
prices_by_week = prices_by_week.loc[:][[
'listing_price', 'listing_price_count'
]]
prices_by_week.rename({'listing_price': 'average_price'},
axis=1,
inplace=True)
prices_by_week.rename({'listing_price_count': 'num_listings'},
axis=1,
inplace=True)
return prices_by_week
def do_harvest(self, start_date: pendulum.Pendulum, end_date: pendulum.Pendulum, set_spec=None) -> list:
url = furl(self.url)
if set_spec:
url.args['set'] = set_spec
url.args['verb'] = 'ListRecords'
url.args['metadataPrefix'] = self.metadata_prefix
if self.time_granularity:
url.args[self.from_param] = start_date.format('YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z'
url.args[self.until_param] = end_date.format('YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z'
else:
url.args[self.from_param] = start_date.date().isoformat()
url.args[self.until_param] = end_date.date().isoformat()
return self.fetch_records(url)
def do_harvest(self,
start_date: pendulum.Pendulum,
end_date: pendulum.Pendulum,
set_spec=None) -> list:
url = furl(self.url)
if set_spec:
url.args['set'] = set_spec
url.args['verb'] = 'ListRecords'
url.args['metadataPrefix'] = self.metadata_prefix
if self.time_granularity:
url.args[self.from_param] = start_date.format(
'YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z'
url.args[self.until_param] = end_date.format(
'YYYY-MM-DDT00:00:00', formatter='alternative') + 'Z'
else:
url.args[self.from_param] = start_date.date().isoformat()
url.args[self.until_param] = end_date.date().isoformat()
return self.fetch_records(url)
def test_date(self):
dt = Pendulum(2016, 10, 20, 10, 40, 34, 123456)
d = dt.date()
self.assertIsInstanceOfDate(d)
self.assertDate(d, 2016, 10, 20)
class BehaviorTest(AbstractTestCase):
def setUp(self):
super(BehaviorTest, self).setUp()
self.p = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris')
self.p1 = self.p.in_tz('America/New_York')
self.tz = timezone('Europe/Paris')
self.d = self.tz.convert(datetime(2016, 8, 27, 12, 34, 56, 123456))
def test_timetuple(self):
self.assertEqual(self.d.timetuple(), self.p.timetuple())
def test_utctimetuple(self):
self.assertEqual(self.d.utctimetuple(), self.p.utctimetuple())
def test_date(self):
self.assertEqual(self.d.date(), self.p.date())
def test_time(self):
self.assertEqual(self.p.time(), self.d.time())
def test_timetz(self):
self.assertEqual(self.d.timetz(), self.p.timetz())
def test_astimezone(self):
self.assertEqual(self.d.astimezone(self.p1.tzinfo),
self.p.astimezone(self.p1.tzinfo))
def test_ctime(self):
self.assertEqual(self.d.ctime(), self.p.ctime())
def test_isoformat(self):
self.assertEqual(self.d.isoformat(), self.p.isoformat())
def test_utcoffset(self):
self.assertEqual(self.d.utcoffset(), self.p.utcoffset())
def test_tzname(self):
self.assertEqual(self.d.tzname(), self.p.tzname())
def test_dst(self):
self.assertEqual(self.d.dst(), self.p.dst())
def test_toordinal(self):
self.assertEqual(self.d.toordinal(), self.p.toordinal())
def test_weekday(self):
self.assertEqual(self.d.weekday(), self.p.weekday())
def test_isoweekday(self):
self.assertEqual(self.d.isoweekday(), self.p.isoweekday())
def test_isocalendar(self):
self.assertEqual(self.d.isocalendar(), self.p.isocalendar())
def test_fromtimestamp(self):
self.assertEqual(datetime.fromtimestamp(0, pendulum.UTC),
pendulum.fromtimestamp(0, pendulum.UTC))
def test_utcfromtimestamp(self):
self.assertEqual(datetime.utcfromtimestamp(0),
pendulum.utcfromtimestamp(0))
def test_fromordinal(self):
self.assertEqual(datetime.fromordinal(730120),
pendulum.fromordinal(730120))
def test_combine(self):
self.assertEqual(
datetime.combine(date(2016, 1, 1), time(1, 2, 3, 123456)),
pendulum.combine(date(2016, 1, 1), time(1, 2, 3, 123456)))
def test_hash(self):
dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris')
dt2 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris')
dt3 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'America/Toronto')
self.assertEqual(hash(dt1), hash(dt2))
self.assertNotEqual(hash(dt1), hash(dt3))
def test_pickle(self):
dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris')
s = pickle.dumps(dt1)
dt2 = pickle.loads(s)
self.assertEqual(dt1, dt2)
def test_pickle_with_integer_tzinfo(self):
dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 0)
s = pickle.dumps(dt1)
dt2 = pickle.loads(s)
self.assertEqual(dt1, dt2)
def test_proper_dst(self):
dt = pendulum.create(1941, 7, 1, tz='Europe/Amsterdam')
self.assertEqual(timedelta(0, 6000), dt.dst())
def test_deepcopy(self):
dt = pendulum.create(1941, 7, 1, tz='Europe/Amsterdam')
self.assertEqual(dt, deepcopy(dt))
def test_deepcopy_datetime(self):
dt = pendulum.create(1941, 7, 1, tz='Europe/Amsterdam')
self.assertEqual(dt._datetime, deepcopy(dt._datetime))
def test_pickle_timezone(self):
dt1 = pendulum.timezone('Europe/Amsterdam')
s = pickle.dumps(dt1)
dt2 = pickle.loads(s)
self.assertTrue(isinstance(dt2, Timezone))
dt1 = pendulum.timezone('UTC')
s = pickle.dumps(dt1)
dt2 = pickle.loads(s)
self.assertTrue(isinstance(dt2, Timezone))
class BehaviorTest(AbstractTestCase):
def setUp(self):
super(BehaviorTest, self).setUp()
self.p = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris')
self.p1 = self.p.in_tz('America/New_York')
self.tz = timezone('Europe/Paris')
self.d = self.tz.convert(datetime(2016, 8, 27, 12, 34, 56, 123456))
def test_timetuple(self):
self.assertEqual(self.d.timetuple(), self.p.timetuple())
def test_utctimetuple(self):
self.assertEqual(self.d.utctimetuple(), self.p.utctimetuple())
def test_date(self):
self.assertEqual(self.d.date(), self.p.date())
def test_time(self):
self.assertEqual(self.d.time(), self.p.time())
def test_timetz(self):
self.assertEqual(self.d.timetz(), self.p.timetz())
def test_astimezone(self):
self.assertEqual(self.d.astimezone(self.p1.tzinfo),
self.p.astimezone(self.p1.tzinfo))
def test_ctime(self):
self.assertEqual(self.d.ctime(), self.p.ctime())
def test_isoformat(self):
self.assertEqual(self.d.isoformat(), self.p.isoformat())
def test_utcoffset(self):
self.assertEqual(self.d.utcoffset(), self.p.utcoffset())
def test_tzname(self):
self.assertEqual(self.d.tzname(), self.p.tzname())
def test_dst(self):
self.assertEqual(self.d.dst(), self.p.dst())
def test_toordinal(self):
self.assertEqual(self.d.toordinal(), self.p.toordinal())
def test_weekday(self):
self.assertEqual(self.d.weekday(), self.p.weekday())
def test_isoweekday(self):
self.assertEqual(self.d.isoweekday(), self.p.isoweekday())
def test_isocalendar(self):
self.assertEqual(self.d.isocalendar(), self.p.isocalendar())
def test_fromtimestamp(self):
self.assertEqual(datetime.fromtimestamp(0, pendulum.UTC),
pendulum.fromtimestamp(0, pendulum.UTC))
def test_utcfromtimestamp(self):
self.assertEqual(datetime.utcfromtimestamp(0),
pendulum.utcfromtimestamp(0))
def test_fromordinal(self):
self.assertEqual(datetime.fromordinal(730120),
pendulum.fromordinal(730120))
def test_combine(self):
self.assertEqual(
datetime.combine(date(2016, 1, 1), time(1, 2, 3, 123456)),
pendulum.combine(date(2016, 1, 1), time(1, 2, 3, 123456)))
def test_hash(self):
dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris')
dt2 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris')
dt3 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'America/Toronto')
self.assertEqual(hash(dt1), hash(dt2))
self.assertNotEqual(hash(dt1), hash(dt3))
def test_pickle(self):
dt1 = Pendulum(2016, 8, 27, 12, 34, 56, 123456, 'Europe/Paris')
s = pickle.dumps(dt1)
dt2 = pickle.loads(s)
self.assertEqual(dt1, dt2)
