def test_cache_file(self):
c = CityIq(self.config)
dt = c.convert_time('2020-01-01')
l = c.get_location('09a66ff9a2c0cb63106fe0054412c2af')
cf = CacheFile(self.config.cache_objects, l, event_type='PKIN', dt=dt, format='csv')
print(cf.path)
# Bare CF
cf.delete()
self.assertFalse(cf.exists())
cf.write(['foo!'])
self.assertTrue(cf.exists())
self.assertEqual(cf.read()[0], 'foo!')
# From the location
cf = l.cache_file(event_type='PKIN', dt=dt)
self.assertTrue(cf.exists(), cf.path)
self.assertEqual(cf.read()[0], 'foo!')
cf.delete()
self.assertFalse(cf.exists())
cf.write(['foo!'])
self.assertTrue(cf.exists())
self.assertEqual(cf.read()[0], 'foo!')
def test_generate_time(self):
c = CityIq(self.config)
st = c.convert_time('2019-01-01')
et = c.convert_time('2020-01-01')
for sm, em in generate_months(st, et):
print(sm, em)
for sd, ed in generate_days(sm, em):
print(' ', sd.date(), ed.date())
def test_cache_assets_locations(self):
from time import time
c = CityIq(self.config)
c.clear_meta_cache()
# These should take gonger because
# they are making the full request
t = time()
o = c.get_locations()
self.assertGreater(time() - t, 1)
self.assertGreater(len(o), 500)
t = time()
c.get_assets()
self.assertGreater(time() - t, 4)
o = self.assertGreater(len(o), 500)
# These should be fast, because they are cached.
t = time()
o = c.get_locations()
self.assertLess(time() - t, 1)
self.assertGreater(len(o), 500)
t = time()
o = c.get_assets()
self.assertLess(time() - t, 1)
self.assertGreater(len(o), 500)
def test_asset_events(self):
c = CityIq(Config(cache_objects='/tmp'))
start = c.tz.localize(datetime(2019, 1, 1, 0, 0, 0))
end = c.tz.localize(datetime(2019, 1, 31, 0, 0, 0))
a = c.get_asset('ff0cfb97-d1d5-463a-b36f-2626421b5e8d')
ft = FetchTask(0, a, 'PKIN', start, end, overwrite=True)
events1 = ft.run()
print(len(events1))
def test_time(self):
c = CityIq(self.config)
dt = c.tz.localize(datetime(2020, 1, 1, 0, 0, 0))
for t in ('2020-01-01', 1577865600, datetime(2020, 1, 1, 0, 0, 0)):
self.assertEqual(dt, c.convert_time(t))
now = c.tz.localize(datetime.now())
for t in ('now', None):
self.assertEqual(int(now.timestamp()), int(c.convert_time(t).timestamp()))
print(c.convert_time('now').replace(hour=0, minute=0, second=0, microsecond=0))
def test_dont_cache_today(self):
from dateutil.relativedelta import relativedelta
c = CityIq(self.config)
api_logger.setLevel(logging.INFO)
assets = list(c.assets_by_event('BICYCLE'))
now =c.convert_time('now').replace(hour=0, minute=0, second=0, microsecond=0)
d1 = relativedelta(days=2)
st = now - d1
et = now + d1
assets[100].get_events('BICYCLE',st, et)
def test_generate_tasks(self):
c = CityIq(self.config)
locations = c.locations
tasks = list(
FetchTask.make_tasks(locations[:10], ['PKIN', 'PKOUT'],
c.convert_time('2019-01-01'),
c.convert_time('2019-04-11')))
#for t in tasks:
# t.run()
print(sum(t.exists() for t in tasks))
self.assertEqual(2000, len(tasks))
def test_async_scrape(self):
from tqdm import tqdm
c = CityIq(self.config)
#task_logger.setLevel(logging.DEBUG)
events = ['BICYCLE'] # '['PKIN', 'PKOUT']
assets = list(c.assets_by_event(events))[:4]
tasks = c.make_tasks(assets, events, '2020-01-01', '2020-01-10')
df = c.events_dataframe(tasks)
print(len(df))
print(df.head().T)
def test_async_runner_locations(self):
c = CityIq(config)
start = c.tz.localize(datetime(2020, 5, 1, 0, 0, 0))
end = c.tz.localize(datetime(2020, 5, 5, 0, 0, 0))
assets = [
e for e in c.get_assets()
if e.eventTypes and 'BICYCLE' in e.eventTypes
]
afr = AsyncFetchRunner(c.config, assets[0:10], 'BICYCLE', start, end)
for e in afr.get_events():
print(e)
def test_locations_detail(self):
c = CityIq(self.config)
for l in c.locations[60:70]:
print(l.data.keys())
d = l.detail
print(d)
def test_make_events(self):
c = CityIq(self.config)
a = c.get_asset('0e5d0a42-8c9e-49fd-9bd7-6ccdf69f840e')
tasks = c.make_tasks([a], ['PKIN', 'PKOUT'], '2020-01-01',
'2020-03-01')
self.assertEqual(24, len(tasks))
dates = list(sorted([t.start_date.isoformat() for t in tasks]))
self.assertEqual([
'2020-01-01T00:00:00-08:00', '2020-01-01T00:00:00-08:00',
'2020-01-06T00:00:00-08:00', '2020-01-06T00:00:00-08:00',
'2020-01-11T00:00:00-08:00', '2020-01-11T00:00:00-08:00',
'2020-01-16T00:00:00-08:00', '2020-01-16T00:00:00-08:00'
], dates[:8])
def test_total_bbox(self):
c = CityIq(self.config)
bb = c.total_bounds
print(bb)
print(c.total_bounds_str)
a = (bb[2] - bb[0]) * (bb[1] - bb[3])
self.assertEqual(0.13, round(a, 2))
def test_consec_days_raw(self):
"""Test that two requests for consecutive days returns the
same number of records as one request for both days. """
import requests, json
c = CityIq(self.config)
headers = c.request_headers(zone='SD-IE-PARKING')
def make_url(start, end):
url = 'https://sandiego.cityiq.io/api/v2/event/locations/0d152fbd26c5baad229556c01d3eb43b/events'
params = c._event_params(start, end, 'PKIN', bbox=False)
url = c.process_url(url, params)
return params, url,
p, url = make_url('2020-01-01', '2020-01-03')
r1 = requests.get(url, headers=headers)
r1.raise_for_status()
print(json.dumps(r1.json()['metaData'], indent=4))
l1 = int(r1.json()['metaData']['totalRecords'])
p, url = make_url('2020-01-01', '2020-01-02')
r2 = requests.get(url, headers=headers)
l2 = int(r2.json()['metaData']['totalRecords'])
p, url = make_url('2020-01-02', '2020-01-03')
r3 = requests.get(url, headers=headers)
l3 = int(r3.json()['metaData']['totalRecords'])
print(l1, l2, l3, l2 + l3)
self.assertEquals(l1, l2 + l3)
def test_nodes(self):
c = CityIq(self.config)
n_childs = 0
for i, n in enumerate(c.nodes):
for c in n.children:
n_childs += 1
if i > 5:
break
self.assertEqual(42, n_childs)
def test_detail(self):
c = CityIq(self.config)
assets = c.assets
a = assets[50]
pprint(a.detail.data)
children = list(a.children)
pprint(children[0].parent.detail.data)
for c in children:
print(c)
def test_event_overlaps(self):
from cityiq.task import request_ranges
from dateutil.relativedelta import relativedelta
d1 = relativedelta(days=1)
c = CityIq(self.config)
extant = list(generate_days(c.convert_time('2020-01-01'), c.convert_time('2020-01-05'))) + \
list(generate_days(c.convert_time('2020-01-10'), c.convert_time('2020-01-15'))) + \
list(generate_days(c.convert_time('2020-01-20'), c.convert_time('2020-01-25')))
extant = [ e[0] for e in extant]
rr = request_ranges(c.convert_time('2020-01-01'), c.convert_time('2020-02-01'), extant)
dts = sorted(c.convert_time(e[0]).date().isoformat() for e in rr)
dte = sorted(c.convert_time(e[1]).date().isoformat() for e in rr)
self.assertEqual(dts, ['2020-01-05', '2020-01-15', '2020-01-25'])
self.assertEqual(dte, ['2020-01-10', '2020-01-20', '2020-02-01'])
def test_location_events(self):
from datetime import datetime
c = CityIq(Config(cache_dir='/tmp', cache_errors='/tmp/errors'))
locations = list(c.walkways)
start = c.tz.localize(datetime(2019, 1, 1, 0, 0, 0))
end = c.tz.localize(datetime(2019, 1, 31, 0, 0, 0))
ft = FetchTask(locations[100], overwrite=True)
events1 = ft.events('PEDEVT', start, end)
print(len(events1))
events2 = locations[100].events('PEDEVT', start, end)
print(len(events2))
def test_async_events(self):
api_logger.setLevel(logging.DEBUG)
c = CityIq(self.config)
assets = list(c.assets_by_event(['PKIN', 'PKOUT']))
self.assertEqual(2590, len(assets))
assets = [
c.get_asset(u) for u in [
'094c05dc-6378-476b-817d-21ba3b99f8ab',
'09741091-c77e-4d61-9a14-d489bd061975',
'09aa6fad-0ae3-4256-906b-8e82c82eeacb',
'09b58810-dd7b-40f2-b183-aef265db4681',
'09f05731-a47e-41e6-970b-9fae943bfd3f',
'09fdcb8e-9498-4c9e-8612-afdfd0487f64',
'0a2f68d2-b283-4835-a775-a209645cebb2',
'0a2ff822-6ba2-4f11-a6a0-b7f080b7ad08',
'0a31fd96-f288-411c-8e50-f07b0b532462',
'0a3e5df5-0738-4f3f-9ab8-1498adfde99c'
]
]
tasks = c.make_tasks(assets, ['PKIN', 'PKOUT'], '2020-01-01',
'2020-02-01')
self.assertEqual(140, len(tasks)) # one per asset/event type
list(c.run_async(tasks))
df = c.get_cached_events(assets, ['PKIN', 'PKOUT'], '2020-01-01',
'2020-02-01')
print(len(df))
print(df.head())
def test_bicycle_events(self):
api_logger.setLevel(logging.DEBUG)
c = CityIq(self.config)
a = c.get_asset('0e5d0a42-8c9e-49fd-9bd7-6ccdf69f840e')
# Clean everything
c._clean_cache(a, 'BICYCLE', '2020-01-01', '2020-02-01')
e = c.get_cached_events(a, 'BICYCLE', '2020-01-10', '2020-01-15')
self.assertEqual(0, len(e))
# Cache events in ranges.
c.cache_events(a, 'BICYCLE', '2020-01-01', '2020-01-05')
e = c.get_cached_events(a, 'BICYCLE', '2020-01-01', '2020-02-01')
self.assertEqual(22942, len(e))
self.assertEqual('2020-01-01', str(e.timestamp.min().date()))
self.assertEqual('2020-01-04', str(e.timestamp.max().date()))
c.cache_events(a, 'BICYCLE', '2020-01-10', '2020-01-15')
c.cache_events(a, 'BICYCLE', '2020-01-20', '2020-01-25')
e = c.get_cached_events(a, 'BICYCLE', '2020-01-01', '2020-02-01')
self.assertEqual(80212, len(e))
self.assertEqual('2020-01-01', str(e.timestamp.min().date()))
self.assertEqual('2020-01-24', str(e.timestamp.max().date()))
c.cache_events(a, 'BICYCLE', '2020-01-04', '2020-01-11')
c.cache_events(a, 'BICYCLE', '2020-01-01', '2020-02-01')
e = c.get_cached_events(a, 'BICYCLE', '2020-01-01', '2020-02-01')
self.assertEqual(177568, len(e))
self.assertEqual('2020-01-01', str(e.timestamp.min().date()))
self.assertEqual('2020-01-31', str(e.timestamp.max().date()))
# Check a subrange.
e = c.get_cached_events(a, 'BICYCLE', '2020-01-10', '2020-01-15')
self.assertEqual(28670, len(e))
self.assertEqual('2020-01-10', str(e.timestamp.min().date()))
self.assertEqual('2020-01-14', str(e.timestamp.max().date()))
c._clean_cache(a, 'BICYCLE', '2020-01-01', '2020-02-01')
e = c.get_cached_events(a, 'BICYCLE', '2020-01-10', '2020-01-15')
self.assertEqual(0, len(e))
def test_get_token(self):
c = CityIq(self.config)
self.assertTrue(len(c.token) > 100)
def test_list_by_event(self):
c = CityIq(self.config)
for a in c.assets_by_event('PEDEVT'):
print(a)
def test_has_event(self):
c = CityIq(self.config)
for i, n in enumerate(c.assets_by_event('TFEVT')):
print(i, n.uid, n.assetType)