def render_home_page_to_cache(force=False):
"""If Home page HTML is not in pref table, render the HTML for the home page and save it to prefs table.
Always return the rendered html
if force == True, ignore the expiration date and always render the html fresh
"""
#get the time cache expiration TTL
try:
from app import app
cache_exp_minutes = app.config['JUMP_DATA_CACHE_TTL']
except:
cache_exp_minutes = 20
page_name = "cache_home_page"
cache = Pref(g.db).select_one(where='name = "{}"'.format(page_name, ))
# see if the page is in cache
if cache and cache.expires >= datetime_as_string(
local_datetime_now()) and force == False:
# deliver from cache
content = cache.value
else:
#otherwise render a new page and save it to cache
rendered_html = render_markdown_for(__file__, mod, 'index.md')
report_data = get_report_data()
summary_data = jump.make_data_dict()
hourly_data = hourlies(1)
hourly_graph_html = hourly_graph.hourly_graph(hourly_data)
content = render_template(
'index_body.html',
rendered_html=rendered_html,
data=summary_data,
report_data=report_data,
hourly_data=hourly_data,
hourly_graph_html=hourly_graph_html,
)
if not cache:
cache = Pref(g.db).new()
cache.name = page_name
cache.value = content
expires = local_datetime_now() + timedelta(
seconds=60 * cache_exp_minutes) # cache TTL
cache.expires = datetime_as_string(expires)
Pref(g.db).save(cache)
try:
g.db.commit()
except:
g.db.rollback()
#satify without saving to cache
return content
def new_sighting(sightings,data,shapes_list,**kwargs):
"""
Create a new sighting record object and return it.
data is single row of the Jump Bike response object
"""
returned_to_service = kwargs.get('returned_to_service',0)
rec = sightings.new()
rec.jump_bike_id = data.get('id',None)
rec.bike_name = data.get('name',None)
rec.retrieved = data.get('retrieved',local_datetime_now())
rec.sighted = rec.retrieved
rec.address = data.get('address',None)
rec.network_id = data.get('network_id',None)
rec.lng = data.get('lng',None)
rec.lat = data.get('lat',None)
rec.returned_to_service = returned_to_service
rec.city = get_city(shapes_list,rec.lng,rec.lat)
rec.batt_level = data.get('ebike_battery_level',None)
rec.batt_distance = data.get('ebike_battery_distance',None)
rec.hub_id = data.get('hub_id',None)
rec.day_number = day_number()
## Get the bonuses
rec.bonuses = get_bonuses(data.get('bonuses',None),rec)
return rec
def new_sighting(sightings, data, shapes_list, **kwargs):
"""
Create a new sighting record object and return it.
data is single row of the Jump Bike response object
"""
returned_to_service = kwargs.get('returned_to_service', 0)
rec = sightings.new()
rec.jump_bike_id = data.get('bike_id', None)
rec.bike_name = data.get('name', None)
rec.retrieved = data.get('retrieved', local_datetime_now())
rec.sighted = rec.retrieved
#rec.address = data.get('address',None)
rec.network_id = data.get(app.config['JUMP_NETWORK_NAME'], None)
rec.lng = data.get('lon', None)
rec.lat = data.get('lat', None)
rec.returned_to_service = returned_to_service
rec.city = get_city(rec.lng, rec.lat, shapes_list)
rec.batt_level = data.get('jump_ebike_battery_level', None)
rec.vehicle_type = data.get('jump_vehicle_type', None)
#rec.hub_id = data.get('hub_id',None)
rec.day_number = day_number()
## Get the bonuses
#rec.bonuses = get_bonuses(data.get('bonuses',None),rec)
return rec
def test_local_datetime_now():
from app import app
app.config['TIME_ZONE'] = 'US/Pacific'
now = datetime.now()
local_now = dates.local_datetime_now()
assert now.day == local_now.day
assert now.hour == local_now.hour
assert now.month == local_now.month
assert local_now.tzinfo is not None
assert local_now.tzinfo.zone == app.config["TIME_ZONE"]
# spcecify a time zone
zone = "US/Eastern"
east_now = dates.local_datetime_now(zone)
assert east_now.tzinfo is not None
assert east_now.tzinfo.zone == zone
def contact():
setExits()
g.name = 'Contact Us'
from app import app
from takeabeltof.mailer import send_message
rendered_html = render_markdown_for(__file__, mod, 'contact.md')
show_form = True
context = {}
if request.form:
if request.form['name'] and request.form['email'] and request.form[
'comment']:
context['name'] = request.form['name']
context['email'] = request.form['email']
context['comment'] = request.form['comment']
context['date'] = local_datetime_now().isoformat(sep=" ")
print(context)
send_message(
None,
subject="Comment from {}".format(app.config['SITE_NAME']),
html_template="home/email/contact_email.html",
context=context,
reply_to=request.form['email'],
)
show_form = False
else:
context = request.form
flash('You left some stuff out.')
return render_template('contact.html',
rendered_html=rendered_html,
show_form=show_form,
context=context)
def get_free_bike_url():
"""get the url to the free bike list based on the site JUMP_GBFS_ROOT_URL from app.config"""
url = app.config['JUMP_GBFS_ROOT_URL']
request_data = requests.get(url).text
if "error" in request_data or '"feeds":' not in request_data: # Not sure what an error looks like
mes = """An error occured while attempting to import Jump Bike feed from {} in "get_free_bike_url()".
Time: {}
Error: {}""".format(url,
local_datetime_now().isoformat(),
str(request_data))
alert_admin(mes)
return None
#convert data from json
try:
request_data = json.loads(request_data)
except:
# alert on conversion error
mes = """An error occured while attempting to convert json data in "get_free_bike_url() step 2".
Time: {}
Error: {}""".format(local_datetime_now().isoformat(),
str(request_data))
alert_admin(mes)
return None
#Find the free bike url
url = None
for item in request_data['data']['en']['feeds']:
if item.get('name', None) == 'free_bike_status':
url = item.get('url', None)
break
if url:
return url
else:
mes = """No valid feeds were retrievd in "get_free_bike_url()" step 3.
Time: {}
Error: {}""".format(local_datetime_now().isoformat(),
str(request_data))
alert_admin(mes)
return None
def test_make_tz_aware():
# make_tz_aware(the_datetime,time_zone=None)
from app import app
zone = "US/Central"
aware = dates.make_tz_aware(datetime.now(), zone)
now = dates.local_datetime_now()
assert aware.tzinfo.zone == zone
aware = dates.make_tz_aware(datetime.now())
assert aware.tzinfo.zone == app.config["TIME_ZONE"]
def update_sighting(data,sight):
"""
Update the sighting record with the latest data
"""
sight.retrieved = data.get('retrieved',local_datetime_now())
sight.day_number = day_number()
## Don't think I want to update the batt level between new sightings
#sight.batt_level = data.get('batt_level',None)
#sight.batt_distance = data.get('batt_distance',None)
sight.bonuses = get_bonuses(data.get('bonuses',sight.bonuses),sight)
return sight
def long_time_no_see(previous_date, newer_date=None):
"""
Return True if it has been over the time limit since we last saw this bike
else False
"""
if newer_date == None:
newer_date = local_datetime_now()
if previous_date <= newer_date - timedelta(hours=2):
#it's been a while
return True
return False
def get_city(lng, lat, shapes_list=None):
"""
Return the name of the city where we found this bike
9/3/18 - Now use mapping.shapes.get_shape_list to provide
a list of dictionaries with shapely shape objects to test
to see if the bike is in there
10/1/18 - If city can't be found in shape files, try reverse geocode lookup
"""
city = None
if shapes_list:
point = Point(lng, lat) # longitude, latitude
for city_dict in shapes_list:
if shape(city_dict['shape']).contains(point):
city = city_dict['city_name']
break
if not city:
#attempt geocode lookup
url = "https://nominatim.openstreetmap.org/reverse?format=json&lat={}&lon={}&zoom=18&addressdetails=1".format(
lat,
lng,
)
geo_data = "Data Not Retrieved Yet"
try:
geo_data = requests.get(url).text
#convert data from json
geo_data = json.loads(geo_data)
#look in address:{..., "city":'XXXXXX',...}
add = geo_data.get('address', None)
city = add.get('city', None)
if not city:
#Not technically in the city?
city = add.get('county', None)
except:
# alert on conversion error
mes = """An error occured while attempting to convert json data.
URL: {}
Time: {}
Data: {}""".format(url,
local_datetime_now().isoformat(),
str(geo_data))
alert_admin(mes)
if not city:
city = "Unknown"
return city
def day_number():
return int(local_datetime_now().strftime('%Y%m%d'))
def hourlies(days_to_report=1):
"""
Return a dictionary designed to be used to report on
the hourly bike usage and availability
Returns 12 hours of data stating 24 hours before now in a dictionary like so:
{'start_date': 'a date string','end_date: 'a date string, 'days': nnumber of days in report,
'hours': [ hours 0-23 in the order to report],
'trips':[ int values of trips for each our ], 'bikes': [ int values of bike counts for each hour],
'max_bikes': highest in 'bikes', 'max_trips': highest in 'trips',
}
If no data is found at all, retruns None
"""
if days_to_report <= 0:
days_to_report = 1
hourly_data = {}
hourly_data['hours'] = []
end_date = local_datetime_now().replace(
minute=0, second=0, microsecond=0) - timedelta(minutes=1)
start_date = end_date - timedelta(days=days_to_report) + timedelta(
minutes=1)
hourly_data['start_date'] = start_date.strftime('%B %-d, %Y %-I:%M %p')
hourly_data['end_date'] = end_date.strftime('%B %-d, %Y %-I:%M %p')
trip_list = []
bike_list = []
max_bikes = 0
max_trips = 0
has_data = False
for hour in range(24):
query_start = start_date
query_end = query_start + timedelta(minutes=59)
hourly_data['hours'].append(start_date.hour)
sql = """select count(trip.id) as trip_count from trip
join sighting on destination_sighting_id = sighting.id
where sighting.retrieved >= '{}' and sighting.retrieved <= '{}'
""".format(query_start.isoformat(sep=" "),
query_end.isoformat(sep=" "))
trip_cnt = g.db.execute(sql).fetchone()
sql = """select avg(bikes_available) as bikes from available_bike_count
where retrieved >= '{}' and retrieved <= '{}' group by city
""".format(query_start.isoformat(sep=" "),
query_end.isoformat(sep=" "))
bike_cnt = g.db.execute(sql).fetchall()
if trip_cnt and bike_cnt:
has_data = True
cnt = int(trip_cnt['trip_count'] / days_to_report)
trip_list.append(cnt)
if cnt > max_trips:
max_trips = cnt
cnt = 0
for x in bike_cnt:
cnt += x['bikes']
cnt = int(cnt)
bike_list.append(cnt)
if cnt > max_bikes:
max_bikes = cnt
else:
trip_list.append(0)
bike_list.append(0)
start_date = start_date + timedelta(hours=1)
if has_data:
hourly_data['trips'] = trip_list
hourly_data['bikes'] = bike_list
hourly_data['max_bikes'] = max_bikes
hourly_data['max_trips'] = max_trips
hourly_data['days'] = days_to_report
return hourly_data
else:
return None
def test_date_to_string():
assert len(dates.date_to_string(dates.local_datetime_now(),
"%Y-%m-%d")) == 10
assert len(dates.date_to_string("11/15/18", "%Y-%m-%d")) == 10
def get_report_data():
"""
Create some containers for our data
"""
report_data = []
#make a cities list plus one element for monthly totals
totals_title = "Network Wide *" # used when displaying data
#cities = ['Davis','Sacramento','West Sacramento',totals_title]
#for this month and last month
for x in range(0, 2):
#import pdb;pdb.set_trace()
network_wide_bikes_available = 0
now = local_datetime_now().replace(day=1) #first day of this month
now = now - timedelta(
days=1 * x
) #now is now the first day of this month, or the last day of the prev. month
dr = calendar.monthrange(
now.year, now.month) # -> `(first day_number, number of last day)`
start_date = now.replace(day=1)
month_name = start_date.strftime('%B %Y') # Month Name / Year
end_date = now.replace(day=dr[1]) # last day of month
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
# get a list of all the cities we actually saw bikes in
sql = 'select distinct city from sighting where retrieved >= "{start_date}" and retrieved <= "{end_date}" order by city'.format(
start_date=start_date_str, end_date=end_date_str)
city_recs = g.db.execute(sql)
cities = []
for city in city_recs:
cities.append(city['city'])
cities.append(totals_title)
# First test is there are any sightings for this month
#import pdb;pdb.set_trace()
rec = g.db.execute(
'select min(retrieved) as first, max(retrieved) as last from sighting where retrieved >= "{start_date}" and retrieved <= "{end_date}"'
.format(start_date=start_date_str,
end_date=end_date_str)).fetchone()
if rec and rec['first'] != None and rec['last'] != None:
# Get the number of days in this date range
days_to_average_over = int(rec['last'][8:10]) - (int(
rec['first'][8:10])) + 1
if days_to_average_over < 1:
days_to_average_over = 1 #protect against divide by zero
monthly_data = {}
monthly_data['month_name'] = month_name
monthly_data['days_to_average_over'] = days_to_average_over
monthly_data['cities'] = []
# Get bikes and trips observed for all cities
for current_city in cities:
if current_city != totals_title:
city_clause = get_sql_for_city_clause().format(
city=current_city)
else:
city_clause = ''
sql = get_sql_for_bikes_and_trips().format(
city_clause=city_clause,
start_date=start_date_str,
end_date=end_date_str)
rec = g.db.execute(sql).fetchone()
if rec:
# Get the median of available bikes for these sightings
avg_bikes_available = 0 #place holder
sql = get_available_bikes_sql().format(
city_clause=city_clause,
start_date=start_date_str,
end_date=end_date_str)
avail = g.db.execute(sql).fetchall()
if not avail:
avg_bikes_available = 0
elif current_city != totals_title:
avg_bikes_available = int(
median([x['bikes_available'] for x in avail]))
network_wide_bikes_available += avg_bikes_available
if current_city == totals_title:
avg_bikes_available = network_wide_bikes_available
city_dict = {}
city_dict['city'] = current_city
city_dict['city_bikes'] = rec['city_bikes']
city_dict['city_trips'] = rec['city_trips']
city_dict['avg_bikes_available'] = avg_bikes_available
"""
Averages should only be divided by the number of FULL days.
If it's not the last day of the month we are currently processing
then we need to get the counts for just the full days of this month.
"""
#import pdb;pdb.set_trace()
city_trips = 0
city_bikes = 0
day_adjust = 0
if start_date.day < end_date.day and days_to_average_over >= 2:
#get the data for just the full days of this month
avg_end_date_str = end_date.strftime('%Y-%m-%d')
sql = get_sql_for_bikes_and_trips().format(
city_clause=city_clause,
start_date=start_date_str,
end_date=avg_end_date_str)
avg_rec = g.db.execute(sql).fetchone()
if avg_rec:
city_trips = avg_rec['city_trips']
city_bikes = avg_rec['city_bikes']
day_adjust = 1
else:
city_trips = rec['city_trips']
city_bikes = rec['city_bikes']
day_adjust = 0
else:
# to avoid divide by zero error
# This should only happen during the first day of data collection each month
city_dict['trips_per_day'] = 'N/A'
city_dict['trips_per_bike_per_day'] = 'N/A'
if city_bikes <= 0:
city_bikes = 1 # Protect against divide by zero
if days_to_average_over >= 2:
city_dict['trips_per_day'] = '{:.2f}'.format(
round(
city_trips /
(days_to_average_over - day_adjust), 2))
city_dict['trips_per_bike_per_day'] = '{:.2f}'.format(
round((city_trips /
(days_to_average_over - day_adjust) /
city_bikes), 2))
monthly_data['cities'].append(city_dict)
report_data.append(monthly_data)
return report_data
def get_jump_data():
try:
mes = 'No message Yet...'
from takeabeltof.database import Database
db = Database(working_path + "/" + app.config['DATABASE_PATH']).connect()
init_tables(db) # creates tables if needed
# Use this to determine if bikes have been off the system too long (getting recharged?)
last_sighting_limit = (local_datetime_now() - timedelta(hours=2)).isoformat(sep=' ')
#size=10
#network = 165
size = app.config['JUMP_REQUEST_SIZE']
network = app.config['JUMP_NETWORK_ID']
shapes_list = get_shape_list()
if not shapes_list:
# most likely the json files don't exist or are in the wrong place
alert_admin("Error: No shape files were found when trying to get shapes_list in jump.get_data")
# I now have 2 Jump accounts to use for polling the server, so I can poll more often
# if the minutes are odd, or even...
if (local_datetime_now().minute % 2 == 0): #even
row = 0
else: #odd
row = 1
username = app.config['JUMP_LOGIN'][row][0]
password = app.config['JUMP_LOGIN'][row][1]
url = 'https://app.socialbicycles.com/api/bikes.json?page=1&per_page={}&network_id={}'.format(size,network)
request_data = requests.get(url,auth=(username,password)).json()
if "error" in request_data or 'items' not in request_data: # {"error":"Not Authenticated","code":401}
db.close()
mes = """An error occured while attempting to import Jump Bike data.
Time: {}
Error: {}""".format(local_datetime_now().isoformat(),str(request_data))
alert_admin(mes)
return "Error received while accessing Jump Data: {}".format(str(request_data))
observations = request_data['items']
retrieval_dt = local_datetime_now()
sightings = Sighting(db)
bikes = Bike(db)
trips = Trip(db)
new_data = {'sighting':0, 'bike': 0, 'trip': 0, 'available': 0,}
avail_city_data = {}
for ob in observations:
lng = ob['lng'] = ob['current_position']['coordinates'][0]
lat = ob['lat'] = ob['current_position']['coordinates'][1]
ob['retrieved'] = retrieval_dt
sql = 'jump_bike_id = {}'.format(ob['id'])
bike = bikes.select_one(where=sql)
new_data['available'] += 1
city = get_city(shapes_list,lng,lat)
if city in avail_city_data:
avail_city_data[city] += 1
else:
avail_city_data[city] = 1
if not bike:
# A new bike...
bike = bikes.new()
bike.jump_bike_id = ob.get('id',None)
bike.name = ob.get('name',None)
bikes.save(bike)
new_data['bike'] += 1
sightings.save(new_sighting(sightings,ob,shapes_list))
new_data['sighting'] += 1
continue
#Get the last time we saw this bike
where = 'jump_bike_id = {}'.format(bike.jump_bike_id)
order_by = 'id desc' # should be by datetime, but there are issues
sight = sightings.select_one(where=where, order_by=order_by)
if not sight:
#This should really never happen...
sightings.save(new_sighting(sightings,ob,shapes_list))
new_data['sighting'] += 1
continue
if long_time_no_see(datetime.strptime(sight.retrieved,'%Y-%m-%d %H:%M:%S.%f')):
#This bike has been off getting service
sight = new_sighting(sightings,ob,shapes_list,returned_to_service=1)
sightings.save(sight)
#print('Returned to service: {}'.format(sight))
new_data['sighting'] += 1
continue
# Seeing the bike again so how far has it moved
distance = miles_traveled(lng, lat, sight.lng, sight.lat)
if distance >= 0.128:
#bike moved at least 1/8 mile
origin_id = sight.id
sight = new_sighting(sightings,ob,shapes_list)
sightings.save(sight)
#print('New Trip Sighting: {}'.format(sight))
new_data['sighting'] += 1
# Make a trip
trip = new_trip(trips,bike.jump_bike_id,origin_id,sight.id,distance)
trips.save(trip)
#print('New Trip : {}'.format(sight))
new_data['trip'] += 1
else:
#too short a move, Just update the sighting record
sightings.save(update_sighting(ob,sight))
#end ob loop
# record the number of available bikes
if new_data['available'] > 0:
for city in avail_city_data.keys():
avail = AvailableBikeCount(db).new()
avail.bikes_available = avail_city_data[city]
avail.city = city
avail.retrieved = retrieval_dt
avail.day_number = day_number()
AvailableBikeCount(db).save(avail)
db.commit()
mes = 'At {}; New Data added: Available: {}, Sightings: {}, Bikes: {}, Trips: {}'.format(local_datetime_now().isoformat(),new_data['available'],new_data['sighting'],new_data['bike'],new_data['trip'])
return(mes)
except Exception as e:
try:
if db:
db.rollback()
except Exception as e:
mes = """Could not connect to db while attempting to import Jump Bike data.
Error: {}
""".format(str(e))
mes = printException(mes,"error",e)
alert_admin(mes)
return mes
mes = """An error occured while attempting to import Jump Bike data.
Error: {}
""".format(str(e))
mes = printException(mes,"error",e)
alert_admin(mes)
return mes
def time_lapse_map():
"""
Display an automated map of bike sightings over time
"""
setExits()
days = 1
start_date = local_datetime_now() + timedelta(
days=-1) # Always starts at midnight, yesterday
start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
end_date = start_date + timedelta(days=days, seconds=-1)
frame_duration = 10 * 60 # this many seconds of real time elapse between each frame
seconds_per_frame = 1 # display each frame for this many seconds
sql = """select id, lng, lat, sighted, retrieved from sighting where
retrieved >= '{start_date}' and sighted <= '{end_date}'
order by sighted
""".format(start_date=start_date.isoformat(sep=' '),
end_date=end_date.isoformat(sep=' '))
recs = g.db.execute(sql).fetchall()
markerData = {"markers": []}
markerData["zoomToFit"] = False # can/t zoom if there are no markers.
if recs:
"""
The Marker is a list of lists containing:
lng,
lat,
display start seconds,
display end seconds
At play time in javascript, every frame_duration seconds loop through Markers:
if display start seconds <= frame start time and display end seconds >= frame end time,
set the marker opacity to 1
else
set opacity to 0
"""
total_seconds = int(round((end_date - start_date).total_seconds(), 0))
markerData["zoomToFit"] = True
markerData['total_seconds'] = total_seconds
markerData['frame_duration'] = frame_duration
markerData['seconds_per_frame'] = seconds_per_frame
#import pdb;pdb.set_trace()
for rec in recs:
sighted_dt = getDatetimeFromString(rec['sighted'])
if sighted_dt.day == 17:
#import pdb;pdb.set_trace()
pass
#print('sighted_dt: {}'.format(sighted_dt))
retrieved_dt = getDatetimeFromString(rec['retrieved'])
#print('retrieved_dt: {}'.format(retrieved_dt))
markerData["markers"].append([
round(rec['lng'], 5),
round(rec['lat'], 5),
int(round((sighted_dt - start_date).total_seconds(), 0)),
int(round((retrieved_dt - start_date).total_seconds(), 0)),
])
return render_template('JSONmap.html',
markerData=markerData,
start_date=start_date)
def get_gbfs_data():
try:
#import pdb;pdb.set_trace()
mes = 'No message Yet...'
raw_data = "No data fetched yet" #just a place-holder for now
db = g.db
shapes_list = get_shape_list()
if not shapes_list:
# most likely the json files don't exist or are in the wrong place
alert_admin(
"Error: No shape files were found when trying to get shapes_list in jump.get_data"
)
# Get free bike status feed url
url = get_free_bike_url()
if not url:
mes = """No Free bike status URL while attempting to import Jump Bike data for {}.
Time: {}
URL: {}""".format(app.config['JUMP_NETWORK_NAME'],
local_datetime_now().isoformat(), str(url))
alert_admin(mes)
raw_data = requests.get(url).text
if "error" in raw_data or '"bikes":' not in raw_data: # Not sure what an error looks like
mes = """An error occured while attempting to import Jump Bike data from {}.
Time: {}
Error: {}""".format(url,
local_datetime_now().isoformat(), raw_data)
alert_admin(mes)
return mes
#convert data from json
try:
request_data = json.loads(raw_data)
except:
# alert on conversion error
mes = """An error occured while attempting to convert json data in "get_gbfs_data()".
Time: {}
Error: {}""".format(local_datetime_now().isoformat(), raw_data)
alert_admin(mes)
return mes
#Are there any bikes?
if not request_data['data']['bikes']:
mes = """No bikes were retrievd in "get_gbfs_data()" step 2.
Time: {}
Error: {}""".format(local_datetime_now().isoformat(),
str(request_data))
alert_admin(mes)
return mes3
#got data!
observations = request_data['data']['bikes']
retrieval_dt = local_datetime_now()
sightings = Sighting(db)
bikes = Bike(db)
trips = Trip(db)
new_data = {
'sighting': 0,
'bike': 0,
'trip': 0,
'available': 0,
}
avail_city_data = {}
# Temporary to only send email once per retireval
found_a_different_vehicle = ''
for ob in observations:
# Jump added a new property for vehicle type. We are only interested in bikes
if ob.get("jump_vehicle_type", "bike") != "bike":
# Alert me when we see a vehicle type I don't recognize
temp_veh = ob.get("jump_vehicle_type", "None")
if found_a_different_vehicle != temp_veh:
found_a_different_vehicle = temp_veh
mes = """Received response with vehicle type of '{}'. Time: {}
Data:
{}
""".format(temp_veh,
local_datetime_now().isoformat(), ob)
alert_admin(mes)
lng = ob['lon']
lat = ob['lat']
ob['retrieved'] = retrieval_dt
# the bike_id for jump is prepended with 'bike_' so we need to strip that off
bike_id = ob.get('bike_id', None)
if not bike_id:
mes = """bike_id not in data.
Time: {}""".format(local_datetime_now().isoformat(), )
alert_admin(mes)
continue
pos = ob['bike_id'].find('_')
if pos > 0:
ob['bike_id'] = ob['bike_id'][pos + 1:]
# jump has a custom field 'jump_ebike_battery_level' with a '%' sign. Drop the sign
batt_pct = ob.get('jump_ebike_battery_level', None)
if batt_pct:
pos = batt_pct.find("%")
if pos > 0:
ob['jump_ebike_battery_level'] = batt_pct[:pos]
sql = 'jump_bike_id = {}'.format(ob['bike_id'])
bike = bikes.select_one(where=sql)
new_data['available'] += 1
city = get_city(lng, lat, shapes_list)
if city in avail_city_data:
avail_city_data[city] += 1
else:
avail_city_data[city] = 1
if not bike:
# A new bike...
bike = bikes.new()
bike.jump_bike_id = ob['bike_id']
bike.name = ob.get('name', None)
bike.vehicle_type = ob.get("jump_vehicle_type", None)
bikes.save(bike)
new_data['bike'] += 1
sightings.save(new_sighting(sightings, ob, shapes_list))
new_data['sighting'] += 1
# no need to look for previous sightings
continue
#Get the last time we saw this bike
where = 'jump_bike_id = {}'.format(bike.jump_bike_id)
order_by = 'id desc' # should be by datetime, but there are issues
sight = sightings.select_one(where=where, order_by=order_by)
if not sight:
#This should really never happen...
sightings.save(new_sighting(sightings, ob, shapes_list))
new_data['sighting'] += 1
continue
#import pdb;pdb.set_trace()
# sight.retrieved is a string so try to convert it to a datetime
# Because some dates were stored as time zone aware and some not, just truncat it to seconds
prev_sighting_date = datetime.strptime(sight.retrieved[:19],
'%Y-%m-%d %H:%M:%S')
if long_time_no_see(make_tz_aware(prev_sighting_date)):
#This bike has been off getting service
sight = new_sighting(sightings,
ob,
shapes_list,
returned_to_service=1)
sightings.save(sight)
#print('Returned to service: {}'.format(sight))
new_data['sighting'] += 1
continue
# Seeing the bike again so how far has it moved
distance = miles_traveled(lng, lat, sight.lng, sight.lat)
if distance >= 0.128:
#import pdb;pdb.set_trace()
#bike moved at least 1/8 mile
origin_id = sight.id
sight = new_sighting(sightings, ob, shapes_list)
sightings.save(sight)
#print('New Trip Sighting: {}'.format(sight))
new_data['sighting'] += 1
# Make a trip
trip = new_trip(trips, bike.jump_bike_id, origin_id, sight.id,
distance)
trips.save(trip)
#print('New Trip : {}'.format(sight))
new_data['trip'] += 1
else:
#too short a move, Just update the sighting record
sightings.save(update_sighting(ob, sight))
#end ob loop
# record the number of available bikes
if new_data['available'] > 0:
for city in avail_city_data.keys():
avail = AvailableBikeCount(db).new()
avail.bikes_available = avail_city_data[city]
avail.city = city
avail.retrieved = retrieval_dt
avail.day_number = day_number()
AvailableBikeCount(db).save(avail)
db.commit()
mes = 'At {}; New Data added: Available: {}, Sightings: {}, Bikes: {}, Trips: {}'.format(
local_datetime_now().isoformat(), new_data['available'],
new_data['sighting'], new_data['bike'], new_data['trip'])
# Update the home page cache with the new data
render_home_page_to_cache(force=True)
return (mes)
except Exception as e:
try:
mes_data_response = raw_data
except:
mes_data_response = "No Data Retrieved"
mes = """An un-caught error occured while attempting to fetch bike data.
Error: {}
Data Retrived: {}
""".format(str(e), mes_data_response)
mes = printException(mes, "error", e)
alert_admin(mes)
return mes