def combine_versions(items, a, b, props=['d1']):
results = []
keys_a = set(items[a].keys())
keys_b = set(items[a].keys())
# keys_a = set(['boo', 'bar'])
# keys_b = set(['foo', 'bar', 'hala'])
intersected = keys_a.intersection(keys_b)
diverse = keys_a.union(keys_b) - intersected
if diverse:
print 'key sets are not same! They differ at {}'.format(diverse)
for key in intersected:
result = { 'name': key }
for p in props:
val_b = [float(x) for x in pluck(items[b][key], p) if x is not None]
val_a = [float(x) for x in pluck(items[a][key], p) if x is not None]
if not val_a or not val_b:
# print 'missing property on {key}'.format(key=key)
result[p] = 1.0
else:
# fix calculated slow-down-factor for frame f if number of collected results differ
# only if missing collected results are results fixing random element
# tests MUST be the same, otherwise stretching does not make sense
result[p] = (sum(val_b) / len(val_b)) / (sum(val_a) / len(val_a))
# result[p] = sum(val_b) / sum(val_a)
results.append(result)
return results
def measure(self, cls, name, timeout=None, tries=None, processes=None):
timeout = timeout if timeout is not None else self.timeout
tries = tries if tries is not None else self.tries
processes = processes if processes is not None else self.processes
pb = ProgressBar(maximum=tries, width=30, prefix="{self.name:35}",
suffix=" {self.last_progress}/{self.maximum}")
measure_result = list()
for no_cpu in processes:
pb.name = "{:s} {:d} {:s}".format(name, no_cpu, 'core' if no_cpu == 1 else 'cores')
results = list()
for i in range(0, tries):
if self.print_output:
pb.progress(i)
targets = [cls() for j in range(0, no_cpu)]
with timer.measured("{:s} {:d}".format(name, i), False):
# start processes
for target in targets:
target.start()
# wait for timeout
time.sleep(timeout)
# send exit status
for target in targets:
target.shutdown()
# join threads
for target in targets:
target.join()
tmp = dict()
tmp['duration'] = timer.time()
tmp['value'] = sum(pluck(targets, 'result.value'))
tmp['exit'] = not max(pluck(targets, 'terminated'))
results.append(tmp)
if self.print_output:
pb.end()
result = dict()
result['processes'] = no_cpu
# result['exit'] = min(pluck(results, 'exit'))
result['duration'] = sum(pluck(results, 'duration')) / float(tries)
result['value'] = sum(pluck(results, 'value')) / float(tries)
result['performance'] = result['value'] / result['duration']
result['effectiveness'] = (result['value'] / result['duration']) / no_cpu
if self.human_format:
result['value'] = human_readable(result['value'])
result['performance'] = human_readable(result['performance'])
result['effectiveness'] = human_readable(result['effectiveness'])
measure_result.append(result)
return measure_result
def test_getattr_based_plucks(self):
l = [
datetime(2012, 10, 1, 3),
datetime(2012, 10, 15, 4),
datetime(2012, 10, 30, 8),
]
assert pluck(l, 'year') == [2012, 2012, 2012]
assert pluck(l, 'day') == [1, 15, 30]
assert pluck(l, 'hour') == [3, 4, 8]
def test_getitem_based_plucks(self):
l = [
{'id': 282, 'name': 'Alice', 'age': 30},
{'id': 217, 'name': 'Bob', 'age': 56},
{'id': 328, 'name': 'Charlie', 'age': 56},
{'id': '167'},
]
assert pluck(l, 'id') == [282, 217, 328, '167']
self.assertRaises(ValueError, pluck, l, 'name')
assert pluck(l, 'name', default=None) == ['Alice', 'Bob', 'Charlie', None]
assert pluck(l, 'name', default='Mr. X') == ['Alice', 'Bob', 'Charlie', 'Mr. X']
assert pluck(l, 'age', default=None) == [30, 56, 56, None]
def rpluck(obj, path):
names = path.split(".")
last = names[-1]
del names[-1]
o = rget(obj, ".".join(names), [])
return pluck(o, last)
def phase_classify_render(request):
query = {'timestamp': {'$exists': True, '$ne': None}}
# print(request.POST)
min_phase_time = float(request.POST.get('min-phase-seconds'))
cruise_window = float(request.POST.get('cruise-window'))
if request.POST.getall('trips[]'):
query['trip_id'] = {'$in': request.POST.getall('trips[]')}
else:
query['trip_id'] = {
'$nin':
pluck(
request.db.webcan_trip_filters.find(
{'vid': {
'$in': request.POST.getall('devices[]')
}}), 'trip_id')
}
if request.POST.getall('devices[]'):
query['vid'] = {'$in': request.POST.getall('devices[]')}
readings = request.db.rpi_readings.with_options(codec_options=CodecOptions(
tz_aware=True, tzinfo=pytz.timezone('Australia/Adelaide')))
cursor = readings.find(query, {'_id': False}).sort(SORT_TRIP_SEQ)
print(query)
print("Len readings:", cursor.count())
readings = list(cursor)
speed = _classify_readings_with_phases_pas(readings,
min_phase_time,
cruise_avg_window=cruise_window)
summary = _summarise_readings(readings)
return {
'readings': readings,
# 'readings': [],
'summary': summary,
'speed_field': speed
}
def get_device_trips_for_user(request):
return list(
request.db.rpi_readings.distinct(
'trip_id',
{'vid': {
'$in': pluck(request.user['devices'], 'name')
}}))
def __init__(self, request_file):
if type(request_file) is dict:
request = request_file
else:
with open(request_file, 'r') as fp:
request = json.load(fp)
self.username = request.get('username', None)
self.nameuser = request.get('nameuser', None)
self.reference = request.get('reference', None)
self.timestamp = datetime.datetime.fromtimestamp(request.get('timestamp', 0))
self.root = request.get('root', None)
self.filename = request.get('filename', None)
self.main_file = os.path.join(self.root, self.filename)
self.main_file_name, self.main_file_ext = os.path.splitext(self.main_file)
self.lang = Langs.get(request.get('lang_id', None))
self.problem = Problems.get(request.get('problem_id', None))
try:
self.cases = request.get('cases', pluck(self.problem.input if self.problem else [], 'id'))
except Exception as e:
print(e)
self.cases = []
raise
self.result_file = os.path.join(self.root, 'result.json')
self.output_root = os.path.join(self.root, 'output')
self.delete_me_file = os.path.join(self.root, '.delete-me')
def rpluck(obj, path):
names = path.split('.')
last = names[-1]
del names[-1]
o = rget(obj, '.'.join(names), [])
return pluck(o, last)
def get(self):
words = [score.to_json() for score in Word.query.all()]
random_word = random.choice(pluck(words, "word"))
response_object = {
"word": random_word,
}
return response_object, 200
def main():
conf = configparser.ConfigParser()
conf.read('../../development.ini')
uri = conf['app:webcan']['mongo_uri']
conn = MongoClient(uri)['webcan']
filtered_trips = pluck(conn.webcan_trip_filters.find(), 'trip_id')
# vid_re = 'rocco_phev'
vid_re = '^adl_metro'
num_trips = len(set(x.split('_')[2] for x in
conn.rpi_readings.distinct('trip_id', {'vid': {'$regex': vid_re}}))
- set(x.split('_')[2] for x in filtered_trips))
# generate a fuel consumption report
query = {
# 'vid': vid_re,
'vid': {'$regex': vid_re},
}
if filtered_trips:
query['trip_id'] = {'$nin': filtered_trips}
readings = conn.rpi_readings.with_options(
codec_options=CodecOptions(tz_aware=True, tzinfo=pytz.timezone('Australia/Adelaide')))
cursor = readings.find(query).sort([('trip_key', 1)])
report = []
prog = tqdm.tqdm(desc='Trip Reports: ', total=num_trips, unit=' trips')
def on_complete(r):
# put this in the db
# print(r)
conn.trip_summary.insert_one({k: parse(v) for k, v in r.items()})
if r['Distance (km)'] >= 10:
report.append(r)
prog.update()
def summary_exists(trip_key):
return conn.trip_summary.find_one({'trip_key': trip_key}) is not None
pool = Pool()
i = 0
for trip_id, readings in groupby(cursor, key=lambda x: x['trip_key']):
if summary_exists(trip_id):
continue
readings = list(readings)
# on_complete(fuel_report_trip(trip_id, readings))
pool.apply_async(fuel_report_trip, args=(trip_id, readings), callback=on_complete)
i += 1
pool.close()
pool.join()
prog.close()
print(tabulate.tabulate(report, headers='keys'))
exit()
import csv
with open('adl_metro_report_phev.csv', 'w') as out:
writer = csv.DictWriter(out, fieldnames=list(report[0].keys()))
writer.writeheader()
writer.writerows(report)
def get_statuses(self, format):
status_getter = lambda x: x[1]
statuses = pluck(self.threads, 'pretty_name', 'status', 'cpus',
'timer')
sorted_statuses = sorted(statuses, key=status_getter)
if format is LogStatusFormat.SIMPLE:
msg = 'Worker statuses:\n'
msgs = list()
for key, group in itertools.groupby(sorted_statuses,
key=status_getter):
grp = list(group)
if key is WorkerStatus.RUNNING:
msgs.append(' %2d x %s %s' %
(len(grp), key.name, [x[0] for x in grp]))
else:
msgs.append(' %2d x %s' % (len(grp), key.name))
return msg + '\n'.join(msgs)
elif format is LogStatusFormat.COMPLEX:
msg = 'Worker statuses:\n'
msgs = list()
for key, group in itertools.groupby(sorted_statuses,
key=status_getter):
msgs.append('%s:' % key.name)
for name, status, cpus, timer in group:
if timer.duration > 0.0:
msgs.append(' - %dx %s [%1.3f sec]' %
(cpus, name, timer.duration))
else:
msgs.append(' - %dx %s' % (cpus, name))
return msg + '\n'.join(msgs)
elif format is LogStatusFormat.ONELINE:
statuses = sorted(
[x[0] for x in pluck(self.threads, 'status.name')])
msg = ''.join(statuses).upper(
) # .replace('W', '◦').replace('R', '▸').replace('F', ' ') # ⧖⧗⚡
return msg
elif format is LogStatusFormat.ONELINE_GROUPED:
statuses = sorted([x for x in pluck(self.threads, 'status')])
msg = list()
for g, d in itertools.groupby(statuses):
msg.append('[%d x %s]' % (len(list(d)), g.name))
return ' > '.join(msg)
def reduce_groups(items, reductions=[]):
result = {}
for key, grp in items.items():
item = {'_id': key, '_count': len(grp)}
for reduce_field, new_field, reduce_func in reductions:
item[new_field] = reduce_func(pluck(grp, reduce_field))
result[key] = item
return result
def reduce_groups(items, reductions=[]):
result = { }
for key, grp in items.items():
item = { '_id': key, '_count': len(grp) }
for reduce_field, new_field, reduce_func in reductions:
item[new_field] = reduce_func(pluck(grp, reduce_field))
result[key] = item
return result
def test_works_with_iterables(self):
l = iter([
{'id': 282, 'name': 'Alice', 'age': 30},
{'id': 217, 'name': 'Bob', 'age': 56},
{'id': 328, 'name': 'Charlie', 'age': 56},
{'id': 167},
])
assert pluck(l, 'id', 'name', defaults={'name': 'Mr. X'}) == \
[(282, 'Alice'), (217, 'Bob'), (328, 'Charlie'), (167, 'Mr. X')]
def make_stats(_readings):
out = {}
for key, val in _readings.items():
if not val:
continue
dist = 0
duration = 0
phases = {"Phase {}".format(i): 0.0 for i in range(7)}
# print(val)
start = val[0]['timestamp']
for r1, r2 in zip(val, val[1:]):
if r1['trip_id'] != r2['trip_id']:
duration += (r1['timestamp'] - start).total_seconds()
start = r2['timestamp']
continue
dist += vincenty(r1['pos']['coordinates'][::-1],
r2['pos']['coordinates'][::-1]).kilometers
for phase, data in groupby(
val, key=lambda x: f"{x['phase']}:{x['trip_key']}"):
phase = phase.split(':')[0]
data = list(data)
phase_duration = (data[-1]['timestamp'] -
data[0]['timestamp']).total_seconds()
phases[f"Phase {phase}"] += phase_duration
duration += (val[-1]['timestamp'] - start).total_seconds()
h, m, s = seconds_2_hms(duration)
usages = [
'Total CO2e (g)',
'Total Energy (kWh)',
'Petrol Used (ml)',
'Petrol CO2e (g)',
'Petrol cost (c)',
'P Used (kWh)',
'E Used (kWh)',
'E CO2e (g)',
'E cost (c)',
]
out[key] = {
'Duration': "%d:%02d:%02d" % (h, m, s),
'Distance (km)': round(dist, 4),
}
for field in usages:
out[key][field] = round(sum(pluck(val, field, default=0)), 2)
total_time = sum(phases.values())
out[key].update({
k: "{} ({:.2f}%)".format(
"{:02d}:{:02d}:{:02d}".format(
*(int(x) for x in seconds_2_hms(v))),
100 * v / total_time)
for k, v in phases.items()
})
out[key]['Start'] = val[0]['timestamp'].isoformat(' ')
out[key]['End'] = val[-1]['timestamp'].isoformat(' ')
return out
def get_diagrams(intersections, request):
intersections = pluck(intersections, 'site_no')
return request.db.locations.find({
'site_no': {
'$in': intersections
},
'scats_diagram': {
'$exists': True
}
})
def test_get_multiple_fields(self):
l = [
{'id': 282, 'name': 'Alice', 'age': 30},
{'id': 217, 'name': 'Bob', 'age': 56},
{'id': 328, 'name': 'Charlie', 'age': 56},
{'id': '167'},
]
self.assertRaises(ValueError, pluck, l, 'id', 'name')
assert pluck(l, 'id', 'name', defaults={'name': 'Mr. X'}) == \
[(282, 'Alice'), (217, 'Bob'), (328, 'Charlie'), ('167', 'Mr. X')]
# Specifying a default works, as long as all other fields are found
self.assertRaises(ValueError, pluck, l, 'name', 'age')
# This still fails, because there are missing age fields
self.assertRaises(ValueError, pluck, l, 'name', 'age', defaults={'name': 'Mr. X'})
assert pluck(l, 'id', 'age', defaults={'age': 100}) == \
[(282, 30), (217, 56), (328, 56), ('167', 100)]
def trips_of_device(request):
req_devices = set(request.GET.getall('devices[]'))
user_devices = set(pluck(request.user['devices'], 'name'))
if len(req_devices) == 0:
devices = user_devices
else:
devices = req_devices & user_devices
trips = list(request.db.rpi_readings.distinct('trip_id', {'vid': {'$in': list(devices)}}))
trips_with_vid = [request.db.rpi_readings.find_one({'trip_id': x, 'vid': {'$exists': True}},
{'_id': False, 'trip_id': True, 'vid': True}) for x in trips]
return {
'trips': trips_with_vid
}
def list_groups(request):
# a regular user will have their groups listed
# admin will see all groups
user = request.user
query = request.db_session.query(LedGroup)
users_groups = request.db_session.query(LedGroupUser).filter(LedGroupUser.led_user == user).all()
if not user.admin:
query.filter(LedGroup.id.in_(pluck(users_groups, 'led_group_id')))
groups_admin = [group.led_group_id for group in users_groups if group.access_level == 2]
return {
'user_admins': groups_admin,
'groups': query.all()
}
def get_comments(request):
comments = Comments.objects.filter(article_id=request.GET['id']).order_by('-id')
comments = pagination(comments, request.GET.get('page'))
login_ids = pluck(comments, 'login_id')
user_details = Login.objects.filter(id__in=login_ids)
all_comments = []
for comment in comments:
for user_detail in user_details:
if comment.login_id == user_detail.id:
data = {
'name' : user_detail.first_name + ' ' + user_detail.last_name,
'comment': comment.comment
}
all_comments.append(data)
return render(request, 'comments.html', {'comments':comments, 'all_comments':all_comments, 'id':request.GET['id']})
def home(request):
if request.user.is_authenticated and 'category' not in request.GET and 'all' not in request.GET:
intrests = Login.objects.filter(id=request.user.id).first()
intrests = intrests.intrests.replace('[', '')
intrests = intrests.replace(']', '')
intrests = intrests.replace("'", '')
intrests = intrests.split(',')
intrests_id = []
for d in intrests:
intrests_id.append(int(d))
articles = Article.objects.filter(category_id__in=intrests_id).order_by('-created_at')
if len(articles) == 0:
articles = Article.objects.order_by('-created_at')
elif 'category' in request.GET:
articles = Article.objects.filter(category_id=request.GET['category']).order_by('-created_at')
elif 'all' in request.GET:
articles = Article.objects.filter().order_by('-created_at')
else:
articles = Article.objects.filter().order_by('-created_at')
categories = Category.objects.filter()
total = Article.objects.count()
articles = pagination(articles, request.GET.get('page'))
login_ids = pluck(articles, 'login_id')
user_details = Login.objects.filter(id__in=login_ids)
categories = Category.objects.filter()
all_articles = []
for article in articles:
for user_detail in user_details:
if article.login_id == user_detail.id:
data = {
'id':article.id,
'author':user_detail.first_name + ' ' + user_detail.last_name,
'title':article.title,
'image':article.image,
'content':article.content,
'created_at':article.created_at if not article.updated_at else article.updated_at,
'login_id':article.login_id,
'category_id':article.category_id,
}
all_articles.append(data)
for d in all_articles:
for category in categories:
if category.id == d['category_id']:
d.update({'category_name':category.name})
return render(request, 'home.html', {'articles': articles, 'all_articles':all_articles, 'total':total, 'categories':categories})
def target():
# no work to be done? just finish up here
# and return the thread
if not self.threads:
return
last_print = 0
while True:
if (time.time() - last_print) > log_period:
logger.debug(self.get_statuses(format))
last_print = time.time()
if set(pluck(self.threads,
'status')) == {WorkerStatus.FINISHED}:
break
time.sleep(update_period)
async def get_my_all_jobs(token: str = Header(None)):
authenticated_user = verify_token(token)
if not authenticated_user:
response = {'detail': 'UNAUTHORIZED ACCESS', 'status': 401}
return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED,
content=response)
my_jobs = session.query(AppliedJob).filter(
AppliedJob.c.user_id == authenticated_user['user_id'])
if my_jobs.count() == 0:
response = {'detail': 'JOBS NOT FOUND', 'status': 404}
return JSONResponse(status_code=status.HTTP_404_NOT_FOUND,
content=response)
job_ids = pluck(my_jobs, 'job_id')
job_ids = set(job_ids)
total_jobs = len(job_ids)
jobs = session.query(Job).filter(Job.c.id.in_(job_ids))
if jobs.count() == 0:
response = {'detail': 'JOBS NOT FOUND', 'status': 404}
return JSONResponse(status_code=status.HTTP_404_NOT_FOUND,
content=response)
my_all_jobs = []
for my_job in my_jobs:
for job in jobs:
if my_job.job_id == job.id:
data = {
'my_status':
my_job.status.value if my_job.status else 'NA',
'job_details': {
'id': job.id,
'company_name': job.company_name,
'job_title': job.job_title,
'job_type': job.job_type.value,
'experiance_min': job.experiance_min,
'experiance_max': job.experiance_max,
'job_count': job.job_count,
'location': job.location,
'job_status': job.status.value if job.status else 'NA',
}
}
my_all_jobs.append(data)
response = {'Total': total_jobs, 'Jobs': my_all_jobs, 'status': 200}
return JSONResponse(status_code=status.HTTP_200_OK, content=response)
def test_tile_list(self):
game = Game.objects.create()
board = Board.objects.create(game=game)
simple_tile_group = TileGroup.objects.create(board=board)
colored_tile_group = TileGroup.objects.create(board=board)
chance_tile_group = TileGroup.objects.create(board=board)
chancellery_tile_group = TileGroup.objects.create(board=board)
SimpleTile.objects.create(tile_group=simple_tile_group)
ColoredTile.objects.create(tile_group=colored_tile_group)
ChanceTile.objects.create(tile_group=chance_tile_group)
ChancelleryTile.objects.create(tile_group=chancellery_tile_group)
response = self.client.get("/api/tiles/")
self.assertEqual(
pluck(response.json(), "resourcetype"),
["SimpleTile", "ColoredTile", "ChanceTile", "ChancelleryTile"],
)
def fuel_report_trip(trip_id, p):
report = {'trip_key': trip_id}
prev = None
p.sort(key=lambda x: x['trip_sequence'])
for r in p:
r.update(calc_extra(r, prev))
prev = r
fms_spd = 'FMS_CRUISE_CONTROL_VEHICLE_SPEED (km/h)'
gps_spd = 'spd_over_grnd'
duration = (p[-1]['timestamp'] - p[0]['timestamp']).total_seconds()
speeds_fms = np.array(pluck(p, fms_spd, default=0))
speeds = np.array(pluck(p, gps_spd, default=0))
fuel_rates = np.array(pluck(p, 'FMS_FUEL_ECONOMY (L/h)', default=0))
# durations = np.array(pluck(p, '_duration', default=0)) / 3600
# fuels = fuel_rates * durations # should be in ml
# if not any(fuel_rates):
# fuel_rates = np.array(pluck(p, 'Petrol Used (ml)', default=0)) / 1000 / durations
fuels = pluck(p, 'Petrol Used (ml)', default=0)
co2s = pluck(p, 'Total CO2e (g)', default=0)
idle_time = 0
for r in p:
if gps_spd in r and '_duration' in r and r[gps_spd] < 2:
idle_time += r['_duration']
energy = np.array(pluck(p, 'Total Energy (kWh)', default=0))
report.update({
'vid': p[0]['vid'],
'Start Time': p[0]['timestamp'],
'Finish Time': p[-1]['timestamp'],
'Duration (s)': duration,
'Idle Duration (s)': idle_time,
'Distance (km)': sum(
vincenty(r1['pos']['coordinates'], r2['pos']['coordinates']).kilometers for r2, r1 in zip(p, p[1:])),
'GPS Min Speed (km/h)': np.min(speeds),
'GPS Max Speed (km/h)': np.max(speeds),
'GPS Mean Speed (km/h)': np.mean(speeds),
'GPS STDEV Speed (km/h)': np.std(speeds),
'FMS Min Speed (km/h)': np.min(speeds_fms),
'FMS Max Speed (km/h)': np.max(speeds_fms),
'FSM Mean Speed (km/h)': np.mean(speeds_fms),
'FMS STDEV Speed (km/h)': np.std(speeds_fms),
'Total Fuel (ml)': np.sum(fuels),
'Min Fuel rate (L/h)': np.min(fuel_rates),
'Max Fuel Rate (L/h)': np.max(fuel_rates),
'Mean Fuel Rate (L/h)': np.mean(fuel_rates),
'STDEV Fuel Rate (L/h)': np.std(fuel_rates),
'Mean CO2 (g)': np.mean(co2s),
'STDEV CO2 (g)': np.std(co2s),
'Total CO2e (g)': np.sum(co2s),
'Total Energy (kWh)': energy.sum(),
'% Idle': idle_time / duration * 100
})
return report
def main():
import matplotlib.pyplot as plt
# get all the bus data
conf = configparser.ConfigParser()
conf.read('../../development.ini')
uri = conf['app:webcan']['mongo_uri']
conn = MongoClient(uri)['webcan']
filtered_trips = pluck(conn.webcan_trip_filters.find(), 'trip_id')
# vid_re = '^adl_metro_1905'
query = {
# 'vid': {'$in': ['adl_metro_1905','adl_metro_1901', 'adl_metro_2451', 'adl_metro_2450']},
'Distance (km)': {
'$gte': 5
}
}
if filtered_trips:
query['trip_id'] = {'$nin': filtered_trips}
data = list(conn['trip_summary'].find(query))
series = defaultdict(lambda: {'x': [], 'y': []})
xf = 'Total Fuel (ml)'
yf = 'Total CO2e (g)'
for trip_summary in data:
fuel_litres = trip_summary[xf] * 0.001
dist_100km = trip_summary['Distance (km)'] / 100.
fuel_usage_rate_per_100km = fuel_litres / dist_100km
series[trip_summary['vid']]['x'].append(fuel_usage_rate_per_100km)
series[trip_summary['vid']]['y'].append(trip_summary[yf])
colors = cm.rainbow(np.linspace(0, 1, len(series)))
it = cycle(colors)
for vid, xy in series.items():
x, y = np.array(xy['x']), np.array(xy['y'])
color = next(it)
plt.scatter(x, y, label="{} ({})".format(vid, len(y)), color=color)
plt.legend()
plt.xlabel(xf)
plt.ylabel(yf)
allys = np.array(sum((vals['y'] for vals in series.values()), []))
plt.title("{} vs {} n={}".format(xf, yf, allys.size))
plt.show()
def update(self, customer, validated_data):
addresses = validated_data.pop("addresses")
primary_addresses = filter(
lambda address: address.get("is_primary", False), addresses)
if len(list(primary_addresses)) > 1:
raise ValidationError("There are more than one primary address")
customer = super().update(customer, validated_data)
customer.addresses.exclude(
id__in=pluck(addresses, "id", default=None)).delete()
for address in addresses:
address["customer_id"] = customer.id
Address.objects.update_or_create(id=address.get("id"),
defaults=address)
return customer
def crash_in_polygon(request):
with _get_mongo_client() as client:
crashes_collection = client[mongo_database]['crashes']
points = request.json_body
# make sure it's a list of lists of floats
points.append(points[0])
crashes = crashes_collection.find({'loc': {
'$geoWithin': {
'$geometry': {
'type': 'Polygon',
'coordinates': [points]
}
}
}})
readings_collection = client[mongo_database]['readings']
crashes = list(crashes)
td = timedelta(minutes=5)
for i, crash in enumerate(crashes):
# find the nearest 2 intersections
# and get the readings for the downstream one
sites = client[mongo_database]['locations'].find({
'loc': {
'$geoNear': {
'$geometry': crash['loc']
}
}
}).limit(2)
sites = pluck(list(sites), 'intersection_number')
readings = readings_collection.find({
'datetime': {'$gte': crash['datetime'] - td, '$lte': crash['datetime'] + td},
'site_no': {'$in': sites}
}).limit(6).sort([['site_no', pymongo.ASCENDING], ['datetime', pymongo.ASCENDING]])
crashes[i]['readings'] = list(readings)
crashes[i]['sites'] = sites
return {
'crashes': crashes
}
def show_group(request):
# a regular user will have their groups listed
# admin will see all groups
user = request.user
group = request.db_session.query(LedGroup).filter(LedGroup.id == request.matchdict['group_id']).first()
users = request.db_session.query(LedGroupUser).filter(LedGroupUser.led_group == group).all()
if not (user.admin or user.id in pluck(users, 'led_user_id')):
raise exc.HTTPForbidden("Only site admins or group members can view this")
schedule = request.db_session.query(LedSchedule).filter(LedSchedule.led_group == group).order_by(LedSchedule.position.asc()).all()
subquery = request.db_session.query(LedGroupUser.led_user_id).filter(LedGroupUser.led_group == group)
other_users = request.db_session.query(LedUser).filter(~LedUser.id.in_(subquery))
subquery = request.db_session.query(LedSchedule.led_plugin_id).filter(LedSchedule.led_group == group)
# other_plugins = request.db_session.query(LedPlugin).filter(~LedPlugin.id.in_(subquery))
other_plugins = request.db_session.query(LedPlugin)
return {
'group': group,
'users': users,
'schedule': schedule,
'other_users': other_users,
'other_plugins': other_plugins,
'group_admin': can_modify_group(request, group.id, False)
}
def export(results, props):
result = 'name\t' + '\t'.join(pluck(results, 'name')) + '\n'
for p in props:
result += p + '\t' + '\t'.join(['%1.3f' % x for x in pluck(results, p)]) + '\n'
return result
def get_diagrams(intersections):
with _get_mongo_client() as client:
db = client[mongo_database]
intersections = pluck(intersections, 'intersection_number')
return db['locations'].find({'intersection_number': {'$in': intersections}, 'scats_diagram': {'$exists': True}})
def avgStdSpeed(l):
speeds = pluck(l, speed)
return np.mean(speeds), np.std(speeds)
from pluck import pluck, ipluck
from datetime import datetime
# is the simplest way of plucking “fields” from an iterable of values.
# “Fields” are either item.field or item[field]. Pluck tries both,
# in that order. If nothing is found, and no default value is specified,
# it throws an exception.
#pluck(iterable, key)
#pluck(iterable, *keys)
dates = [
datetime(2012, 10, 22, 12, 00),
datetime(2012, 10, 22, 15, 14),
datetime(2012, 10, 22, 21, 44),
]
print(pluck(dates, 'day'))
#[22, 22, 22]
print(pluck(dates, 'hour'))
#[12, 15, 21]
#It also works on dictionary-like access (__getitem__):
objects = [
{
'id': 282,
'name': 'Alice',
'age': 30,
'sex': 'female'
},
{
'id': 217,
'name': 'Bob',
def phase_classify_csv_render(request):
query = {
# 'timestamp': {'$exists': True, '$ne': None},
}
min_phase_time = int(request.POST.get('min-phase-seconds'))
cruise_window = int(request.POST.get('cruise-window'))
if request.POST.get('select-trips'):
query['trip_id'] = {'$in': request.POST.get('select-trips').split(',')}
else:
query['trip_id'] = {
'$nin':
pluck(
request.db.webcan_trip_filters.find({
'vid': {
'$in': request.POST.get('select-trips').split(',')
}
}), 'trip_id')
}
if request.POST.get('select-vids'):
query['vid'] = {'$in': request.POST.get('select-vids').split(',')}
if any(request.POST.getall('map-hull')):
query['pos'] = {
'$geoWithin': {
'$geometry': {
'type': 'Polygon',
'coordinates': request.POST.getall('map-hull')
}
}
}
print(request.POST)
print(query)
cursor = request.db.rpi_readings.find(query, {
'_id': False
}).sort(SORT_TRIP_SEQ)
trips = groupby(cursor, lambda x: x['trip_key'])
rows = []
for trip_id, readings in trips:
readings = list(readings)
prev = None
for i in readings:
i.update(calc_extra(i, prev))
prev = i
_classify_readings_with_phases_pas(readings, min_phase_time,
cruise_window)
phase_report = per_phase_report(readings, min_phase_time)
rows.extend(phase_report)
if len(rows) == 0:
raise exc.HTTPBadRequest('No data returned for query')
headers = rows[0].keys()
data = {
'header': headers,
'rows': rows,
}
renderer_obj = get_renderer('csv')
request.response.content_type = renderer_obj.content_type
request.response.content_disposition = 'attachment;filename=phase_report_{}.{}'.format(
datetime.now().strftime('%Y%m%d_%H%M%S'),
renderer_obj.content_type.split('/')[1])
return render_to_response(renderer_name='csv',
value=data,
request=request,
response=request.response)
def gen_summary_report_for_vehicle(vid):
# get all distinct trip_keys for this vehicle
# trip_keys = list([x.split('_')[2] for x in request.db.rpi_readings.distinct('trip_id', {'vid': vid})])
# get all the trip_summary data, if not exists, generate
report = dict(trips=0,
distance=0,
time=0,
first=datetime(9999, 1, 1),
last=datetime(2000, 1, 1))
report['vehicle'] = request.db.webcan_devices.find_one({'name': vid}, {
'_id': 0,
'secret': 0
})
def merge(trip_info):
x, trip_id = trip_info
if x['distance'] >= min_trip_distance:
for k, v in x.items():
report[k] += v
else:
excluded.append(trip_id)
filtered_trips = pluck(
request.db.webcan_trip_filters.find({'vid': vid}), 'trip_id')
pool = Pool()
query = {
'vid': vid,
'trip_key': {
'$nin': filtered_trips
},
'Distance (km)': {
'$gte': min_trip_distance
},
'Fuel Economy (L/100km)': {
'$gt': 1,
'$lt': 100
}
}
vehicle_obj = request.db.webcan_devices.find_one({'name': vid})
if vehicle_obj is not None and vehicle_obj['type'] == 'Electric':
del query['Fuel Economy (L/100km)']
for trip_summary in request.db.trip_summary.find(query, {'phases': 0}):
# if trip_summary is None:
# cursor = request.db.rpi_readings.find({
# 'vid': vid,
# 'trip_id': {'$nin': filtered_trips}
# }, {'_id': False}).sort('trip_key', 1)
# trips = groupby(cursor, lambda x: x['trip_key'])
# print("Doing summary for", trip_key)
# for trip_id, readings in trips:
# # summarise_trip(trip_id, list(readings), report)
# lreadings = list(readings)
# report['first'] = min(lreadings[0]['timestamp'], report['first'])
# report['last'] = max(lreadings[-1]['timestamp'], report['last'])
# pool.apply_async(summarise_trip, args=(trip_id, lreadings), callback=merge, error_callback=print)
# else:
report['first'] = min(trip_summary['Start Time'], report['first'])
report['last'] = max(trip_summary['Finish Time'], report['last'])
# print("Used cached", trip_key)
ts = {
'trips': 1,
'distance': trip_summary['Distance (km)'],
'time': trip_summary['Duration (s)']
}
merge((ts, trip_summary['trip_key']))
pool.close()
pool.join()
return report
def summary_report_do(request):
min_trip_distance = float(request.POST.get('min-trip-distance', 5))
excluded = []
def gen_summary_report_for_vehicle(vid):
# get all distinct trip_keys for this vehicle
# trip_keys = list([x.split('_')[2] for x in request.db.rpi_readings.distinct('trip_id', {'vid': vid})])
# get all the trip_summary data, if not exists, generate
report = dict(trips=0,
distance=0,
time=0,
first=datetime(9999, 1, 1),
last=datetime(2000, 1, 1))
report['vehicle'] = request.db.webcan_devices.find_one({'name': vid}, {
'_id': 0,
'secret': 0
})
def merge(trip_info):
x, trip_id = trip_info
if x['distance'] >= min_trip_distance:
for k, v in x.items():
report[k] += v
else:
excluded.append(trip_id)
filtered_trips = pluck(
request.db.webcan_trip_filters.find({'vid': vid}), 'trip_id')
pool = Pool()
query = {
'vid': vid,
'trip_key': {
'$nin': filtered_trips
},
'Distance (km)': {
'$gte': min_trip_distance
},
'Fuel Economy (L/100km)': {
'$gt': 1,
'$lt': 100
}
}
vehicle_obj = request.db.webcan_devices.find_one({'name': vid})
if vehicle_obj is not None and vehicle_obj['type'] == 'Electric':
del query['Fuel Economy (L/100km)']
for trip_summary in request.db.trip_summary.find(query, {'phases': 0}):
# if trip_summary is None:
# cursor = request.db.rpi_readings.find({
# 'vid': vid,
# 'trip_id': {'$nin': filtered_trips}
# }, {'_id': False}).sort('trip_key', 1)
# trips = groupby(cursor, lambda x: x['trip_key'])
# print("Doing summary for", trip_key)
# for trip_id, readings in trips:
# # summarise_trip(trip_id, list(readings), report)
# lreadings = list(readings)
# report['first'] = min(lreadings[0]['timestamp'], report['first'])
# report['last'] = max(lreadings[-1]['timestamp'], report['last'])
# pool.apply_async(summarise_trip, args=(trip_id, lreadings), callback=merge, error_callback=print)
# else:
report['first'] = min(trip_summary['Start Time'], report['first'])
report['last'] = max(trip_summary['Finish Time'], report['last'])
# print("Used cached", trip_key)
ts = {
'trips': 1,
'distance': trip_summary['Distance (km)'],
'time': trip_summary['Duration (s)']
}
merge((ts, trip_summary['trip_key']))
pool.close()
pool.join()
return report
"""
Summary is a dict of
vehicle_id: {
'trips': int,
'distance': float,
'time': float,
'petrol_used': float
}
"""
summary = {}
# group everything by trip_id
vids = request.POST.getall('devices[]')
if not vids:
return exc.HTTPBadRequest('Please enter at least 1 vehicle')
for vid in vids:
summary[vid] = gen_summary_report_for_vehicle(vid)
# print(summary)
vals = list(summary.values())
summary['Aggregate'] = {
key: sum(pluck(vals, key))
for key in ['trips', 'distance', 'time']
}
summary['Aggregate']['last'] = max(pluck(vals, 'last'))
summary['Aggregate']['first'] = min(pluck(vals, 'first'))
return {'summary': summary, 'excluded': excluded}
def process_readings(readings, intersection, write_anomaly, progress=True, multi_model=False, smoothing=0):
counter = 0
total = readings.count(True)
if multi_model:
loc = locations_collection.find_one({'intersection_number': intersection})
models = {}
for sensor in loc['sensors']:
models[sensor] = Worker(sensor, intersection)
models[sensor].start()
else:
model = createModel(intersection)
anomaly_likelihood_helper = anomaly_likelihood.AnomalyLikelihood(1000, 200)
if model is None:
print "No model could be made for intersection", intersection
return
pfield = model.getInferenceArgs()['predictedField']
encoders = get_encoders(model)
if progress:
progBar = pyprind.ProgBar(total, width=50)
_smoothing = smoothing >= 1
if _smoothing:
previous = deque(maxlen=smoothing)
for i in readings:
counter += 1
if progress:
progBar.update()
timestamp = i['datetime']
if multi_model:
predictions, anomalies = {}, {}
for sensor, proc in models.iteritems():
vc = i['readings'][sensor]
if vc > max_vehicles:
vc = None
elif _smoothing and len(previous):
vc = (vc + sum(pluck(sensor, previous)))/float(len(previous) + 1)
fields = {"timestamp": timestamp, sensor: vc}
proc.queue_in.put(fields)
for sensor, proc in models.iteritems():
result = proc.queue_out.get()
# (self.sensor, prediction, anomaly_score, likelihood)
anomalies[result[0]] = {'score': result[2], 'likelihood': result[3]}
predictions[result[0]] = result[1]
else:
fields = {"timestamp": timestamp}
for p, j in enumerate(i['readings'].items()):
if j[0] not in encoders:
continue
vc = j[1]
if vc > max_vehicles:
vc = None
fields[j[0]] = vc
result = model.run(fields)
prediction = result.inferences["multiStepBestPredictions"][1]
anomaly_score = result.inferences["anomalyScore"]
predictions = {pfield: prediction}
likelihood = anomaly_likelihood_helper.anomalyProbability(
i['readings'][pfield], anomaly_score, timestamp)
anomalies = {pfield: {'score': anomaly_score, 'likelihood': likelihood}}
if write_anomaly:
write_anomaly_out(i, anomalies, predictions)
if _smoothing:
previous.append(i['readings'])
locations_collection.find_one_and_update({'intersection_number': intersection}, {'$unset': {'running': ''}})
if multi_model:
for proc in models.values():
proc.terminate()
else:
save_model(model, intersection)
if progress:
print
print "Read", counter, "lines"
assert duration >= 0.1, "To ensure machine safety, duration must be greater than 100 ms"
print 'Watching memory usage (max: {limit} MB, check interval: {duration:1.2f} ms)'.format(limit=limit, duration=float(duration*1000))
while True:
output = subprocess.check_output("ps aux --sort '-rss' --cols 200 | head -n 200", shell=True)
lines = output.splitlines()
headers = re.split(r'\s+', lines[0], 10)
del lines[0]
processes = list()
for line in lines:
attrs = re.split(r'\s+', line, 10)
processes.append(dict(zip(headers, attrs)))
rsss = [int(x) / 1000 for x in pluck(processes, 'RSS')]
pids = pluck(processes, 'PID')
pairs = dict(zip(pids, rsss))
measurements.append(pairs)
if len(measurements) > history_size:
del measurements[0]
top = dict(
pid=pids[0],
rss=rsss[0],
name=get_name_by_pid(pids[0])
)
measure_index = (measure_index + 1) % 2
print '\r ' + points[measure_index] + ' Top process "{name}:{pid}" with {rss} MB'.format(**top),
sys.stdout.flush()
def refresh_schedule():
"""
Get the latest scheduling from the database
The next plugin to run is at the right
:return:class Runner:
def __init__(self, board_dimensions):
self.board_dimensions = board_dimensions
import numpy as np
self.np = np
self.pixels = self.np.random.normal(128, 128, (self.board_dimensions[0], self.board_dimensions[1], 3)).astype(self.np.uint8)
def run(self):
# shift everything to the left, put a new random column on the end
self.pixels = self.np.roll(self.pixels, 1, 0)
col = self.np.random.normal(128, 128, [17, 3]).astype(dtype=self.np.uint8)
for idx, i in enumerate(col):
self.pixels[0][idx] = i
self.pixels.sort(1)
return self.pixels
"""
global current_plugin
global do_exception
#print "Updating schedule"
try:
connection = get_connection()
except pymysql.err.OperationalError as e:
print e
print "Using default schedule"
return test_sched()
cursor = connection.cursor()
# need to fix this so that it loads the right schedule
#
group_id = get_current_schedule(connection)
if group_id is None:
return test_sched()
sql = """SELECT * FROM `led_schedule`
LEFT OUTER JOIN led_plugin
ON led_schedule.led_plugin_id = led_plugin.id
WHERE
`enabled` = 1 AND
`led_group_id` = {}
ORDER BY `position` DESC """.format(group_id['id'])
cursor.execute(sql)
schedule.clear()
cursor.close()
for row in cursor:
schedule.append(row)
cursor.close()
cursor = connection.cursor()
sql = """SELECT * FROM `led_skip`"""
cursor.execute(sql)
if cursor.fetchone():
do_exception = True
cursor.execute("TRUNCATE TABLE `led_skip`")
cursor.close()
connection.close()
# if there schedule has > 2 elems,
# roll the schedule until we get old_schedule[0] at the start
if current_plugin and len(schedule) > 2 and current_plugin['id'] in pluck(schedule, 'id'):
while schedule[-1]['id'] != current_plugin['id']:
schedule.rotate(-1)
else:
schedule.rotate(-1)
# show_schedule(schedule)
return schedule
def per_phase_report(readings, min_duration=5):
"""
For a bunch of readings, calculate some stats about every phase it did,
splitting up by trip_id
:param readings:
:return:
"""
phases = []
for idx, phase_group in enumerate([(key, list(group)) for key, group in (
groupby(readings, key=lambda x: f"{x['phase']}:{x['trip_id']}"))]):
phase_type = phase_group[0].split(':')[0]
phase_no = idx
p = phase_group[1]
duration = (p[-1]['timestamp'] - p[0]['timestamp']).total_seconds()
if duration < min_duration:
continue
if len(p) < 2:
continue
speeds = pluck(p, 'speed')
fuel_rates = np.array(pluck(p, 'FMS_FUEL_ECONOMY (L/h)',
default=0)) / 1000
durations = np.array(pluck(p, '_duration', default=1e-9)) / 3600
fuels = fuel_rates * durations # should be in ml
if not any(fuel_rates):
fuel_rates = np.array(pluck(p, 'Petrol Used (ml)',
default=0)) / 1000 / durations
fuels = pluck(p, 'Petrol Used (ml)', default=0)
co2s = pluck(p, 'Total CO2e (g)', default=0)
accels = [(s2['speed'] - s1['speed']) /
((s2['timestamp'] - s1['timestamp']).total_seconds())
for s1, s2 in zip(p, p[1:])]
times = pluck(p, 'timestamp')
slope, intercept, r_value, p_value, std_err = stats.linregress(
np.sqrt([(x - times[0]).total_seconds() for x in times]), speeds)
energy = np.array(pluck(p, 'Total Energy (kWh)', default=0))
phases.append({
'phasetype':
int(phase_type),
'phase_no':
phase_no,
'trip_id':
p[0]['trip_id'],
'Start Time':
p[0]['timestamp'],
'Finish Time':
p[-1]['timestamp'],
'Duration (s)':
duration,
'Avg Temp (°C)':
np.mean(pluck(p, 'FMS_ENGINE_TEMP (°C)', default=0)),
'Distance (km)':
sum(
vincenty(r1['pos']['coordinates'][::-1], r2['pos']
['coordinates'][::-1]).kilometers
for r2, r1 in zip(p, p[1:])),
'Start Speed (km/h)':
speeds[0],
'Finish Speed (km/h)':
speeds[-1],
'Min Speed (km/h)':
np.min(speeds),
'Max Speed (km/h)':
np.max(speeds),
'Mean Speed (km/h)':
np.mean(speeds),
'STDEV Speed (km/h)':
np.std(speeds),
'Coeff Beta (km/h)/√(Δt)':
slope,
'Y Intercept (km/h)':
intercept,
'r_squared_value':
r_value**2,
# 'p_value': p_value,
'Min Acc ((Δkm/h)/s)':
np.min(accels),
'Max Acc ((Δkm/h)/s)':
np.max(accels),
'Mean Acc ((Δkm/h)/s)':
np.mean(accels),
'Total Acc ((Δkm/h)/s)': (p[-1]['speed'] - p[0]['speed']) /
duration, # change in speed km/h
'STDEV Acc ((Δkm/h)/s)':
np.std(accels),
'Total Fuel (ml)':
np.sum(fuels),
'Min Fuel rate (L/h)':
np.min(fuel_rates),
'Max Fuel Rate (L/h)':
np.max(fuel_rates),
'Mean Fuel Rate (L/h)':
np.mean(fuel_rates),
'STDEV Fuel Rate (L/h)':
np.std(fuel_rates),
# 'Mean NOX': 0,
# 'STDEV NOx': 0,
# 'Mean HC': 0,
# 'STDEV HC': 0,
# 'Mean CH4': 0,
# 'STDEV CH4': 0,
# 'Mean CO': 0,
# 'STDEV CO': 0,
'Mean CO2 (g)':
np.mean(co2s),
'STDEV CO2 (g)':
np.std(co2s),
'Total CO2 (g)':
np.sum(co2s),
# 'Mean FC': 0,
# 'STDEV FC': 0,
# 'Mean PM': 0,
# 'STDEV PM': 0,
# 'Mean OP': 0,
# 'STDEV OP': 0
'Min Energy (kWh)':
energy.min(),
'Max Energy (kWh)':
energy.max(),
'Mean Energy (kWh)':
energy.mean(),
'STDEV Energy (kWh)':
energy.std(),
'Total Energy (kWh)':
energy.sum()
})
return phases
row_count += 1
return step_predictions, data, model, it, row_count, len(data)
if __name__ == "__main__":
import sys
for i in sys.argv[1:]:
print("Running ", i)
fname = i.split('/')[-1]
predictions, data, model, it, row_count, data_len = run_data(i, limit=datetime(2013, 4, 23), sensors=[5])
model.save('/scratch/model_store/3002_model_sensor_5')
# turn the data into numpy arrays
split_idx = int(len(data) * 0.4)
flow_values = np.array(pluck(data[split_idx:], 'downstream'))
print()
# print (predictions)
predictions = {
k: np.array(v[split_idx:]) for k, v in predictions.items()
}
print()
table = []
print(' & '.join(['step', 'geh', 'mape', 'rmse'])+' \\\\')
for step in steps:
# true values
stepped_vals = flow_values[step:len(predictions)]
# predicted values
pred_vals = predictions[step][:-step] + eps
def getSecurityHistory(strEspecie):
print('fetching security data...')
strEspecie = strEspecie.upper()
strFilename = 'data/SecurityHistory/' + strEspecie + '.json'
if (os.path.exists(strFilename)):
print('from file: ' + strFilename)
with open(strFilename) as jsonFile:
jsondata = json.load(jsonFile)
else:
print('from bolsar.com: ' + strEspecie)
body = '{"strEspecie":"' + strEspecie + '","intVto":"4","intPeriodoId":4}'
headers = {"Content-type": "application/json",
"Accept": "application/json"}
conn = httplib.HTTPSConnection("www.bolsar.com")
conn.request("POST", "/VistasDL/PaginaIntradiarioEspecies.aspx/GetIntradiarioHistorico", body, headers)
response = conn.getresponse()
print response.status, response.reason
if 200 == response.status:
jsondata = json.load(response)
conn.close()
with open(strFilename, 'w') as outfile:
json.dump(jsondata, outfile)
outfile.close()
else:
return
series = jsondata['d'].pop()['EspeciesSeries']
# all dates represented here are at GMT-3 (Buenos Aires)
intDates = map(dateToInt, pluck(series, 'Fecha'))
intTimes = map(timeToInt, pluck(series, 'Hora'))
intTimestamps = np.sum([np.array(intDates).astype(np.float), np.array(intTimes).astype(np.float)], axis=0)
datetimeDates = map(timestampToDatetime, intTimestamps)
strDates = map(timestampToStr, intTimestamps)
data = np.array([
intTimestamps,
pluck(series, 'VolumenNominal'),
pluck(series, 'PrecioUltimo'),
pluck(series, 'PrecioCierre'),
pluck(series, 'PrecioOperacion'),
pluck(series, 'PrecioApertura'),
pluck(series, 'PrecioMaximo'),
pluck(series, 'PrecioMinimo'),
pluck(series, 'VariacionPrecio'),
pluck(series, 'Operaciones'),
pluck(series, 'TotalOperadoVn'),
pluck(series, 'TotalOperadoMonto')
#datetimeDates,
#strDates,
])
data = data.transpose()
data = data[data[:,0].argsort()] # sort data by column 0 (timepstamps)
return data
def plot(x_title,
y_title,
plt_title,
# xlim,
# ylim
):
# get all the bus data
conf = configparser.ConfigParser()
conf.read('../../development.ini')
uri = conf['app:webcan']['mongo_uri']
conn = MongoClient(uri)['webcan']
bus = 'adl_metro_1999'
readings_col = conn['rpi_readings']
trip_key = conn['trip_summary'].find_one({'vid': bus, 'Distance (km)': {'$gte': 5}})['trip_key']
readings = []
for _r in readings_col.find({
'trip_key': trip_key,
'vid': bus,
}):
# filter out the garbage
rpm = _r.get('FMS_ELECTRONIC_ENGINE_CONTROLLER_1 (RPM)')
if rpm is not None and rpm > 8000:
continue
else:
readings.append(_r)
readings.sort(key=lambda r: r['trip_sequence'])
prev = None
for p in readings:
p.update(calc_extra(p, prev))
prev = p
_classify_readings_with_phases_pas(readings, 3, 1)
accel_phases = []
for g, vals in groupby(readings, key=lambda x: x['phase']):
vals = list(vals)
if g == 1 and (vals[-1]['timestamp'] - vals[0]['timestamp']).total_seconds() > 5:
accel_phases.append(vals)
# pick the an accel phase somewhere
phase = accel_phases[0]
x, y = [], []
start_time = phase[0]['timestamp']
for r in phase:
x.append((r['timestamp'] - start_time).total_seconds())
y.append(r['spd_over_grnd'])
x = np.array(x)
y = np.array(y)
times = pluck(phase, 'timestamp')
sqrt_x = np.sqrt([(t - times[0]).total_seconds() for t in times]),
bus_short = bus.split('_')[-1]
plt.figure()
plt.scatter(x, y, s=4, label='Speed Over time')
plt.scatter(sqrt_x, y, s=4, label='sqrt speed over time')
trend = stats.linregress(x, y)
slope, intercept, r_value, p_value, std_err = trend
plt.plot(x, intercept + slope * x,
label=trend_lbl(bus_short, len(x), slope, intercept, r_value),
color='k',
path_effects=[pe.Stroke(linewidth=5, foreground='r'), pe.Normal()]
)
# plt.ylim(*ylim)
# plt.xlim(*xlim)
plt.xlabel(x_title)
plt.ylabel(y_title)
plt.legend()
plt.title(plt_title + bus_short)
plt.savefig('./out/' + plt_title + bus_short + '.png')
plt.show()
def test_empty_iterables(self):
assert pluck([], 'foo') == []
assert set(pluck(set(), 'foo')) == set()
