def get_fake_points(origin, destination):
all_points.append(origin)
params = { 'origin': "%s,%s" % (origin[LAT], origin[LON]),
'destination': "%s,%s" % (destination[LAT], destination[LON]),
'mode': "driving",
# 'avoid': "highways",
'sensor': "false"
}
params = net.urlencode(params)
url = "http://maps.googleapis.com/maps/api/directions/json?" + params
response = net.read(url)
directions = json.loads(response)
try:
steps = directions['routes'][0]['legs'][0]['steps']
except Exception as e:
print(response)
exit()
points = []
for i, step in enumerate(steps):
instruction = step['html_instructions']
instructions.append(instruction)
if i != 0:
point = [step['start_location']['lng'], step['start_location']['lat'], None, None, None]
points.append(point)
if i != len(steps) - 1:
point = [step['end_location']['lng'], step['end_location']['lat'], None, None, None]
points.append(point)
for point in points:
point[X] = util.scale(point[LON], min_lon, max_lon)
point[Y] = util.scale(point[LAT], min_lat, max_lat)
all_points.append(point)
all_points.append(destination)
return points
def run(self):
while True:
value, t, duration = self.queue.get()
data = [{'device': config['device'], 'kind': "motion", 'value': value, 't': t, 'duration': duration}]
try:
response = net.read("http://%s:%s" % (config['server']['host'], config['server']['port']), json.dumps(data))
log.info(response)
except Exception as e:
log.error(log.exc(e))
def upload(self, t, filename):
log.info("upload %s" % filename)
try:
s3.upload(filename)
log.info("--> uploaded. Pinging server...")
data = {'t': t}
response = net.read("http://%s:%s" % (config['server']['host'], config['server']['port']), json.dumps(data).encode('utf-8'))
log.info(response)
os.remove(filename)
except Exception as e:
log.error(log.exc(e))
def handle(t, data):
stations = model.fetch_stations()
events = {}
for s in data["results"]:
try:
if s["id"] not in stations:
data = json.loads(net.read(LOOKUP, {"lat": s["latitude"], "lng": s["longitude"], "username": "******"}))
zipcode = data["postalCodes"][0]["postalCode"]
model.insert_station(s["id"], s["longitude"], s["latitude"], zipcode, t, s["availableBikes"])
continue
station = stations[s["id"]]
if s["availableBikes"] != station["bikes"] and t > station["t"]:
json.dumps(s, indent=4)
model.update_station(s["id"], t, s["availableBikes"])
events[s["id"]] = s["availableBikes"] - station["bikes"]
except Exception as e:
log.error(log.exc(e))
model.insert_beat(t, events)
model.update_station(s["id"], t, s["availableBikes"])
events[s["id"]] = s["availableBikes"] - station["bikes"]
except Exception as e:
log.error(log.exc(e))
model.insert_beat(t, events)
if len(sys.argv) > 1:
for filename in os.listdir("files"):
log.info("Parsing %s" % filename)
t = int(filename.split(".")[0])
if model.check_t(t):
log.info("skipping...")
continue
try:
with open("files/" + filename) as f:
data = json.loads(f.read())
except Exception as e:
log.error(log.exc(e))
exit()
handle(t, data)
else:
log.info("Grabbing from endpoint...")
try:
t = time.time()
data = json.loads(net.read(ENDPOINT))
except Exception as e:
log.error(log.exc(e))
exit()
handle(t, data)
total_venues = 0
for anchor in anchors:
log.info("checking for %s" % anchor)
# params = {'intent': "browse", 'll': anchor, 'radius': 8000, 'limit': 50, 'client_id': config['foursquare']['key'], 'client_secret': config['foursquare']['secret'], 'v': "20130704"}
# params = net.urlencode(params)
# request_string = "https://api.foursquare.com/v2/venues/search?%s" % params
params = {'ll': anchor, 'radius': 8000, 'limit': 50, 'time': "any", 'day': "any", 'client_id': config['foursquare']['key'], 'client_secret': config['foursquare']['secret'], 'v': "20130704"}
params = net.urlencode(params)
request_string = "https://api.foursquare.com/v2/venues/explore?%s" % params
try:
response = net.read(request_string)
except Exception as e:
log.error(log.exc(e))
continue
data = json.loads(response)
# print(json.dumps(data, indent=4))
try:
venues = data['response']['groups'][0]['items']
for venue in venues:
venue = venue['venue']
checkins = venue['stats']['checkinsCount']
if checkins == 0:
continue
if 'hereNow' in venue:
people = venue['hereNow']['count']
else:
import json, model, datetime, math
from housepy import config, log, net
log.info("////////// foursquare //////////")
update = []
checkins = 0
checkouts = 0
for venue in model.get_venues():
log.debug("checking %s" % venue['venue_id'])
try:
params = {'client_id': config['foursquare']['key'], 'client_secret': config['foursquare']['secret'], 'v': "20130704"}
params = net.urlencode(params)
request_string = "https://api.foursquare.com/v2/venues/%s/herenow?%s" % (venue['venue_id'], params)
response = net.read(request_string, timeout=1)
data = json.loads(response)
people = data['response']['hereNow']['count']
if people != venue['people']:
if people > venue['people']:
checkins += people - venue['people']
else:
checkouts += venue['people'] - people
venue['people'] = people
update.append(venue)
except Exception as e:
log.error(log.exc(e))
# print(json.dumps(update, indent=4))
model.update_venues(update)
#!/usr/bin/env python
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
import json, model
from housepy import config, log, net
log.info("////////// weather_station //////////")
try:
response = net.read("http://api.wunderground.com/api/%s/conditions/q/NY/Acra.json" % config['weather'])
data = json.loads(response)
model.insert_reading('server', 'heat', float(data['current_observation']['temp_f'])) # changing from feelslike to temp
rain_v = float(data['current_observation']['precip_today_in']) # occasional bad values
if abs(rain_v) > 50:
rain_v = 0.0
model.insert_reading('server', 'rain', rain_v, cumulative=True)
model.insert_reading('server', 'wind', (float(data['current_observation']['wind_mph']) + float(data['current_observation']['wind_gust_mph'])) * 0.5)
model.insert_reading('server', 'visibility', float(data['current_observation']['visibility_mi']))
model.insert_reading('server', 'humidity', float(data['current_observation']['relative_humidity'][:-1]))
except Exception as e:
log.error(log.exc(e))
exit()
"""
{
"current_observation": {
"heat_index_c": 31,
"local_tz_long": "America/New_York",
def process(t):
log.info("////////// process %s //////////" % t)
filename = "audio_tmp/%s.wav" % t
sample_rate, signal = wavfile.read(filename)
log.info("AUDIO SAMPLES %s" % len(signal))
log.info("SAMPLE RATE %s" % sample_rate)
duration = float(len(signal)) / sample_rate
log.info("AUDIO DURATION %ss" % util.format_time(duration))
signal = (np.array(signal).astype('float') / (2**16 * 0.5)) # assuming 16-bit PCM, -1 - 1
log.info("--> preprocessing")
magnitude = abs(signal)
thresholded_magnitude = (magnitude > THRESHOLD) * magnitude
# level = sp.smooth(thresholded_magnitude, size=10000) # shit -- smooth is too expensive for raspi
level = thresholded_magnitude
log.info("--> scanning")
TOLERANCE = sample_rate / 10 # within a tenth of a second, same sound (poor man's smoothing?)
indexes = []
maxes = []
durations = []
zeros = 0
on_chunk = False
for index, sample in enumerate(level):
if sample > 0.0:
if not on_chunk:
indexes.append(index)
durations.append(0)
maxes.append(0)
on_chunk = True
durations[-1] += 1
if sample > maxes[-1]:
maxes[-1] = sample
zeros = 0
if sample == 0.0:
if on_chunk:
zeros += 1
if zeros == TOLERANCE:
on_chunk = False
events = []
for i in xrange(len(indexes)):
value, t_, duration = maxes[i], t + int(float(indexes[i]) / sample_rate), float(durations[i]) / sample_rate
events.append((value, t_, duration))
for event in events:
log.debug(event)
if 'draw' in config and config['draw']:
from housepy import drawing
log.info("--> drawing")
ctx = drawing.Context(width=2000, height=500, background=(0., 0., 1.), hsv=True, flip=True, relative=True)
ctx.line([(float(i) / len(magnitude), sample) for (i, sample) in enumerate(magnitude)], thickness=1, stroke=(0., 0., 0.5))
ctx.line([(float(i) / len(thresholded_magnitude), sample) for (i, sample) in enumerate(thresholded_magnitude)], thickness=1, stroke=(0., 0., 0.))
ctx.line([(float(i) / len(level), sample) for (i, sample) in enumerate(level)], thickness=1, stroke=(0., 1., 1.))
level = sp.normalize(level)
ctx.line([(float(i) / len(level), sample) for (i, sample) in enumerate(level)], thickness=1, stroke=(0.15, 1., 1.))
ctx.line(0.0, THRESHOLD, 1.0, THRESHOLD, thickness=1, stroke=(0.55, 1., 1.))
ctx.show()
try:
data = []
for event in events:
value, t_, duration = event
data.append({'device': config['device'], 'kind': "sound", 'value': value, 't': t_, 'duration': duration})
response = net.read("http://%s:%s" % (config['server']['host'], config['server']['port']), json.dumps(data))
log.info(response)
except Exception as e:
log.error(log.exc(e))
if config['device'] != "Granu":
os.remove(filename)
def traverse(pd):
log.info("Checking %s..." % pd)
for i, filename in enumerate(os.listdir(pd)):
if filename[0] == ".":
continue
elif os.path.isdir(os.path.join(pd, filename)):
traverse(os.path.join(pd, filename))
elif filename[-3:] == "sml":
try:
log.info("Reading %s..." % os.path.join(pd, filename))
with open(os.path.join(pd, filename)) as f:
content = f.read()
except Exception as e:
log.error("Could not read file: %s" % log.exc(e))
else:
try:
log.info("Parsing...")
data = xmltodict.parse(content)
# log.debug(json.dumps(data, indent=4))
serial_number = str(data['sml']['DeviceLog']['Device']['SerialNumber'])
try:
member = config['ambits'][serial_number]
except KeyError:
log.warning("Ambit serial number not linked to a Member")
log.debug(serial_number)
log.debug(config['ambits'])
continue
log.info("Member: %s" % member)
samples = data['sml']['DeviceLog']['Samples']['Sample']
start_t = None
for s, sample in enumerate(samples):
if s == 0:
dt = util.parse_date(sample['UTC']) # these are marked UTC in the data
start_t = util.timestamp(dt)
sample['Member'] = member
if 'Satellites' in sample: # ingest satellite location data
try:
url = "%s/ingest/ambit_geo" % config['url']
log.info("Sending to %s..." % url)
response = net.read(url, str(json.dumps(sample)).encode('utf-8'))
log.info("--> %s" % response)
except Exception as e:
log.error(log.exc(e))
else: # ingest energy data sample
## bh16
# this data is not interesting, and mucks up the estimating
continue
# try:
# url = "%s/ingest/ambit" % config['url']
# log.info("Sending to %s..." % url)
# response = net.read(url, str(json.dumps(sample)).encode('utf-8'))
# log.info("--> %s" % response)
# except Exception as e:
# log.error(log.exc(e))
try:
beats = [strings.as_numeric(beat) for beat in data['sml']['DeviceLog']['R-R']['Data'].split()]
d = {'Member': member, 't_utc': start_t, 'Beats': beats}
url = "%s/ingest/ambit_hr" % config['url']
log.info("Sending to %s..." % url)
response = net.read(url, str(json.dumps(d)).encode('utf-8'))
log.info("--> %s" % response)
except Exception as e:
log.error(log.exc(e))
except Exception as e:
log.error("Parsing error: %s" % log.exc(e))
else:
log.warning("--> unknown file type %s, skipping..." % filename)
def main():
log.info("Checking email...")
messages = emailer.fetch()
log.info("--> found %s new messages" % len(messages))
for m, message in enumerate(messages):
log.info("Processing message %s..." % m)
if message['from'] not in config['allowed_senders']:
log.warning("Received bunk email from %s" % message['from'])
continue
subject = message['subject'].lower().strip()
log.info("--> subject: %s" % subject)
def check_satellites(subject):
for satellite in config['satellites']:
if satellite.lower() in subject:
return True
return False
if check_satellites(subject):
# relay a beacon (body post)
url = "%s/ingest/beacon" % config['url']
log.info("Sending to %s..." % url)
try:
response = net.read(url, ("%s\n%s" % (str(subject), str(message['body']))).encode('utf-8'))
log.info("--> %s" % response)
except Exception as e:
log.error("--> main server error: %s" % log.exc(e))
continue
else:
# unpack the ambit zip and post each sample (to ambit or to ambit_geo)
log.info("--> %s attachments" % len(message['attachments']))
for attachment in message['attachments']:
try:
path = os.path.join(os.path.dirname(__file__), "..", "uploads", "%s-%s_%s" % (util.timestamp(), m, attachment['filename'].lower()))
def write_file():
with open(path, 'wb') as f:
f.write(attachment['data'])
if path[-3:] != "zip":
log.info("--> skipping non-zip file %s" % path)
continue
write_file()
if zipfile.is_zipfile(path) is False:
log.warning("--> zip file is corrupt %s" % path)
continue
p = path[:-4]
os.mkdir(p)
with zipfile.ZipFile(path, 'r') as archive:
archive.extractall(p)
log.info("--> zip file extracted")
def traverse(pd):
log.info("Checking %s..." % pd)
for i, filename in enumerate(os.listdir(pd)):
if filename[0] == ".":
continue
elif os.path.isdir(os.path.join(pd, filename)):
traverse(os.path.join(pd, filename))
elif filename[-3:] == "sml":
try:
log.info("Reading %s..." % os.path.join(pd, filename))
with open(os.path.join(pd, filename)) as f:
content = f.read()
except Exception as e:
log.error("Could not read file: %s" % log.exc(e))
else:
try:
log.info("Parsing...")
data = xmltodict.parse(content)
# log.debug(json.dumps(data, indent=4))
serial_number = str(data['sml']['DeviceLog']['Device']['SerialNumber'])
try:
member = config['ambits'][serial_number]
except KeyError:
log.warning("Ambit serial number not linked to a Member")
log.debug(serial_number)
log.debug(config['ambits'])
continue
log.info("Member: %s" % member)
samples = data['sml']['DeviceLog']['Samples']['Sample']
start_t = None
for s, sample in enumerate(samples):
if s == 0:
dt = util.parse_date(sample['UTC']) # these are marked UTC in the data
start_t = util.timestamp(dt)
sample['Member'] = member
if 'Satellites' in sample: # ingest satellite location data
try:
url = "%s/ingest/ambit_geo" % config['url']
log.info("Sending to %s..." % url)
response = net.read(url, str(json.dumps(sample)).encode('utf-8'))
log.info("--> %s" % response)
except Exception as e:
log.error(log.exc(e))
else: # ingest energy data sample
## bh16
# this data is not interesting, and mucks up the estimating
continue
# try:
# url = "%s/ingest/ambit" % config['url']
# log.info("Sending to %s..." % url)
# response = net.read(url, str(json.dumps(sample)).encode('utf-8'))
# log.info("--> %s" % response)
# except Exception as e:
# log.error(log.exc(e))
try:
beats = [strings.as_numeric(beat) for beat in data['sml']['DeviceLog']['R-R']['Data'].split()]
d = {'Member': member, 't_utc': start_t, 'Beats': beats}
url = "%s/ingest/ambit_hr" % config['url']
log.info("Sending to %s..." % url)
response = net.read(url, str(json.dumps(d)).encode('utf-8'))
log.info("--> %s" % response)
except Exception as e:
log.error(log.exc(e))
except Exception as e:
log.error("Parsing error: %s" % log.exc(e))
else:
log.warning("--> unknown file type %s, skipping..." % filename)
traverse(p)
except Exception as e:
log.error(log.exc(e))