def run(self): while True: try: data = self.xbee.wait_read_frame() if self.verbose: log.debug(data) response = {} if 'source_addr' in data: response['sensor'] = int(data['source_addr'][1]) if 'frame_id' in data: response['frame'] = str(data['frame_id'], 'ascii') if 'parameter' in data: response['parameter'] = int.from_bytes(data['parameter'], 'little') if 'rssi' in data: response['rssi'] = int.from_bytes(data['rssi'], 'little') if 'samples' in data: response['samples'] = [] for each in data['samples']: samples = list(each.items()) samples.sort(key=lambda item: item[0]) response['samples'].append([s[1] for s in samples]) if len(response['samples']) == 1: response['samples'] = response['samples'][0] if self.message_handler is not None: self.message_handler(response) except Exception as e: log.error(log.exc(e))
def parse(request): log.info("ambit_geo.parse") sample = ingest_json_body(request) if sample is None: return sample, "Could not parse" data = {} for key, value in sample.items(): if key == "UTC": dt = util.parse_date(sample['UTC']) # these are marked UTC in the data t = util.timestamp(dt) data['t_utc'] = t continue if key == "Longitude": data['longitude'] = math.degrees(float(sample['Longitude'])) continue if key == "Latitude": data['latitude'] = math.degrees(float(sample['Latitude'])) continue if key == "GPSAltitude": data['altitude'] = float(sample['GPSAltitude']) continue if type(value) != str: continue data[key] = strings.as_numeric(value) try: log.debug("%s %s %s" % (data['longitude'], data['latitude'], data['altitude'])) except: log.error("MISSING GEO") return data
def ingest_json_api(path): log.info("ingest_json_api %s" % path) d = open(path) txt = d.read(); data = json.loads(txt) t = data['t_utc'] lat = data['Longitude'] lon = data['Latitude'] coords = [float(lat),float(lon),0] log.debug(data) feature = geojson.Feature(geometry={'type': "Point", 'coordinates': coords},properties=data) if ('Exhaustion' in data): feature_id = model.insert_feature('ethnographic', t, geojson.dumps(feature)) log.info("ingest_json_api ETHNO") elif ('Hardness' in data): feature_id = model.insert_feature('hydro', t, geojson.dumps(feature)) log.info("ingest_json_api HYDRO") else: feature_id = model.insert_feature('sighting', t, geojson.dumps(feature)) log.info("ingest_json_api SIGHTING") d.close()
def on_data(self, data): try: tweet = json.loads(data) log.debug("@%s: %s" % (tweet['user']['screen_name'], tweet['text'])) # log.debug(json.dumps(data, indent=4)) self.queue.put(1) return True except Exception as e: log.error(log.exc(e)) return True
def find_future(user_id, host): if Model.build(user_id, host) is None: return None if __name__ == "__main__": for host, site in Model.sites.items(): log.debug(site) log.debug("Novelty: %f" % Model.calc_novelty()) if len(Model.sites) < max(2, MIN_MODEL_SIZE): # the model is too small return None future = Model.get(host).find_next() return future
def fetch_walks(db, hidden=False, desc=False): try: log.debug(hidden) query = "SELECT * FROM walks %s ORDER BY start_time %s" % ("WHERE hidden=0" if not hidden else "", "DESC" if desc else "") log.debug(query) db.execute(query) rows = [dict(gd) for gd in db.fetchall()] except Exception as e: log.error(log.exc(e)) rows = [] return rows
def geocode(self): try: url = "https://maps.googleapis.com/maps/api/geocode/json?latlng=%s,%s" % ( self.lat, self.lon) result = requests.get(url).json() self.address = result['results'][0]['formatted_address'] self.address = self.address.split(", NY ")[0].replace( ", New York", "") except Exception as e: log.error(log.exc(e)) log.debug(json.dumps(result, indent=4))
def generate(): day = [] index = random.choice(range(len(X))) x = X[index] for i in range(PERIODS): distribution = model.predict(np.array([x[-MEMORY:]]), verbose=0, batch_size=1)[0] y = sample(distribution, config['temperature']) x = np.append(x, to_categorical(y, CATEGORIES), axis=0) day.append(y) log.debug(day) return day
def get(self, collar_id=None, session_id=None, start_t=None, stop_t=None): log.info("GET") session_list = [] if not len(collar_id) or not len(session_id): collar_ids = list(self.db.entries.find().distinct("collar_id")) for collar_id in collar_ids: sessions = list( self.db.entries.find({ 'collar_id': collar_id }).distinct("session")) session_list.append({ 'collar_id': collar_id, 'sessions': sessions }) return self.render("index.html", session_list=session_list) if not len(start_t): start_t = 0 else: start_t = int(start_t) if not len(stop_t): stop_t = 86400000 else: stop_t = int(stop_t) collar_id = strings.as_numeric(collar_id) session_id = strings.as_numeric(session_id) log.info("%d (%s-%s)" % (collar_id, start_t, stop_t)) template = { 't': { '$gt': start_t, '$lt': stop_t }, 'collar_id': collar_id, 'session': session_id } log.debug(template) results = list(self.db.entries.find(template).sort('t')) start_segment = None stop_segment = None if len(results): start_segment = timeutil.seconds_to_string(results[0]['t']) stop_segment = timeutil.seconds_to_string(results[-1]['t']) for result in results: del result['_id'] del result['session'] log.debug("Returned %s entries" % len(results)) return self.render("home.html", data=results, collar_id=collar_id, session=session_id, start_segment=start_segment, stop_segment=stop_segment)
def fetch_features(db, kinds, start_t, stop_t, skip=1): kindq = [] for kind in kinds: kindq.append(" OR kind='%s'" % kind) query = "SELECT rowid, data FROM features WHERE rowid %% ? = 0 AND (1=0%s) AND t>=? AND t<? ORDER BY t" % ''.join(kindq) log.debug(query) db.execute(query, (skip, start_t, stop_t)) features = [] # this is slow for row in db.fetchall(): feature = geojson.loads(row['data']) feature.id = row['rowid'] features.append(feature) return features
def temporal_filter(features, resolution): try: log.info("--> starting temporal_filter") first_t = features[0]['properties']['t_utc'] dt = datetime.datetime.utcfromtimestamp(first_t) dt = dt.replace(hour=0, minute=0, second=0, microsecond=0) start_t = util.timestamp(dt) log.debug("start_date %s" % util.datestring(start_t)) log.debug("stop_date %s" % util.datestring(features[-1]['properties']['t_utc'])) log.debug("start_t %s" % start_t) log.debug("step %s" % resolution) results = [] index_t = start_t index = 0 while True: # log.debug("Checking %s..." % util.datestring(index_t)) while index < len(features) and features[index]['properties']['t_utc'] < index_t: index += 1 if index == len(features): break if not (features[index]['properties']['t_utc'] > index_t + resolution): # log.debug("--> %s %s %s" % (index, features[index]['id'], util.datestring(features[index]['properties']['t_utc']))) results.append(features[index]) index_t += resolution log.info("--> done temporal_filter") return results except Exception as e: log.error(log.exc(e))
def ingest_audio(path, i, t_protect): log.info("ingest_audio %s" % path) dt = datetime.datetime.strptime(path.split('/')[-1], "audio %d%m%Y_%H%M.mp3") tz = pytz.timezone(config['local_tz']) dt = tz.localize(dt) t = util.timestamp(dt) # if t <= t_protect: # log.warning("Protected t, skipping...") # return fixed_path = path.replace(".mp3", ".amr") shutil.move(path, fixed_path) new_path = os.path.join(os.path.dirname(__file__), "static", "data", "audio", "%s-%s.wav" % (t, i)) log.debug("CONVERTING SOUND.") try: log.debug("--> converting [%s] to [%s]" % (fixed_path, new_path)) log.debug("%s -y -i '%s' '%s'" % (config['ffmpeg'], os.path.abspath(fixed_path), os.path.abspath(new_path))) subprocess.check_call("%s -y -i '%s' '%s'" % (config['ffmpeg'], os.path.abspath(fixed_path), os.path.abspath(new_path)), shell=True) except Exception as e: log.error(log.exc(e)) return log.debug("DONE CONVERTING SOUND.") feature = geojson.Feature(properties={'utc_t': t, 'ContentType': "audio", 'url': "/static/data/audio/%s-%s.wav" % (t, i), 'DateTime': dt.astimezone(pytz.timezone(config['local_tz'])).strftime("%Y-%m-%dT%H:%M:%S%z")}) feature_id = model.insert_feature('audio', t, geojson.dumps(feature))
def parse(request): log.info("sensor.parse") try: message = ingest_plain_body(request) data = net.urldecode(message) log.debug(data) number = data['From'] body = data['Body'] content = strings.singlespace(body.strip()) # 1436477098,Lat,-18.111946,Long,21.665733,TDS,4.000,Sal,0.000,WatTemp,16.88,AirTemp,18.50,Hum,48.90 tokens = content.split(',') t = util.delocalize_timestamp(strings.as_numeric(tokens[0]), config['local_tz']) data = {'t_utc': t, 'FeatureType': "sensor", 'CoreExpedition': False} tokens = tokens[1:] keys = tokens[0::2] values = tokens[1::2] for k, key in enumerate(keys): data[strings.camelcase(key)] = strings.as_numeric(values[k]) # tokens = content.split(" ") # if tokens[0] == 'location': # lon, lat = tokens[3], tokens[2] ## note reversal # # insert a record for this location update # data = {'t_utc': util.timestamp(), 'SensorName': tokens[1], 'longitude': lon, 'latitude': lat, 'LocationUpdate': True} # # send a verification # send_sms(number, tokens[1], lon, lat) # else: # # find the most recent location update for this sensor # db = Application.instance.db # results = list(db.features.find({'properties.FeatureType': "sensor", 'properties.SensorName': tokens[1], 'properties.LocationUpdate': True}).sort([('properties.t_utc', DESCENDING)]).limit(1)) # georef = results[0] if len(results) else None # geometry = georef['geometry'] if georef is not None else None # log.debug(geometry) # if tokens[0] == "*": # t = util.timestamp() # else: # t = util.delocalize_timestamp(tokens[0], config['local_tz']) # converts from local time timestamp to t_utc # data = {'t_utc': t, 'SensorName': tokens[1], strings.camelcase(tokens[2]): tokens[3], 'LocationUpdate': False, 'geometry': geometry} except Exception as e: log.error("Twilio post is malformed: %s" % log.exc(e)) return None return data
def get_timeline(self): skip = self.get_argument('skip', 1) kinds = self.get_argument('types', "beacon").split(',') kinds = [kind.rstrip('s') for kind in kinds if kind.rstrip('s') in ['ambit', 'ambit_geo', 'sighting', 'breadcrumb', 'image', 'audio', 'breadcrumb', 'beacon', 'heart_spike']] # sanitizes try: dt = self.get_argument('date', datetime.datetime.now(pytz.timezone(config['local_tz'])).strftime("%Y-%m-%d")) log.debug(dt) dt = util.parse_date(dt, tz=config['local_tz']) days = int(self.get_argument('days', 1)) except Exception as e: return self.error("Bad parameters: %s" % log.exc(e)) t = util.timestamp(dt) log.debug("--> search for kinds: %s" % kinds) features = model.fetch_features(kinds, t, t + (days * (24 * 60 * 60)), skip) feature_collection = geojson.FeatureCollection(features) return self.json(feature_collection)
def on_close(self): log.info("//////////// WebSocket.on_close") user_id = None for uid, instance in WebSocket.sockets.items(): if instance == self: user_id = uid log.info("--> closing user_id %s" % user_id) if user_id is None: log.warning("socket for %s not found" % user_id) return if user_id in WebSocket.users: del WebSocket.users[user_id] for uid, partner_id in WebSocket.users.items(): if partner_id == user_id: WebSocket.send(uid, "unentangled") WebSocket.users[uid] = None log.debug("--> users %s" % WebSocket.users) log.info("--> complete")
def retrieve(db, source, start, end, filters, page=None): if filters == None: filters = {} sources = [clean(source) for source in source.split(",")] start_t = 0 if start == "*" else util.timestamp(util.parse_date(start, tz=config['tz'])) end_t = min(2147483647, sys.maxsize) if end == "*" else util.timestamp(util.parse_date(end, tz=config['tz'])) template = {'t_utc': {'$gt': start_t, '$lt': end_t}, '$or': [{'source': source} for source in sources]} template.update(filters) log.info("QUERY %s" % template) results = db.entries.find(template).sort('t_utc') count = results.count() if page is None: page = (count // 100) + 1 skip = (page - 1) * 100 log.debug("page %s, skip %s" % (page, skip)) results = results.skip(skip).limit(100) log.info("--> done") return list(results), start_t, end_t, count, page
def open(self): log.info("//////////// WebSocket.open") user_id = strings.random_string(10) WebSocket.sockets[user_id] = self log.info("--> new user_id %s" % user_id) available_users = [uid for uid in WebSocket.users.keys() if WebSocket.users[uid] is None] log.info("--> available users: %s" % available_users) if len(available_users): partner_id = random.choice(available_users) WebSocket.users[partner_id] = user_id WebSocket.users[user_id] = partner_id log.info("--> entangled %s with %s" % (user_id, partner_id)) WebSocket.send(user_id, "entangled") WebSocket.send(partner_id, "entangled") else: WebSocket.users[user_id] = None log.debug("--> no partner to entangle") log.debug("--> users %s" % WebSocket.users) WebSocket.send(user_id, user_id)
def ingest_beacon(content): log.info("ingest_beacon") t_protect = model.get_protect('beacon') properties = {} coordinates = [None, None, None] t = None try: lines = content.split('\n') for line in lines: log.debug("%s" % line) try: if "Position Time:" in line: line = line.replace("Position Time:", "").strip() dt = util.parse_date(line) t = util.timestamp(dt) properties['DateTime'] = dt.astimezone(pytz.timezone(config['local_tz'])).strftime("%Y-%m-%dT%H:%M:%S%z") properties['t_utc'] = t if "Map:" in line: line = line.split('?')[1].strip() result = net.urldecode(line) lat, lon = result['q'].split(' ')[0].split(',') coordinates[0], coordinates[1] = strings.as_numeric(lon), strings.as_numeric(lat) if "Altitude:" in line: altitude = strings.as_numeric(line.replace("Altitude:", "").replace("meters", "").strip()) coordinates[2] = altitude if "Speed:" in line: speed = strings.as_numeric(line.replace("Speed:", "").replace("Knots", "").strip()) properties['Speed'] = speed if "Heading:" in line: heading = strings.as_numeric(line.replace("Heading:", "").replace("°", "").strip()) properties['Heading'] = heading except Exception as e: log.error(log.exc(e)) continue # if t <= t_protect: # log.warning("Protected t, skipping...") # return feature = geojson.Feature(geometry={'type': "Point", 'coordinates': coordinates}, properties=properties) feature_id = model.insert_feature('beacon', t, geojson.dumps(feature)) except Exception as e: log.error(log.exc(e))
def format_csv(data): import csv features = data[0]['features'] # build header header = [] for feature in features: feature.update(feature['properties']) if 'Taxonomy' in feature and feature['Taxonomy'] is not None: feature.update(feature['Taxonomy']) del feature['Taxonomy'] if feature['geometry'] is not None: feature.update({"Longitude": feature['geometry']['coordinates'][0], "Latitude": feature['geometry']['coordinates'][1]}) del feature['properties'] del feature['geometry'] for key in feature: if key not in header: header.append(key) header.sort() log.debug(header) # populate rows csv = [] csv.append(','.join(header)) with open('data.csv', 'w', newline='') as csvfile: for feature in features: row = [] for column in header: if column in feature: value = feature[column] if type(value) == str: value = strings.singlespace(value) value.replace('"', "'") value = "%s" % value row.append(str(value).replace(",", "")) else: row.append("None") csv.append(','.join(row)) return '\n'.join(csv) # print(json.dumps(features, indent=4, default=lambda x: str(x)))
def main(): global t, db, ctx t = int(time.time()) # t = int(time.mktime(util.parse_date('2013-07-10 19:00:00').timetuple())) filename = "signals/%s_%s.json" % (t, DURATION) log.info("Generating %s..." % filename) db = CrashDB(filename) if config['draw']: from housepy import drawing ctx = drawing.Context(width=2000, height=500, background=(0., 0., 1.), hsv=True, flip=True, relative=True) process_readings('heat', (0., 1., 1.)) # red # # process_readings('rain', (.1, 1., 1.)) # orange process_readings('humidity', (.1, 1., 1.)) # orange process_readings('wind', (.3, 1., 1.)) # green process_readings('visibility', (.6, 1., 1.)) # blue process_readings('sun', (0., 0., 0.)) # black # process_readings('tide', (0., 0., 0.5)) # gray process_readings('checkins', (.8, .8, 1.)) # purple process_readings('checkouts', (.9, .8, 1.), 1) # thin purple process_events('tweets', (0.55, 1., 1.)) # matrix # process_events('motion', (0.76, 1., 1.)) # process_events('sound', (0.92, 1., 1.)) # crimson db.close() log.debug("%s" % (db.keys(),)) log.info("--> ok") if config['draw']: image_filename = "signals/%s_%s.png" % (t, DURATION) ctx.image.save(image_filename, "PNG") if __name__ == "__main__": ctx.show() return filename
def parse(request): log.info("sensornet.parse") data = ingest_json_body(request) try: t_local = strings.as_numeric(data['t_local']) data = data['data'] log.debug(json.dumps(data, indent=4, default=lambda x: str(x))) if 'gps_long' in data and 'gps_lat' in data: data['Longitude'] = data['gps_long'] del data['gps_long'] data['Latitude'] = data['gps_lat'] del data['gps_lat'] data['FeatureType'] = "sensor" data['FeatureSubType'] = "hybrid" data['SensorName'] = "sensornet" data['CoreExpedition'] = False data['t_utc'] = util.delocalize_timestamp(t_local, tz=config['local_tz']) except Exception as e: log.error("--> failed: %s" % log.exc(e)) return None, "Unexpected format" return data
def get_data(hidden=False): log.debug("HIDDEN %s" % hidden) data = {} try: walks = model.fetch_walks(hidden=hidden) notes = [] v = 0 ids = [] for walk in walks: sequence = model.fetch_sequence(walk['id']) if len(sequence) < config['min_steps']: continue for step in sequence: #[:config['max_steps']]: notes.append((step[0], v, 0 if step[1] == 'left' else 1)) v += 1 ids.append(walk['id']) # sort and normalize onsets notes.sort(key=lambda x: x[0]) onsets = [note[0] for note in notes] onsets = sp.normalize(onsets) notes = [(onsets[i], note[1], note[2]) for (i, note) in enumerate(notes)] log.info("NOTES %s" % len(notes)) data['notes'] = notes data['walk_ids'] = ids except Exception as e: log.error(log.exc(e)) return {} return json.dumps(data)
def get_data(hidden=False): log.debug("HIDDEN %s" % hidden) data = {} try: walks = model.fetch_walks(hidden=hidden) notes = [] v = 0 ids = [] for walk in walks: sequence = model.fetch_sequence(walk['id']) if len(sequence) < config['min_steps']: continue for step in sequence:#[:config['max_steps']]: notes.append((step[0], v, 0 if step[1] == 'left' else 1)) v += 1 ids.append(walk['id']) # sort and normalize onsets notes.sort(key=lambda x: x[0]) onsets = [note[0] for note in notes] onsets = sp.normalize(onsets) notes = [(onsets[i], note[1], note[2]) for (i, note) in enumerate(notes)] log.info("NOTES %s" % len(notes)) data['notes'] = notes data['walk_ids'] = ids except Exception as e: log.error(log.exc(e)) return {} return json.dumps(data)
def run(self): while True: try: data = self.xbee.wait_read_frame() if self.verbose: log.debug(data) response = {} if 'source_addr' in data: response['sensor'] = int(data['source_addr'][1]) if 'frame_id' in data: response['frame'] = str(data['frame_id'], 'ascii') if 'parameter' in data: response['parameter'] = int.from_bytes(data['parameter'], 'little') if 'rssi' in data: response['rssi'] = int.from_bytes(data['rssi'], 'little') if 'samples' in data: samples = list(data['samples'][0].items()) samples.sort(key=lambda item: item[0]) response['samples'] = [s[1] for s in samples] if self.message_handler is not None: self.message_handler(response) except Exception as e: log.error(log.exc(e))
def parse(request): log.info("beacon.parse") content = ingest_plain_body(request) if content is None: return content, "Could not parse" data = {} lines = content.split('\n') for line in lines: log.debug("%s" % line) try: if "sat4rent" in line.lower(): data['Satellite'] = line[-8:].upper() if "Position Time:" in line: line = line.replace("Position Time:", "").strip() dt = util.parse_date(line) t = util.timestamp(dt) data['t_utc'] = t if "Map:" in line: line = line.split('?')[1].strip() result = net.urldecode(line) lat, lon = result['q'].split(' ')[0].split(',') data['longitude'], data['latitude'] = strings.as_numeric(lon), strings.as_numeric(lat) if "Altitude:" in line: altitude = strings.as_numeric(line.replace("Altitude:", "").replace("meters", "").strip()) data['altitude'] = altitude if "Speed:" in line: speed = strings.as_numeric(line.replace("Speed:", "").replace("Knots", "").strip()) data['Speed'] = speed if "Heading:" in line: heading = strings.as_numeric(line.replace("Heading:", "").replace("°", "").strip()) data['Heading'] = heading except Exception as e: log.error(log.exc(e)) continue return data
def parse(request): log.info("databoat.parse") data = ingest_json_file(request) try: for key, reading in data.items(): t_local = strings.as_numeric(reading['t_local']) reading = reading['data'] log.debug(json.dumps(reading, indent=4, default=lambda x: str(x))) if 'gps_long' in reading and 'gps_lat' in reading: reading['Longitude'] = reading['gps_long'] del reading['gps_long'] reading['Latitude'] = reading['gps_lat'] del reading['gps_lat'] reading['FeatureType'] = "sensor" reading['FeatureSubType'] = "hybrid" reading['SensorName'] = "databoat" reading['t_utc'] = util.delocalize_timestamp(t_local, tz=config['local_tz']) success, value = ingest_data("sensor", reading) if not success: log.error("--> failed: %s" % value) except Exception as e: log.error("--> failed: %s" % log.exc(e)) return None, "Unexpected format" return True
def current_week(self): now = datetime.datetime.now() year = now.year week = now.isocalendar()[1] - 1 log.debug("Current year: %s" % year) log.debug("Current week: %s" % week) try: current_week = self[year][week] except Exception: log.debug("Week not found") else: return current_week
def on_mouse_press(data): global waiting, transmitting, current_string, incoming_message # waiting mode, nothing happens if waiting: log.debug("MODE: waiting") return # transmitting mode, we've clicked, so go into receiving mode if transmitting: log.debug("MODE: transmitting") ctx.textures = [] incoming_message = [] label.text = "" ctx.objects = [o for o in ctx.objects if o is label] draw_reception() transmitting = False flush_messages() return # receiving mode: process clicks and build message log.debug("MODE: receiving") x, y, button, modifiers = data x *= ctx.width y *= ctx.height for c, coord in enumerate(coords): if x > coord[0][0] and x < coord[1][0] and y > coord[0][ 1] and y < coord[1][1]: if c == len(CHARACTERS) + 1: sender.messages.put("DONE") ctx.textures = [] incoming_message = [] label.text = "" ctx.objects = [o for o in ctx.objects if o is label] result = subprocess.run( ["osascript", "focus.scpt", "main_terminal"], stdout=subprocess.PIPE) log.info(result) waiting = True elif c == len(CHARACTERS): if len(incoming_message): incoming_message.pop() sender.messages.put("ERASE") else: character = CHARACTERS[c] if len(incoming_message) < 30: incoming_message.append(character) sender.messages.put(character) label.text = "".join(incoming_message) break
params = net.urlencode(params) request_string = "https://api.foursquare.com/v2/venues/explore?%s" % params try: response = net.read(request_string) except Exception as e: log.error(log.exc(e)) continue data = json.loads(response) # print(json.dumps(data, indent=4)) try: venues = data['response']['groups'][0]['items'] for venue in venues: venue = venue['venue'] checkins = venue['stats']['checkinsCount'] if checkins == 0: continue if 'hereNow' in venue: people = venue['hereNow']['count'] else: people = 0 venue_id = venue['id'] venue = {'venue_id': venue_id, 'people': people} model.add_venue(venue) total_venues += 1 except Exception as e: log.error(log.exc(e)) log.error(json.dumps(data, indent=4)) log.debug("added %s venues" % total_venues)
#!/usr/bin/env python import os, sys sys.path.append(os.path.join(os.path.dirname(__file__), "..")) import json, model, datetime, math from housepy import config, log, net log.info("////////// foursquare //////////") update = [] checkins = 0 checkouts = 0 for venue in model.get_venues(): log.debug("checking %s" % venue['venue_id']) try: params = {'client_id': config['foursquare']['key'], 'client_secret': config['foursquare']['secret'], 'v': "20130704"} params = net.urlencode(params) request_string = "https://api.foursquare.com/v2/venues/%s/herenow?%s" % (venue['venue_id'], params) response = net.read(request_string, timeout=1) data = json.loads(response) people = data['response']['hereNow']['count'] if people != venue['people']: if people > venue['people']: checkins += people - venue['people'] else: checkouts += venue['people'] - people venue['people'] = people update.append(venue) except Exception as e: log.error(log.exc(e))
site = random.choice(self.nexts) # duplicate entries mean prob distribution is correct, abusing memory space a bit, how's that gonna scale... else: site = random.choice(Model.sites_exclude(self)) # jump the chain to a random site in the model page = random.choice(site.pages) if len(site.pages) else '/' # keep with deep paths if possible return site.host, page, int(seconds * 1000) def __str__(self): return "%s\n\t%f, %f\n\t%s\n" % (self.host, np.mean(self.durations) if len(self.durations) else 0.0, np.std(self.durations) if len(self.durations) else 0.0, self.pages) def find_future(user_id, host): if Model.build(user_id, host) is None: return None if __name__ == "__main__": for host, site in Model.sites.items(): log.debug(site) log.debug("Novelty: %f" % Model.calc_novelty()) if len(Model.sites) < max(2, MIN_MODEL_SIZE): # the model is too small return None future = Model.get(host).find_next() return future if __name__ == "__main__": user_id = "e29d909b34d7ced11f67440a74f95f1e" t = time.clock() future = find_future(user_id, "twitter.com") log.debug(future) log.info("%fms" % ((time.clock() - t) * 1000))
def hide(db, walk_id, hidden=True): log.debug("Hiding walk %s: %s" % (walk_id, hidden)) db.execute("UPDATE walks SET hidden=? WHERE id=?", (hidden, walk_id,))
def process(t): log.info("////////// process %s //////////" % t) filename = "audio_tmp/%s.wav" % t sample_rate, signal = wavfile.read(filename) log.info("AUDIO SAMPLES %s" % len(signal)) log.info("SAMPLE RATE %s" % sample_rate) duration = float(len(signal)) / sample_rate log.info("AUDIO DURATION %ss" % util.format_time(duration)) signal = (np.array(signal).astype('float') / (2**16 * 0.5)) # assuming 16-bit PCM, -1 - 1 log.info("--> preprocessing") magnitude = abs(signal) thresholded_magnitude = (magnitude > THRESHOLD) * magnitude # level = sp.smooth(thresholded_magnitude, size=10000) # shit -- smooth is too expensive for raspi level = thresholded_magnitude log.info("--> scanning") TOLERANCE = sample_rate / 10 # within a tenth of a second, same sound (poor man's smoothing?) indexes = [] maxes = [] durations = [] zeros = 0 on_chunk = False for index, sample in enumerate(level): if sample > 0.0: if not on_chunk: indexes.append(index) durations.append(0) maxes.append(0) on_chunk = True durations[-1] += 1 if sample > maxes[-1]: maxes[-1] = sample zeros = 0 if sample == 0.0: if on_chunk: zeros += 1 if zeros == TOLERANCE: on_chunk = False events = [] for i in xrange(len(indexes)): value, t_, duration = maxes[i], t + int(float(indexes[i]) / sample_rate), float(durations[i]) / sample_rate events.append((value, t_, duration)) for event in events: log.debug(event) if 'draw' in config and config['draw']: from housepy import drawing log.info("--> drawing") ctx = drawing.Context(width=2000, height=500, background=(0., 0., 1.), hsv=True, flip=True, relative=True) ctx.line([(float(i) / len(magnitude), sample) for (i, sample) in enumerate(magnitude)], thickness=1, stroke=(0., 0., 0.5)) ctx.line([(float(i) / len(thresholded_magnitude), sample) for (i, sample) in enumerate(thresholded_magnitude)], thickness=1, stroke=(0., 0., 0.)) ctx.line([(float(i) / len(level), sample) for (i, sample) in enumerate(level)], thickness=1, stroke=(0., 1., 1.)) level = sp.normalize(level) ctx.line([(float(i) / len(level), sample) for (i, sample) in enumerate(level)], thickness=1, stroke=(0.15, 1., 1.)) ctx.line(0.0, THRESHOLD, 1.0, THRESHOLD, thickness=1, stroke=(0.55, 1., 1.)) ctx.show() try: data = [] for event in events: value, t_, duration = event data.append({'device': config['device'], 'kind': "sound", 'value': value, 't': t_, 'duration': duration}) response = net.read("http://%s:%s" % (config['server']['host'], config['server']['port']), json.dumps(data)) log.info(response) except Exception as e: log.error(log.exc(e)) if config['device'] != "Granu": os.remove(filename)
import time, json, threading, subprocess, queue, platform, os, sys import numpy as np from housepy import log, config, strings, net, s3, util, process, drawing from scipy.io import wavfile DURATION = 10 AUDIO_TMP = os.path.abspath(os.path.join(os.path.dirname(__file__), "audio_tmp")) t = sys.argv[1] filename = "%s/%s.wav" % (AUDIO_TMP, t) sample_rate, signal = wavfile.read(filename) log.debug("samples %s" % len(signal)) log.debug("sample_rate %s" % sample_rate) duration = float(len(signal)) / sample_rate log.debug("duration %ss" % strings.format_time(duration)) signal = (np.array(signal).astype('float') / (2**16 * 0.5)) # assuming 16-bit PCM, -1 - 1 signal = abs(signal) # magnitude ctx = drawing.Context() ctx.plot(signal) ctx.line(0, config['noise_threshold'], 1, config['noise_threshold'], stroke=(255, 0, 0)) ctx.output("screenshots") log.debug("noise threshold is %s" % config['noise_threshold']) log.debug("found magnitude") content_samples = 0 for sample in signal: if sample > config['noise_threshold']: content_samples += 1 total_content_time = float(content_samples) / sample_rate
} } } }) log.info("POINTS %s" % results.count()) users = len(results.distinct('user_id')) log.info("USERS %s" % users) points = np.array([(result['location']['coordinates'][0], result['location']['coordinates'][1], result['user_id']) for result in results]) min_lon, max_lon = (np.min(points[:, 0]), np.max(points[:, 0])) min_lat, max_lat = (np.min(points[:, 1]), np.max(points[:, 1])) log.debug("%f %f %f %f" % (min_lon, max_lon, min_lat, max_lat)) min_x, max_y = geo.project((min_lon, max_lat)) max_x, min_y = geo.project((max_lon, min_lat)) ratio = (max_x - min_x) / (max_y - min_y) ctx = drawing.Context(1000, int(1000 / ratio), relative=True, flip=True, hsv=True) log.info("Drawing %d %d..." % (ctx.width, ctx.height)) for point in points:
def estimate_geometry(data, db): """Estimate the location of a geotagged object for a new feature that's missing it""" """For data tagged to a Member, find something else that's geotagged with that Member, best case ambit_geo, worst case take the beacon if they are core, otherwise fail""" """For non-member data, just tag it to the beacon""" log.info("Estimating geometry...") t = data['properties']['t_utc'] feature_type = data['properties']['FeatureType'] log.info("--> t is %s (%s)" % (t, util.datestring(t, tz=config['local_tz']))) try: # find geodata from this Member ## bh16 restrict this to ambit_geo. why wasnt it before? member_closest_before = None member_closest_after = None if 'Member' in data['properties'] and data['properties']['Member'] is not None: member = data['properties']['Member'] log.info("--> member is %s" % member) try: # member_closest_before = list(db.features.find({'properties.Member': member, 'geometry': {'$ne': None}, 'properties.t_utc': {'$lte': t}, 'properties.EstimatedGeometry': {'$exists': False}}).sort('properties.t_utc', -1).limit(1))[0] # member_closest_after = list(db.features.find({'properties.Member': member, 'geometry': {'$ne': None}, 'properties.t_utc': {'$gte': t}, 'properties.EstimatedGeometry': {'$exists': False}}).sort('properties.t_utc', 1).limit(1))[0] member_closest_before = list(db.features.find({'properties.Member': member, 'properties.FeatureType': "ambit_geo", 'properties.t_utc': {'$lte': t}}).sort('properties.t_utc', -1).limit(1))[0] member_closest_after = list(db.features.find({'properties.Member': member, 'properties.FeatureType': "ambit_geo", 'properties.t_utc': {'$gte': t}}).sort('properties.t_utc', 1).limit(1))[0] except IndexError: pass # # core? # eliminate, bh16 # if 'CoreExpedition' in data['properties']: # core = data['properties']['CoreExpedition'] # else: # there should never be an else # log.warning("--> no CoreExpedition for estimator") # core = False # log.info("--> core is %s" % core) # find geodata from the nearest beacon # but only do it if there is no Member (always core, unless overridden), or the Member is/was core at that point core_sat = config['satellites'][0] # first satellite is core expedition beacon_closest_before = None beacon_closest_after = None # bh16 temporarily ignoring satellites # # if core and not feature_type == "ambit": ## don't let ambit readings pop to beacons # try: # beacon_closest_before = list(db.features.find({'$or': [{'properties.t_utc': {'$lte': t}, 'properties.FeatureType': 'beacon', 'properties.Satellite': {'$exists': False}}, {'properties.t_utc': {'$lte': t}, 'properties.FeatureType': 'beacon', 'properties.Satellite': {'$eq': core_sat}}]}).sort('properties.t_utc', -1).limit(1))[0] # beacon_closest_after = list(db.features.find({'$or': [{'properties.t_utc': {'$gte': t}, 'properties.FeatureType': 'beacon', 'properties.Satellite': {'$exists': False}}, {'properties.t_utc': {'$gte': t}, 'properties.FeatureType': 'beacon', 'properties.Satellite': {'$eq': core_sat}}]}).sort('properties.t_utc', 1).limit(1))[0] # except IndexError: # pass # pick the best ones if member_closest_before is not None and beacon_closest_before is not None: closest_before = beacon_closest_before if beacon_closest_before['properties']['t_utc'] > member_closest_before['properties']['t_utc'] else member_closest_before elif member_closest_before is not None: closest_before = member_closest_before else: closest_before = beacon_closest_before if member_closest_after is not None and beacon_closest_after is not None: closest_after = beacon_closest_after if beacon_closest_after['properties']['t_utc'] < member_closest_after['properties']['t_utc'] else member_closest_after elif member_closest_after is not None: closest_after = member_closest_after else: closest_after = beacon_closest_after if closest_before is None or closest_after is None: data['properties']['EstimatedGeometry'] = None log.warning("--> closest not found") return data # average the times and positions: this is naive calculation not taking projection into account data['geometry'] = closest_before['geometry'] # make sure the fields are there (if theres an error it will default to this assignment) if len(data['geometry']['coordinates']) == 2: # add altitude if it's missing data['geometry']['coordinates'].append(None) data['properties']['EstimatedGeometry'] = closest_before['properties']['FeatureType'] # note what we used t1 = closest_before['properties']['t_utc'] t2 = closest_after['properties']['t_utc'] if t1 != t2: p = (t - t1) / (t2 - t1) data['geometry']['coordinates'][0] = (closest_before['geometry']['coordinates'][0] * (1 - p)) + (closest_after['geometry']['coordinates'][0] * p) data['geometry']['coordinates'][1] = (closest_before['geometry']['coordinates'][1] * (1 - p)) + (closest_after['geometry']['coordinates'][1] * p) try: data['geometry']['coordinates'][2] = (closest_before['geometry']['coordinates'][2] * (1 - p)) + (closest_after['geometry']['coordinates'][2] * p) except: data['geometry']['coordinates'][2] = None log.debug(data['geometry']['coordinates']) log.info("--> derived from %s" % data['properties']['EstimatedGeometry']) except Exception as e: log.error(log.exc(e)) return data
def parse(request): log.info("sighting.parse") paths = save_files(request) if not len(paths): return None, "No files" # process the json data = None for path in paths: if path[-4:] == "json": try: with open(path) as f: data = json.loads(f.read()) except Exception as e: log.error(log.exc(e)) return None, "Could not parse JSON" break if data is None: return None, "No data" # make corrections # Bird Name should be SpeciesName for key, item in data.items(): modkey = key.strip().lower().replace(' ', '') if modkey == "birdname": data['SpeciesName'] = item del data[key] if 'TeamMember' in data: data['Member'] = data['TeamMember'] del data['TeamMember'] # purge blanks data = {key: value for (key, value) in data.items() if type(value) != str or len(value.strip())} if 'SpeciesName' not in data: log.error("Missing SpeciesName") return None, "Missing SpeciesName" data['SpeciesName'] = strings.titlecase(data['SpeciesName']) if 'Count' not in data and 'count' not in data: data['Count'] = 1 log.debug(json.dumps(data, indent=4)) data['Taxonomy'] = get_taxonomy(data['SpeciesName']) # process the image images = [] for path in paths: if path[-4:] != "json": log.info("Inserting image... %s" % path.split('/')[-1]) image_data = process_image(path, data['Member'] if 'Member' in data else None, data['t_utc'] if 't_utc' in data else None) if image_data is None: log.info("--> no image data") continue success, value = ingest_data("image", image_data.copy()) # make a second request for the image featuretype if not success: log.error(value) if 'Member' in image_data: del image_data['Member'] images.append(image_data) log.info("--> image added") data['Images'] = images # use image data to assign a timestamp to the sighting if 'getImageTimestamp' in data and data['getImageTimestamp'] == True and len(data['Images']) and 't_utc' in data['Images'][0]: data['t_utc'] = data['Images'][0]['t_utc'] log.info("--> replaced sighting t_utc with image data") if 'getImageTimestamp' in data: del data['getImageTimestamp'] return data
#!/usr/bin/env python3 import json, os import sender_new, sender_exit, sender_persisting, sender_alone from housepy import config, log from tweeter import t # initialize Twitter and retrieve followers try: result = t.followers.ids(screen_name="angryhermitbot") except Exception as e: log.debug(log.exc(e)) exit() current_ids = result['ids'] # we are so happy if not len(current_ids): log.info("No followers! So happy.") sender_alone.send() exit() # sort followers past_ids = [] if os.path.isfile('followers.txt'): with open('followers.txt', 'r') as f: for line in f.readlines(): past_ids.append(int(line.strip())) new_ids = list(set(current_ids) - set(past_ids)) exit_ids = list(set(past_ids) - set(current_ids)) persisting_ids = list(set(current_ids) - set(new_ids)) #put run info into log file
def on_data(data): if config['debug']: log.debug("Epoch time: [{0}] - X: {1}, Y: {2}, Z: {3}".format( data[0], *data[1])) rater.queue.put(1) monitor.queue.put(data)