def retrieve(db, start, end, type_):
start = util.parse_date(start, tz=config['tz'])
start_t = util.timestamp(start)
end = util.parse_date(end, tz=config['tz'])
end_t = util.timestamp(end)
results = db.entries.find({'t_utc': {'$gt': start_t, '$lt': end_t}, 'type': type_}).sort('t_utc')
return list(results)
def assemble(self, search, limit, order, resolution):
log.info("expeditions.assemble")
expeditions = {}
try:
results = self.db.features.find(search).distinct('properties.Expedition')
for expedition in results:
start_date = util.parse_date(str(config['start_date'][expedition]))
last_feature = list(self.db.features.find({'properties.Expedition': expedition}).sort([('properties.t_utc', DESCENDING)]).limit(1))[0]
end_date = util.parse_date(last_feature['properties']['DateTime'])
duration = end_date - start_date
expeditions[expedition] = {'StartDate': start_date, 'Days': duration.days}
except Exception as e:
return self.error(log.exc(e))
total = returned = len(expeditions)
return expeditions, total, returned
def parse(request):
log.info("ambit_geo.parse")
sample = ingest_json_body(request)
if sample is None:
return sample, "Could not parse"
data = {}
for key, value in sample.items():
if key == "UTC":
dt = util.parse_date(sample['UTC']) # these are marked UTC in the data
t = util.timestamp(dt)
data['t_utc'] = t
continue
if key == "Longitude":
data['longitude'] = math.degrees(float(sample['Longitude']))
continue
if key == "Latitude":
data['latitude'] = math.degrees(float(sample['Latitude']))
continue
if key == "GPSAltitude":
data['altitude'] = float(sample['GPSAltitude'])
continue
if type(value) != str:
continue
data[key] = strings.as_numeric(value)
try:
log.debug("%s %s %s" % (data['longitude'], data['latitude'], data['altitude']))
except:
log.error("MISSING GEO")
return data
def ingest_geo_feature(path, kind):
log.info("ingest_geo_feature %s" % path)
t_protect = model.get_protect(kind)
sightings = []
headings = {}
with open(path) as f:
rows = csv.reader(f)
for r, row in enumerate(rows):
if r == 0:
for i, item in enumerate(row):
headings[item] = i
continue
try:
dt = util.parse_date("%s %s" % (row[headings['Date']], row[headings['Time']]), tz=config['local_tz'], dayfirst=True)
t = util.timestamp(dt)
if t <= t_protect:
log.warning("Protected t, skipping...")
continue
try:
coordinates = strings.as_numeric(row[headings['Longitude']]), strings.as_numeric(row[headings['Latitude']]), strings.as_numeric(row[headings['Altitude']])
except Exception as e:
log.error("Missing coordinates! Skipping...")
continue
properties = {'DateTime': dt.strftime("%Y-%m-%dT%H:%M:%S%z"), 't_utc': t, 'ContentType': kind}
for heading in headings:
if heading not in ['Date', 'Time', 'Latitude', 'Longitude', 'Altitude']:
try:
properties[heading] = strings.as_numeric(row[headings[heading]])
except IndexError:
pass
feature = geojson.Feature(geometry={'type': "Point", 'coordinates': coordinates}, properties=properties)
model.insert_feature(kind, t, geojson.dumps(feature))
except Exception as e:
log.error("Row failed: " + log.exc(e))
continue
def retrieve(db, source, start, end, filters, page=None):
if filters == None:
filters = {}
sources = [clean(source) for source in source.split(",")]
start_t = 0 if start == "*" else util.timestamp(util.parse_date(start, tz=config['tz']))
end_t = min(2147483647, sys.maxsize) if end == "*" else util.timestamp(util.parse_date(end, tz=config['tz']))
template = {'t_utc': {'$gt': start_t, '$lt': end_t}, '$or': [{'source': source} for source in sources]}
template.update(filters)
log.info("QUERY %s" % template)
results = db.entries.find(template).sort('t_utc')
count = results.count()
if page is None:
page = (count // 100) + 1
skip = (page - 1) * 100
log.debug("page %s, skip %s" % (page, skip))
results = results.skip(skip).limit(100)
log.info("--> done")
return list(results), start_t, end_t, count, page
def main(): ## called via tweet_grabber.py
for a, account in enumerate(ACCOUNTS):
log.info("Checking %s..." % account)
try:
feed = "https://medium.com/feed/@%s" % account
data = feedparser.parse(feed)['entries']
except Exception as e:
log.error(log.exc(e))
continue
for entry in data:
try:
entry = {strings.camelcase(key): value for (key, value) in entry.items() if key in ['title', 'link', 'summary', 'published']}
entry['Member'] = MEMBERS[a]
entry['t_utc'] = util.timestamp(util.parse_date(entry['Published']))
if entry['t_utc'] < (util.timestamp(util.parse_date(str(config['start_date'][config['expedition']])))) - (3 * 24 * 60 * 60): ## hack, minus three days to get jer's post
log.info("--> skipping too early blog post")
continue
del entry['Published']
entry['Url'] = entry['Link']
del entry['Link']
entry['Summary'] = strings.strip_html(entry['Summary']).replace("Continue reading on Medium \u00bb", "")
entry['FeatureType'] = "blog"
dup = db.features.find_one({'properties.FeatureType': 'blog', 'properties.Url': entry['Url']})
if dup is not None:
log.info("--> skipping duplicate blog post")
continue
log.info("--> %s" % entry)
success, value = ingest_data("tweet", entry)
if not success:
log.error("--> failed: %s" % value)
else:
log.info("--> %s" % value)
except Exception as e:
log.error(log.exc(e))
continue
def get_timeline(self):
skip = self.get_argument('skip', 1)
kinds = self.get_argument('types', "beacon").split(',')
kinds = [kind.rstrip('s') for kind in kinds if kind.rstrip('s') in ['ambit', 'ambit_geo', 'sighting', 'breadcrumb', 'image', 'audio', 'breadcrumb', 'beacon', 'heart_spike']] # sanitizes
try:
dt = self.get_argument('date', datetime.datetime.now(pytz.timezone(config['local_tz'])).strftime("%Y-%m-%d"))
log.debug(dt)
dt = util.parse_date(dt, tz=config['local_tz'])
days = int(self.get_argument('days', 1))
except Exception as e:
return self.error("Bad parameters: %s" % log.exc(e))
t = util.timestamp(dt)
log.debug("--> search for kinds: %s" % kinds)
features = model.fetch_features(kinds, t, t + (days * (24 * 60 * 60)), skip)
feature_collection = geojson.FeatureCollection(features)
return self.json(feature_collection)
def parse(request):
log.info("ambit.parse")
sample = ingest_json_body(request)
if sample is None:
return sample, "Could not parse"
data = {}
for key, value in sample.items():
if key == "UTC":
dt = util.parse_date(value) # these are marked UTC in the data
t = util.timestamp(dt)
data['t_utc'] = t
continue
if type(value) != str:
continue
data[key] = strings.as_numeric(value)
return data
def ingest_hydrosensor(hydrosensor_id, content, dt):
log.info("ingest_hydrosensor")
t_protect = model.get_protect('hydrosensor')
lat, lon = model.get_drop_by_id(hydrosensor_id)
#HACK
coordinates = [0, 0, 0]
t = util.timestamp(dt)
#properties = {'DateTime': dt.strftime("%Y-%m-%dT%H:%M:%S%z"), 't_utc': t, 'ContentType': "hydrosensor"}
properties = {'ContentType': "hydrosensor"}
try:
lines = content.split('\n')
for line in lines:
if not len(line.strip()):
continue
try:
#Date: Sat, Sep 13, 2014 at 5:23 AM
if "Date" in line:
dt = util.parse_date(line.replace("Date: ", "").strip(), tz=config['local_tz'])
t = util.timestamp(dt)
properties['DateTime'] = dt.strftime("%Y-%m-%dT%H:%M:%S%z")
properties['t_utc'] = t
log.info(dt.strftime("%Y-%m-%dT%H:%M:%S%z"))
if "Temp" in line:
temperature = strings.as_numeric(line.replace("Temp (deg C) = (", "").replace(")", "").strip())
properties['temperature'] = temperature
if "pH" in line:
ph = strings.as_numeric(line.replace("pH = (", "").replace(")", "").strip())
properties['ph'] = ph
if "Conductivity" in line:
conductivity = line.replace("Conductivity (Cond,TDS,Sal,SG) = (", "").replace(")", "").strip()
conductivity = [strings.as_numeric(element) for element in conductivity.split(",")]
properties['conductivity'] = conductivity
except Exception as e:
log.error(log.exc(e))
continue
# if t <= t_protect:
# log.warning("Protected t, skipping...")
# return
feature = geojson.Feature(geometry={'type': "Point", 'coordinates': coordinates}, properties=properties)
feature_id = model.insert_feature('hydrosensor', t, geojson.dumps(feature))
except Exception as e:
log.error(log.exc(e))
def ingest_beacon(content):
log.info("ingest_beacon")
t_protect = model.get_protect('beacon')
properties = {}
coordinates = [None, None, None]
t = None
try:
lines = content.split('\n')
for line in lines:
log.debug("%s" % line)
try:
if "Position Time:" in line:
line = line.replace("Position Time:", "").strip()
dt = util.parse_date(line)
t = util.timestamp(dt)
properties['DateTime'] = dt.astimezone(pytz.timezone(config['local_tz'])).strftime("%Y-%m-%dT%H:%M:%S%z")
properties['t_utc'] = t
if "Map:" in line:
line = line.split('?')[1].strip()
result = net.urldecode(line)
lat, lon = result['q'].split(' ')[0].split(',')
coordinates[0], coordinates[1] = strings.as_numeric(lon), strings.as_numeric(lat)
if "Altitude:" in line:
altitude = strings.as_numeric(line.replace("Altitude:", "").replace("meters", "").strip())
coordinates[2] = altitude
if "Speed:" in line:
speed = strings.as_numeric(line.replace("Speed:", "").replace("Knots", "").strip())
properties['Speed'] = speed
if "Heading:" in line:
heading = strings.as_numeric(line.replace("Heading:", "").replace("°", "").strip())
properties['Heading'] = heading
except Exception as e:
log.error(log.exc(e))
continue
# if t <= t_protect:
# log.warning("Protected t, skipping...")
# return
feature = geojson.Feature(geometry={'type': "Point", 'coordinates': coordinates}, properties=properties)
feature_id = model.insert_feature('beacon', t, geojson.dumps(feature))
except Exception as e:
log.error(log.exc(e))
def process_image(path, member=None, t_utc=None):
# try to get EXIF data
log.info("process_image %s..." % path)
data = {}
if member is not None:
data['Member'] = member
if t_utc is not None:
data['t_utc'] = t_utc
try:
image = Image.open(path)
width, height = image.size
data['Dimensions'] = width, height
try:
exif = {ExifTags.TAGS[k]: v for (k, v) in image._getexif().items() if k in ExifTags.TAGS}
except Exception as e:
log.warning("--> no EXIF data in image: %s" % e)
if 't_utc' not in data:
log.warning("--> substituting current time for t_utc")
data['t_utc'] = util.timestamp()
else:
# log.debug(json.dumps(exif, indent=4, default=lambda x: str(x)))
date_field = exif['DateTimeOriginal'] if 'DateTimeOriginal' in exif else exif['DateTime']
if date_field[4] == ":" and date_field[7] == ":":
date_field = list(date_field)
date_field[4] = "-"
date_field[7] = "-"
date_field = ''.join(date_field)
date = util.parse_date(date_field, tz=config['local_tz'])
data['t_utc'] = util.timestamp(date) ## careful about this overriding
data['DateTime'] = util.datestring(data['t_utc'], tz=config['local_tz'])
data['Make'] = exif['Make'].replace("\u0000", '').strip() if 'Make' in exif else None
data['Model'] = exif['Model'].replace("\u0000", '').strip() if 'Model' in exif else None
filename = "%s_%s.jpg" % (data['t_utc'], str(uuid.uuid4()))
new_path = os.path.join(os.path.dirname(__file__), "..", "static", "data", "images", filename)
shutil.copy(path, new_path)
data['Url'] = "/static/data/images/%s" % filename
except Exception as e:
log.error(log.exc(e))
return None
return data
def parse(request):
log.info("beacon.parse")
content = ingest_plain_body(request)
if content is None:
return content, "Could not parse"
data = {}
lines = content.split('\n')
for line in lines:
log.debug("%s" % line)
try:
if "sat4rent" in line.lower():
data['Satellite'] = line[-8:].upper()
if "Position Time:" in line:
line = line.replace("Position Time:", "").strip()
dt = util.parse_date(line)
t = util.timestamp(dt)
data['t_utc'] = t
if "Map:" in line:
line = line.split('?')[1].strip()
result = net.urldecode(line)
lat, lon = result['q'].split(' ')[0].split(',')
data['longitude'], data['latitude'] = strings.as_numeric(lon), strings.as_numeric(lat)
if "Altitude:" in line:
altitude = strings.as_numeric(line.replace("Altitude:", "").replace("meters", "").strip())
data['altitude'] = altitude
if "Speed:" in line:
speed = strings.as_numeric(line.replace("Speed:", "").replace("Knots", "").strip())
data['Speed'] = speed
if "Heading:" in line:
heading = strings.as_numeric(line.replace("Heading:", "").replace("°", "").strip())
data['Heading'] = heading
except Exception as e:
log.error(log.exc(e))
continue
return data
def traverse(pd):
log.info("Checking %s..." % pd)
for i, filename in enumerate(os.listdir(pd)):
if filename[0] == ".":
continue
elif os.path.isdir(os.path.join(pd, filename)):
traverse(os.path.join(pd, filename))
elif filename[-3:] == "sml":
try:
log.info("Reading %s..." % os.path.join(pd, filename))
with open(os.path.join(pd, filename)) as f:
content = f.read()
except Exception as e:
log.error("Could not read file: %s" % log.exc(e))
else:
try:
log.info("Parsing...")
data = xmltodict.parse(content)
# log.debug(json.dumps(data, indent=4))
serial_number = str(data['sml']['DeviceLog']['Device']['SerialNumber'])
try:
member = config['ambits'][serial_number]
except KeyError:
log.warning("Ambit serial number not linked to a Member")
log.debug(serial_number)
log.debug(config['ambits'])
continue
log.info("Member: %s" % member)
samples = data['sml']['DeviceLog']['Samples']['Sample']
start_t = None
for s, sample in enumerate(samples):
if s == 0:
dt = util.parse_date(sample['UTC']) # these are marked UTC in the data
start_t = util.timestamp(dt)
sample['Member'] = member
if 'Satellites' in sample: # ingest satellite location data
try:
url = "%s/ingest/ambit_geo" % config['url']
log.info("Sending to %s..." % url)
response = net.read(url, str(json.dumps(sample)).encode('utf-8'))
log.info("--> %s" % response)
except Exception as e:
log.error(log.exc(e))
else: # ingest energy data sample
## bh16
# this data is not interesting, and mucks up the estimating
continue
# try:
# url = "%s/ingest/ambit" % config['url']
# log.info("Sending to %s..." % url)
# response = net.read(url, str(json.dumps(sample)).encode('utf-8'))
# log.info("--> %s" % response)
# except Exception as e:
# log.error(log.exc(e))
try:
beats = [strings.as_numeric(beat) for beat in data['sml']['DeviceLog']['R-R']['Data'].split()]
d = {'Member': member, 't_utc': start_t, 'Beats': beats}
url = "%s/ingest/ambit_hr" % config['url']
log.info("Sending to %s..." % url)
response = net.read(url, str(json.dumps(d)).encode('utf-8'))
log.info("--> %s" % response)
except Exception as e:
log.error(log.exc(e))
except Exception as e:
log.error("Parsing error: %s" % log.exc(e))
else:
log.warning("--> unknown file type %s, skipping..." % filename)
def ingest_ambit(path, t_protect):
log.info("ingest_ambit %s" % path)
with open(path, 'r') as f:
content = f.read()
content = content.split("<IBI>")[0]
parts = content.split("</header>")
header = parts[0] + "</header>"
header = xmltodict.parse(header.encode('utf-8'))
person = header['header']['Activity'].replace("OWBS ", "")
person = "Chris" if person == "Trail running" else person
person = "Jer" if person == "John" else person
content = parts[-1].encode('utf-8')
samples = xmltodict.parse(content)['Samples']['Sample']
c = 0;
for s, sample in enumerate(samples):
try:
#if 'VerticalSpeed' not in sample:
if 'Satellites' in sample:
# satellite data sample
lon, lat, alt = None, None, None
t, dt = None, None
for key, value in sample.items():
if key == "UTC":
dt = util.parse_date(sample['UTC']) # these are marked UTC in the data
t = util.timestamp(dt)
del sample[key]
continue
if key == "Longitude":
lon = math.degrees(float(sample['Longitude']))
del sample[key]
continue
if key == "Latitude":
lat = math.degrees(float(sample['Latitude']))
del sample[key]
continue
if key == "GPSAltitude":
alt = float(sample['Latitude'])
del sample[key]
continue
if key[:3] == "Nav":
del sample[key]
continue
if type(value) != str:
del sample[key]
continue
sample[key] = strings.as_numeric(value)
# if t <= t_protect:
# log.warning("Protected t, skipping...")
# continue
sample['DateTime'] = dt.astimezone(pytz.timezone(config['local_tz'])).strftime("%Y-%m-%dT%H:%M:%S%z")
sample['t_utc'] = t
sample['ContentType'] = 'ambit_geo'
sample['Person'] = person
feature = geojson.Feature(geometry={'type': "Point", 'coordinates': [lon, lat, alt]}, properties=sample)
model.insert_feature('ambit_geo', t, geojson.dumps(feature))
elif 'VerticalSpeed' in sample:
# energy data sample
for key, value in sample.items():
if key == "UTC":
dt = util.parse_date(value) # these are marked UTC in the data
t = util.timestamp(dt)
del sample[key]
continue
if type(value) != str:
del sample[key]
continue
sample[key] = strings.as_numeric(value)
# if t <= t_protect:
# log.warning("Protected t, skipping...")
# continue
sample['DateTime'] = dt.astimezone(pytz.timezone(config['local_tz'])).strftime("%Y-%m-%dT%H:%M:%S%z")
sample['t_utc'] = t
sample['ContentType'] = 'ambit'
sample['Person'] = person
if((person == 'Chris' and c % 10 == 0) or person != 'Chris'):
feature = geojson.Feature(properties=sample)
model.insert_feature('ambit', t, geojson.dumps(feature))
c = c + 1
else:
log.info("extra ambit field")
except Exception as e:
log.error(log.exc(e))
from housepy import net, science, drawing, util
from openpaths_video import *
LON = 0
LAT = 1
T = 2
X = 3
Y = 4
ZOOM = 1000
DURATION = 60*30 # if points are within 10 minutes, group them
THRESHOLD = 3 # less than threshold points isnt much of a path
points = json.loads(open("thief_points.json").read())
points = np.array([(float(point['lon']), float(point['lat']), time.mktime(util.parse_date(point['time']).timetuple()), None, None) for point in points])
median_lon = np.median(points[:,0])
median_lat = np.median(points[:,1])
points = np.array([point for point in points if abs(point[0] - median_lon) < ZOOM and abs(point[1] - median_lat) < ZOOM])
max_lon = np.max(points[:,0])
min_lon = np.min(points[:,0])
max_lat = np.max(points[:,1])
min_lat = np.min(points[:,1])
points = list(points)
for point in points:
point[X] = util.scale(point[LON], min_lon, max_lon)
point[Y] = util.scale(point[LAT], min_lat, max_lat)
all_points = []
#!/usr/bin/env python3
import pymongo, json
import signal_processing as sp
from housepy import drawing, config, log, util
from mongo import db
START = "2016-07-05 00:00:00"
END = "2016-07-06"
log.info("Retrieving data...")
results = list(
db.stream.find(
{
"t_utc": {
"$gt": util.timestamp(util.parse_date(START, tz="America/New_York")),
"$lt": util.timestamp(util.parse_date(END, tz="America/New_York")),
}
}
).sort([("t_utc", pymongo.ASCENDING)])
)
log.info("--> done")
##
ts = [r["t_utc"] for r in results]
xs = [r["x"] for r in results]
duration = ts[-1] - ts[0]
SAMPLING_RATE = 100
def generate():
# load data into t and count arrays per species
species = OrderedDict()
start_t = util.timestamp(util.parse_date(str(config['start'])))
end_t = util.timestamp(util.parse_date(str(config['end'])))
max_count = 0
with open("data.csv") as f:
data = csv.reader(f)
for r, row in enumerate(data):
if r == 0:
continue
plot = row[1]
name = row[2]
if len(config['species_list']) and name not in config['species_list']:
continue
dt = datetime.datetime(int(row[3]), 1, 1) + datetime.timedelta(int(row[4]) - 1)
t = util.timestamp(dt)
if t < start_t or t > end_t:
continue
count = 0 if row[5] == "NA" else int(row[5])
if count > max_count:
max_count = count
if name not in species:
species[name] = {'ts': [start_t, t - 1], 'counts': [0, 0]}
species[name]['ts'].append(t)
species[name]['counts'].append(count)
species = OrderedDict(sorted(species.items()))
print("--> loaded")
# add a zero count at the start and end of every year
yts = [util.timestamp(datetime.datetime(y, 1, 1)) for y in range(1974, 2017)]
for name in species:
ts = species[name]['ts']
for yt in yts:
i = 0
while i < len(ts) and ts[i] < yt:
i += 1
if i > 0:
end_season_t = ts[i-1]
if i < len(ts):
start_season_t = ts[i]
ts.insert(i, start_season_t - config['tail'])
species[name]['counts'].insert(i, 0)
ts.insert(i, end_season_t + config['tail'])
species[name]['counts'].insert(i, 0)
species[name]['ts'].append(end_t)
species[name]['counts'].append(0)
print("--> onsets added")
# create and draw signals
signals = []
names = []
i = 0
for name, data in species.items():
print("Processing %s..." % name)
# create signal from bloom counts
signal = sp.resample(data['ts'], data['counts'])
if config['normalize']:
signal = sp.normalize(signal)
else:
signal = sp.normalize(signal, 0, max_count)
signal = sp.smooth(signal, size=8)
signal = sp.limit(signal, max(signal)) # get rid of noise below 0 for onset detection
# add spikes for peaks
if config['peak_spikes']:
peaks, valleys = sp.detect_peaks(signal, lookahead=50)
peak_signal = np.zeros(len(signal))
for peak in peaks:
peak_signal[peak[0]] = 1.0
signal += peak_signal
# add spikes for onsets
if config['onset_spikes']:
onsets = sp.detect_onsets(signal)
onset_signal = np.zeros(len(signal))
for onset in onsets:
onset_signal[onset] = 0.5
onset_signal[onset+1] = 0.4
onset_signal[onset+2] = 0.25
signal += onset_signal
# limit
signal = sp.limit(signal, 1.0)
signal *= 0.9 # hack, just controlling gain
signals.append(signal)
names.append(name)
i += 1
return signals, names
def generate():
# load data into t and count arrays per species
species = OrderedDict()
start_t = util.timestamp(util.parse_date(str(config['start'])))
end_t = util.timestamp(util.parse_date(str(config['end'])))
max_count = 0
with open("data.csv") as f:
data = csv.reader(f)
for r, row in enumerate(data):
if r == 0:
continue
plot = row[1]
name = row[2]
if len(config['species_list']
) and name not in config['species_list']:
continue
dt = datetime.datetime(int(row[3]), 1,
1) + datetime.timedelta(int(row[4]) - 1)
t = util.timestamp(dt)
if t < start_t or t > end_t:
continue
count = 0 if row[5] == "NA" else int(row[5])
if count > max_count:
max_count = count
if name not in species:
species[name] = {'ts': [start_t, t - 1], 'counts': [0, 0]}
species[name]['ts'].append(t)
species[name]['counts'].append(count)
species = OrderedDict(sorted(species.items()))
print("--> loaded")
# add a zero count at the start and end of every year
yts = [
util.timestamp(datetime.datetime(y, 1, 1)) for y in range(1974, 2017)
]
for name in species:
ts = species[name]['ts']
for yt in yts:
i = 0
while i < len(ts) and ts[i] < yt:
i += 1
if i > 0:
end_season_t = ts[i - 1]
if i < len(ts):
start_season_t = ts[i]
ts.insert(i, start_season_t - config['tail'])
species[name]['counts'].insert(i, 0)
ts.insert(i, end_season_t + config['tail'])
species[name]['counts'].insert(i, 0)
species[name]['ts'].append(end_t)
species[name]['counts'].append(0)
print("--> onsets added")
# create and draw signals
signals = []
names = []
i = 0
for name, data in species.items():
print("Processing %s..." % name)
# create signal from bloom counts
signal = sp.resample(data['ts'], data['counts'])
if config['normalize']:
signal = sp.normalize(signal)
else:
signal = sp.normalize(signal, 0, max_count)
signal = sp.smooth(signal, size=8)
signal = sp.limit(
signal,
max(signal)) # get rid of noise below 0 for onset detection
# add spikes for peaks
if config['peak_spikes']:
peaks, valleys = sp.detect_peaks(signal, lookahead=50)
peak_signal = np.zeros(len(signal))
for peak in peaks:
peak_signal[peak[0]] = 1.0
signal += peak_signal
# add spikes for onsets
if config['onset_spikes']:
onsets = sp.detect_onsets(signal)
onset_signal = np.zeros(len(signal))
for onset in onsets:
onset_signal[onset] = 0.5
onset_signal[onset + 1] = 0.4
onset_signal[onset + 2] = 0.25
signal += onset_signal
# limit
signal = sp.limit(signal, 1.0)
signal *= 0.9 # hack, just controlling gain
signals.append(signal)
names.append(name)
i += 1
return signals, names
def get(self, view_name=None, output=None):
# add a header for unrestricted access
self.set_header("Access-Control-Allow-Origin", "*")
csv = False
# do the routing and load view module
if not len(view_name):
log.info("Listing views...")
views = ["/api/%s" % filename.split('.')[0] for filename in os.listdir(os.path.abspath(os.path.dirname(__file__))) if filename[0] != "_" and filename[-3:] == ".py"]
response = {'description': "API view endpoints", "views": views}
return self.json(response)
module_name = "api.%s" % view_name
try:
view = importlib.import_module(module_name)
log.info("Loaded %s module" % module_name)
except ImportError as e:
log.error(log.exc(e))
return self.error("View \"%s\" not recognized" % view_name)
if len(output):
if output == "csv":
csv = True
else:
feature_type = self.get_argument('FeatureType', None)
try:
return self.render("api/%s.html" % output, query=(self.request.uri).replace("/%s" % output, ""), feature_type=feature_type)
except Exception as e:
return self.error("Could not render %s" % output)
# time to build our search filter
search = {}
# special parsing for startDate and endDate
start_string = self.get_argument('startDate', None)
if start_string is not None:
try:
start_dt = util.parse_date(start_string, tz=config['local_tz'])
start_t = util.timestamp(start_dt)
end_string = self.get_argument('endDate', (start_dt + datetime.timedelta(days=1)).strftime("%Y-%m-%d"))
end_dt = util.parse_date(end_string, tz=config['local_tz'])
end_t = util.timestamp(end_dt)
log.debug("startDate %s" % start_dt)
log.debug("endDate %s" % end_dt)
search['t_utc'] = {'$gt': start_t, '$lt': end_t}
except Exception as e:
log.error(log.exc(e))
return self.error("Bad dates")
# special parsing for location
# expecting bounds (upper left (NW), lower right (SE)): lon_1,lat_1,lon_2,lat_2
# oka: 20,-17,26,-22 nyc: -75,41,-71,40
geo_bounds = self.get_argument('geoBounds', None)
if geo_bounds is not None:
try:
lon_1, lat_1, lon_2, lat_2 = [float(coord) for coord in geo_bounds.split(',')]
log.debug("geo_bounds %f,%f %f,%f" % (lon_1, lat_1, lon_2, lat_2))
search['geometry'] = {'$geoWithin': {'$geometry': {'type': "Polygon", 'coordinates': [[ [lon_1, lat_1], [lon_2, lat_1], [lon_2, lat_2], [lon_1, lat_2], [lon_1, lat_1] ]]}}}
except Exception as e:
log.error(log.exc(e))
return self.error("Bad geometry")
# special parsing for expeditionDay (overrides startDate / endDate)
expedition_day = self.get_argument('expeditionDay', None)
if expedition_day is not None:
try:
expedition = self.get_argument('expedition', config['expedition'])
expedition = self.get_argument('Expedition', expedition)
start_dt = util.parse_date(str(config['start_date'][expedition]), tz=config['local_tz'])
expedition_day = int(expedition_day) - 1
log.debug("%s days after %s" % (expedition_day, start_dt))
gt_t = util.timestamp(start_dt + datetime.timedelta(days=expedition_day))
lt_t = util.timestamp(start_dt + datetime.timedelta(days=expedition_day + 1))
search['t_utc'] = {'$gt': gt_t, '$lt': lt_t}
except Exception as e:
log.error(log.exc(e))
return self.error("Bad day")
# special parsing for resolution
resolution = strings.as_numeric(self.request.arguments['resolution'][0]) if 'resolution' in self.request.arguments else 0
# special parsing for SpeciesSearch
species_search = self.get_argument('speciesSearch', None)
if species_search is not None:
search['$text'] = {'$search': species_search}
# get limit and order
# limit = self.get_argument('limit', 100) # this fails on int arguments, which I think is a tornado bug
limit = strings.as_numeric(self.request.arguments['limit'][0]) if 'limit' in self.request.arguments else 100
order = self.request.arguments['order'][0].lower() if 'order' in self.request.arguments else 'ascending'
order = ASCENDING if order == "ascending" else DESCENDING
# get all the rest of the arguments and format as properties
try:
for param, value in self.request.arguments.items():
for i, item in enumerate(value):
item = item.decode('utf-8')
item = strings.as_numeric(item)
item = True if type(item) == str and item.lower() == "true" else item
item = False if type(item) == str and item.lower() == "false" else item
item = {'$exists': True} if item == '*' else item
value[i] = item
search[param] = value[0] if len(value) == 1 else value
search = {('properties.%s' % (strings.camelcase(param) if param != 't_utc' else 't_utc') if param != 'geometry' and param != '$text' else param): value for (param, value) in search.items() if param not in ['geoBounds', 'startDate', 'endDate', 'expeditionDay', 'limit', 'order', 'resolution', 'speciesSearch']}
except Exception as e:
log.error(log.exc(e))
return self.error("bad parameters")
# http://localhost:7777/api?geoBounds=20,-17,26,-22&startDate=2014-08-01&endDate=2014-09-01&Member=Jer
log.info("FILTER %s" % search)
# pass our search to the view module for execution and formatting
# try:
result = view.assemble(self, search, limit, order, resolution)
if result is None:
return
if csv:
return self.csv(format_csv(result), "data.csv")
results, total, returned = result
search = {key.replace('properties.', ''): value for (key, value) in search.items()}
return self.json({'order': order, 'limit': limit, 'total': total, 'returned': len(results) if returned is None else returned, 'filter': search, 'results': results, 'resolution': resolution if resolution != 0 else "full"})
import pymongo, time
from housepy import video, config, log, animation, util
from collections import deque
from mongo import db
PATH = "/Users/house/Projects/rats/bronx_lab/1467738431_.mov"
# PATH = "/Users/house/Projects/rats/bronx_lab/BDMV_5.mov"
# PATH = "/Users/house/Projects/rats/bronx_lab/video-720p-h264.mov"
START = "2016-07-05 12:30:00"
END = "2016-07-06"
log.info("Retrieving data...")
results = db.stream.find({'t_utc': {'$gt': util.timestamp(util.parse_date(START, tz='America/New_York')), '$lt': util.timestamp(util.parse_date(END, tz='America/New_York'))}}).sort([('t_utc', pymongo.ASCENDING)])
log.info("--> done")
ctx = animation.Context(640, 480, background=(1.0, 1.0, 1.0, 1.), fullscreen=False, title="collar stream")
graph = deque()
start_time = time.time()
current_data = results.next()
data_start_time = current_data['t_utc']
def draw():
global start_time, data_start_time, current_data
elapsed_time = time.time() - start_time
while True:
data_elapsed_time = current_data['t_utc'] - data_start_time
# center = -73.959486, 40.685193 # brooklyn
# center = -72.723889, 43.173611 # vermont # no points!
# center = -71.009755, 41.569593 # new bedford
# center = -93.219539, 44.933524 # minneapolis
# center = -77.059081, 38.948266 # dc
# center = -104.890219, 39.698841 # denver
# center = -83.961412, 35.935478 # knoxville
# center = -73.490419, 41.908486 # berkshires
# center = -74.035318, 41.498944 # hudson valley
# center = 127.032687, 37.635063 # seoul
# center = -71.221729, 42.306461 # boston
# center = -68.700278, 45.658056 # millinocket
# center = -118.334105, 34.045948 # LA
# almanac = Almanac.build(data, time.mktime(util.parse_date('2011-05-25').timetuple()), time.mktime(util.parse_date('2012-05-27').timetuple()), center)
almanac = Almanac.build(data, time.mktime(util.parse_date('2011-08-01').timetuple()), time.mktime(util.parse_date('2012-05-27').timetuple()), center)
# almanac = Almanac.build(data, time.mktime(util.parse_date('2012-01-01').timetuple()), time.mktime(util.parse_date('2012-05-27').timetuple()), center)
print
print "POINTS"
print np.array(almanac.points)
print
print "PATHS"
for path in almanac.paths:
print path
print
print "PLACES"
for place in almanac.places:
print place
