def strips(points, user_id=None):
log.info("Drawing strips for user %s..." % user_id)
lines = []
q = 0
for p, point in enumerate(points):
prev = points[p - 1] if p > 0 else None
if prev is not None and point.period < prev.period:
q += 1
color = colors[point.location % len(colors)]
lines.append([(point.period / PERIODS) * 1000, q,
((point.period + point.duration) / PERIODS) * 1000, q,
color, 8.0])
if point.period + point.duration > PERIODS:
overflow = (point.period + point.duration) - PERIODS
lines.append(
[0, q + 1, (overflow / PERIODS) * 1000, q + 1, color, 8.0])
ctx = drawing.Context(1000, ((q + 2) * 10) + 2,
relative=False,
flip=False,
hsv=False,
background=(0., 0., 0., 1.))
for line in lines:
line[1] = line[3] = (((line[1] / (q + 2))) * ((q + 2) * 10)) + 6
ctx.line(*line)
ctx.output("images/%s_strips.png" % user_id, False)
def plot(walk_id, xs, ys, zs, ds, peaks, total_samples, fs):
try:
from housepy import drawing
except:
log.error("Can't draw")
return
# plot
ctx = drawing.Context(5000, 600, relative=True, flip=True)
ctx.line(200.0 / total_samples,
0.5,
350.0 / total_samples,
0.5,
thickness=10.0)
ctx.line([(float(i) / total_samples, x) for (i, x) in enumerate(xs)],
stroke=(1., 0., 0., 1.0)) # thickness=3.0)
ctx.line([(float(i) / total_samples, y) for (i, y) in enumerate(ys)],
stroke=(0., 1., 0., 1.0)) #, thickness=3.0)
ctx.line([(float(i) / total_samples, z) for (i, z) in enumerate(zs)],
stroke=(0., 0., 1., 1.0)) #, thickness=3.0)
ctx.line([(float(i) / total_samples, d) for (i, d) in enumerate(ds)],
stroke=(0., 0., 0.),
thickness=3.0)
ctx.line([(float(i) / total_samples, f) for (i, f) in enumerate(fs)],
stroke=(1., 0., 1.),
thickness=5.0)
for peak in peaks:
x, y = peak
x = float(x) / total_samples
ctx.arc(x,
y, (10.0 / ctx.width), (10.0 / ctx.height),
fill=(1., 0., 0.),
thickness=0.0)
ctx.output("charts/steps_%s_%s.png" % (walk_id, int(time.time())))
def path_print(points, index):
t = str(timeutil.timestamp(ms=True)).replace(".", "-")
log.info("Drawing path...")
ctx = drawing.Context(3000,
int(3000 / RATIO),
relative=True,
flip=True,
hsv=True)
ctx.image("basemap/basemap.png")
midline = sum([point.x for point in points]) / len(points)
poss = []
for p in range(len(points)):
x1, y1 = points[p].x, points[p].y
if p < len(points) - 1:
x2, y2 = points[p + 1].x, points[p + 1].y
ctx.line(x1, y1, x2, y2, stroke=(0., 0., .5, 1.), thickness=5.0)
ctx.arc(x1,
y1,
15 / ctx.width,
15 / ctx.height,
fill=(0., 0., 0., 1.),
thickness=0.0)
flip = False
if x1 < midline:
flip = True
for pos in poss:
dist_x = abs(x1 - pos[0]) * ctx.height
dist_y = abs(y1 - pos[1]) * ctx.height
if dist_y <= 100 and dist_x <= 400:
flip = not flip
if not flip:
x = x1 + (30 / ctx.width)
else:
x = x1 - (50 / ctx.width)
y = y1 - (12 / ctx.height)
poss.append((x, y))
ctx.label(x,
y,
str(p + 1),
stroke=(0., 0., 0., 1.),
font="Monaco",
size=36)
for p, point in enumerate(points):
label = "%d) %s %s%s" % (p + 1, "Wake up at" if p == 0 else "%s," %
point.display_time, point.address, "" if p !=
(len(points) - 1) else " ... sleep")
ctx.label((200 / ctx.width),
1.0 - ((200 + (40 * p)) / ctx.height),
label,
stroke=(0., 0., 0., 1.),
font="Monaco",
size=36)
ctx.output("images/%s_path.png" % (index, ))
log.info("--> done")
def gradient_test():
ctx = drawing.Context(1000, 250, relative=True, flip=True, hsv=True)
for x in range(1440):
# c = ((x/288) * 0.35) + .3
# c = ((x/288) * 0.35) + .0
c = ((x / 1440) * 0.65) + .0
c = x / 1440
ctx.line(x / 1440,
0,
x / 1440,
1,
stroke=(c, 1., 1., 1.),
thickness=(ctx.width / 1440) + 1)
ctx.output("gradient.png")
def days(days, user_id=None):
ctx = drawing.Context(1000,
len(days) * 10,
relative=True,
flip=False,
hsv=True,
background=(0., 0., 0., 1.))
for d, day in enumerate(days):
for period, point in enumerate(day):
# color = colors[point.location % len(colors)]
color = point.location / 100, 1., 1., 1.
ctx.line(period / PERIODS, (d / len(days)) + 5 / ctx.height,
(period + 1) / PERIODS, (d / len(days)) + 5 / ctx.height,
stroke=color,
thickness=8)
ctx.output("images/%s_days.png" % user_id, True)
def map(points, user_id=None):
log.info("Drawing map for user %s..." % user_id)
ctx = drawing.Context(3000,
int(3000 / RATIO),
relative=True,
flip=True,
hsv=True)
ctx.image("basemap/basemap.png")
for point in points:
color = point.location / 100, 1., 1., 1.
ctx.arc(point.x,
point.y,
6 / ctx.width,
6 / ctx.height,
fill=color,
thickness=0.0)
ctx.output("images/%d_map.png" % user_id, True)
log.info("--> done")
def path(points):
t = str(timeutil.timestamp(ms=True)).replace(".", "-")
log.info("Drawing path...")
ctx = drawing.Context(3000,
int(3000 / RATIO),
relative=True,
flip=True,
hsv=True)
ctx.image("basemap/basemap.png")
for p in range(len(points)):
x1, y1 = points[p].x, points[p].y
color = points[p].period / PERIODS, 1., 1., 1.
ctx.arc(x1,
y1,
5 / ctx.width,
5 / ctx.height,
fill=color,
thickness=0.0)
if p < len(points) - 1:
x2, y2 = points[p + 1].x, points[p + 1].y
ctx.line(x1, y1, x2, y2, stroke=color, thickness=1.0)
ctx.output("images/%s_path.png" % t)
log.info("--> done")
def main(session_id):
ctx = drawing.Context(2000, 1000)
for sensor in [2, 3, 4]:
result = db.branches.find({
'session': session_id,
'sensor': sensor
}).sort([('t', ASCENDING)])
if not result.count():
continue
result = list(result)
ts = [r['t'] for r in result]
xs = [r['sample'][0] for r in result]
ys = [r['sample'][1] for r in result]
zs = [r['sample'][2] for r in result]
rms = [r['sample'][3] for r in result]
duration = ts[-1] - ts[0]
SAMPLING_RATE = 50 # hz
x_signal = (sp.resample(ts, xs, duration * SAMPLING_RATE) -
RANGE[0]) / (RANGE[1] - RANGE[0])
y_signal = (sp.resample(ts, ys, duration * SAMPLING_RATE) -
RANGE[0]) / (RANGE[1] - RANGE[0])
z_signal = (sp.resample(ts, zs, duration * SAMPLING_RATE) -
RANGE[0]) / (RANGE[1] - RANGE[0])
rms_signal = (sp.resample(ts, rms, duration * SAMPLING_RATE) -
RANGE[0]) / (RANGE[1] - RANGE[0])
# ctx.plot(x_signal, stroke=(1.0, 0.0, 0.0, 1.0), thickness=2.0)
# ctx.plot(y_signal, stroke=(0.0, 1.0, 0.0, 1.0), thickness=2.0)
# ctx.plot(z_signal, stroke=(0.0, 0.0, 1.0, 1.0), thickness=2.0)
rms_signal = sp.normalize(rms_signal)
ctx.plot(rms_signal, stroke=colors[sensor], thickness=2.0)
ctx.output("graphs")
signals = []
labels = list(streams.keys())
log.info("LABELS %s" % labels)
for label in labels:
log.info(label)
ts = tses[label]
ts = [t_min] + ts + [t_max]
values = [d[label] if label in d else None for d in streams[label]]
values = [values[0]] + values + [values[-1]]
values = sp.remove_shots(values, nones=True) # repair missing values
signal = sp.resample(ts, values)
num_samples = len(signal)
sample_rate = num_samples / duration
signal = sp.normalize(signal)
signal = sp.smooth(signal, 15)
signals.append(signal)
log.info("Drawing...")
ctx = drawing.Context(1200, 500, margin=20, hsv=True)
for b in range(12):
ctx.line(b / 12, 0, b / 12, 1, stroke=(0.5, 0.5, 0.5, 0.5), thickness=0.5)
ctx.line(1, 0, 1, 1, stroke=(0.5, 0.5, 0.5, 0.5), thickness=0.5)
for i, signal in enumerate(signals):
color = i / (len(signals) + 4) + .1, 1., .8, 1.
ctx.plot(signal, stroke=color, thickness=1.5)
ctx.line(10 / ctx.width, 1 - ((10 + (i * 10)) / ctx.height), 30 / ctx.width, 1 - ((10 + (i * 10)) / ctx.height), stroke=color, thickness=2)
ctx.label(35 / ctx.width, 1 - ((13 + (i * 10)) / ctx.height), labels[i].upper(), size=8)
ctx.output("charts/")
# limit
signal = sp.limit(signal, 1.0)
signal *= 0.9 # hack, just controlling gain
signals.append(signal)
names.append(name)
i += 1
return signals, names
if __name__ == "__main__":
signals, names = generate()
ctx = drawing.Context(1500, 750)
for i, signal in enumerate(signals):
color = colors[i % len(colors)]
ctx.plot(signal, stroke=color, thickness=2)
ctx.line(10 / ctx.width,
1 - ((10 + (i * 10)) / ctx.height),
30 / ctx.width,
1 - ((10 + (i * 10)) / ctx.height),
stroke=color,
thickness=2)
ctx.label(35 / ctx.width,
1 - ((13 + (i * 10)) / ctx.height),
names[i].upper(),
size=10)
ctx.output("charts")
from housepy import log, config, strings, net, s3, util, process, drawing
from scipy.io import wavfile
DURATION = 10
AUDIO_TMP = os.path.abspath(os.path.join(os.path.dirname(__file__), "audio_tmp"))
t = sys.argv[1]
filename = "%s/%s.wav" % (AUDIO_TMP, t)
sample_rate, signal = wavfile.read(filename)
log.debug("samples %s" % len(signal))
log.debug("sample_rate %s" % sample_rate)
duration = float(len(signal)) / sample_rate
log.debug("duration %ss" % strings.format_time(duration))
signal = (np.array(signal).astype('float') / (2**16 * 0.5)) # assuming 16-bit PCM, -1 - 1
signal = abs(signal) # magnitude
ctx = drawing.Context()
ctx.plot(signal)
ctx.line(0, config['noise_threshold'], 1, config['noise_threshold'], stroke=(255, 0, 0))
ctx.output("screenshots")
log.debug("noise threshold is %s" % config['noise_threshold'])
log.debug("found magnitude")
content_samples = 0
for sample in signal:
if sample > config['noise_threshold']:
content_samples += 1
total_content_time = float(content_samples) / sample_rate
log.info("total_content_time %s" % total_content_time)
points = np.array([(result['location']['coordinates'][0],
result['location']['coordinates'][1], result['user_id'])
for result in results])
min_lon, max_lon = (np.min(points[:, 0]), np.max(points[:, 0]))
min_lat, max_lat = (np.min(points[:, 1]), np.max(points[:, 1]))
log.debug("%f %f %f %f" % (min_lon, max_lon, min_lat, max_lat))
min_x, max_y = geo.project((min_lon, max_lat))
max_x, min_y = geo.project((max_lon, min_lat))
ratio = (max_x - min_x) / (max_y - min_y)
ctx = drawing.Context(1000,
int(1000 / ratio),
relative=True,
flip=True,
hsv=True)
log.info("Drawing %d %d..." % (ctx.width, ctx.height))
for point in points:
x, y = geo.project((point[0], point[1]))
if x > max_x or x < min_x or y > max_y or y < min_y:
continue
x = (x - min_x) / (max_x - min_x)
y = (y - min_y) / (max_y - min_y)
ctx.arc(x,
def sample(draw=False):
log.info("START SAMPLE")
# get the time
# dt = timeutil.get_dt(tz=config['tz'])
# dt -= datetime.timedelta(days=300) # time adjustment if necessary for testing
# t_utc = timeutil.t_utc(dt)
t_utc = timeutil.t_utc()
dt = timeutil.get_dt(t_utc, tz=config['tz'])
log.info("CURRENT TIME %s" % timeutil.get_string(t_utc, tz=config['tz']))
# pull the last 24 hours worth -- we're going to normalize over that to set our dynamic levels
log.info(config['sites'][config['sample']])
# # this is the real-time last 24 hours
# query = {'site': config['sample'], 't_utc': {'$gt': t_utc - 86400, '$lt': t_utc}}
# log.info(query)
# results = db.entries.find(query)
# this is the last 24 hours we have
# assume updating every 15 minutes, last 24 hours is the last 96 results
results = db.entries.find({
'site': config['sample']
}).sort([('t_utc', DESCENDING)]).limit(96)
results = list(results)
results.reverse()
log.info("%s results" % len(results)) # should be 96
log.info(json.dumps(results[-1], indent=4,
default=lambda d: str(d))) # show the last one
# resample signals for each
ts = [d['t_utc'] for d in results]
duration = ts[-1] - ts[0]
log.info("DURATION %s %s" % (duration, timeutil.format_seconds(duration)))
signals = []
rates = []
labels = list(config['labels'].values())
labels.sort()
for i, label in enumerate(labels):
# log.debug(label)
try:
values = [d[label] if label in d else None for d in results]
values = sp.remove_shots(values,
nones=True) # repair missing values
signal = sp.resample(ts, values)
num_samples = len(signal)
sample_rate = num_samples / duration
rates.append(sample_rate)
signal = sp.normalize(signal)
signal = sp.smooth(signal, 15)
signals.append(signal)
except KeyError as e:
log.error(log.exc(e))
log.error(values)
# draw if desired
if draw:
from housepy import drawing
ctx = drawing.Context(1200, 500, margin=20, hsv=True)
for i, label in enumerate(labels):
color = i / len(labels), .8, .8, 1.
signal = signals[i]
ctx.plot(signal, stroke=color, thickness=2)
ctx.output("charts/")
# collapse into n-dimensional points
points = []
for i in range(len(signals[0])):
point = [signal[i] for signal in signals]
points.append(point)
# PCA to 4D -- this takes whatever data we've got and maximizes variation for our four panels
points = np.array(points)
# log.debug("INPUT: %s POINTS, %s DIMENSIONS" % points.shape)
points = decomposition.PCA(n_components=4).fit_transform(points)
# log.debug("OUTPUT: %s POINTS, %s DIMENSIONS" % points.shape)
# normalize each dimension independently, again amplifying dynamics
points = np.column_stack((sp.normalize(points[:, 0], np.min(points[:, 0]),
np.max(points[:, 0])),
sp.normalize(points[:, 1], np.min(points[:, 1]),
np.max(points[:, 1])),
sp.normalize(points[:, 1], np.min(points[:, 1]),
np.max(points[:, 2])),
sp.normalize(points[:, 2], np.min(points[:, 3]),
np.max(points[:, 3]))))
# now, for each time this is queried we want to return an interpolation between the last two points
# this essentially implements a delay that closes in on the most recent query
# ...hopefully to be refreshed with a new USGS reading when it gets there
# if that reading doesnt come, it's ok, it just hovers there until we proceed
# aaandd actually we want a couple of hours delay, because these come in at bulk every 1-4 hours
# we know we have 96 points. four hours back is 16 points
# interpolating between points -17 and -16 should give the most recent guaranteed smooth transitions
# transduction takes time, pues
point_a = points[-17]
point_b = points[-16]
# log.debug(point_a)
# log.debug(point_b)
# linear interpolation over 15 minutes
position = (((dt.minute % 15) * 60) + dt.second) / (15 * 60)
# log.debug(position)
point = [(point_a[i] * (1.0 - position)) + (point_b[i] * position)
for i in range(len(point_a))]
log.info("RESULT: %s" % point)
return point
def main(session_id):
result = db.branches.find({'session': session_id}).sort([('t', ASCENDING)])
if not result.count():
print("NO DATA!")
exit()
log.info("Start processing...")
result = list(result)
ts = [r['t'] for r in result]
rms = [r['sample'][3] for r in result]
duration = ts[-1] - ts[0]
SAMPLING_RATE = 60 # hz
log.info("DURATION %fs" % duration)
signal = sp.resample(ts, rms, duration * SAMPLING_RATE)
signal = sp.remove_shots(signal)
signal = sp.normalize(signal)
signal = sp.smooth(signal, 15)
# this number should match some lower frequency bound. ie, put this in hz.
# the smaller the number, the more it will affect small motion
# so this should be higher than the slowest motion we care about
# ie, dont care about motion over 0.5hz, which is 120 samples
trend = sp.smooth(signal, 120)
signal -= trend
signal += 0.5
atrend = sp.smooth(signal, 500)
## autocorrelation
auto = sp.autocorrelate(signal)
# this should be small -- if 60hz, fastest gesture would reasonably be half of that, so 30
peaks, valleys = sp.detect_peaks(auto, 10)
peaks = [peak for peak in peaks[1:] if peak[1] > 0.5]
partials = []
for peak in peaks:
frequency = SAMPLING_RATE / peak[0]
partial = frequency * 1000
partials.append([partial, float(peak[1])])
log.info("%d samps\t%fhz\t%f magnitude\t%f map" % (peak[0], frequency, peak[1], partial))
log.info(partials)
ctx = drawing.Context(2000, 750)
ctx.plot(auto, stroke=(0.0, 0.0, 0.0, 1.0), thickness=2.0)
for peak in peaks:
x = peak[0] / len(auto)
ctx.line(x, 0.0, x, peak[1], stroke=(1.0, 0.0, 0.0, 1.0))
ctx.output("graphs")
## audio
audio_signal = sp.make_audio(signal)
spectrum(audio_signal, SAMPLING_RATE)
AUDIO_RATE = 11025
filename = "%s.wav" % util.timestamp()
sound.write_audio(audio_signal, filename, AUDIO_RATE)
subprocess.call(["open", filename])
log.info("AUDIO DURATION %fs" % (duration / (AUDIO_RATE / SAMPLING_RATE)))
ctx = drawing.Context(2000, 750)
ctx.plot(signal, stroke=(0.0, 0.0, 0.0, 1.0), thickness=2.0)
ctx.plot(trend, stroke=(1.0, 0.0, 0.0, 1.0), thickness=2.0)
ctx.plot(atrend, stroke=(0.0, 0.0, 1.0, 1.0), thickness=2.0)
ctx.output("graphs")
log.info("--> done") # around 300ms
def spectrum(signal, rate):
log.info("Computing spectrogram...")
block_size = 512
block_overlap = block_size / 2 # power of two, default is 128
# freqs, ts, spectrum = spectrogram(sound.signal, fs=sound.rate, noverlap=block_overlap, nfft=block_size, detrend='constant', return_onesided=True, scaling='density', axis=-1, mode='psd', window=('tukey', 0.25), nperseg=block_overlap*2)
spectrum, freqs, ts, image = plt.specgram(signal,
NFFT=block_size,
Fs=rate,
noverlap=block_overlap)
# (plt is 3k smaller)
# print("spectrum", spectrum) # freq rows of time columns.
# print()
# print(freqs)
# print()
# print(ts)
log.info("--> done")
log.info("--> freq bins %s" % len(freqs))
log.info("--> time columns %s" % len(ts))
log.info("Drawing...")
# with gzip.open("spectrum.pklz", 'wb') as f:
# f.write(pickle.dumps(spectrum))
ctx = drawing.Context(
len(ts) * 1, len(freqs) * 1,
relative=True) # if it's not an even multiple, artifacts happen
pixel_width = ctx.width / len(spectrum[0])
pixel_height = ctx.height / len(spectrum)
# for y, row in enumerate(spectrum):
# for x, value in enumerate(row):
# v = min(value / (allmax / 500), 1.0)
# v = 1 - v
# # print((x * pixel_width) / ctx.width, (y * pixel_height) / ctx.height, pixel_width / ctx.width, pixel_height / ctx.height)
# ctx.rect((x * pixel_width) / ctx.width, (y * pixel_height) / ctx.height, pixel_width / ctx.width, pixel_height / ctx.height, fill=(v, v, v, 1.), stroke=(1., 0., 0., 0.), thickness=0.0)
# mx = spectrum.flatten().max()
# print("maximum", spectrum.flatten().max())
# print("minimum", spectrum.flatten().min())
spectrum = sp.normalize(
np.sqrt(spectrum), 0.0,
200.0) # sqrt compresses, good for power. 200 is a clipping threshold.
for y, row in enumerate(spectrum):
for x, v in enumerate(row):
ctx.line((x * pixel_width) / ctx.width,
(y * pixel_height) / ctx.height,
((x * pixel_width) + pixel_width) / ctx.width,
(y * pixel_height) / ctx.height,
stroke=(v, v, v, 1.),
thickness=pixel_height)
log.info("--> done")
ctx.output("charts/")