def _execute(self, sources, alignment_stream, interval):
data = list(sources[0].window(interval, force_calculation=True))
flattened = map(
lambda x: dict(
dict(timestamp=x.timestamp,
fold=next(iter(x.value['annotations']['Experiment']), None
),
location=next(iter(x.value['annotations']['Location']),
None)), **(x.value['rssi'])), data)
df = pd.DataFrame(flattened)
yield StreamInstance(interval.end, df)
def _execute(self, sources, alignment_stream, interval):
if self.breaks is not None:
breaks = self.breaks
else:
breaks = [
self.first_break + i * self.break_width
for i in range(self.n_breaks)
]
breaks = [-np.inf] + breaks + [np.inf]
for t, d in sources[0].window(interval, force_calculation=True):
yield StreamInstance(t, np.histogram(d, breaks)[0].tolist())
def reformat(self, doc):
doc = deepcopy(doc)
dt = doc.pop('datetime')
if 'hid' in doc and doc['hid'] is not None:
house_id = doc.pop('hid')
meta_data = ('house', house_id)
else:
house_id = self.default_house
meta_data = ('house', house_id)
return StreamMetaInstance(stream_instance=StreamInstance(dt, doc),
meta_data=meta_data)
def _execute(self, sources, alignment_stream, interval):
data = list(sources[0].window(interval, force_calculation=True))
flattened = map(lambda x: dict(dict(
experiment_id=construct_experiment_id(TimeInterval(x.value['start'], x.value['end'])),
start=x.value['start'],
end=x.value['end'],
annotator=x.value['annotator']
), **(x.value['notes'])), data)
df = pd.DataFrame(flattened)
df['id'] = range(1, len(df) + 1)
yield StreamInstance(interval.end, df)
def _execute(self, sources, alignment_stream, interval):
def get_chunk(data, time):
items = [d for t,d in data if t >= time.start and t <= time.end]
return [(time.end, items),]
for time, inter in sources[0].window(interval, force_calculation=True):
activities = list()
for t,items in get_chunk(sources[1].window(interval, force_calculation=True), inter):
activities = [i['video-Activity'] for i in items if 'video-Activity' in i.keys()]
yield StreamInstance(inter.end, activities)
def _execute(self, sources, alignment_stream, interval):
for time, data in sources[0].window(interval, force_calculation=True):
res = self.aids[:]
for i in range(len(res)):
res[i] = self.default_value
if "gw" in data.keys():
for subdoc in data["gw"]:
for i in range(len(self.aids)):
if subdoc["aid"]==self.aids[i]:
res[i] = subdoc["rssi"]
yield StreamInstance(time, res)
def _execute(self, sources, alignment_stream, interval):
for tt, rows in sources[0].window(interval, force_calculation=True):
vals = defaultdict(list)
for row in rows:
for kk, vv in row.value.iteritems():
if isinstance(vv, (int, float)):
vals[kk].append(vv)
yield StreamInstance(
tt, {kk: self.func(vv)
for kk, vv in vals.iteritems()})
def _execute(self, sources, alignment_stream, interval):
def get_chunk(data, time):
items = [d for t,d in data if t >= time.start and t <= time.end]
return [(time.end, items),]
for time, inter in sources[0].window(interval, force_calculation=True):
collection = list()
for t,items in get_chunk(sources[1].window(interval, force_calculation=True), inter):
collection = [i[self.element] for i in items if self.element in i.keys() and len(i[self.element]) > 0]
if len(collection) > 0:
yield StreamInstance(inter.end, collection)
def _execute(self, source, splitting_stream, interval, output_plate):
for timestamp, data in source.window(interval, force_calculation=True):
if self.element is None:
data_element = data
else:
if self.element in data:
data_element = data[self.element]
else:
continue
for key, value in data_element.items():
yield StreamMetaInstance(
StreamInstance(timestamp=timestamp, value=value),
(output_plate.meta_data_id, key))
def _execute(self, sources, alignment_stream, interval):
data_stream = sources[0]
data = iter(data_stream.window(interval + self.relative_interval))
window = []
future = []
for (t, _) in alignment_stream.window(interval,
force_calculation=True):
while (len(window) > 0) and (window[0][0] <= t + timedelta(
seconds=self.relative_interval.start)):
window = window[1:]
while (len(future) > 0) and (future[0][0] <= t + timedelta(
seconds=self.relative_interval.end)):
doc = future[0]
future = future[1:]
if t + timedelta(seconds=self.relative_interval.start) < doc[0] \
<= t + timedelta(seconds=self.relative_interval.end):
window.append(doc)
while True:
try:
doc = next(data)
if t + timedelta(seconds=self.relative_interval.start) < doc[0] \
<= t + timedelta(seconds=self.relative_interval.end):
window.append(doc)
elif doc[0] > t + timedelta(
seconds=self.relative_interval.end):
future.append(doc)
break
else:
pass
except StopIteration:
break
if self.values_only:
yield StreamInstance(t, map(lambda si: si.value, window))
else:
yield StreamInstance(t, list(window))
def _execute(self, sources, alignment_stream, interval):
if self.names and len(self.names) != len(sources):
raise TypeError(
"Tool AlignedMerge expected {} streams as input, got {} instead"
.format(len(self.names), len(sources)))
streams = [
list(source.window(interval, force_calculation=True))
for source in sources
]
# Get all available timestamps. see https://goo.gl/uyA7pt
timestamps = sorted(set().union(*([i.timestamp for i in s]
for s in streams)))
for timestamp in timestamps:
try:
tt, values = zip(*[s[0] for s in streams])
except IndexError:
logging.warn("{}: Stream empty (lengths {})".format(
self.name, ", ".join(map(str, map(len, streams)))))
return
# If one of the streams doesn't have this timestamp, it's a misalignment
matches = list(map(lambda t: t == timestamp, tt))
if all(matches):
if not self.names:
yield StreamInstance(timestamp, values)
else:
# noinspection PyTypeChecker
yield StreamInstance(
timestamp,
dict([(name, values[i])
for i, name in enumerate(self.names)]))
for i, match in enumerate(matches):
if match:
# remove from the lists
streams[i].pop(0)
def _execute(self, sources, alignment_stream, interval):
field = sources[0].stream_id.as_dict()['meta_data'][2][
1] # ['env_field']
params = self.field_specific_params[field]
if params['breaks'] is not None:
breaks = params['breaks']
else:
breaks = [
params['first_break'] + i * params['break_width']
for i in range(params['n_breaks'])
]
breaks = [-np.inf] + breaks + [np.inf]
for t, d in sources[0].window(interval, force_calculation=True):
yield StreamInstance(t, np.histogram(d, breaks)[0].tolist())
def _execute(self, sources, alignment_stream, interval):
for time, data in sources[0].window(interval, force_calculation=True):
annotation = data['annotation']
rssi = data['rssi']
yield StreamInstance(
time,
dict(camera=annotation['camera_id'],
exper=annotation['exper_id'],
person=annotation['person_id'],
rssi1=rssi[0],
rssi2=rssi[1],
rssi3=rssi[2],
na1=(1 if rssi[0] == -120 else 0),
na2=(1 if rssi[1] == -120 else 0),
na3=(1 if rssi[2] == -120 else 0)))
def generate_instance(self, timestamp, value, meta_data):
"""
Generate a stream instance, checking if the filters are matched
:param timestamp: The timestamp
:param value: The value
:param meta_data: The meta data
:return: The stream instance (or None)
"""
if not self.filters or meta_data in self.filters:
if self.filters:
logging.debug('Filtered in for {} from {}'.format(meta_data, self.filters))
return StreamMetaInstance(StreamInstance(timestamp=timestamp, value=value), meta_data)
if self.filters:
logging.debug('Filtered out for {} from {}'.format(meta_data, self.filters))
return None
def _execute(self, sources, alignment_stream, interval):
if interval.start < self.first:
interval.start = self.first
n_widths = int((interval.start - self.first).total_seconds() //
self.width.total_seconds())
lower = self.first + n_widths * self.width
upper = lower + self.width
while upper <= interval.end:
yield StreamInstance(upper, TimeInterval(lower, upper))
lower += self.increment
upper += self.increment
def _execute(self, sources, alignment_stream, interval):
data = sources[0].window(interval, force_calculation=True)
try:
experiment_ids = sources[1].window(
interval, force_calculation=True).last().value
except AttributeError:
return
mappings = []
for x in data:
experiment_interval = TimeInterval(x.value['start'],
x.value['end'])
experiment_id = construct_experiment_id(experiment_interval)
if experiment_id in experiment_ids:
mappings.append((experiment_id, experiment_interval))
yield StreamInstance(interval.end, mappings)
def _execute(self, sources, alignment_stream, interval):
s0 = sources[0].window(interval, force_calculation=True).items()
s1 = sources[1].window(interval, force_calculation=True).items()
if len(s0) == 0 or len(s1) == 0:
return
i = 0
j = 0
while i < len(s0) or j < len(s1):
if s0[i].timestamp == s1[j].timestamp:
yield StreamInstance(s0[i].timestamp,
float(s0[i].value) / s1[j].value)
i += 1
j += 1
elif s0[i].timestamp < s1[j].timestamp:
i += 1
else:
j += 1
def _execute(self, source, splitting_stream, interval, output_plate):
for timestamp, data in source.window(interval, force_calculation=True):
if self.element is None:
data_element = data
else:
if self.element in data:
data_element = data[self.element]
else:
continue
for key, value in data_element.items():
if not self.filters or key in self.filters:
if self.filters:
logging.debug('Filtered in for {} from {}'.format(
key, self.filters))
yield StreamMetaInstance(
StreamInstance(timestamp=timestamp, value=value),
(output_plate.meta_data_id, key))
else:
if self.filters:
logging.debug('Filtered out for {} from {}'.format(
key, self.filters))
def _execute(self, sources, alignment_stream, interval):
for time, data in sources[0].window(interval, force_calculation=True):
time_str = time.strftime("%Y-%m-%dT%H_%M_%S")
filename = [
os.path.join('output', time_str + self.filename_suffix)
]
params = filename + [
str(self.missing_impute_value),
str(self.n_histogram_bins),
str(self.n_color_bins),
str(self.width_inches),
str(self.height_inches)
]
cmd_with_params = ' '.join(R_COMMAND + R_SCRIPT + params)
logging.info("starting {}".format(cmd_with_params))
try:
p = Popen(R_COMMAND + R_SCRIPT + params, stdin=PIPE)
stdout_data = p.communicate(input=data)[0]
logging.info(stdout_data)
except:
pass
logging.info("finished {}".format(cmd_with_params))
yield StreamInstance(time, filename)
def _execute(self, sources, alignment_stream, interval):
expected_status_stream = sources[0].window(interval,
force_calculation=True)
counts_stream = (x for x in sources[1].window(
interval, force_calculation=self.force_calculation))
counts_stream_docs = [
x for x in sources[1].window(
interval, force_calculation=self.force_calculation)
]
counts_stream_docs = [
x for x in sources[1].window(
interval, force_calculation=self.force_calculation)
]
counts_stream_docs = [
x for x in sources[1].window(
interval, force_calculation=self.force_calculation)
]
counts_doc = None
for time, expected_status in expected_status_stream:
res = expected_status.copy()
res['data'] = False
if counts_doc is None:
try:
counts_doc = next(counts_stream)
except:
pass
if (counts_doc is not None) and (counts_doc.timestamp == time):
counts = counts_doc.value
try:
res['data'] = counts > 0
except:
try:
res['data'] = sum(counts) > 0
except:
res['data'] = sum(counts.values())
counts_doc = None
yield StreamInstance(time, res)
return w
def print_head(w, node_id, parent_plate_values, plate_values, interval, n=10, print_func=print):
print_func("Node: {}".format(node_id))
w.nodes[node_id].print_head(parent_plate_values, plate_values, interval, n, print_func)
# Some plate values for display purposes
h1 = (('house', '1'),)
wA = (('wearable', 'A'),)
locs = tuple(("location", loc) for loc in ["kitchen", "hallway", "lounge"])
RSS_DEV_AVG = {
(('house', '1'), ('location', 'kitchen')): [
StreamInstance(datetime(2015, 8, 6, 13, 35, 43, 2000, tzinfo=UTC), -85.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 45, 404000, tzinfo=UTC), -99.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 46, 806000, tzinfo=UTC), -89.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 47, 406000, tzinfo=UTC), -90.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 47, 606000, tzinfo=UTC), -90.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 47, 807000, tzinfo=UTC), -93.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 49, 8000, tzinfo=UTC), -93.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 49, 208000, tzinfo=UTC), -93.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 49, 408000, tzinfo=UTC), -93.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 49, 608000, tzinfo=UTC), -95.0)
],
(('house', '1'), ('location', 'hallway')): [
StreamInstance(datetime(2015, 8, 6, 13, 35, 36, 196000, tzinfo=UTC), -82.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 36, 396000, tzinfo=UTC), -82.0),
StreamInstance(datetime(2015, 8, 6, 13, 35, 36, 596000, tzinfo=UTC), -83.0),
def _execute(self, sources, alignment_stream, interval):
for (time, _) in sources[0].window(interval, force_calculation=True):
yield StreamInstance(
time, TimeInterval(time + self._lower, time + self._upper))
def _execute(self, sources, alignment_stream, interval):
sliding_window = sources[0].window(interval, force_calculation=True)
try:
t, doc = sliding_window.first()
sliding_window_start = doc.start
except:
return
data = iter(sources[1].window(TimeInterval(sliding_window_start,
interval.end),
force_calculation=True))
window = []
future = []
for time, rel_window in sliding_window:
lower = rel_window.start
upper = rel_window.end
# Prune the old data points from the window
num_to_remove = 0
for win_time, win_data in window:
if lower < win_time <= upper: # MK: changed from lower <= win_time <= upper
break
num_to_remove += 1
window = window[num_to_remove:]
# Add those stolen from the future
num_to_remove = 0
for doc in future:
fut_time, fut_data = doc
# if lower <= fut_time <= upper: (MK: this was a bug because things in the far future were thrown away from the future
# break
if fut_time > upper: # added by MK: if in the far future, then must remain in future
break
num_to_remove += 1
if fut_time >= lower:
window.append(doc)
future = future[num_to_remove:]
# Take data from the execute
while True:
try:
doc = next(data)
tt, dd = doc
if lower < tt <= upper: # MK: changed from lower <= win_time <= upper
window.append(doc)
elif tt > upper:
future.append(doc)
break
except StopIteration:
break
if self.include_time:
value = [stream_instance for stream_instance in window]
else:
value = [stream_instance.value for stream_instance in window]
if len(value) > 0:
yield StreamInstance(time, value)
else:
# TODO: Should we yield anything???
# yield StreamInstance(time, {})
pass
def _execute(self, sources, alignment_stream, interval):
data = list(sources[1].window(interval, force_calculation=True))
timestamps = [x.timestamp for x in data]
try:
interval2 = TimeInterval(min(timestamps) - datetime.timedelta(microseconds=1), max(timestamps))
except ValueError:
# TODO: Something more intelligent - This is raised when the source stream is empty
return
windows = iter(sources[0].window(interval2, force_calculation=True))
data = iter(data)
# run from self.start_time to interval.start to find out the state at the beginning of the interval
# then from there on yield documents according to the sliding window,
# containing the active annotations within the window
win_future = []
data_future = []
annotations = {}
while True:
if len(data_future) == 0:
try:
doc = next(data)
data_future.append(doc)
except StopIteration:
pass
if len(win_future) == 0:
try:
_, win = next(windows)
win_future.append(win)
win_start = win_future[0].start
win_end = win_future[0].end
win_annotations = deepcopy(annotations)
except StopIteration:
pass
if len(win_future) == 0:
return
# the current annotation has potential effect on the current window
if (len(data_future) > 0) and (data_future[0].timestamp <= win_end):
doc = data_future.pop(0)
tt, dd = doc
trigger = dd['trigger']
tier = dd['tier']
label = dd['label']
if not annotations.has_key(tier):
annotations[tier] = set()
if trigger == 1:
annotations[tier].add(label)
elif trigger == -1:
try:
annotations[tier].remove(label) # raises KeyError if removing a label which is not present
except KeyError:
logging.warn("At time {} label {} of tier {} ending without a start".
format(tt, label, tier))
else:
raise ValueError("trigger must have value +1 or -1")
if tt > win_start: # the current annotation is changing a tier within the current window
win_annotations[tier] = {'MIX'}
else:
win_annotations[tier] = annotations[tier].copy()
else:
# must yield a window because it has been finished
yield StreamInstance(win_end, win_annotations)
win_future.pop(0)
def _execute(self, sources, alignment_stream, interval):
for time, data in sources[0].window(interval, force_calculation=True):
res = dict()
for key in data.keys():
res[key] = self.const
yield StreamInstance(time, res)
def reformat(doc):
dt = doc['datetime']
del doc['datetime']
return StreamInstance(dt, doc)
def _execute(self, sources, alignment_stream, interval):
interval2 = TimeInterval(self.start_time, interval.end)
windows = iter(sources[0].window(interval, force_calculation=True))
data = iter(sources[1].window(interval2, force_calculation=True))
# run from self.start_time to interval.start to find out the state at the beginning of the interval
# then from there on yield documents according to the sliding window,
# containing the active annotations within the window
win_future = []
data_future = []
annotations = {}
last_experiment = 0
while True:
if len(data_future) == 0:
try:
doc = next(data)
data_future.append(doc)
except StopIteration:
pass
if len(win_future) == 0:
try:
_, win = next(windows)
win_future.append(win)
win_start = win_future[0].start
win_end = win_future[0].end
win_annotations = deepcopy(annotations)
except StopIteration:
pass
if len(win_future) == 0:
return
# the current annotation has potential effect on the current window
if (len(data_future) > 0) and (data_future[0].timestamp <=
win_end):
doc = data_future.pop(0)
tt, dd = doc
trigger = dd['trigger']
tier = dd['tier']
label = dd['label']
if label == "Experiment Time":
tier = "Experiment"
if trigger == 1:
last_experiment += 1
label = last_experiment
if not annotations.has_key(tier):
annotations[tier] = set()
if trigger == 1:
annotations[tier].add(label)
elif trigger == -1:
try:
annotations[tier].remove(
label
) # raises KeyError if removing a label which is not present
except KeyError:
logging.warn(
"At time {} label {} of tier {} ending without a start"
.format(tt, label, tier))
else:
raise ValueError("trigger must have value +1 or -1")
if tt > win_start: # the current annotation is changing a tier within the current window
win_annotations[tier] = set(['MIX'])
else:
win_annotations[tier] = annotations[tier].copy()
else: # must yield a window because it has been finished
if win_annotations.has_key("Experiment") and len(
win_annotations["Experiment"]) > 0:
yield StreamInstance(win_end, win_annotations)
win_future.pop(0)
def _execute(self, sources, alignment_stream, interval):
for time, data in sources[0].window(interval, force_calculation=True):
if self.key in data:
yield StreamInstance(time, data[self.key])
def _execute(self, sources, alignment_stream, interval):
for t, d in sources[0].window(interval, force_calculation=True):
if self.include_timestamps:
yield StreamInstance(t, self.func(t, d))
else:
yield StreamInstance(t, self.func(d))
def _execute(self, sources, alignment_stream, interval):
for source in sources:
for ts, items in source.window(interval, force_calculation=True):
yield StreamInstance(ts, items)
