def run(house, wearables, delete_existing_workflows=True, loglevel=logging.INFO):
from hyperstream import HyperStream, TimeInterval, StreamNotFoundError
from workflows.asset_splitter import split_sphere_assets
from workflows.deploy_localisation_model import create_workflow_localisation_predict
# from workflows.deploy_localisation_model_new_api import create_workflow_localisation_predict
hyperstream = HyperStream(loglevel=loglevel, file_logger=None)
D = hyperstream.channel_manager.mongo
A = hyperstream.channel_manager.assets
experiment_ids = A.find_stream(name="experiments_selected", house=house).window(
TimeInterval.up_to_now()).last().value
experiment_ids_str = '_'.join(experiment_ids)
workflow_id0 = "asset_splitter"
workflow_id1 = "lda_localisation_model_predict_"+experiment_ids_str
if delete_existing_workflows:
hyperstream.workflow_manager.delete_workflow(workflow_id0)
hyperstream.workflow_manager.delete_workflow(workflow_id1)
split_sphere_assets(hyperstream, house)
try:
w = hyperstream.workflow_manager.workflows[workflow_id1]
except KeyError:
w = create_workflow_localisation_predict(hyperstream, house=house, experiment_ids=experiment_ids, safe=False)
hyperstream.workflow_manager.commit_workflow(workflow_id1)
# def safe_purge(channel, stream_id):
# try:
# channel.purge_stream(stream_id)
# except StreamNotFoundError:
# pass
# A.purge_node("wearables_by_house")
# A.purge_node("access_points_by_house")
# D.purge_node("predicted_locations_broadcasted")
# for h in [1, 2, 1176, 1116]:
# safe_purge(A, StreamId(name="wearables_by_house", meta_data=(('house', h),)))
# safe_purge(A, StreamId(name="access_points_by_house", meta_data=(('house', h),)))
# for w in wearables:
# safe_purge(D, StreamId(name="predicted_locations_broadcasted", meta_data=(('house', h), ('wearable', w))))
ti0 = TimeInterval.up_to_now()
ti1 = TimeInterval.now_minus(minutes=1)
# ti0 = TimeInterval(MIN_DATE, parse("2016-12-02 17:14:25.075Z"))
# ti1 = TimeInterval(start=ti0.end - timedelta(minutes=1), end=ti0.end)
w.execute(ti1)
print('number of non_empty_streams: {}'.format(
len(hyperstream.channel_manager.memory.non_empty_streams)))
from display_localisation_predictions import display_predictions
display_predictions(hyperstream, ti1, house, wearables=wearables)
def run(house,
selection,
delete_existing_workflows=True,
loglevel=logging.INFO):
from hyperstream import HyperStream, StreamId, TimeInterval
from workflows.display_experiments import create_workflow_list_technicians_walkarounds
from workflows.learn_localisation_model import create_workflow_lda_localisation_model_learner
from hyperstream.utils import StreamNotFoundError, reconstruct_interval
hyperstream = HyperStream(loglevel=loglevel, file_logger=None)
M = hyperstream.channel_manager.memory
D = hyperstream.channel_manager.mongo
A = hyperstream.channel_manager.assets
workflow_id0 = "list_technicians_walkarounds"
if delete_existing_workflows:
hyperstream.workflow_manager.delete_workflow(workflow_id0)
try:
w0 = hyperstream.workflow_manager.workflows[workflow_id0]
except KeyError:
w0 = create_workflow_list_technicians_walkarounds(hyperstream,
house=house,
safe=False)
hyperstream.workflow_manager.commit_workflow(workflow_id0)
time_interval = TimeInterval.up_to_now()
w0.execute(time_interval)
# from datetime import timedelta
# time_interval.end += timedelta(milliseconds=1)
df = M[StreamId('experiments_dataframe',
(('house', house), ))].window().values()[0]
experiment_ids = set([df['experiment_id'][i - 1] for i in selection])
experiment_ids_str = '_'.join(experiment_ids)
create_selected_localisation_plates(hyperstream)
# Ensure the model is overwritten if it's already there
for model_name in ('lda', 'svm', 'room_rssi_hmm'):
model_id = StreamId(name="location_prediction",
meta_data=(('house', house), ('localisation_model',
model_name)))
try:
hyperstream.channel_manager.mongo.purge_stream(model_id)
except StreamNotFoundError:
pass
workflow_id1 = "lda_localisation_model_learner_" + experiment_ids_str
if delete_existing_workflows:
hyperstream.workflow_manager.delete_workflow(workflow_id1)
try:
w1 = hyperstream.workflow_manager.workflows[workflow_id1]
except KeyError:
w1 = create_workflow_lda_localisation_model_learner(
hyperstream,
house=house,
experiment_ids=experiment_ids,
safe=False)
hyperstream.workflow_manager.commit_workflow(workflow_id1)
# Put the experiments selected into an asset stream
from hyperstream import StreamInstance
from hyperstream.utils import utcnow
A.write_to_stream(stream_id=StreamId(name="experiments_selected",
meta_data=(('house', house), )),
data=StreamInstance(timestamp=utcnow(),
value=list(experiment_ids)))
time_interval = TimeInterval.up_to_now()
w1.execute(time_interval)
print('number of non_empty_streams: {}'.format(
len(hyperstream.channel_manager.memory.non_empty_streams)))
for model_name in ('lda', 'svm', 'room_rssi_hmm'):
print("Model: {}".format(model_name))
model_id = StreamId(name="location_prediction",
meta_data=(('house', house), ('localisation_model',
model_name)))
try:
model = D[model_id].window().last().value
except (AttributeError, KeyError):
print(
"No {} model was learnt - not requested or no data recorded?".
format(model_name))
for experiment_id in list(experiment_ids):
print("Experiment id: {}".format(experiment_id))
print("Time interval: {}".format(
reconstruct_interval(experiment_id)))
print("Accuracy: {}".format(
pformat(model['performance'][experiment_id]['accuracy'])))
print("Macro F1: {}".format(
pformat(
model['performance'][experiment_id]['f1_score_macro'])))
print("Micro F1: {}".format(
pformat(
model['performance'][experiment_id]['f1_score_micro'])))
print("Confusion Matrix:")
pprint(model['performance'][experiment_id]['confusion_matrix'])
print("")
return True
def run(house,
selection,
delete_existing_workflows=True,
loglevel=logging.INFO):
from hyperstream import HyperStream, StreamId, TimeInterval
from workflows.display_experiments import create_workflow_list_technicians_walkarounds
from workflows.rssi_distributions_per_room import create_workflow_rssi_distributions_per_room
hyperstream = HyperStream(loglevel=loglevel, file_logger=None)
M = hyperstream.channel_manager.memory
workflow_id0 = "list_technicians_walkarounds"
if delete_existing_workflows:
hyperstream.workflow_manager.delete_workflow(workflow_id0)
try:
w0 = hyperstream.workflow_manager.workflows[workflow_id0]
except KeyError:
w0 = create_workflow_list_technicians_walkarounds(hyperstream,
house=house,
safe=False)
hyperstream.workflow_manager.commit_workflow(workflow_id0)
time_interval = TimeInterval.up_to_now()
w0.execute(time_interval)
df = M[StreamId('experiments_dataframe', (('house', house), ))].window(
TimeInterval.up_to_now()).values()[0]
experiment_indices = selection
experiment_ids = set([df['experiment_id'][i - 1] for i in selection])
hyperstream.plate_manager.delete_plate("H.SelectedLocalisationExperiment")
hyperstream.plate_manager.create_plate(
plate_id="H.SelectedLocalisationExperiment",
description=
"Localisation experiments selected by the technician in SPHERE house",
meta_data_id="localisation-experiment",
values=[],
complement=True,
parent_plate="H")
experiment_ids_str = '_'.join(experiment_ids)
# Create a simple one step workflow for querying
workflow_id1 = "rssi_distributions_per_room_" + experiment_ids_str
if delete_existing_workflows:
hyperstream.workflow_manager.delete_workflow(workflow_id1)
try:
w1 = hyperstream.workflow_manager.workflows[workflow_id1]
except KeyError:
w1 = create_workflow_rssi_distributions_per_room(
hyperstream,
house=house,
experiment_indices=experiment_indices,
experiment_ids=experiment_ids,
safe=False)
hyperstream.workflow_manager.commit_workflow(workflow_id1)
time_interval = TimeInterval.up_to_now()
w1.execute(time_interval)
df = M[StreamId('dataframe_' + experiment_ids_str,
(('house', house), ))].window(
TimeInterval.up_to_now()).values()[0]
df.to_csv(
os.path.join(hyperstream.config.output_path,
'dataframe_{}.csv'.format(experiment_ids_str)))
print('number of non_empty_streams: {}'.format(
len(hyperstream.channel_manager.memory.non_empty_streams)))
def run(house, delete_existing_workflows=True, loglevel=logging.INFO):
from hyperstream import HyperStream, TimeInterval
from hyperstream.utils import duration2str
from workflows.display_experiments import create_workflow_list_technicians_walkarounds
from workflows.asset_splitter import split_sphere_assets
hyperstream = HyperStream(loglevel=loglevel, file_logger=None)
# Various channels
M = hyperstream.channel_manager.memory
S = hyperstream.channel_manager.sphere
if delete_existing_workflows:
hyperstream.workflow_manager.delete_workflow("asset_splitter")
split_sphere_assets(hyperstream, house)
hyperstream.plate_manager.delete_plate("H")
hyperstream.plate_manager.create_plate(plate_id="H",
description="All houses",
meta_data_id="house",
values=[],
complement=True,
parent_plate=None)
workflow_id = "list_technicians_walkarounds"
if delete_existing_workflows:
hyperstream.workflow_manager.delete_workflow(workflow_id)
try:
w = hyperstream.workflow_manager.workflows[workflow_id]
except KeyError:
w = create_workflow_list_technicians_walkarounds(hyperstream,
house,
safe=False)
hyperstream.workflow_manager.commit_workflow(workflow_id)
time_interval = TimeInterval.up_to_now()
w.execute(time_interval)
print('number of sphere non_empty_streams: {}'.format(
len(S.non_empty_streams)))
print('number of memory non_empty_streams: {}'.format(
len(M.non_empty_streams)))
df = M.find_stream(name='experiments_dataframe',
house=house).window().values()[0]
if len(df) > 0:
# TODO: the following line gave a timezone error in some circumstances (when only 1 experiment?)
df['duration'] = df['end'] - df['start']
# TODO: the following 3 lines worked in that cas but give an error in some cases (when more than 1 experiment?)
# df['start_utc'] = pd.Series(pd.DatetimeIndex(df['start']).tz_convert('UTC'))
# df['end_utc'] = pd.Series(pd.DatetimeIndex(df['end']).tz_convert('UTC'))
# df['duration'] = df['end_utc'] - df['start_utc']
df['start'] = map(lambda x: '{:%Y-%m-%d %H:%M:%S}'.format(x),
df['start'])
df['end'] = map(lambda x: '{:%Y-%m-%d %H:%M:%S}'.format(x), df['end'])
# df['duration'] = map(lambda x:'{:%Mmin %Ssec}'.format(x),df['duration'])
df['start_as_text'] = map(lambda x: arrow.get(x).humanize(),
df['start'])
df['duration_as_text'] = map(lambda x: duration2str(x), df['duration'])
pd.set_option('display.width', 1000)
print(df[[
'id', 'start_as_text', 'duration_as_text', 'start', 'end',
'annotator'
]].to_string(index=False))
return True
else:
print("DataFrame is empty")
return False