Ejemplo n.º 1
0
def run(house, wearables, delete_existing_workflows=True, loglevel=logging.INFO):
    from hyperstream import HyperStream, TimeInterval, StreamNotFoundError
    from workflows.asset_splitter import split_sphere_assets
    from workflows.deploy_localisation_model import create_workflow_localisation_predict
    # from workflows.deploy_localisation_model_new_api import create_workflow_localisation_predict

    hyperstream = HyperStream(loglevel=loglevel, file_logger=None)
    D = hyperstream.channel_manager.mongo
    A = hyperstream.channel_manager.assets

    experiment_ids = A.find_stream(name="experiments_selected", house=house).window(
        TimeInterval.up_to_now()).last().value

    experiment_ids_str = '_'.join(experiment_ids)
    workflow_id0 = "asset_splitter"
    workflow_id1 = "lda_localisation_model_predict_"+experiment_ids_str

    if delete_existing_workflows:
        hyperstream.workflow_manager.delete_workflow(workflow_id0)
        hyperstream.workflow_manager.delete_workflow(workflow_id1)

    split_sphere_assets(hyperstream, house)

    try:
        w = hyperstream.workflow_manager.workflows[workflow_id1]
    except KeyError:
        w = create_workflow_localisation_predict(hyperstream, house=house, experiment_ids=experiment_ids, safe=False)
        hyperstream.workflow_manager.commit_workflow(workflow_id1)

    # def safe_purge(channel, stream_id):
    #     try:
    #         channel.purge_stream(stream_id)
    #     except StreamNotFoundError:
    #         pass

    # A.purge_node("wearables_by_house")
    # A.purge_node("access_points_by_house")
    # D.purge_node("predicted_locations_broadcasted")

    # for h in [1, 2, 1176, 1116]:
    #     safe_purge(A, StreamId(name="wearables_by_house", meta_data=(('house', h),)))
    #     safe_purge(A, StreamId(name="access_points_by_house", meta_data=(('house', h),)))
    #     for w in wearables:
    #         safe_purge(D, StreamId(name="predicted_locations_broadcasted", meta_data=(('house', h), ('wearable', w))))

    ti0 = TimeInterval.up_to_now()
    ti1 = TimeInterval.now_minus(minutes=1)

    # ti0 = TimeInterval(MIN_DATE, parse("2016-12-02 17:14:25.075Z"))
    # ti1 = TimeInterval(start=ti0.end - timedelta(minutes=1), end=ti0.end)

    w.execute(ti1)

    print('number of non_empty_streams: {}'.format(
        len(hyperstream.channel_manager.memory.non_empty_streams)))

    from display_localisation_predictions import display_predictions
    display_predictions(hyperstream, ti1, house, wearables=wearables)
Ejemplo n.º 2
0
def run(house,
        selection,
        delete_existing_workflows=True,
        loglevel=logging.INFO):
    from hyperstream import HyperStream, StreamId, TimeInterval
    from workflows.display_experiments import create_workflow_list_technicians_walkarounds
    from workflows.learn_localisation_model import create_workflow_lda_localisation_model_learner
    from hyperstream.utils import StreamNotFoundError, reconstruct_interval

    hyperstream = HyperStream(loglevel=loglevel, file_logger=None)
    M = hyperstream.channel_manager.memory
    D = hyperstream.channel_manager.mongo
    A = hyperstream.channel_manager.assets

    workflow_id0 = "list_technicians_walkarounds"

    if delete_existing_workflows:
        hyperstream.workflow_manager.delete_workflow(workflow_id0)

    try:
        w0 = hyperstream.workflow_manager.workflows[workflow_id0]
    except KeyError:
        w0 = create_workflow_list_technicians_walkarounds(hyperstream,
                                                          house=house,
                                                          safe=False)
        hyperstream.workflow_manager.commit_workflow(workflow_id0)
    time_interval = TimeInterval.up_to_now()
    w0.execute(time_interval)

    # from datetime import timedelta
    # time_interval.end += timedelta(milliseconds=1)
    df = M[StreamId('experiments_dataframe',
                    (('house', house), ))].window().values()[0]
    experiment_ids = set([df['experiment_id'][i - 1] for i in selection])

    experiment_ids_str = '_'.join(experiment_ids)

    create_selected_localisation_plates(hyperstream)

    # Ensure the model is overwritten if it's already there
    for model_name in ('lda', 'svm', 'room_rssi_hmm'):
        model_id = StreamId(name="location_prediction",
                            meta_data=(('house', house), ('localisation_model',
                                                          model_name)))
        try:
            hyperstream.channel_manager.mongo.purge_stream(model_id)
        except StreamNotFoundError:
            pass

    workflow_id1 = "lda_localisation_model_learner_" + experiment_ids_str

    if delete_existing_workflows:
        hyperstream.workflow_manager.delete_workflow(workflow_id1)

    try:
        w1 = hyperstream.workflow_manager.workflows[workflow_id1]
    except KeyError:
        w1 = create_workflow_lda_localisation_model_learner(
            hyperstream,
            house=house,
            experiment_ids=experiment_ids,
            safe=False)
        hyperstream.workflow_manager.commit_workflow(workflow_id1)

    # Put the experiments selected into an asset stream
    from hyperstream import StreamInstance
    from hyperstream.utils import utcnow

    A.write_to_stream(stream_id=StreamId(name="experiments_selected",
                                         meta_data=(('house', house), )),
                      data=StreamInstance(timestamp=utcnow(),
                                          value=list(experiment_ids)))

    time_interval = TimeInterval.up_to_now()
    w1.execute(time_interval)

    print('number of non_empty_streams: {}'.format(
        len(hyperstream.channel_manager.memory.non_empty_streams)))

    for model_name in ('lda', 'svm', 'room_rssi_hmm'):
        print("Model: {}".format(model_name))
        model_id = StreamId(name="location_prediction",
                            meta_data=(('house', house), ('localisation_model',
                                                          model_name)))
        try:
            model = D[model_id].window().last().value
        except (AttributeError, KeyError):
            print(
                "No {} model was learnt - not requested or no data recorded?".
                format(model_name))

        for experiment_id in list(experiment_ids):
            print("Experiment id: {}".format(experiment_id))
            print("Time interval: {}".format(
                reconstruct_interval(experiment_id)))
            print("Accuracy: {}".format(
                pformat(model['performance'][experiment_id]['accuracy'])))
            print("Macro F1: {}".format(
                pformat(
                    model['performance'][experiment_id]['f1_score_macro'])))
            print("Micro F1: {}".format(
                pformat(
                    model['performance'][experiment_id]['f1_score_micro'])))
            print("Confusion Matrix:")
            pprint(model['performance'][experiment_id]['confusion_matrix'])
            print("")
    return True
def run(house,
        selection,
        delete_existing_workflows=True,
        loglevel=logging.INFO):
    from hyperstream import HyperStream, StreamId, TimeInterval
    from workflows.display_experiments import create_workflow_list_technicians_walkarounds
    from workflows.rssi_distributions_per_room import create_workflow_rssi_distributions_per_room

    hyperstream = HyperStream(loglevel=loglevel, file_logger=None)
    M = hyperstream.channel_manager.memory

    workflow_id0 = "list_technicians_walkarounds"

    if delete_existing_workflows:
        hyperstream.workflow_manager.delete_workflow(workflow_id0)

    try:
        w0 = hyperstream.workflow_manager.workflows[workflow_id0]
    except KeyError:
        w0 = create_workflow_list_technicians_walkarounds(hyperstream,
                                                          house=house,
                                                          safe=False)
        hyperstream.workflow_manager.commit_workflow(workflow_id0)
    time_interval = TimeInterval.up_to_now()
    w0.execute(time_interval)

    df = M[StreamId('experiments_dataframe', (('house', house), ))].window(
        TimeInterval.up_to_now()).values()[0]
    experiment_indices = selection
    experiment_ids = set([df['experiment_id'][i - 1] for i in selection])

    hyperstream.plate_manager.delete_plate("H.SelectedLocalisationExperiment")
    hyperstream.plate_manager.create_plate(
        plate_id="H.SelectedLocalisationExperiment",
        description=
        "Localisation experiments selected by the technician in SPHERE house",
        meta_data_id="localisation-experiment",
        values=[],
        complement=True,
        parent_plate="H")

    experiment_ids_str = '_'.join(experiment_ids)
    # Create a simple one step workflow for querying
    workflow_id1 = "rssi_distributions_per_room_" + experiment_ids_str

    if delete_existing_workflows:
        hyperstream.workflow_manager.delete_workflow(workflow_id1)

    try:
        w1 = hyperstream.workflow_manager.workflows[workflow_id1]
    except KeyError:
        w1 = create_workflow_rssi_distributions_per_room(
            hyperstream,
            house=house,
            experiment_indices=experiment_indices,
            experiment_ids=experiment_ids,
            safe=False)
        hyperstream.workflow_manager.commit_workflow(workflow_id1)

    time_interval = TimeInterval.up_to_now()
    w1.execute(time_interval)

    df = M[StreamId('dataframe_' + experiment_ids_str,
                    (('house', house), ))].window(
                        TimeInterval.up_to_now()).values()[0]
    df.to_csv(
        os.path.join(hyperstream.config.output_path,
                     'dataframe_{}.csv'.format(experiment_ids_str)))

    print('number of non_empty_streams: {}'.format(
        len(hyperstream.channel_manager.memory.non_empty_streams)))
def run(house, delete_existing_workflows=True, loglevel=logging.INFO):
    from hyperstream import HyperStream, TimeInterval
    from hyperstream.utils import duration2str
    from workflows.display_experiments import create_workflow_list_technicians_walkarounds
    from workflows.asset_splitter import split_sphere_assets

    hyperstream = HyperStream(loglevel=loglevel, file_logger=None)

    # Various channels
    M = hyperstream.channel_manager.memory
    S = hyperstream.channel_manager.sphere

    if delete_existing_workflows:
        hyperstream.workflow_manager.delete_workflow("asset_splitter")

    split_sphere_assets(hyperstream, house)

    hyperstream.plate_manager.delete_plate("H")
    hyperstream.plate_manager.create_plate(plate_id="H",
                                           description="All houses",
                                           meta_data_id="house",
                                           values=[],
                                           complement=True,
                                           parent_plate=None)

    workflow_id = "list_technicians_walkarounds"

    if delete_existing_workflows:
        hyperstream.workflow_manager.delete_workflow(workflow_id)

    try:
        w = hyperstream.workflow_manager.workflows[workflow_id]
    except KeyError:
        w = create_workflow_list_technicians_walkarounds(hyperstream,
                                                         house,
                                                         safe=False)
        hyperstream.workflow_manager.commit_workflow(workflow_id)
    time_interval = TimeInterval.up_to_now()
    w.execute(time_interval)

    print('number of sphere non_empty_streams: {}'.format(
        len(S.non_empty_streams)))
    print('number of memory non_empty_streams: {}'.format(
        len(M.non_empty_streams)))

    df = M.find_stream(name='experiments_dataframe',
                       house=house).window().values()[0]

    if len(df) > 0:
        # TODO: the following line gave a timezone error in some circumstances (when only 1 experiment?)
        df['duration'] = df['end'] - df['start']
        # TODO: the following 3 lines worked in that cas but give an error in some cases (when more than 1 experiment?)
        # df['start_utc'] = pd.Series(pd.DatetimeIndex(df['start']).tz_convert('UTC'))
        # df['end_utc'] = pd.Series(pd.DatetimeIndex(df['end']).tz_convert('UTC'))
        # df['duration'] = df['end_utc'] - df['start_utc']
        df['start'] = map(lambda x: '{:%Y-%m-%d %H:%M:%S}'.format(x),
                          df['start'])
        df['end'] = map(lambda x: '{:%Y-%m-%d %H:%M:%S}'.format(x), df['end'])
        # df['duration'] = map(lambda x:'{:%Mmin %Ssec}'.format(x),df['duration'])

        df['start_as_text'] = map(lambda x: arrow.get(x).humanize(),
                                  df['start'])
        df['duration_as_text'] = map(lambda x: duration2str(x), df['duration'])

        pd.set_option('display.width', 1000)
        print(df[[
            'id', 'start_as_text', 'duration_as_text', 'start', 'end',
            'annotator'
        ]].to_string(index=False))
        return True
    else:
        print("DataFrame is empty")
        return False