예제 #1
0
def export_aggregated_events(pj: dict, parameters: dict, obsId: str):
    """
    export aggregated events

    Args:
        pj (dict): BORIS project
        parameters (dict): subjects, behaviors
        obsId (str): observation id

    Returns:
        tablib.Dataset:

    """
    logging.debug(f"function: export aggregated events {parameters} {obsId}")

    interval = parameters["time"]
    start_time = parameters[START_TIME]
    end_time = parameters[END_TIME]

    data = tablib.Dataset()
    observation = pj[OBSERVATIONS][obsId]

    # obs description
    obs_description = observation["description"]

    duration1 = []   # in seconds
    if observation[TYPE] in [MEDIA]:
        try:
            for mediaFile in observation[FILE][PLAYER1]:
                if MEDIA_INFO in observation:
                    duration1.append(observation[MEDIA_INFO]["length"][mediaFile])
        except Exception:
            duration1 = []

    obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obsId])
    if obs_length == Decimal("-1"):  # media length not available
        interval = TIME_EVENTS

    logging.debug(f"obs_length: {obs_length}")

    ok, msg, connector = db_functions.load_aggregated_events_in_db(pj,
                                                                   parameters[SELECTED_SUBJECTS],
                                                                   [obsId],
                                                                   parameters[SELECTED_BEHAVIORS])
    if connector is None:
        logging.critical(f"error when loading aggregated events in DB")
        return data

    # time
    cursor = connector.cursor()

    if interval == TIME_FULL_OBS:
        min_time = float(0)
        max_time = float(obs_length)

    if interval == TIME_EVENTS:
        try:
            min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0])
        except Exception:
            min_time = float(0)
        try:
            max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0])
        except Exception:
            max_time = float(obs_length)

    if interval == TIME_ARBITRARY_INTERVAL:
        min_time = float(start_time)
        max_time = float(end_time)

    # adapt start and stop to the selected time interval
    cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
                   (min_time, obsId, min_time, min_time, max_time, ))
    cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
                   (max_time, obsId, max_time, min_time, max_time, ))

    cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
                   (min_time, max_time, obsId, min_time, max_time, ))

    cursor.execute("DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)",
                   (obsId, min_time, min_time, max_time, max_time, ))


    behavioral_category = project_functions.behavior_category(pj[ETHOGRAM])

    for subject in parameters[SELECTED_SUBJECTS]:

        for behavior in parameters[SELECTED_BEHAVIORS]:

            cursor.execute("SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers",
                           (subject, behavior,))

            rows_distinct_modifiers = list(x[0] for x in cursor.fetchall())

            for distinct_modifiers in rows_distinct_modifiers:

                cursor.execute(("SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events "
                                "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"),
                               (subject, behavior, distinct_modifiers))
                rows = list(cursor.fetchall())

                for row in rows:

                    if observation[TYPE] in [MEDIA]:
                        if duration1:
                            mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1])][-1]
                            mediaFileString = observation[FILE][PLAYER1][mediaFileIdx]
                            try:
                                fpsString = observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]]
                            except Exception:
                                fpsString = "NA"
                        else:
                            try:
                                if len(observation[FILE][PLAYER1]) == 1:
                                    mediaFileString = observation[FILE][PLAYER1][0]
                                else:
                                    mediaFileString = "NA"
                            except Exception:
                                mediaFileString = "NA"
                            fpsString = "NA"

                    if observation[TYPE] in [LIVE]:
                        mediaFileString = "LIVE"
                        fpsString = "NA"

                    if row["type"] == POINT:

                        row_data = []
                        row_data.extend([obsId,
                                         observation["date"].replace("T", " "),
                                         obs_description,
                                         mediaFileString,
                                         f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA",
                                         fpsString])

                        # independent variables
                        if INDEPENDENT_VARIABLES in pj:
                            for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]):
                                if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]:
                                    row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]])
                                else:
                                    row_data.append("")

                        row_data.extend([subject,
                                         behavior,
                                         behavioral_category[behavior],
                                         row["modifiers"],
                                         POINT,
                                         f"{row['start']:.3f}",  # start
                                         f"{row['stop']:.3f}",  # stop
                                         "NA",  # duration
                                         row["comment"],
                                         ""
                                         ])
                        data.append(row_data)

                    if row["type"] == STATE:
                        if idx % 2 == 0:
                            row_data = []
                            row_data.extend([obsId,
                                             observation["date"].replace("T", " "),
                                             obs_description,
                                             mediaFileString,
                                             f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA",
                                             fpsString])

                            # independent variables
                            if INDEPENDENT_VARIABLES in pj:
                                for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]):
                                    if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]:
                                        row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]])
                                    else:
                                        row_data.append("")

                            row_data.extend([subject,
                                             behavior,
                                             behavioral_category[behavior],
                                             row["modifiers"],
                                             STATE,
                                             f"{row['start']:.3f}",
                                             f"{row['stop']:.3f}",
                                             f"{row['stop'] - row['start']:.3f}",
                                             row["comment"],
                                             row["comment_stop"]
                                             ])
                            data.append(row_data)

    return data
예제 #2
0
def create_events_plot(pj,
                       selected_observations,
                       parameters,
                       plot_colors=BEHAVIORS_PLOT_COLORS,
                       plot_directory="",
                       file_format="png"):

    """
    create a time diagram plot (sort of gantt chart)
    with matplotlib barh function (https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.barh.html)
    """

    selected_subjects = parameters[SELECTED_SUBJECTS]
    selected_behaviors = parameters[SELECTED_BEHAVIORS]
    include_modifiers = parameters[INCLUDE_MODIFIERS]
    interval = parameters[TIME_INTERVAL]
    start_time = parameters[START_TIME]
    end_time = parameters[END_TIME]

    ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
                                                                      selected_subjects,
                                                                      selected_observations,
                                                                      selected_behaviors)

    if not ok:
        return False, msg, None
    cursor = db_connector.cursor()

    # if modifiers not to be included set modifiers to ""
    if not include_modifiers:
        cursor.execute("UPDATE aggregated_events SET modifiers = ''")

    cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
    distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]

    # add selected behaviors that are not observed
    for behav in selected_behaviors:
        if [x for x in distinct_behav_modif if x[0] == behav] == []:
            distinct_behav_modif.append([behav, "-"])

    distinct_behav_modif = sorted(distinct_behav_modif)
    max_len = len(distinct_behav_modif)

    all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])]

    par1 = 1
    bar_height = 0.5
    init = dt.datetime(2017, 1, 1)

    for obs_id in selected_observations:

        if len(selected_subjects) > 1:
            fig, axs = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True)
        else:
            fig, ax = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True)
            axs = np.ndarray(shape=(1), dtype=type(ax))
            axs[0] = ax

        ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
            pj,
            selected_subjects,
            [obs_id],
            selected_behaviors)

        cursor = db_connector.cursor()
        # if modifiers not to be included set modifiers to ""
        if not include_modifiers:
            cursor.execute("UPDATE aggregated_events SET modifiers = ''")
        cursor = db_connector.cursor()

        cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
        distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]

        # add selected behaviors that are not observed
        if not parameters["exclude behaviors"]:
            for behav in selected_behaviors:
                if [x for x in distinct_behav_modif if x[0] == behav] == []:
                    distinct_behav_modif.append([behav, "-"])

        distinct_behav_modif = sorted(distinct_behav_modif)
        max_len = len(distinct_behav_modif)


        # time
        obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id])
        if obs_length == -1:  # media length not available
            interval = TIME_EVENTS

        if interval == TIME_FULL_OBS:
            min_time = float(0)
            max_time = float(obs_length)

        if interval == TIME_EVENTS:
            try:
                min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
            except Exception:
                min_time = float(0)
            try:
                max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
            except Exception:
                max_time = float(obs_length)

        if interval == TIME_ARBITRARY_INTERVAL:
            min_time = float(start_time)
            max_time = float(end_time)

        cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
                       (min_time, obs_id, min_time, min_time, max_time, ))
        cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
                       (max_time, obs_id, max_time, min_time, max_time, ))
        cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
                       (min_time, max_time, obs_id, min_time, max_time, ))

        ylabels = [" ".join(x) for x in distinct_behav_modif]
        for ax_idx, subject in enumerate(selected_subjects):

            if parameters["exclude behaviors"]:
                cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?", (subject, ))
                distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]

                # add selected behaviors that are not observed
                if not parameters["exclude behaviors"]:
                    for behav in selected_behaviors:
                        if [x for x in distinct_behav_modif if x[0] == behav] == []:
                            distinct_behav_modif.append([behav, "-"])

                distinct_behav_modif = sorted(distinct_behav_modif)
                max_len = len(distinct_behav_modif)
                ylabels = [" ".join(x) for x in distinct_behav_modif]

            if not ax_idx:
                axs[ax_idx].set_title(f"Observation {obs_id}\n{subject}", fontsize=14)
            else:
                axs[ax_idx].set_title(subject, fontsize=14)
            bars = {}
            i = 0
            for behavior_modifiers in distinct_behav_modif:
                behavior, modifiers = behavior_modifiers
                behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
                bars[behavior_modifiers_str] = []

                # total duration
                cursor.execute(("SELECT start, stop FROM aggregated_events "
                                "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"),
                               (obs_id, subject, behavior, modifiers,))
                for row in cursor.fetchall():
                    bars[behavior_modifiers_str].append((row["start"], row["stop"]))

                    start_date = matplotlib.dates.date2num(init + dt.timedelta(seconds=row["start"]))
                    end_date = matplotlib.dates.date2num(
                        init + dt.timedelta(seconds=row["stop"] + POINT_EVENT_PLOT_DURATION * (row["stop"] == row["start"])))
                    try:
                        bar_color = utilities.behavior_color(plot_colors, all_behaviors.index(behavior))
                    except Exception:
                        bar_color = "darkgray"
                    bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row["start"] else bar_color

                    # sage colors removed from matplotlib colors list
                    if bar_color in ["sage", "darksage", "lightsage"]:
                        bar_color = {"darksage": "#598556", "lightsage": "#bcecac", "sage": "#87ae73"}[bar_color]

                    try:
                        axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height,
                                         align="center", edgecolor=bar_color, color=bar_color, alpha=1)
                    except Exception:
                        axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height,
                                         align="center", edgecolor="darkgray", color="darkgray", alpha=1)

                i += 1

            axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1)
            pos = np.arange(par1, max_len * par1 + par1 + 1, par1)
            axs[ax_idx].set_yticks(pos[:len(ylabels)])

            axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10})

            axs[ax_idx].set_ylabel("Behaviors" + " (modifiers)" * include_modifiers, fontdict={"fontsize": 10})

            axs[ax_idx].set_xlim(left=matplotlib.dates.date2num(init + dt.timedelta(seconds=min_time)),
                                 right=matplotlib.dates.date2num(init + dt.timedelta(seconds=max_time + 1)))

            axs[ax_idx].grid(color="g", linestyle=":")
            axs[ax_idx].xaxis_date()
            axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
            axs[ax_idx].set_xlabel("Time (HH:MM:SS)", fontdict={"fontsize": 12})
            axs[ax_idx].invert_yaxis()

        fig.autofmt_xdate()
        plt.tight_layout()

        if len(selected_observations) > 1:
            plt.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.{file_format}")
        else:
            plt.show()
예제 #3
0
def synthetic_time_budget(pj: dict, selected_observations: list,
                          parameters_obs: dict):
    """
    create a synthetic time budget

    Args:
        pj (dict): project dictionary
        selected_observations (list): list of observations to include in time budget
        parameters_obs (dict):

    Returns:
        bool: True if everything OK
        str: message
        tablib.Dataset: dataset containing synthetic time budget data
    """
    try:
        selected_subjects = parameters_obs[SELECTED_SUBJECTS]
        selected_behaviors = parameters_obs[SELECTED_BEHAVIORS]
        include_modifiers = parameters_obs[INCLUDE_MODIFIERS]
        interval = parameters_obs["time"]
        start_time = parameters_obs["start time"]
        end_time = parameters_obs["end time"]

        parameters = [
            ["duration", "Total duration"],
            ["number", "Number of occurrences"],
            ["duration mean", "Duration mean"],
            ["duration stdev", "Duration std dev"],
            ["proportion of time", "Proportion of time"],
        ]

        data_report = tablib.Dataset()
        data_report.title = "Synthetic time budget"

        ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
            pj, selected_subjects, selected_observations, selected_behaviors)

        if not ok:
            return False, msg, None

        db_connector.create_aggregate("stdev", 1, StdevFunc)
        cursor = db_connector.cursor()

        # modifiers
        if include_modifiers:
            cursor.execute(
                "SELECT distinct behavior, modifiers FROM aggregated_events")
            distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
                                    for rows in cursor.fetchall()]
        else:
            cursor.execute("SELECT distinct behavior FROM aggregated_events")
            distinct_behav_modif = [[rows["behavior"], ""]
                                    for rows in cursor.fetchall()]

        # add selected behaviors that are not observed
        for behav in selected_behaviors:
            if [x for x in distinct_behav_modif if x[0] == behav] == []:
                distinct_behav_modif.append([behav, ""])

        behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects,
                                     distinct_behav_modif, include_modifiers,
                                     parameters)

        param_header = ["", "Total length (s)"]
        subj_header, behav_header, modif_header = [""] * len(param_header), [
            ""
        ] * len(param_header), [""] * len(param_header)

        for subj in selected_subjects:
            for behavior_modifiers in distinct_behav_modif:
                behavior, modifiers = behavior_modifiers
                behavior_modifiers_str = "|".join(
                    behavior_modifiers) if modifiers else behavior
                for param in parameters:
                    subj_header.append(subj)
                    behav_header.append(behavior)
                    modif_header.append(modifiers)
                    param_header.append(param[1])
        '''
        if parameters_obs["group observations"]:
            cursor.execute("UPDATE aggregated_events SET observation = 'all' " )
            #selected_observations = ["all"]
        '''

        data_report.append(subj_header)
        data_report.append(behav_header)
        if include_modifiers:
            data_report.append(modif_header)
        data_report.append(param_header)

        # select time interval
        for obs_id in selected_observations:

            ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
                pj, selected_subjects, [obs_id], selected_behaviors)

            if not ok:
                return False, msg, None

            db_connector.create_aggregate("stdev", 1, StdevFunc)
            cursor = db_connector.cursor()

            # if modifiers not to be included set modifiers to ""
            if not include_modifiers:
                cursor.execute("UPDATE aggregated_events SET modifiers = ''")

            # time
            obs_length = project_functions.observation_total_length(
                pj[OBSERVATIONS][obs_id])
            if obs_length == -1:
                obs_length = 0

            if interval == TIME_FULL_OBS:
                min_time = float(0)
                max_time = float(obs_length)

            if interval == TIME_EVENTS:
                try:
                    min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
                except Exception:
                    min_time = float(0)
                try:
                    max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
                except Exception:
                    max_time = float(obs_length)

            if interval == TIME_ARBITRARY_INTERVAL:
                min_time = float(start_time)
                max_time = float(end_time)

            # adapt start and stop to the selected time interval
            cursor.execute(
                "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
                (
                    min_time,
                    obs_id,
                    min_time,
                    min_time,
                    max_time,
                ))
            cursor.execute(
                "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
                (
                    max_time,
                    obs_id,
                    max_time,
                    min_time,
                    max_time,
                ))

            cursor.execute(
                "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
                (
                    min_time,
                    max_time,
                    obs_id,
                    min_time,
                    max_time,
                ))

            cursor.execute(
                "DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)",
                (
                    obs_id,
                    min_time,
                    min_time,
                    max_time,
                    max_time,
                ))

            for subject in selected_subjects:

                # check if behaviors are to exclude from total time
                time_to_subtract = 0
                if EXCLUDED_BEHAVIORS in parameters_obs:
                    for excluded_behav in parameters_obs[EXCLUDED_BEHAVIORS]:
                        cursor.execute((
                            "SELECT SUM(stop-start) "
                            "FROM aggregated_events "
                            "WHERE observation = ? AND subject = ? AND behavior = ? "
                        ), (
                            obs_id,
                            subject,
                            excluded_behav,
                        ))
                        for row in cursor.fetchall():
                            if row[0] is not None:
                                time_to_subtract += row[0]

                for behavior_modifiers in distinct_behav_modif:
                    behavior, modifiers = behavior_modifiers
                    behavior_modifiers_str = "|".join(
                        behavior_modifiers) if modifiers else behavior

                    cursor.execute((
                        "SELECT SUM(stop-start), COUNT(*), AVG(stop-start), stdev(stop-start) "
                        "FROM aggregated_events "
                        "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ? "
                    ), (
                        obs_id,
                        subject,
                        behavior,
                        modifiers,
                    ))

                    for row in cursor.fetchall():
                        behaviors[subject][behavior_modifiers_str][
                            "duration"] = (0 if row[0] is None else
                                           f"{row[0]:.3f}")

                        behaviors[subject][behavior_modifiers_str][
                            "number"] = 0 if row[1] is None else row[1]
                        behaviors[subject][behavior_modifiers_str][
                            "duration mean"] = (0 if row[2] is None else
                                                f"{row[2]:.3f}")
                        behaviors[subject][behavior_modifiers_str][
                            "duration stdev"] = (0 if row[3] is None else
                                                 f"{row[3]:.3f}")

                        if behavior not in parameters_obs[EXCLUDED_BEHAVIORS]:
                            try:
                                behaviors[subject][behavior_modifiers_str][
                                    "proportion of time"] = (
                                        0 if row[0] is None else
                                        f"{row[0] / ((max_time - min_time) - time_to_subtract):.3f}"
                                    )
                            except ZeroDivisionError:
                                behaviors[subject][behavior_modifiers_str][
                                    "proportion of time"] = "-"
                        else:
                            # behavior subtracted
                            behaviors[subject][behavior_modifiers_str][
                                "proportion of time"] = (
                                    0 if row[0] is None else
                                    f"{row[0] / (max_time - min_time):.3f}")

            columns = [obs_id, f"{max_time - min_time:0.3f}"]
            for subj in selected_subjects:
                for behavior_modifiers in distinct_behav_modif:
                    behavior, modifiers = behavior_modifiers
                    behavior_modifiers_str = "|".join(
                        behavior_modifiers) if modifiers else behavior

                    for param in parameters:
                        columns.append(
                            behaviors[subj][behavior_modifiers_str][param[0]])

            data_report.append(columns)
    except Exception:

        error_type, error_file_name, error_lineno = utilities.error_info(
            sys.exc_info())
        logging.critical(
            f"Error in edit_event function: {error_type} {error_file_name} {error_lineno}"
        )

        msg = f"Error type: {error_type}\nError file name: {error_file_name}\nError line number: {error_lineno}"

        logging.critical(msg)

        return (False, msg, tablib.Dataset())

    return True, msg, data_report
예제 #4
0
def create_behaviors_bar_plot(pj: dict,
                       selected_observations: list,
                       param: dict,
                       plot_directory: str,
                       output_format: str,
                       plot_colors:list=BEHAVIORS_PLOT_COLORS):
    """
    time budget bar plot

    Args:
        pj (dict): project
        param (dict): parameters
        plot_directory (str): path of directory
        output_format (str): image format

    Returns:
        dict:
    """

    selected_subjects = param[SELECTED_SUBJECTS]
    selected_behaviors = param[SELECTED_BEHAVIORS]
    time_interval = param["time"]
    start_time = param[START_TIME]
    end_time = param[END_TIME]

    parameters = ["duration", "number of occurences"]

    ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
                                                                      selected_subjects,
                                                                      selected_observations,
                                                                      selected_behaviors)

    if not ok:
        return {"error": True, "message": msg}
    try:

        # extract all behaviors from ethogram for colors in plot
        all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])]

        for obs_id in selected_observations:

            cursor = db_connector.cursor()
            # distinct behaviors
            cursor.execute("SELECT distinct behavior FROM aggregated_events WHERE observation = ?",
                           (obs_id,))
            distinct_behav = [rows["behavior"] for rows in cursor.fetchall()]

            # add selected behaviors that are not observed
            '''
            if not param[EXCLUDE_BEHAVIORS]:
                for behavior in selected_behaviors:
                    if [x for x in distinct_behav if x == behavior] == []:
                        distinct_behav.append(behavior)
            '''

            # distinct subjects
            cursor.execute("SELECT distinct subject FROM aggregated_events WHERE observation = ?",
                           (obs_id,))
            distinct_subjects = [rows["subject"] for rows in cursor.fetchall()]

            behaviors = init_behav(pj[ETHOGRAM],
                                distinct_subjects,
                                distinct_behav,
                                parameters)

            # plot creation
            if len(distinct_subjects) > 1:
                fig, axs = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True)
                fig2, axs2 = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True)

            else:
                fig, ax = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True)
                axs = np.ndarray(shape=(1), dtype=type(ax))
                axs[0] = ax

                fig2, ax2 = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True)
                axs2 = np.ndarray(shape=(1), dtype=type(ax2))
                axs2[0] = ax2

            fig.suptitle("Durations of behaviors")
            fig2.suptitle("Number of occurences of behaviors")

            # if modifiers not to be included set modifiers to ""
            cursor.execute("UPDATE aggregated_events SET modifiers = ''")

            # time
            obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id])
            if obs_length == -1:
                obs_length = 0

            if param["time"] == TIME_FULL_OBS:
                min_time = float(0)
                max_time = float(obs_length)

            if param["time"] == TIME_EVENTS:
                try:
                    min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
                except Exception:
                    min_time = float(0)
                try:
                    max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
                except Exception:
                    max_time = float(obs_length)

            if param["time"] == TIME_ARBITRARY_INTERVAL:
                min_time = float(start_time)
                max_time = float(end_time)

            cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
                        (min_time, obs_id, min_time, min_time, max_time, ))
            cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
                        (max_time, obs_id, max_time, min_time, max_time, ))
            cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
                        (min_time, max_time, obs_id, min_time, max_time, ))

            for ax_idx, subject in enumerate(sorted(distinct_subjects)):

                for behavior in distinct_behav:

                    # number of occurences
                    cursor.execute(("SELECT COUNT(*) AS count FROM aggregated_events "
                                    "WHERE observation = ? AND subject = ? AND behavior = ?"),
                                (obs_id, subject, behavior, ))
                    for row in cursor.fetchall():
                        behaviors[subject][behavior]["number of occurences"] = 0 if row["count"] is None else row["count"]

                    # total duration
                    if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]):
                        cursor.execute(("SELECT SUM(stop - start) AS duration FROM aggregated_events "
                                        "WHERE observation = ? AND subject = ? AND behavior = ?"),
                                    (obs_id, subject, behavior, ))
                        for row in cursor.fetchall():
                            behaviors[subject][behavior]["duration"] = 0 if row["duration"] is None else row["duration"]

                durations, n_occurences, colors, x_labels, colors_duration, x_labels_duration = [], [], [], [], [], []

                for behavior in sorted(distinct_behav):

                    if param[EXCLUDE_BEHAVIORS] and behaviors[subject][behavior]["number of occurences"] == 0:
                        continue

                    n_occurences.append(behaviors[subject][behavior]["number of occurences"])
                    x_labels.append(behavior)
                    try:
                        colors.append(utilities.behavior_color(plot_colors, all_behaviors.index(behavior)))
                    except Exception:
                        colors.append("darkgray")

                    if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]):
                        durations.append(behaviors[subject][behavior]["duration"])
                        x_labels_duration.append(behavior)
                        try:
                            colors_duration.append(utilities.behavior_color(plot_colors, all_behaviors.index(behavior)))
                        except Exception:
                            colors_duration.append("darkgray")


                #width = 0.35       # the width of the bars: can also be len(x) sequence

                axs2[ax_idx].bar(np.arange(len(n_occurences)),
                                n_occurences,
                                #width,
                                color=colors
                                )


                axs[ax_idx].bar(np.arange(len(durations)),
                                durations,
                                #width,
                                color=colors_duration
                                )

                if ax_idx == 0:
                    axs[ax_idx].set_ylabel("Duration (s)")
                axs[ax_idx].set_xlabel("Behaviors")
                axs[ax_idx].set_title(f"{subject}")

                axs[ax_idx].set_xticks(np.arange(len(durations)))
                axs[ax_idx].set_xticklabels(x_labels_duration, rotation='vertical', fontsize=8)

                if ax_idx == 0:
                    axs2[ax_idx].set_ylabel("Number of occurences")
                axs2[ax_idx].set_xlabel("Behaviors")
                axs2[ax_idx].set_title(f"{subject}")

                axs2[ax_idx].set_xticks(np.arange(len(n_occurences)))
                axs2[ax_idx].set_xticklabels(x_labels, rotation='vertical', fontsize=8)


            fig.align_labels()
            fig.tight_layout(rect=[0, 0.03, 1, 0.95])

            fig2.align_labels()
            fig2.tight_layout(rect=[0, 0.03, 1, 0.95])


            if plot_directory:
                # output_file_name = f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.{output_format}"
                fig.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.duration.{output_format}")
                fig2.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.number_of_occurences.{output_format}")
                plt.close()
            else:
                fig.show()
                fig2.show()

        return {}

    except Exception:
        error_type, error_file_name, error_lineno = utilities.error_info(sys.exc_info())
        logging.critical(f"Error in time budget bar plot: {error_type} {error_file_name} {error_lineno}")
        return {"error": True, "exception": sys.exc_info()}
def synthetic_time_budget_bin(pj: dict, selected_observations: list,
                              parameters_obs: dict):
    """
    create a synthetic time budget divised in time bin

    Args:
        pj (dict): project dictionary
        selected_observations (list): list of observations to include in time budget
        parameters_obs (dict):

    Returns:
        bool: True if everything OK
        str: message
        tablib.Dataset: dataset containing synthetic time budget data
    """
    def interval_len(interval):
        if interval.empty:
            return dec(0)
        else:
            return sum([x.upper - x.lower for x in interval])

    def interval_number(interval):
        if interval.empty:
            return dec(0)
        else:
            return len(interval)

    def interval_mean(interval):
        if interval.empty:
            return dec(0)
        else:
            return sum([x.upper - x.lower for x in interval]) / len(interval)

    def interval_std_dev(interval):
        if interval.empty:
            return "NA"
        else:
            try:
                return round(
                    statistics.stdev([x.upper - x.lower for x in interval]), 3)
            except:
                return "NA"

    try:
        selected_subjects = parameters_obs[SELECTED_SUBJECTS]
        selected_behaviors = parameters_obs[SELECTED_BEHAVIORS]
        include_modifiers = parameters_obs[INCLUDE_MODIFIERS]
        time_interval = parameters_obs["time"]
        start_time = parameters_obs[START_TIME]
        end_time = parameters_obs[END_TIME]
        time_bin_size = dec(parameters_obs[TIME_BIN_SIZE])

        parameters = [
            ["duration", "Total duration"],
            ["number", "Number of occurrences"],
            ["duration mean", "Duration mean"],
            ["duration stdev", "Duration std dev"],
            ["proportion of time", "Proportion of time"],
        ]

        data_report = tablib.Dataset()
        data_report.title = "Synthetic time budget with time bin"

        distinct_behav_modif = []
        for obs_id in selected_observations:
            for event in pj[OBSERVATIONS][obs_id][EVENTS]:
                if include_modifiers:
                    if (event[EVENT_BEHAVIOR_FIELD_IDX],
                            event[EVENT_MODIFIER_FIELD_IDX]
                        ) not in distinct_behav_modif:
                        distinct_behav_modif.append(
                            (event[EVENT_BEHAVIOR_FIELD_IDX],
                             event[EVENT_MODIFIER_FIELD_IDX]))
                else:
                    if (event[EVENT_BEHAVIOR_FIELD_IDX],
                            "") not in distinct_behav_modif:
                        distinct_behav_modif.append(
                            (event[EVENT_BEHAVIOR_FIELD_IDX], ""))

        distinct_behav_modif.sort()
        '''
        print("distinct_behav_modif", distinct_behav_modif)
        '''

        # add selected behaviors that are not observed
        for behav in selected_behaviors:
            if [x for x in distinct_behav_modif if x[0] == behav] == []:
                distinct_behav_modif.append([behav, ""])
        '''
        print("distinct_behav_modif with not observed behav", distinct_behav_modif)
        '''

        behaviors = init_behav_modif_bin(pj[ETHOGRAM], selected_subjects,
                                         distinct_behav_modif,
                                         include_modifiers, parameters)
        '''
        print("init behaviors", behaviors)

        print(f"selected subjects: {selected_subjects}")
        '''

        param_header = [
            "Observations id", "Total length (s)", "Time interval (s)"
        ]
        subj_header, behav_header, modif_header = [""] * len(param_header), [
            ""
        ] * len(param_header), [""] * len(param_header)
        subj_header[1] = "Subjects:"
        behav_header[1] = "Behaviors:"
        modif_header[1] = "Modifiers:"

        for subj in selected_subjects:
            for behavior_modifiers in distinct_behav_modif:
                behavior, modifiers = behavior_modifiers
                behavior_modifiers_str = "|".join(
                    behavior_modifiers) if modifiers else behavior
                for param in parameters:
                    subj_header.append(subj)
                    behav_header.append(behavior)
                    modif_header.append(modifiers)
                    param_header.append(param[1])

        data_report.append(subj_header)
        data_report.append(behav_header)
        if include_modifiers:
            data_report.append(modif_header)
        data_report.append(param_header)

        state_events_list = [
            pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in pj[ETHOGRAM]
            if STATE in pj[ETHOGRAM][x][TYPE].upper()
        ]
        # select time interval
        for obs_id in selected_observations:

            obs_length = project_functions.observation_total_length(
                pj[OBSERVATIONS][obs_id])

            if obs_length == -1:
                obs_length = 0
            if time_interval == TIME_FULL_OBS:
                min_time = dec(0)
                max_time = dec(obs_length)

            if time_interval == TIME_EVENTS:
                try:
                    min_time = dec(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
                except Exception:
                    min_time = dec(0)
                try:
                    max_time = dec(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
                except Exception:
                    max_time = dec(obs_length)

            if time_interval == TIME_ARBITRARY_INTERVAL:
                min_time = dec(start_time)
                max_time = dec(end_time)

            #print("observation:", obs_id)
            events_interval = {}
            mem_events_interval = {}

            for event in pj[OBSERVATIONS][obs_id][EVENTS]:
                if event[EVENT_SUBJECT_FIELD_IDX] == "":
                    current_subject = NO_FOCAL_SUBJECT
                else:
                    current_subject = event[EVENT_SUBJECT_FIELD_IDX]

                if current_subject not in selected_subjects:
                    continue
                if current_subject not in events_interval:
                    events_interval[current_subject] = {}
                    mem_events_interval[current_subject] = {}

                if include_modifiers:
                    modif = event[EVENT_MODIFIER_FIELD_IDX]
                else:
                    modif = ""
                if (event[EVENT_BEHAVIOR_FIELD_IDX],
                        modif) not in distinct_behav_modif:
                    continue

                if (event[EVENT_BEHAVIOR_FIELD_IDX],
                        modif) not in events_interval[current_subject]:
                    events_interval[current_subject][(
                        event[EVENT_BEHAVIOR_FIELD_IDX], modif)] = I.empty()
                    mem_events_interval[current_subject][(
                        event[EVENT_BEHAVIOR_FIELD_IDX], modif)] = []

                if event[EVENT_BEHAVIOR_FIELD_IDX] in state_events_list:
                    mem_events_interval[current_subject][(
                        event[EVENT_BEHAVIOR_FIELD_IDX],
                        modif)].append(event[EVENT_TIME_FIELD_IDX])
                    if len(mem_events_interval[current_subject][(
                            event[EVENT_BEHAVIOR_FIELD_IDX], modif)]) == 2:
                        events_interval[current_subject][(event[EVENT_BEHAVIOR_FIELD_IDX], modif)] |= \
                                I.closedopen(mem_events_interval[current_subject][(event[EVENT_BEHAVIOR_FIELD_IDX], modif)][0],
                                            mem_events_interval[current_subject][(event[EVENT_BEHAVIOR_FIELD_IDX], modif)][1])
                        mem_events_interval[current_subject][(
                            event[EVENT_BEHAVIOR_FIELD_IDX], modif)] = []
                else:
                    events_interval[current_subject][(
                        event[EVENT_BEHAVIOR_FIELD_IDX],
                        modif)] |= I.singleton(event[EVENT_TIME_FIELD_IDX])
            '''
            print("\n\n events interval", events_interval)
            '''

            time_bin_start = min_time

            if time_bin_size:
                time_bin_end = time_bin_start + time_bin_size
                if time_bin_end > max_time:
                    time_bin_end = max_time
            else:
                time_bin_end = max_time
            '''
            print("time_bin_start type", type(time_bin_start))
            '''
            while True:

                for subject in events_interval:

                    # check behavior to exclude from total time
                    time_to_subtract = 0
                    if EXCLUDED_BEHAVIORS in parameters_obs:
                        for behav in events_interval[subject]:
                            if behav[0] in parameters_obs.get(
                                    EXCLUDED_BEHAVIORS, []):
                                interval_intersec = events_interval[
                                    subject][behav] & I.closed(
                                        time_bin_start, time_bin_end)
                                time_to_subtract += interval_len(
                                    interval_intersec)

                    for behav in events_interval[subject]:

                        interval_intersec = events_interval[subject][
                            behav] & I.closed(time_bin_start, time_bin_end)

                        dur = interval_len(interval_intersec)
                        nocc = interval_number(interval_intersec)
                        mean = interval_mean(interval_intersec)
                        '''
                        print(interval_intersec)
                        print(subject, behav)

                        print("duration", dur)
                        print("n. occ", nocc)
                        print("mean", mean)
                        print("std dev", interval_std_dev(interval_intersec))

                        print(time_bin_start)
                        print(time_bin_end)
                        print(time_to_subtract)
                        '''
                        if behav[0] in parameters_obs.get(
                                EXCLUDED_BEHAVIORS, []):
                            proportion = dur / (
                                (time_bin_end - time_bin_start))
                        else:
                            proportion = dur / (
                                (time_bin_end - time_bin_start) -
                                time_to_subtract)

                        behaviors[subject][behav]["duration"] = dur
                        behaviors[subject][behav]["number"] = nocc
                        behaviors[subject][behav]["duration mean"] = mean
                        behaviors[subject][behav][
                            "duration stdev"] = interval_std_dev(
                                interval_intersec)
                        behaviors[subject][behav][
                            "proportion of time"] = f"{proportion:.3f}"

                columns = [
                    obs_id, f"{max_time - min_time:0.3f}",
                    f"{time_bin_start:.3f}-{time_bin_end:.3f}"
                ]
                for subject in selected_subjects:
                    for behavior_modifiers in distinct_behav_modif:
                        behavior, modifiers = behavior_modifiers
                        #behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
                        behavior_modifiers_str = behavior_modifiers

                        for param in parameters:
                            columns.append(behaviors[subject]
                                           [behavior_modifiers_str][param[0]])
                            #columns.append(behaviors[subject][behavior][param[0]])

                data_report.append(columns)

                time_bin_start = time_bin_end
                time_bin_end = time_bin_start + time_bin_size
                if time_bin_end > max_time:
                    time_bin_end = max_time
                #print(f"start: {time_bin_start} end: {time_bin_end}  max time: {max_time}")

                if time_bin_start == time_bin_end:
                    break

    except Exception:
        dialog.error_message("synthetic_time_budget_bin", sys.exc_info())
        return (False, tablib.Dataset())

    return True, data_report