def synthetic_time_budget(pj, selected_observations, parameters_obs ): selected_subjects = parameters_obs["selected subjects"] selected_behaviors = parameters_obs["selected behaviors"] include_modifiers = parameters_obs["include modifiers"] interval = parameters_obs["time"] start_time = parameters_obs["start time"] end_time = parameters_obs["end time"] parameters = [["duration", "Total duration"], ["number", "Number of occurrences"], ["duration mean", "Duration mean"], ["duration stdev", "Duration std dev"], ["proportion of time", "Proportion of time"], ] data_report = tablib.Dataset() data_report.title = "Synthetic time budget" ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None db_connector.create_aggregate("stdev", 1, StdevFunc) cursor = db_connector.cursor() # modifiers if include_modifiers: cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] else: cursor.execute("SELECT distinct behavior FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()] # add selected behaviors that are not observed #if not parameters_obs["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects, distinct_behav_modif, include_modifiers, parameters) param_header = ["", "Total length (s)"] subj_header, behav_header, modif_header= [""]*len(param_header), [""]*len(param_header), [""]*len(param_header) for subj in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior for param in parameters: subj_header.append(subj) behav_header.append(behavior) modif_header.append(modifiers) param_header.append(param[1]) ''' if parameters_obs["group observations"]: cursor.execute("UPDATE aggregated_events SET observation = 'all' " ) #selected_observations = ["all"] ''' data_report.append(subj_header) data_report.append(behav_header) if include_modifiers: data_report.append(modif_header) data_report.append(param_header) # select time interval for obs_id in selected_observations: '''if not parameters_obs["group observations"]:''' ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, [obs_id], selected_behaviors) if not ok: return False, msg, None db_connector.create_aggregate("stdev", 1, StdevFunc) cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) #duration = end_time - start_time cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obs_id, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obs_id, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obs_id, min_time, max_time, )) for subject in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior cursor.execute(("SELECT SUM(stop-start), COUNT(*), AVG(stop-start), stdev(stop-start) " "FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"), (obs_id, subject, behavior, modifiers,)) for row in cursor.fetchall(): behaviors[subject][behavior_modifiers_str]["duration"] = 0 if row[0] is None else row[0] behaviors[subject][behavior_modifiers_str]["number"] = 0 if row[1] is None else row[1] behaviors[subject][behavior_modifiers_str]["duration mean"] = 0 if row[2] is None else row[2] behaviors[subject][behavior_modifiers_str]["duration stdev"] = 0 if row[3] is None else row[3] behaviors[subject][behavior_modifiers_str]["proportion of time"] = 0 if row[0] is None else row[0]/(max_time - min_time) columns = [obs_id, "{:0.3f}".format(max_time - min_time)] for subj in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior for param in parameters: columns.append(behaviors[subj][behavior_modifiers_str][param[0]]) data_report.append(columns) return True, msg, data_report
def behaviors_bar_plot(pj, selected_observations, selected_subjects, selected_behaviors, include_modifiers, interval, start_time, end_time, plot_directory, output_format): """ scatter plot """ parameters = [ ["duration", "Total duration"], ] ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() if include_modifiers: cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] else: cursor.execute("SELECT distinct behavior FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects, distinct_behav_modif, include_modifiers, parameters) # select time interval for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True) else: fig, ax = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length( pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute( "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", ( min_time, obs_id, min_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", ( max_time, obs_id, max_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", ( min_time, max_time, obs_id, min_time, max_time, )) for subject in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers # skip if behavior defined as POINT if POINT in project_functions.event_type( behavior, pj[ETHOGRAM]): continue behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior # total duration cursor.execute(( "SELECT SUM(stop-start) FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?" ), ( obs_id, subject, behavior, modifiers, )) for row in cursor.fetchall(): behaviors[subject][behavior_modifiers_str][ "duration"] = 0 if row[0] is None else row[0] print("behaviors") print(behaviors) print() print("sorted(distinct_behav_modif)", sorted(distinct_behav_modif)) max_length = 0 behaviors_duration = {} mb = {} for ax_idx, subj in enumerate(selected_subjects): behaviors_duration[subj] = {} behavior_ticks = [] for behavior_modifiers in sorted(distinct_behav_modif): behavior, modifiers = behavior_modifiers # skip if behavior defined as POINT if POINT in project_functions.event_type( behavior, pj[ETHOGRAM]): continue if behavior not in behaviors_duration[subj]: behaviors_duration[subj][behavior] = [[], []] behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior if behavior not in behavior_ticks: behavior_ticks.append(behavior) for param in parameters: behaviors_duration[subj][behavior][0].append( behaviors[subj][behavior_modifiers_str][param[0]]) behaviors_duration[subj][behavior][1].append(modifiers) max_length = max( max_length, len(behaviors_duration[subj][behavior][1])) print() print("behaviors_duration", behaviors_duration) print() print("behavior_ticks", behavior_ticks) print() behavior_mod_ticks = behavior_ticks[:] for ax_idx, subj in enumerate(selected_subjects): print("subject", subj) md_lgd = [] b = {} for i in range(max_length): b[i] = [] for behavior in sorted(behaviors_duration[subj].keys()): try: b[i].append(behaviors_duration[subj][behavior][0][i]) if include_modifiers: if behaviors_duration[subj][behavior][1][i]: md_lgd.append( behavior + " " + behaviors_duration[subj][behavior][1][i]) else: md_lgd.append(behavior) except Exception: b[i].append(0) print() print("behavior_mod_ticks", behavior_mod_ticks) print() print("b") print(b) print() print("md_lgd") print(md_lgd) ind = np.arange(len(behavior_ticks)) width = 0.35 # the width of the bars: can also be len(x) sequence pp = [] max_obs = 0 bottom_ = [] idx_color = 0 for i in sorted(b.keys()): if i == 0: pp.append(axs[ax_idx].bar( ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])])) else: pp.append(axs[ax_idx].bar( ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])], bottom=bottom_)) idx_color += len(b[i]) if not bottom_: bottom_ = b[i] else: bottom_ = [x + bottom_[idx] for idx, x in enumerate(b[i])] max_obs = max(max_obs, sum(b[i])) if ax_idx == 0: axs[ax_idx].set_ylabel("Duration (s)") axs[ax_idx].set_xlabel("Behaviors") axs[ax_idx].set_title(f"{subj}") axs[ax_idx].set_xticks(ind) axs[ax_idx].set_xticklabels(behavior_mod_ticks, rotation=90) axs[ax_idx].set_yticks(np.arange(0, max(bottom_), 50)) lgd_col = [] for p in pp: for r in p: if r.get_height(): lgd_col.append(r) plt.legend(lgd_col, md_lgd) if plot_directory: output_file_name = str( pathlib.Path( pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix("." + file_format)) plt.savefig(output_file_name) else: plt.show()
def export_aggregated_events(pj: dict, parameters: dict, obsId: str): """ export aggregated events Args: pj (dict): BORIS project parameters (dict): subjects, behaviors obsId (str): observation id Returns: tablib.Dataset: """ logging.debug(f"function: export aggregated events {parameters} {obsId}") interval = parameters["time"] start_time = parameters[START_TIME] end_time = parameters[END_TIME] data = tablib.Dataset() observation = pj[OBSERVATIONS][obsId] duration1 = [] # in seconds if observation[TYPE] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: if MEDIA_INFO in observation: duration1.append( observation[MEDIA_INFO]["length"][mediaFile]) except Exception: duration1 = [] obs_length = project_functions.observation_total_length( pj[OBSERVATIONS][obsId]) if obs_length == Decimal("-1"): # media length not available interval = TIME_EVENTS logging.debug(f"obs_length: {obs_length}") ok, msg, connector = db_functions.load_aggregated_events_in_db( pj, parameters[SELECTED_SUBJECTS], [obsId], parameters[SELECTED_BEHAVIORS]) if connector is None: logging.critical(f"error when loading aggregated events in DB") return data # time cursor = connector.cursor() if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) # adapt start and stop to the selected time interval cursor.execute( "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", ( min_time, obsId, min_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", ( max_time, obsId, max_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", ( min_time, max_time, obsId, min_time, max_time, )) cursor.execute( "DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)", ( obsId, min_time, min_time, max_time, max_time, )) behavioral_category = project_functions.behavior_category(pj[ETHOGRAM]) for subject in parameters[SELECTED_SUBJECTS]: for behavior in parameters[SELECTED_BEHAVIORS]: cursor.execute( "SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers", ( subject, behavior, )) rows_distinct_modifiers = list(x[0] for x in cursor.fetchall()) for distinct_modifiers in rows_distinct_modifiers: cursor.execute(( "SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events " "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start" ), (subject, behavior, distinct_modifiers)) rows = list(cursor.fetchall()) for row in rows: if observation[TYPE] in [MEDIA]: if duration1: mediaFileIdx = [ idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1]) ][-1] mediaFileString = observation[FILE][PLAYER1][ mediaFileIdx] try: fpsString = observation[MEDIA_INFO]["fps"][ observation[FILE][PLAYER1][mediaFileIdx]] except Exception: fpsString = "NA" else: try: if len(observation[FILE][PLAYER1]) == 1: mediaFileString = observation[FILE][ PLAYER1][0] else: mediaFileString = "NA" except Exception: mediaFileString = "NA" fpsString = "NA" if observation[TYPE] in [LIVE]: mediaFileString = "LIVE" fpsString = "NA" if row["type"] == POINT: row_data = [] row_data.extend([ obsId, observation["date"].replace("T", " "), mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString ]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys( pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var][ "label"] in observation[ INDEPENDENT_VARIABLES]: row_data.append( observation[INDEPENDENT_VARIABLES][ pj[INDEPENDENT_VARIABLES][idx_var] ["label"]]) else: row_data.append("") row_data.extend([ subject, behavior, behavioral_category[behavior], row["modifiers"], POINT, f"{row['start']:.3f}", # start f"{row['stop']:.3f}", # stop "NA", # duration row["comment"], "" ]) data.append(row_data) if row["type"] == STATE: if idx % 2 == 0: row_data = [] row_data.extend([ obsId, observation["date"].replace("T", " "), mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString ]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys( pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var][ "label"] in observation[ INDEPENDENT_VARIABLES]: row_data.append( observation[INDEPENDENT_VARIABLES] [pj[INDEPENDENT_VARIABLES][idx_var] ["label"]]) else: row_data.append("") row_data.extend([ subject, behavior, behavioral_category[behavior], row["modifiers"], STATE, f"{row['start']:.3f}", f"{row['stop']:.3f}", f"{row['stop'] - row['start']:.3f}", row["comment"], row["comment_stop"] ]) data.append(row_data) return data
def create_behaviors_bar_plot(pj: dict, selected_observations: list, param: dict, plot_directory: str, output_format: str, plot_colors:list=BEHAVIORS_PLOT_COLORS): """ time budget bar plot Args: pj (dict): project param (dict): parameters plot_directory (str): path of directory output_format (str): image format Returns: dict: """ selected_subjects = param[SELECTED_SUBJECTS] selected_behaviors = param[SELECTED_BEHAVIORS] time_interval = param["time"] start_time = param[START_TIME] end_time = param[END_TIME] parameters = ["duration", "number of occurences"] ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return {"error": True, "message": msg} try: # extract all behaviors from ethogram for colors in plot all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])] for obs_id in selected_observations: cursor = db_connector.cursor() # distinct behaviors cursor.execute("SELECT distinct behavior FROM aggregated_events WHERE observation = ?", (obs_id,)) distinct_behav = [rows["behavior"] for rows in cursor.fetchall()] # add selected behaviors that are not observed ''' if not param[EXCLUDE_BEHAVIORS]: for behavior in selected_behaviors: if [x for x in distinct_behav if x == behavior] == []: distinct_behav.append(behavior) ''' # distinct subjects cursor.execute("SELECT distinct subject FROM aggregated_events WHERE observation = ?", (obs_id,)) distinct_subjects = [rows["subject"] for rows in cursor.fetchall()] behaviors = init_behav(pj[ETHOGRAM], distinct_subjects, distinct_behav, parameters) # plot creation if len(distinct_subjects) > 1: fig, axs = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True) fig2, axs2 = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True) else: fig, ax = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax fig2, ax2 = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True) axs2 = np.ndarray(shape=(1), dtype=type(ax2)) axs2[0] = ax2 fig.suptitle("Durations of behaviors") fig2.suptitle("Number of occurences of behaviors") # if modifiers not to be included set modifiers to "" cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if param["time"] == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if param["time"] == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if param["time"] == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obs_id, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obs_id, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obs_id, min_time, max_time, )) for ax_idx, subject in enumerate(sorted(distinct_subjects)): for behavior in distinct_behav: # number of occurences cursor.execute(("SELECT COUNT(*) AS count FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ?"), (obs_id, subject, behavior, )) for row in cursor.fetchall(): behaviors[subject][behavior]["number of occurences"] = 0 if row["count"] is None else row["count"] # total duration if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]): cursor.execute(("SELECT SUM(stop - start) AS duration FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ?"), (obs_id, subject, behavior, )) for row in cursor.fetchall(): behaviors[subject][behavior]["duration"] = 0 if row["duration"] is None else row["duration"] durations, n_occurences, colors, x_labels, colors_duration, x_labels_duration = [], [], [], [], [], [] for behavior in sorted(distinct_behav): if param[EXCLUDE_BEHAVIORS] and behaviors[subject][behavior]["number of occurences"] == 0: continue n_occurences.append(behaviors[subject][behavior]["number of occurences"]) x_labels.append(behavior) try: colors.append(utilities.behavior_color(plot_colors, all_behaviors.index(behavior))) except Exception: colors.append("darkgray") if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]): durations.append(behaviors[subject][behavior]["duration"]) x_labels_duration.append(behavior) try: colors_duration.append(utilities.behavior_color(plot_colors, all_behaviors.index(behavior))) except Exception: colors_duration.append("darkgray") #width = 0.35 # the width of the bars: can also be len(x) sequence axs2[ax_idx].bar(np.arange(len(n_occurences)), n_occurences, #width, color=colors ) axs[ax_idx].bar(np.arange(len(durations)), durations, #width, color=colors_duration ) if ax_idx == 0: axs[ax_idx].set_ylabel("Duration (s)") axs[ax_idx].set_xlabel("Behaviors") axs[ax_idx].set_title(f"{subject}") axs[ax_idx].set_xticks(np.arange(len(durations))) axs[ax_idx].set_xticklabels(x_labels_duration, rotation='vertical', fontsize=8) if ax_idx == 0: axs2[ax_idx].set_ylabel("Number of occurences") axs2[ax_idx].set_xlabel("Behaviors") axs2[ax_idx].set_title(f"{subject}") axs2[ax_idx].set_xticks(np.arange(len(n_occurences))) axs2[ax_idx].set_xticklabels(x_labels, rotation='vertical', fontsize=8) fig.align_labels() fig.tight_layout(rect=[0, 0.03, 1, 0.95]) fig2.align_labels() fig2.tight_layout(rect=[0, 0.03, 1, 0.95]) if plot_directory: # output_file_name = f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.{output_format}" fig.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.duration.{output_format}") fig2.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.number_of_occurences.{output_format}") plt.close() else: fig.show() fig2.show() return {} except Exception: error_type, error_file_name, error_lineno = utilities.error_info(sys.exc_info()) logging.critical(f"Error in time budget bar plot: {error_type} {error_file_name} {error_lineno}") return {"error": True, "exception": sys.exc_info()}
def create_events_plot(pj, selected_observations, parameters, plot_colors=BEHAVIORS_PLOT_COLORS, plot_directory="", file_format="png"): selected_subjects = parameters[SELECTED_SUBJECTS] selected_behaviors = parameters[SELECTED_BEHAVIORS] include_modifiers = parameters[INCLUDE_MODIFIERS] interval = parameters[TIME_INTERVAL] start_time = parameters[START_TIME] end_time = parameters[END_TIME] ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) all_behaviors = [ pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM]) ] par1 = 1 bar_height = 0.5 init = dt.datetime(2017, 1, 1) for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) else: fig, ax = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, [obs_id], selected_behaviors) cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor = db_connector.cursor() cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) # time obs_length = project_functions.observation_total_length( pj[OBSERVATIONS][obs_id]) if obs_length == -1: # media length not available interval = TIME_EVENTS if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute( "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", ( min_time, obs_id, min_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", ( max_time, obs_id, max_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", ( min_time, max_time, obs_id, min_time, max_time, )) ylabels = [" ".join(x) for x in distinct_behav_modif] for ax_idx, subject in enumerate(selected_subjects): if parameters["exclude behaviors"]: cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?", (subject, )) distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) ylabels = [" ".join(x) for x in distinct_behav_modif] if not ax_idx: axs[ax_idx].set_title(f"Observation {obs_id}\n{subject}", fontsize=14) else: axs[ax_idx].set_title(subject, fontsize=14) bars = {} i = 0 for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior bars[behavior_modifiers_str] = [] # total duration cursor.execute(( "SELECT start,stop FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?" ), ( obs_id, subject, behavior, modifiers, )) for row in cursor.fetchall(): bars[behavior_modifiers_str].append( (row["start"], row["stop"])) start_date = matplotlib.dates.date2num(init + dt.timedelta( seconds=row["start"])) end_date = matplotlib.dates.date2num(init + dt.timedelta( seconds=row["stop"] + POINT_EVENT_PLOT_DURATION * (row["stop"] == row["start"]))) try: bar_color = utilities.behavior_color( plot_colors, all_behaviors.index(behavior)) except Exception: bar_color = "darkgray" bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row[ "start"] else bar_color # sage colors removed from matplotlib colors list if bar_color in ["sage", "darksage", "lightsage"]: bar_color = { "darksage": "#598556", "lightsage": "#bcecac", "sage": "#87ae73" }[bar_color] try: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor=bar_color, color=bar_color, alpha=1) except Exception: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor="darkgray", color="darkgray", alpha=1) i += 1 axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1) pos = np.arange(par1, max_len * par1 + par1 + 1, par1) axs[ax_idx].set_yticks(pos[:len(ylabels)]) axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10}) axs[ax_idx].set_ylabel("Behaviors" + " (modifiers)" * include_modifiers, fontdict={"fontsize": 10}) axs[ax_idx].set_xlim( left=matplotlib.dates.date2num(init + dt.timedelta(seconds=min_time)), right=matplotlib.dates.date2num(init + dt.timedelta( seconds=max_time + 1))) axs[ax_idx].grid(color="g", linestyle=":") axs[ax_idx].xaxis_date() axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S")) axs[ax_idx].set_xlabel("Time (HH:MM:SS)", fontdict={"fontsize": 12}) axs[ax_idx].invert_yaxis() fig.autofmt_xdate() plt.tight_layout() if len(selected_observations) > 1: output_file_name = str( pathlib.Path( pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix("." + file_format)) plt.savefig(output_file_name) else: plt.show()
def export_aggregated_events(pj, parameters, obsId): """ export aggregated events Args: pj (dict): BORIS project parameters (dict): subjects, behaviors obsId (str): observation id Returns: tablib.Dataset: """ data = tablib.Dataset() observation = pj[OBSERVATIONS][obsId] duration1 = [] # in seconds if observation[TYPE] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: if "media_info" in observation: duration1.append( observation["media_info"]["length"][mediaFile]) except: duration1 = [] total_length = "{0:.3f}".format( project_functions.observation_total_length(observation)) ok, msg, connector = db_functions.load_aggregated_events_in_db( pj, parameters["selected subjects"], [obsId], parameters["selected behaviors"]) if not ok: data cursor = connector.cursor() for subject in parameters["selected subjects"]: for behavior in parameters["selected behaviors"]: cursor.execute( "select distinct modifiers from aggregated_events where subject=? AND behavior=? order by modifiers", ( subject, behavior, )) rows_distinct_modifiers = list(x[0].strip() for x in cursor.fetchall()) for distinct_modifiers in rows_distinct_modifiers: cursor.execute(( "SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events " "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start" ), (subject, behavior, distinct_modifiers)) rows = list(cursor.fetchall()) for row in rows: if observation[TYPE] in [MEDIA]: if duration1: mediaFileIdx = [ idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1]) ][-1] mediaFileString = observation[FILE][PLAYER1][ mediaFileIdx] fpsString = observation["media_info"]["fps"][ observation[FILE][PLAYER1][mediaFileIdx]] else: mediaFileString = "-" fpsString = "NA" if observation[TYPE] in [LIVE]: mediaFileString = "LIVE" fpsString = "NA" #if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]): if row["type"] == POINT: row_data = [] row_data.extend([ obsId, observation["date"].replace("T", " "), mediaFileString, total_length, fpsString ]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys( pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var][ "label"] in observation[ INDEPENDENT_VARIABLES]: row_data.append( observation[INDEPENDENT_VARIABLES][ pj[INDEPENDENT_VARIABLES][idx_var] ["label"]]) else: row_data.append("") row_data.extend([ subject, behavior, row["modifiers"].strip(), POINT, "{0:.3f}".format(row["start"]), # start "{0:.3f}".format(row["stop"]), # stop "NA", # duration row["comment"], "" ]) data.append(row_data) #if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]): if row["type"] == STATE: if idx % 2 == 0: row_data = [] row_data.extend([ obsId, observation["date"].replace("T", " "), mediaFileString, total_length, fpsString ]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys( pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var][ "label"] in observation[ INDEPENDENT_VARIABLES]: row_data.append( observation[INDEPENDENT_VARIABLES] [pj[INDEPENDENT_VARIABLES][idx_var] ["label"]]) else: row_data.append("") row_data.extend([ subject, behavior, row["modifiers"].strip(), STATE, "{0:.3f}".format(row["start"]), "{0:.3f}".format(row["stop"]), "{0:.3f}".format(row["stop"] - row["start"]), row["comment"], row["comment_stop"] ]) data.append(row_data) return data
def export_events(parameters, obsId, observation, ethogram, file_name, output_format): """ export events Args: parameters (dict): subjects, behaviors obsId (str): observation id observation (dict): observation ethogram (dict): ethogram of project file_name (str): file name for exporting events output_format (str): output for exporting events Returns: bool: result: True if OK else False str: error message """ total_length = "{0:.3f}".format( project_functions.observation_total_length(observation)) eventsWithStatus = project_functions.events_start_stop( ethogram, observation[EVENTS]) # check max number of modifiers max_modifiers = 0 for event in eventsWithStatus: for c in pj_events_fields: if c == "modifier" and event[pj_obs_fields[c]]: max_modifiers = max(max_modifiers, len(event[pj_obs_fields[c]].split("|"))) # media file number mediaNb = 0 if observation["type"] in [MEDIA]: for idx in observation[FILE]: for media in observation[FILE][idx]: mediaNb += 1 rows = [] # observation id rows.append(["Observation id", obsId]) rows.append([""]) # media file name if observation["type"] in [MEDIA]: rows.append(["Media file(s)"]) else: rows.append(["Live observation"]) rows.append([""]) if observation[TYPE] in [MEDIA]: for idx in observation[FILE]: for media in observation[FILE][idx]: rows.append(["Player #{0}".format(idx), media]) rows.append([""]) # date if "date" in observation: rows.append( ["Observation date", observation["date"].replace("T", " ")]) rows.append([""]) # description if "description" in observation: rows.append( ["Description", utilities.eol2space(observation["description"])]) rows.append([""]) # time offset if "time offset" in observation: rows.append(["Time offset (s)", observation["time offset"]]) rows.append([""]) # independent variables if INDEPENDENT_VARIABLES in observation: rows.extend([["independent variables"], ["variable", "value"]]) for variable in observation[INDEPENDENT_VARIABLES]: rows.append( [variable, observation[INDEPENDENT_VARIABLES][variable]]) rows.append([""]) # write table header col = 0 header = ["Time"] header.extend(["Media file path", "Total length", "FPS"]) header.extend(["Subject", "Behavior"]) for x in range(1, max_modifiers + 1): header.append("Modifier {}".format(x)) header.extend(["Comment", "Status"]) rows.append(header) duration1 = [] # in seconds if observation["type"] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: duration1.append( observation["media_info"]["length"][mediaFile]) except: pass for event in eventsWithStatus: if (((event[SUBJECT_EVENT_FIELD] in parameters["selected subjects"]) or (event[SUBJECT_EVENT_FIELD] == "" and NO_FOCAL_SUBJECT in parameters["selected subjects"])) and (event[BEHAVIOR_EVENT_FIELD] in parameters["selected behaviors"])): fields = [] fields.append( utilities.intfloatstr(str(event[EVENT_TIME_FIELD_IDX]))) if observation["type"] in [MEDIA]: time_ = event[EVENT_TIME_FIELD_IDX] - observation[TIME_OFFSET] if time_ < 0: time_ = 0 mediaFileIdx = [ idx1 for idx1, x in enumerate(duration1) if time_ >= sum(duration1[0:idx1]) ][-1] fields.append( utilities.intfloatstr( str(observation[FILE][PLAYER1][mediaFileIdx]))) fields.append(total_length) fields.append(observation["media_info"]["fps"][ observation[FILE][PLAYER1][mediaFileIdx]]) # fps if observation["type"] in [LIVE]: fields.append(LIVE) # media fields.append(total_length) # total length fields.append("NA") # FPS fields.append(event[EVENT_SUBJECT_FIELD_IDX]) fields.append(event[EVENT_BEHAVIOR_FIELD_IDX]) modifiers = event[EVENT_MODIFIER_FIELD_IDX].split("|") while len(modifiers) < max_modifiers: modifiers.append("") for m in modifiers: fields.append(m) fields.append(event[EVENT_COMMENT_FIELD_IDX].replace( os.linesep, " ")) # status fields.append(event[-1]) rows.append(fields) maxLen = max([len(r) for r in rows]) data = tablib.Dataset() data.title = obsId # check if worksheet name will be > 31 char if output_format in ["xls", "xlsx"]: for forbidden_char in r"\/*[]:?": data.title = data.title.replace(forbidden_char, " ") if output_format in ["xls"]: if len(data.title) > 31: data.title = data.title[0:31] for row in rows: data.append(utilities.complete(row, maxLen)) r, msg = dataset_write(data, file_name, output_format) return r, msg
def synthetic_time_budget(pj: dict, selected_observations: list, parameters_obs: dict): """ create a synthetic time budget Args: pj (dict): project dictionary selected_observations (list): list of observations to include in time budget parameters_obs (dict): Returns: bool: True if everything OK str: message tablib.Dataset: dataset containing synthetic time budget data """ try: selected_subjects = parameters_obs[SELECTED_SUBJECTS] selected_behaviors = parameters_obs[SELECTED_BEHAVIORS] include_modifiers = parameters_obs[INCLUDE_MODIFIERS] interval = parameters_obs["time"] start_time = parameters_obs["start time"] end_time = parameters_obs["end time"] parameters = [ ["duration", "Total duration"], ["number", "Number of occurrences"], ["duration mean", "Duration mean"], ["duration stdev", "Duration std dev"], ["proportion of time", "Proportion of time"], ] data_report = tablib.Dataset() data_report.title = "Synthetic time budget" ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None db_connector.create_aggregate("stdev", 1, StdevFunc) cursor = db_connector.cursor() # modifiers if include_modifiers: cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] else: cursor.execute("SELECT distinct behavior FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, ""]) behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects, distinct_behav_modif, include_modifiers, parameters) param_header = ["", "Total length (s)"] subj_header, behav_header, modif_header = [""] * len(param_header), [ "" ] * len(param_header), [""] * len(param_header) for subj in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior for param in parameters: subj_header.append(subj) behav_header.append(behavior) modif_header.append(modifiers) param_header.append(param[1]) ''' if parameters_obs["group observations"]: cursor.execute("UPDATE aggregated_events SET observation = 'all' " ) #selected_observations = ["all"] ''' data_report.append(subj_header) data_report.append(behav_header) if include_modifiers: data_report.append(modif_header) data_report.append(param_header) # select time interval for obs_id in selected_observations: ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, [obs_id], selected_behaviors) if not ok: return False, msg, None db_connector.create_aggregate("stdev", 1, StdevFunc) cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length( pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) # adapt start and stop to the selected time interval cursor.execute( "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", ( min_time, obs_id, min_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", ( max_time, obs_id, max_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", ( min_time, max_time, obs_id, min_time, max_time, )) cursor.execute( "DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)", ( obs_id, min_time, min_time, max_time, max_time, )) for subject in selected_subjects: # check if behaviors are to exclude from total time time_to_subtract = 0 if EXCLUDED_BEHAVIORS in parameters_obs: for excluded_behav in parameters_obs[EXCLUDED_BEHAVIORS]: cursor.execute(( "SELECT SUM(stop-start) " "FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? " ), ( obs_id, subject, excluded_behav, )) for row in cursor.fetchall(): if row[0] is not None: time_to_subtract += row[0] for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior cursor.execute(( "SELECT SUM(stop-start), COUNT(*), AVG(stop-start), stdev(stop-start) " "FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ? " ), ( obs_id, subject, behavior, modifiers, )) for row in cursor.fetchall(): behaviors[subject][behavior_modifiers_str][ "duration"] = (0 if row[0] is None else f"{row[0]:.3f}") behaviors[subject][behavior_modifiers_str][ "number"] = 0 if row[1] is None else row[1] behaviors[subject][behavior_modifiers_str][ "duration mean"] = (0 if row[2] is None else f"{row[2]:.3f}") behaviors[subject][behavior_modifiers_str][ "duration stdev"] = (0 if row[3] is None else f"{row[3]:.3f}") if behavior not in parameters_obs[EXCLUDED_BEHAVIORS]: try: behaviors[subject][behavior_modifiers_str][ "proportion of time"] = ( 0 if row[0] is None else f"{row[0] / ((max_time - min_time) - time_to_subtract):.3f}" ) except ZeroDivisionError: behaviors[subject][behavior_modifiers_str][ "proportion of time"] = "-" else: # behavior subtracted behaviors[subject][behavior_modifiers_str][ "proportion of time"] = ( 0 if row[0] is None else f"{row[0] / (max_time - min_time):.3f}") columns = [obs_id, f"{max_time - min_time:0.3f}"] for subj in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior for param in parameters: columns.append( behaviors[subj][behavior_modifiers_str][param[0]]) data_report.append(columns) except Exception: error_type, error_file_name, error_lineno = utilities.error_info( sys.exc_info()) logging.critical( f"Error in edit_event function: {error_type} {error_file_name} {error_lineno}" ) msg = f"Error type: {error_type}\nError file name: {error_file_name}\nError line number: {error_lineno}" logging.critical(msg) return (False, msg, tablib.Dataset()) return True, msg, data_report
def behaviors_bar_plot(pj, selected_observations, selected_subjects, selected_behaviors, include_modifiers, interval, start_time, end_time, plot_directory, output_format): """ scatter plot """ parameters = [["duration", "Total duration"], ] ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() if include_modifiers: cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] else: cursor.execute("SELECT distinct behavior FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects, distinct_behav_modif, include_modifiers, parameters) # select time interval for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True) else: fig, ax = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obs_id, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obs_id, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obs_id, min_time, max_time, )) for subject in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers # skip if behavior defined as POINT if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]): continue behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior # total duration cursor.execute(("SELECT SUM(stop-start) FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"), (obs_id, subject, behavior, modifiers,)) for row in cursor.fetchall(): behaviors[subject][behavior_modifiers_str]["duration"] = 0 if row[0] is None else row[0] print("behaviors") print(behaviors) print() print("sorted(distinct_behav_modif)", sorted(distinct_behav_modif)) max_length = 0 behaviors_duration = {} mb = {} for ax_idx, subj in enumerate(selected_subjects): behaviors_duration[subj] = {} behavior_ticks = [] for behavior_modifiers in sorted(distinct_behav_modif): behavior, modifiers = behavior_modifiers # skip if behavior defined as POINT if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]): continue if behavior not in behaviors_duration[subj]: behaviors_duration[subj][behavior] = [[],[]] behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior #print(subj, behavior, modifiers) if behavior not in behavior_ticks: behavior_ticks.append(behavior) for param in parameters: behaviors_duration[subj][behavior][0].append(behaviors[subj][behavior_modifiers_str][param[0]]) behaviors_duration[subj][behavior][1].append(modifiers) max_length = max(max_length, len(behaviors_duration[subj][behavior][1])) print() print("behaviors_duration", behaviors_duration) print() print("behavior_ticks", behavior_ticks) print() #b = {} behavior_mod_ticks = behavior_ticks[:] for ax_idx, subj in enumerate(selected_subjects): print("subject", subj) md_lgd = [] b = {} for i in range(max_length): b[i] = [] for behavior in sorted(behaviors_duration[subj].keys()): try: b[i].append(behaviors_duration[subj][behavior][0][i]) if include_modifiers: ''' if behaviors_duration[subj][behavior][1][i]: behavior_mod_ticks[behavior_ticks.index(behavior)] = behavior_mod_ticks[behavior_ticks.index(behavior)] + "\n" + \ behaviors_duration[subj][behavior][1][i] ''' if behaviors_duration[subj][behavior][1][i]: md_lgd.append(behavior + " " + behaviors_duration[subj][behavior][1][i]) else: md_lgd.append(behavior) except: b[i].append(0) print() print("behavior_mod_ticks", behavior_mod_ticks) print() print("b") print(b) print() print("md_lgd") print(md_lgd) #ind = np.arange(len(behavior_ticks)) # the x locations for the groups ind = np.arange(len(behavior_ticks)) width = 0.35 # the width of the bars: can also be len(x) sequence pp = [] max_obs = 0 bottom_ = [] idx_color = 0 for i in sorted(b.keys()): print(i, b[i]) if i == 0: pp.append(axs[ax_idx].bar(ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])])) else: pp.append(axs[ax_idx].bar(ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])], bottom=bottom_)) idx_color += len(b[i]) if not bottom_: bottom_ = b[i] else: bottom_ = [x + bottom_[idx] for idx,x in enumerate(b[i])] max_obs = max(max_obs, sum(b[i])) if ax_idx == 0: axs[ax_idx].set_ylabel('Duration (s)') axs[ax_idx].set_xlabel('Behaviors') axs[ax_idx].set_title('{}'.format(subj)) axs[ax_idx].set_xticks(ind) axs[ax_idx].set_xticklabels(behavior_mod_ticks,rotation=90) axs[ax_idx].set_yticks(np.arange(0, max(bottom_), 50)) lgd_col = [] for p in pp: for r in p: if r.get_height(): lgd_col.append(r) plt.legend(lgd_col, md_lgd) if plot_directory: output_file_name = str(pathlib.Path(pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix("." + file_format)) plt.savefig(output_file_name) else: plt.show()
def create_events_plot(pj, selected_observations, parameters, plot_colors=BEHAVIORS_PLOT_COLORS, plot_directory="", file_format="png"): selected_subjects = parameters["selected subjects"] selected_behaviors = parameters["selected behaviors"] include_modifiers = parameters["include modifiers"] interval = parameters[TIME_INTERVAL] start_time = parameters[START_TIME] end_time = parameters[END_TIME] ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])] par1 = 1 bar_height = 0.5 init = dt.datetime(2017, 1, 1) for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) else: fig, ax = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, [obs_id], selected_behaviors) cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor = db_connector.cursor() cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) # time obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id]) if obs_length == -1: # media length not available interval = TIME_EVENTS if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obs_id, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obs_id, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obs_id, min_time, max_time, )) ylabels = [" ".join(x) for x in distinct_behav_modif] for ax_idx, subject in enumerate(selected_subjects): if parameters["exclude behaviors"]: cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?", (subject, )) distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) ylabels = [" ".join(x) for x in distinct_behav_modif] if not ax_idx: axs[ax_idx].set_title("Observation {}\n{}".format(obs_id, subject), fontsize=14) else: axs[ax_idx].set_title(subject, fontsize=14) bars = {} i = 0 for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior bars[behavior_modifiers_str] = [] # total duration cursor.execute(("SELECT start,stop FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"), (obs_id, subject, behavior, modifiers,)) for row in cursor.fetchall(): bars[behavior_modifiers_str].append((row["start"], row["stop"])) start_date = matplotlib.dates.date2num(init + dt.timedelta(seconds=row["start"])) end_date = matplotlib.dates.date2num( init + dt.timedelta(seconds=row["stop"] + POINT_EVENT_PLOT_DURATION * (row["stop"] == row["start"]))) try: bar_color = utilities.behavior_color(plot_colors, all_behaviors.index(behavior)) except Exception: bar_color = "darkgray" bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row["start"] else bar_color # sage colors removed from matplotlib colors list if bar_color in ["sage", "darksage", "lightsage"]: bar_color = {"darksage": "#598556", "lightsage": "#bcecac", "sage": "#87ae73"}[bar_color] try: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor=bar_color, color=bar_color, alpha=1) except Exception: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor="darkgray", color="darkgray", alpha=1) i += 1 axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1) pos = np.arange(par1, max_len * par1 + par1 + 1, par1) axs[ax_idx].set_yticks(pos[:len(ylabels)]) axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10}) axs[ax_idx].set_ylabel("Behaviors" + " (modifiers)" * include_modifiers, fontdict={"fontsize": 10}) axs[ax_idx].set_xlim(left=matplotlib.dates.date2num(init + dt.timedelta(seconds=min_time)), right=matplotlib.dates.date2num(init + dt.timedelta(seconds=max_time + 1))) axs[ax_idx].grid(color="g", linestyle=":") axs[ax_idx].xaxis_date() axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S")) axs[ax_idx].set_xlabel("Time (HH:MM:SS)", fontdict={"fontsize": 12}) axs[ax_idx].invert_yaxis() fig.autofmt_xdate() plt.tight_layout() if len(selected_observations) > 1: output_file_name = str(pathlib.Path(pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix( "." + file_format)) plt.savefig(output_file_name) else: plt.show()
def export_aggregated_events(pj: dict, parameters: dict, obsId: str): """ export aggregated events Args: pj (dict): BORIS project parameters (dict): subjects, behaviors obsId (str): observation id Returns: tablib.Dataset: """ logging.debug(f"function: export aggregated events {parameters} {obsId}") interval = parameters["time"] start_time = parameters[START_TIME] end_time = parameters[END_TIME] data = tablib.Dataset() observation = pj[OBSERVATIONS][obsId] duration1 = [] # in seconds if observation[TYPE] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: if MEDIA_INFO in observation: duration1.append(observation[MEDIA_INFO]["length"][mediaFile]) except Exception: duration1 = [] obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obsId]) if obs_length == Decimal("-1"): # media length not available interval = TIME_EVENTS logging.debug(f"obs_length: {obs_length}") ok, msg, connector = db_functions.load_aggregated_events_in_db(pj, parameters[SELECTED_SUBJECTS], [obsId], parameters[SELECTED_BEHAVIORS]) if connector is None: logging.critical(f"error when loading aggregated events in DB") return data # time cursor = connector.cursor() if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) # adapt start and stop to the selected time interval cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obsId, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obsId, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obsId, min_time, max_time, )) cursor.execute("DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)", (obsId, min_time, min_time, max_time, max_time, )) behavioral_category = project_functions.behavior_category(pj[ETHOGRAM]) for subject in parameters[SELECTED_SUBJECTS]: for behavior in parameters[SELECTED_BEHAVIORS]: cursor.execute("SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers", (subject, behavior,)) rows_distinct_modifiers = list(x[0] for x in cursor.fetchall()) for distinct_modifiers in rows_distinct_modifiers: cursor.execute(("SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events " "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"), (subject, behavior, distinct_modifiers)) rows = list(cursor.fetchall()) for row in rows: if observation[TYPE] in [MEDIA]: if duration1: mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1])][-1] mediaFileString = observation[FILE][PLAYER1][mediaFileIdx] try: fpsString = observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]] except Exception: fpsString = "NA" else: try: if len(observation[FILE][PLAYER1]) == 1: mediaFileString = observation[FILE][PLAYER1][0] else: mediaFileString = "NA" except Exception: mediaFileString = "NA" fpsString = "NA" if observation[TYPE] in [LIVE]: mediaFileString = "LIVE" fpsString = "NA" if row["type"] == POINT: row_data = [] row_data.extend([obsId, observation["date"].replace("T", " "), mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]: row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]]) else: row_data.append("") row_data.extend([subject, behavior, behavioral_category[behavior], row["modifiers"], POINT, "{0:.3f}".format(row["start"]), # start "{0:.3f}".format(row["stop"]), # stop "NA", # duration row["comment"], "" ]) data.append(row_data) if row["type"] == STATE: if idx % 2 == 0: row_data = [] row_data.extend([obsId, observation["date"].replace("T", " "), mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]: row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]]) else: row_data.append("") row_data.extend([subject, behavior, behavioral_category[behavior], row["modifiers"], STATE, "{0:.3f}".format(row["start"]), "{0:.3f}".format(row["stop"]), "{0:.3f}".format(row["stop"] - row["start"]), row["comment"], row["comment_stop"] ]) data.append(row_data) return data
def export_events(parameters, obsId, observation, ethogram, file_name, output_format): """ export events Args: parameters (dict): subjects, behaviors obsId (str): observation id observation (dict): observation ethogram (dict): ethogram of project file_name (str): file name for exporting events output_format (str): output for exporting events Returns: bool: result: True if OK else False str: error message """ total_length = "{0:.3f}".format(project_functions.observation_total_length(observation)) eventsWithStatus = project_functions.events_start_stop(ethogram, observation[EVENTS]) # check max number of modifiers max_modifiers = 0 for event in eventsWithStatus: for c in pj_events_fields: if c == "modifier" and event[pj_obs_fields[c]]: max_modifiers = max(max_modifiers, len(event[pj_obs_fields[c]].split("|"))) # media file number mediaNb = 0 if observation["type"] == MEDIA: for player in observation[FILE]: mediaNb += len(observation[FILE][player]) rows = [] # observation id rows.append(["Observation id", obsId]) rows.append([""]) # media file name if observation["type"] in [MEDIA]: rows.append(["Media file(s)"]) elif observation["type"] in [LIVE]: rows.append(["Live observation"]) else: rows.append(["?"]) rows.append([""]) if observation[TYPE] in [MEDIA]: for player in sorted(list(observation[FILE].keys())): for media in observation[FILE][player]: rows.append(["Player #{0}".format(player), media]) rows.append([""]) # date if "date" in observation: rows.append(["Observation date", observation["date"].replace("T", " ")]) rows.append([""]) # description if "description" in observation: rows.append(["Description", utilities.eol2space(observation["description"])]) rows.append([""]) # time offset if "time offset" in observation: rows.append(["Time offset (s)", observation["time offset"]]) rows.append([""]) # independent variables if INDEPENDENT_VARIABLES in observation: rows.extend([["independent variables"], ["variable", "value"]]) for variable in observation[INDEPENDENT_VARIABLES]: rows.append([variable, observation[INDEPENDENT_VARIABLES][variable]]) rows.append([""]) # write table header col = 0 header = ["Time"] header.extend(["Media file path", "Total length", "FPS"]) header.extend(["Subject", "Behavior", "Behavioral category"]) behavioral_category = project_functions.behavior_category(ethogram) for x in range(1, max_modifiers + 1): header.append("Modifier {}".format(x)) header.extend(["Comment", "Status"]) rows.append(header) duration1 = [] # in seconds if observation["type"] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: duration1.append(observation[MEDIA_INFO]["length"][mediaFile]) except KeyError: pass for event in eventsWithStatus: if (((event[SUBJECT_EVENT_FIELD] in parameters["selected subjects"]) or (event[SUBJECT_EVENT_FIELD] == "" and NO_FOCAL_SUBJECT in parameters["selected subjects"])) and (event[BEHAVIOR_EVENT_FIELD] in parameters["selected behaviors"])): fields = [] fields.append(utilities.intfloatstr(str(event[EVENT_TIME_FIELD_IDX]))) if observation["type"] in [MEDIA]: time_ = event[EVENT_TIME_FIELD_IDX] - observation[TIME_OFFSET] if time_ < 0: time_ = 0 if duration1: mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if time_ >= sum(duration1[0:idx1])][-1] fields.append(observation[FILE][PLAYER1][mediaFileIdx]) fields.append(total_length) # FPS try: fields.append(observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]]) # fps except KeyError: fields.append("NA") else: fields.append("NA") # media file fields.append("NA") # FPS if observation["type"] in [LIVE]: fields.append(LIVE) # media fields.append(total_length) # total length fields.append("NA") # FPS fields.append(event[EVENT_SUBJECT_FIELD_IDX]) fields.append(event[EVENT_BEHAVIOR_FIELD_IDX]) # behavioral category try: behav_category = behavioral_category[event[EVENT_BEHAVIOR_FIELD_IDX]] except Exception: behav_category = "" fields.append(behav_category) # modifiers if max_modifiers: modifiers = event[EVENT_MODIFIER_FIELD_IDX].split("|") while len(modifiers) < max_modifiers: modifiers.append("") for m in modifiers: fields.append(m) # comment fields.append(event[EVENT_COMMENT_FIELD_IDX].replace(os.linesep, " ")) # status fields.append(event[-1]) rows.append(fields) maxLen = max([len(r) for r in rows]) data = tablib.Dataset() data.title = obsId # check if worksheet name will be > 31 char if output_format in ["xls", "xlsx"]: for forbidden_char in EXCEL_FORBIDDEN_CHARACTERS: data.title = data.title.replace(forbidden_char, " ") if output_format in ["xls"]: if len(data.title) > 31: data.title = data.title[0:31] for row in rows: data.append(utilities.complete(row, maxLen)) r, msg = dataset_write(data, file_name, output_format) return r, msg