def test_no_observation(self): """ test with no observation """ pj = json.loads(open("files/test.boris").read()) ok, msg, db = db_functions.load_aggregated_events_in_db(pj, [], [], []) assert ok == False
def test_not_ok(self): """ test with observation with state events NOT PAIRED """ pj = json.loads(open("files/test.boris").read()) ok, msg, db = db_functions.load_aggregated_events_in_db(pj, [], ["live not paired"], []) assert ok == False
def test_dump(self): pj = json.loads(open("files/test.boris").read()) ok, msg, db = db_functions.load_aggregated_events_in_db(pj, [], ["observation #1", "observation #2"], []) out = "" for line in db.iterdump(): out += line + "\n" print(out == open("files/test_db_functions_test1").read())
def test_cohen_kappa_very_similar_obs_interval_state_3s(self): pj = json.loads(open("files/test.boris").read()) ethogram = pj[config.ETHOGRAM] selected_observations = ["observation #2", "observation #2 (copy)"] selected_subjects = ["No focal subject", "subject1", "subject2"] r, s, db = cursor = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, ["s"]) cursor = db.cursor() K, _ = irr.cohen_kappa(cursor, obsid1=selected_observations[0], obsid2=selected_observations[1], interval=decimal.Decimal("3"), selected_subjects=selected_subjects, include_modifiers=False) assert K == 1
def test_cohen_kappa_same_observation(self): pj = json.loads(open("files/test.boris").read()) ethogram = pj[config.ETHOGRAM] selected_observations = ["observation #1", "observation #1"] selected_subjects = ["subject1", "subject2"] r, s, db = cursor = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, ["s", "p"]) assert r == True cursor = db.cursor() K, msg = irr.cohen_kappa(cursor, obsid1=selected_observations[0], obsid2=selected_observations[1], interval=decimal.Decimal("1.0"), selected_subjects=selected_subjects, include_modifiers=False) assert K == 1
def test_needleman_wunsch_identity_very_similar_obs_point_3s(self): pj = json.loads(open("files/test.boris").read()) ethogram = pj[config.ETHOGRAM] selected_observations = ["observation #2", "observation #2 (copy)"] selected_subjects = ["No focal subject", "subject1", "subject2"] r, s, db = cursor = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, ["p"]) cursor = db.cursor() identity, msg = irr.needleman_wunsch_identity( cursor, obsid1=selected_observations[0], obsid2=selected_observations[1], interval=decimal.Decimal("3.0"), selected_subjects=selected_subjects, include_modifiers=False) print(identity) assert identity == 97.91666666666666
def test_needleman_wunsch_identity_very_different_obs(self): pj = json.loads(open("files/test.boris").read()) ethogram = pj[config.ETHOGRAM] selected_observations = ["observation #1", "observation #2"] selected_subjects = ["subject1", "subject2"] r, s, db = cursor = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, ["s"]) cursor = db.cursor() identity, msg = irr.needleman_wunsch_identity( cursor, obsid1=selected_observations[0], obsid2=selected_observations[1], interval=decimal.Decimal("1.0"), selected_subjects=selected_subjects, include_modifiers=False) # print(identity) assert identity == 85.84070796460178
def export_aggregated_events(pj: dict, parameters: dict, obsId: str): """ export aggregated events Args: pj (dict): BORIS project parameters (dict): subjects, behaviors obsId (str): observation id Returns: tablib.Dataset: """ logging.debug(f"function: export aggregated events {parameters} {obsId}") interval = parameters["time"] start_time = parameters[START_TIME] end_time = parameters[END_TIME] data = tablib.Dataset() observation = pj[OBSERVATIONS][obsId] # obs description obs_description = observation["description"] duration1 = [] # in seconds if observation[TYPE] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: if MEDIA_INFO in observation: duration1.append(observation[MEDIA_INFO]["length"][mediaFile]) except Exception: duration1 = [] obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obsId]) if obs_length == Decimal("-1"): # media length not available interval = TIME_EVENTS logging.debug(f"obs_length: {obs_length}") ok, msg, connector = db_functions.load_aggregated_events_in_db(pj, parameters[SELECTED_SUBJECTS], [obsId], parameters[SELECTED_BEHAVIORS]) if connector is None: logging.critical(f"error when loading aggregated events in DB") return data # time cursor = connector.cursor() if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) # adapt start and stop to the selected time interval cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obsId, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obsId, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obsId, min_time, max_time, )) cursor.execute("DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)", (obsId, min_time, min_time, max_time, max_time, )) behavioral_category = project_functions.behavior_category(pj[ETHOGRAM]) for subject in parameters[SELECTED_SUBJECTS]: for behavior in parameters[SELECTED_BEHAVIORS]: cursor.execute("SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers", (subject, behavior,)) rows_distinct_modifiers = list(x[0] for x in cursor.fetchall()) for distinct_modifiers in rows_distinct_modifiers: cursor.execute(("SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events " "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"), (subject, behavior, distinct_modifiers)) rows = list(cursor.fetchall()) for row in rows: if observation[TYPE] in [MEDIA]: if duration1: mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1])][-1] mediaFileString = observation[FILE][PLAYER1][mediaFileIdx] try: fpsString = observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]] except Exception: fpsString = "NA" else: try: if len(observation[FILE][PLAYER1]) == 1: mediaFileString = observation[FILE][PLAYER1][0] else: mediaFileString = "NA" except Exception: mediaFileString = "NA" fpsString = "NA" if observation[TYPE] in [LIVE]: mediaFileString = "LIVE" fpsString = "NA" if row["type"] == POINT: row_data = [] row_data.extend([obsId, observation["date"].replace("T", " "), obs_description, mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]: row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]]) else: row_data.append("") row_data.extend([subject, behavior, behavioral_category[behavior], row["modifiers"], POINT, f"{row['start']:.3f}", # start f"{row['stop']:.3f}", # stop "NA", # duration row["comment"], "" ]) data.append(row_data) if row["type"] == STATE: if idx % 2 == 0: row_data = [] row_data.extend([obsId, observation["date"].replace("T", " "), obs_description, mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]: row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]]) else: row_data.append("") row_data.extend([subject, behavior, behavioral_category[behavior], row["modifiers"], STATE, f"{row['start']:.3f}", f"{row['stop']:.3f}", f"{row['stop'] - row['start']:.3f}", row["comment"], row["comment_stop"] ]) data.append(row_data) return data
def create_events_plot(pj, selected_observations, parameters, plot_colors=BEHAVIORS_PLOT_COLORS, plot_directory="", file_format="png"): """ create a time diagram plot (sort of gantt chart) with matplotlib barh function (https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.barh.html) """ selected_subjects = parameters[SELECTED_SUBJECTS] selected_behaviors = parameters[SELECTED_BEHAVIORS] include_modifiers = parameters[INCLUDE_MODIFIERS] interval = parameters[TIME_INTERVAL] start_time = parameters[START_TIME] end_time = parameters[END_TIME] ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])] par1 = 1 bar_height = 0.5 init = dt.datetime(2017, 1, 1) for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) else: fig, ax = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, [obs_id], selected_behaviors) cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor = db_connector.cursor() cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) # time obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id]) if obs_length == -1: # media length not available interval = TIME_EVENTS if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obs_id, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obs_id, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obs_id, min_time, max_time, )) ylabels = [" ".join(x) for x in distinct_behav_modif] for ax_idx, subject in enumerate(selected_subjects): if parameters["exclude behaviors"]: cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?", (subject, )) distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) ylabels = [" ".join(x) for x in distinct_behav_modif] if not ax_idx: axs[ax_idx].set_title(f"Observation {obs_id}\n{subject}", fontsize=14) else: axs[ax_idx].set_title(subject, fontsize=14) bars = {} i = 0 for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior bars[behavior_modifiers_str] = [] # total duration cursor.execute(("SELECT start, stop FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"), (obs_id, subject, behavior, modifiers,)) for row in cursor.fetchall(): bars[behavior_modifiers_str].append((row["start"], row["stop"])) start_date = matplotlib.dates.date2num(init + dt.timedelta(seconds=row["start"])) end_date = matplotlib.dates.date2num( init + dt.timedelta(seconds=row["stop"] + POINT_EVENT_PLOT_DURATION * (row["stop"] == row["start"]))) try: bar_color = utilities.behavior_color(plot_colors, all_behaviors.index(behavior)) except Exception: bar_color = "darkgray" bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row["start"] else bar_color # sage colors removed from matplotlib colors list if bar_color in ["sage", "darksage", "lightsage"]: bar_color = {"darksage": "#598556", "lightsage": "#bcecac", "sage": "#87ae73"}[bar_color] try: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor=bar_color, color=bar_color, alpha=1) except Exception: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor="darkgray", color="darkgray", alpha=1) i += 1 axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1) pos = np.arange(par1, max_len * par1 + par1 + 1, par1) axs[ax_idx].set_yticks(pos[:len(ylabels)]) axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10}) axs[ax_idx].set_ylabel("Behaviors" + " (modifiers)" * include_modifiers, fontdict={"fontsize": 10}) axs[ax_idx].set_xlim(left=matplotlib.dates.date2num(init + dt.timedelta(seconds=min_time)), right=matplotlib.dates.date2num(init + dt.timedelta(seconds=max_time + 1))) axs[ax_idx].grid(color="g", linestyle=":") axs[ax_idx].xaxis_date() axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S")) axs[ax_idx].set_xlabel("Time (HH:MM:SS)", fontdict={"fontsize": 12}) axs[ax_idx].invert_yaxis() fig.autofmt_xdate() plt.tight_layout() if len(selected_observations) > 1: plt.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.{file_format}") else: plt.show()
def create_behaviors_bar_plot(pj: dict, selected_observations: list, param: dict, plot_directory: str, output_format: str, plot_colors:list=BEHAVIORS_PLOT_COLORS): """ time budget bar plot Args: pj (dict): project param (dict): parameters plot_directory (str): path of directory output_format (str): image format Returns: dict: """ selected_subjects = param[SELECTED_SUBJECTS] selected_behaviors = param[SELECTED_BEHAVIORS] time_interval = param["time"] start_time = param[START_TIME] end_time = param[END_TIME] parameters = ["duration", "number of occurences"] ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return {"error": True, "message": msg} try: # extract all behaviors from ethogram for colors in plot all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])] for obs_id in selected_observations: cursor = db_connector.cursor() # distinct behaviors cursor.execute("SELECT distinct behavior FROM aggregated_events WHERE observation = ?", (obs_id,)) distinct_behav = [rows["behavior"] for rows in cursor.fetchall()] # add selected behaviors that are not observed ''' if not param[EXCLUDE_BEHAVIORS]: for behavior in selected_behaviors: if [x for x in distinct_behav if x == behavior] == []: distinct_behav.append(behavior) ''' # distinct subjects cursor.execute("SELECT distinct subject FROM aggregated_events WHERE observation = ?", (obs_id,)) distinct_subjects = [rows["subject"] for rows in cursor.fetchall()] behaviors = init_behav(pj[ETHOGRAM], distinct_subjects, distinct_behav, parameters) # plot creation if len(distinct_subjects) > 1: fig, axs = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True) fig2, axs2 = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True) else: fig, ax = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax fig2, ax2 = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True) axs2 = np.ndarray(shape=(1), dtype=type(ax2)) axs2[0] = ax2 fig.suptitle("Durations of behaviors") fig2.suptitle("Number of occurences of behaviors") # if modifiers not to be included set modifiers to "" cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if param["time"] == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if param["time"] == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if param["time"] == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obs_id, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obs_id, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obs_id, min_time, max_time, )) for ax_idx, subject in enumerate(sorted(distinct_subjects)): for behavior in distinct_behav: # number of occurences cursor.execute(("SELECT COUNT(*) AS count FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ?"), (obs_id, subject, behavior, )) for row in cursor.fetchall(): behaviors[subject][behavior]["number of occurences"] = 0 if row["count"] is None else row["count"] # total duration if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]): cursor.execute(("SELECT SUM(stop - start) AS duration FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ?"), (obs_id, subject, behavior, )) for row in cursor.fetchall(): behaviors[subject][behavior]["duration"] = 0 if row["duration"] is None else row["duration"] durations, n_occurences, colors, x_labels, colors_duration, x_labels_duration = [], [], [], [], [], [] for behavior in sorted(distinct_behav): if param[EXCLUDE_BEHAVIORS] and behaviors[subject][behavior]["number of occurences"] == 0: continue n_occurences.append(behaviors[subject][behavior]["number of occurences"]) x_labels.append(behavior) try: colors.append(utilities.behavior_color(plot_colors, all_behaviors.index(behavior))) except Exception: colors.append("darkgray") if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]): durations.append(behaviors[subject][behavior]["duration"]) x_labels_duration.append(behavior) try: colors_duration.append(utilities.behavior_color(plot_colors, all_behaviors.index(behavior))) except Exception: colors_duration.append("darkgray") #width = 0.35 # the width of the bars: can also be len(x) sequence axs2[ax_idx].bar(np.arange(len(n_occurences)), n_occurences, #width, color=colors ) axs[ax_idx].bar(np.arange(len(durations)), durations, #width, color=colors_duration ) if ax_idx == 0: axs[ax_idx].set_ylabel("Duration (s)") axs[ax_idx].set_xlabel("Behaviors") axs[ax_idx].set_title(f"{subject}") axs[ax_idx].set_xticks(np.arange(len(durations))) axs[ax_idx].set_xticklabels(x_labels_duration, rotation='vertical', fontsize=8) if ax_idx == 0: axs2[ax_idx].set_ylabel("Number of occurences") axs2[ax_idx].set_xlabel("Behaviors") axs2[ax_idx].set_title(f"{subject}") axs2[ax_idx].set_xticks(np.arange(len(n_occurences))) axs2[ax_idx].set_xticklabels(x_labels, rotation='vertical', fontsize=8) fig.align_labels() fig.tight_layout(rect=[0, 0.03, 1, 0.95]) fig2.align_labels() fig2.tight_layout(rect=[0, 0.03, 1, 0.95]) if plot_directory: # output_file_name = f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.{output_format}" fig.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.duration.{output_format}") fig2.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.number_of_occurences.{output_format}") plt.close() else: fig.show() fig2.show() return {} except Exception: error_type, error_file_name, error_lineno = utilities.error_info(sys.exc_info()) logging.critical(f"Error in time budget bar plot: {error_type} {error_file_name} {error_lineno}") return {"error": True, "exception": sys.exc_info()}
def synthetic_time_budget(pj: dict, selected_observations: list, parameters_obs: dict): """ create a synthetic time budget Args: pj (dict): project dictionary selected_observations (list): list of observations to include in time budget parameters_obs (dict): Returns: bool: True if everything OK str: message tablib.Dataset: dataset containing synthetic time budget data """ try: selected_subjects = parameters_obs[SELECTED_SUBJECTS] selected_behaviors = parameters_obs[SELECTED_BEHAVIORS] include_modifiers = parameters_obs[INCLUDE_MODIFIERS] interval = parameters_obs["time"] start_time = parameters_obs["start time"] end_time = parameters_obs["end time"] parameters = [ ["duration", "Total duration"], ["number", "Number of occurrences"], ["duration mean", "Duration mean"], ["duration stdev", "Duration std dev"], ["proportion of time", "Proportion of time"], ] data_report = tablib.Dataset() data_report.title = "Synthetic time budget" ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None db_connector.create_aggregate("stdev", 1, StdevFunc) cursor = db_connector.cursor() # modifiers if include_modifiers: cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] else: cursor.execute("SELECT distinct behavior FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, ""]) behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects, distinct_behav_modif, include_modifiers, parameters) param_header = ["", "Total length (s)"] subj_header, behav_header, modif_header = [""] * len(param_header), [ "" ] * len(param_header), [""] * len(param_header) for subj in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior for param in parameters: subj_header.append(subj) behav_header.append(behavior) modif_header.append(modifiers) param_header.append(param[1]) ''' if parameters_obs["group observations"]: cursor.execute("UPDATE aggregated_events SET observation = 'all' " ) #selected_observations = ["all"] ''' data_report.append(subj_header) data_report.append(behav_header) if include_modifiers: data_report.append(modif_header) data_report.append(param_header) # select time interval for obs_id in selected_observations: ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, [obs_id], selected_behaviors) if not ok: return False, msg, None db_connector.create_aggregate("stdev", 1, StdevFunc) cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length( pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) # adapt start and stop to the selected time interval cursor.execute( "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", ( min_time, obs_id, min_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", ( max_time, obs_id, max_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", ( min_time, max_time, obs_id, min_time, max_time, )) cursor.execute( "DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)", ( obs_id, min_time, min_time, max_time, max_time, )) for subject in selected_subjects: # check if behaviors are to exclude from total time time_to_subtract = 0 if EXCLUDED_BEHAVIORS in parameters_obs: for excluded_behav in parameters_obs[EXCLUDED_BEHAVIORS]: cursor.execute(( "SELECT SUM(stop-start) " "FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? " ), ( obs_id, subject, excluded_behav, )) for row in cursor.fetchall(): if row[0] is not None: time_to_subtract += row[0] for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior cursor.execute(( "SELECT SUM(stop-start), COUNT(*), AVG(stop-start), stdev(stop-start) " "FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ? " ), ( obs_id, subject, behavior, modifiers, )) for row in cursor.fetchall(): behaviors[subject][behavior_modifiers_str][ "duration"] = (0 if row[0] is None else f"{row[0]:.3f}") behaviors[subject][behavior_modifiers_str][ "number"] = 0 if row[1] is None else row[1] behaviors[subject][behavior_modifiers_str][ "duration mean"] = (0 if row[2] is None else f"{row[2]:.3f}") behaviors[subject][behavior_modifiers_str][ "duration stdev"] = (0 if row[3] is None else f"{row[3]:.3f}") if behavior not in parameters_obs[EXCLUDED_BEHAVIORS]: try: behaviors[subject][behavior_modifiers_str][ "proportion of time"] = ( 0 if row[0] is None else f"{row[0] / ((max_time - min_time) - time_to_subtract):.3f}" ) except ZeroDivisionError: behaviors[subject][behavior_modifiers_str][ "proportion of time"] = "-" else: # behavior subtracted behaviors[subject][behavior_modifiers_str][ "proportion of time"] = ( 0 if row[0] is None else f"{row[0] / (max_time - min_time):.3f}") columns = [obs_id, f"{max_time - min_time:0.3f}"] for subj in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior for param in parameters: columns.append( behaviors[subj][behavior_modifiers_str][param[0]]) data_report.append(columns) except Exception: error_type, error_file_name, error_lineno = utilities.error_info( sys.exc_info()) logging.critical( f"Error in edit_event function: {error_type} {error_file_name} {error_lineno}" ) msg = f"Error type: {error_type}\nError file name: {error_file_name}\nError line number: {error_lineno}" logging.critical(msg) return (False, msg, tablib.Dataset()) return True, msg, data_report
def create_subtitles(pj: dict, selected_observations: list, parameters: dict, export_dir: str) -> tuple: """ create subtitles for selected observations, subjects and behaviors Args: pj (dict): project selected_observations (list): list of observations parameters (dict): export_dir (str): directory to save subtitles Returns: bool: True if OK else False str: error message """ def subject_color(subject): """ subject color Args: subject (str): subject name Returns: str: HTML tag for color font (beginning) str: HTML tag for color font (closing) """ if subject == NO_FOCAL_SUBJECT: return "", "" else: return ( """<font color="{}">""".format( subtitlesColors[parameters["selected subjects"].index( row["subject"]) % len(subtitlesColors)]), "</font>", ) try: ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, parameters["selected subjects"], selected_observations, parameters["selected behaviors"]) if not ok: return False, msg cursor = db_connector.cursor() flag_ok = True msg = "" for obsId in selected_observations: if pj[OBSERVATIONS][obsId][TYPE] in [LIVE]: out = "" cursor.execute( ("SELECT subject, behavior, start, stop, type, modifiers FROM aggregated_events " "WHERE observation = ? AND subject in ({}) " "AND behavior in ({}) " "ORDER BY start").format( ",".join(["?"] * len(parameters["selected subjects"])), ",".join(["?"] * len(parameters["selected behaviors"]))), [obsId] + parameters["selected subjects"] + parameters["selected behaviors"], ) for idx, row in enumerate(cursor.fetchall()): col1, col2 = subject_color(row["subject"]) if parameters["include modifiers"]: modifiers_str = "\n{}".format(row["modifiers"].replace( "|", ", ")) else: modifiers_str = "" out += ("{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format( idx=idx + 1, start=utilities.seconds2time( row["start"]).replace(".", ","), stop=utilities.seconds2time( row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION).replace(".", ","), col1=col1, col2=col2, subject=row["subject"], behavior=row["behavior"], modifiers=modifiers_str, ) ''' file_name = str(pathlib.Path(pathlib.Path(export_dir) / utilities.safeFileName(obsId)).with suffix(".srt")) ''' file_name = f"{pathlib.Path(export_dir) / utilities.safeFileName(obsId)}.srt" try: with open(file_name, "w") as f: f.write(out) except Exception: flag_ok = False msg += "observation: {}\ngave the following error:\n{}\n".format( obsId, str(sys.exc_info()[1])) elif pj[OBSERVATIONS][obsId][TYPE] in [MEDIA]: for nplayer in ALL_PLAYERS: if not pj[OBSERVATIONS][obsId][FILE][nplayer]: continue init = 0 for mediaFile in pj[OBSERVATIONS][obsId][FILE][nplayer]: try: end = init + pj[OBSERVATIONS][obsId][MEDIA_INFO][ LENGTH][mediaFile] except KeyError: return False, f"The length for media file {mediaFile} is not available" out = "" cursor.execute( ("SELECT subject, behavior, type, start, stop, modifiers FROM aggregated_events " "WHERE observation = ? AND start BETWEEN ? and ? " "AND subject in ({}) " "AND behavior in ({}) " "ORDER BY start").format( ",".join( ["?"] * len(parameters["selected subjects"])), ",".join( ["?"] * len(parameters["selected behaviors"])), ), [obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"], ) for idx, row in enumerate(cursor.fetchall()): col1, col2 = subject_color(row["subject"]) if parameters["include modifiers"]: modifiers_str = "\n{}".format( row["modifiers"].replace("|", ", ")) else: modifiers_str = "" out += ( "{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format( idx=idx + 1, start=utilities.seconds2time(row["start"] - init).replace( ".", ","), stop=utilities.seconds2time( (row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION) - init).replace(".", ","), col1=col1, col2=col2, subject=row["subject"], behavior=row["behavior"], modifiers=modifiers_str, ) ''' file_name = str(pathlib.Path(pathlib.Path(export_dir) / pathlib.Path(mediaFile).name).with suffix(".srt")) ''' file_name = f"{pathlib.Path(export_dir) / pathlib.Path(mediaFile).name}.srt" try: with open(file_name, "w") as f: f.write(out) except Exception: flag_ok = False msg += f"observation: {obsId}\ngave the following error:\n{sys.exc_info()[1]}\n" init = end return flag_ok, msg except Exception: return False, str(sys.exc_info()[1])
def event_filtering(pj: dict): """ advanced event filtering the python-intervals module is used to do operations on intervals (intersection, union) """ result, selected_observations = select_observations.select_observations( pj, MULTIPLE, "Select observations for advanced event filtering") if not selected_observations: return # check if state events are paired out = "" not_paired_obs_list = [] for obs_id in selected_observations: r, msg = project_functions.check_state_events_obs( obs_id, pj[ETHOGRAM], pj[OBSERVATIONS][obs_id]) if not r: out += f"Observation: <strong>{obs_id}</strong><br>{msg}<br>" not_paired_obs_list.append(obs_id) if out: out = f"The observations with UNPAIRED state events will be removed from tha analysis<br><br>{out}" results = dialog.Results_dialog() results.setWindowTitle(f"{programName} - Check selected observations") results.ptText.setReadOnly(True) results.ptText.appendHtml(out) results.pbSave.setVisible(False) results.pbCancel.setVisible(True) if not results.exec_(): return selected_observations = [ x for x in selected_observations if x not in not_paired_obs_list ] if not selected_observations: return # observations length max_obs_length, selectedObsTotalMediaLength = project_functions.observation_length( pj, selected_observations) if max_obs_length == -1: # media length not available, user choose to not use events return parameters = dialog.choose_obs_subj_behav_category( pj, selected_observations, maxTime=max_obs_length, flagShowIncludeModifiers=False, flagShowExcludeBehaviorsWoEvents=False, by_category=False) if not parameters[SELECTED_SUBJECTS] or not parameters[SELECTED_BEHAVIORS]: QMessageBox.warning(None, programName, "Select subject(s) and behavior(s) to analyze") return ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, parameters[SELECTED_SUBJECTS], selected_observations, parameters[SELECTED_BEHAVIORS]) cursor = db_connector.cursor() # create intervals from DB cursor.execute( "SELECT observation, subject, behavior, start, stop FROM aggregated_events" ) events = {} for row in cursor.fetchall(): for event in row: obs, subj, behav, start, stop = row # check if start and stop are in selected time interval if stop < parameters[START_TIME]: continue if start > parameters[END_TIME]: continue if start < parameters[START_TIME]: start = float(parameters[START_TIME]) if stop > parameters[END_TIME]: stop = float(parameters[END_TIME]) if obs not in events: events[obs] = {} # use function in base at event (state or point) interval_func = icc if start == stop else ico if f"{subj}|{behav}" not in events[obs]: events[obs][f"{subj}|{behav}"] = interval_func([start, stop]) else: events[obs][f"{subj}|{behav}"] = events[obs][ f"{subj}|{behav}"] | interval_func([start, stop]) w = Advanced_event_filtering_dialog(events) w.exec_()