def create_subtitles(pj: dict, selected_observations: list, parameters: dict, export_dir: str) -> tuple: """ create subtitles for selected observations, subjects and behaviors Args: pj (dict): project selected_observations (list): list of observations parameters (dict): export_dir (str): directory to save subtitles Returns: bool: True if OK else False str: error message """ def subject_color(subject): """ subject color Args: subject (str): subject name Returns: str: HTML tag for color font (beginning) str: HTML tag for color font (closing) """ if subject == NO_FOCAL_SUBJECT: return "", "" else: return ( """<font color="{}">""".format( subtitlesColors[parameters["selected subjects"].index( row["subject"]) % len(subtitlesColors)]), "</font>", ) try: ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, parameters["selected subjects"], selected_observations, parameters["selected behaviors"]) if not ok: return False, msg cursor = db_connector.cursor() flag_ok = True msg = "" for obsId in selected_observations: if pj[OBSERVATIONS][obsId][TYPE] in [LIVE]: out = "" cursor.execute( ("SELECT subject, behavior, start, stop, type, modifiers FROM aggregated_events " "WHERE observation = ? AND subject in ({}) " "AND behavior in ({}) " "ORDER BY start").format( ",".join(["?"] * len(parameters["selected subjects"])), ",".join(["?"] * len(parameters["selected behaviors"]))), [obsId] + parameters["selected subjects"] + parameters["selected behaviors"], ) for idx, row in enumerate(cursor.fetchall()): col1, col2 = subject_color(row["subject"]) if parameters["include modifiers"]: modifiers_str = "\n{}".format(row["modifiers"].replace( "|", ", ")) else: modifiers_str = "" out += ("{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format( idx=idx + 1, start=utilities.seconds2time( row["start"]).replace(".", ","), stop=utilities.seconds2time( row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION).replace(".", ","), col1=col1, col2=col2, subject=row["subject"], behavior=row["behavior"], modifiers=modifiers_str, ) file_name = str( pathlib.Path( pathlib.Path(export_dir) / utilities.safeFileName(obsId)).with_suffix(".srt")) try: with open(file_name, "w") as f: f.write(out) except Exception: flag_ok = False msg += "observation: {}\ngave the following error:\n{}\n".format( obsId, str(sys.exc_info()[1])) elif pj[OBSERVATIONS][obsId][TYPE] in [MEDIA]: for nplayer in ALL_PLAYERS: if not pj[OBSERVATIONS][obsId][FILE][nplayer]: continue init = 0 for mediaFile in pj[OBSERVATIONS][obsId][FILE][nplayer]: try: end = init + pj[OBSERVATIONS][obsId][MEDIA_INFO][ LENGTH][mediaFile] except KeyError: return False, f"The length for media file {mediaFile} is not available" out = "" print(",".join(["?"] * len(parameters["selected subjects"]))) print(",".join(["?"] * len(parameters["selected behaviors"]))) print([obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"]) cursor.execute( ("SELECT subject, behavior, type, start, stop, modifiers FROM aggregated_events " "WHERE observation = ? AND start BETWEEN ? and ? " "AND subject in ({}) " "AND behavior in ({}) " "ORDER BY start").format( ",".join( ["?"] * len(parameters["selected subjects"])), ",".join( ["?"] * len(parameters["selected behaviors"])), ), [obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"], ) for idx, row in enumerate(cursor.fetchall()): col1, col2 = subject_color(row["subject"]) if parameters["include modifiers"]: modifiers_str = "\n{}".format( row["modifiers"].replace("|", ", ")) else: modifiers_str = "" out += ( "{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format( idx=idx + 1, start=utilities.seconds2time(row["start"] - init).replace( ".", ","), stop=utilities.seconds2time( (row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION) - init).replace(".", ","), col1=col1, col2=col2, subject=row["subject"], behavior=row["behavior"], modifiers=modifiers_str, ) file_name = str( pathlib.Path( pathlib.Path(export_dir) / pathlib.Path(mediaFile).name).with_suffix( ".srt")) try: with open(file_name, "w") as f: f.write(out) except Exception: flag_ok = False msg += "observation: {}\ngave the following error:\n{}\n".format( obsId, str(sys.exc_info()[1])) init = end return flag_ok, msg except Exception: return False, str(sys.exc_info()[1])
def behavior_binary_table(pj: dict): """ ask user for parameters for behavior binary table call create_behavior_binary_table """ result, selected_observations = select_observations.select_observations( pj, MULTIPLE, "Select observations for the behavior binary table") if not selected_observations: return # check if state events are paired out = "" not_paired_obs_list = [] for obs_id in selected_observations: r, msg = project_functions.check_state_events_obs( obs_id, pj[ETHOGRAM], pj[OBSERVATIONS][obs_id]) if not r: out += f"Observation: <strong>{obs_id}</strong><br>{msg}<br>" not_paired_obs_list.append(obs_id) if out: out = f"The observations with UNPAIRED state events will be removed from the analysis<br><br>{out}" results = dialog.Results_dialog() results.setWindowTitle(f"{programName} - Check selected observations") results.ptText.setReadOnly(True) results.ptText.appendHtml(out) results.pbSave.setVisible(False) results.pbCancel.setVisible(True) if not results.exec_(): return selected_observations = [ x for x in selected_observations if x not in not_paired_obs_list ] if not selected_observations: return max_obs_length, selectedObsTotalMediaLength = project_functions.observation_length( pj, selected_observations) if max_obs_length == -1: # media length not available, user choose to not use events return parameters = dialog.choose_obs_subj_behav_category( pj, selected_observations, maxTime=max_obs_length, flagShowIncludeModifiers=True, flagShowExcludeBehaviorsWoEvents=True, by_category=False) if not parameters[SELECTED_SUBJECTS] or not parameters[SELECTED_BEHAVIORS]: QMessageBox.warning(None, programName, "Select subject(s) and behavior(s) to analyze") return # ask for time interval i, ok = QInputDialog.getDouble(None, "Behavior binary table", "Time interval (in seconds):", 1.0, 0.001, 86400, 3) if not ok: return time_interval = utilities.float2decimal(i) ''' iw = dialog.Info_widget() iw.lwi.setVisible(False) iw.resize(350, 200) iw.setWindowFlags(Qt.WindowStaysOnTopHint) iw.setWindowTitle("Behavior binary table") iw.label.setText("Creating the behavior binary table...") iw.show() QApplication.processEvents() ''' results_df = create_behavior_binary_table(pj, selected_observations, parameters, time_interval) ''' iw.hide() ''' if "error" in results_df: QMessageBox.warning(None, programName, results_df["msg"]) return # save results if len(selected_observations) == 1: extended_file_formats = [ "Tab Separated Values (*.tsv)", "Comma Separated Values (*.csv)", "Open Document Spreadsheet ODS (*.ods)", "Microsoft Excel Spreadsheet XLSX (*.xlsx)", "Legacy Microsoft Excel Spreadsheet XLS (*.xls)", "HTML (*.html)" ] file_formats = ["tsv", "csv", "ods", "xlsx", "xls", "html"] file_name, filter_ = QFileDialog().getSaveFileName( None, "Save results", "", ";;".join(extended_file_formats)) if not file_name: return output_format = file_formats[extended_file_formats.index(filter_)] if pathlib.Path(file_name).suffix != "." + output_format: file_name = str(pathlib.Path(file_name)) + "." + output_format # check if file with new extension already exists if pathlib.Path(file_name).is_file(): if dialog.MessageDialog( programName, f"The file {file_name} already exists.", [CANCEL, OVERWRITE]) == CANCEL: return else: items = ("Tab Separated Values (*.tsv)", "Comma separated values (*.csv)", "Open Document Spreadsheet (*.ods)", "Microsoft Excel Spreadsheet XLSX (*.xlsx)", "Legacy Microsoft Excel Spreadsheet XLS (*.xls)", "HTML (*.html)") item, ok = QInputDialog.getItem(None, "Save results", "Available formats", items, 0, False) if not ok: return output_format = re.sub(".* \(\*\.", "", item)[:-1] export_dir = QFileDialog().getExistingDirectory( None, "Choose a directory to save results", os.path.expanduser("~"), options=QFileDialog.ShowDirsOnly) if not export_dir: return mem_command = "" for obs_id in results_df: for subject in results_df[obs_id]: if len(selected_observations) > 1: file_name_with_subject = str( pathlib.Path(export_dir) / utilities.safeFileName(obs_id + "_" + subject)) + "." + output_format else: file_name_with_subject = str( os.path.splitext(file_name)[0] + utilities.safeFileName("_" + subject)) + "." + output_format # check if file with new extension already exists if mem_command != OVERWRITE_ALL and pathlib.Path( file_name_with_subject).is_file(): if mem_command == "Skip all": continue mem_command = dialog.MessageDialog( programName, f"The file {file_name_with_subject} already exists.", [OVERWRITE, OVERWRITE_ALL, "Skip", "Skip all", CANCEL]) if mem_command == CANCEL: return if mem_command in ["Skip", "Skip all"]: continue try: if output_format in ["csv", "tsv", "html"]: with open(file_name_with_subject, "wb") as f: f.write( str.encode(results_df[obs_id][subject].export( output_format))) if output_format in ["ods", "xlsx", "xls"]: with open(file_name_with_subject, "wb") as f: f.write( results_df[obs_id][subject].export(output_format)) except Exception: error_type, error_file_name, error_lineno = utilities.error_info( sys.exc_info()) logging.critical( f"Error in behavior binary table function: {error_type} {error_file_name} {error_lineno}" ) QMessageBox.critical(None, programName, f"Error saving file: {error_type}") return
pj[SUBJECTS][k]["name"] for k in utilities.sorted_keys(pj[SUBJECTS]) ] + [NO_FOCAL_SUBJECT] output_format = "tsv" if len(args.command) > 1: output_format = args.command[1] for observation_id in observations_id_list: ok, msg = export_observation.export_events( { "selected subjects": subjects, "selected behaviors": behaviors }, observation_id, pj[OBSERVATIONS][observation_id], pj[ETHOGRAM], utilities.safeFileName(observation_id + "." + output_format), output_format) if not ok: print(msg) sys.exit() if "irr" in args.command: if len(observations_id_list) != 2: print("select 2 observations") sys.exit() behaviors = [ pj[ETHOGRAM][k]["code"] for k in utilities.sorted_keys(pj[ETHOGRAM]) ]
def create_events_plot(pj, selected_observations, parameters, plot_colors=BEHAVIORS_PLOT_COLORS, plot_directory="", file_format="png"): selected_subjects = parameters[SELECTED_SUBJECTS] selected_behaviors = parameters[SELECTED_BEHAVIORS] include_modifiers = parameters[INCLUDE_MODIFIERS] interval = parameters[TIME_INTERVAL] start_time = parameters[START_TIME] end_time = parameters[END_TIME] ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) all_behaviors = [ pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM]) ] par1 = 1 bar_height = 0.5 init = dt.datetime(2017, 1, 1) for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) else: fig, ax = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, [obs_id], selected_behaviors) cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor = db_connector.cursor() cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) # time obs_length = project_functions.observation_total_length( pj[OBSERVATIONS][obs_id]) if obs_length == -1: # media length not available interval = TIME_EVENTS if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute( "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", ( min_time, obs_id, min_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", ( max_time, obs_id, max_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", ( min_time, max_time, obs_id, min_time, max_time, )) ylabels = [" ".join(x) for x in distinct_behav_modif] for ax_idx, subject in enumerate(selected_subjects): if parameters["exclude behaviors"]: cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?", (subject, )) distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) ylabels = [" ".join(x) for x in distinct_behav_modif] if not ax_idx: axs[ax_idx].set_title(f"Observation {obs_id}\n{subject}", fontsize=14) else: axs[ax_idx].set_title(subject, fontsize=14) bars = {} i = 0 for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior bars[behavior_modifiers_str] = [] # total duration cursor.execute(( "SELECT start,stop FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?" ), ( obs_id, subject, behavior, modifiers, )) for row in cursor.fetchall(): bars[behavior_modifiers_str].append( (row["start"], row["stop"])) start_date = matplotlib.dates.date2num(init + dt.timedelta( seconds=row["start"])) end_date = matplotlib.dates.date2num(init + dt.timedelta( seconds=row["stop"] + POINT_EVENT_PLOT_DURATION * (row["stop"] == row["start"]))) try: bar_color = utilities.behavior_color( plot_colors, all_behaviors.index(behavior)) except Exception: bar_color = "darkgray" bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row[ "start"] else bar_color # sage colors removed from matplotlib colors list if bar_color in ["sage", "darksage", "lightsage"]: bar_color = { "darksage": "#598556", "lightsage": "#bcecac", "sage": "#87ae73" }[bar_color] try: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor=bar_color, color=bar_color, alpha=1) except Exception: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor="darkgray", color="darkgray", alpha=1) i += 1 axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1) pos = np.arange(par1, max_len * par1 + par1 + 1, par1) axs[ax_idx].set_yticks(pos[:len(ylabels)]) axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10}) axs[ax_idx].set_ylabel("Behaviors" + " (modifiers)" * include_modifiers, fontdict={"fontsize": 10}) axs[ax_idx].set_xlim( left=matplotlib.dates.date2num(init + dt.timedelta(seconds=min_time)), right=matplotlib.dates.date2num(init + dt.timedelta( seconds=max_time + 1))) axs[ax_idx].grid(color="g", linestyle=":") axs[ax_idx].xaxis_date() axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S")) axs[ax_idx].set_xlabel("Time (HH:MM:SS)", fontdict={"fontsize": 12}) axs[ax_idx].invert_yaxis() fig.autofmt_xdate() plt.tight_layout() if len(selected_observations) > 1: output_file_name = str( pathlib.Path( pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix("." + file_format)) plt.savefig(output_file_name) else: plt.show()
def behaviors_bar_plot(pj, selected_observations, selected_subjects, selected_behaviors, include_modifiers, interval, start_time, end_time, plot_directory, output_format): """ scatter plot """ parameters = [ ["duration", "Total duration"], ] ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() if include_modifiers: cursor.execute( "SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] else: cursor.execute("SELECT distinct behavior FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects, distinct_behav_modif, include_modifiers, parameters) # select time interval for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True) else: fig, ax = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length( pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute( "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", ( min_time, obs_id, min_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", ( max_time, obs_id, max_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", ( min_time, max_time, obs_id, min_time, max_time, )) for subject in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers # skip if behavior defined as POINT if POINT in project_functions.event_type( behavior, pj[ETHOGRAM]): continue behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior # total duration cursor.execute(( "SELECT SUM(stop-start) FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?" ), ( obs_id, subject, behavior, modifiers, )) for row in cursor.fetchall(): behaviors[subject][behavior_modifiers_str][ "duration"] = 0 if row[0] is None else row[0] print("behaviors") print(behaviors) print() print("sorted(distinct_behav_modif)", sorted(distinct_behav_modif)) max_length = 0 behaviors_duration = {} mb = {} for ax_idx, subj in enumerate(selected_subjects): behaviors_duration[subj] = {} behavior_ticks = [] for behavior_modifiers in sorted(distinct_behav_modif): behavior, modifiers = behavior_modifiers # skip if behavior defined as POINT if POINT in project_functions.event_type( behavior, pj[ETHOGRAM]): continue if behavior not in behaviors_duration[subj]: behaviors_duration[subj][behavior] = [[], []] behavior_modifiers_str = "|".join( behavior_modifiers) if modifiers else behavior if behavior not in behavior_ticks: behavior_ticks.append(behavior) for param in parameters: behaviors_duration[subj][behavior][0].append( behaviors[subj][behavior_modifiers_str][param[0]]) behaviors_duration[subj][behavior][1].append(modifiers) max_length = max( max_length, len(behaviors_duration[subj][behavior][1])) print() print("behaviors_duration", behaviors_duration) print() print("behavior_ticks", behavior_ticks) print() behavior_mod_ticks = behavior_ticks[:] for ax_idx, subj in enumerate(selected_subjects): print("subject", subj) md_lgd = [] b = {} for i in range(max_length): b[i] = [] for behavior in sorted(behaviors_duration[subj].keys()): try: b[i].append(behaviors_duration[subj][behavior][0][i]) if include_modifiers: if behaviors_duration[subj][behavior][1][i]: md_lgd.append( behavior + " " + behaviors_duration[subj][behavior][1][i]) else: md_lgd.append(behavior) except Exception: b[i].append(0) print() print("behavior_mod_ticks", behavior_mod_ticks) print() print("b") print(b) print() print("md_lgd") print(md_lgd) ind = np.arange(len(behavior_ticks)) width = 0.35 # the width of the bars: can also be len(x) sequence pp = [] max_obs = 0 bottom_ = [] idx_color = 0 for i in sorted(b.keys()): if i == 0: pp.append(axs[ax_idx].bar( ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])])) else: pp.append(axs[ax_idx].bar( ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])], bottom=bottom_)) idx_color += len(b[i]) if not bottom_: bottom_ = b[i] else: bottom_ = [x + bottom_[idx] for idx, x in enumerate(b[i])] max_obs = max(max_obs, sum(b[i])) if ax_idx == 0: axs[ax_idx].set_ylabel("Duration (s)") axs[ax_idx].set_xlabel("Behaviors") axs[ax_idx].set_title(f"{subj}") axs[ax_idx].set_xticks(ind) axs[ax_idx].set_xticklabels(behavior_mod_ticks, rotation=90) axs[ax_idx].set_yticks(np.arange(0, max(bottom_), 50)) lgd_col = [] for p in pp: for r in p: if r.get_height(): lgd_col.append(r) plt.legend(lgd_col, md_lgd) if plot_directory: output_file_name = str( pathlib.Path( pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix("." + file_format)) plt.savefig(output_file_name) else: plt.show()
def test_filename_with_spaces(self): assert utilities.safeFileName("aaa bbb.ccc") == "aaa bbb.ccc"
def test_filename_with_forbidden_chars(self): # ["/", "\\", ":", "*", "?", '"', "<", ">", "|"] assert utilities.safeFileName("aaa/bb\\b.c:cc ddd* ? \"www\" <> |" ) == "aaa_bb_b.c_cc ddd_ _ _www_ __ _"
def behaviors_bar_plot(pj, selected_observations, selected_subjects, selected_behaviors, include_modifiers, interval, start_time, end_time, plot_directory, output_format): """ scatter plot """ parameters = [["duration", "Total duration"], ] ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() if include_modifiers: cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] else: cursor.execute("SELECT distinct behavior FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects, distinct_behav_modif, include_modifiers, parameters) # select time interval for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True) else: fig, ax = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") # time obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id]) if obs_length == -1: obs_length = 0 if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obs_id, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obs_id, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obs_id, min_time, max_time, )) for subject in selected_subjects: for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers # skip if behavior defined as POINT if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]): continue behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior # total duration cursor.execute(("SELECT SUM(stop-start) FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"), (obs_id, subject, behavior, modifiers,)) for row in cursor.fetchall(): behaviors[subject][behavior_modifiers_str]["duration"] = 0 if row[0] is None else row[0] print("behaviors") print(behaviors) print() print("sorted(distinct_behav_modif)", sorted(distinct_behav_modif)) max_length = 0 behaviors_duration = {} mb = {} for ax_idx, subj in enumerate(selected_subjects): behaviors_duration[subj] = {} behavior_ticks = [] for behavior_modifiers in sorted(distinct_behav_modif): behavior, modifiers = behavior_modifiers # skip if behavior defined as POINT if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]): continue if behavior not in behaviors_duration[subj]: behaviors_duration[subj][behavior] = [[],[]] behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior #print(subj, behavior, modifiers) if behavior not in behavior_ticks: behavior_ticks.append(behavior) for param in parameters: behaviors_duration[subj][behavior][0].append(behaviors[subj][behavior_modifiers_str][param[0]]) behaviors_duration[subj][behavior][1].append(modifiers) max_length = max(max_length, len(behaviors_duration[subj][behavior][1])) print() print("behaviors_duration", behaviors_duration) print() print("behavior_ticks", behavior_ticks) print() #b = {} behavior_mod_ticks = behavior_ticks[:] for ax_idx, subj in enumerate(selected_subjects): print("subject", subj) md_lgd = [] b = {} for i in range(max_length): b[i] = [] for behavior in sorted(behaviors_duration[subj].keys()): try: b[i].append(behaviors_duration[subj][behavior][0][i]) if include_modifiers: ''' if behaviors_duration[subj][behavior][1][i]: behavior_mod_ticks[behavior_ticks.index(behavior)] = behavior_mod_ticks[behavior_ticks.index(behavior)] + "\n" + \ behaviors_duration[subj][behavior][1][i] ''' if behaviors_duration[subj][behavior][1][i]: md_lgd.append(behavior + " " + behaviors_duration[subj][behavior][1][i]) else: md_lgd.append(behavior) except: b[i].append(0) print() print("behavior_mod_ticks", behavior_mod_ticks) print() print("b") print(b) print() print("md_lgd") print(md_lgd) #ind = np.arange(len(behavior_ticks)) # the x locations for the groups ind = np.arange(len(behavior_ticks)) width = 0.35 # the width of the bars: can also be len(x) sequence pp = [] max_obs = 0 bottom_ = [] idx_color = 0 for i in sorted(b.keys()): print(i, b[i]) if i == 0: pp.append(axs[ax_idx].bar(ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])])) else: pp.append(axs[ax_idx].bar(ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])], bottom=bottom_)) idx_color += len(b[i]) if not bottom_: bottom_ = b[i] else: bottom_ = [x + bottom_[idx] for idx,x in enumerate(b[i])] max_obs = max(max_obs, sum(b[i])) if ax_idx == 0: axs[ax_idx].set_ylabel('Duration (s)') axs[ax_idx].set_xlabel('Behaviors') axs[ax_idx].set_title('{}'.format(subj)) axs[ax_idx].set_xticks(ind) axs[ax_idx].set_xticklabels(behavior_mod_ticks,rotation=90) axs[ax_idx].set_yticks(np.arange(0, max(bottom_), 50)) lgd_col = [] for p in pp: for r in p: if r.get_height(): lgd_col.append(r) plt.legend(lgd_col, md_lgd) if plot_directory: output_file_name = str(pathlib.Path(pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix("." + file_format)) plt.savefig(output_file_name) else: plt.show()
def create_events_plot(pj, selected_observations, parameters, plot_colors=BEHAVIORS_PLOT_COLORS, plot_directory="", file_format="png"): selected_subjects = parameters["selected subjects"] selected_behaviors = parameters["selected behaviors"] include_modifiers = parameters["include modifiers"] interval = parameters[TIME_INTERVAL] start_time = parameters[START_TIME] end_time = parameters[END_TIME] ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj, selected_subjects, selected_observations, selected_behaviors) if not ok: return False, msg, None cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])] par1 = 1 bar_height = 0.5 init = dt.datetime(2017, 1, 1) for obs_id in selected_observations: if len(selected_subjects) > 1: fig, axs = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) else: fig, ax = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True) axs = np.ndarray(shape=(1), dtype=type(ax)) axs[0] = ax ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, selected_subjects, [obs_id], selected_behaviors) cursor = db_connector.cursor() # if modifiers not to be included set modifiers to "" if not include_modifiers: cursor.execute("UPDATE aggregated_events SET modifiers = ''") cursor = db_connector.cursor() cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events") distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) # time obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id]) if obs_length == -1: # media length not available interval = TIME_EVENTS if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obs_id, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obs_id, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obs_id, min_time, max_time, )) ylabels = [" ".join(x) for x in distinct_behav_modif] for ax_idx, subject in enumerate(selected_subjects): if parameters["exclude behaviors"]: cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?", (subject, )) distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()] # add selected behaviors that are not observed if not parameters["exclude behaviors"]: for behav in selected_behaviors: if [x for x in distinct_behav_modif if x[0] == behav] == []: distinct_behav_modif.append([behav, "-"]) distinct_behav_modif = sorted(distinct_behav_modif) max_len = len(distinct_behav_modif) ylabels = [" ".join(x) for x in distinct_behav_modif] if not ax_idx: axs[ax_idx].set_title("Observation {}\n{}".format(obs_id, subject), fontsize=14) else: axs[ax_idx].set_title(subject, fontsize=14) bars = {} i = 0 for behavior_modifiers in distinct_behav_modif: behavior, modifiers = behavior_modifiers behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior bars[behavior_modifiers_str] = [] # total duration cursor.execute(("SELECT start,stop FROM aggregated_events " "WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"), (obs_id, subject, behavior, modifiers,)) for row in cursor.fetchall(): bars[behavior_modifiers_str].append((row["start"], row["stop"])) start_date = matplotlib.dates.date2num(init + dt.timedelta(seconds=row["start"])) end_date = matplotlib.dates.date2num( init + dt.timedelta(seconds=row["stop"] + POINT_EVENT_PLOT_DURATION * (row["stop"] == row["start"]))) try: bar_color = utilities.behavior_color(plot_colors, all_behaviors.index(behavior)) except Exception: bar_color = "darkgray" bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row["start"] else bar_color # sage colors removed from matplotlib colors list if bar_color in ["sage", "darksage", "lightsage"]: bar_color = {"darksage": "#598556", "lightsage": "#bcecac", "sage": "#87ae73"}[bar_color] try: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor=bar_color, color=bar_color, alpha=1) except Exception: axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height, align="center", edgecolor="darkgray", color="darkgray", alpha=1) i += 1 axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1) pos = np.arange(par1, max_len * par1 + par1 + 1, par1) axs[ax_idx].set_yticks(pos[:len(ylabels)]) axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10}) axs[ax_idx].set_ylabel("Behaviors" + " (modifiers)" * include_modifiers, fontdict={"fontsize": 10}) axs[ax_idx].set_xlim(left=matplotlib.dates.date2num(init + dt.timedelta(seconds=min_time)), right=matplotlib.dates.date2num(init + dt.timedelta(seconds=max_time + 1))) axs[ax_idx].grid(color="g", linestyle=":") axs[ax_idx].xaxis_date() axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S")) axs[ax_idx].set_xlabel("Time (HH:MM:SS)", fontdict={"fontsize": 12}) axs[ax_idx].invert_yaxis() fig.autofmt_xdate() plt.tight_layout() if len(selected_observations) > 1: output_file_name = str(pathlib.Path(pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix( "." + file_format)) plt.savefig(output_file_name) else: plt.show()
def create_subtitles(pj: dict, selected_observations: list, parameters: dict, export_dir: str) -> tuple: """ create subtitles for selected observations, subjects and behaviors Args: pj (dict): project selected_observations (list): list of observations parameters (dict): export_dir (str): directory to save subtitles Returns: bool: True if OK else False str: error message """ def subject_color(subject): """ subject color Args: subject (str): subject name Returns: str: HTML tag for color font (beginning) str: HTML tag for color font (closing) """ if subject == NO_FOCAL_SUBJECT: return "", "" else: return ( """<font color="{}">""".format( subtitlesColors[parameters["selected subjects"].index(row["subject"]) % len(subtitlesColors)] ), "</font>", ) try: ok, msg, db_connector = db_functions.load_aggregated_events_in_db( pj, parameters["selected subjects"], selected_observations, parameters["selected behaviors"] ) if not ok: return False, msg cursor = db_connector.cursor() flag_ok = True msg = "" for obsId in selected_observations: if pj[OBSERVATIONS][obsId][TYPE] in [LIVE]: out = "" cursor.execute( ( "SELECT subject, behavior, start, stop, type, modifiers FROM aggregated_events " "WHERE observation = ? AND subject in ({}) " "AND behavior in ({}) " "ORDER BY start" ).format(",".join(["?"] * len(parameters["selected subjects"])), ",".join(["?"] * len(parameters["selected behaviors"]))), [obsId] + parameters["selected subjects"] + parameters["selected behaviors"], ) for idx, row in enumerate(cursor.fetchall()): col1, col2 = subject_color(row["subject"]) if parameters["include modifiers"]: modifiers_str = "\n{}".format(row["modifiers"].replace("|", ", ")) else: modifiers_str = "" out += ("{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format( idx=idx + 1, start=utilities.seconds2time(row["start"]).replace(".", ","), stop=utilities.seconds2time(row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION).replace( ".", "," ), col1=col1, col2=col2, subject=row["subject"], behavior=row["behavior"], modifiers=modifiers_str, ) file_name = str(pathlib.Path(pathlib.Path(export_dir) / utilities.safeFileName(obsId)).with_suffix(".srt")) try: with open(file_name, "w") as f: f.write(out) except Exception: flag_ok = False msg += "observation: {}\ngave the following error:\n{}\n".format(obsId, str(sys.exc_info()[1])) elif pj[OBSERVATIONS][obsId][TYPE] in [MEDIA]: for nplayer in ALL_PLAYERS: if not pj[OBSERVATIONS][obsId][FILE][nplayer]: continue init = 0 for mediaFile in pj[OBSERVATIONS][obsId][FILE][nplayer]: try: end = init + pj[OBSERVATIONS][obsId][MEDIA_INFO][LENGTH][mediaFile] except KeyError: return False, f"The length for media file {mediaFile} is not available" out = "" print(",".join(["?"] * len(parameters["selected subjects"]))) print(",".join(["?"] * len(parameters["selected behaviors"]))) print([obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"]) cursor.execute( ( "SELECT subject, behavior, type, start, stop, modifiers FROM aggregated_events " "WHERE observation = ? AND start BETWEEN ? and ? " "AND subject in ({}) " "AND behavior in ({}) " "ORDER BY start" ).format( ",".join(["?"] * len(parameters["selected subjects"])), ",".join(["?"] * len(parameters["selected behaviors"])), ), [obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"], ) for idx, row in enumerate(cursor.fetchall()): col1, col2 = subject_color(row["subject"]) if parameters["include modifiers"]: modifiers_str = "\n{}".format(row["modifiers"].replace("|", ", ")) else: modifiers_str = "" out += ("{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format( idx=idx + 1, start=utilities.seconds2time(row["start"] - init).replace(".", ","), stop=utilities.seconds2time( (row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION) - init ).replace(".", ","), col1=col1, col2=col2, subject=row["subject"], behavior=row["behavior"], modifiers=modifiers_str, ) file_name = str(pathlib.Path(pathlib.Path(export_dir) / pathlib.Path(mediaFile).name).with_suffix(".srt")) try: with open(file_name, "w") as f: f.write(out) except Exception: flag_ok = False msg += "observation: {}\ngave the following error:\n{}\n".format(obsId, str(sys.exc_info()[1])) init = end return flag_ok, msg except Exception: return False, str(sys.exc_info()[1])