def create_subtitles(pj: dict, selected_observations: list, parameters: dict,
export_dir: str) -> tuple:
"""
create subtitles for selected observations, subjects and behaviors
Args:
pj (dict): project
selected_observations (list): list of observations
parameters (dict):
export_dir (str): directory to save subtitles
Returns:
bool: True if OK else False
str: error message
"""
def subject_color(subject):
"""
subject color
Args:
subject (str): subject name
Returns:
str: HTML tag for color font (beginning)
str: HTML tag for color font (closing)
"""
if subject == NO_FOCAL_SUBJECT:
return "", ""
else:
return (
"""<font color="{}">""".format(
subtitlesColors[parameters["selected subjects"].index(
row["subject"]) % len(subtitlesColors)]),
"</font>",
)
try:
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, parameters["selected subjects"], selected_observations,
parameters["selected behaviors"])
if not ok:
return False, msg
cursor = db_connector.cursor()
flag_ok = True
msg = ""
for obsId in selected_observations:
if pj[OBSERVATIONS][obsId][TYPE] in [LIVE]:
out = ""
cursor.execute(
("SELECT subject, behavior, start, stop, type, modifiers FROM aggregated_events "
"WHERE observation = ? AND subject in ({}) "
"AND behavior in ({}) "
"ORDER BY start").format(
",".join(["?"] *
len(parameters["selected subjects"])),
",".join(["?"] *
len(parameters["selected behaviors"]))),
[obsId] + parameters["selected subjects"] +
parameters["selected behaviors"],
)
for idx, row in enumerate(cursor.fetchall()):
col1, col2 = subject_color(row["subject"])
if parameters["include modifiers"]:
modifiers_str = "\n{}".format(row["modifiers"].replace(
"|", ", "))
else:
modifiers_str = ""
out += ("{idx}\n"
"{start} --> {stop}\n"
"{col1}{subject}: {behavior}"
"{modifiers}"
"{col2}\n\n").format(
idx=idx + 1,
start=utilities.seconds2time(
row["start"]).replace(".", ","),
stop=utilities.seconds2time(
row["stop"] if row["type"] ==
STATE else row["stop"] +
POINT_EVENT_ST_DURATION).replace(".", ","),
col1=col1,
col2=col2,
subject=row["subject"],
behavior=row["behavior"],
modifiers=modifiers_str,
)
file_name = str(
pathlib.Path(
pathlib.Path(export_dir) /
utilities.safeFileName(obsId)).with_suffix(".srt"))
try:
with open(file_name, "w") as f:
f.write(out)
except Exception:
flag_ok = False
msg += "observation: {}\ngave the following error:\n{}\n".format(
obsId, str(sys.exc_info()[1]))
elif pj[OBSERVATIONS][obsId][TYPE] in [MEDIA]:
for nplayer in ALL_PLAYERS:
if not pj[OBSERVATIONS][obsId][FILE][nplayer]:
continue
init = 0
for mediaFile in pj[OBSERVATIONS][obsId][FILE][nplayer]:
try:
end = init + pj[OBSERVATIONS][obsId][MEDIA_INFO][
LENGTH][mediaFile]
except KeyError:
return False, f"The length for media file {mediaFile} is not available"
out = ""
print(",".join(["?"] *
len(parameters["selected subjects"])))
print(",".join(["?"] *
len(parameters["selected behaviors"])))
print([obsId, init, end] +
parameters["selected subjects"] +
parameters["selected behaviors"])
cursor.execute(
("SELECT subject, behavior, type, start, stop, modifiers FROM aggregated_events "
"WHERE observation = ? AND start BETWEEN ? and ? "
"AND subject in ({}) "
"AND behavior in ({}) "
"ORDER BY start").format(
",".join(
["?"] *
len(parameters["selected subjects"])),
",".join(
["?"] *
len(parameters["selected behaviors"])),
),
[obsId, init, end] +
parameters["selected subjects"] +
parameters["selected behaviors"],
)
for idx, row in enumerate(cursor.fetchall()):
col1, col2 = subject_color(row["subject"])
if parameters["include modifiers"]:
modifiers_str = "\n{}".format(
row["modifiers"].replace("|", ", "))
else:
modifiers_str = ""
out += (
"{idx}\n"
"{start} --> {stop}\n"
"{col1}{subject}: {behavior}"
"{modifiers}"
"{col2}\n\n").format(
idx=idx + 1,
start=utilities.seconds2time(row["start"] -
init).replace(
".", ","),
stop=utilities.seconds2time(
(row["stop"] if row["type"] ==
STATE else row["stop"] +
POINT_EVENT_ST_DURATION) -
init).replace(".", ","),
col1=col1,
col2=col2,
subject=row["subject"],
behavior=row["behavior"],
modifiers=modifiers_str,
)
file_name = str(
pathlib.Path(
pathlib.Path(export_dir) /
pathlib.Path(mediaFile).name).with_suffix(
".srt"))
try:
with open(file_name, "w") as f:
f.write(out)
except Exception:
flag_ok = False
msg += "observation: {}\ngave the following error:\n{}\n".format(
obsId, str(sys.exc_info()[1]))
init = end
return flag_ok, msg
except Exception:
return False, str(sys.exc_info()[1])
def behavior_binary_table(pj: dict):
"""
ask user for parameters for behavior binary table
call create_behavior_binary_table
"""
result, selected_observations = select_observations.select_observations(
pj, MULTIPLE, "Select observations for the behavior binary table")
if not selected_observations:
return
# check if state events are paired
out = ""
not_paired_obs_list = []
for obs_id in selected_observations:
r, msg = project_functions.check_state_events_obs(
obs_id, pj[ETHOGRAM], pj[OBSERVATIONS][obs_id])
if not r:
out += f"Observation: <strong>{obs_id}</strong><br>{msg}<br>"
not_paired_obs_list.append(obs_id)
if out:
out = f"The observations with UNPAIRED state events will be removed from the analysis<br><br>{out}"
results = dialog.Results_dialog()
results.setWindowTitle(f"{programName} - Check selected observations")
results.ptText.setReadOnly(True)
results.ptText.appendHtml(out)
results.pbSave.setVisible(False)
results.pbCancel.setVisible(True)
if not results.exec_():
return
selected_observations = [
x for x in selected_observations if x not in not_paired_obs_list
]
if not selected_observations:
return
max_obs_length, selectedObsTotalMediaLength = project_functions.observation_length(
pj, selected_observations)
if max_obs_length == -1: # media length not available, user choose to not use events
return
parameters = dialog.choose_obs_subj_behav_category(
pj,
selected_observations,
maxTime=max_obs_length,
flagShowIncludeModifiers=True,
flagShowExcludeBehaviorsWoEvents=True,
by_category=False)
if not parameters[SELECTED_SUBJECTS] or not parameters[SELECTED_BEHAVIORS]:
QMessageBox.warning(None, programName,
"Select subject(s) and behavior(s) to analyze")
return
# ask for time interval
i, ok = QInputDialog.getDouble(None, "Behavior binary table",
"Time interval (in seconds):", 1.0, 0.001,
86400, 3)
if not ok:
return
time_interval = utilities.float2decimal(i)
'''
iw = dialog.Info_widget()
iw.lwi.setVisible(False)
iw.resize(350, 200)
iw.setWindowFlags(Qt.WindowStaysOnTopHint)
iw.setWindowTitle("Behavior binary table")
iw.label.setText("Creating the behavior binary table...")
iw.show()
QApplication.processEvents()
'''
results_df = create_behavior_binary_table(pj, selected_observations,
parameters, time_interval)
'''
iw.hide()
'''
if "error" in results_df:
QMessageBox.warning(None, programName, results_df["msg"])
return
# save results
if len(selected_observations) == 1:
extended_file_formats = [
"Tab Separated Values (*.tsv)", "Comma Separated Values (*.csv)",
"Open Document Spreadsheet ODS (*.ods)",
"Microsoft Excel Spreadsheet XLSX (*.xlsx)",
"Legacy Microsoft Excel Spreadsheet XLS (*.xls)", "HTML (*.html)"
]
file_formats = ["tsv", "csv", "ods", "xlsx", "xls", "html"]
file_name, filter_ = QFileDialog().getSaveFileName(
None, "Save results", "", ";;".join(extended_file_formats))
if not file_name:
return
output_format = file_formats[extended_file_formats.index(filter_)]
if pathlib.Path(file_name).suffix != "." + output_format:
file_name = str(pathlib.Path(file_name)) + "." + output_format
# check if file with new extension already exists
if pathlib.Path(file_name).is_file():
if dialog.MessageDialog(
programName, f"The file {file_name} already exists.",
[CANCEL, OVERWRITE]) == CANCEL:
return
else:
items = ("Tab Separated Values (*.tsv)",
"Comma separated values (*.csv)",
"Open Document Spreadsheet (*.ods)",
"Microsoft Excel Spreadsheet XLSX (*.xlsx)",
"Legacy Microsoft Excel Spreadsheet XLS (*.xls)",
"HTML (*.html)")
item, ok = QInputDialog.getItem(None, "Save results",
"Available formats", items, 0, False)
if not ok:
return
output_format = re.sub(".* \(\*\.", "", item)[:-1]
export_dir = QFileDialog().getExistingDirectory(
None,
"Choose a directory to save results",
os.path.expanduser("~"),
options=QFileDialog.ShowDirsOnly)
if not export_dir:
return
mem_command = ""
for obs_id in results_df:
for subject in results_df[obs_id]:
if len(selected_observations) > 1:
file_name_with_subject = str(
pathlib.Path(export_dir) /
utilities.safeFileName(obs_id + "_" +
subject)) + "." + output_format
else:
file_name_with_subject = str(
os.path.splitext(file_name)[0] +
utilities.safeFileName("_" +
subject)) + "." + output_format
# check if file with new extension already exists
if mem_command != OVERWRITE_ALL and pathlib.Path(
file_name_with_subject).is_file():
if mem_command == "Skip all":
continue
mem_command = dialog.MessageDialog(
programName,
f"The file {file_name_with_subject} already exists.",
[OVERWRITE, OVERWRITE_ALL, "Skip", "Skip all", CANCEL])
if mem_command == CANCEL:
return
if mem_command in ["Skip", "Skip all"]:
continue
try:
if output_format in ["csv", "tsv", "html"]:
with open(file_name_with_subject, "wb") as f:
f.write(
str.encode(results_df[obs_id][subject].export(
output_format)))
if output_format in ["ods", "xlsx", "xls"]:
with open(file_name_with_subject, "wb") as f:
f.write(
results_df[obs_id][subject].export(output_format))
except Exception:
error_type, error_file_name, error_lineno = utilities.error_info(
sys.exc_info())
logging.critical(
f"Error in behavior binary table function: {error_type} {error_file_name} {error_lineno}"
)
QMessageBox.critical(None, programName,
f"Error saving file: {error_type}")
return
pj[SUBJECTS][k]["name"]
for k in utilities.sorted_keys(pj[SUBJECTS])
] + [NO_FOCAL_SUBJECT]
output_format = "tsv"
if len(args.command) > 1:
output_format = args.command[1]
for observation_id in observations_id_list:
ok, msg = export_observation.export_events(
{
"selected subjects": subjects,
"selected behaviors": behaviors
}, observation_id, pj[OBSERVATIONS][observation_id],
pj[ETHOGRAM],
utilities.safeFileName(observation_id + "." + output_format),
output_format)
if not ok:
print(msg)
sys.exit()
if "irr" in args.command:
if len(observations_id_list) != 2:
print("select 2 observations")
sys.exit()
behaviors = [
pj[ETHOGRAM][k]["code"]
for k in utilities.sorted_keys(pj[ETHOGRAM])
]
def create_events_plot(pj,
selected_observations,
parameters,
plot_colors=BEHAVIORS_PLOT_COLORS,
plot_directory="",
file_format="png"):
selected_subjects = parameters[SELECTED_SUBJECTS]
selected_behaviors = parameters[SELECTED_BEHAVIORS]
include_modifiers = parameters[INCLUDE_MODIFIERS]
interval = parameters[TIME_INTERVAL]
start_time = parameters[START_TIME]
end_time = parameters[END_TIME]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, selected_behaviors)
if not ok:
return False, msg, None
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
all_behaviors = [
pj[ETHOGRAM][x][BEHAVIOR_CODE]
for x in utilities.sorted_keys(pj[ETHOGRAM])
]
par1 = 1
bar_height = 0.5
init = dt.datetime(2017, 1, 1)
for obs_id in selected_observations:
if len(selected_subjects) > 1:
fig, axs = plt.subplots(figsize=(20, 8),
nrows=len(selected_subjects),
ncols=1,
sharex=True)
else:
fig, ax = plt.subplots(figsize=(20, 8),
nrows=len(selected_subjects),
ncols=1,
sharex=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, [obs_id], selected_behaviors)
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
cursor = db_connector.cursor()
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
if not parameters["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
# time
obs_length = project_functions.observation_total_length(
pj[OBSERVATIONS][obs_id])
if obs_length == -1: # media length not available
interval = TIME_EVENTS
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute(
"UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(
min_time,
obs_id,
min_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(
max_time,
obs_id,
max_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(
min_time,
max_time,
obs_id,
min_time,
max_time,
))
ylabels = [" ".join(x) for x in distinct_behav_modif]
for ax_idx, subject in enumerate(selected_subjects):
if parameters["exclude behaviors"]:
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?",
(subject, ))
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
if not parameters["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif
if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
ylabels = [" ".join(x) for x in distinct_behav_modif]
if not ax_idx:
axs[ax_idx].set_title(f"Observation {obs_id}\n{subject}",
fontsize=14)
else:
axs[ax_idx].set_title(subject, fontsize=14)
bars = {}
i = 0
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
bars[behavior_modifiers_str] = []
# total duration
cursor.execute((
"SELECT start,stop FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"
), (
obs_id,
subject,
behavior,
modifiers,
))
for row in cursor.fetchall():
bars[behavior_modifiers_str].append(
(row["start"], row["stop"]))
start_date = matplotlib.dates.date2num(init + dt.timedelta(
seconds=row["start"]))
end_date = matplotlib.dates.date2num(init + dt.timedelta(
seconds=row["stop"] + POINT_EVENT_PLOT_DURATION *
(row["stop"] == row["start"])))
try:
bar_color = utilities.behavior_color(
plot_colors, all_behaviors.index(behavior))
except Exception:
bar_color = "darkgray"
bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row[
"start"] else bar_color
# sage colors removed from matplotlib colors list
if bar_color in ["sage", "darksage", "lightsage"]:
bar_color = {
"darksage": "#598556",
"lightsage": "#bcecac",
"sage": "#87ae73"
}[bar_color]
try:
axs[ax_idx].barh((i * par1) + par1,
end_date - start_date,
left=start_date,
height=bar_height,
align="center",
edgecolor=bar_color,
color=bar_color,
alpha=1)
except Exception:
axs[ax_idx].barh((i * par1) + par1,
end_date - start_date,
left=start_date,
height=bar_height,
align="center",
edgecolor="darkgray",
color="darkgray",
alpha=1)
i += 1
axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1)
pos = np.arange(par1, max_len * par1 + par1 + 1, par1)
axs[ax_idx].set_yticks(pos[:len(ylabels)])
axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10})
axs[ax_idx].set_ylabel("Behaviors" +
" (modifiers)" * include_modifiers,
fontdict={"fontsize": 10})
axs[ax_idx].set_xlim(
left=matplotlib.dates.date2num(init +
dt.timedelta(seconds=min_time)),
right=matplotlib.dates.date2num(init + dt.timedelta(
seconds=max_time + 1)))
axs[ax_idx].grid(color="g", linestyle=":")
axs[ax_idx].xaxis_date()
axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
axs[ax_idx].set_xlabel("Time (HH:MM:SS)",
fontdict={"fontsize": 12})
axs[ax_idx].invert_yaxis()
fig.autofmt_xdate()
plt.tight_layout()
if len(selected_observations) > 1:
output_file_name = str(
pathlib.Path(
pathlib.Path(plot_directory) /
utilities.safeFileName(obs_id)).with_suffix("." +
file_format))
plt.savefig(output_file_name)
else:
plt.show()
def behaviors_bar_plot(pj, selected_observations, selected_subjects,
selected_behaviors, include_modifiers, interval,
start_time, end_time, plot_directory, output_format):
"""
scatter plot
"""
parameters = [
["duration", "Total duration"],
]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, selected_behaviors)
if not ok:
return False, msg, None
cursor = db_connector.cursor()
if include_modifiers:
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
else:
cursor.execute("SELECT distinct behavior FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], ""]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects,
distinct_behav_modif, include_modifiers,
parameters)
# select time interval
for obs_id in selected_observations:
if len(selected_subjects) > 1:
fig, axs = plt.subplots(nrows=1,
ncols=len(selected_subjects),
sharey=True)
else:
fig, ax = plt.subplots(nrows=1,
ncols=len(selected_subjects),
sharey=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
# time
obs_length = project_functions.observation_total_length(
pj[OBSERVATIONS][obs_id])
if obs_length == -1:
obs_length = 0
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute(
"UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(
min_time,
obs_id,
min_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(
max_time,
obs_id,
max_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(
min_time,
max_time,
obs_id,
min_time,
max_time,
))
for subject in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
# skip if behavior defined as POINT
if POINT in project_functions.event_type(
behavior, pj[ETHOGRAM]):
continue
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
# total duration
cursor.execute((
"SELECT SUM(stop-start) FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"
), (
obs_id,
subject,
behavior,
modifiers,
))
for row in cursor.fetchall():
behaviors[subject][behavior_modifiers_str][
"duration"] = 0 if row[0] is None else row[0]
print("behaviors")
print(behaviors)
print()
print("sorted(distinct_behav_modif)", sorted(distinct_behav_modif))
max_length = 0
behaviors_duration = {}
mb = {}
for ax_idx, subj in enumerate(selected_subjects):
behaviors_duration[subj] = {}
behavior_ticks = []
for behavior_modifiers in sorted(distinct_behav_modif):
behavior, modifiers = behavior_modifiers
# skip if behavior defined as POINT
if POINT in project_functions.event_type(
behavior, pj[ETHOGRAM]):
continue
if behavior not in behaviors_duration[subj]:
behaviors_duration[subj][behavior] = [[], []]
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
if behavior not in behavior_ticks:
behavior_ticks.append(behavior)
for param in parameters:
behaviors_duration[subj][behavior][0].append(
behaviors[subj][behavior_modifiers_str][param[0]])
behaviors_duration[subj][behavior][1].append(modifiers)
max_length = max(
max_length, len(behaviors_duration[subj][behavior][1]))
print()
print("behaviors_duration", behaviors_duration)
print()
print("behavior_ticks", behavior_ticks)
print()
behavior_mod_ticks = behavior_ticks[:]
for ax_idx, subj in enumerate(selected_subjects):
print("subject", subj)
md_lgd = []
b = {}
for i in range(max_length):
b[i] = []
for behavior in sorted(behaviors_duration[subj].keys()):
try:
b[i].append(behaviors_duration[subj][behavior][0][i])
if include_modifiers:
if behaviors_duration[subj][behavior][1][i]:
md_lgd.append(
behavior + " " +
behaviors_duration[subj][behavior][1][i])
else:
md_lgd.append(behavior)
except Exception:
b[i].append(0)
print()
print("behavior_mod_ticks", behavior_mod_ticks)
print()
print("b")
print(b)
print()
print("md_lgd")
print(md_lgd)
ind = np.arange(len(behavior_ticks))
width = 0.35 # the width of the bars: can also be len(x) sequence
pp = []
max_obs = 0
bottom_ = []
idx_color = 0
for i in sorted(b.keys()):
if i == 0:
pp.append(axs[ax_idx].bar(
ind,
b[i],
width,
color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color +
len(b[i])]))
else:
pp.append(axs[ax_idx].bar(
ind,
b[i],
width,
color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color +
len(b[i])],
bottom=bottom_))
idx_color += len(b[i])
if not bottom_:
bottom_ = b[i]
else:
bottom_ = [x + bottom_[idx] for idx, x in enumerate(b[i])]
max_obs = max(max_obs, sum(b[i]))
if ax_idx == 0:
axs[ax_idx].set_ylabel("Duration (s)")
axs[ax_idx].set_xlabel("Behaviors")
axs[ax_idx].set_title(f"{subj}")
axs[ax_idx].set_xticks(ind)
axs[ax_idx].set_xticklabels(behavior_mod_ticks, rotation=90)
axs[ax_idx].set_yticks(np.arange(0, max(bottom_), 50))
lgd_col = []
for p in pp:
for r in p:
if r.get_height():
lgd_col.append(r)
plt.legend(lgd_col, md_lgd)
if plot_directory:
output_file_name = str(
pathlib.Path(
pathlib.Path(plot_directory) /
utilities.safeFileName(obs_id)).with_suffix("." +
file_format))
plt.savefig(output_file_name)
else:
plt.show()
def test_filename_with_spaces(self):
assert utilities.safeFileName("aaa bbb.ccc") == "aaa bbb.ccc"
def test_filename_with_forbidden_chars(self):
# ["/", "\\", ":", "*", "?", '"', "<", ">", "|"]
assert utilities.safeFileName("aaa/bb\\b.c:cc ddd* ? \"www\" <> |"
) == "aaa_bb_b.c_cc ddd_ _ _www_ __ _"
def test_filename_with_forbidden_chars(self):
# ["/", "\\", ":", "*", "?", '"', "<", ">", "|"]
assert utilities.safeFileName("aaa/bb\\b.c:cc ddd* ? \"www\" <> |"
) == "aaa_bb_b.c_cc ddd_ _ _www_ __ _"
def test_filename_with_spaces(self):
assert utilities.safeFileName("aaa bbb.ccc") == "aaa bbb.ccc"
def behaviors_bar_plot(pj, selected_observations, selected_subjects, selected_behaviors, include_modifiers,
interval, start_time, end_time,
plot_directory, output_format):
"""
scatter plot
"""
parameters = [["duration", "Total duration"],
]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
selected_subjects,
selected_observations,
selected_behaviors)
if not ok:
return False, msg, None
cursor = db_connector.cursor()
if include_modifiers:
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
else:
cursor.execute("SELECT distinct behavior FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
behaviors = init_behav_modif(pj[ETHOGRAM],
selected_subjects,
distinct_behav_modif,
include_modifiers,
parameters)
# select time interval
for obs_id in selected_observations:
if len(selected_subjects) > 1:
fig, axs = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True)
else:
fig, ax = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
# time
obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id])
if obs_length == -1:
obs_length = 0
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(min_time, obs_id, min_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(max_time, obs_id, max_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(min_time, max_time, obs_id, min_time, max_time, ))
for subject in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
# skip if behavior defined as POINT
if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]):
continue
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
# total duration
cursor.execute(("SELECT SUM(stop-start) FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"),
(obs_id, subject, behavior, modifiers,))
for row in cursor.fetchall():
behaviors[subject][behavior_modifiers_str]["duration"] = 0 if row[0] is None else row[0]
print("behaviors")
print(behaviors)
print()
print("sorted(distinct_behav_modif)", sorted(distinct_behav_modif))
max_length = 0
behaviors_duration = {}
mb = {}
for ax_idx, subj in enumerate(selected_subjects):
behaviors_duration[subj] = {}
behavior_ticks = []
for behavior_modifiers in sorted(distinct_behav_modif):
behavior, modifiers = behavior_modifiers
# skip if behavior defined as POINT
if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]):
continue
if behavior not in behaviors_duration[subj]:
behaviors_duration[subj][behavior] = [[],[]]
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
#print(subj, behavior, modifiers)
if behavior not in behavior_ticks:
behavior_ticks.append(behavior)
for param in parameters:
behaviors_duration[subj][behavior][0].append(behaviors[subj][behavior_modifiers_str][param[0]])
behaviors_duration[subj][behavior][1].append(modifiers)
max_length = max(max_length, len(behaviors_duration[subj][behavior][1]))
print()
print("behaviors_duration", behaviors_duration)
print()
print("behavior_ticks", behavior_ticks)
print()
#b = {}
behavior_mod_ticks = behavior_ticks[:]
for ax_idx, subj in enumerate(selected_subjects):
print("subject", subj)
md_lgd = []
b = {}
for i in range(max_length):
b[i] = []
for behavior in sorted(behaviors_duration[subj].keys()):
try:
b[i].append(behaviors_duration[subj][behavior][0][i])
if include_modifiers:
'''
if behaviors_duration[subj][behavior][1][i]:
behavior_mod_ticks[behavior_ticks.index(behavior)] = behavior_mod_ticks[behavior_ticks.index(behavior)] + "\n" + \
behaviors_duration[subj][behavior][1][i]
'''
if behaviors_duration[subj][behavior][1][i]:
md_lgd.append(behavior + " " + behaviors_duration[subj][behavior][1][i])
else:
md_lgd.append(behavior)
except:
b[i].append(0)
print()
print("behavior_mod_ticks", behavior_mod_ticks)
print()
print("b")
print(b)
print()
print("md_lgd")
print(md_lgd)
#ind = np.arange(len(behavior_ticks)) # the x locations for the groups
ind = np.arange(len(behavior_ticks))
width = 0.35 # the width of the bars: can also be len(x) sequence
pp = []
max_obs = 0
bottom_ = []
idx_color = 0
for i in sorted(b.keys()):
print(i, b[i])
if i == 0:
pp.append(axs[ax_idx].bar(ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])]))
else:
pp.append(axs[ax_idx].bar(ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])], bottom=bottom_))
idx_color += len(b[i])
if not bottom_:
bottom_ = b[i]
else:
bottom_ = [x + bottom_[idx] for idx,x in enumerate(b[i])]
max_obs = max(max_obs, sum(b[i]))
if ax_idx == 0:
axs[ax_idx].set_ylabel('Duration (s)')
axs[ax_idx].set_xlabel('Behaviors')
axs[ax_idx].set_title('{}'.format(subj))
axs[ax_idx].set_xticks(ind)
axs[ax_idx].set_xticklabels(behavior_mod_ticks,rotation=90)
axs[ax_idx].set_yticks(np.arange(0, max(bottom_), 50))
lgd_col = []
for p in pp:
for r in p:
if r.get_height():
lgd_col.append(r)
plt.legend(lgd_col, md_lgd)
if plot_directory:
output_file_name = str(pathlib.Path(pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix("." + file_format))
plt.savefig(output_file_name)
else:
plt.show()
def create_events_plot(pj,
selected_observations,
parameters,
plot_colors=BEHAVIORS_PLOT_COLORS,
plot_directory="",
file_format="png"):
selected_subjects = parameters["selected subjects"]
selected_behaviors = parameters["selected behaviors"]
include_modifiers = parameters["include modifiers"]
interval = parameters[TIME_INTERVAL]
start_time = parameters[START_TIME]
end_time = parameters[END_TIME]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
selected_subjects,
selected_observations,
selected_behaviors)
if not ok:
return False, msg, None
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])]
par1 = 1
bar_height = 0.5
init = dt.datetime(2017, 1, 1)
for obs_id in selected_observations:
if len(selected_subjects) > 1:
fig, axs = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True)
else:
fig, ax = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj,
selected_subjects,
[obs_id],
selected_behaviors)
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
cursor = db_connector.cursor()
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
if not parameters["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
# time
obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id])
if obs_length == -1: # media length not available
interval = TIME_EVENTS
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(min_time, obs_id, min_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(max_time, obs_id, max_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(min_time, max_time, obs_id, min_time, max_time, ))
ylabels = [" ".join(x) for x in distinct_behav_modif]
for ax_idx, subject in enumerate(selected_subjects):
if parameters["exclude behaviors"]:
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?", (subject, ))
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
if not parameters["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
ylabels = [" ".join(x) for x in distinct_behav_modif]
if not ax_idx:
axs[ax_idx].set_title("Observation {}\n{}".format(obs_id, subject), fontsize=14)
else:
axs[ax_idx].set_title(subject, fontsize=14)
bars = {}
i = 0
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
bars[behavior_modifiers_str] = []
# total duration
cursor.execute(("SELECT start,stop FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"),
(obs_id, subject, behavior, modifiers,))
for row in cursor.fetchall():
bars[behavior_modifiers_str].append((row["start"], row["stop"]))
start_date = matplotlib.dates.date2num(init + dt.timedelta(seconds=row["start"]))
end_date = matplotlib.dates.date2num(
init + dt.timedelta(seconds=row["stop"] + POINT_EVENT_PLOT_DURATION * (row["stop"] == row["start"])))
try:
bar_color = utilities.behavior_color(plot_colors, all_behaviors.index(behavior))
except Exception:
bar_color = "darkgray"
bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row["start"] else bar_color
# sage colors removed from matplotlib colors list
if bar_color in ["sage", "darksage", "lightsage"]:
bar_color = {"darksage": "#598556", "lightsage": "#bcecac", "sage": "#87ae73"}[bar_color]
try:
axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height,
align="center", edgecolor=bar_color, color=bar_color, alpha=1)
except Exception:
axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height,
align="center", edgecolor="darkgray", color="darkgray", alpha=1)
i += 1
axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1)
pos = np.arange(par1, max_len * par1 + par1 + 1, par1)
axs[ax_idx].set_yticks(pos[:len(ylabels)])
axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10})
axs[ax_idx].set_ylabel("Behaviors" + " (modifiers)" * include_modifiers, fontdict={"fontsize": 10})
axs[ax_idx].set_xlim(left=matplotlib.dates.date2num(init + dt.timedelta(seconds=min_time)),
right=matplotlib.dates.date2num(init + dt.timedelta(seconds=max_time + 1)))
axs[ax_idx].grid(color="g", linestyle=":")
axs[ax_idx].xaxis_date()
axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
axs[ax_idx].set_xlabel("Time (HH:MM:SS)", fontdict={"fontsize": 12})
axs[ax_idx].invert_yaxis()
fig.autofmt_xdate()
plt.tight_layout()
if len(selected_observations) > 1:
output_file_name = str(pathlib.Path(pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix(
"." + file_format))
plt.savefig(output_file_name)
else:
plt.show()
def create_subtitles(pj: dict,
selected_observations: list,
parameters: dict,
export_dir: str) -> tuple:
"""
create subtitles for selected observations, subjects and behaviors
Args:
pj (dict): project
selected_observations (list): list of observations
parameters (dict):
export_dir (str): directory to save subtitles
Returns:
bool: True if OK else False
str: error message
"""
def subject_color(subject):
"""
subject color
Args:
subject (str): subject name
Returns:
str: HTML tag for color font (beginning)
str: HTML tag for color font (closing)
"""
if subject == NO_FOCAL_SUBJECT:
return "", ""
else:
return (
"""<font color="{}">""".format(
subtitlesColors[parameters["selected subjects"].index(row["subject"]) % len(subtitlesColors)]
),
"</font>",
)
try:
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, parameters["selected subjects"], selected_observations, parameters["selected behaviors"]
)
if not ok:
return False, msg
cursor = db_connector.cursor()
flag_ok = True
msg = ""
for obsId in selected_observations:
if pj[OBSERVATIONS][obsId][TYPE] in [LIVE]:
out = ""
cursor.execute(
(
"SELECT subject, behavior, start, stop, type, modifiers FROM aggregated_events "
"WHERE observation = ? AND subject in ({}) "
"AND behavior in ({}) "
"ORDER BY start"
).format(",".join(["?"] * len(parameters["selected subjects"])),
",".join(["?"] * len(parameters["selected behaviors"]))),
[obsId] + parameters["selected subjects"] + parameters["selected behaviors"],
)
for idx, row in enumerate(cursor.fetchall()):
col1, col2 = subject_color(row["subject"])
if parameters["include modifiers"]:
modifiers_str = "\n{}".format(row["modifiers"].replace("|", ", "))
else:
modifiers_str = ""
out += ("{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format(
idx=idx + 1,
start=utilities.seconds2time(row["start"]).replace(".", ","),
stop=utilities.seconds2time(row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION).replace(
".", ","
),
col1=col1,
col2=col2,
subject=row["subject"],
behavior=row["behavior"],
modifiers=modifiers_str,
)
file_name = str(pathlib.Path(pathlib.Path(export_dir) / utilities.safeFileName(obsId)).with_suffix(".srt"))
try:
with open(file_name, "w") as f:
f.write(out)
except Exception:
flag_ok = False
msg += "observation: {}\ngave the following error:\n{}\n".format(obsId, str(sys.exc_info()[1]))
elif pj[OBSERVATIONS][obsId][TYPE] in [MEDIA]:
for nplayer in ALL_PLAYERS:
if not pj[OBSERVATIONS][obsId][FILE][nplayer]:
continue
init = 0
for mediaFile in pj[OBSERVATIONS][obsId][FILE][nplayer]:
try:
end = init + pj[OBSERVATIONS][obsId][MEDIA_INFO][LENGTH][mediaFile]
except KeyError:
return False, f"The length for media file {mediaFile} is not available"
out = ""
print(",".join(["?"] * len(parameters["selected subjects"])))
print(",".join(["?"] * len(parameters["selected behaviors"])))
print([obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"])
cursor.execute(
(
"SELECT subject, behavior, type, start, stop, modifiers FROM aggregated_events "
"WHERE observation = ? AND start BETWEEN ? and ? "
"AND subject in ({}) "
"AND behavior in ({}) "
"ORDER BY start"
).format(
",".join(["?"] * len(parameters["selected subjects"])),
",".join(["?"] * len(parameters["selected behaviors"])),
),
[obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"],
)
for idx, row in enumerate(cursor.fetchall()):
col1, col2 = subject_color(row["subject"])
if parameters["include modifiers"]:
modifiers_str = "\n{}".format(row["modifiers"].replace("|", ", "))
else:
modifiers_str = ""
out += ("{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format(
idx=idx + 1,
start=utilities.seconds2time(row["start"] - init).replace(".", ","),
stop=utilities.seconds2time(
(row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION) - init
).replace(".", ","),
col1=col1,
col2=col2,
subject=row["subject"],
behavior=row["behavior"],
modifiers=modifiers_str,
)
file_name = str(pathlib.Path(pathlib.Path(export_dir) / pathlib.Path(mediaFile).name).with_suffix(".srt"))
try:
with open(file_name, "w") as f:
f.write(out)
except Exception:
flag_ok = False
msg += "observation: {}\ngave the following error:\n{}\n".format(obsId, str(sys.exc_info()[1]))
init = end
return flag_ok, msg
except Exception:
return False, str(sys.exc_info()[1])
