def test_no_observation(self):
"""
test with no observation
"""
pj = json.loads(open("files/test.boris").read())
ok, msg, db = db_functions.load_aggregated_events_in_db(pj, [], [], [])
assert ok == False
def test_no_observation(self):
"""
test with no observation
"""
pj = json.loads(open("files/test.boris").read())
ok, msg, db = db_functions.load_aggregated_events_in_db(pj, [], [], [])
assert ok == False
def test_dump(self):
pj = json.loads(open("files/test.boris").read())
ok, msg, db = db_functions.load_aggregated_events_in_db(pj, [], ["observation #1", "observation #2"], [])
out = ""
for line in db.iterdump():
out += line + "\n"
print(out == open("files/test_db_functions_test1").read())
def test_not_ok(self):
"""
test with observation with state events NOT PAIRED
"""
pj = json.loads(open("files/test.boris").read())
ok, msg, db = db_functions.load_aggregated_events_in_db(pj, [], ["live not paired"], [])
assert ok == False
def test_not_ok(self):
"""
test with observation with state events NOT PAIRED
"""
pj = json.loads(open("files/test.boris").read())
ok, msg, db = db_functions.load_aggregated_events_in_db(
pj, [], ["live not paired"], [])
assert ok == False
def test_dump(self):
pj = json.loads(open("files/test.boris").read())
ok, msg, db = db_functions.load_aggregated_events_in_db(
pj, [], ["observation #1", "observation #2"], [])
out = ""
for line in db.iterdump():
out += line + "\n"
print(out == open("files/test_db_functions_test1").read())
def test_cohen_kappa_very_similar_obs_interval_state_3s(self):
pj = json.loads(open("files/test.boris").read())
ethogram = pj[ETHOGRAM]
selected_observations = ["observation #2", "observation #2 (copy)"]
selected_subjects = ["No focal subject", "subject1", "subject2"]
r, s, db = cursor = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, ["s"])
cursor = db.cursor()
K, _ = irr.cohen_kappa(cursor,
obsid1=selected_observations[0],
obsid2=selected_observations[1],
interval=decimal.Decimal("3"),
selected_subjects=selected_subjects,
include_modifiers=False)
assert K == 1
def test_cohen_kappa_same_observation(self):
pj = json.loads(open("files/test.boris").read())
ethogram = pj[ETHOGRAM]
selected_observations = ["observation #1", "observation #1"]
selected_subjects = ["subject1", "subject2"]
r, s, db = cursor = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, ["s", "p"])
assert r == True
cursor = db.cursor()
K, msg = irr.cohen_kappa(cursor,
obsid1=selected_observations[0],
obsid2=selected_observations[1],
interval=decimal.Decimal("1.0"),
selected_subjects=selected_subjects,
include_modifiers=False)
assert K == 1
def test_needleman_wunsch_identity_very_similar_obs_point_3s(self):
pj = json.loads(open("files/test.boris").read())
ethogram = pj[ETHOGRAM]
selected_observations = ["observation #2", "observation #2 (copy)"]
selected_subjects = ["No focal subject", "subject1", "subject2"]
r, s, db = cursor = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, ["p"])
cursor = db.cursor()
identity, msg = irr.needleman_wunsch_identity(
cursor,
obsid1=selected_observations[0],
obsid2=selected_observations[1],
interval=decimal.Decimal("3.0"),
selected_subjects=selected_subjects,
include_modifiers=False)
print(identity)
assert identity == 97.91666666666666
def test_needleman_wunsch_identity_very_different_obs(self):
pj = json.loads(open("files/test.boris").read())
ethogram = pj[ETHOGRAM]
selected_observations = ["observation #1", "observation #2"]
selected_subjects = ["subject1", "subject2"]
r, s, db = cursor = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, ["s"])
cursor = db.cursor()
identity, msg = irr.needleman_wunsch_identity(
cursor,
obsid1=selected_observations[0],
obsid2=selected_observations[1],
interval=decimal.Decimal("1.0"),
selected_subjects=selected_subjects,
include_modifiers=False)
# print(identity)
assert identity == 85.84070796460178
def export_aggregated_events(pj: dict, parameters: dict, obsId: str):
"""
export aggregated events
Args:
pj (dict): BORIS project
parameters (dict): subjects, behaviors
obsId (str): observation id
Returns:
tablib.Dataset:
"""
logging.debug(f"function: export aggregated events {parameters} {obsId}")
interval = parameters["time"]
start_time = parameters[START_TIME]
end_time = parameters[END_TIME]
data = tablib.Dataset()
observation = pj[OBSERVATIONS][obsId]
duration1 = [] # in seconds
if observation[TYPE] in [MEDIA]:
try:
for mediaFile in observation[FILE][PLAYER1]:
if MEDIA_INFO in observation:
duration1.append(observation[MEDIA_INFO]["length"][mediaFile])
except Exception:
duration1 = []
obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obsId])
if obs_length == Decimal("-1"): # media length not available
interval = TIME_EVENTS
logging.debug(f"obs_length: {obs_length}")
ok, msg, connector = db_functions.load_aggregated_events_in_db(pj,
parameters[SELECTED_SUBJECTS],
[obsId],
parameters[SELECTED_BEHAVIORS])
if connector is None:
logging.critical(f"error when loading aggregated events in DB")
return data
# time
cursor = connector.cursor()
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
# adapt start and stop to the selected time interval
cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(min_time, obsId, min_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(max_time, obsId, max_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(min_time, max_time, obsId, min_time, max_time, ))
cursor.execute("DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)",
(obsId, min_time, min_time, max_time, max_time, ))
behavioral_category = project_functions.behavior_category(pj[ETHOGRAM])
for subject in parameters[SELECTED_SUBJECTS]:
for behavior in parameters[SELECTED_BEHAVIORS]:
cursor.execute("SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers",
(subject, behavior,))
rows_distinct_modifiers = list(x[0] for x in cursor.fetchall())
for distinct_modifiers in rows_distinct_modifiers:
cursor.execute(("SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events "
"WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"),
(subject, behavior, distinct_modifiers))
rows = list(cursor.fetchall())
for row in rows:
if observation[TYPE] in [MEDIA]:
if duration1:
mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1])][-1]
mediaFileString = observation[FILE][PLAYER1][mediaFileIdx]
try:
fpsString = observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]]
except Exception:
fpsString = "NA"
else:
try:
if len(observation[FILE][PLAYER1]) == 1:
mediaFileString = observation[FILE][PLAYER1][0]
else:
mediaFileString = "NA"
except Exception:
mediaFileString = "NA"
fpsString = "NA"
if observation[TYPE] in [LIVE]:
mediaFileString = "LIVE"
fpsString = "NA"
if row["type"] == POINT:
row_data = []
row_data.extend([obsId,
observation["date"].replace("T", " "),
mediaFileString,
f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA",
fpsString])
# independent variables
if INDEPENDENT_VARIABLES in pj:
for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]):
if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]:
row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]])
else:
row_data.append("")
row_data.extend([subject,
behavior,
behavioral_category[behavior],
row["modifiers"],
POINT,
"{0:.3f}".format(row["start"]), # start
"{0:.3f}".format(row["stop"]), # stop
"NA", # duration
row["comment"],
""
])
data.append(row_data)
if row["type"] == STATE:
if idx % 2 == 0:
row_data = []
row_data.extend([obsId,
observation["date"].replace("T", " "),
mediaFileString,
f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA",
fpsString])
# independent variables
if INDEPENDENT_VARIABLES in pj:
for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]):
if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]:
row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]])
else:
row_data.append("")
row_data.extend([subject,
behavior,
behavioral_category[behavior],
row["modifiers"],
STATE,
"{0:.3f}".format(row["start"]),
"{0:.3f}".format(row["stop"]),
"{0:.3f}".format(row["stop"] - row["start"]),
row["comment"],
row["comment_stop"]
])
data.append(row_data)
return data
def create_events_plot(pj,
selected_observations,
parameters,
plot_colors=BEHAVIORS_PLOT_COLORS,
plot_directory="",
file_format="png"):
selected_subjects = parameters["selected subjects"]
selected_behaviors = parameters["selected behaviors"]
include_modifiers = parameters["include modifiers"]
interval = parameters[TIME_INTERVAL]
start_time = parameters[START_TIME]
end_time = parameters[END_TIME]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
selected_subjects,
selected_observations,
selected_behaviors)
if not ok:
return False, msg, None
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])]
par1 = 1
bar_height = 0.5
init = dt.datetime(2017, 1, 1)
for obs_id in selected_observations:
if len(selected_subjects) > 1:
fig, axs = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True)
else:
fig, ax = plt.subplots(figsize=(20, 8), nrows=len(selected_subjects), ncols=1, sharex=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj,
selected_subjects,
[obs_id],
selected_behaviors)
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
cursor = db_connector.cursor()
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
if not parameters["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
# time
obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id])
if obs_length == -1: # media length not available
interval = TIME_EVENTS
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(min_time, obs_id, min_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(max_time, obs_id, max_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(min_time, max_time, obs_id, min_time, max_time, ))
ylabels = [" ".join(x) for x in distinct_behav_modif]
for ax_idx, subject in enumerate(selected_subjects):
if parameters["exclude behaviors"]:
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?", (subject, ))
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
if not parameters["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
ylabels = [" ".join(x) for x in distinct_behav_modif]
if not ax_idx:
axs[ax_idx].set_title("Observation {}\n{}".format(obs_id, subject), fontsize=14)
else:
axs[ax_idx].set_title(subject, fontsize=14)
bars = {}
i = 0
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
bars[behavior_modifiers_str] = []
# total duration
cursor.execute(("SELECT start,stop FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"),
(obs_id, subject, behavior, modifiers,))
for row in cursor.fetchall():
bars[behavior_modifiers_str].append((row["start"], row["stop"]))
start_date = matplotlib.dates.date2num(init + dt.timedelta(seconds=row["start"]))
end_date = matplotlib.dates.date2num(
init + dt.timedelta(seconds=row["stop"] + POINT_EVENT_PLOT_DURATION * (row["stop"] == row["start"])))
try:
bar_color = utilities.behavior_color(plot_colors, all_behaviors.index(behavior))
except Exception:
bar_color = "darkgray"
bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row["start"] else bar_color
# sage colors removed from matplotlib colors list
if bar_color in ["sage", "darksage", "lightsage"]:
bar_color = {"darksage": "#598556", "lightsage": "#bcecac", "sage": "#87ae73"}[bar_color]
try:
axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height,
align="center", edgecolor=bar_color, color=bar_color, alpha=1)
except Exception:
axs[ax_idx].barh((i * par1) + par1, end_date - start_date, left=start_date, height=bar_height,
align="center", edgecolor="darkgray", color="darkgray", alpha=1)
i += 1
axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1)
pos = np.arange(par1, max_len * par1 + par1 + 1, par1)
axs[ax_idx].set_yticks(pos[:len(ylabels)])
axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10})
axs[ax_idx].set_ylabel("Behaviors" + " (modifiers)" * include_modifiers, fontdict={"fontsize": 10})
axs[ax_idx].set_xlim(left=matplotlib.dates.date2num(init + dt.timedelta(seconds=min_time)),
right=matplotlib.dates.date2num(init + dt.timedelta(seconds=max_time + 1)))
axs[ax_idx].grid(color="g", linestyle=":")
axs[ax_idx].xaxis_date()
axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
axs[ax_idx].set_xlabel("Time (HH:MM:SS)", fontdict={"fontsize": 12})
axs[ax_idx].invert_yaxis()
fig.autofmt_xdate()
plt.tight_layout()
if len(selected_observations) > 1:
output_file_name = str(pathlib.Path(pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix(
"." + file_format))
plt.savefig(output_file_name)
else:
plt.show()
def behaviors_bar_plot(pj, selected_observations, selected_subjects,
selected_behaviors, include_modifiers, interval,
start_time, end_time, plot_directory, output_format):
"""
scatter plot
"""
parameters = [
["duration", "Total duration"],
]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, selected_behaviors)
if not ok:
return False, msg, None
cursor = db_connector.cursor()
if include_modifiers:
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
else:
cursor.execute("SELECT distinct behavior FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], ""]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects,
distinct_behav_modif, include_modifiers,
parameters)
# select time interval
for obs_id in selected_observations:
if len(selected_subjects) > 1:
fig, axs = plt.subplots(nrows=1,
ncols=len(selected_subjects),
sharey=True)
else:
fig, ax = plt.subplots(nrows=1,
ncols=len(selected_subjects),
sharey=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
# time
obs_length = project_functions.observation_total_length(
pj[OBSERVATIONS][obs_id])
if obs_length == -1:
obs_length = 0
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute(
"UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(
min_time,
obs_id,
min_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(
max_time,
obs_id,
max_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(
min_time,
max_time,
obs_id,
min_time,
max_time,
))
for subject in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
# skip if behavior defined as POINT
if POINT in project_functions.event_type(
behavior, pj[ETHOGRAM]):
continue
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
# total duration
cursor.execute((
"SELECT SUM(stop-start) FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"
), (
obs_id,
subject,
behavior,
modifiers,
))
for row in cursor.fetchall():
behaviors[subject][behavior_modifiers_str][
"duration"] = 0 if row[0] is None else row[0]
print("behaviors")
print(behaviors)
print()
print("sorted(distinct_behav_modif)", sorted(distinct_behav_modif))
max_length = 0
behaviors_duration = {}
mb = {}
for ax_idx, subj in enumerate(selected_subjects):
behaviors_duration[subj] = {}
behavior_ticks = []
for behavior_modifiers in sorted(distinct_behav_modif):
behavior, modifiers = behavior_modifiers
# skip if behavior defined as POINT
if POINT in project_functions.event_type(
behavior, pj[ETHOGRAM]):
continue
if behavior not in behaviors_duration[subj]:
behaviors_duration[subj][behavior] = [[], []]
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
if behavior not in behavior_ticks:
behavior_ticks.append(behavior)
for param in parameters:
behaviors_duration[subj][behavior][0].append(
behaviors[subj][behavior_modifiers_str][param[0]])
behaviors_duration[subj][behavior][1].append(modifiers)
max_length = max(
max_length, len(behaviors_duration[subj][behavior][1]))
print()
print("behaviors_duration", behaviors_duration)
print()
print("behavior_ticks", behavior_ticks)
print()
behavior_mod_ticks = behavior_ticks[:]
for ax_idx, subj in enumerate(selected_subjects):
print("subject", subj)
md_lgd = []
b = {}
for i in range(max_length):
b[i] = []
for behavior in sorted(behaviors_duration[subj].keys()):
try:
b[i].append(behaviors_duration[subj][behavior][0][i])
if include_modifiers:
if behaviors_duration[subj][behavior][1][i]:
md_lgd.append(
behavior + " " +
behaviors_duration[subj][behavior][1][i])
else:
md_lgd.append(behavior)
except Exception:
b[i].append(0)
print()
print("behavior_mod_ticks", behavior_mod_ticks)
print()
print("b")
print(b)
print()
print("md_lgd")
print(md_lgd)
ind = np.arange(len(behavior_ticks))
width = 0.35 # the width of the bars: can also be len(x) sequence
pp = []
max_obs = 0
bottom_ = []
idx_color = 0
for i in sorted(b.keys()):
if i == 0:
pp.append(axs[ax_idx].bar(
ind,
b[i],
width,
color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color +
len(b[i])]))
else:
pp.append(axs[ax_idx].bar(
ind,
b[i],
width,
color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color +
len(b[i])],
bottom=bottom_))
idx_color += len(b[i])
if not bottom_:
bottom_ = b[i]
else:
bottom_ = [x + bottom_[idx] for idx, x in enumerate(b[i])]
max_obs = max(max_obs, sum(b[i]))
if ax_idx == 0:
axs[ax_idx].set_ylabel("Duration (s)")
axs[ax_idx].set_xlabel("Behaviors")
axs[ax_idx].set_title(f"{subj}")
axs[ax_idx].set_xticks(ind)
axs[ax_idx].set_xticklabels(behavior_mod_ticks, rotation=90)
axs[ax_idx].set_yticks(np.arange(0, max(bottom_), 50))
lgd_col = []
for p in pp:
for r in p:
if r.get_height():
lgd_col.append(r)
plt.legend(lgd_col, md_lgd)
if plot_directory:
output_file_name = str(
pathlib.Path(
pathlib.Path(plot_directory) /
utilities.safeFileName(obs_id)).with_suffix("." +
file_format))
plt.savefig(output_file_name)
else:
plt.show()
def export_aggregated_events(pj: dict, parameters: dict, obsId: str):
"""
export aggregated events
Args:
pj (dict): BORIS project
parameters (dict): subjects, behaviors
obsId (str): observation id
Returns:
tablib.Dataset:
"""
logging.debug(f"function: export aggregated events {parameters} {obsId}")
interval = parameters["time"]
start_time = parameters[START_TIME]
end_time = parameters[END_TIME]
data = tablib.Dataset()
observation = pj[OBSERVATIONS][obsId]
duration1 = [] # in seconds
if observation[TYPE] in [MEDIA]:
try:
for mediaFile in observation[FILE][PLAYER1]:
if MEDIA_INFO in observation:
duration1.append(
observation[MEDIA_INFO]["length"][mediaFile])
except Exception:
duration1 = []
obs_length = project_functions.observation_total_length(
pj[OBSERVATIONS][obsId])
if obs_length == Decimal("-1"): # media length not available
interval = TIME_EVENTS
logging.debug(f"obs_length: {obs_length}")
ok, msg, connector = db_functions.load_aggregated_events_in_db(
pj, parameters[SELECTED_SUBJECTS], [obsId],
parameters[SELECTED_BEHAVIORS])
if connector is None:
logging.critical(f"error when loading aggregated events in DB")
return data
# time
cursor = connector.cursor()
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
# adapt start and stop to the selected time interval
cursor.execute(
"UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(
min_time,
obsId,
min_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(
max_time,
obsId,
max_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(
min_time,
max_time,
obsId,
min_time,
max_time,
))
cursor.execute(
"DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)",
(
obsId,
min_time,
min_time,
max_time,
max_time,
))
behavioral_category = project_functions.behavior_category(pj[ETHOGRAM])
for subject in parameters[SELECTED_SUBJECTS]:
for behavior in parameters[SELECTED_BEHAVIORS]:
cursor.execute(
"SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers",
(
subject,
behavior,
))
rows_distinct_modifiers = list(x[0] for x in cursor.fetchall())
for distinct_modifiers in rows_distinct_modifiers:
cursor.execute((
"SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events "
"WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"
), (subject, behavior, distinct_modifiers))
rows = list(cursor.fetchall())
for row in rows:
if observation[TYPE] in [MEDIA]:
if duration1:
mediaFileIdx = [
idx1 for idx1, x in enumerate(duration1)
if row["start"] >= sum(duration1[0:idx1])
][-1]
mediaFileString = observation[FILE][PLAYER1][
mediaFileIdx]
try:
fpsString = observation[MEDIA_INFO]["fps"][
observation[FILE][PLAYER1][mediaFileIdx]]
except Exception:
fpsString = "NA"
else:
try:
if len(observation[FILE][PLAYER1]) == 1:
mediaFileString = observation[FILE][
PLAYER1][0]
else:
mediaFileString = "NA"
except Exception:
mediaFileString = "NA"
fpsString = "NA"
if observation[TYPE] in [LIVE]:
mediaFileString = "LIVE"
fpsString = "NA"
if row["type"] == POINT:
row_data = []
row_data.extend([
obsId, observation["date"].replace("T", " "),
mediaFileString, f"{obs_length:.3f}"
if obs_length != Decimal("-1") else "NA", fpsString
])
# independent variables
if INDEPENDENT_VARIABLES in pj:
for idx_var in utilities.sorted_keys(
pj[INDEPENDENT_VARIABLES]):
if pj[INDEPENDENT_VARIABLES][idx_var][
"label"] in observation[
INDEPENDENT_VARIABLES]:
row_data.append(
observation[INDEPENDENT_VARIABLES][
pj[INDEPENDENT_VARIABLES][idx_var]
["label"]])
else:
row_data.append("")
row_data.extend([
subject,
behavior,
behavioral_category[behavior],
row["modifiers"],
POINT,
f"{row['start']:.3f}", # start
f"{row['stop']:.3f}", # stop
"NA", # duration
row["comment"],
""
])
data.append(row_data)
if row["type"] == STATE:
if idx % 2 == 0:
row_data = []
row_data.extend([
obsId, observation["date"].replace("T", " "),
mediaFileString, f"{obs_length:.3f}"
if obs_length != Decimal("-1") else "NA",
fpsString
])
# independent variables
if INDEPENDENT_VARIABLES in pj:
for idx_var in utilities.sorted_keys(
pj[INDEPENDENT_VARIABLES]):
if pj[INDEPENDENT_VARIABLES][idx_var][
"label"] in observation[
INDEPENDENT_VARIABLES]:
row_data.append(
observation[INDEPENDENT_VARIABLES]
[pj[INDEPENDENT_VARIABLES][idx_var]
["label"]])
else:
row_data.append("")
row_data.extend([
subject, behavior,
behavioral_category[behavior],
row["modifiers"], STATE, f"{row['start']:.3f}",
f"{row['stop']:.3f}",
f"{row['stop'] - row['start']:.3f}",
row["comment"], row["comment_stop"]
])
data.append(row_data)
return data
def sql():
cursor = db_functions.load_aggregated_events_in_db(pj, selected_subjects,
selected_observations,
selected_behaviors)
return cursor
def create_events_plot(pj,
selected_observations,
parameters,
plot_colors=BEHAVIORS_PLOT_COLORS,
plot_directory="",
file_format="png"):
selected_subjects = parameters[SELECTED_SUBJECTS]
selected_behaviors = parameters[SELECTED_BEHAVIORS]
include_modifiers = parameters[INCLUDE_MODIFIERS]
interval = parameters[TIME_INTERVAL]
start_time = parameters[START_TIME]
end_time = parameters[END_TIME]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, selected_behaviors)
if not ok:
return False, msg, None
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
all_behaviors = [
pj[ETHOGRAM][x][BEHAVIOR_CODE]
for x in utilities.sorted_keys(pj[ETHOGRAM])
]
par1 = 1
bar_height = 0.5
init = dt.datetime(2017, 1, 1)
for obs_id in selected_observations:
if len(selected_subjects) > 1:
fig, axs = plt.subplots(figsize=(20, 8),
nrows=len(selected_subjects),
ncols=1,
sharex=True)
else:
fig, ax = plt.subplots(figsize=(20, 8),
nrows=len(selected_subjects),
ncols=1,
sharex=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, [obs_id], selected_behaviors)
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
cursor = db_connector.cursor()
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
if not parameters["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
# time
obs_length = project_functions.observation_total_length(
pj[OBSERVATIONS][obs_id])
if obs_length == -1: # media length not available
interval = TIME_EVENTS
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute(
"UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(
min_time,
obs_id,
min_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(
max_time,
obs_id,
max_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(
min_time,
max_time,
obs_id,
min_time,
max_time,
))
ylabels = [" ".join(x) for x in distinct_behav_modif]
for ax_idx, subject in enumerate(selected_subjects):
if parameters["exclude behaviors"]:
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events WHERE subject = ?",
(subject, ))
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
if not parameters["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif
if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
distinct_behav_modif = sorted(distinct_behav_modif)
max_len = len(distinct_behav_modif)
ylabels = [" ".join(x) for x in distinct_behav_modif]
if not ax_idx:
axs[ax_idx].set_title(f"Observation {obs_id}\n{subject}",
fontsize=14)
else:
axs[ax_idx].set_title(subject, fontsize=14)
bars = {}
i = 0
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
bars[behavior_modifiers_str] = []
# total duration
cursor.execute((
"SELECT start,stop FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"
), (
obs_id,
subject,
behavior,
modifiers,
))
for row in cursor.fetchall():
bars[behavior_modifiers_str].append(
(row["start"], row["stop"]))
start_date = matplotlib.dates.date2num(init + dt.timedelta(
seconds=row["start"]))
end_date = matplotlib.dates.date2num(init + dt.timedelta(
seconds=row["stop"] + POINT_EVENT_PLOT_DURATION *
(row["stop"] == row["start"])))
try:
bar_color = utilities.behavior_color(
plot_colors, all_behaviors.index(behavior))
except Exception:
bar_color = "darkgray"
bar_color = POINT_EVENT_PLOT_COLOR if row["stop"] == row[
"start"] else bar_color
# sage colors removed from matplotlib colors list
if bar_color in ["sage", "darksage", "lightsage"]:
bar_color = {
"darksage": "#598556",
"lightsage": "#bcecac",
"sage": "#87ae73"
}[bar_color]
try:
axs[ax_idx].barh((i * par1) + par1,
end_date - start_date,
left=start_date,
height=bar_height,
align="center",
edgecolor=bar_color,
color=bar_color,
alpha=1)
except Exception:
axs[ax_idx].barh((i * par1) + par1,
end_date - start_date,
left=start_date,
height=bar_height,
align="center",
edgecolor="darkgray",
color="darkgray",
alpha=1)
i += 1
axs[ax_idx].set_ylim(bottom=0, top=(max_len * par1) + par1)
pos = np.arange(par1, max_len * par1 + par1 + 1, par1)
axs[ax_idx].set_yticks(pos[:len(ylabels)])
axs[ax_idx].set_yticklabels(ylabels, fontdict={"fontsize": 10})
axs[ax_idx].set_ylabel("Behaviors" +
" (modifiers)" * include_modifiers,
fontdict={"fontsize": 10})
axs[ax_idx].set_xlim(
left=matplotlib.dates.date2num(init +
dt.timedelta(seconds=min_time)),
right=matplotlib.dates.date2num(init + dt.timedelta(
seconds=max_time + 1)))
axs[ax_idx].grid(color="g", linestyle=":")
axs[ax_idx].xaxis_date()
axs[ax_idx].xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
axs[ax_idx].set_xlabel("Time (HH:MM:SS)",
fontdict={"fontsize": 12})
axs[ax_idx].invert_yaxis()
fig.autofmt_xdate()
plt.tight_layout()
if len(selected_observations) > 1:
output_file_name = str(
pathlib.Path(
pathlib.Path(plot_directory) /
utilities.safeFileName(obs_id)).with_suffix("." +
file_format))
plt.savefig(output_file_name)
else:
plt.show()
def create_behaviors_bar_plot(pj: dict,
selected_observations: list,
param: dict,
plot_directory: str,
output_format: str,
plot_colors:list=BEHAVIORS_PLOT_COLORS):
"""
time budget bar plot
Args:
pj (dict): project
param (dict): parameters
plot_directory (str): path of directory
output_format (str): image format
Returns:
dict:
"""
selected_subjects = param[SELECTED_SUBJECTS]
selected_behaviors = param[SELECTED_BEHAVIORS]
time_interval = param["time"]
start_time = param[START_TIME]
end_time = param[END_TIME]
parameters = ["duration", "number of occurences"]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
selected_subjects,
selected_observations,
selected_behaviors)
if not ok:
return {"error": True, "message": msg}
try:
# extract all behaviors from ethogram for colors in plot
all_behaviors = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in utilities.sorted_keys(pj[ETHOGRAM])]
for obs_id in selected_observations:
cursor = db_connector.cursor()
# distinct behaviors
cursor.execute("SELECT distinct behavior FROM aggregated_events WHERE observation = ?",
(obs_id,))
distinct_behav = [rows["behavior"] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
'''
if not param[EXCLUDE_BEHAVIORS]:
for behavior in selected_behaviors:
if [x for x in distinct_behav if x == behavior] == []:
distinct_behav.append(behavior)
'''
# distinct subjects
cursor.execute("SELECT distinct subject FROM aggregated_events WHERE observation = ?",
(obs_id,))
distinct_subjects = [rows["subject"] for rows in cursor.fetchall()]
behaviors = init_behav(pj[ETHOGRAM],
distinct_subjects,
distinct_behav,
parameters)
# plot creation
if len(distinct_subjects) > 1:
fig, axs = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True)
fig2, axs2 = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True)
else:
fig, ax = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
fig2, ax2 = plt.subplots(nrows=1, ncols=len(distinct_subjects), sharey=True)
axs2 = np.ndarray(shape=(1), dtype=type(ax2))
axs2[0] = ax2
fig.suptitle("Durations of behaviors")
fig2.suptitle("Number of occurences of behaviors")
# if modifiers not to be included set modifiers to ""
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
# time
obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id])
if obs_length == -1:
obs_length = 0
if param["time"] == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if param["time"] == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if param["time"] == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(min_time, obs_id, min_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(max_time, obs_id, max_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(min_time, max_time, obs_id, min_time, max_time, ))
for ax_idx, subject in enumerate(sorted(distinct_subjects)):
for behavior in distinct_behav:
# number of occurences
cursor.execute(("SELECT COUNT(*) AS count FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ?"),
(obs_id, subject, behavior, ))
for row in cursor.fetchall():
behaviors[subject][behavior]["number of occurences"] = 0 if row["count"] is None else row["count"]
# total duration
if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]):
cursor.execute(("SELECT SUM(stop - start) AS duration FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ?"),
(obs_id, subject, behavior, ))
for row in cursor.fetchall():
behaviors[subject][behavior]["duration"] = 0 if row["duration"] is None else row["duration"]
durations, n_occurences, colors, x_labels, colors_duration, x_labels_duration = [], [], [], [], [], []
for behavior in sorted(distinct_behav):
if param[EXCLUDE_BEHAVIORS] and behaviors[subject][behavior]["number of occurences"] == 0:
continue
n_occurences.append(behaviors[subject][behavior]["number of occurences"])
x_labels.append(behavior)
try:
colors.append(utilities.behavior_color(plot_colors, all_behaviors.index(behavior)))
except Exception:
colors.append("darkgray")
if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]):
durations.append(behaviors[subject][behavior]["duration"])
x_labels_duration.append(behavior)
try:
colors_duration.append(utilities.behavior_color(plot_colors, all_behaviors.index(behavior)))
except Exception:
colors_duration.append("darkgray")
#width = 0.35 # the width of the bars: can also be len(x) sequence
axs2[ax_idx].bar(np.arange(len(n_occurences)),
n_occurences,
#width,
color=colors
)
axs[ax_idx].bar(np.arange(len(durations)),
durations,
#width,
color=colors_duration
)
if ax_idx == 0:
axs[ax_idx].set_ylabel("Duration (s)")
axs[ax_idx].set_xlabel("Behaviors")
axs[ax_idx].set_title(f"{subject}")
axs[ax_idx].set_xticks(np.arange(len(durations)))
axs[ax_idx].set_xticklabels(x_labels_duration, rotation='vertical', fontsize=8)
if ax_idx == 0:
axs2[ax_idx].set_ylabel("Number of occurences")
axs2[ax_idx].set_xlabel("Behaviors")
axs2[ax_idx].set_title(f"{subject}")
axs2[ax_idx].set_xticks(np.arange(len(n_occurences)))
axs2[ax_idx].set_xticklabels(x_labels, rotation='vertical', fontsize=8)
fig.align_labels()
fig.tight_layout(rect=[0, 0.03, 1, 0.95])
fig2.align_labels()
fig2.tight_layout(rect=[0, 0.03, 1, 0.95])
if plot_directory:
# output_file_name = f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.{output_format}"
fig.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.duration.{output_format}")
fig2.savefig(f"{pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)}.number_of_occurences.{output_format}")
plt.close()
else:
fig.show()
fig2.show()
return {}
except Exception:
error_type, error_file_name, error_lineno = utilities.error_info(sys.exc_info())
logging.critical(f"Error in time budget bar plot: {error_type} {error_file_name} {error_lineno}")
return {"error": True, "exception": sys.exc_info()}
def create_subtitles(pj: dict,
selected_observations: list,
parameters: dict,
export_dir: str) -> tuple:
"""
create subtitles for selected observations, subjects and behaviors
Args:
pj (dict): project
selected_observations (list): list of observations
parameters (dict):
export_dir (str): directory to save subtitles
Returns:
bool: True if OK else False
str: error message
"""
def subject_color(subject):
"""
subject color
Args:
subject (str): subject name
Returns:
str: HTML tag for color font (beginning)
str: HTML tag for color font (closing)
"""
if subject == NO_FOCAL_SUBJECT:
return "", ""
else:
return (
"""<font color="{}">""".format(
subtitlesColors[parameters["selected subjects"].index(row["subject"]) % len(subtitlesColors)]
),
"</font>",
)
try:
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, parameters["selected subjects"], selected_observations, parameters["selected behaviors"]
)
if not ok:
return False, msg
cursor = db_connector.cursor()
flag_ok = True
msg = ""
for obsId in selected_observations:
if pj[OBSERVATIONS][obsId][TYPE] in [LIVE]:
out = ""
cursor.execute(
(
"SELECT subject, behavior, start, stop, type, modifiers FROM aggregated_events "
"WHERE observation = ? AND subject in ({}) "
"AND behavior in ({}) "
"ORDER BY start"
).format(",".join(["?"] * len(parameters["selected subjects"])),
",".join(["?"] * len(parameters["selected behaviors"]))),
[obsId] + parameters["selected subjects"] + parameters["selected behaviors"],
)
for idx, row in enumerate(cursor.fetchall()):
col1, col2 = subject_color(row["subject"])
if parameters["include modifiers"]:
modifiers_str = "\n{}".format(row["modifiers"].replace("|", ", "))
else:
modifiers_str = ""
out += ("{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format(
idx=idx + 1,
start=utilities.seconds2time(row["start"]).replace(".", ","),
stop=utilities.seconds2time(row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION).replace(
".", ","
),
col1=col1,
col2=col2,
subject=row["subject"],
behavior=row["behavior"],
modifiers=modifiers_str,
)
file_name = str(pathlib.Path(pathlib.Path(export_dir) / utilities.safeFileName(obsId)).with_suffix(".srt"))
try:
with open(file_name, "w") as f:
f.write(out)
except Exception:
flag_ok = False
msg += "observation: {}\ngave the following error:\n{}\n".format(obsId, str(sys.exc_info()[1]))
elif pj[OBSERVATIONS][obsId][TYPE] in [MEDIA]:
for nplayer in ALL_PLAYERS:
if not pj[OBSERVATIONS][obsId][FILE][nplayer]:
continue
init = 0
for mediaFile in pj[OBSERVATIONS][obsId][FILE][nplayer]:
try:
end = init + pj[OBSERVATIONS][obsId][MEDIA_INFO][LENGTH][mediaFile]
except KeyError:
return False, f"The length for media file {mediaFile} is not available"
out = ""
print(",".join(["?"] * len(parameters["selected subjects"])))
print(",".join(["?"] * len(parameters["selected behaviors"])))
print([obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"])
cursor.execute(
(
"SELECT subject, behavior, type, start, stop, modifiers FROM aggregated_events "
"WHERE observation = ? AND start BETWEEN ? and ? "
"AND subject in ({}) "
"AND behavior in ({}) "
"ORDER BY start"
).format(
",".join(["?"] * len(parameters["selected subjects"])),
",".join(["?"] * len(parameters["selected behaviors"])),
),
[obsId, init, end] + parameters["selected subjects"] + parameters["selected behaviors"],
)
for idx, row in enumerate(cursor.fetchall()):
col1, col2 = subject_color(row["subject"])
if parameters["include modifiers"]:
modifiers_str = "\n{}".format(row["modifiers"].replace("|", ", "))
else:
modifiers_str = ""
out += ("{idx}\n" "{start} --> {stop}\n" "{col1}{subject}: {behavior}" "{modifiers}" "{col2}\n\n").format(
idx=idx + 1,
start=utilities.seconds2time(row["start"] - init).replace(".", ","),
stop=utilities.seconds2time(
(row["stop"] if row["type"] == STATE else row["stop"] + POINT_EVENT_ST_DURATION) - init
).replace(".", ","),
col1=col1,
col2=col2,
subject=row["subject"],
behavior=row["behavior"],
modifiers=modifiers_str,
)
file_name = str(pathlib.Path(pathlib.Path(export_dir) / pathlib.Path(mediaFile).name).with_suffix(".srt"))
try:
with open(file_name, "w") as f:
f.write(out)
except Exception:
flag_ok = False
msg += "observation: {}\ngave the following error:\n{}\n".format(obsId, str(sys.exc_info()[1]))
init = end
return flag_ok, msg
except Exception:
return False, str(sys.exc_info()[1])
if "irr" in args.command:
if len(observations_id_list) != 2:
print("select 2 observations")
sys.exit()
behaviors = [
pj[ETHOGRAM][k]["code"]
for k in utilities.sorted_keys(pj[ETHOGRAM])
]
subjects = [
pj[SUBJECTS][k]["name"]
for k in utilities.sorted_keys(pj[SUBJECTS])
] + [NO_FOCAL_SUBJECT]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, subjects, observations_id_list, behaviors)
if not ok:
print(cleanhtml(msg))
sys.exit()
cursor = db_connector.cursor()
interval = 1
if len(args.command) > 1:
interval = utilities.float2decimal(args.command[1])
include_modifiers = True
if len(args.command) > 2:
include_modifiers = "TRUE" in args.command[2].upper()
def behaviors_bar_plot(pj, selected_observations, selected_subjects, selected_behaviors, include_modifiers,
interval, start_time, end_time,
plot_directory, output_format):
"""
scatter plot
"""
parameters = [["duration", "Total duration"],
]
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
selected_subjects,
selected_observations,
selected_behaviors)
if not ok:
return False, msg, None
cursor = db_connector.cursor()
if include_modifiers:
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
else:
cursor.execute("SELECT distinct behavior FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
behaviors = init_behav_modif(pj[ETHOGRAM],
selected_subjects,
distinct_behav_modif,
include_modifiers,
parameters)
# select time interval
for obs_id in selected_observations:
if len(selected_subjects) > 1:
fig, axs = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True)
else:
fig, ax = plt.subplots(nrows=1, ncols=len(selected_subjects), sharey=True)
axs = np.ndarray(shape=(1), dtype=type(ax))
axs[0] = ax
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
# time
obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id])
if obs_length == -1:
obs_length = 0
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(min_time, obs_id, min_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(max_time, obs_id, max_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(min_time, max_time, obs_id, min_time, max_time, ))
for subject in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
# skip if behavior defined as POINT
if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]):
continue
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
# total duration
cursor.execute(("SELECT SUM(stop-start) FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"),
(obs_id, subject, behavior, modifiers,))
for row in cursor.fetchall():
behaviors[subject][behavior_modifiers_str]["duration"] = 0 if row[0] is None else row[0]
print("behaviors")
print(behaviors)
print()
print("sorted(distinct_behav_modif)", sorted(distinct_behav_modif))
max_length = 0
behaviors_duration = {}
mb = {}
for ax_idx, subj in enumerate(selected_subjects):
behaviors_duration[subj] = {}
behavior_ticks = []
for behavior_modifiers in sorted(distinct_behav_modif):
behavior, modifiers = behavior_modifiers
# skip if behavior defined as POINT
if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]):
continue
if behavior not in behaviors_duration[subj]:
behaviors_duration[subj][behavior] = [[],[]]
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
#print(subj, behavior, modifiers)
if behavior not in behavior_ticks:
behavior_ticks.append(behavior)
for param in parameters:
behaviors_duration[subj][behavior][0].append(behaviors[subj][behavior_modifiers_str][param[0]])
behaviors_duration[subj][behavior][1].append(modifiers)
max_length = max(max_length, len(behaviors_duration[subj][behavior][1]))
print()
print("behaviors_duration", behaviors_duration)
print()
print("behavior_ticks", behavior_ticks)
print()
#b = {}
behavior_mod_ticks = behavior_ticks[:]
for ax_idx, subj in enumerate(selected_subjects):
print("subject", subj)
md_lgd = []
b = {}
for i in range(max_length):
b[i] = []
for behavior in sorted(behaviors_duration[subj].keys()):
try:
b[i].append(behaviors_duration[subj][behavior][0][i])
if include_modifiers:
'''
if behaviors_duration[subj][behavior][1][i]:
behavior_mod_ticks[behavior_ticks.index(behavior)] = behavior_mod_ticks[behavior_ticks.index(behavior)] + "\n" + \
behaviors_duration[subj][behavior][1][i]
'''
if behaviors_duration[subj][behavior][1][i]:
md_lgd.append(behavior + " " + behaviors_duration[subj][behavior][1][i])
else:
md_lgd.append(behavior)
except:
b[i].append(0)
print()
print("behavior_mod_ticks", behavior_mod_ticks)
print()
print("b")
print(b)
print()
print("md_lgd")
print(md_lgd)
#ind = np.arange(len(behavior_ticks)) # the x locations for the groups
ind = np.arange(len(behavior_ticks))
width = 0.35 # the width of the bars: can also be len(x) sequence
pp = []
max_obs = 0
bottom_ = []
idx_color = 0
for i in sorted(b.keys()):
print(i, b[i])
if i == 0:
pp.append(axs[ax_idx].bar(ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])]))
else:
pp.append(axs[ax_idx].bar(ind, b[i], width, color=BEHAVIORS_PLOT_COLORS[idx_color:idx_color + len(b[i])], bottom=bottom_))
idx_color += len(b[i])
if not bottom_:
bottom_ = b[i]
else:
bottom_ = [x + bottom_[idx] for idx,x in enumerate(b[i])]
max_obs = max(max_obs, sum(b[i]))
if ax_idx == 0:
axs[ax_idx].set_ylabel('Duration (s)')
axs[ax_idx].set_xlabel('Behaviors')
axs[ax_idx].set_title('{}'.format(subj))
axs[ax_idx].set_xticks(ind)
axs[ax_idx].set_xticklabels(behavior_mod_ticks,rotation=90)
axs[ax_idx].set_yticks(np.arange(0, max(bottom_), 50))
lgd_col = []
for p in pp:
for r in p:
if r.get_height():
lgd_col.append(r)
plt.legend(lgd_col, md_lgd)
if plot_directory:
output_file_name = str(pathlib.Path(pathlib.Path(plot_directory) / utilities.safeFileName(obs_id)).with_suffix("." + file_format))
plt.savefig(output_file_name)
else:
plt.show()
def event_filtering(pj: dict):
"""
advanced event filtering
the python-intervals module is used to do operations on intervals (intersection, union)
"""
result, selected_observations = select_observations.select_observations(
pj, MULTIPLE, "Select observations for advanced event filtering")
if not selected_observations:
return
# check if state events are paired
out = ""
not_paired_obs_list = []
for obs_id in selected_observations:
r, msg = project_functions.check_state_events_obs(
obs_id, pj[ETHOGRAM], pj[OBSERVATIONS][obs_id])
if not r:
out += f"Observation: <strong>{obs_id}</strong><br>{msg}<br>"
not_paired_obs_list.append(obs_id)
if out:
out = f"The observations with UNPAIRED state events will be removed from tha analysis<br><br>{out}"
results = dialog.Results_dialog()
results.setWindowTitle(f"{programName} - Check selected observations")
results.ptText.setReadOnly(True)
results.ptText.appendHtml(out)
results.pbSave.setVisible(False)
results.pbCancel.setVisible(True)
if not results.exec_():
return
selected_observations = [
x for x in selected_observations if x not in not_paired_obs_list
]
if not selected_observations:
return
# observations length
max_obs_length, selectedObsTotalMediaLength = project_functions.observation_length(
pj, selected_observations)
if max_obs_length == -1: # media length not available, user choose to not use events
return
parameters = dialog.choose_obs_subj_behav_category(
pj,
selected_observations,
maxTime=max_obs_length,
flagShowIncludeModifiers=False,
flagShowExcludeBehaviorsWoEvents=False,
by_category=False)
if not parameters[SELECTED_SUBJECTS] or not parameters[SELECTED_BEHAVIORS]:
QMessageBox.warning(None, programName,
"Select subject(s) and behavior(s) to analyze")
return
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, parameters[SELECTED_SUBJECTS], selected_observations,
parameters[SELECTED_BEHAVIORS])
cursor = db_connector.cursor()
# create intervals from DB
cursor.execute(
"SELECT observation, subject, behavior, start, stop FROM aggregated_events"
)
events = {}
for row in cursor.fetchall():
for event in row:
obs, subj, behav, start, stop = row
# check if start and stop are in selected time interval
if stop < parameters[START_TIME]:
continue
if start > parameters[END_TIME]:
continue
if start < parameters[START_TIME]:
start = float(parameters[START_TIME])
if stop > parameters[END_TIME]:
stop = float(parameters[END_TIME])
if obs not in events:
events[obs] = {}
# use function in base at event (state or point)
interval_func = icc if start == stop else ico
if f"{subj}|{behav}" not in events[obs]:
events[obs][f"{subj}|{behav}"] = interval_func([start, stop])
else:
events[obs][f"{subj}|{behav}"] = events[obs][
f"{subj}|{behav}"] | interval_func([start, stop])
w = Advanced_event_filtering_dialog(events)
w.exec_()
def synthetic_time_budget(pj,
selected_observations,
parameters_obs
):
selected_subjects = parameters_obs["selected subjects"]
selected_behaviors = parameters_obs["selected behaviors"]
include_modifiers = parameters_obs["include modifiers"]
interval = parameters_obs["time"]
start_time = parameters_obs["start time"]
end_time = parameters_obs["end time"]
parameters = [["duration", "Total duration"],
["number", "Number of occurrences"],
["duration mean", "Duration mean"],
["duration stdev", "Duration std dev"],
["proportion of time", "Proportion of time"],
]
data_report = tablib.Dataset()
data_report.title = "Synthetic time budget"
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
selected_subjects,
selected_observations,
selected_behaviors)
if not ok:
return False, msg, None
db_connector.create_aggregate("stdev", 1, StdevFunc)
cursor = db_connector.cursor()
# modifiers
if include_modifiers:
cursor.execute("SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]] for rows in cursor.fetchall()]
else:
cursor.execute("SELECT distinct behavior FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], ""] for rows in cursor.fetchall()]
# add selected behaviors that are not observed
#if not parameters_obs["exclude behaviors"]:
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, "-"])
behaviors = init_behav_modif(pj[ETHOGRAM],
selected_subjects,
distinct_behav_modif,
include_modifiers,
parameters)
param_header = ["", "Total length (s)"]
subj_header, behav_header, modif_header= [""]*len(param_header), [""]*len(param_header), [""]*len(param_header)
for subj in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
for param in parameters:
subj_header.append(subj)
behav_header.append(behavior)
modif_header.append(modifiers)
param_header.append(param[1])
'''
if parameters_obs["group observations"]:
cursor.execute("UPDATE aggregated_events SET observation = 'all' " )
#selected_observations = ["all"]
'''
data_report.append(subj_header)
data_report.append(behav_header)
if include_modifiers:
data_report.append(modif_header)
data_report.append(param_header)
# select time interval
for obs_id in selected_observations:
'''if not parameters_obs["group observations"]:'''
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(pj,
selected_subjects,
[obs_id],
selected_behaviors)
if not ok:
return False, msg, None
db_connector.create_aggregate("stdev", 1, StdevFunc)
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
# time
obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obs_id])
if obs_length == -1:
obs_length = 0
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
#duration = end_time - start_time
cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(min_time, obs_id, min_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(max_time, obs_id, max_time, min_time, max_time, ))
cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(min_time, max_time, obs_id, min_time, max_time, ))
for subject in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
cursor.execute(("SELECT SUM(stop-start), COUNT(*), AVG(stop-start), stdev(stop-start) "
"FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ?"),
(obs_id, subject, behavior, modifiers,))
for row in cursor.fetchall():
behaviors[subject][behavior_modifiers_str]["duration"] = 0 if row[0] is None else row[0]
behaviors[subject][behavior_modifiers_str]["number"] = 0 if row[1] is None else row[1]
behaviors[subject][behavior_modifiers_str]["duration mean"] = 0 if row[2] is None else row[2]
behaviors[subject][behavior_modifiers_str]["duration stdev"] = 0 if row[3] is None else row[3]
behaviors[subject][behavior_modifiers_str]["proportion of time"] = 0 if row[0] is None else row[0]/(max_time - min_time)
columns = [obs_id, "{:0.3f}".format(max_time - min_time)]
for subj in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(behavior_modifiers) if modifiers else behavior
for param in parameters:
columns.append(behaviors[subj][behavior_modifiers_str][param[0]])
data_report.append(columns)
return True, msg, data_report
def synthetic_time_budget(pj: dict, selected_observations: list,
parameters_obs: dict):
"""
create a synthetic time budget
Args:
pj (dict): project dictionary
selected_observations (list): list of observations to include in time budget
parameters_obs (dict):
Returns:
bool: True if everything OK
str: message
tablib.Dataset: dataset containing synthetic time budget data
"""
try:
selected_subjects = parameters_obs[SELECTED_SUBJECTS]
selected_behaviors = parameters_obs[SELECTED_BEHAVIORS]
include_modifiers = parameters_obs[INCLUDE_MODIFIERS]
interval = parameters_obs["time"]
start_time = parameters_obs["start time"]
end_time = parameters_obs["end time"]
parameters = [
["duration", "Total duration"],
["number", "Number of occurrences"],
["duration mean", "Duration mean"],
["duration stdev", "Duration std dev"],
["proportion of time", "Proportion of time"],
]
data_report = tablib.Dataset()
data_report.title = "Synthetic time budget"
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, selected_observations, selected_behaviors)
if not ok:
return False, msg, None
db_connector.create_aggregate("stdev", 1, StdevFunc)
cursor = db_connector.cursor()
# modifiers
if include_modifiers:
cursor.execute(
"SELECT distinct behavior, modifiers FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], rows["modifiers"]]
for rows in cursor.fetchall()]
else:
cursor.execute("SELECT distinct behavior FROM aggregated_events")
distinct_behav_modif = [[rows["behavior"], ""]
for rows in cursor.fetchall()]
# add selected behaviors that are not observed
for behav in selected_behaviors:
if [x for x in distinct_behav_modif if x[0] == behav] == []:
distinct_behav_modif.append([behav, ""])
behaviors = init_behav_modif(pj[ETHOGRAM], selected_subjects,
distinct_behav_modif, include_modifiers,
parameters)
param_header = ["", "Total length (s)"]
subj_header, behav_header, modif_header = [""] * len(param_header), [
""
] * len(param_header), [""] * len(param_header)
for subj in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
for param in parameters:
subj_header.append(subj)
behav_header.append(behavior)
modif_header.append(modifiers)
param_header.append(param[1])
'''
if parameters_obs["group observations"]:
cursor.execute("UPDATE aggregated_events SET observation = 'all' " )
#selected_observations = ["all"]
'''
data_report.append(subj_header)
data_report.append(behav_header)
if include_modifiers:
data_report.append(modif_header)
data_report.append(param_header)
# select time interval
for obs_id in selected_observations:
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, selected_subjects, [obs_id], selected_behaviors)
if not ok:
return False, msg, None
db_connector.create_aggregate("stdev", 1, StdevFunc)
cursor = db_connector.cursor()
# if modifiers not to be included set modifiers to ""
if not include_modifiers:
cursor.execute("UPDATE aggregated_events SET modifiers = ''")
# time
obs_length = project_functions.observation_total_length(
pj[OBSERVATIONS][obs_id])
if obs_length == -1:
obs_length = 0
if interval == TIME_FULL_OBS:
min_time = float(0)
max_time = float(obs_length)
if interval == TIME_EVENTS:
try:
min_time = float(pj[OBSERVATIONS][obs_id][EVENTS][0][0])
except Exception:
min_time = float(0)
try:
max_time = float(pj[OBSERVATIONS][obs_id][EVENTS][-1][0])
except Exception:
max_time = float(obs_length)
if interval == TIME_ARBITRARY_INTERVAL:
min_time = float(start_time)
max_time = float(end_time)
# adapt start and stop to the selected time interval
cursor.execute(
"UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
(
min_time,
obs_id,
min_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
(
max_time,
obs_id,
max_time,
min_time,
max_time,
))
cursor.execute(
"UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
(
min_time,
max_time,
obs_id,
min_time,
max_time,
))
cursor.execute(
"DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)",
(
obs_id,
min_time,
min_time,
max_time,
max_time,
))
for subject in selected_subjects:
# check if behaviors are to exclude from total time
time_to_subtract = 0
if EXCLUDED_BEHAVIORS in parameters_obs:
for excluded_behav in parameters_obs[EXCLUDED_BEHAVIORS]:
cursor.execute((
"SELECT SUM(stop-start) "
"FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? "
), (
obs_id,
subject,
excluded_behav,
))
for row in cursor.fetchall():
if row[0] is not None:
time_to_subtract += row[0]
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
cursor.execute((
"SELECT SUM(stop-start), COUNT(*), AVG(stop-start), stdev(stop-start) "
"FROM aggregated_events "
"WHERE observation = ? AND subject = ? AND behavior = ? AND modifiers = ? "
), (
obs_id,
subject,
behavior,
modifiers,
))
for row in cursor.fetchall():
behaviors[subject][behavior_modifiers_str][
"duration"] = (0 if row[0] is None else
f"{row[0]:.3f}")
behaviors[subject][behavior_modifiers_str][
"number"] = 0 if row[1] is None else row[1]
behaviors[subject][behavior_modifiers_str][
"duration mean"] = (0 if row[2] is None else
f"{row[2]:.3f}")
behaviors[subject][behavior_modifiers_str][
"duration stdev"] = (0 if row[3] is None else
f"{row[3]:.3f}")
if behavior not in parameters_obs[EXCLUDED_BEHAVIORS]:
try:
behaviors[subject][behavior_modifiers_str][
"proportion of time"] = (
0 if row[0] is None else
f"{row[0] / ((max_time - min_time) - time_to_subtract):.3f}"
)
except ZeroDivisionError:
behaviors[subject][behavior_modifiers_str][
"proportion of time"] = "-"
else:
# behavior subtracted
behaviors[subject][behavior_modifiers_str][
"proportion of time"] = (
0 if row[0] is None else
f"{row[0] / (max_time - min_time):.3f}")
columns = [obs_id, f"{max_time - min_time:0.3f}"]
for subj in selected_subjects:
for behavior_modifiers in distinct_behav_modif:
behavior, modifiers = behavior_modifiers
behavior_modifiers_str = "|".join(
behavior_modifiers) if modifiers else behavior
for param in parameters:
columns.append(
behaviors[subj][behavior_modifiers_str][param[0]])
data_report.append(columns)
except Exception:
error_type, error_file_name, error_lineno = utilities.error_info(
sys.exc_info())
logging.critical(
f"Error in edit_event function: {error_type} {error_file_name} {error_lineno}"
)
msg = f"Error type: {error_type}\nError file name: {error_file_name}\nError line number: {error_lineno}"
logging.critical(msg)
return (False, msg, tablib.Dataset())
return True, msg, data_report
def create_subtitles(pj: dict, selected_observations: list, parameters: dict,
export_dir: str) -> tuple:
"""
create subtitles for selected observations, subjects and behaviors
Args:
pj (dict): project
selected_observations (list): list of observations
parameters (dict):
export_dir (str): directory to save subtitles
Returns:
bool: True if OK else False
str: error message
"""
def subject_color(subject):
"""
subject color
Args:
subject (str): subject name
Returns:
str: HTML tag for color font (beginning)
str: HTML tag for color font (closing)
"""
if subject == NO_FOCAL_SUBJECT:
return "", ""
else:
return (
"""<font color="{}">""".format(
subtitlesColors[parameters["selected subjects"].index(
row["subject"]) % len(subtitlesColors)]),
"</font>",
)
try:
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, parameters["selected subjects"], selected_observations,
parameters["selected behaviors"])
if not ok:
return False, msg
cursor = db_connector.cursor()
flag_ok = True
msg = ""
for obsId in selected_observations:
if pj[OBSERVATIONS][obsId][TYPE] in [LIVE]:
out = ""
cursor.execute(
("SELECT subject, behavior, start, stop, type, modifiers FROM aggregated_events "
"WHERE observation = ? AND subject in ({}) "
"AND behavior in ({}) "
"ORDER BY start").format(
",".join(["?"] *
len(parameters["selected subjects"])),
",".join(["?"] *
len(parameters["selected behaviors"]))),
[obsId] + parameters["selected subjects"] +
parameters["selected behaviors"],
)
for idx, row in enumerate(cursor.fetchall()):
col1, col2 = subject_color(row["subject"])
if parameters["include modifiers"]:
modifiers_str = "\n{}".format(row["modifiers"].replace(
"|", ", "))
else:
modifiers_str = ""
out += ("{idx}\n"
"{start} --> {stop}\n"
"{col1}{subject}: {behavior}"
"{modifiers}"
"{col2}\n\n").format(
idx=idx + 1,
start=utilities.seconds2time(
row["start"]).replace(".", ","),
stop=utilities.seconds2time(
row["stop"] if row["type"] ==
STATE else row["stop"] +
POINT_EVENT_ST_DURATION).replace(".", ","),
col1=col1,
col2=col2,
subject=row["subject"],
behavior=row["behavior"],
modifiers=modifiers_str,
)
file_name = str(
pathlib.Path(
pathlib.Path(export_dir) /
utilities.safeFileName(obsId)).with_suffix(".srt"))
try:
with open(file_name, "w") as f:
f.write(out)
except Exception:
flag_ok = False
msg += "observation: {}\ngave the following error:\n{}\n".format(
obsId, str(sys.exc_info()[1]))
elif pj[OBSERVATIONS][obsId][TYPE] in [MEDIA]:
for nplayer in ALL_PLAYERS:
if not pj[OBSERVATIONS][obsId][FILE][nplayer]:
continue
init = 0
for mediaFile in pj[OBSERVATIONS][obsId][FILE][nplayer]:
try:
end = init + pj[OBSERVATIONS][obsId][MEDIA_INFO][
LENGTH][mediaFile]
except KeyError:
return False, f"The length for media file {mediaFile} is not available"
out = ""
print(",".join(["?"] *
len(parameters["selected subjects"])))
print(",".join(["?"] *
len(parameters["selected behaviors"])))
print([obsId, init, end] +
parameters["selected subjects"] +
parameters["selected behaviors"])
cursor.execute(
("SELECT subject, behavior, type, start, stop, modifiers FROM aggregated_events "
"WHERE observation = ? AND start BETWEEN ? and ? "
"AND subject in ({}) "
"AND behavior in ({}) "
"ORDER BY start").format(
",".join(
["?"] *
len(parameters["selected subjects"])),
",".join(
["?"] *
len(parameters["selected behaviors"])),
),
[obsId, init, end] +
parameters["selected subjects"] +
parameters["selected behaviors"],
)
for idx, row in enumerate(cursor.fetchall()):
col1, col2 = subject_color(row["subject"])
if parameters["include modifiers"]:
modifiers_str = "\n{}".format(
row["modifiers"].replace("|", ", "))
else:
modifiers_str = ""
out += (
"{idx}\n"
"{start} --> {stop}\n"
"{col1}{subject}: {behavior}"
"{modifiers}"
"{col2}\n\n").format(
idx=idx + 1,
start=utilities.seconds2time(row["start"] -
init).replace(
".", ","),
stop=utilities.seconds2time(
(row["stop"] if row["type"] ==
STATE else row["stop"] +
POINT_EVENT_ST_DURATION) -
init).replace(".", ","),
col1=col1,
col2=col2,
subject=row["subject"],
behavior=row["behavior"],
modifiers=modifiers_str,
)
file_name = str(
pathlib.Path(
pathlib.Path(export_dir) /
pathlib.Path(mediaFile).name).with_suffix(
".srt"))
try:
with open(file_name, "w") as f:
f.write(out)
except Exception:
flag_ok = False
msg += "observation: {}\ngave the following error:\n{}\n".format(
obsId, str(sys.exc_info()[1]))
init = end
return flag_ok, msg
except Exception:
return False, str(sys.exc_info()[1])
def export_aggregated_events(pj, parameters, obsId):
"""
export aggregated events
Args:
pj (dict): BORIS project
parameters (dict): subjects, behaviors
obsId (str): observation id
Returns:
tablib.Dataset:
"""
data = tablib.Dataset()
observation = pj[OBSERVATIONS][obsId]
duration1 = [] # in seconds
if observation[TYPE] in [MEDIA]:
try:
for mediaFile in observation[FILE][PLAYER1]:
if "media_info" in observation:
duration1.append(
observation["media_info"]["length"][mediaFile])
except:
duration1 = []
total_length = "{0:.3f}".format(
project_functions.observation_total_length(observation))
ok, msg, connector = db_functions.load_aggregated_events_in_db(
pj, parameters["selected subjects"], [obsId],
parameters["selected behaviors"])
if not ok:
data
cursor = connector.cursor()
for subject in parameters["selected subjects"]:
for behavior in parameters["selected behaviors"]:
cursor.execute(
"select distinct modifiers from aggregated_events where subject=? AND behavior=? order by modifiers",
(
subject,
behavior,
))
rows_distinct_modifiers = list(x[0].strip()
for x in cursor.fetchall())
for distinct_modifiers in rows_distinct_modifiers:
cursor.execute((
"SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events "
"WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"
), (subject, behavior, distinct_modifiers))
rows = list(cursor.fetchall())
for row in rows:
if observation[TYPE] in [MEDIA]:
if duration1:
mediaFileIdx = [
idx1 for idx1, x in enumerate(duration1)
if row["start"] >= sum(duration1[0:idx1])
][-1]
mediaFileString = observation[FILE][PLAYER1][
mediaFileIdx]
fpsString = observation["media_info"]["fps"][
observation[FILE][PLAYER1][mediaFileIdx]]
else:
mediaFileString = "-"
fpsString = "NA"
if observation[TYPE] in [LIVE]:
mediaFileString = "LIVE"
fpsString = "NA"
#if POINT in project_functions.event_type(behavior, pj[ETHOGRAM]):
if row["type"] == POINT:
row_data = []
row_data.extend([
obsId, observation["date"].replace("T", " "),
mediaFileString, total_length, fpsString
])
# independent variables
if INDEPENDENT_VARIABLES in pj:
for idx_var in utilities.sorted_keys(
pj[INDEPENDENT_VARIABLES]):
if pj[INDEPENDENT_VARIABLES][idx_var][
"label"] in observation[
INDEPENDENT_VARIABLES]:
row_data.append(
observation[INDEPENDENT_VARIABLES][
pj[INDEPENDENT_VARIABLES][idx_var]
["label"]])
else:
row_data.append("")
row_data.extend([
subject,
behavior,
row["modifiers"].strip(),
POINT,
"{0:.3f}".format(row["start"]), # start
"{0:.3f}".format(row["stop"]), # stop
"NA", # duration
row["comment"],
""
])
data.append(row_data)
#if STATE in project_functions.event_type(behavior, pj[ETHOGRAM]):
if row["type"] == STATE:
if idx % 2 == 0:
row_data = []
row_data.extend([
obsId, observation["date"].replace("T", " "),
mediaFileString, total_length, fpsString
])
# independent variables
if INDEPENDENT_VARIABLES in pj:
for idx_var in utilities.sorted_keys(
pj[INDEPENDENT_VARIABLES]):
if pj[INDEPENDENT_VARIABLES][idx_var][
"label"] in observation[
INDEPENDENT_VARIABLES]:
row_data.append(
observation[INDEPENDENT_VARIABLES]
[pj[INDEPENDENT_VARIABLES][idx_var]
["label"]])
else:
row_data.append("")
row_data.extend([
subject, behavior, row["modifiers"].strip(),
STATE, "{0:.3f}".format(row["start"]),
"{0:.3f}".format(row["stop"]),
"{0:.3f}".format(row["stop"] - row["start"]),
row["comment"], row["comment_stop"]
])
data.append(row_data)
return data
