def test_observation_ok(self):
pj = json.loads(open("files/test.boris").read())
results = project_functions.check_state_events_obs(
"offset positif", pj[config.ETHOGRAM],
pj[config.OBSERVATIONS]["offset positif"], config.HHMMSS)
# print(results)
assert results == (True, 'No problem detected')
def test_observation_not_paired(self):
pj = json.loads(open("files/test.boris").read())
results = project_functions.check_state_events_obs(
"live not paired", pj[config.ETHOGRAM],
pj[config.OBSERVATIONS]["live not paired"], config.HHMMSS)
# print(results)
assert results == (
False,
'The behavior <b>s</b> is not PAIRED for subject "<b>No focal subject</b>" at <b>00:00:26.862</b><br>'
)
def select_observations(pj: dict, mode: str, windows_title: str = "") -> tuple:
"""
allow user to select observations
mode: accepted values: OPEN, EDIT, SINGLE, MULTIPLE, SELECT1
Args:
pj (dict): BORIS project dictionary
mode (str): mode for selection: OPEN, EDIT, SINGLE, MULTIPLE, SELECT1
windows_title (str): title for windows
Returns:
str: selected mode: OPEN, EDIT, VIEW
list: list of selected observations
"""
obsListFields = [
"id", "date", "description", "subjects", "observation duration",
"exhaustivity %", "media"
]
indepVarHeader, column_type = [], [
TEXT, TEXT, TEXT, TEXT, NUMERIC, NUMERIC, TEXT
]
if INDEPENDENT_VARIABLES in pj:
for idx in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]):
indepVarHeader.append(pj[INDEPENDENT_VARIABLES][idx]["label"])
column_type.append(pj[INDEPENDENT_VARIABLES][idx]["type"])
data = []
not_paired = []
state_events_list = [
pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in pj[ETHOGRAM]
if STATE in pj[ETHOGRAM][x][TYPE].upper()
]
for obs in sorted(list(pj[OBSERVATIONS].keys())):
date = pj[OBSERVATIONS][obs]["date"].replace("T", " ")
descr = utilities.eol2space(pj[OBSERVATIONS][obs][DESCRIPTION])
# subjects
observedSubjects = [
NO_FOCAL_SUBJECT if x == "" else x
for x in project_functions.extract_observed_subjects(pj, [obs])
]
subjectsList = ", ".join(observedSubjects)
# observed time interval
interval = project_functions.observed_interval(pj[OBSERVATIONS][obs])
observed_interval_str = str(interval[1] - interval[0])
# media
mediaList = []
if pj[OBSERVATIONS][obs][TYPE] in [MEDIA]:
if pj[OBSERVATIONS][obs][FILE]:
for player in sorted(pj[OBSERVATIONS][obs][FILE].keys()):
for media in pj[OBSERVATIONS][obs][FILE][player]:
mediaList.append(f"#{player}: {media}")
if len(mediaList) > 8:
media = " ".join(mediaList)
else:
media = "\n".join(mediaList)
elif pj[OBSERVATIONS][obs][TYPE] in [LIVE]:
media = LIVE
# independent variables
indepvar = []
if INDEPENDENT_VARIABLES in pj[OBSERVATIONS][obs]:
for var_label in indepVarHeader:
if var_label in pj[OBSERVATIONS][obs][INDEPENDENT_VARIABLES]:
indepvar.append(pj[OBSERVATIONS][obs]
[INDEPENDENT_VARIABLES][var_label])
else:
indepvar.append("")
# check unpaired events
ok, _ = project_functions.check_state_events_obs(
obs, pj[ETHOGRAM], pj[OBSERVATIONS][obs], HHMMSS)
if not ok:
not_paired.append(obs)
# check exhaustivity of observation
exhaustivity = project_functions.check_observation_exhaustivity(
pj[OBSERVATIONS][obs][EVENTS], [], state_events_list)
data.append([
obs, date, descr, subjectsList, observed_interval_str,
str(exhaustivity), media
] + indepvar)
obsList = observations_list.observationsList_widget(
data,
header=obsListFields + indepVarHeader,
column_type=column_type,
not_paired=not_paired)
if windows_title:
obsList.setWindowTitle(windows_title)
obsList.pbOpen.setVisible(False)
obsList.pbView.setVisible(False)
obsList.pbEdit.setVisible(False)
obsList.pbOk.setVisible(False)
obsList.pbSelectAll.setVisible(False)
obsList.pbUnSelectAll.setVisible(False)
obsList.mode = mode
if mode == OPEN:
obsList.view.setSelectionMode(QAbstractItemView.SingleSelection)
obsList.pbOpen.setVisible(True)
if mode == VIEW:
obsList.view.setSelectionMode(QAbstractItemView.SingleSelection)
obsList.pbView.setVisible(True)
if mode == EDIT:
obsList.view.setSelectionMode(QAbstractItemView.SingleSelection)
obsList.pbEdit.setVisible(True)
if mode == SINGLE:
obsList.view.setSelectionMode(QAbstractItemView.SingleSelection)
obsList.pbOpen.setVisible(True)
obsList.pbView.setVisible(True)
obsList.pbEdit.setVisible(True)
if mode == MULTIPLE:
obsList.view.setSelectionMode(QAbstractItemView.MultiSelection)
obsList.pbOk.setVisible(True)
obsList.pbSelectAll.setVisible(True)
obsList.pbUnSelectAll.setVisible(True)
if mode == SELECT1:
obsList.view.setSelectionMode(QAbstractItemView.SingleSelection)
obsList.pbOk.setVisible(True)
# restore window geometry
gui_utilities.restore_geometry(obsList, "observations list", (900, 600))
obsList.view.sortItems(0, Qt.AscendingOrder)
for row in range(obsList.view.rowCount()):
obsList.view.resizeRowToContents(row)
selected_observations = []
result = obsList.exec_()
# saving window geometry in ini file
gui_utilities.save_geometry(obsList, "observations list")
if result:
if obsList.view.selectedIndexes():
for idx in obsList.view.selectedIndexes():
if idx.column() == 0: # first column
selected_observations.append(idx.data())
if result == 0: # cancel
resultStr = ""
if result == 1: # select
resultStr = "ok"
if result == 2: # open
resultStr = OPEN
if result == 3: # edit
resultStr = EDIT
if result == 4: # view
resultStr = VIEW
return resultStr, selected_observations
def behavior_binary_table(pj: dict):
"""
ask user for parameters for behavior binary table
call create_behavior_binary_table
"""
result, selected_observations = select_observations.select_observations(
pj, MULTIPLE, "Select observations for the behavior binary table")
if not selected_observations:
return
# check if state events are paired
out = ""
not_paired_obs_list = []
for obs_id in selected_observations:
r, msg = project_functions.check_state_events_obs(
obs_id, pj[ETHOGRAM], pj[OBSERVATIONS][obs_id])
if not r:
out += f"Observation: <strong>{obs_id}</strong><br>{msg}<br>"
not_paired_obs_list.append(obs_id)
if out:
out = f"The observations with UNPAIRED state events will be removed from the analysis<br><br>{out}"
results = dialog.Results_dialog()
results.setWindowTitle(f"{programName} - Check selected observations")
results.ptText.setReadOnly(True)
results.ptText.appendHtml(out)
results.pbSave.setVisible(False)
results.pbCancel.setVisible(True)
if not results.exec_():
return
selected_observations = [
x for x in selected_observations if x not in not_paired_obs_list
]
if not selected_observations:
return
max_obs_length, selectedObsTotalMediaLength = project_functions.observation_length(
pj, selected_observations)
if max_obs_length == -1: # media length not available, user choose to not use events
return
parameters = dialog.choose_obs_subj_behav_category(
pj,
selected_observations,
maxTime=max_obs_length,
flagShowIncludeModifiers=True,
flagShowExcludeBehaviorsWoEvents=True,
by_category=False)
if not parameters[SELECTED_SUBJECTS] or not parameters[SELECTED_BEHAVIORS]:
QMessageBox.warning(None, programName,
"Select subject(s) and behavior(s) to analyze")
return
# ask for time interval
i, ok = QInputDialog.getDouble(None, "Behavior binary table",
"Time interval (in seconds):", 1.0, 0.001,
86400, 3)
if not ok:
return
time_interval = utilities.float2decimal(i)
'''
iw = dialog.Info_widget()
iw.lwi.setVisible(False)
iw.resize(350, 200)
iw.setWindowFlags(Qt.WindowStaysOnTopHint)
iw.setWindowTitle("Behavior binary table")
iw.label.setText("Creating the behavior binary table...")
iw.show()
QApplication.processEvents()
'''
results_df = create_behavior_binary_table(pj, selected_observations,
parameters, time_interval)
'''
iw.hide()
'''
if "error" in results_df:
QMessageBox.warning(None, programName, results_df["msg"])
return
# save results
if len(selected_observations) == 1:
extended_file_formats = [
"Tab Separated Values (*.tsv)", "Comma Separated Values (*.csv)",
"Open Document Spreadsheet ODS (*.ods)",
"Microsoft Excel Spreadsheet XLSX (*.xlsx)",
"Legacy Microsoft Excel Spreadsheet XLS (*.xls)", "HTML (*.html)"
]
file_formats = ["tsv", "csv", "ods", "xlsx", "xls", "html"]
file_name, filter_ = QFileDialog().getSaveFileName(
None, "Save results", "", ";;".join(extended_file_formats))
if not file_name:
return
output_format = file_formats[extended_file_formats.index(filter_)]
if pathlib.Path(file_name).suffix != "." + output_format:
file_name = str(pathlib.Path(file_name)) + "." + output_format
# check if file with new extension already exists
if pathlib.Path(file_name).is_file():
if dialog.MessageDialog(
programName, f"The file {file_name} already exists.",
[CANCEL, OVERWRITE]) == CANCEL:
return
else:
items = ("Tab Separated Values (*.tsv)",
"Comma separated values (*.csv)",
"Open Document Spreadsheet (*.ods)",
"Microsoft Excel Spreadsheet XLSX (*.xlsx)",
"Legacy Microsoft Excel Spreadsheet XLS (*.xls)",
"HTML (*.html)")
item, ok = QInputDialog.getItem(None, "Save results",
"Available formats", items, 0, False)
if not ok:
return
output_format = re.sub(".* \(\*\.", "", item)[:-1]
export_dir = QFileDialog().getExistingDirectory(
None,
"Choose a directory to save results",
os.path.expanduser("~"),
options=QFileDialog.ShowDirsOnly)
if not export_dir:
return
mem_command = ""
for obs_id in results_df:
for subject in results_df[obs_id]:
if len(selected_observations) > 1:
file_name_with_subject = str(
pathlib.Path(export_dir) /
utilities.safeFileName(obs_id + "_" +
subject)) + "." + output_format
else:
file_name_with_subject = str(
os.path.splitext(file_name)[0] +
utilities.safeFileName("_" +
subject)) + "." + output_format
# check if file with new extension already exists
if mem_command != OVERWRITE_ALL and pathlib.Path(
file_name_with_subject).is_file():
if mem_command == "Skip all":
continue
mem_command = dialog.MessageDialog(
programName,
f"The file {file_name_with_subject} already exists.",
[OVERWRITE, OVERWRITE_ALL, "Skip", "Skip all", CANCEL])
if mem_command == CANCEL:
return
if mem_command in ["Skip", "Skip all"]:
continue
try:
if output_format in ["csv", "tsv", "html"]:
with open(file_name_with_subject, "wb") as f:
f.write(
str.encode(results_df[obs_id][subject].export(
output_format)))
if output_format in ["ods", "xlsx", "xls"]:
with open(file_name_with_subject, "wb") as f:
f.write(
results_df[obs_id][subject].export(output_format))
except Exception:
error_type, error_file_name, error_lineno = utilities.error_info(
sys.exc_info())
logging.critical(
f"Error in behavior binary table function: {error_type} {error_file_name} {error_lineno}"
)
QMessageBox.critical(None, programName,
f"Error saving file: {error_type}")
return
def load_aggregated_events_in_db(pj: dict, selectedSubjects: list,
selectedObservations: list,
selectedBehaviors: list):
"""
populate a memory sqlite database with aggregated events from selectedObservations, selectedSubjects and selectedBehaviors
Args:
pj (dict): project dictionary
selectedObservations (list):
selectedSubjects (list):
selectedBehaviors (list):
Returns:
bool: True if OK else False
str: error message
database connector: db connector if bool True else None
"""
logging.debug(f"function: load_aggregated_events_in_db")
# if no observation selected select all
if not selectedObservations:
selectedObservations = sorted([x for x in pj[OBSERVATIONS]])
# if no subject selected select all
if not selectedSubjects:
selectedSubjects = sorted(
[pj[SUBJECTS][x][SUBJECT_NAME]
for x in pj[SUBJECTS]] + [NO_FOCAL_SUBJECT])
# if no behavior selected select all
if not selectedBehaviors:
selectedBehaviors = sorted(
[pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in pj[ETHOGRAM]])
# check if state events are paired
out = ""
for obsId in selectedObservations:
r, msg = project_functions.check_state_events_obs(
obsId, pj[ETHOGRAM], pj[OBSERVATIONS][obsId], HHMMSS)
if not r:
out += f"Observation: <strong>{obsId}</strong><br>{msg}<br>"
if out:
return False, out, None
# selected behaviors defined as state event
state_behaviors_codes = [
pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in pj[ETHOGRAM]
if STATE in pj[ETHOGRAM][x][TYPE].upper()
and pj[ETHOGRAM][x][BEHAVIOR_CODE] in selectedBehaviors
]
# selected behaviors defined as point event
point_behaviors_codes = [
pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in pj[ETHOGRAM]
if POINT in pj[ETHOGRAM][x][TYPE].upper()
and pj[ETHOGRAM][x][BEHAVIOR_CODE] in selectedBehaviors
]
db = sqlite3.connect(":memory:")
#db = sqlite3.connect("/tmp/1.sqlite", isolation_level=None)
db.row_factory = sqlite3.Row
cursor2 = db.cursor()
cursor2.execute(("CREATE TABLE aggregated_events "
"(id INTEGER PRIMARY KEY ASC, "
"observation TEXT, "
"subject TEXT, "
"behavior TEXT, "
"type TEXT, "
"modifiers TEXT, "
"start FLOAT, "
"stop FLOAT, "
"comment TEXT, "
"comment_stop TEXT)"))
cursor2.execute(
"CREATE INDEX observation_idx ON aggregated_events(observation)")
cursor2.execute("CREATE INDEX subject_idx ON aggregated_events(subject)")
cursor2.execute("CREATE INDEX behavior_idx ON aggregated_events(behavior)")
cursor2.execute(
"CREATE INDEX modifiers_idx ON aggregated_events(modifiers)")
# too slow! cursor1 = load_events_in_db(pj, selectedSubjects, selectedObservations, selectedBehaviors)
for obsId in selectedObservations:
cursor1 = load_events_in_db(pj, selectedSubjects, [obsId],
selectedBehaviors)
for subject in selectedSubjects:
for behavior in selectedBehaviors:
cursor1.execute(
"SELECT DISTINCT modifiers FROM events WHERE subject=? AND code=? ORDER BY modifiers",
(
subject,
behavior,
))
rows_distinct_modifiers = list(x[0]
for x in cursor1.fetchall())
for distinct_modifiers in rows_distinct_modifiers:
cursor1.execute((
"SELECT occurence, comment FROM events "
"WHERE subject = ? AND code = ? AND modifiers = ? ORDER by occurence"
), (subject, behavior, distinct_modifiers))
rows = list(cursor1.fetchall())
for idx, row in enumerate(rows):
if behavior in point_behaviors_codes:
cursor2.execute((
"INSERT INTO aggregated_events (observation, subject, behavior, type, modifiers, "
" start, stop, comment, comment_stop) "
"VALUES (?,?,?,?,?,?,?,?,?)"), (
obsId,
subject,
behavior,
POINT,
distinct_modifiers,
row["occurence"],
row["occurence"],
row["comment"],
"",
))
if behavior in state_behaviors_codes:
if idx % 2 == 0:
cursor2.execute((
"INSERT INTO aggregated_events (observation, subject, behavior, type, modifiers,"
" start, stop, comment, comment_stop) "
"VALUES (?,?,?,?,?,?,?,?,?)"),
(obsId, subject, behavior,
STATE, distinct_modifiers,
row["occurence"],
rows[idx + 1]["occurence"],
row["comment"],
rows[idx + 1]["comment"]))
db.commit()
return True, "", db
def load_aggregated_events_in_intervals(pj: dict,
selected_subjects: list,
selected_observations: list,
selected_behaviors: list):
"""
populate a memory sqlite database with aggregated events from selected_observations, selected_subjects and selected_behaviors
Args:
pj (dict): project dictionary
selected_observations (list):
selected_subjects (list):
selected_behaviors (list):
Returns:
bool: True if OK else False
str: error message
database connector: db connector if bool True else None
"""
logging.debug(f"function: load_aggregated_events_in_db")
# if no observation selected select all
if not selected_observations:
selected_observations = sorted([x for x in pj[OBSERVATIONS]])
# if no subject selected select all
if not selected_subjects:
selected_subjects = sorted([pj[SUBJECTS][x][SUBJECT_NAME] for x in pj[SUBJECTS]] + [NO_FOCAL_SUBJECT])
# if no behavior selected select all
if not selected_behaviors:
selected_behaviors = sorted([pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in pj[ETHOGRAM]])
intervals = {}
# check if state events are paired
out = ""
for obsId in selected_observations:
r, msg = project_functions.check_state_events_obs(obsId, pj[ETHOGRAM],
pj[OBSERVATIONS][obsId],
HHMMSS)
if not r:
out += f"Observation: <strong>{obsId}</strong><br>{msg}<br>"
if out:
return False, out, None
# selected behaviors defined as state event
state_behaviors_codes = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in pj[ETHOGRAM]
if STATE in pj[ETHOGRAM][x][TYPE].upper()
and pj[ETHOGRAM][x][BEHAVIOR_CODE] in selected_behaviors]
# selected behaviors defined as point event
point_behaviors_codes = [pj[ETHOGRAM][x][BEHAVIOR_CODE] for x in pj[ETHOGRAM]
if POINT in pj[ETHOGRAM][x][TYPE].upper()
and pj[ETHOGRAM][x][BEHAVIOR_CODE] in selected_behaviors]
db = sqlite3.connect(":memory:")
for obsId in selected_observations:
cursor1 = load_events_in_db(pj, selected_subjects, [obsId], selected_behaviors)
for subject in selected_subjects:
intervals[subject] = {}
for behavior in selected_behaviors:
intervals[subject][behaviors] = []
cursor1.execute("SELECT DISTINCT modifiers FROM events WHERE subject=? AND code=? ORDER BY modifiers",
(subject, behavior, ))
rows_distinct_modifiers = list(x[0] for x in cursor1.fetchall())
for distinct_modifiers in rows_distinct_modifiers:
cursor1.execute(("SELECT occurence, comment FROM events "
"WHERE subject = ? AND code = ? AND modifiers = ? ORDER by occurence"),
(subject, behavior, distinct_modifiers))
rows = list(cursor1.fetchall())
for idx, row in enumerate(rows):
if behavior in point_behaviors_codes:
intervals[subject][behavior].append = P.singleton(row["occurence"])
cursor2.execute(("INSERT INTO aggregated_events (observation, subject, behavior, type, modifiers, "
" start, stop, comment, comment_stop) "
"VALUES (?,?,?,?,?,?,?,?,?)"),
(obsId, subject, behavior, POINT, distinct_modifiers,
row["occurence"], row["occurence"], row["comment"], "",))
if behavior in state_behaviors_codes:
if idx % 2 == 0:
cursor2.execute(("INSERT INTO aggregated_events (observation, subject, behavior, type, modifiers,"
" start, stop, comment, comment_stop) "
"VALUES (?,?,?,?,?,?,?,?,?)"),
(obsId, subject, behavior, STATE, distinct_modifiers,
row["occurence"], rows[idx + 1]["occurence"], row["comment"], rows[idx + 1]["comment"]))
db.commit()
return True, "", db
def event_filtering(pj: dict):
"""
advanced event filtering
the python-intervals module is used to do operations on intervals (intersection, union)
"""
result, selected_observations = select_observations.select_observations(
pj, MULTIPLE, "Select observations for advanced event filtering")
if not selected_observations:
return
# check if state events are paired
out = ""
not_paired_obs_list = []
for obs_id in selected_observations:
r, msg = project_functions.check_state_events_obs(
obs_id, pj[ETHOGRAM], pj[OBSERVATIONS][obs_id])
if not r:
out += f"Observation: <strong>{obs_id}</strong><br>{msg}<br>"
not_paired_obs_list.append(obs_id)
if out:
out = f"The observations with UNPAIRED state events will be removed from tha analysis<br><br>{out}"
results = dialog.Results_dialog()
results.setWindowTitle(f"{programName} - Check selected observations")
results.ptText.setReadOnly(True)
results.ptText.appendHtml(out)
results.pbSave.setVisible(False)
results.pbCancel.setVisible(True)
if not results.exec_():
return
selected_observations = [
x for x in selected_observations if x not in not_paired_obs_list
]
if not selected_observations:
return
# observations length
max_obs_length, selectedObsTotalMediaLength = project_functions.observation_length(
pj, selected_observations)
if max_obs_length == -1: # media length not available, user choose to not use events
return
parameters = dialog.choose_obs_subj_behav_category(
pj,
selected_observations,
maxTime=max_obs_length,
flagShowIncludeModifiers=False,
flagShowExcludeBehaviorsWoEvents=False,
by_category=False)
if not parameters[SELECTED_SUBJECTS] or not parameters[SELECTED_BEHAVIORS]:
QMessageBox.warning(None, programName,
"Select subject(s) and behavior(s) to analyze")
return
ok, msg, db_connector = db_functions.load_aggregated_events_in_db(
pj, parameters[SELECTED_SUBJECTS], selected_observations,
parameters[SELECTED_BEHAVIORS])
cursor = db_connector.cursor()
# create intervals from DB
cursor.execute(
"SELECT observation, subject, behavior, start, stop FROM aggregated_events"
)
events = {}
for row in cursor.fetchall():
for event in row:
obs, subj, behav, start, stop = row
# check if start and stop are in selected time interval
if stop < parameters[START_TIME]:
continue
if start > parameters[END_TIME]:
continue
if start < parameters[START_TIME]:
start = float(parameters[START_TIME])
if stop > parameters[END_TIME]:
stop = float(parameters[END_TIME])
if obs not in events:
events[obs] = {}
# use function in base at event (state or point)
interval_func = icc if start == stop else ico
if f"{subj}|{behav}" not in events[obs]:
events[obs][f"{subj}|{behav}"] = interval_func([start, stop])
else:
events[obs][f"{subj}|{behav}"] = events[obs][
f"{subj}|{behav}"] | interval_func([start, stop])
w = Advanced_event_filtering_dialog(events)
w.exec_()