def test_1(self): pj = json.loads(open("files/test.boris").read()) assert project_functions.behavior_category(pj[ETHOGRAM]) == { 'p': '', 's': '', 'q': '', 'r': '', 'm': '' }
def export_aggregated_events(pj: dict, parameters: dict, obsId: str): """ export aggregated events Args: pj (dict): BORIS project parameters (dict): subjects, behaviors obsId (str): observation id Returns: tablib.Dataset: """ logging.debug(f"function: export aggregated events {parameters} {obsId}") interval = parameters["time"] start_time = parameters[START_TIME] end_time = parameters[END_TIME] data = tablib.Dataset() observation = pj[OBSERVATIONS][obsId] duration1 = [] # in seconds if observation[TYPE] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: if MEDIA_INFO in observation: duration1.append( observation[MEDIA_INFO]["length"][mediaFile]) except Exception: duration1 = [] obs_length = project_functions.observation_total_length( pj[OBSERVATIONS][obsId]) if obs_length == Decimal("-1"): # media length not available interval = TIME_EVENTS logging.debug(f"obs_length: {obs_length}") ok, msg, connector = db_functions.load_aggregated_events_in_db( pj, parameters[SELECTED_SUBJECTS], [obsId], parameters[SELECTED_BEHAVIORS]) if connector is None: logging.critical(f"error when loading aggregated events in DB") return data # time cursor = connector.cursor() if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) # adapt start and stop to the selected time interval cursor.execute( "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", ( min_time, obsId, min_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", ( max_time, obsId, max_time, min_time, max_time, )) cursor.execute( "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", ( min_time, max_time, obsId, min_time, max_time, )) cursor.execute( "DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)", ( obsId, min_time, min_time, max_time, max_time, )) behavioral_category = project_functions.behavior_category(pj[ETHOGRAM]) for subject in parameters[SELECTED_SUBJECTS]: for behavior in parameters[SELECTED_BEHAVIORS]: cursor.execute( "SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers", ( subject, behavior, )) rows_distinct_modifiers = list(x[0] for x in cursor.fetchall()) for distinct_modifiers in rows_distinct_modifiers: cursor.execute(( "SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events " "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start" ), (subject, behavior, distinct_modifiers)) rows = list(cursor.fetchall()) for row in rows: if observation[TYPE] in [MEDIA]: if duration1: mediaFileIdx = [ idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1]) ][-1] mediaFileString = observation[FILE][PLAYER1][ mediaFileIdx] try: fpsString = observation[MEDIA_INFO]["fps"][ observation[FILE][PLAYER1][mediaFileIdx]] except Exception: fpsString = "NA" else: try: if len(observation[FILE][PLAYER1]) == 1: mediaFileString = observation[FILE][ PLAYER1][0] else: mediaFileString = "NA" except Exception: mediaFileString = "NA" fpsString = "NA" if observation[TYPE] in [LIVE]: mediaFileString = "LIVE" fpsString = "NA" if row["type"] == POINT: row_data = [] row_data.extend([ obsId, observation["date"].replace("T", " "), mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString ]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys( pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var][ "label"] in observation[ INDEPENDENT_VARIABLES]: row_data.append( observation[INDEPENDENT_VARIABLES][ pj[INDEPENDENT_VARIABLES][idx_var] ["label"]]) else: row_data.append("") row_data.extend([ subject, behavior, behavioral_category[behavior], row["modifiers"], POINT, f"{row['start']:.3f}", # start f"{row['stop']:.3f}", # stop "NA", # duration row["comment"], "" ]) data.append(row_data) if row["type"] == STATE: if idx % 2 == 0: row_data = [] row_data.extend([ obsId, observation["date"].replace("T", " "), mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString ]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys( pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var][ "label"] in observation[ INDEPENDENT_VARIABLES]: row_data.append( observation[INDEPENDENT_VARIABLES] [pj[INDEPENDENT_VARIABLES][idx_var] ["label"]]) else: row_data.append("") row_data.extend([ subject, behavior, behavioral_category[behavior], row["modifiers"], STATE, f"{row['start']:.3f}", f"{row['stop']:.3f}", f"{row['stop'] - row['start']:.3f}", row["comment"], row["comment_stop"] ]) data.append(row_data) return data
def export_events(parameters, obsId, observation, ethogram, file_name, output_format): """ export events Args: parameters (dict): subjects, behaviors obsId (str): observation id observation (dict): observation ethogram (dict): ethogram of project file_name (str): file name for exporting events output_format (str): output for exporting events Returns: bool: result: True if OK else False str: error message """ total_length = f"{project_functions.observation_total_length(observation):.3f}" eventsWithStatus = project_functions.events_start_stop( ethogram, observation[EVENTS]) # check max number of modifiers max_modifiers = 0 for event in eventsWithStatus: for c in pj_events_fields: if c == "modifier" and event[pj_obs_fields[c]]: max_modifiers = max(max_modifiers, len(event[pj_obs_fields[c]].split("|"))) # media file number mediaNb = 0 if observation["type"] == MEDIA: for player in observation[FILE]: mediaNb += len(observation[FILE][player]) rows = [] # observation id rows.append(["Observation id", obsId]) rows.append([""]) # media file name if observation["type"] in [MEDIA]: rows.append(["Media file(s)"]) elif observation["type"] in [LIVE]: rows.append(["Live observation"]) else: rows.append(["?"]) rows.append([""]) if observation[TYPE] in [MEDIA]: for player in sorted(list(observation[FILE].keys())): for media in observation[FILE][player]: rows.append([f"Player #{player}", media]) rows.append([""]) # date if "date" in observation: rows.append( ["Observation date", observation["date"].replace("T", " ")]) rows.append([""]) # description if "description" in observation: rows.append( ["Description", utilities.eol2space(observation["description"])]) rows.append([""]) # time offset if "time offset" in observation: rows.append(["Time offset (s)", observation["time offset"]]) rows.append([""]) # independent variables if INDEPENDENT_VARIABLES in observation: rows.extend([["independent variables"], ["variable", "value"]]) for variable in observation[INDEPENDENT_VARIABLES]: rows.append( [variable, observation[INDEPENDENT_VARIABLES][variable]]) rows.append([""]) # write table header col = 0 header = ["Time"] header.extend(["Media file path", "Total length", "FPS"]) header.extend(["Subject", "Behavior", "Behavioral category"]) behavioral_category = project_functions.behavior_category(ethogram) for x in range(1, max_modifiers + 1): header.append(f"Modifier {x}") header.extend(["Comment", "Status"]) rows.append(header) duration1 = [] # in seconds if observation["type"] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: duration1.append(observation[MEDIA_INFO]["length"][mediaFile]) except KeyError: pass for event in eventsWithStatus: if (((event[SUBJECT_EVENT_FIELD] in parameters["selected subjects"]) or (event[SUBJECT_EVENT_FIELD] == "" and NO_FOCAL_SUBJECT in parameters["selected subjects"])) and (event[BEHAVIOR_EVENT_FIELD] in parameters["selected behaviors"])): fields = [] fields.append( utilities.intfloatstr(str(event[EVENT_TIME_FIELD_IDX]))) if observation["type"] in [MEDIA]: time_ = event[EVENT_TIME_FIELD_IDX] - observation[TIME_OFFSET] if time_ < 0: time_ = 0 if duration1: mediaFileIdx = [ idx1 for idx1, x in enumerate(duration1) if time_ >= sum(duration1[0:idx1]) ][-1] fields.append(observation[FILE][PLAYER1][mediaFileIdx]) fields.append(total_length) # FPS try: fields.append(observation[MEDIA_INFO]["fps"][ observation[FILE][PLAYER1][mediaFileIdx]]) # fps except KeyError: fields.append("NA") else: fields.append("NA") # media file fields.append("NA") # FPS if observation["type"] in [LIVE]: fields.append(LIVE) # media fields.append(total_length) # total length fields.append("NA") # FPS fields.append(event[EVENT_SUBJECT_FIELD_IDX]) fields.append(event[EVENT_BEHAVIOR_FIELD_IDX]) # behavioral category try: behav_category = behavioral_category[ event[EVENT_BEHAVIOR_FIELD_IDX]] except Exception: behav_category = "" fields.append(behav_category) # modifiers if max_modifiers: modifiers = event[EVENT_MODIFIER_FIELD_IDX].split("|") while len(modifiers) < max_modifiers: modifiers.append("") for m in modifiers: fields.append(m) # comment fields.append(event[EVENT_COMMENT_FIELD_IDX].replace( os.linesep, " ")) # status fields.append(event[-1]) rows.append(fields) maxLen = max([len(r) for r in rows]) data = tablib.Dataset() data.title = utilities.safe_xl_worksheet_title(obsId, output_format) ''' if output_format in ["xls", "xlsx"]: for forbidden_char in EXCEL_FORBIDDEN_CHARACTERS: data.title = data.title.replace(forbidden_char, " ") if output_format in ["xls"]: if len(data.title) > 31: data.title = data.title[0:31] ''' for row in rows: data.append(utilities.complete(row, maxLen)) r, msg = dataset_write(data, file_name, output_format) return r, msg
def export_aggregated_events(pj: dict, parameters: dict, obsId: str): """ export aggregated events Args: pj (dict): BORIS project parameters (dict): subjects, behaviors obsId (str): observation id Returns: tablib.Dataset: """ logging.debug(f"function: export aggregated events {parameters} {obsId}") interval = parameters["time"] start_time = parameters[START_TIME] end_time = parameters[END_TIME] data = tablib.Dataset() observation = pj[OBSERVATIONS][obsId] duration1 = [] # in seconds if observation[TYPE] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: if MEDIA_INFO in observation: duration1.append(observation[MEDIA_INFO]["length"][mediaFile]) except Exception: duration1 = [] obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obsId]) if obs_length == Decimal("-1"): # media length not available interval = TIME_EVENTS logging.debug(f"obs_length: {obs_length}") ok, msg, connector = db_functions.load_aggregated_events_in_db(pj, parameters[SELECTED_SUBJECTS], [obsId], parameters[SELECTED_BEHAVIORS]) if connector is None: logging.critical(f"error when loading aggregated events in DB") return data # time cursor = connector.cursor() if interval == TIME_FULL_OBS: min_time = float(0) max_time = float(obs_length) if interval == TIME_EVENTS: try: min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0]) except Exception: min_time = float(0) try: max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0]) except Exception: max_time = float(obs_length) if interval == TIME_ARBITRARY_INTERVAL: min_time = float(start_time) max_time = float(end_time) # adapt start and stop to the selected time interval cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?", (min_time, obsId, min_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?", (max_time, obsId, max_time, min_time, max_time, )) cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?", (min_time, max_time, obsId, min_time, max_time, )) cursor.execute("DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)", (obsId, min_time, min_time, max_time, max_time, )) behavioral_category = project_functions.behavior_category(pj[ETHOGRAM]) for subject in parameters[SELECTED_SUBJECTS]: for behavior in parameters[SELECTED_BEHAVIORS]: cursor.execute("SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers", (subject, behavior,)) rows_distinct_modifiers = list(x[0] for x in cursor.fetchall()) for distinct_modifiers in rows_distinct_modifiers: cursor.execute(("SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events " "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"), (subject, behavior, distinct_modifiers)) rows = list(cursor.fetchall()) for row in rows: if observation[TYPE] in [MEDIA]: if duration1: mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1])][-1] mediaFileString = observation[FILE][PLAYER1][mediaFileIdx] try: fpsString = observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]] except Exception: fpsString = "NA" else: try: if len(observation[FILE][PLAYER1]) == 1: mediaFileString = observation[FILE][PLAYER1][0] else: mediaFileString = "NA" except Exception: mediaFileString = "NA" fpsString = "NA" if observation[TYPE] in [LIVE]: mediaFileString = "LIVE" fpsString = "NA" if row["type"] == POINT: row_data = [] row_data.extend([obsId, observation["date"].replace("T", " "), mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]: row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]]) else: row_data.append("") row_data.extend([subject, behavior, behavioral_category[behavior], row["modifiers"], POINT, "{0:.3f}".format(row["start"]), # start "{0:.3f}".format(row["stop"]), # stop "NA", # duration row["comment"], "" ]) data.append(row_data) if row["type"] == STATE: if idx % 2 == 0: row_data = [] row_data.extend([obsId, observation["date"].replace("T", " "), mediaFileString, f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA", fpsString]) # independent variables if INDEPENDENT_VARIABLES in pj: for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]): if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]: row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]]) else: row_data.append("") row_data.extend([subject, behavior, behavioral_category[behavior], row["modifiers"], STATE, "{0:.3f}".format(row["start"]), "{0:.3f}".format(row["stop"]), "{0:.3f}".format(row["stop"] - row["start"]), row["comment"], row["comment_stop"] ]) data.append(row_data) return data
def export_events(parameters, obsId, observation, ethogram, file_name, output_format): """ export events Args: parameters (dict): subjects, behaviors obsId (str): observation id observation (dict): observation ethogram (dict): ethogram of project file_name (str): file name for exporting events output_format (str): output for exporting events Returns: bool: result: True if OK else False str: error message """ total_length = "{0:.3f}".format(project_functions.observation_total_length(observation)) eventsWithStatus = project_functions.events_start_stop(ethogram, observation[EVENTS]) # check max number of modifiers max_modifiers = 0 for event in eventsWithStatus: for c in pj_events_fields: if c == "modifier" and event[pj_obs_fields[c]]: max_modifiers = max(max_modifiers, len(event[pj_obs_fields[c]].split("|"))) # media file number mediaNb = 0 if observation["type"] == MEDIA: for player in observation[FILE]: mediaNb += len(observation[FILE][player]) rows = [] # observation id rows.append(["Observation id", obsId]) rows.append([""]) # media file name if observation["type"] in [MEDIA]: rows.append(["Media file(s)"]) elif observation["type"] in [LIVE]: rows.append(["Live observation"]) else: rows.append(["?"]) rows.append([""]) if observation[TYPE] in [MEDIA]: for player in sorted(list(observation[FILE].keys())): for media in observation[FILE][player]: rows.append(["Player #{0}".format(player), media]) rows.append([""]) # date if "date" in observation: rows.append(["Observation date", observation["date"].replace("T", " ")]) rows.append([""]) # description if "description" in observation: rows.append(["Description", utilities.eol2space(observation["description"])]) rows.append([""]) # time offset if "time offset" in observation: rows.append(["Time offset (s)", observation["time offset"]]) rows.append([""]) # independent variables if INDEPENDENT_VARIABLES in observation: rows.extend([["independent variables"], ["variable", "value"]]) for variable in observation[INDEPENDENT_VARIABLES]: rows.append([variable, observation[INDEPENDENT_VARIABLES][variable]]) rows.append([""]) # write table header col = 0 header = ["Time"] header.extend(["Media file path", "Total length", "FPS"]) header.extend(["Subject", "Behavior", "Behavioral category"]) behavioral_category = project_functions.behavior_category(ethogram) for x in range(1, max_modifiers + 1): header.append("Modifier {}".format(x)) header.extend(["Comment", "Status"]) rows.append(header) duration1 = [] # in seconds if observation["type"] in [MEDIA]: try: for mediaFile in observation[FILE][PLAYER1]: duration1.append(observation[MEDIA_INFO]["length"][mediaFile]) except KeyError: pass for event in eventsWithStatus: if (((event[SUBJECT_EVENT_FIELD] in parameters["selected subjects"]) or (event[SUBJECT_EVENT_FIELD] == "" and NO_FOCAL_SUBJECT in parameters["selected subjects"])) and (event[BEHAVIOR_EVENT_FIELD] in parameters["selected behaviors"])): fields = [] fields.append(utilities.intfloatstr(str(event[EVENT_TIME_FIELD_IDX]))) if observation["type"] in [MEDIA]: time_ = event[EVENT_TIME_FIELD_IDX] - observation[TIME_OFFSET] if time_ < 0: time_ = 0 if duration1: mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if time_ >= sum(duration1[0:idx1])][-1] fields.append(observation[FILE][PLAYER1][mediaFileIdx]) fields.append(total_length) # FPS try: fields.append(observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]]) # fps except KeyError: fields.append("NA") else: fields.append("NA") # media file fields.append("NA") # FPS if observation["type"] in [LIVE]: fields.append(LIVE) # media fields.append(total_length) # total length fields.append("NA") # FPS fields.append(event[EVENT_SUBJECT_FIELD_IDX]) fields.append(event[EVENT_BEHAVIOR_FIELD_IDX]) # behavioral category try: behav_category = behavioral_category[event[EVENT_BEHAVIOR_FIELD_IDX]] except Exception: behav_category = "" fields.append(behav_category) # modifiers if max_modifiers: modifiers = event[EVENT_MODIFIER_FIELD_IDX].split("|") while len(modifiers) < max_modifiers: modifiers.append("") for m in modifiers: fields.append(m) # comment fields.append(event[EVENT_COMMENT_FIELD_IDX].replace(os.linesep, " ")) # status fields.append(event[-1]) rows.append(fields) maxLen = max([len(r) for r in rows]) data = tablib.Dataset() data.title = obsId # check if worksheet name will be > 31 char if output_format in ["xls", "xlsx"]: for forbidden_char in EXCEL_FORBIDDEN_CHARACTERS: data.title = data.title.replace(forbidden_char, " ") if output_format in ["xls"]: if len(data.title) > 31: data.title = data.title[0:31] for row in rows: data.append(utilities.complete(row, maxLen)) r, msg = dataset_write(data, file_name, output_format) return r, msg