def test_1(self):
     pj = json.loads(open("files/test.boris").read())
     assert project_functions.behavior_category(pj[ETHOGRAM]) == {
         'p': '',
         's': '',
         'q': '',
         'r': '',
         'm': ''
     }
Example #2
0
def export_aggregated_events(pj: dict, parameters: dict, obsId: str):
    """
    export aggregated events

    Args:
        pj (dict): BORIS project
        parameters (dict): subjects, behaviors
        obsId (str): observation id

    Returns:
        tablib.Dataset:

    """
    logging.debug(f"function: export aggregated events {parameters} {obsId}")

    interval = parameters["time"]
    start_time = parameters[START_TIME]
    end_time = parameters[END_TIME]

    data = tablib.Dataset()
    observation = pj[OBSERVATIONS][obsId]

    duration1 = []  # in seconds
    if observation[TYPE] in [MEDIA]:
        try:
            for mediaFile in observation[FILE][PLAYER1]:
                if MEDIA_INFO in observation:
                    duration1.append(
                        observation[MEDIA_INFO]["length"][mediaFile])
        except Exception:
            duration1 = []

    obs_length = project_functions.observation_total_length(
        pj[OBSERVATIONS][obsId])
    if obs_length == Decimal("-1"):  # media length not available
        interval = TIME_EVENTS

    logging.debug(f"obs_length: {obs_length}")

    ok, msg, connector = db_functions.load_aggregated_events_in_db(
        pj, parameters[SELECTED_SUBJECTS], [obsId],
        parameters[SELECTED_BEHAVIORS])
    if connector is None:
        logging.critical(f"error when loading aggregated events in DB")
        return data

    # time
    cursor = connector.cursor()

    if interval == TIME_FULL_OBS:
        min_time = float(0)
        max_time = float(obs_length)

    if interval == TIME_EVENTS:
        try:
            min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0])
        except Exception:
            min_time = float(0)
        try:
            max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0])
        except Exception:
            max_time = float(obs_length)

    if interval == TIME_ARBITRARY_INTERVAL:
        min_time = float(start_time)
        max_time = float(end_time)

    # adapt start and stop to the selected time interval
    cursor.execute(
        "UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
        (
            min_time,
            obsId,
            min_time,
            min_time,
            max_time,
        ))
    cursor.execute(
        "UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
        (
            max_time,
            obsId,
            max_time,
            min_time,
            max_time,
        ))

    cursor.execute(
        "UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
        (
            min_time,
            max_time,
            obsId,
            min_time,
            max_time,
        ))

    cursor.execute(
        "DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)",
        (
            obsId,
            min_time,
            min_time,
            max_time,
            max_time,
        ))

    behavioral_category = project_functions.behavior_category(pj[ETHOGRAM])

    for subject in parameters[SELECTED_SUBJECTS]:

        for behavior in parameters[SELECTED_BEHAVIORS]:

            cursor.execute(
                "SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers",
                (
                    subject,
                    behavior,
                ))

            rows_distinct_modifiers = list(x[0] for x in cursor.fetchall())

            for distinct_modifiers in rows_distinct_modifiers:

                cursor.execute((
                    "SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events "
                    "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"
                ), (subject, behavior, distinct_modifiers))
                rows = list(cursor.fetchall())

                for row in rows:

                    if observation[TYPE] in [MEDIA]:
                        if duration1:
                            mediaFileIdx = [
                                idx1 for idx1, x in enumerate(duration1)
                                if row["start"] >= sum(duration1[0:idx1])
                            ][-1]
                            mediaFileString = observation[FILE][PLAYER1][
                                mediaFileIdx]
                            try:
                                fpsString = observation[MEDIA_INFO]["fps"][
                                    observation[FILE][PLAYER1][mediaFileIdx]]
                            except Exception:
                                fpsString = "NA"
                        else:
                            try:
                                if len(observation[FILE][PLAYER1]) == 1:
                                    mediaFileString = observation[FILE][
                                        PLAYER1][0]
                                else:
                                    mediaFileString = "NA"
                            except Exception:
                                mediaFileString = "NA"
                            fpsString = "NA"

                    if observation[TYPE] in [LIVE]:
                        mediaFileString = "LIVE"
                        fpsString = "NA"

                    if row["type"] == POINT:

                        row_data = []
                        row_data.extend([
                            obsId, observation["date"].replace("T", " "),
                            mediaFileString, f"{obs_length:.3f}"
                            if obs_length != Decimal("-1") else "NA", fpsString
                        ])

                        # independent variables
                        if INDEPENDENT_VARIABLES in pj:
                            for idx_var in utilities.sorted_keys(
                                    pj[INDEPENDENT_VARIABLES]):
                                if pj[INDEPENDENT_VARIABLES][idx_var][
                                        "label"] in observation[
                                            INDEPENDENT_VARIABLES]:
                                    row_data.append(
                                        observation[INDEPENDENT_VARIABLES][
                                            pj[INDEPENDENT_VARIABLES][idx_var]
                                            ["label"]])
                                else:
                                    row_data.append("")

                        row_data.extend([
                            subject,
                            behavior,
                            behavioral_category[behavior],
                            row["modifiers"],
                            POINT,
                            f"{row['start']:.3f}",  # start
                            f"{row['stop']:.3f}",  # stop
                            "NA",  # duration
                            row["comment"],
                            ""
                        ])
                        data.append(row_data)

                    if row["type"] == STATE:
                        if idx % 2 == 0:
                            row_data = []
                            row_data.extend([
                                obsId, observation["date"].replace("T", " "),
                                mediaFileString, f"{obs_length:.3f}"
                                if obs_length != Decimal("-1") else "NA",
                                fpsString
                            ])

                            # independent variables
                            if INDEPENDENT_VARIABLES in pj:
                                for idx_var in utilities.sorted_keys(
                                        pj[INDEPENDENT_VARIABLES]):
                                    if pj[INDEPENDENT_VARIABLES][idx_var][
                                            "label"] in observation[
                                                INDEPENDENT_VARIABLES]:
                                        row_data.append(
                                            observation[INDEPENDENT_VARIABLES]
                                            [pj[INDEPENDENT_VARIABLES][idx_var]
                                             ["label"]])
                                    else:
                                        row_data.append("")

                            row_data.extend([
                                subject, behavior,
                                behavioral_category[behavior],
                                row["modifiers"], STATE, f"{row['start']:.3f}",
                                f"{row['stop']:.3f}",
                                f"{row['stop'] - row['start']:.3f}",
                                row["comment"], row["comment_stop"]
                            ])
                            data.append(row_data)

    return data
Example #3
0
def export_events(parameters, obsId, observation, ethogram, file_name,
                  output_format):
    """
    export events

    Args:
        parameters (dict): subjects, behaviors
        obsId (str): observation id
        observation (dict): observation
        ethogram (dict): ethogram of project
        file_name (str): file name for exporting events
        output_format (str): output for exporting events

    Returns:
        bool: result: True if OK else False
        str: error message
    """

    total_length = f"{project_functions.observation_total_length(observation):.3f}"

    eventsWithStatus = project_functions.events_start_stop(
        ethogram, observation[EVENTS])

    # check max number of modifiers
    max_modifiers = 0
    for event in eventsWithStatus:
        for c in pj_events_fields:
            if c == "modifier" and event[pj_obs_fields[c]]:
                max_modifiers = max(max_modifiers,
                                    len(event[pj_obs_fields[c]].split("|")))

    # media file number
    mediaNb = 0
    if observation["type"] == MEDIA:
        for player in observation[FILE]:
            mediaNb += len(observation[FILE][player])

    rows = []

    # observation id
    rows.append(["Observation id", obsId])
    rows.append([""])

    # media file name
    if observation["type"] in [MEDIA]:
        rows.append(["Media file(s)"])
    elif observation["type"] in [LIVE]:
        rows.append(["Live observation"])
    else:
        rows.append(["?"])
    rows.append([""])

    if observation[TYPE] in [MEDIA]:
        for player in sorted(list(observation[FILE].keys())):
            for media in observation[FILE][player]:
                rows.append([f"Player #{player}", media])
    rows.append([""])

    # date
    if "date" in observation:
        rows.append(
            ["Observation date", observation["date"].replace("T", " ")])
    rows.append([""])

    # description
    if "description" in observation:
        rows.append(
            ["Description",
             utilities.eol2space(observation["description"])])
    rows.append([""])

    # time offset
    if "time offset" in observation:
        rows.append(["Time offset (s)", observation["time offset"]])
    rows.append([""])

    # independent variables
    if INDEPENDENT_VARIABLES in observation:
        rows.extend([["independent variables"], ["variable", "value"]])

        for variable in observation[INDEPENDENT_VARIABLES]:
            rows.append(
                [variable, observation[INDEPENDENT_VARIABLES][variable]])

    rows.append([""])

    # write table header
    col = 0
    header = ["Time"]
    header.extend(["Media file path", "Total length", "FPS"])

    header.extend(["Subject", "Behavior", "Behavioral category"])

    behavioral_category = project_functions.behavior_category(ethogram)

    for x in range(1, max_modifiers + 1):
        header.append(f"Modifier {x}")
    header.extend(["Comment", "Status"])

    rows.append(header)

    duration1 = []  # in seconds
    if observation["type"] in [MEDIA]:
        try:
            for mediaFile in observation[FILE][PLAYER1]:
                duration1.append(observation[MEDIA_INFO]["length"][mediaFile])
        except KeyError:
            pass

    for event in eventsWithStatus:
        if (((event[SUBJECT_EVENT_FIELD] in parameters["selected subjects"]) or
             (event[SUBJECT_EVENT_FIELD] == ""
              and NO_FOCAL_SUBJECT in parameters["selected subjects"])) and
            (event[BEHAVIOR_EVENT_FIELD] in parameters["selected behaviors"])):

            fields = []
            fields.append(
                utilities.intfloatstr(str(event[EVENT_TIME_FIELD_IDX])))

            if observation["type"] in [MEDIA]:

                time_ = event[EVENT_TIME_FIELD_IDX] - observation[TIME_OFFSET]
                if time_ < 0:
                    time_ = 0

                if duration1:
                    mediaFileIdx = [
                        idx1 for idx1, x in enumerate(duration1)
                        if time_ >= sum(duration1[0:idx1])
                    ][-1]
                    fields.append(observation[FILE][PLAYER1][mediaFileIdx])
                    fields.append(total_length)
                    # FPS
                    try:
                        fields.append(observation[MEDIA_INFO]["fps"][
                            observation[FILE][PLAYER1][mediaFileIdx]])  # fps
                    except KeyError:
                        fields.append("NA")
                else:
                    fields.append("NA")  # media file
                    fields.append("NA")  # FPS

            if observation["type"] in [LIVE]:
                fields.append(LIVE)  # media
                fields.append(total_length)  # total length
                fields.append("NA")  # FPS

            fields.append(event[EVENT_SUBJECT_FIELD_IDX])
            fields.append(event[EVENT_BEHAVIOR_FIELD_IDX])

            # behavioral category

            try:
                behav_category = behavioral_category[
                    event[EVENT_BEHAVIOR_FIELD_IDX]]
            except Exception:
                behav_category = ""
            fields.append(behav_category)

            # modifiers
            if max_modifiers:
                modifiers = event[EVENT_MODIFIER_FIELD_IDX].split("|")
                while len(modifiers) < max_modifiers:
                    modifiers.append("")

                for m in modifiers:
                    fields.append(m)

            # comment
            fields.append(event[EVENT_COMMENT_FIELD_IDX].replace(
                os.linesep, " "))
            # status
            fields.append(event[-1])

            rows.append(fields)

    maxLen = max([len(r) for r in rows])
    data = tablib.Dataset()

    data.title = utilities.safe_xl_worksheet_title(obsId, output_format)
    '''
    if output_format in ["xls", "xlsx"]:
        for forbidden_char in EXCEL_FORBIDDEN_CHARACTERS:
            data.title = data.title.replace(forbidden_char, " ")
        if output_format in ["xls"]:
            if len(data.title) > 31:
                data.title = data.title[0:31]
    '''

    for row in rows:
        data.append(utilities.complete(row, maxLen))

    r, msg = dataset_write(data, file_name, output_format)

    return r, msg
def export_aggregated_events(pj: dict, parameters: dict, obsId: str):
    """
    export aggregated events

    Args:
        pj (dict): BORIS project
        parameters (dict): subjects, behaviors
        obsId (str): observation id

    Returns:
        tablib.Dataset:

    """
    logging.debug(f"function: export aggregated events {parameters} {obsId}")

    interval = parameters["time"]
    start_time = parameters[START_TIME]
    end_time = parameters[END_TIME]

    data = tablib.Dataset()
    observation = pj[OBSERVATIONS][obsId]


    duration1 = []   # in seconds
    if observation[TYPE] in [MEDIA]:
        try:
            for mediaFile in observation[FILE][PLAYER1]:
                if MEDIA_INFO in observation:
                    duration1.append(observation[MEDIA_INFO]["length"][mediaFile])
        except Exception:
            duration1 = []


    obs_length = project_functions.observation_total_length(pj[OBSERVATIONS][obsId])
    if obs_length == Decimal("-1"): # media length not available
        interval = TIME_EVENTS

    logging.debug(f"obs_length: {obs_length}")

    ok, msg, connector = db_functions.load_aggregated_events_in_db(pj,
                                                                   parameters[SELECTED_SUBJECTS],
                                                                   [obsId],
                                                                   parameters[SELECTED_BEHAVIORS])
    if connector is None:
        logging.critical(f"error when loading aggregated events in DB")
        return data

    # time
    cursor = connector.cursor()

    if interval == TIME_FULL_OBS:
        min_time = float(0)
        max_time = float(obs_length)

    if interval == TIME_EVENTS:
        try:
            min_time = float(pj[OBSERVATIONS][obsId][EVENTS][0][0])
        except Exception:
            min_time = float(0)
        try:
            max_time = float(pj[OBSERVATIONS][obsId][EVENTS][-1][0])
        except Exception:
            max_time = float(obs_length)

    if interval == TIME_ARBITRARY_INTERVAL:
        min_time = float(start_time)
        max_time = float(end_time)

    # adapt start and stop to the selected time interval
    cursor.execute("UPDATE aggregated_events SET start = ? WHERE observation = ? AND start < ? AND stop BETWEEN ? AND ?",
                   (min_time, obsId, min_time, min_time, max_time, ))
    cursor.execute("UPDATE aggregated_events SET stop = ? WHERE observation = ? AND stop > ? AND start BETWEEN ? AND ?",
                   (max_time, obsId, max_time, min_time, max_time, ))

    cursor.execute("UPDATE aggregated_events SET start = ?, stop = ? WHERE observation = ? AND start < ? AND stop > ?",
                   (min_time, max_time, obsId, min_time, max_time, ))

    cursor.execute("DELETE FROM aggregated_events WHERE observation = ? AND (start < ? AND stop < ?) OR (start > ? AND stop > ?)",
                   (obsId, min_time, min_time, max_time, max_time, ))


    behavioral_category = project_functions.behavior_category(pj[ETHOGRAM])

    for subject in parameters[SELECTED_SUBJECTS]:

        for behavior in parameters[SELECTED_BEHAVIORS]:

            cursor.execute("SELECT distinct modifiers FROM aggregated_events where subject=? AND behavior=? order by modifiers",
                           (subject, behavior,))

            rows_distinct_modifiers = list(x[0] for x in cursor.fetchall())

            for distinct_modifiers in rows_distinct_modifiers:

                cursor.execute(("SELECT start, stop, type, modifiers, comment, comment_stop FROM aggregated_events "
                                "WHERE subject = ? AND behavior = ? AND modifiers = ? ORDER by start"),
                               (subject, behavior, distinct_modifiers))
                rows = list(cursor.fetchall())

                for row in rows:

                    if observation[TYPE] in [MEDIA]:
                        if duration1:
                            mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if row["start"] >= sum(duration1[0:idx1])][-1]
                            mediaFileString = observation[FILE][PLAYER1][mediaFileIdx]
                            try:
                                fpsString = observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]]
                            except Exception:
                                fpsString = "NA"
                        else:
                            try:
                                if len(observation[FILE][PLAYER1]) == 1:
                                    mediaFileString = observation[FILE][PLAYER1][0]
                                else:
                                    mediaFileString = "NA"
                            except Exception:
                                mediaFileString = "NA"
                            fpsString = "NA"

                    if observation[TYPE] in [LIVE]:
                        mediaFileString = "LIVE"
                        fpsString = "NA"

                    if row["type"] == POINT:

                        row_data = []
                        row_data.extend([obsId,
                                         observation["date"].replace("T", " "),
                                         mediaFileString,
                                         f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA",
                                         fpsString])

                        # independent variables
                        if INDEPENDENT_VARIABLES in pj:
                            for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]):
                                if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]:
                                    row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]])
                                else:
                                    row_data.append("")

                        row_data.extend([subject,
                                         behavior,
                                         behavioral_category[behavior],
                                         row["modifiers"],
                                         POINT,
                                         "{0:.3f}".format(row["start"]),  # start
                                         "{0:.3f}".format(row["stop"]),  # stop
                                         "NA",  # duration
                                         row["comment"],
                                         ""
                                         ])
                        data.append(row_data)

                    if row["type"] == STATE:
                        if idx % 2 == 0:
                            row_data = []
                            row_data.extend([obsId,
                                             observation["date"].replace("T", " "),
                                             mediaFileString,
                                             f"{obs_length:.3f}" if obs_length != Decimal("-1") else "NA",
                                             fpsString])

                            # independent variables
                            if INDEPENDENT_VARIABLES in pj:
                                for idx_var in utilities.sorted_keys(pj[INDEPENDENT_VARIABLES]):
                                    if pj[INDEPENDENT_VARIABLES][idx_var]["label"] in observation[INDEPENDENT_VARIABLES]:
                                        row_data.append(observation[INDEPENDENT_VARIABLES][pj[INDEPENDENT_VARIABLES][idx_var]["label"]])
                                    else:
                                        row_data.append("")

                            row_data.extend([subject,
                                             behavior,
                                             behavioral_category[behavior],
                                             row["modifiers"],
                                             STATE,
                                             "{0:.3f}".format(row["start"]),
                                             "{0:.3f}".format(row["stop"]),
                                             "{0:.3f}".format(row["stop"] - row["start"]),
                                             row["comment"],
                                             row["comment_stop"]
                                             ])
                            data.append(row_data)

    return data
def export_events(parameters, obsId, observation, ethogram, file_name, output_format):
    """
    export events

    Args:
        parameters (dict): subjects, behaviors
        obsId (str): observation id
        observation (dict): observation
        ethogram (dict): ethogram of project
        file_name (str): file name for exporting events
        output_format (str): output for exporting events

    Returns:
        bool: result: True if OK else False
        str: error message
    """

    total_length = "{0:.3f}".format(project_functions.observation_total_length(observation))

    eventsWithStatus = project_functions.events_start_stop(ethogram, observation[EVENTS])

    # check max number of modifiers
    max_modifiers = 0
    for event in eventsWithStatus:
        for c in pj_events_fields:
            if c == "modifier" and event[pj_obs_fields[c]]:
                max_modifiers = max(max_modifiers, len(event[pj_obs_fields[c]].split("|")))

    # media file number
    mediaNb = 0
    if observation["type"] == MEDIA:
        for player in observation[FILE]:
            mediaNb += len(observation[FILE][player])

    rows = []

    # observation id
    rows.append(["Observation id", obsId])
    rows.append([""])

    # media file name
    if observation["type"] in [MEDIA]:
        rows.append(["Media file(s)"])
    elif observation["type"] in [LIVE]:
        rows.append(["Live observation"])
    else:
        rows.append(["?"])
    rows.append([""])

    if observation[TYPE] in [MEDIA]:
        for player in sorted(list(observation[FILE].keys())):
            for media in observation[FILE][player]:
                rows.append(["Player #{0}".format(player), media])
    rows.append([""])

    # date
    if "date" in observation:
        rows.append(["Observation date", observation["date"].replace("T", " ")])
    rows.append([""])

    # description
    if "description" in observation:
        rows.append(["Description", utilities.eol2space(observation["description"])])
    rows.append([""])

    # time offset
    if "time offset" in observation:
        rows.append(["Time offset (s)", observation["time offset"]])
    rows.append([""])

    # independent variables
    if INDEPENDENT_VARIABLES in observation:
        rows.extend([["independent variables"], ["variable", "value"]])

        for variable in observation[INDEPENDENT_VARIABLES]:
            rows.append([variable, observation[INDEPENDENT_VARIABLES][variable]])

    rows.append([""])

    # write table header
    col = 0
    header = ["Time"]
    header.extend(["Media file path", "Total length", "FPS"])

    header.extend(["Subject", "Behavior", "Behavioral category"])

    behavioral_category = project_functions.behavior_category(ethogram)

    for x in range(1, max_modifiers + 1):
        header.append("Modifier {}".format(x))
    header.extend(["Comment", "Status"])

    rows.append(header)

    duration1 = []   # in seconds
    if observation["type"] in [MEDIA]:
        try:
            for mediaFile in observation[FILE][PLAYER1]:
                duration1.append(observation[MEDIA_INFO]["length"][mediaFile])
        except KeyError:
            pass

    for event in eventsWithStatus:
        if (((event[SUBJECT_EVENT_FIELD] in parameters["selected subjects"]) or
                (event[SUBJECT_EVENT_FIELD] == "" and NO_FOCAL_SUBJECT in parameters["selected subjects"])) and
                (event[BEHAVIOR_EVENT_FIELD] in parameters["selected behaviors"])):

            fields = []
            fields.append(utilities.intfloatstr(str(event[EVENT_TIME_FIELD_IDX])))

            if observation["type"] in [MEDIA]:

                time_ = event[EVENT_TIME_FIELD_IDX] - observation[TIME_OFFSET]
                if time_ < 0:
                    time_ = 0

                if duration1:
                    mediaFileIdx = [idx1 for idx1, x in enumerate(duration1) if time_ >= sum(duration1[0:idx1])][-1]
                    fields.append(observation[FILE][PLAYER1][mediaFileIdx])
                    fields.append(total_length)
                    # FPS
                    try:
                        fields.append(observation[MEDIA_INFO]["fps"][observation[FILE][PLAYER1][mediaFileIdx]])  # fps
                    except KeyError:
                        fields.append("NA")
                else:
                    fields.append("NA") # media file
                    fields.append("NA") # FPS

            if observation["type"] in [LIVE]:
                fields.append(LIVE)  # media
                fields.append(total_length)  # total length
                fields.append("NA")   # FPS

            fields.append(event[EVENT_SUBJECT_FIELD_IDX])
            fields.append(event[EVENT_BEHAVIOR_FIELD_IDX])

            # behavioral category

            try:
                behav_category = behavioral_category[event[EVENT_BEHAVIOR_FIELD_IDX]]
            except Exception:
                behav_category = ""
            fields.append(behav_category)

            # modifiers
            if max_modifiers:
                modifiers = event[EVENT_MODIFIER_FIELD_IDX].split("|")
                while len(modifiers) < max_modifiers:
                    modifiers.append("")

                for m in modifiers:
                    fields.append(m)

            # comment
            fields.append(event[EVENT_COMMENT_FIELD_IDX].replace(os.linesep, " "))
            # status
            fields.append(event[-1])

            rows.append(fields)

    maxLen = max([len(r) for r in rows])
    data = tablib.Dataset()

    data.title = obsId
    # check if worksheet name will be > 31 char
    if output_format in ["xls", "xlsx"]:
        for forbidden_char in EXCEL_FORBIDDEN_CHARACTERS:
            data.title = data.title.replace(forbidden_char, " ")
        if output_format in ["xls"]:
            if len(data.title) > 31:
                data.title = data.title[0:31]

    for row in rows:
        data.append(utilities.complete(row, maxLen))

    r, msg = dataset_write(data, file_name, output_format)

    return r, msg