def open_load_file_dialog(browser):
    nids = browser.selectedNotes()
    if nids:
        try:
            ext = ".csv"
            default_path = QStandardPaths.writableLocation(
                QStandardPaths.DocumentsLocation)
            path = os.path.join(default_path, f"changes{ext}")

            options = QFileDialog.Options()

            # native doesn't seem to works
            options |= QFileDialog.DontUseNativeDialog

            result = QFileDialog.getOpenFileName(browser,
                                                 "Import CSV for Batch Update",
                                                 path,
                                                 f"CSV (*{ext})",
                                                 options=options)

            if not isinstance(result, tuple):
                raise Exception("Expected a tuple from save dialog")
            file = result[0]
            if file:
                BatchUpdateDialog(browser, nids, file).exec_()

        except Exception as e:
            tooltip("Failed: {}".format(e))
    else:
        tooltip("You must select some cards first")
Exemplo n.º 2
0
    def onDiff(self):
        """Produces HTML diff of the updates that would be made"""
        append_to_log = self.log.appendPlainText

        lines = []
        try:
            self.log.clear()
            nids = self.nids
            field_name = self.field_selection.currentText()

            cnt = 0
            need_clean = 0
            failed_notes = []
            for nid in nids:
                note = self.browser.mw.col.getNote(nid)
                if field_name in note:
                    content = note[field_name]
                    try:
                        cleaned_content = self.clean_content(
                            content, output_html_diff=True)
                        if content != cleaned_content:
                            lines.append((nid, cleaned_content))
                            need_clean += 1
                    except TextProcessingError as e:
                        failed_notes.append((nid, content, str(e)))
                    cnt += 1
            if failed_notes:
                append_to_log(
                    "Found {} notes that failed to be processed:".format(
                        len(failed_notes)))
                for nid, _, exc in failed_notes:
                    append_to_log("{}: {}\n".format(nid, exc))
            append_to_log(
                "Checked {} notes. Found {} notes need updating.".format(
                    cnt, need_clean))
            if failed_notes:
                append_to_log(
                    "Found {} notes that failed to be processed".format(
                        len(failed_notes)))

            if len(lines) > 0:
                ext = ".html"
                default_path = QStandardPaths.writableLocation(
                    QStandardPaths.DocumentsLocation)
                path = os.path.join(default_path, f"diff{ext}")

                options = QFileDialog.Options()

                # native doesn't seem to works
                options |= QFileDialog.DontUseNativeDialog

                # we'll confirm ourselves
                options |= QFileDialog.DontConfirmOverwrite

                result = QFileDialog.getSaveFileName(self,
                                                     "Save HTML diff",
                                                     path,
                                                     f"HTML (*{ext})",
                                                     options=options)

                if not isinstance(result, tuple):
                    raise Exception("Expected a tuple from save dialog")
                file = result[0]
                if file:
                    do_save = True
                    if not file.lower().endswith(ext):
                        file += ext
                    if os.path.exists(file):
                        if not askUser(
                                "{} already exists. Are you sure you want to overwrite it?"
                                .format(file),
                                parent=self):
                            do_save = False
                    if do_save:
                        append_to_log("Saving to {}".format(file))
                        with open(file, "w", encoding="utf-8") as outf:
                            outf.write(DIFF_PRE)
                            for nid, line in lines:
                                outf.write("<p>nid {}:</p>\n".format(nid))
                                outf.write("<p>{}</p>\n".format(line))
                            outf.write(DIFF_POST)
                        append_to_log("Done")

            # Ensure QPlainTextEdit refreshes (not clear why this is necessary)
            self.log.repaint()

        except Exception:
            append_to_log("Failed while checking notes:\n{}".format(
                traceback.format_exc()))
    def onCheck(self, *, mode):
        self.log.clear()
        try:
            # Mapping from field name in file to field combo boxes for notes.
            # We need to check each of the selections for the combo boxes.
            zipped_fields = zip(
                [fn for fn in self.file_field_names if fn != "nid"],
                self.mapping_field_selections)

            # mapping from file join key name to note join key name
            file_join_key_name = self.file_join_key_selection.currentText()
            note_join_key_name = self.note_join_key_selection.currentText()

            self.log.appendPlainText(
                "Join key: File field '{}' -> Note field '{}'".format(
                    file_join_key_name, note_join_key_name))

            # Check which of the field combo boxes having a non-nothing selection and
            # build the mapping from fields in file to fields in notes.
            file_to_note_mappings = {}
            for file_field_name, note_field_cb in zipped_fields:
                note_field_name = note_field_cb.currentText()
                if note_field_name != NOTHING_VALUE:
                    file_to_note_mappings[file_field_name] = note_field_name
                    self.log.appendPlainText(
                        "File field '{}' -> Note field '{}'".format(
                            file_field_name, note_field_name))
            if not file_to_note_mappings:
                self.log.appendPlainText("ERROR: No mappings selected")
                return

            # Check which key values exist and to make sure there are no duplicate values.
            # Build a mapping form these key values to the row which contains the field values.
            file_key_to_values = {}
            duplicate_file_key_values = set()
            with open(self.file, encoding="utf-8") as inf:
                reader = csv.DictReader(inf)
                for row in reader:
                    join_key_val = row[file_join_key_name]
                    if join_key_val in file_key_to_values:
                        duplicate_file_key_values.add(join_key_val)
                    else:
                        file_key_to_values[join_key_val] = row
            if duplicate_file_key_values:
                self.log.appendPlainText(
                    "ERROR: Found {} key values for '{}' that appear more than once:"
                    .format(len(duplicate_file_key_values),
                            file_join_key_name))
                for val in duplicate_file_key_values:
                    self.log.appendPlainText(val)
                return
            self.log.appendPlainText("Found {} records for '{}' in {}".format(
                len(file_key_to_values), file_join_key_name, self.file))

            # When we aren't joining by nid, we need to create an additional mapping from the join
            # key to the nid value, because we can only look up by nid.
            note_join_key_to_nid = {}
            if note_join_key_name != "nid":
                self.log.appendPlainText(
                    "Joining to notes by '{}', so finding all values.".format(
                        note_join_key_name))
                for nid in self.nids:
                    note = self.browser.mw.col.getNote(nid)
                    if note.mid != self.model_id:
                        self.log.appendPlainText(
                            "ERROR: Note {} has different model ID {} than expected {} based on first note. "
                            .format(nid, note.mid, self.model_id) +
                            "Please only select notes of the same model.")
                        return
                    if note_join_key_name in note:
                        if note[note_join_key_name] in note_join_key_to_nid:
                            self.log.appendPlainText(
                                "ERROR: Value '{}' already exists in notes".
                                format(note[note_join_key_name]))
                            return
                        else:
                            note_join_key_to_nid[
                                note[note_join_key_name]] = nid
                    else:
                        self.log.appendPlainText(
                            "ERROR: Field '{}' not found in note {}".format(
                                note_join_key_name, nid))
                        return

            # these store the changes we will propose to make (grouped by nid)
            note_changes = defaultdict(list)

            # track join keys that were not found in notes
            missing_note_keys = set()

            # how many fields being updated are empty
            empty_note_field_count = 0
            notes_with_empty_fields = set()

            for file_key, file_values in file_key_to_values.items():

                if note_join_key_name == "nid":
                    nid = file_key
                else:
                    if file_key in note_join_key_to_nid:
                        nid = note_join_key_to_nid[file_key]
                        self.log.appendPlainText(
                            "Found note {} with value {} for '{}'".format(
                                nid, file_key, note_join_key_name))
                    else:
                        self.log.appendPlainText(
                            "Could not find note with value {} for '{}'".
                            format(file_key, note_join_key_name))
                        missing_note_keys.add(file_key)
                        continue

                self.log.appendPlainText("Checking note {}".format(nid))

                try:
                    note = self.browser.mw.col.getNote(nid)
                except TypeError:
                    self.log.appendPlainText(
                        "ERROR: Note {} was not found".format(nid))
                    return

                # Get the current values for fields we're updating in the note.
                note_values = {}
                for note_field_name in file_to_note_mappings.values():
                    if note_field_name in note:
                        note_values[note_field_name] = note[note_field_name]
                    else:
                        self.log.appendPlainText(
                            "ERROR: Field '{}' not found in note {}".format(
                                note_field_name, nid))
                        return

                # Compare the file field values to the note field values and see if anything is different
                # and therefore needs to be updated.
                for file_field_name, note_field_name in file_to_note_mappings.items(
                ):
                    file_value = file_values[file_field_name]
                    note_value = note_values[note_field_name]
                    if file_value != note_value:
                        self.log.appendPlainText(
                            "Need to update note field '{}':".format(
                                note_field_name))
                        self.log.appendPlainText("{}\n=>\n{}".format(
                            note_value or "<empty>", file_value))
                        note_changes[nid].append(
                            NoteChange(nid=nid,
                                       fld=note_field_name,
                                       old=note_value,
                                       new=file_value))
                        if not note_value:
                            empty_note_field_count += 1
                            notes_with_empty_fields.add(nid)

            if missing_note_keys:
                self.log.appendPlainText(
                    "ERROR: {} values were not found in notes for field '{}'".
                    format(len(missing_note_keys), note_join_key_name))
                return

            if note_changes:
                self.log.appendPlainText(
                    "Need to make changes to {} notes".format(
                        len(note_changes)))
                if empty_note_field_count:
                    self.log.appendPlainText(
                        "{} fields across {} notes are empty".format(
                            empty_note_field_count,
                            len(notes_with_empty_fields)))

                if mode == "dryrun":
                    # nothing to do
                    pass
                elif mode == "diff":
                    ext = ".html"
                    default_path = QStandardPaths.writableLocation(
                        QStandardPaths.DocumentsLocation)
                    path = os.path.join(default_path, f"diff{ext}")

                    options = QFileDialog.Options()

                    # native doesn't seem to works
                    options |= QFileDialog.DontUseNativeDialog

                    # we'll confirm ourselves
                    options |= QFileDialog.DontConfirmOverwrite

                    result = QFileDialog.getSaveFileName(self,
                                                         "Save HTML diff",
                                                         path,
                                                         f"HTML (*{ext})",
                                                         options=options)

                    if not isinstance(result, tuple):
                        raise Exception("Expected a tuple from save dialog")
                    file = result[0]
                    if file:
                        do_save = True
                        if not file.lower().endswith(ext):
                            file += ext
                        if os.path.exists(file):
                            if not askUser(
                                    "{} already exists. Are you sure you want to overwrite it?"
                                    .format(file),
                                    parent=self):
                                do_save = False
                        if do_save:
                            self.log.appendPlainText(
                                "Saving to {}".format(file))
                            with open(file, "w", encoding="utf-8") as outf:
                                outf.write(DIFF_PRE)
                                for nid, changes in note_changes.items():
                                    outf.write("<p>nid {}:</p>\n".format(nid))
                                    for change in changes:
                                        outf.write("<p>{}: {}</p>\n".format(
                                            change.fld,
                                            html_diff(html.escape(change.old),
                                                      html.escape(
                                                          change.new))))
                                outf.write(DIFF_POST)
                            self.log.appendPlainText("Done")
                elif mode == "update":
                    if askUser(
                            "{} notes will be updated.  Are you sure you want to do this?"
                            .format(len(note_changes)),
                            parent=self):
                        self.log.appendPlainText("Beginning update")

                        self.browser.mw.checkpoint("{} ({} {})".format(
                            self.checkpoint_name, len(note_changes),
                            "notes" if len(note_changes) > 1 else "note"))
                        self.browser.model.beginReset()
                        updated_count = 0
                        try:
                            init_ts = int(time.time() * 1000)

                            for nid, changes in note_changes.items():
                                note = self.browser.mw.col.getNote(nid)
                                for change in changes:
                                    ts = int(time.time() * 1000)
                                    note[change.fld] = change.new
                                    self.changelog.record_change(
                                        "batch_update", init_ts,
                                        ChangeLogEntry(ts=ts,
                                                       nid=nid,
                                                       fld=change.fld,
                                                       old=change.old,
                                                       new=change.new))
                                note.flush()
                                updated_count += 1
                            self.log.appendPlainText(
                                "Updated {} notes".format(updated_count))
                        finally:
                            if updated_count:
                                self.changelog.commit_changes()
                                self.browser.mw.requireReset()
                            self.browser.model.endReset()
                else:
                    self.log.appendPlainText(
                        "ERROR: Unexpected mode: {}".format(mode))
                    return
            else:
                self.log.appendPlainText("No changes need to be made")

        except Exception:
            self.log.appendPlainText("Failed during dry run:\n{}".format(
                traceback.format_exc()))

        finally:
            # Ensure QPlainTextEdit refreshes (not clear why this is necessary)
            self.log.repaint()
Exemplo n.º 4
0
    def onExport(self):
        append_to_log = self.log.appendPlainText

        if not self.has_records:
            tooltip("Log is empty")
            return

        try:
            ext = ".csv"
            default_path = QStandardPaths.writableLocation(
                QStandardPaths.DocumentsLocation)
            path = os.path.join(default_path, f"changes{ext}")

            options = QFileDialog.Options()

            # native doesn't seem to works
            options |= QFileDialog.DontUseNativeDialog

            # we'll confirm ourselves
            options |= QFileDialog.DontConfirmOverwrite

            result = QFileDialog.getSaveFileName(self,
                                                 "Save CSV",
                                                 path,
                                                 f"CSV (*{ext})",
                                                 options=options)

            if not isinstance(result, tuple):
                raise Exception("Expected a tuple from save dialog")
            file = result[0]
            if file:
                do_save = True
                if not file.lower().endswith(ext):
                    file += ext
                if os.path.exists(file):
                    if not askUser(
                            "{} already exists. Are you sure you want to overwrite it?"
                            .format(file),
                            parent=self):
                        do_save = False
                if do_save:
                    append_to_log("Saving to {}".format(file))
                    with open(file, "w", encoding="utf-8") as outf:
                        field_names = ["ts", "op", "nid", "fld", "old", "new"]
                        writer = csv.DictWriter(outf, fieldnames=field_names)
                        writer.writeheader()
                        for rec in self.changelog.db.all("""
                                select op, ts, nid, fld, old, new from changelog
                                order by ts asc
                                """):
                            op, ts, nid, fld, old, new = rec
                            writer.writerow({
                                "op": op,
                                "ts": ts,
                                "nid": nid,
                                "fld": fld,
                                "old": old,
                                "new": new
                            })

                    append_to_log("Done")
        except Exception:
            append_to_log("Failed while writing CSV:\n{}".format(
                traceback.format_exc()))