def open_load_file_dialog(browser):
nids = browser.selectedNotes()
if nids:
try:
ext = ".csv"
default_path = QStandardPaths.writableLocation(
QStandardPaths.DocumentsLocation)
path = os.path.join(default_path, f"changes{ext}")
options = QFileDialog.Options()
# native doesn't seem to works
options |= QFileDialog.DontUseNativeDialog
result = QFileDialog.getOpenFileName(browser,
"Import CSV for Batch Update",
path,
f"CSV (*{ext})",
options=options)
if not isinstance(result, tuple):
raise Exception("Expected a tuple from save dialog")
file = result[0]
if file:
BatchUpdateDialog(browser, nids, file).exec_()
except Exception as e:
tooltip("Failed: {}".format(e))
else:
tooltip("You must select some cards first")
def onDiff(self):
"""Produces HTML diff of the updates that would be made"""
append_to_log = self.log.appendPlainText
lines = []
try:
self.log.clear()
nids = self.nids
field_name = self.field_selection.currentText()
cnt = 0
need_clean = 0
failed_notes = []
for nid in nids:
note = self.browser.mw.col.getNote(nid)
if field_name in note:
content = note[field_name]
try:
cleaned_content = self.clean_content(
content, output_html_diff=True)
if content != cleaned_content:
lines.append((nid, cleaned_content))
need_clean += 1
except TextProcessingError as e:
failed_notes.append((nid, content, str(e)))
cnt += 1
if failed_notes:
append_to_log(
"Found {} notes that failed to be processed:".format(
len(failed_notes)))
for nid, _, exc in failed_notes:
append_to_log("{}: {}\n".format(nid, exc))
append_to_log(
"Checked {} notes. Found {} notes need updating.".format(
cnt, need_clean))
if failed_notes:
append_to_log(
"Found {} notes that failed to be processed".format(
len(failed_notes)))
if len(lines) > 0:
ext = ".html"
default_path = QStandardPaths.writableLocation(
QStandardPaths.DocumentsLocation)
path = os.path.join(default_path, f"diff{ext}")
options = QFileDialog.Options()
# native doesn't seem to works
options |= QFileDialog.DontUseNativeDialog
# we'll confirm ourselves
options |= QFileDialog.DontConfirmOverwrite
result = QFileDialog.getSaveFileName(self,
"Save HTML diff",
path,
f"HTML (*{ext})",
options=options)
if not isinstance(result, tuple):
raise Exception("Expected a tuple from save dialog")
file = result[0]
if file:
do_save = True
if not file.lower().endswith(ext):
file += ext
if os.path.exists(file):
if not askUser(
"{} already exists. Are you sure you want to overwrite it?"
.format(file),
parent=self):
do_save = False
if do_save:
append_to_log("Saving to {}".format(file))
with open(file, "w", encoding="utf-8") as outf:
outf.write(DIFF_PRE)
for nid, line in lines:
outf.write("<p>nid {}:</p>\n".format(nid))
outf.write("<p>{}</p>\n".format(line))
outf.write(DIFF_POST)
append_to_log("Done")
# Ensure QPlainTextEdit refreshes (not clear why this is necessary)
self.log.repaint()
except Exception:
append_to_log("Failed while checking notes:\n{}".format(
traceback.format_exc()))
def onCheck(self, *, mode):
self.log.clear()
try:
# Mapping from field name in file to field combo boxes for notes.
# We need to check each of the selections for the combo boxes.
zipped_fields = zip(
[fn for fn in self.file_field_names if fn != "nid"],
self.mapping_field_selections)
# mapping from file join key name to note join key name
file_join_key_name = self.file_join_key_selection.currentText()
note_join_key_name = self.note_join_key_selection.currentText()
self.log.appendPlainText(
"Join key: File field '{}' -> Note field '{}'".format(
file_join_key_name, note_join_key_name))
# Check which of the field combo boxes having a non-nothing selection and
# build the mapping from fields in file to fields in notes.
file_to_note_mappings = {}
for file_field_name, note_field_cb in zipped_fields:
note_field_name = note_field_cb.currentText()
if note_field_name != NOTHING_VALUE:
file_to_note_mappings[file_field_name] = note_field_name
self.log.appendPlainText(
"File field '{}' -> Note field '{}'".format(
file_field_name, note_field_name))
if not file_to_note_mappings:
self.log.appendPlainText("ERROR: No mappings selected")
return
# Check which key values exist and to make sure there are no duplicate values.
# Build a mapping form these key values to the row which contains the field values.
file_key_to_values = {}
duplicate_file_key_values = set()
with open(self.file, encoding="utf-8") as inf:
reader = csv.DictReader(inf)
for row in reader:
join_key_val = row[file_join_key_name]
if join_key_val in file_key_to_values:
duplicate_file_key_values.add(join_key_val)
else:
file_key_to_values[join_key_val] = row
if duplicate_file_key_values:
self.log.appendPlainText(
"ERROR: Found {} key values for '{}' that appear more than once:"
.format(len(duplicate_file_key_values),
file_join_key_name))
for val in duplicate_file_key_values:
self.log.appendPlainText(val)
return
self.log.appendPlainText("Found {} records for '{}' in {}".format(
len(file_key_to_values), file_join_key_name, self.file))
# When we aren't joining by nid, we need to create an additional mapping from the join
# key to the nid value, because we can only look up by nid.
note_join_key_to_nid = {}
if note_join_key_name != "nid":
self.log.appendPlainText(
"Joining to notes by '{}', so finding all values.".format(
note_join_key_name))
for nid in self.nids:
note = self.browser.mw.col.getNote(nid)
if note.mid != self.model_id:
self.log.appendPlainText(
"ERROR: Note {} has different model ID {} than expected {} based on first note. "
.format(nid, note.mid, self.model_id) +
"Please only select notes of the same model.")
return
if note_join_key_name in note:
if note[note_join_key_name] in note_join_key_to_nid:
self.log.appendPlainText(
"ERROR: Value '{}' already exists in notes".
format(note[note_join_key_name]))
return
else:
note_join_key_to_nid[
note[note_join_key_name]] = nid
else:
self.log.appendPlainText(
"ERROR: Field '{}' not found in note {}".format(
note_join_key_name, nid))
return
# these store the changes we will propose to make (grouped by nid)
note_changes = defaultdict(list)
# track join keys that were not found in notes
missing_note_keys = set()
# how many fields being updated are empty
empty_note_field_count = 0
notes_with_empty_fields = set()
for file_key, file_values in file_key_to_values.items():
if note_join_key_name == "nid":
nid = file_key
else:
if file_key in note_join_key_to_nid:
nid = note_join_key_to_nid[file_key]
self.log.appendPlainText(
"Found note {} with value {} for '{}'".format(
nid, file_key, note_join_key_name))
else:
self.log.appendPlainText(
"Could not find note with value {} for '{}'".
format(file_key, note_join_key_name))
missing_note_keys.add(file_key)
continue
self.log.appendPlainText("Checking note {}".format(nid))
try:
note = self.browser.mw.col.getNote(nid)
except TypeError:
self.log.appendPlainText(
"ERROR: Note {} was not found".format(nid))
return
# Get the current values for fields we're updating in the note.
note_values = {}
for note_field_name in file_to_note_mappings.values():
if note_field_name in note:
note_values[note_field_name] = note[note_field_name]
else:
self.log.appendPlainText(
"ERROR: Field '{}' not found in note {}".format(
note_field_name, nid))
return
# Compare the file field values to the note field values and see if anything is different
# and therefore needs to be updated.
for file_field_name, note_field_name in file_to_note_mappings.items(
):
file_value = file_values[file_field_name]
note_value = note_values[note_field_name]
if file_value != note_value:
self.log.appendPlainText(
"Need to update note field '{}':".format(
note_field_name))
self.log.appendPlainText("{}\n=>\n{}".format(
note_value or "<empty>", file_value))
note_changes[nid].append(
NoteChange(nid=nid,
fld=note_field_name,
old=note_value,
new=file_value))
if not note_value:
empty_note_field_count += 1
notes_with_empty_fields.add(nid)
if missing_note_keys:
self.log.appendPlainText(
"ERROR: {} values were not found in notes for field '{}'".
format(len(missing_note_keys), note_join_key_name))
return
if note_changes:
self.log.appendPlainText(
"Need to make changes to {} notes".format(
len(note_changes)))
if empty_note_field_count:
self.log.appendPlainText(
"{} fields across {} notes are empty".format(
empty_note_field_count,
len(notes_with_empty_fields)))
if mode == "dryrun":
# nothing to do
pass
elif mode == "diff":
ext = ".html"
default_path = QStandardPaths.writableLocation(
QStandardPaths.DocumentsLocation)
path = os.path.join(default_path, f"diff{ext}")
options = QFileDialog.Options()
# native doesn't seem to works
options |= QFileDialog.DontUseNativeDialog
# we'll confirm ourselves
options |= QFileDialog.DontConfirmOverwrite
result = QFileDialog.getSaveFileName(self,
"Save HTML diff",
path,
f"HTML (*{ext})",
options=options)
if not isinstance(result, tuple):
raise Exception("Expected a tuple from save dialog")
file = result[0]
if file:
do_save = True
if not file.lower().endswith(ext):
file += ext
if os.path.exists(file):
if not askUser(
"{} already exists. Are you sure you want to overwrite it?"
.format(file),
parent=self):
do_save = False
if do_save:
self.log.appendPlainText(
"Saving to {}".format(file))
with open(file, "w", encoding="utf-8") as outf:
outf.write(DIFF_PRE)
for nid, changes in note_changes.items():
outf.write("<p>nid {}:</p>\n".format(nid))
for change in changes:
outf.write("<p>{}: {}</p>\n".format(
change.fld,
html_diff(html.escape(change.old),
html.escape(
change.new))))
outf.write(DIFF_POST)
self.log.appendPlainText("Done")
elif mode == "update":
if askUser(
"{} notes will be updated. Are you sure you want to do this?"
.format(len(note_changes)),
parent=self):
self.log.appendPlainText("Beginning update")
self.browser.mw.checkpoint("{} ({} {})".format(
self.checkpoint_name, len(note_changes),
"notes" if len(note_changes) > 1 else "note"))
self.browser.model.beginReset()
updated_count = 0
try:
init_ts = int(time.time() * 1000)
for nid, changes in note_changes.items():
note = self.browser.mw.col.getNote(nid)
for change in changes:
ts = int(time.time() * 1000)
note[change.fld] = change.new
self.changelog.record_change(
"batch_update", init_ts,
ChangeLogEntry(ts=ts,
nid=nid,
fld=change.fld,
old=change.old,
new=change.new))
note.flush()
updated_count += 1
self.log.appendPlainText(
"Updated {} notes".format(updated_count))
finally:
if updated_count:
self.changelog.commit_changes()
self.browser.mw.requireReset()
self.browser.model.endReset()
else:
self.log.appendPlainText(
"ERROR: Unexpected mode: {}".format(mode))
return
else:
self.log.appendPlainText("No changes need to be made")
except Exception:
self.log.appendPlainText("Failed during dry run:\n{}".format(
traceback.format_exc()))
finally:
# Ensure QPlainTextEdit refreshes (not clear why this is necessary)
self.log.repaint()
def onExport(self):
append_to_log = self.log.appendPlainText
if not self.has_records:
tooltip("Log is empty")
return
try:
ext = ".csv"
default_path = QStandardPaths.writableLocation(
QStandardPaths.DocumentsLocation)
path = os.path.join(default_path, f"changes{ext}")
options = QFileDialog.Options()
# native doesn't seem to works
options |= QFileDialog.DontUseNativeDialog
# we'll confirm ourselves
options |= QFileDialog.DontConfirmOverwrite
result = QFileDialog.getSaveFileName(self,
"Save CSV",
path,
f"CSV (*{ext})",
options=options)
if not isinstance(result, tuple):
raise Exception("Expected a tuple from save dialog")
file = result[0]
if file:
do_save = True
if not file.lower().endswith(ext):
file += ext
if os.path.exists(file):
if not askUser(
"{} already exists. Are you sure you want to overwrite it?"
.format(file),
parent=self):
do_save = False
if do_save:
append_to_log("Saving to {}".format(file))
with open(file, "w", encoding="utf-8") as outf:
field_names = ["ts", "op", "nid", "fld", "old", "new"]
writer = csv.DictWriter(outf, fieldnames=field_names)
writer.writeheader()
for rec in self.changelog.db.all("""
select op, ts, nid, fld, old, new from changelog
order by ts asc
"""):
op, ts, nid, fld, old, new = rec
writer.writerow({
"op": op,
"ts": ts,
"nid": nid,
"fld": fld,
"old": old,
"new": new
})
append_to_log("Done")
except Exception:
append_to_log("Failed while writing CSV:\n{}".format(
traceback.format_exc()))