def foreignNotes(self):
"""Parse notes from the rows read in ``open``.
"""
self.open()
out = []
for word, senses in groupby(0, self._rows).items():
back = []
usually_written_using_kana_alone = False
for row in senses:
reading = row[1]
if row[-1]:
usually_written_using_kana_alone = True
back.append(f'{row[3]}: {row[2]}')
back = '\n'.join(back)
note = ForeignNote()
note.fields = [
reading if usually_written_using_kana_alone else word,
reading,
back,
]
out.append(note)
return out
def foreignNotes(self) -> List[ForeignNote]:
md = markdown.Markdown()
self.open()
self.log = []
notes = []
if not self.allowHTML:
raise Exception('"Allow HTML in fields" must be checked. Aborting.')
rows = islice(self.worksheet.rows, 4, None)
for row in rows:
note = ForeignNote()
for i in range(self.numFields):
if i >= len(row) or row[i].value is None:
item = ""
else:
item = str(row[i].value)
if self.fieldTypes[i] == "text":
item = html.escape(item, quote=False)
item = item.strip()
item = item.replace("\n", "<br>")
elif self.fieldTypes[i] == "markdown":
item = md.reset().convert(item)
else:
#html
pass
note.fields.append(item)
note.tags.extend(self.tagsToAdd)
notes.append(note)
self.workbook.close()
self.opened = False
return notes
def processNote(self, noteString, globalTags):
newNote = ForeignNote()
newNote.tags.extend(globalTags)
fieldCommands = [{'beginfun': lambda string: self.findCommand(string, ur"field"),
'endfun': lambda string: self.findCommand(string, ur"endfield"),
'process': lambda string: self.processLatexField(string, newNote)},
{'beginfun': lambda string: self.findCommand(string, ur"begin", ur"field"),
'endfun': lambda string: self.findCommand(string, ur"end", ur"field", warning=True),
'process': lambda string: self.processLatexField(string, newNote)},
{'beginfun': lambda string: self.findCommand(string, ur"xfield", ur"?"),
'endfun': self.findClosingBrace,
'process': lambda string: self.processLatexField(string, newNote)},
{'beginfun': lambda string: self.findCommand(string, ur"plain"),
'endfun': lambda string: self.findCommand(string, ur"endplain"),
'process': lambda string: self.processPlainField(string, newNote)},
{'beginfun': lambda string: self.findCommand(string, ur"begin", ur"plain"),
'endfun': lambda string: self.findCommand(string, ur"end", ur"plain", warning=True),
'process': lambda string: self.processPlainField(string, newNote)},
{'beginfun': lambda string: self.findCommand(string, ur"xplain", ur"?"),
'endfun': self.findClosingBrace,
'process': lambda string: self.processPlainField(string, newNote)},
{'beginfun': lambda string: self.findCommand(string, ur"tags", ur"?"),
'endfun': self.findClosingBrace,
'process': lambda string: self.processTags(string, newNote)}
]
def noteFromJson(self, jsonDict):
note = None
if (jsonDict[u'object'] == 'vocabulary'
and jsonDict[u'id'] in self.availableIds):
note = ForeignNote()
# appending as a number anki throws an Exception
note.fields.append('{}'.format(jsonDict[u'id']))
# appending as a number anki throws an Exception
note.fields.append('{}'.format(jsonDict[u'data'][u'level']))
# appending as a number anki throws an Exception
note.fields.append('{}'.format(
jsonDict[u'data'][u'lesson_position']))
note.fields.append(jsonDict[u'data'][u'characters'])
note.fields.append(
joinValues(jsonDict[u'data'][u'meanings'], u'meaning')[1])
note.fields.append(jsonDict[u'data'][u'meaning_mnemonic'])
note.fields.append(
joinValues(jsonDict[u'data'][u'readings'], u'reading')[1])
note.fields.append(jsonDict[u'data'][u'reading_mnemonic'])
note.fields.append(", ".join(
jsonDict[u'data'][u'parts_of_speech']))
return note
def _addCloze(self, notes):
data = []
notes = notes.values()
for orig in notes:
# create a foreign note object
n = ForeignNote()
n.fields = []
fld = orig.get("text", "")
fld = re.sub("\r?\n", "<br>", fld)
state = dict(n=1)
def repl(match):
# replace [...] with cloze refs
res = ("{{c%d::%s}}" % (state['n'], match.group(1)))
state['n'] += 1
return res
fld = re.sub("\[(.+?)\]", repl, fld)
fld = self._mungeField(fld)
n.fields.append(fld)
n.fields.append("") # extra
n.tags = orig['tags']
n.cards = orig.get('cards', {})
data.append(n)
# add cloze model
model = addClozeModel(self.col)
model['name'] = "Mnemosyne-Cloze"
mm = self.col.models
mm.save(model)
mm.setCurrent(model)
self.model = model
self._fields = len(model['flds'])
self.initMapping()
self.importNotes(data)
def foreignNotes(self):
with open(self.file, 'r') as f:
content = f.read().decode('utf-8')
notes = []
entries = json.loads(content)
for entry in entries:
row = []
empty = True
for f in self.mappingFields:
value = entry.get(f)
if value:
empty = False
if '.' in value:
ext = value[value.rfind('.') + 1:].lower()
if ext in MEDIA_EXTENSIONS:
path = os.path.join(self.fileDir, value)
if os.path.exists(path):
filename = self.col.media.addFile(
os.path.join(self.fileDir, value))
if ext in AUDIO_EXTENSIONS:
value = u'[sound:%s]' % filename
else:
value = u'<img src="%s">' % filename
row.append(value)
if empty: # empty entry
continue
note = ForeignNote()
note.fields = row
notes.append(note)
return notes
def foreignNotes(self):
"""Build and return a list of notes."""
notes = []
try:
f = gzip.open(self.file)
tree = ET.parse(f)
lesson = tree.getroot()
assert lesson.tag == "Lesson"
finally:
f.close()
index = -4
for batch in lesson.findall('./Batch'):
index += 1
for card in batch.findall('./Card'):
# Create a note for this card.
front = card.findtext('./FrontSide/Text')
back = card.findtext('./ReverseSide/Text')
note = ForeignNote()
note.fields = [
cgi.escape(x.strip()).replace('\n', '<br>').replace(
' ', ' ') for x in [front, back]
]
notes.append(note)
# Determine due date for cards.
frontdue = card.find('./FrontSide[@LearnedTimestamp]')
backdue = card.find('./ReverseSide[@Batch][@LearnedTimestamp]')
if frontdue is not None:
note.cards[0] = self._learnedCard(
index, int(frontdue.attrib['LearnedTimestamp']))
if backdue is not None:
note.cards[1] = self._learnedCard(
int(backdue.attrib['Batch']),
int(backdue.attrib['LearnedTimestamp']))
return notes
def _addCloze(self, notes):
data = []
notes = list(notes.values())
for orig in notes:
# create a foreign note object
n = ForeignNote()
n.fields = []
fld = orig.get("text", "")
fld = re.sub("\r?\n", "<br>", fld)
state = dict(n=1)
def repl(match):
# pylint: disable=cell-var-from-loop
# replace [...] with cloze refs
res = "{{c%d::%s}}" % (state["n"], match.group(1))
state["n"] += 1
return res
fld = re.sub(r"\[(.+?)\]", repl, fld)
fld = self._mungeField(fld)
n.fields.append(fld)
n.fields.append("") # extra
n.tags = orig["tags"]
n.cards = orig.get("cards", {})
data.append(n)
# add cloze model
model = addClozeModel(self.col)
model["name"] = "Mnemosyne-Cloze"
mm = self.col.models
mm.save(model)
mm.setCurrent(model)
self.model = model
self._fields = len(model["flds"])
self.initMapping()
self.importNotes(data)
def addItemToCards(self,item):
"This method actually do conversion"
# new anki card
note = ForeignNote()
# clean Q and A
note.fields.append(self._fudgeText(self._decode_htmlescapes(item.Question)))
note.fields.append(self._fudgeText(self._decode_htmlescapes(item.Answer)))
note.tags = []
# pre-process scheduling data
# convert learning data
if (not self.META.resetLearningData
and int(item.Interval) >= 1
and getattr(item, "LastRepetition", None)):
# migration of LearningData algorithm
tLastrep = time.mktime(time.strptime(item.LastRepetition, '%d.%m.%Y'))
tToday = time.time()
card = ForeignCard()
card.ivl = int(item.Interval)
card.lapses = int(item.Lapses)
card.reps = int(item.Repetitions) + int(item.Lapses)
nextDue = tLastrep + (float(item.Interval) * 86400.0)
remDays = int((nextDue - time.time())/86400)
card.due = self.col.sched.today+remDays
card.factor = int(self._afactor2efactor(float(item.AFactor.replace(',','.')))*1000)
note.cards[0] = card
# categories & tags
# it's worth to have every theme (tree structure of sm collection) stored in tags, but sometimes not
# you can deceide if you are going to tag all toppics or just that containing some pattern
tTaggTitle = False
for pattern in self.META.pathsToBeTagged:
if item.lTitle != None and pattern.lower() in " ".join(item.lTitle).lower():
tTaggTitle = True
break
if tTaggTitle or self.META.tagAllTopics:
# normalize - remove diacritic punctuation from unicode chars to ascii
item.lTitle = [ self._unicode2ascii(topic) for topic in item.lTitle]
# Transfrom xyz / aaa / bbb / ccc on Title path to Tag xyzAaaBbbCcc
# clean things like [999] or [111-2222] from title path, example: xyz / [1000-1200] zyx / xyz
# clean whitespaces
# set Capital letters for first char of the word
tmp = list(set([ re.sub('(\[[0-9]+\])' , ' ' , i ).replace('_',' ') for i in item.lTitle ]))
tmp = list(set([ re.sub('(\W)',' ', i ) for i in tmp ]))
tmp = list(set([ re.sub( '^[0-9 ]+$','',i) for i in tmp ]))
tmp = list(set([ capwords(i).replace(' ','') for i in tmp ]))
tags = [ j[0].lower() + j[1:] for j in tmp if j.strip() != '']
note.tags += tags
if self.META.tagMemorizedItems and int(item.Interval) >0:
note.tags.append("Memorized")
self.logger('Element tags\t- ' + repr(note.tags), level=3)
self.notes.append(note)
def entriesToNotes(self, entries):
notes = []
for entry in entries:
row = self._read_row(entry)
if len(row) == 0: # empty entry
continue
note = ForeignNote()
note.fields = row
notes.append(note)
return notes
def questionToNote(self, question):
"""Convert a Question object to a ForeignNote."""
displayedAnswers = []
for letter, answer in zip(string.lowercase, question.answers):
displayedAnswers.append(letter + ") " + answer)
questionField = question.question + "\n\n" + "\n".join(displayedAnswers)
answerField = displayedAnswers[question.iCorrectAnswer]
note = ForeignNote()
note.fields = [questionField, answerField]
return note
def foreignNotes(self):
"""Build and return a list of notes."""
notes = []
try:
f = gzip.open(self.file)
tree = ET.parse(f) # type: ignore
lesson = tree.getroot()
assert lesson.tag == "Lesson"
finally:
f.close()
index = -4
for batch in lesson.findall("./Batch"):
index += 1
for card in batch.findall("./Card"):
# Create a note for this card.
front = card.findtext("./FrontSide/Text")
back = card.findtext("./ReverseSide/Text")
note = ForeignNote()
assert front and back
note.fields = [
html.escape(x.strip())
.replace("\n", "<br>")
.replace(" ", " ")
for x in [front, back]
]
notes.append(note)
# Determine due date for cards.
frontdue = card.find("./FrontSide[@LearnedTimestamp]")
backdue = card.find("./ReverseSide[@Batch][@LearnedTimestamp]")
if frontdue is not None:
note.cards[0] = self._learnedCard(
index, int(frontdue.attrib["LearnedTimestamp"])
)
if backdue is not None:
note.cards[1] = self._learnedCard(
int(backdue.attrib["Batch"]),
int(backdue.attrib["LearnedTimestamp"]),
)
return notes
def noteFromJson(self, jsonDict):
note = None
if (jsonDict[u'object'] == 'kanji'
and jsonDict[u'id'] in self.availableIds):
note = ForeignNote()
# appending as a number anki throws an Exception
note.fields.append('{}'.format(jsonDict[u'id']))
# appending as a number anki throws an Exception
note.fields.append('{}'.format(jsonDict[u'data'][u'level']))
# appending as a number anki throws an Exception
note.fields.append('{}'.format(
jsonDict[u'data'][u'lesson_position']))
note.fields.append(jsonDict[u'data'][u'characters'])
note.fields.append(
joinValues(jsonDict[u'data'][u'meanings'], u'meaning')[1])
note.fields.append(jsonDict[u'data'][u'meaning_mnemonic'])
opcount, onyomi = joinValues(jsonDict[u'data'][u'readings'],
u'reading', u'onyomi')
kpcount, kunyomi = joinValues(jsonDict[u'data'][u'readings'],
u'reading', u'kunyomi')
note.fields.append(onyomi)
# if kanji has primary onyomi and kunyomi reading too, then reading mnemonic
# is put both in onyomi and kunyomi reading mnemonic
if (opcount > 0 and kpcount == 0) or (kpcount > 0 and opcount > 0):
note.fields.append(jsonDict[u'data'][u'reading_mnemonic'])
else:
note.fields.append(u'')
note.fields.append(kunyomi)
# if kanji has primary onyomi and kunyomi reading too, then reading mnemonic
# is put both in onyomi and kunyomi reading mnemonic
if (kpcount > 0 and opcount == 0) or (kpcount > 0 and opcount > 0):
note.fields.append(jsonDict[u'data'][u'reading_mnemonic'])
else:
note.fields.append(u'')
return note
def readData(self, mapping):
from anki.importing.noteimp import ForeignNote
if not self.hasOpenWorkbook():
self._log.append("xlimport: Error reading data: no open worksheet.")
return
assert self._dataDim
assert self._headerCells
self._notes = []
for row in tuple(self.worksheet.iter_rows(self._dataDim, row_offset = self._rowOffset)):
flds = []
isEmpty = True
for c in xrange(0, len(mapping)):
hdr = self._headerCells[c].value
if hdr and hdr in mapping:
v = row[c].value if row[c].value else ''
isEmpty = False if row[c].value else isEmpty
flds.append(v if row[c].data_type == 's' else str(v))
if not isEmpty:
note = ForeignNote()
note.fields = flds
self._notes.append(note)
def foreignNotes(self):
notes = []
for record in self.getRecords():
note = ForeignNote()
modelFields = self.model['flds']
for field in modelFields:
key = field['name']
if key == 'Definition':
note.fields.append(record["definition"])
elif key == 'Word':
note.fields.append(
f'<a href="{record["entryUrl"]}">{record["headword"]}</a>'
)
notes.append(note)
return notes
def foreignNotes(self):
notes = []
for record in self.getRecords():
note = ForeignNote()
fields = record["fields"]
modelFields = self.model["flds"]
for field in modelFields:
key = field['name']
# sys.stderr.write(key + "\n")
if key in fields:
if isinstance(fields[key], list):
if len(fields[key]) == 0:
note.fields.append("")
elif isinstance(fields[key][0], dict):
note.fields.append(
self.downloadToCollection(fields[key]))
else:
strings = [str(e) for e in fields[key]]
note.fields.append(str(", ".join(strings)))
else:
if not isinstance(fields[key], str) and not isinstance(
fields[key], str):
fields[key] = str(fields[key])
asciiToHtml = fields[key].replace("\n", "<br/>")
note.fields.append(asciiToHtml)
else:
note.fields.append("")
# This has some encoding problems.
# sys.stderr.write(key + " => " + note.fields[len(note.fields) - 1] + "\n")
notes.append(note)
return notes
def _addFronts(self, notes, model=None, fields=("f", "b")):
data = []
for orig in notes:
# create a foreign note object
n = ForeignNote()
n.fields = []
for f in fields:
n.fields.append(orig.get(f, ''))
n.tags = orig['tags']
n.cards = orig.get('cards', {})
data.append(n)
# add a basic model
if not model:
model = addBasicModel(self.col)
model['name'] = "Mnemosyne-FrontOnly"
mm = self.col.models
mm.save(model)
mm.setCurrent(model)
self.model = model
self._fields = len(model['flds'])
self.initMapping()
# import
self.importNotes(data)
def _addFronts(self, notes, model=None, fields=("f", "b")):
data = []
for orig in notes:
# create a foreign note object
n = ForeignNote()
n.fields = []
for f in fields:
fld = self._mungeField(orig.get(f, ""))
n.fields.append(fld)
n.tags = orig["tags"]
n.cards = orig.get("cards", {})
data.append(n)
# add a basic model
if not model:
model = _legacy_add_basic_model(self.col)
model["name"] = "Mnemosyne-FrontOnly"
mm = self.col.models
mm.save(model)
mm.set_current(model)
self.model = model
self._fields = len(model["flds"])
self.initMapping()
# import
self.importNotes(data)
def noteFromFields(self, id, front, back, tags):
note = ForeignNote()
note.fields.extend([id, front, back, ""])
note.tags.extend(tags + self.tagsToAdd)
return note
def noteFromFields(self, fields: List[str]) -> ForeignNote:
note = ForeignNote()
note.fields.extend([x for x in fields])
note.tags.extend(self.tagsToAdd)
return note
def noteFromFields(self, fields):
note = ForeignNote()
note.fields.extend([x.strip().replace("\n", "<br>") for x in fields])
note.tags.extend(self.tagsToAdd)
return note
def processNote(self, noteString, globalTags):
newNote = ForeignNote()
newNote.tags.extend(globalTags)
fieldCommands = [{
'beginfun':
lambda string: self.findCommand(string, r"field"),
'endfun':
lambda string: self.findCommand(string, r"endfield"),
'process':
lambda string: self.processLatexField(string, newNote)
}, {
'beginfun':
lambda string: self.findCommand(string, r"begin", r"field"),
'endfun':
lambda string: self.findCommand(
string, r"end", r"field", warning=True),
'process':
lambda string: self.processLatexField(string, newNote)
}, {
'beginfun':
lambda string: self.findCommand(string, r"xfield", r"?"),
'endfun':
self.findClosingBrace,
'process':
lambda string: self.processLatexField(string, newNote)
}, {
'beginfun':
lambda string: self.findCommand(string, r"plain"),
'endfun':
lambda string: self.findCommand(string, r"endplain"),
'process':
lambda string: self.processPlainField(string, newNote)
}, {
'beginfun':
lambda string: self.findCommand(string, r"begin", r"plain"),
'endfun':
lambda string: self.findCommand(
string, r"end", r"plain", warning=True),
'process':
lambda string: self.processPlainField(string, newNote)
}, {
'beginfun':
lambda string: self.findCommand(string, r"xplain", r"?"),
'endfun':
self.findClosingBrace,
'process':
lambda string: self.processPlainField(string, newNote)
}, {
'beginfun':
lambda string: self.findCommand(string, r"tags", r"?"),
'endfun':
self.findClosingBrace,
'process':
lambda string: self.processTags(string, newNote)
}]
pieces, post = self.cutIntoPieces(noteString, fieldCommands)
for pre, value, ci in pieces:
self.rubbishList.append(pre)
fieldCommands[ci]['process'](value)
self.rubbishList.append(post)
self.noteList.append(newNote)