def test_note_from_file_with_bad_notebook_format(self):
path_to_test_note = os.path.join(THIS_DIR, 'data/note-with_bad_nb_format.md')
note = Note()
with self.assertRaises(RuntimeError) as cm:
note.from_file(path_to_test_note)
the_exception = cm.exception
self.assertEqual(the_exception.message, 'Bad notebook line format')
def new(title, notebook_name="personal", body=""):
"""
Create new note in notebook
:param title:
:param notebook_name: optional
:param body: optional
:return: new_note.id
"""
# Ensure that note with the same title does not exist already in the database
try:
note = Note.get(Note.title == title)
notification.show_error("Note with same name already exists", note.title)
raise Exception(f"Note with same name already exists")
except Note.DoesNotExist:
# All good, it should not exist
pass
# Retrieve notebook id
try:
notebook = Folder.get(Folder.title == notebook_name)
except Folder.DoesNotExist:
notification.show_error("Notebook not found", notebook_name)
raise Folder.DoesNotExist
# Create new note instance
# Get unique identifier following Joplin format
# Source: https://github.com/laurent22/joplin/issues/305#issuecomment-374123414
import uuid
uid = str(uuid.uuid4()).replace("-", "")
# Get current time in Joplin format (int epoch in milliseconds)
uint_current_timestamp_msec = time_joplin()
new_note = Note(
body=body,
created_time=uint_current_timestamp_msec,
id=uid,
parent=notebook.id,
source="pyjoplin",
source_application="pyjoplin",
title=title,
updated_time=uint_current_timestamp_msec,
user_created_time=uint_current_timestamp_msec,
user_updated_time=uint_current_timestamp_msec,
)
num_saved_notes = new_note.save(force_insert=True)
if num_saved_notes != 1:
notification.show_error_and_raise("Creating note", new_note.title)
if config.DO_NOTIFY:
notification.show("New note created in nb %s" % notebook.title, title)
return new_note.id
def imfeelinglucky(uid):
"""
Try to make most straightforward action on note,
similar in spirit to "I'm feeling lucky" feature in Google search.
:param uid:
:return:
"""
subprocess.Popen(
"increase_hit_history_for pyjoplin %s" % inspect.currentframe().f_code.co_name,
shell=True,
)
# NOTE: For now only implements searching first code stub in Solution:
# In other cases it could open url for "link-type" notes
# Find note entry by uid
note = Note.get(Note.id == uid)
# Parse first code stub (if any)
import re
stub = ""
if not stub:
# Typical example:
# # Solution...
# ... (one or several lines)
# ```<lang>
# <code_stub>
# ```
pattern_str = r"#?\s*Solution.*?```.*?\n(.*?)```"
m = re.search(pattern_str, note.body, re.DOTALL)
if m:
stub = m.group(1)
if not stub:
# Inline solution:
# # Solution...
# ... (one or several lines)
# `<code_stub>`
pattern_str = r"#?\s*Solution.*?`(.*?)`"
m = re.search(pattern_str, note.body, re.DOTALL)
if m:
stub = m.group(1)
if not stub:
notification.show("No code stub found, opening file", note.title)
edit(uid)
return None
# Strip newlines to avoid unexpected execution e.g. in shell
stub = stub.strip("\n")
# Copy stub into the clipboard
# NOTE: Using xclip because pyperclip does not work
# Create temporary file with text
path_tempfile = os.path.join(config.PATH_TEMP, "stub")
with open(path_tempfile, "w") as f:
f.write(stub)
cmd = "xclip -selection clipboard %s" % path_tempfile
subprocess.call(cmd, shell=True)
notification.show("Extracted code stub:", note.title, stub)
return stub
def rebuild_fts_index():
"""
Rebuild virtual table for FTS (fulltext search)
populating with all existing notes
:return:
"""
# Create empty FTS table from scratch
try:
# Remove table in case it existed
NoteIndex.drop_table()
except Exception:
pass
NoteIndex.create_table()
# Add all entries into virtual FTS table
notes = Note.select()
with db.atomic():
for note in notes:
NoteIndex.insert(
{
NoteIndex.uid: note.id,
NoteIndex.title: note.title,
NoteIndex.body: note.body,
}
).execute()
if config.DO_NOTIFY:
notification.show("Rebuilt index", message="FTS index populated from scratch")
def rename_conflicting_notes():
notes = Note.select().where(Note.is_conflict == 1)
with db.atomic():
for note in notes:
if not note.title.endswith("(CONFLICT)"):
note.title = note.title + " (CONFLICT)"
num_saved_notes = note.save()
if num_saved_notes != 1:
notification.show_error("Renaming conflicting note", note.title)
def edit_by_title(title):
if isinstance(title, list):
# For case coming from CLI
title = " ".join(title)
try:
note = Note.get(Note.title == title.rstrip())
edit(note.id)
except Note.DoesNotExist:
print("No exact match found for title query " + title)
def list_conflicts():
"""
List conflicting notes
:return:
"""
notes = Note.select().where(Note.is_conflict == 1)
print("List of conflicting notes:")
with db.atomic():
for note in notes:
print("-----------------------------")
print("%s %s" % (note.id, note.title))
def print_for_uid(uid):
"""
Print note content
:param uid:
:return: returncode
"""
try:
note = Note.get(Note.id == uid)
print(note.to_string(), end="")
return 0
except Note.DoesNotExist:
print(f"No note found with id {uid}")
return 2
def test_note_to_file_with_unicode(self):
test_title = 'pyjoplin_test test_note_to_file_with_unicode'
test_body = u"This is a unicode string: \u2192"
note_id = commands.new(test_title, 'test', body=test_body)
new_note = Note.get(Note.id == note_id)
path_tempfile = '/tmp/pyjoplin/test_note_to_file_with_unicode'
new_note.to_file(path_tempfile)
backup_body = new_note.body
new_note.from_file(path_tempfile)
self.assertEqual(new_note.body, backup_body)
def get_notes_by_id(ids, ordered=False):
# Find notes with id in uids
# Peewee: `x << y` stands for `x in y`
# NOTE: Order in query is unrelated to order in uids
query = Note.select().where(Note.id << ids).dicts()
# Debug: [note['id'] for note in query]
if ordered:
# Order query list in same order as uids
dict_query = {note["id"]: note for note in query}
# Grab note only if in the query
ordered_notes = [dict_query[id] for id in ids if id in dict_query]
query = ordered_notes
# Debug: [note['id'] for note in query]
return query
def find_empty_notes(delete=False):
"""
Find and report empty notes
Useful e.g. to find if some note was left empty due to synchronization issues
:return:
"""
notes = Note.select().order_by(Note.title)
print("Listing empty notes:")
with db.atomic():
for note in notes:
if not note.body:
print("%s %s" % (note.id, note.title))
if delete:
note.delete_instance()
print("Deleted")
def delete(uid):
# Find note entry by uid
note = Note.get(Note.id == uid)
note.delete_instance()
if config.DO_NOTIFY:
notification.show("Deleted note", note.title)
def find_title(title):
try:
note = Note.get(Note.title == title)
return note.id
except Note.DoesNotExist:
return None
def test_note_from_file(self):
path_to_test_note = os.path.join(THIS_DIR, 'data/note.md')
note = Note()
note.from_file(path_to_test_note)
def edit(uid):
subprocess.Popen(
"increase_hit_history_for pyjoplin %s" % inspect.currentframe().f_code.co_name,
shell=True,
)
# Find note entry by uid
note = Note.get(Note.id == uid)
# Save previous title and body for reference
init_title = note.title
init_body = note.body
# Populate temporary file from note content
path_sentinel = os.path.join(config.PATH_TEMP, f"{uid}")
if os.path.exists(path_sentinel):
notification.show_error("Note is already under edit", note.title, note.id)
raise Exception("Note is already under edit")
else:
# Create sentinel to block this note
open(path_sentinel, "a").close()
path_tempfile = os.path.join(config.PATH_TEMP, f"{note.title.replace('/', '_')}.md")
# Create named symlink to sentinel note
# for better readability
# NOTE:
# Because of this link, the sentinel is no longer *just* a sentinel.
# Instead, it is a plain text file containing the actual note content.
try:
os.symlink(path_sentinel, path_tempfile)
except FileExistsError:
notification.show_error(
"Note file with same name is already under edit", note.title, note.id
)
raise Exception(f"Temp file named {note.title} already exists")
note.to_file(path_tempfile)
# Open file with editor
# NOTE: Stop using template, this command gets too complicated for a single string
# It's better to have a list of inputs to build the last complicated command
# Nesting bash becomes necessary to execute source for non-interactive customization
# NOTE: Using vimx since aliases do not seem to apply in non-interactive (even if I define it inside bashrc!?)
proc = subprocess.Popen(
# [
# 'xfce4-terminal',
# '--disable-server',
# '--title',
# 'pyjoplin - {title}'.format(title=note.title),
# '-e',
# 'bash -c "source ~/.bashrc && unset PYTHONPATH && vimx \'{path}\' || vi \'{path}\'"'.format(path=path_tempfile)
# ],
[
"run-standalone.sh",
"bash",
"-c",
# NOTE: `stty -ixon` to disable "flow control characters" for vim-shell
# NOTE: `unset PYTHONPATH` seems necessary for this to work when called through ulauncher extension
f"stty -ixon && unset PYTHONPATH && vim -u ~/Code/python/pyjoplin/vim/vimrc '{path_tempfile}'"
# NOTE: Version below works when called from terminal,
# e.g. `pyjoplin edit 170b3c8de5034f7c8023a6a39f02219c`
# but it immediately exits when called via ulauncher
# 'vimx',
# '{path}'.format(path=path_tempfile)
],
env=dict(os.environ, OPT_TITLE=f"pyjoplin - {note.title}"),
stdout=subprocess.PIPE,
)
# cmd_str = config.EDITOR_CALL_TEMPLATE.format(title=note.title, path=path_tempfile)
# proc = subprocess.Popen(
# cmd_str, shell=True,
# # NOTE: This is needed for non-gedit editors, but DISPLAY seems to be giving issues
# # env=dict(
# # os.environ,
# # PYTHONPATH='', # avoid issues when calling Python3 scripts from Python2
# # DISPLAY=":0.0" # ensure some GUI apps catch the right display
# # ),
# stdout=subprocess.PIPE
# )
# Loop during edition to save incremental changes
import time
last_modification_time_sec = os.path.getmtime(path_tempfile)
try:
# Sync note under edition with database note in a loop
while proc.poll() is None:
time.sleep(0.5)
if os.path.getmtime(path_tempfile) > last_modification_time_sec:
last_modification_time_sec = os.path.getmtime(path_tempfile)
note.from_file(path_tempfile)
num_saved_notes = note.save()
if num_saved_notes != 1:
notification.show_error(
"Saving note changes during edition", note.title
)
# else:
# notification.show("Note saved", note.title)
except:
notification.show_error("During note edition sync loop", note.title)
raise Exception("ERROR during editor-database sync loop")
returncode = proc.wait()
if returncode != 0:
print("ERROR during edition")
print("Output:")
output = proc.stdout.read().decode("utf-8")
print(output)
notification.show_error(
"returncode from editor was not 0. Inspect note to sanity-check.",
note.title,
output,
)
raise Exception("ERROR during edition")
# Save edited file content to Notes table
note.from_file(path_tempfile)
# Cleanup temp files
# NOTE:
# If the editor finished with some error code
# and things didn't go well,
# we do NOT remove the sentinel.
# So we will get an error when trying to open the note again, showcasing the issue
# (even though you should have got an error notification, but just in case!)
os.remove(path_tempfile)
os.remove(path_sentinel)
# Check for note changes
if note.title == init_title and note.body == init_body:
# Changed nothing, no need to save
if config.DO_NOTIFY:
notification.show("Finished edition with no changes", note.title)
return
# Delete entry if empty
if (not note.title) and (not note.body):
note.delete_instance()
if config.DO_NOTIFY:
notification.show("Deleted note", init_title)
return
# Save note changes into database
# NOTE: FTS entry is automatically updated within .save()
num_saved_notes = note.save()
if num_saved_notes != 1:
notification.show_error_and_raise("Saving note changes", note.title)
if config.DO_NOTIFY:
notification.show("Note saved", note.title)
def test_note_from_file_with_bad_notebook_name(self):
path_to_test_note = os.path.join(THIS_DIR, 'data/note-with_bad_nb_name.md')
note = Note()
with self.assertRaises(Folder.DoesNotExist) as cm:
note.from_file(path_to_test_note)
def main():
"""Main program.
Parse command line, then iterate over files and directories under
rootdir and upload all files. Skips some temporary files and
directories, and avoids duplicate uploads by comparing size and
mtime with the server.
"""
# Get list of notes in Facebook notebook
# These will be skipped in synchronization
notebook = Folder.get(Folder.title == "fb")
fb_notebook_id = notebook.id
# NOTE: If fb folder does not exist, will throw FolderDoesNotExist
# TODO: Catch it
query = Note.select().where(Note.parent == fb_notebook_id)
with db.atomic():
fb_note_ids = [note.id for note in query]
args = parser.parse_args()
if sum([bool(b) for b in (args.yes, args.no, args.default)]) > 1:
print("At most one of --yes, --no, --default is allowed")
sys.exit(2)
folder = args.folder
rootdir = os.path.expanduser(args.rootdir)
print("Dropbox folder name:", folder)
print("Local directory:", rootdir)
if not os.path.exists(rootdir):
print(rootdir, "does not exist on your filesystem")
sys.exit(1)
elif not os.path.isdir(rootdir):
print(rootdir, "is not a folder on your filesystem")
sys.exit(1)
for dn, dirs, files in os.walk(rootdir):
subfolder = dn[len(rootdir):].strip(os.path.sep)
listing = list_folder(dbx, folder, subfolder)
print("Descending into", subfolder, "...")
# First do all the files.
for name in files:
fullname = os.path.join(dn, name)
if not isinstance(name, six.text_type):
name = name.decode("utf-8")
nname = unicodedata.normalize("NFC", name)
if name.startswith("."):
print("Skipping dot file:", name)
elif name.startswith("@") or name.endswith("~"):
print("Skipping temporary file:", name)
elif name.endswith(".pyc") or name.endswith(".pyo"):
print("Skipping generated file:", name)
elif nname in listing:
md = listing[nname]
mtime = os.path.getmtime(fullname)
mtime_dt = datetime.datetime(*time.gmtime(mtime)[:6])
size = os.path.getsize(fullname)
if (isinstance(md, dropbox.files.FileMetadata)
and mtime_dt == md.client_modified
and size == md.size):
print(name, "is already synced [stats match]")
else:
print(name, "exists with different stats, downloading")
res = download(dbx, folder, subfolder, name)
with open(fullname) as f:
data = f.read()
if res == data:
print(name, "is already synced [content match]")
else:
print(name, "has changed since last sync")
if yesno("Refresh %s" % name, False, args):
upload(dbx,
fullname,
folder,
subfolder,
name,
overwrite=True)
elif yesno("Upload %s" % name, True, args):
upload(dbx, fullname, folder, subfolder, name)
# Then choose which subdirectories to traverse.
keep = []
for name in dirs:
if name.startswith("."):
print("Skipping dot directory:", name)
elif name.startswith("@") or name.endswith("~"):
print("Skipping temporary directory:", name)
elif name == "__pycache__":
print("Skipping generated directory:", name)
elif yesno("Descend into %s" % name, True, args):
print("Keeping directory:", name)
keep.append(name)
else:
print("OK, skipping directory:", name)
dirs[:] = keep
from __future__ import print_function
from peewee import fn, Entity
from pathlib2 import Path
import subprocess
from termcolor import colored
from pyjoplin.models import Note, NoteIndex, database as db
from pyjoplin.configuration import config
from pyjoplin import notification
path_repo = Path.home() / 'Backup/joplin'
printc = lambda x: print(colored(x, 'cyan'))
notes = Note.select().order_by(Note.title)
print("Listing empty notes:")
for note in notes:
if not note.body:
printc('Empty: %s %s' % (note.id, note.title))
notefile = '%s.md' % note.id
cmd = 'git rev-list HEAD -- %s ' % notefile
out = subprocess.check_output(cmd, shell=True, cwd=str(path_repo))
for sha in out.strip().split('\n'):
cmd = 'git show %s:%s' % (sha, notefile)
note_content = subprocess.check_output(cmd, shell=True, cwd=str(path_repo))
filtered_lines = list()
for idx, line in enumerate(note_content.split('\n')):
if idx in [0, 1]:
continue