def clear_all_notes(self):
res, status = Simplenote(self.user, self.password).get_note_list()
if status == 0:
[
Simplenote(self.user, self.password).delete_note(n["key"])
for n in res
]
def test_simplenote_add_note_content(self):
res, status = Simplenote(self.user, self.password).add_note("new note")
if status == 0:
note, status = Simplenote(self.user,
self.password).get_note(res["key"])
if status == 0:
self.assertEqual("new note", note["content"])
def test_note_with_plus_signs(self):
note, status = Simplenote(self.user, self.password).add_note("++")
if status == 0:
note, status = Simplenote(self.user,
self.password).get_note(note["key"])
if status == 0:
self.assertEqual("++", note["content"])
class MySimpleNote(Simplenote):
def __init__(self,username,password,num=10000):
self._note_list = []
self._handle = Simplenote(username,password)
self._note_list,_ = self._handle.get_note_list(num)
self._get_note_content()
def get_notes(self):
for _note in self._note_list:
yield _note
def get_note_list(self,num):
MySimpleNote.note_list,_ = self._handle.get_note_list(num)
def print_note_list(self):
for note in self._note_list:
self.print_note(note)
def print_note(self,note):
for k,v in note.iteritems():
print k.ljust(20),v
print
def _get_note_content(self):
for _note in self._note_list:
key = _note['key']
content,status = self._handle.get_note(key)
_note['content'] = content['content']
_note['title'] = content['content'].split("\n")[0]
def get_tags_from_note(self,note):
return(note['tags'])
def save_notes(): user = '' pw = '' note = [] simple = Simplenote(user,pw) #Get auth token = simple.authenticate(user, pw) #Get list of notes print "Retrieving list" l = simple.get_note_list() length = len(l[0]) print length count = 0 print "Processing list.." while count < length: print count note += simple.get_note(l[0][count]['key']) count += 1 filename = t + '.txt' file = open(filename, 'w') #To track progress for item in note: print >> file, item file.close() #Dump note object to pickle for use by other programs import pickle pickle.dump(note, open( t + '.pickle', 'wb'))
class MySimpleNote(Simplenote):
def __init__(self, username, password, num=10000):
self._note_list = []
self._handle = Simplenote(username, password)
self._note_list, _ = self._handle.get_note_list(num)
self._get_note_content()
def get_notes(self):
for _note in self._note_list:
yield _note
def get_note_list(self, num):
MySimpleNote.note_list, _ = self._handle.get_note_list(num)
def print_note_list(self):
for note in self._note_list:
self.print_note(note)
def print_note(self, note):
for k, v in note.iteritems():
print k.ljust(20), v
print
def _get_note_content(self):
for _note in self._note_list:
key = _note['key']
content, status = self._handle.get_note(key)
_note['content'] = content['content']
_note['title'] = content['content'].split("\n")[0]
def get_tags_from_note(self, note):
return (note['tags'])
def save_notes():
user = ''
pw = ''
note = []
simple = Simplenote(user, pw)
#Get auth
token = simple.authenticate(user, pw)
#Get list of notes
print "Retrieving list"
l = simple.get_note_list()
length = len(l[0])
print length
count = 0
print "Processing list.."
while count < length:
print count
note += simple.get_note(l[0][count]['key'])
count += 1
filename = t + '.txt'
file = open(filename, 'w')
#To track progress
for item in note:
print >> file, item
file.close()
#Dump note object to pickle for use by other programs
import pickle
pickle.dump(note, open(t + '.pickle', 'wb'))
def savenotes(username, password, note_dir, num_notes, forcesave):
count = 0
sn = Simplenote(username, password)
if num_notes is not None:
output("Getting {0} notes".format(num_notes))
note_list = sn.get_note_list(num_notes)
else:
output("Getting all notes")
note_list = sn.get_note_list()
if note_list[1] == 0:
output("Get Note List successful")
output("Saving notes to {0}".format(note_dir))
for note in note_list[0]:
if note["deleted"] == 0 and (forcesave or newnoteversion(note["key"], note["version"])):
note_object = sn.get_note(note["key"])
if note_object[1] == 0:
note_content = note_object[0]["content"]
filename = savenote(note_content, note_dir)
logsave(note["key"], note["version"], note["syncnum"])
count += 1
output("Note {0} version {1} saved as '{2}'".format(note["key"], note["version"], filename))
else:
output("Unable to get note list. Are your credentials correct?", priority=1)
cleanup(1)
if count > 0:
output("{0} notes saved".format(count), priority=2)
def test_simplenote_update_note(self):
note = {}
note['key'] = self.first_note
note["content"] = "Updated Note."
note, status = Simplenote(self.user, self.password).update_note(note)
if status == 0:
note, status = Simplenote(self.user,
self.password).get_note(note["key"])
if status == 0:
self.assertEqual("Updated Note.",
note["content"].split('\n')[0])
def test_simplenote_trash_note(self):
if self.first_note != False:
note, status = Simplenote(self.user, self.password).trash_note(
self.first_note)
if status == 0:
self.assertEqual(1, note["deleted"])
if self.second_note != False:
note, status = Simplenote(self.user, self.password).trash_note(
self.second_note)
if status == 0:
self.assertEqual(1, note["deleted"])
def __init__(self, config):
utils.SubjectMixin.__init__(self)
self.config = config
# create db dir if it does not exist
if not os.path.exists(config.db_path):
os.mkdir(config.db_path)
self.db_path = config.db_path
now = time.time()
# now read all .json files from disk
fnlist = glob.glob(self.helper_key_to_fname('*'))
self.notes = {}
for fn in fnlist:
n = json.load(open(fn, 'rb'))
# we always have a localkey, also when we don't have a note['key'] yet (no sync)
localkey = os.path.splitext(os.path.basename(fn))[0]
self.notes[localkey] = n
# we maintain in memory a timestamp of the last save
# these notes have just been read, so at this moment
# they're in sync with the disc.
n['savedate'] = now
# initialise the simplenote instance we're going to use
# this does not yet need network access
self.simplenote = Simplenote(config.sn_username, config.sn_password)
# we'll use this to store which notes are currently being synced by
# the background thread, so we don't add them anew if they're still
# in progress. This variable is only used by the background thread.
self.threaded_syncing_keys = {}
# reading a variable or setting this variable is atomic
# so sync thread will write to it, main thread will only
# check it sometimes.
self.waiting_for_simplenote = False
# save and sync queue
self.q_save = Queue()
self.q_save_res = Queue()
thread_save = Thread(target=self.worker_save)
thread_save.setDaemon(True)
thread_save.start()
self.q_sync = Queue()
self.q_sync_res = Queue()
thread_sync = Thread(target=self.worker_sync)
thread_sync.setDaemon(True)
thread_sync.start()
def test_simplenote_get_list_length_longer_than_note_fetch_length(self):
while self.initial_note_count <= simplenote.simplenote.NOTE_FETCH_LENGTH + 1:
note, status = Simplenote(
self.user,
self.password).add_note("Note " +
str(self.initial_note_count + 1))
if status == 0:
self.initial_note_count += 1
res, status = Simplenote(self.user, self.password).get_note_list()
if status == 0:
self.assertTrue(len(res) > simplenote.simplenote.NOTE_FETCH_LENGTH)
def test_simplenote_first_note(self):
if self.first_note != False:
note, status = Simplenote(self.user,
self.password).get_note(self.first_note)
if status == 0:
self.assertTrue(type(note) == dict)
self.assertEqual("First Note.", note["content"].split('\n')[0])
def test_simplenote_delete_note(self):
if self.first_note != False:
note, status = Simplenote(self.user, self.password).delete_note(
self.first_note)
if status == 0:
note, status = Simplenote(self.user, self.password).get_note(
self.first_note)
self.assertEqual(-1, status)
if self.second_note != False:
note, status = Simplenote(self.user, self.password).delete_note(
self.second_note)
if status == 0:
note, status = Simplenote(self.user, self.password).get_note(
self.second_note)
self.assertEqual(-1, status)
def __init__(self, config):
utils.SubjectMixin.__init__(self)
self.config = config
# create db dir if it does not exist
if not os.path.exists(config.db_path):
os.mkdir(config.db_path)
self.db_path = config.db_path
now = time.time()
# now read all .json files from disk
fnlist = glob.glob(self.helper_key_to_fname('*'))
self.notes = {}
for fn in fnlist:
n = json.load(open(fn, 'rb'))
# we always have a localkey, also when we don't have a note['key'] yet (no sync)
localkey = os.path.splitext(os.path.basename(fn))[0]
self.notes[localkey] = n
# we maintain in memory a timestamp of the last save
# these notes have just been read, so at this moment
# they're in sync with the disc.
n['savedate'] = now
# initialise the simplenote instance we're going to use
# this does not yet need network access
self.simplenote = Simplenote(config.sn_username, config.sn_password)
# we'll use this to store which notes are currently being synced by
# the background thread, so we don't add them anew if they're still
# in progress. This variable is only used by the background thread.
self.threaded_syncing_keys = {}
# reading a variable or setting this variable is atomic
# so sync thread will write to it, main thread will only
# check it sometimes.
self.waiting_for_simplenote = False
# save and sync queue
self.q_save = Queue()
self.q_save_res = Queue()
thread_save = Thread(target=self.worker_save)
thread_save.setDaemon(True)
thread_save.start()
self.q_sync = Queue()
self.q_sync_res = Queue()
thread_sync = Thread(target=self.worker_sync)
thread_sync.setDaemon(True)
thread_sync.start()
def setUp(self):
self.user = "******"
self.password = "******"
self.clear_all_notes()
self.unicode_note = "∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ⎧⎡⎛┌─────┐⎞⎤⎫"
self.unicode_note_key = False
self.initial_note_count = 0
self.first_note = False
self.second_note = False
note, status = Simplenote(self.user,
self.password).add_note("First Note.")
if status == 0:
self.initial_note_count += 1
self.first_note = note['key']
note, status = Simplenote(self.user,
self.password).add_note("Second Note.")
if status == 0:
self.initial_note_count += 1
self.second_note = note['key']
note, status = Simplenote(self.user,
self.password).add_note(self.unicode_note)
if status == 0:
self.initial_note_count += 1
self.unicode_note_key = note['key']
def main():
argvs = sys.argv
argc = len(argvs)
if not argc == 2:
print 'Usage: # python %s filename' % argvs[0]
quit()
notes = parseNoteXML(argvs[1])
print "simplenote address:"
address = raw_input()
print "simplenote password:"******"Wrong simplenote address or password."
quit()
def main():
argvs = sys.argv
argc = len(argvs)
if not argc == 2:
print 'Usage: # python %s filename' % argvs[0]
quit()
notes = parseNoteXML(argvs[1])
print "simplenote address:"
address = raw_input()
print "simplenote password:"******"Wrong simplenote address or password."
quit()
def setUp(self):
try:
self.simplenote_instance = None
if not self.simplenote_instance:
self.user = "******"
self.password = "******"
self.simplenote_instance = Simplenote(self.user, self.password)
time.sleep(5) # delays for 5 seconds
self.clear_all_notes()
self.unicode_note = "∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ⎧⎡⎛┌─────┐⎞⎤⎫"
self.unicode_note_key = False
note, status = self.simplenote_instance.get_note_list()
self.initial_note_count = 0
self.tag_note_count = 0
self.first_note = False
self.second_note = False
note, status = self.simplenote_instance.add_note({"content": "First Note.", "tags": ["tag1"]})
if status == 0:
self.initial_note_count += 1
self.tag_note_count += 1
self.first_note = note['key']
else:
self.fail("Setup Failed - First note")
note, status = self.simplenote_instance.add_note({"content": "Second Note.", "tags": ["tag1", "tag2"]})
if status == 0:
self.initial_note_count += 1
self.tag_note_count += 1
self.second_note = note['key']
else:
self.fail("Setup Failed - Second note")
note, status = self.simplenote_instance.add_note(self.unicode_note)
if status == 0:
self.initial_note_count += 1
self.unicode_note_key = note['key']
else:
self.fail("Setup Failed - Unicode note")
except Exception as e:
self.fail("General setup fail")
def main():
"""The main function."""
parser = OptionParser(version='%prog v' + __version__)
parser.add_option('-c', '--config', default='config.ini',
help='Location of config file (default: %default)', metavar='FILE')
parser.add_option('-o', '--output', default='simplenotebak.json.txt',
help='Output file name (default: %default)', metavar='FILE')
(options, args) = parser.parse_args()
# set script's path and add '/' to end
script_path = os.path.abspath(os.path.dirname(sys.argv[0])) + '/'
if args:
print 'debug: you wanted to run command: ' + args[0]
config = SafeConfigParser()
# can pass multiple files to config.read but it merges them, which we don't want
if not config.read(options.config):
# could not read file, try the script's path
if not config.read(script_path + options.config):
# Still can't find it, error out
print 'Could not read any config file'
sys.exit(1)
email = config.get('simplenote', 'email')
password = config.get('simplenote', 'password')
sn = Simplenote(email, password)
if not sn.login():
print 'ERROR:', sn.last_error
sys.exit(1)
index = sn.full_index()
#index = sn.index(5)
if sn.has_error:
print 'ERROR:', sn.last_error
sys.exit(1)
#print '- index -'
#pp = pprint.PrettyPrinter(indent=4)
#pp.pprint(index)
print 'number of notes:', str(len(index))
notecount = float(len(index))
#print '- data -'
notes = []
i = 0
#for note_meta in index['data']:
for note_meta in index:
note = sn.note(note_meta['key'])
if sn.has_error:
print 'ERROR:', sn.last_error
sys.exit(1)
notes.append(note)
#pp.pprint(note)
i += 1
pb.progress(50, math.floor(float(i) / notecount * 100.0))
print 'Number of api calls:', str(sn.api_count)
# xml format
#xmlnotes = ''
#for note in notes:
# if 'xml' in locals():
# xml.append(dict_to_xml(note))
# else:
# xml = dict_to_xml(note)
#xmlnotes = '<?xml version="1.0" encoding="UTF-8"?>' + "\n"
#xmlnotes += ET.tostring(xml, encoding="UTF-8")
#print xmlnotes
# JSON format
jsonnotes = []
i = 0
for note in notes:
if note['deleted'] == 1:
continue
json_note_tmp = {'modifydate': format_date(float(note['modifydate']))}
json_note_tmp.update({'createdate': format_date(float(note['createdate']))})
json_note_tmp.update({'tags': note['tags']})
json_note_tmp.update({'systemtags': note['systemtags']})
json_note_tmp.update({'content': note['content']})
json_note_tmp.update({'key': note['key']})
jsonnotes.append(json_note_tmp)
#print json.dumps(jsonnotes)
fh = open(options.output, 'w')
fh.write(json.dumps(jsonnotes))
fh.close()
def test_simplenote_failed_auth(self):
s = Simplenote(self.user, "")
self.assertRaises(SimplenoteLoginFailed, s.get_token)
def main():
parser = OptionParser()
parser.add_option("--snort",
action="store",
type="string",
help="Import a new file SNORT to Simplenote")
parser.add_option(
"--sniff",
action="store",
nargs=2,
type="string",
help="Link a file with an already existing note in Simplenote",
metavar="<key> <filename>")
parser.add_option("--sneeze",
action="store",
nargs=2,
type="string",
help="Export an existing file from Simplenote",
metavar="<key> <filename>")
parser.add_option("--blow",
action="store",
type="string",
help="Roll back note of key BLOW to previous version")
parser.add_option("--sync",
help="Sync files in index",
default=False,
action='store_true')
parser.add_option("--hanky",
help="Use with --sync to perform a dry run",
default=False,
action='store_true')
parser.add_option("--snot",
help="List notes available for export (tagged snose)",
default=False,
action='store_true')
parser.add_option("--username",
action="store",
type="string",
help="Your Simplenote email address")
parser.add_option("--password",
action="store",
type="string",
help="Your Simplenote password")
(options, args) = parser.parse_args()
if not options.username or not options.password:
#Check to see if stored somewhere
try:
options.username = netrc.netrc().authenticators(
"simple-note.appspot.com")[0]
options.password = netrc.netrc().authenticators(
"simple-note.appspot.com")[2]
except IOError as e:
print 'Username and password must be supplied or exist .netrc file under domain of simple-note.appspot.com'
exit()
snclient = Simplenote(options.username, options.password)
if options.snort:
snort(snclient, options.snort)
elif options.sniff:
sniff(snclient, options.sniff[0], options.sniff[1])
elif options.sneeze:
sneeze(snclient, options.sneeze[0], options.sneeze[1])
elif options.blow:
blow(snclient, options.blow)
elif options.snot:
snot(snclient)
elif options.sync and options.hanky:
sync(snclient, True)
elif options.sync:
sync(snclient)
else:
print 'No options supplied'
from simplenote import Simplenote
from decimal import Decimal
import sys
username = sys.argv[1]
password = sys.argv[2]
simplenote = Simplenote(username, password)
noteresponse = simplenote.get_note_list()
last_update_file = open('notes/.last_update', 'r')
try:
last_update = Decimal(last_update_file.read())
except Exception as e:
last_update = Decimal('0')
last_update_file.close()
most_recently_updated = '0'
notes = noteresponse[0]
print("Checking %d notes..." % len(notes))
for note in notes:
if (Decimal(note['modifydate']) > last_update):
print(' Reading contents of %s' % note['key'])
note_data = simplenote.get_note(note['key'])
print(' Writing contents of %s' % note['key'])
filename = 'notes/%s' % note['key']
f = open(filename, 'w')
f.write(note_data[0]['content'])
f.close()
else:
print('Skipped %s, no changes.' % note['key'])
def grades_note_update(note_id):
simplenote = Simplenote(EMAIL,PASSWORD)
grades = simplenote.get_note(note_id)[0]
grades['content'] = NOTE_NAME + "\n" + get_grades() + "Last updated: " + str(datetime.now())
simplenote.update_note(grades)
class NotesDB(utils.SubjectMixin):
"""NotesDB will take care of the local notes database and syncing with SN.
"""
def __init__(self, config):
utils.SubjectMixin.__init__(self)
self.config = config
# create db dir if it does not exist
if not os.path.exists(config.db_path):
os.mkdir(config.db_path)
self.db_path = config.db_path
# create txt Notes dir if it does not exist
if self.config.notes_as_txt and not os.path.exists(config.txt_path):
os.mkdir(config.txt_path)
now = time.time()
# now read all .json files from disk
fnlist = glob.glob(self.helper_key_to_fname('*'))
txtlist = glob.glob(self.config.txt_path + '/*.txt')
txtlist += glob.glob(self.config.txt_path + '/*.mkdn')
# removing json files and force full full sync if using text files
# and none exists and json files are there
if self.config.notes_as_txt and not txtlist and fnlist:
logging.debug('Forcing resync: using text notes, first usage')
for fn in fnlist:
os.unlink(fn)
fnlist = []
self.notes = {}
if self.config.notes_as_txt:
self.titlelist = {}
for fn in fnlist:
try:
n = json.load(open(fn, 'rb'))
if self.config.notes_as_txt:
nt = utils.get_note_title_file(n)
tfn = os.path.join(self.config.txt_path, nt)
if os.path.isfile(tfn):
self.titlelist[n.get('key')] = nt
txtlist.remove(tfn)
if os.path.getmtime(tfn) > os.path.getmtime(fn):
logging.debug('Text note was changed: %s' % (fn, ))
#with open(tfn, mode='r') as f:
with codecs.open(tfn, mode='rb',
encoding='utf-8') as f:
c = f.read()
n['content'] = c
n['modifydate'] = os.path.getmtime(tfn)
else:
logging.debug('Deleting note : %s' % (fn, ))
if not self.config.simplenote_sync:
os.unlink(fn)
continue
else:
n['deleted'] = 1
n['modifydate'] = now
except ValueError, e:
logging.error('Error parsing %s: %s' % (fn, str(e)))
else:
# we always have a localkey, also when we don't have a note['key'] yet (no sync)
localkey = os.path.splitext(os.path.basename(fn))[0]
self.notes[localkey] = n
# we maintain in memory a timestamp of the last save
# these notes have just been read, so at this moment
# they're in sync with the disc.
n['savedate'] = now
if self.config.notes_as_txt:
for fn in txtlist:
logging.debug('New text note found : %s' % (fn), )
tfn = os.path.join(self.config.txt_path, fn)
#with open(tfn, mode='r') as f:
with codecs.open(tfn, mode='rb', encoding='utf-8') as f:
c = f.read()
nk = self.create_note(c)
nn = os.path.splitext(os.path.basename(fn))[0]
if nn != utils.get_note_title(self.notes[nk]):
self.notes[nk]['content'] = nn + "\n\n" + c
os.unlink(tfn)
# save and sync queue
self.q_save = Queue()
self.q_save_res = Queue()
thread_save = Thread(target=self.worker_save)
thread_save.setDaemon(True)
thread_save.start()
# initialise the simplenote instance we're going to use
# this does not yet need network access
if self.config.simplenote_sync:
self.simplenote = Simplenote(config.sn_username,
config.sn_password)
# we'll use this to store which notes are currently being synced by
# the background thread, so we don't add them anew if they're still
# in progress. This variable is only used by the background thread.
self.threaded_syncing_keys = {}
# reading a variable or setting this variable is atomic
# so sync thread will write to it, main thread will only
# check it sometimes.
self.waiting_for_simplenote = False
self.q_sync = Queue()
self.q_sync_res = Queue()
thread_sync = Thread(target=self.worker_sync)
thread_sync.setDaemon(True)
thread_sync.start()
def add_note(note):
simple = Simplenote(USER, PW)
#Get auth
token = simple.authenticate(USER, PW)
t = simple.add_note(note)
class TestSimplenote(unittest.TestCase):
def setUp(self):
try:
self.simplenote_instance = None
if not self.simplenote_instance:
self.user = "******"
self.password = "******"
self.simplenote_instance = Simplenote(self.user, self.password)
time.sleep(5) # delays for 5 seconds
self.clear_all_notes()
self.unicode_note = "∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ⎧⎡⎛┌─────┐⎞⎤⎫"
self.unicode_note_key = False
note, status = self.simplenote_instance.get_note_list()
self.initial_note_count = 0
self.tag_note_count = 0
self.first_note = False
self.second_note = False
note, status = self.simplenote_instance.add_note({"content": "First Note.", "tags": ["tag1"]})
if status == 0:
self.initial_note_count += 1
self.tag_note_count += 1
self.first_note = note['key']
else:
self.fail("Setup Failed - First note")
note, status = self.simplenote_instance.add_note({"content": "Second Note.", "tags": ["tag1", "tag2"]})
if status == 0:
self.initial_note_count += 1
self.tag_note_count += 1
self.second_note = note['key']
else:
self.fail("Setup Failed - Second note")
note, status = self.simplenote_instance.add_note(self.unicode_note)
if status == 0:
self.initial_note_count += 1
self.unicode_note_key = note['key']
else:
self.fail("Setup Failed - Unicode note")
except Exception as e:
self.fail("General setup fail")
def tearDown(self):
self.clear_all_notes()
def test_simplenote_auth(self):
token = self.simplenote_instance.get_token()
self.assertNotEqual(None, token)
def test_simplenote_failed_auth(self):
s = Simplenote(self.user, "")
self.assertRaises(SimplenoteLoginFailed, s.get_token)
# When get_list_length test fails unexpectedly it's due to a failure in
# teardown to clear all notes. Unfortunately there is no way to guarantee
# all notes are cleared. This test is more likely to fail as a result due
# to the assertEqual requirements The next test also tests get_note_list
# functionality so it makes no sense to let the whole suite fail so set as
# expected failure.
@unittest.expectedFailure
def test_simplenote_get_list_length(self):
res, status = self.simplenote_instance.get_note_list()
if status == 0:
self.assertEqual(self.initial_note_count, len(res))
else:
self.assertEqual(0, len(res))
def test_simplenote_get_list_length_longer_than_note_fetch_length(self):
while self.initial_note_count <= simplenote.simplenote.NOTE_FETCH_LENGTH+1:
note, status = self.simplenote_instance.add_note("Note "+str(self.initial_note_count+1))
if status == 0:
self.initial_note_count += 1
res, status = self.simplenote_instance.get_note_list()
if status == 0:
self.assertTrue(len(res) > simplenote.simplenote.NOTE_FETCH_LENGTH)
def test_simplenote_get_list_with_tags(self):
res, status = self.simplenote_instance.get_note_list(tags=["tag1"])
if status == 0:
self.assertEqual(self.tag_note_count, len(res))
else:
self.assertEqual(0, len(res))
def test_simplenote_first_note(self):
if self.first_note != False:
note, status = self.simplenote_instance.get_note(self.first_note)
if status == 0:
self.assertTrue(type(note) == dict)
self.assertEqual("First Note.", note["content"].split('\n')[0])
def test_simplenote_second_note(self):
if self.second_note != False:
note, status = self.simplenote_instance.get_note(self.second_note)
if status == 0:
self.assertTrue(type(note) == dict)
self.assertEqual("Second Note.", note["content"].split('\n')[0])
def test_simplenote_trash_note(self):
if self.first_note != False:
note, status = self.simplenote_instance.trash_note(self.first_note)
if status == 0:
self.assertEqual(1, note["deleted"])
if self.second_note != False:
note, status = self.simplenote_instance.trash_note(self.second_note)
if status == 0:
self.assertEqual(1, note["deleted"])
def test_simplenote_delete_note(self):
if self.first_note != False:
note, status = self.simplenote_instance.delete_note(self.first_note)
if status == 0:
note, status = self.simplenote_instance.get_note(self.first_note)
self.assertEqual(-1, status)
if self.second_note != False:
note, status = self.simplenote_instance.delete_note(self.second_note)
if status == 0:
note, status = self.simplenote_instance.get_note(self.second_note)
self.assertEqual(-1, status)
def test_simplenote_add_note_object(self):
res, status = self.simplenote_instance.add_note({"content":
"new note"})
if status == 0:
note, status = self.simplenote_instance.get_note(res["key"])
if status == 0:
self.assertEqual("new note", note["content"])
def test_simplenote_add_note_content(self):
res, status = self.simplenote_instance.add_note("new note")
if status == 0:
note, status = self.simplenote_instance.get_note(res["key"])
if status == 0:
self.assertEqual("new note", note["content"])
def test_simplenote_update_note(self):
note = {}
note['key'] = self.first_note
note["content"] = "Updated Note."
note, status = self.simplenote_instance.update_note(note)
if status == 0:
note, status = self.simplenote_instance.get_note(note["key"])
if status == 0:
self.assertEqual("Updated Note.", note["content"].split('\n')[0])
def test_simplenote_is_unicode(self):
if self.unicode_note_key != False:
note, status = self.simplenote_instance.get_note(self.unicode_note_key)
if status == 0:
self.assertTrue(self.is_utf8(note["content"]))
def test_note_with_plus_signs(self):
note, status = self.simplenote_instance.add_note("++")
if status == 0:
note, status = self.simplenote_instance.get_note(note["key"])
if status == 0:
self.assertEqual("++", note["content"])
def test_note_get_previous_version(self):
note_v1, status = self.simplenote_instance.add_note("Hello")
if status == 0:
note_v2 = {}
note_v2['key'] = note_v1["key"]
note_v2["content"] = "Goodbye"
note_v2, status = self.simplenote_instance.update_note(note_v2)
if status == 0:
if note_v2["version"] > 1:
note, status = self.simplenote_instance.get_note(note_v2["key"], note_v2["version"]-1)
if status == 0:
self.assertEqual("Hello", note["content"])
def is_utf8(self, s):
try:
s.decode('utf-8')
return True
except UnicodeDecodeError:
return False
def clear_all_notes(self):
res, status = self.simplenote_instance.get_note_list()
while (len(res) > 0) and (status == 0):
[self.simplenote_instance.delete_note(n["key"]) for n in res]
res, status = self.simplenote_instance.get_note_list()
def test_simplenote_is_unicode(self):
if self.unicode_note_key != False:
note, status = Simplenote(self.user, self.password).get_note(
self.unicode_note_key)
if status == 0:
self.assertTrue(self.is_utf8(note["content"]))
def test_simplenote_auth(self):
token = Simplenote(self.user, self.password).get_token()
self.assertNotEqual(None, token)
def __init__(self, username, password, num=10000):
self._note_list = []
self._handle = Simplenote(username, password)
self._note_list, _ = self._handle.get_note_list(num)
self._get_note_content()
class NotesDB(utils.SubjectMixin):
"""NotesDB will take care of the local notes database and syncing with SN.
"""
def __init__(self, config):
utils.SubjectMixin.__init__(self)
self.config = config
# create db dir if it does not exist
if not os.path.exists(config.db_path):
os.mkdir(config.db_path)
self.db_path = config.db_path
# create txt Notes dir if it does not exist
if self.config.notes_as_txt and not os.path.exists(config.txt_path):
os.mkdir(config.txt_path)
now = time.time()
# now read all .json files from disk
fnlist = glob.glob(self.helper_key_to_fname('*'))
txtlist = []
for ext in config.read_txt_extensions.split(','):
txtlist += glob.glob(
unicode(self.config.txt_path + '/*.' + ext, 'utf-8'))
# removing json files and force full full sync if using text files
# and none exists and json files are there
if self.config.notes_as_txt and not txtlist and fnlist:
logging.debug('Forcing resync: using text notes, first usage')
for fn in fnlist:
os.unlink(fn)
fnlist = []
self.notes = {}
if self.config.notes_as_txt:
self.titlelist = {}
for fn in fnlist:
try:
n = json.load(open(fn, 'rb'))
if self.config.notes_as_txt:
nt = utils.get_note_title_file(n)
tfn = os.path.join(self.config.txt_path, nt)
if os.path.isfile(tfn):
self.titlelist[n.get('key')] = nt
txtlist.remove(tfn)
if os.path.getmtime(tfn) > os.path.getmtime(fn):
logging.debug('Text note was changed: %s' % (fn, ))
with codecs.open(tfn, mode='rb',
encoding='utf-8') as f:
c = f.read()
n['content'] = c
n['modifydate'] = os.path.getmtime(tfn)
else:
logging.debug('Deleting note : %s' % (fn, ))
if not self.config.simplenote_sync:
os.unlink(fn)
continue
else:
n['deleted'] = 1
n['modifydate'] = now
except IOError as e:
logging.error('NotesDB_init: Error opening %s: %s' %
(fn, str(e)))
raise ReadError('Error opening note file')
except ValueError as e:
logging.error('NotesDB_init: Error reading %s: %s' %
(fn, str(e)))
raise ReadError('Error reading note file')
else:
# we always have a localkey, also when we don't have a note['key'] yet (no sync)
localkey = os.path.splitext(os.path.basename(fn))[0]
self.notes[localkey] = n
# we maintain in memory a timestamp of the last save
# these notes have just been read, so at this moment
# they're in sync with the disc.
n['savedate'] = now
if self.config.notes_as_txt:
for fn in txtlist:
logging.debug('New text note found : %s' % (fn), )
tfn = os.path.join(self.config.txt_path, fn)
try:
with codecs.open(tfn, mode='rb', encoding='utf-8') as f:
c = f.read()
except IOError as e:
logging.error('NotesDB_init: Error opening %s: %s' %
(fn, str(e)))
raise ReadError('Error opening note file')
except ValueError as e:
logging.error('NotesDB_init: Error reading %s: %s' %
(fn, str(e)))
raise ReadError('Error reading note file')
else:
nk = self.create_note(c)
nn = os.path.splitext(os.path.basename(fn))[0]
if nn != utils.get_note_title(self.notes[nk]):
self.notes[nk]['content'] = nn + "\n\n" + c
os.unlink(tfn)
# save and sync queue
self.q_save = Queue()
self.q_save_res = Queue()
thread_save = Thread(target=wrap_buggy_function(self.worker_save))
thread_save.setDaemon(True)
thread_save.start()
self.full_syncing = False
# initialise the simplenote instance we're going to use
# this does not yet need network access
if self.config.simplenote_sync:
self.simplenote = Simplenote(config.sn_username,
config.sn_password)
# we'll use this to store which notes are currently being synced by
# the background thread, so we don't add them anew if they're still
# in progress. This variable is only used by the background thread.
self.threaded_syncing_keys = {}
# reading a variable or setting this variable is atomic
# so sync thread will write to it, main thread will only
# check it sometimes.
self.waiting_for_simplenote = False
self.syncing_lock = Lock()
self.q_sync = Queue()
self.q_sync_res = Queue()
thread_sync = Thread(target=wrap_buggy_function(self.worker_sync))
thread_sync.setDaemon(True)
thread_sync.start()
def create_note(self, title):
# need to get a key unique to this database. not really important
# what it is, as long as it's unique.
new_key = utils.generate_random_key()
while new_key in self.notes:
new_key = utils.generate_random_key()
timestamp = time.time()
# note has no internal key yet.
new_note = {
'content': title,
'modifydate': timestamp,
'createdate': timestamp,
'savedate': 0, # never been written to disc
'syncdate': 0, # never been synced with server
'tags': []
}
self.notes[new_key] = new_note
return new_key
def delete_note(self, key):
n = self.notes[key]
n['deleted'] = 1
n['modifydate'] = time.time()
def filter_notes(self, search_string=None):
"""Return list of notes filtered with search string.
Based on the search mode that has been selected in self.config,
this method will call the appropriate helper method to do the
actual work of filtering the notes.
@param search_string: String that will be used for searching.
Different meaning depending on the search mode.
@return: notes filtered with selected search mode and sorted according
to configuration. Two more elements in tuple: a regular expression
that can be used for highlighting strings in the text widget; the
total number of notes in memory.
"""
if self.config.search_mode == 'regexp':
filtered_notes, match_regexp, active_notes = self.filter_notes_regexp(
search_string)
else:
filtered_notes, match_regexp, active_notes = self.filter_notes_gstyle(
search_string)
if self.config.sort_mode == 0:
if self.config.pinned_ontop == 0:
# sort alphabetically on title
filtered_notes.sort(key=lambda o: utils.get_note_title(o.note))
else:
filtered_notes.sort(key=utils.sort_key_by_title_pinned)
elif self.config.sort_mode == 2:
if self.config.pinned_ontop == 0:
# last modified on top
filtered_notes.sort(
key=lambda o: -float(o.note.get('createdate', 0)))
else:
filtered_notes.sort(key=utils.sort_key_by_create_date_pinned,
reverse=True)
else:
if self.config.pinned_ontop == 0:
# last modified on top
filtered_notes.sort(
key=lambda o: -float(o.note.get('modifydate', 0)))
else:
filtered_notes.sort(key=utils.sort_key_by_modify_date_pinned,
reverse=True)
return filtered_notes, match_regexp, active_notes
def _helper_gstyle_tagmatch(self, tag_pats, note):
if tag_pats:
tags = note.get('tags')
# tag: patterns specified, but note has no tags, so no match
if not tags:
return 0
# for each tag_pat, we have to find a matching tag
for tp in tag_pats:
# at the first match between tp and a tag:
if next((tag for tag in tags if tag.startswith(tp)),
None) is not None:
# we found a tag that matches current tagpat, so we move to the next tagpat
continue
else:
# we found no tag that matches current tagpat, so we break out of for loop
break
else:
# for loop never broke out due to no match for tagpat, so:
# all tag_pats could be matched, so note is a go.
return 1
# break out of for loop will have us end up here
# for one of the tag_pats we found no matching tag
return 0
else:
# match because no tag: patterns were specified
return 2
def _helper_gstyle_mswordmatch(self, msword_pats, content):
"""If all words / multi-words in msword_pats are found in the content,
the note goes through, otherwise not.
@param msword_pats:
@param content:
@return:
"""
# no search patterns, so note goes through
if not msword_pats:
return True
# search for the first p that does NOT occur in content
if next((p for p in msword_pats if p not in content), None) is None:
# we only found pats that DO occur in content so note goes through
return True
else:
# we found the first p that does not occur in content
return False
def filter_notes_gstyle(self, search_string=None):
filtered_notes = []
# total number of notes, excluding deleted
active_notes = 0
if not search_string:
for k in self.notes:
n = self.notes[k]
if not n.get('deleted'):
active_notes += 1
filtered_notes.append(
utils.KeyValueObject(key=k, note=n, tagfound=0))
return filtered_notes, [], active_notes
# group0: ag - not used
# group1: t(ag)?:([^\s]+)
# group2: multiple words in quotes
# group3: single words
# example result for 't:tag1 t:tag2 word1 "word2 word3" tag:tag3' ==
# [('', 'tag1', '', ''), ('', 'tag2', '', ''), ('', '', '', 'word1'), ('', '', 'word2 word3', ''), ('ag', 'tag3', '', '')]
groups = re.findall('t(ag)?:([^\s]+)|"([^"]+)"|([^\s]+)',
search_string)
tms_pats = [[] for _ in range(3)]
# we end up with [[tag_pats],[multi_word_pats],[single_word_pats]]
for gi in groups:
for mi in range(1, 4):
if gi[mi]:
tms_pats[mi - 1].append(gi[mi])
for k in self.notes:
n = self.notes[k]
if not n.get('deleted'):
active_notes += 1
c = n.get('content')
# case insensitive mode: WARNING - SLOW!
if not self.config.case_sensitive and c:
c = c.lower()
tagmatch = self._helper_gstyle_tagmatch(tms_pats[0], n)
# case insensitive mode: WARNING - SLOW!
msword_pats = tms_pats[1] + tms_pats[
2] if self.config.case_sensitive else [
p.lower() for p in tms_pats[1] + tms_pats[2]
]
if tagmatch and self._helper_gstyle_mswordmatch(
msword_pats, c):
# we have a note that can go through!
# tagmatch == 1 if a tag was specced and found
# tagmatch == 2 if no tag was specced (so all notes go through)
tagfound = 1 if tagmatch == 1 else 0
# we have to store our local key also
filtered_notes.append(
utils.KeyValueObject(key=k, note=n, tagfound=tagfound))
return filtered_notes, '|'.join(tms_pats[1] +
tms_pats[2]), active_notes
def filter_notes_regexp(self, search_string=None):
"""Return list of notes filtered with search_string,
a regular expression, each a tuple with (local_key, note).
"""
if search_string:
try:
if self.config.case_sensitive == 0:
sspat = re.compile(search_string, re.MULTILINE | re.I)
else:
sspat = re.compile(search_string, re.MULTILINE)
except re.error:
sspat = None
else:
sspat = None
filtered_notes = []
# total number of notes, excluding deleted ones
active_notes = 0
for k in self.notes:
n = self.notes[k]
# we don't do anything with deleted notes (yet)
if n.get('deleted'):
continue
active_notes += 1
c = n.get('content')
if self.config.search_tags == 1:
t = n.get('tags')
if sspat:
# this used to use a filter(), but that would by definition
# test all elements, whereas we can stop when the first
# matching element is found
# now I'm using this awesome trick by Alex Martelli on
# http://stackoverflow.com/a/2748753/532513
# first parameter of next is a generator
# next() executes one step, but due to the if, this will
# either be first matching element or None (second param)
if t and next(
(ti
for ti in t if sspat.search(ti)), None) is not None:
# we have to store our local key also
filtered_notes.append(
utils.KeyValueObject(key=k, note=n, tagfound=1))
elif sspat.search(c):
# we have to store our local key also
filtered_notes.append(
utils.KeyValueObject(key=k, note=n, tagfound=0))
else:
# we have to store our local key also
filtered_notes.append(
utils.KeyValueObject(key=k, note=n, tagfound=0))
else:
if (not sspat or sspat.search(c)):
# we have to store our local key also
filtered_notes.append(
utils.KeyValueObject(key=k, note=n, tagfound=0))
match_regexp = search_string if sspat else ''
return filtered_notes, match_regexp, active_notes
def get_note(self, key):
return self.notes[key]
def get_note_content(self, key):
return self.notes[key].get('content')
def get_note_status(self, key):
o = utils.KeyValueObject(saved=False,
synced=False,
modified=False,
full_syncing=self.full_syncing)
if key is None:
return o
n = self.notes[key]
modifydate = float(n['modifydate'])
savedate = float(n['savedate'])
if savedate > modifydate:
o.saved = True
else:
o.modified = True
if float(n['syncdate']) > modifydate:
o.synced = True
return o
def get_save_queue_len(self):
return self.q_save.qsize()
def get_sync_queue_len(self):
return self.q_sync.qsize()
def helper_key_to_fname(self, k):
return os.path.join(self.db_path, k) + '.json'
def helper_save_note(self, k, note):
"""Save a single note to disc.
"""
if self.config.notes_as_txt:
t = utils.get_note_title_file(note)
if t and not note.get('deleted'):
if k in self.titlelist:
logging.debug('Writing note : %s %s' %
(t, self.titlelist[k]))
if self.titlelist[k] != t:
dfn = os.path.join(self.config.txt_path,
self.titlelist[k])
if os.path.isfile(dfn):
logging.debug('Delete file %s ' % (dfn, ))
os.unlink(dfn)
else:
logging.debug('File not exits %s ' % (dfn, ))
else:
logging.debug('Key not in list %s ' % (k, ))
self.titlelist[k] = t
fn = os.path.join(self.config.txt_path, t)
try:
with codecs.open(fn, mode='wb', encoding='utf-8') as f:
c = note.get('content')
if isinstance(c, str):
c = unicode(c, 'utf-8')
else:
c = unicode(c)
f.write(c)
except IOError as e:
logging.error('NotesDB_save: Error opening %s: %s' %
(fn, str(e)))
raise WriteError('Error opening note file')
except ValueError as e:
logging.error('NotesDB_save: Error writing %s: %s' %
(fn, str(e)))
raise WriteError('Error writing note file')
elif t and note.get('deleted') and k in self.titlelist:
dfn = os.path.join(self.config.txt_path, self.titlelist[k])
if os.path.isfile(dfn):
logging.debug('Delete file %s ' % (dfn, ))
os.unlink(dfn)
fn = self.helper_key_to_fname(k)
if not self.config.simplenote_sync and note.get('deleted'):
if os.path.isfile(fn):
os.unlink(fn)
else:
json.dump(note, codecs.open(fn, 'wb', encoding='utf-8'), indent=2)
# record that we saved this to disc.
note['savedate'] = time.time()
def sync_note_unthreaded(self, k):
"""Sync a single note with the server.
Update existing note in memory with the returned data.
This is a sychronous (blocking) call.
"""
note = self.notes[k]
if Note(note).need_sync_to_server:
# update to server
result = self.update_note_to_server(note)
if result.error_object is None:
# success!
n = result.note
# if content was unchanged, there'll be no content sent back!
new_content = 'content' in n
now = time.time()
# 1. store when we've synced
n['syncdate'] = now
# update our existing note in-place!
note.update(n)
# return the key
return (k, new_content)
else:
return None
else:
# our note is synced up, but we check if server has something new for us
gret = self.simplenote.get_note(note['key'])
if gret[1] == 0:
n = gret[0]
if Note(n).is_newer_than(note):
n['syncdate'] = time.time()
note.update(n)
return (k, True)
else:
return (k, False)
else:
return None
def save_threaded(self):
for k, n in self.notes.items():
if Note(n).need_save:
cn = copy.deepcopy(n)
# put it on my queue as a save
o = utils.KeyValueObject(action=ACTION_SAVE, key=k, note=cn)
self.q_save.put(o)
# in this same call, we process stuff that might have been put on the result queue
nsaved = 0
something_in_queue = True
while something_in_queue:
try:
o = self.q_save_res.get_nowait()
except Empty:
something_in_queue = False
else:
# o (.action, .key, .note) is something that was written to disk
# we only record the savedate.
self.notes[o.key]['savedate'] = o.note['savedate']
self.notify_observers(
'change:note-status',
utils.KeyValueObject(what='savedate', key=o.key))
nsaved += 1
return nsaved
def sync_to_server_threaded(self, wait_for_idle=True):
"""Only sync notes that have been changed / created locally since previous sync.
This function is called by the housekeeping handler, so once every
few seconds.
@param wait_for_idle: Usually, last modification date has to be more
than a few seconds ago before a sync to server is attempted. If
wait_for_idle is set to False, no waiting is applied. Used by exit
cleanup in controller.
"""
# this many seconds of idle time (i.e. modification this long ago)
# before we try to sync.
if wait_for_idle:
lastmod = 3
else:
lastmod = 0
now = time.time()
for k, n in self.notes.items():
# if note has been modified since the sync, we need to sync.
# only do so if note hasn't been touched for 3 seconds
# and if this note isn't still in the queue to be processed by the
# worker (this last one very important)
modifydate = float(n.get('modifydate', -1))
syncdate = float(n.get('syncdate', -1))
if modifydate > syncdate and \
now - modifydate > lastmod and \
k not in self.threaded_syncing_keys:
# record that we've requested a sync on this note,
# so that we don't keep on putting stuff on the queue.
self.threaded_syncing_keys[k] = True
cn = copy.deepcopy(n)
# we store the timestamp when this copy was made as the syncdate
cn['syncdate'] = time.time()
# put it on my queue as a sync
o = utils.KeyValueObject(action=ACTION_SYNC_PARTIAL_TO_SERVER,
key=k,
note=cn)
self.q_sync.put(o)
# in this same call, we read out the result queue
nsynced = 0
nerrored = 0
something_in_queue = True
while something_in_queue:
try:
o = self.q_sync_res.get_nowait()
except Empty:
something_in_queue = False
else:
okey = o.key
if o.error:
nerrored += 1
else:
# o (.action, .key, .note) is something that was synced
# we only apply the changes if the syncdate is newer than
# what we already have, since the main thread could be
# running a full sync whilst the worker thread is putting
# results in the queue.
if float(o.note['syncdate']) > float(
self.notes[okey]['syncdate']):
old_note = copy.deepcopy(self.notes[okey])
if float(o.note['syncdate']) > float(
self.notes[okey]['modifydate']):
# note was synced AFTER the last modification to our local version
# do an in-place update of the existing note
# this could be with or without new content.
self.notes[okey].update(o.note)
else:
# the user has changed stuff since the version that got synced
# just record version that we got from simplenote
# if we don't do this, merging problems start happening.
# VERY importantly: also store the key. It
# could be that we've just created the
# note, but that the user continued
# typing. We need to store the new server
# key, else we'll keep on sending new
# notes.
tkeys = ['version', 'syncdate', 'key']
for tk in tkeys:
self.notes[okey][tk] = o.note[tk]
# notify anyone (probably nvPY) that this note has been changed
self.notify_observers(
'synced:note',
utils.KeyValueObject(lkey=okey, old_note=old_note))
nsynced += 1
self.notify_observers(
'change:note-status',
utils.KeyValueObject(what='syncdate', key=okey))
# after having handled the note that just came back,
# we can take it from this blocker dict
del self.threaded_syncing_keys[okey]
return (nsynced, nerrored)
def sync_full_threaded(self):
def wrapper():
try:
sync_from_server_errors = self.sync_full_unthreaded()
self.notify_observers(
'complete:sync_full',
utils.KeyValueObject(errors=sync_from_server_errors))
except Exception as e:
self.notify_observers(
'error:sync_full',
utils.KeyValueObject(error=e, exc_info=sys.exc_info()))
raise
thread_sync_full = Thread(target=wrap_buggy_function(wrapper))
thread_sync_full.setDaemon(True)
thread_sync_full.start()
def sync_full_unthreaded(self):
"""Perform a full bi-directional sync with server.
After this, it could be that local keys have been changed, so
reset any views that you might have.
"""
try:
self.syncing_lock.acquire()
self.full_syncing = True
local_deletes = {}
now = time.time()
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(msg='Starting full sync.'))
# 1. Synchronize notes when it has locally changed.
# In this phase, synchronized all notes from client to server.
for ni, lk in enumerate(self.notes.keys()):
n = self.notes[lk]
if Note(n).need_sync_to_server:
result = self.update_note_to_server(n)
if result.error_object is None:
# replace n with result.note.
# if this was a new note, our local key is not valid anymore
del self.notes[lk]
# in either case (new or existing note), save note at assigned key
k = result.note.get('key')
# we merge the note we got back (content could be empty!)
n.update(result.note)
# and put it at the new key slot
self.notes[k] = n
# record that we just synced
n['syncdate'] = now
# whatever the case may be, k is now updated
self.helper_save_note(k, self.notes[k])
if lk != k:
# if lk was a different (purely local) key, should be deleted
local_deletes[lk] = True
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Synced modified note %d to server.' %
(ni, )))
else:
key = n.get('key') or lk
raise SyncError(
"Sync step 1 error - Could not update note {0} to server: {1}"
.format(key, str(result.error_object)))
# 2. Retrieves full note list from server.
# In phase 2 to 5, synchronized all notes from server to client.
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg=
'Retrieving full note list from server, could take a while.'
))
self.waiting_for_simplenote = True
nl = self.simplenote.get_note_list()
self.waiting_for_simplenote = False
if nl[1] == 0:
nl = nl[0]
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Retrieved full note list from server.'))
else:
raise SyncError('Could not get note list from server.')
# 3. Delete local notes not included in full note list.
server_keys = {}
for n in nl:
k = n.get('key')
server_keys[k] = True
for lk in list(self.notes.keys()):
if lk not in server_keys:
if self.notes[lk]['syncdate'] == 0:
# This note MUST NOT delete because it was created during phase 1 or phase 2.
continue
if self.config.notes_as_txt:
tfn = os.path.join(
self.config.txt_path,
utils.get_note_title_file(self.notes[lk]))
if os.path.isfile(tfn):
os.unlink(tfn)
del self.notes[lk]
local_deletes[lk] = True
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(msg='Deleted note %d.' %
(len(local_deletes))))
# 4. Update local notes.
lennl = len(nl)
sync_from_server_errors = 0
for ni, n in enumerate(nl):
k = n.get('key')
if k in self.notes:
# n is already exists in local.
if Note(n).is_newer_than(self.notes[k]):
# We must update local note with remote note.
err = 0
if 'content' not in n:
# The content field is missing. Get all data from server.
self.waiting_for_simplenote = True
n, err = self.simplenote.get_note(k)
self.waiting_for_simplenote = False
if err == 0:
self.notes[k].update(n)
self.notes[k]['syncdate'] = now
self.helper_save_note(k, self.notes[k])
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Synced newer note %d (%d) from server.'
% (ni, lennl)))
else:
logging.error(
'Error syncing newer note %s from server: %s' %
(k, err))
sync_from_server_errors += 1
else:
# n is new note.
# We must save it in local.
err = 0
if 'content' not in n:
# The content field is missing. Get all data from server.
self.waiting_for_simplenote = True
n, err = self.simplenote.get_note(k)
self.waiting_for_simplenote = False
if err == 0:
self.notes[k] = n
self.notes[k][
'savedate'] = 0 # never been written to disc
self.notes[k]['syncdate'] = now
self.helper_save_note(k, self.notes[k])
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Synced new note %d (%d) from server.' %
(ni, lennl)))
else:
logging.error(
'Error syncing new note %s from server: %s' %
(k, err))
sync_from_server_errors += 1
# 5. Clean up local notes.
for dk in local_deletes.keys():
fn = self.helper_key_to_fname(dk)
if os.path.exists(fn):
os.unlink(fn)
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(msg='Full sync complete.'))
self.full_syncing = False
return sync_from_server_errors
finally:
self.full_syncing = False
self.syncing_lock.release()
def set_note_content(self, key, content):
n = self.notes[key]
old_content = n.get('content')
if content != old_content:
n['content'] = content
n['modifydate'] = time.time()
self.notify_observers(
'change:note-status',
utils.KeyValueObject(what='modifydate', key=key))
def set_note_tags(self, key, tags):
n = self.notes[key]
old_tags = n.get('tags')
tags = utils.sanitise_tags(tags)
if tags != old_tags:
n['tags'] = tags
n['modifydate'] = time.time()
self.notify_observers(
'change:note-status',
utils.KeyValueObject(what='modifydate', key=key))
def delete_note_tag(self, key, tag):
note = self.notes[key]
note_tags = note.get('tags')
note_tags.remove(tag)
note['tags'] = note_tags
note['modifydate'] = time.time()
self.notify_observers('change:note-status',
utils.KeyValueObject(what='modifydate', key=key))
def add_note_tags(self, key, comma_seperated_tags):
note = self.notes[key]
note_tags = note.get('tags')
new_tags = utils.sanitise_tags(comma_seperated_tags)
note_tags.extend(new_tags)
note['tags'] = note_tags
note['modifydate'] = time.time()
self.notify_observers('change:note-status',
utils.KeyValueObject(what='modifydate', key=key))
def set_note_pinned(self, key, pinned):
n = self.notes[key]
old_pinned = utils.note_pinned(n)
if pinned != old_pinned:
if 'systemtags' not in n:
n['systemtags'] = []
systemtags = n['systemtags']
if pinned:
# which by definition means that it was NOT pinned
systemtags.append('pinned')
else:
systemtags.remove('pinned')
n['modifydate'] = time.time()
self.notify_observers(
'change:note-status',
utils.KeyValueObject(what='modifydate', key=key))
def is_different_note(self, local_note, remote_note):
# for keeping original data.
local_note = dict(local_note)
remote_note = dict(remote_note)
del local_note['savedate']
del local_note['syncdate']
# convert to hashable objects.
for k, v in local_note.items():
if isinstance(v, list):
local_note[k] = tuple(v)
for k, v in remote_note.items():
if isinstance(v, list):
remote_note[k] = tuple(v)
# it will returns an empty set() if each notes is equals.
return set(local_note.items()) ^ set(remote_note.items())
def worker_save(self):
while True:
o = self.q_save.get()
if o.action == ACTION_SAVE:
# this will write the savedate into o.note
# with filename o.key.json
try:
self.helper_save_note(o.key, o.note)
except WriteError as e:
logging.error('FATAL ERROR in access to file system')
print("FATAL ERROR: Check the nvpy.log")
os._exit(1)
else:
# put the whole thing back into the result q
# now we don't have to copy, because this thread
# is never going to use o again.
# somebody has to read out the queue...
self.q_save_res.put(o)
def worker_sync(self):
self.syncing_lock.acquire()
while True:
if self.q_sync.empty():
self.syncing_lock.release()
o = self.q_sync.get()
self.syncing_lock.acquire()
else:
o = self.q_sync.get()
if o.key not in self.threaded_syncing_keys:
# this note was already synced by sync_full thread.
continue
if o.action == ACTION_SYNC_PARTIAL_TO_SERVER:
if 'key' in o.note:
logging.debug(
'Updating note %s (local key %s) to server.' %
(o.note['key'], o.key))
else:
logging.debug(
'Sending new note (local key %s) to server.' %
(o.key, ))
result = self.update_note_to_server(o.note)
if result.error_object is None:
if not result.is_updated:
o.error = 0
self.q_sync_res.put(o)
continue
n = result.note
if not n.get('content', None):
# if note has not been changed, we don't get content back
# delete our own copy too.
del o.note['content']
# syncdate was set when the note was copied into our queue
# we rely on that to determine when a returned note should
# overwrite a note in the main list.
# store the actual note back into o
# in-place update of our existing note copy
o.note.update(n)
# success!
o.error = 0
# and put it on the result queue
self.q_sync_res.put(o)
else:
o.error = 1
self.q_sync_res.put(o)
def update_note_to_server(self, note):
"""Update the note to simplenote server.
:return: UpdateResult object
"""
try:
self.waiting_for_simplenote = True
o, err = self.simplenote.update_note(note)
self.waiting_for_simplenote = False
if err == 0:
# success!
# Keeps the internal fields of nvpy.
new_note = dict(note)
new_note.update(o)
logging.debug('Server replies with updated note ' +
new_note['key'])
return UpdateResult(
note=new_note,
is_updated=True,
error_object=None,
)
elif 'key' in note:
update_error = o
# note has already been saved on the simplenote server.
self.waiting_for_simplenote = True
o, err = self.simplenote.get_note(note['key'])
self.waiting_for_simplenote = False
if err == 0:
local_note = note
remote_note = o
if not self.is_different_note(local_note, remote_note):
# got an error response when updating the note.
# however, the remote note has been updated.
# this phenomenon is rarely occurs.
# if it occurs, housekeeper's is going to repeatedly update this note.
# regard updating error as success for prevent this problem.
logging.info(
'Regard updating error (local key %s, error object %s) as success.'
% (o.key, repr(update_error)))
return UpdateResult(
note=local_note,
is_updated=False,
error_object=None,
)
return UpdateResult(
note=None,
is_updated=False,
error_object=o,
)
except httplib.HTTPException as e:
# workaround for https://github.com/mrtazz/simplenote.py/issues/24
return UpdateResult(
note=None,
is_updated=False,
error_object=e,
)
def main():
"""The main function."""
parser = OptionParser(version='%prog v' + __version__)
parser.add_option('-c', '--config', default='config.ini',
help='Location of config file (default: %default)', metavar='FILE')
parser.add_option('-o', '--output', default='simplenotebak.json.txt',
help='Output file name (default: %default)', metavar='FILE')
(options, args) = parser.parse_args()
log = logging.getLogger('sn')
# get script's path
script_path = os.path.abspath(os.path.dirname(sys.argv[0]))
appdir = AppDirs('simplenote-cli')
local_cache = os.path.join(appdir.user_data_dir, 'data.cache')
if args:
log.debug('you wanted to run command: {}'.format(args[0]))
config = RawConfigParser()
# can pass multiple files to config.read but it merges them, which we don't
# want. Order here:
#
# - exact value of -c option (ie config.ini)
# - xdg_config_dir (ie ~/.config/vrillusions/simplenote-cli)
# - script path, where this file resides
# 1st try exact value
if not config.read(options.config):
log.info('could not read %s' % options.config)
# next try xdg config dir
xdg_config_file = os.path.join(appdir.user_config_dir, 'config.ini')
if not config.read(xdg_config_file):
log.info('could not read %s' % xdg_config_file)
# next try script path
script_config_file = os.path.join(script_path, 'config.ini')
if not config.read(script_config_file):
log.info('could not read %s' % script_config_file)
# Still can't find it, error out
log.critical('could not read any config file')
return 1
email = config.get('simplenote', 'email')
password = config.get('simplenote', 'password')
# TODO: GET PATH TO CACHE FILE (PROBABLY THROUGH XDG)
#sn = Simplenote(email, password, cache_file)
sn = Simplenote(email, password)
sn.login()
index = sn.full_index()
#index = sn.index(5)
#print '- index -'
#pp = pprint.PrettyPrinter(indent=4)
#pp.pprint(index)
log.info('number of notes: {}'.format(len(index)))
notecount = float(len(index))
#print '- data -'
notes = []
i = 0
#for note_meta in index['data']:
for note_meta in index:
note = sn.note(note_meta['key'])
notes.append(note)
#pp.pprint(note)
i += 1
pb.progress(50, math.floor(float(i) / notecount * 100.0))
log.debug('Number of api calls: {}'.format(sn.api_count))
# xml format
#xmlnotes = ''
#for note in notes:
# if 'xml' in locals():
# xml.append(dict_to_xml(note))
# else:
# xml = dict_to_xml(note)
#xmlnotes = '<?xml version="1.0" encoding="UTF-8"?>' + "\n"
#xmlnotes += ET.tostring(xml, encoding="UTF-8")
#print xmlnotes
# JSON format
jsonnotes = []
i = 0
for note in notes:
if note['deleted'] == 1:
continue
json_note_tmp = {'modifydate': format_date(float(note['modifydate']))}
json_note_tmp.update({'createdate': format_date(float(note['createdate']))})
json_note_tmp.update({'tags': note['tags']})
json_note_tmp.update({'systemtags': note['systemtags']})
json_note_tmp.update({'content': note['content']})
json_note_tmp.update({'key': note['key']})
jsonnotes.append(json_note_tmp)
#print json.dumps(jsonnotes)
with open(options.output, 'w') as fh:
fh.write(json.dumps(jsonnotes))
def grades_note_setup():
simplenote = Simplenote(EMAIL,PASSWORD)
return simplenote.add_note(NOTE_NAME)[0]['key']
from simplenote import Simplenote
from decimal import Decimal
import sys
username = sys.argv[1]
password = sys.argv[2]
simplenote = Simplenote(username, password)
noteresponse = simplenote.get_note_list()
last_update_file = open('notes/.last_update', 'r')
try:
last_update = Decimal(last_update_file.read())
except Exception as e:
last_update = Decimal('0')
last_update_file.close()
most_recently_updated = '0'
notes = noteresponse[0]
print "Checking %d notes..." % len(notes)
for note in notes:
if (Decimal(note['modifydate']) > last_update):
print ' Reading contents of %s' % note['key']
note_data = simplenote.get_note(note['key'])
print ' Writing contents of %s' % note['key']
filename = 'notes/%s' % note['key']
f = open(filename, 'w')
f.write(note_data[0]['content'])
f.close()
else:
from simplenote import Simplenote
from decimal import Decimal
import sys
username = sys.argv[1]
password = sys.argv[2]
simplenote = Simplenote(username, password)
# Retrieve timestamp from .last_update
last_update = Decimal('0')
try:
with open('notes/.last_update', 'r') as last_update_file:
last_update = Decimal(last_update_file.read())
except IOError:
pass
# Get note list (with content)
noteresponse = simplenote.get_note_list()
notes = noteresponse[0]
# Write notes to files
print("Checking %d notes..." % len(notes))
most_recently_updated = last_update
for note in notes:
if (Decimal(note['modificationDate']) > last_update):
print(' Writing contents of %s' % note['key'])
filename = 'notes/%s' % note['key']
with open(filename, 'w') as f:
f.write(note['content'])
else:
#A simple Python utility to upload grade information to simple note in real time
#Author: Eden Zik
from simplenote import Simplenote
from datetime import datetime
from sage import get_grades
EMAIL = '???@???.com' #Simplenote email
PASSWORD = '******' #Simplenote password
simplenote = Simplenote(EMAIL, PASSWORD)
GRADES_NOTE_ID = 'bf634100f62811e4a205db0538ecac0d' #Note ID of Grades Note
grades = simplenote.get_note(GRADES_NOTE_ID)[0]
grades['content'] = "Grades\n" + get_grades() + "Last updated: " + str(
datetime.now())
simplenote.update_note(grades)
class NotesDB(utils.SubjectMixin):
"""NotesDB will take care of the local notes database and syncing with SN.
"""
def __init__(self, config):
utils.SubjectMixin.__init__(self)
self.config = config
# create db dir if it does not exist
if not os.path.exists(config.db_path):
os.mkdir(config.db_path)
self.db_path = config.db_path
now = time.time()
# now read all .json files from disk
fnlist = glob.glob(self.helper_key_to_fname('*'))
self.notes = {}
for fn in fnlist:
n = json.load(open(fn, 'rb'))
# we always have a localkey, also when we don't have a note['key'] yet (no sync)
localkey = os.path.splitext(os.path.basename(fn))[0]
self.notes[localkey] = n
# we maintain in memory a timestamp of the last save
# these notes have just been read, so at this moment
# they're in sync with the disc.
n['savedate'] = now
# initialise the simplenote instance we're going to use
# this does not yet need network access
self.simplenote = Simplenote(config.sn_username, config.sn_password)
# we'll use this to store which notes are currently being synced by
# the background thread, so we don't add them anew if they're still
# in progress. This variable is only used by the background thread.
self.threaded_syncing_keys = {}
# reading a variable or setting this variable is atomic
# so sync thread will write to it, main thread will only
# check it sometimes.
self.waiting_for_simplenote = False
# save and sync queue
self.q_save = Queue()
self.q_save_res = Queue()
thread_save = Thread(target=self.worker_save)
thread_save.setDaemon(True)
thread_save.start()
self.q_sync = Queue()
self.q_sync_res = Queue()
thread_sync = Thread(target=self.worker_sync)
thread_sync.setDaemon(True)
thread_sync.start()
def create_note(self, title):
# need to get a key unique to this database. not really important
# what it is, as long as it's unique.
new_key = utils.generate_random_key()
while new_key in self.notes:
new_key = utils.generate_random_key()
timestamp = time.time()
# note has no internal key yet.
new_note = {
'content': title,
'modifydate': timestamp,
'createdate': timestamp,
'savedate': 0, # never been written to disc
'syncdate': 0 # never been synced with server
}
self.notes[new_key] = new_note
return new_key
def delete_note(self, key):
n = self.notes[key]
n['deleted'] = 1
n['modifydate'] = time.time()
def filter_notes(self, search_string=None):
"""Return list of notes filtered with search_string,
a regular expression, each a tuple with (local_key, note).
"""
if search_string:
try:
sspat = re.compile(search_string)
except re.error:
sspat = None
else:
sspat = None
filtered_notes = []
for k in self.notes:
n = self.notes[k]
c = n.get('content')
if not n.get('deleted') and (not sspat or sspat.search(c)):
# we have to store our local key also
filtered_notes.append(utils.KeyValueObject(key=k, note=n))
if self.config.sort_mode == 0:
# sort alphabetically on title
filtered_notes.sort(key=lambda o: utils.get_note_title(o.note))
else:
# last modified on top
filtered_notes.sort(
key=lambda o: -float(o.note.get('modifydate', 0)))
return filtered_notes
def get_note_content(self, key):
return self.notes[key].get('content')
def get_note_status(self, key):
n = self.notes[key]
o = utils.KeyValueObject(saved=False, synced=False, modified=False)
modifydate = float(n['modifydate'])
savedate = float(n['savedate'])
if savedate > modifydate:
o.saved = True
else:
o.modified = True
if float(n['syncdate']) > modifydate:
o.synced = True
return o
def get_save_queue_len(self):
return self.q_save.qsize()
def get_sync_queue_len(self):
return self.q_sync.qsize()
def helper_key_to_fname(self, k):
return os.path.join(self.db_path, k) + '.json'
def helper_save_note(self, k, note):
"""Save a single note to disc.
"""
fn = self.helper_key_to_fname(k)
json.dump(note, open(fn, 'wb'), indent=2)
# record that we saved this to disc.
note['savedate'] = time.time()
def sync_note_unthreaded(self, k):
"""Sync a single note with the server.
Update existing note in memory with the returned data.
This is a sychronous (blocking) call.
"""
note = self.notes[k]
if not note.get('key') or float(note.get('modifydate')) > float(
note.get('syncdate')):
# if has no key, or it has been modified sync last sync,
# update to server
uret = self.simplenote.update_note(note)
if uret[1] == 0:
# success!
n = uret[0]
# if content was unchanged, there'll be no content sent back!
if n.get('content', None):
new_content = True
else:
new_content = False
now = time.time()
# 1. store when we've synced
n['syncdate'] = now
# update our existing note in-place!
note.update(n)
# return the key
return (k, new_content)
else:
return None
else:
# our note is synced up, but we check if server has something new for us
gret = self.simplenote.get_note(note['key'])
if gret[1] == 0:
n = gret[0]
if int(n.get('syncnum')) > int(note.get('syncnum')):
n['syncdate'] = time.time()
note.update(n)
return (k, True)
else:
return (k, False)
else:
return None
def save_threaded(self):
for k, n in self.notes.items():
savedate = float(n.get('savedate'))
if float(n.get('modifydate')) > savedate or \
float(n.get('syncdate')) > savedate:
cn = copy.deepcopy(n)
# put it on my queue as a save
o = utils.KeyValueObject(action=ACTION_SAVE, key=k, note=cn)
self.q_save.put(o)
# in this same call, we process stuff that might have been put on the result queue
nsaved = 0
something_in_queue = True
while something_in_queue:
try:
o = self.q_save_res.get_nowait()
except Empty:
something_in_queue = False
else:
# o (.action, .key, .note) is something that was written to disk
# we only record the savedate.
self.notes[o.key]['savedate'] = o.note['savedate']
self.notify_observers(
'change:note-status',
utils.KeyValueObject(what='savedate', key=o.key))
nsaved += 1
return nsaved
def sync_to_server_threaded(self, wait_for_idle=True):
"""Only sync notes that have been changed / created locally since previous sync.
This function is called by the housekeeping handler, so once every
few seconds.
@param wait_for_idle: Usually, last modification date has to be more
than a few seconds ago before a sync to server is attempted. If
wait_for_idle is set to False, no waiting is applied. Used by exit
cleanup in controller.
"""
# this many seconds of idle time (i.e. modification this long ago)
# before we try to sync.
if wait_for_idle:
lastmod = 3
else:
lastmod = 0
now = time.time()
for k, n in self.notes.items():
# if note has been modified sinc the sync, we need to sync.
# only do so if note hasn't been touched for 3 seconds
# and if this note isn't still in the queue to be processed by the
# worker (this last one very important)
modifydate = float(n.get('modifydate', -1))
syncdate = float(n.get('syncdate', -1))
if modifydate > syncdate and \
now - modifydate > lastmod and \
k not in self.threaded_syncing_keys:
# record that we've requested a sync on this note,
# so that we don't keep on putting stuff on the queue.
self.threaded_syncing_keys[k] = True
cn = copy.deepcopy(n)
# we store the timestamp when this copy was made as the syncdate
cn['syncdate'] = time.time()
# put it on my queue as a sync
o = utils.KeyValueObject(action=ACTION_SYNC_PARTIAL_TO_SERVER,
key=k,
note=cn)
self.q_sync.put(o)
# in this same call, we read out the result queue
nsynced = 0
nerrored = 0
something_in_queue = True
while something_in_queue:
try:
o = self.q_sync_res.get_nowait()
except Empty:
something_in_queue = False
else:
okey = o.key
# this has come back.
del self.threaded_syncing_keys[okey]
if o.error:
nerrored += 1
else:
# o (.action, .key, .note) is something that was synced
# we only apply the changes if the syncdate is newer than
# what we already have, since the main thread could be
# running a full sync whilst the worker thread is putting
# results in the queue.
if float(o.note['syncdate']) > float(
self.notes[okey]['syncdate']):
if float(o.note['syncdate']) > float(
self.notes[okey]['modifydate']):
# note was synced AFTER the last modification to our local version
# do an in-place update of the existing note
# this could be with or without new content.
old_note = copy.deepcopy(self.notes[okey])
self.notes[okey].update(o.note)
# notify anyone (probably nvPY) that this note has been changed
self.notify_observers(
'synced:note',
utils.KeyValueObject(lkey=okey,
old_note=old_note))
else:
# the user has changed stuff since the version that got synced
# just record syncnum and version that we got from simplenote
# if we don't do this, merging problems start happening.
tkeys = ['syncnum', 'version', 'syncdate']
for tk in tkeys:
self.notes[okey][tk] = o.note[tk]
nsynced += 1
self.notify_observers(
'change:note-status',
utils.KeyValueObject(what='syncdate', key=okey))
return (nsynced, nerrored)
def sync_full(self):
"""Perform a full bi-directional sync with server.
This follows the recipe in the SimpleNote 2.0 API documentation.
After this, it could be that local keys have been changed, so
reset any views that you might have.
"""
local_updates = {}
local_deletes = {}
now = time.time()
self.notify_observers('progress:sync_full',
utils.KeyValueObject(msg='Starting full sync.'))
# 1. go through local notes, if anything changed or new, update to server
for ni, lk in enumerate(self.notes.keys()):
n = self.notes[lk]
if not n.get('key') or float(n.get('modifydate')) > float(
n.get('syncdate')):
uret = self.simplenote.update_note(n)
if uret[1] == 0:
# replace n with uret[0]
# if this was a new note, our local key is not valid anymore
del self.notes[lk]
# in either case (new or existing note), save note at assigned key
k = uret[0].get('key')
# we merge the note we got back (content coud be empty!)
n.update(uret[0])
# and put it at the new key slot
self.notes[k] = n
# record that we just synced
uret[0]['syncdate'] = now
# whatever the case may be, k is now updated
local_updates[k] = True
if lk != k:
# if lk was a different (purely local) key, should be deleted
local_deletes[lk] = True
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Synced modified note %d to server.' % (ni, )))
else:
raise SyncError(
"Sync step 1 error: Could not update note to server.")
# 2. if remote syncnum > local syncnum, update our note; if key is new, add note to local.
# this gets the FULL note list, even if multiple gets are required
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Retrieving full note list from server, could take a while.'
))
nl = self.simplenote.get_note_list()
if nl[1] == 0:
nl = nl[0]
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Retrieved full note list from server.'))
else:
raise SyncError('Could not get note list from server.')
server_keys = {}
lennl = len(nl)
for ni, n in enumerate(nl):
k = n.get('key')
server_keys[k] = True
if k in self.notes:
# we already have this
# check if server n has a newer syncnum than mine
if int(n.get('syncnum')) > int(self.notes[k].get(
'syncnum', -1)):
# and the server is newer
ret = self.simplenote.get_note(k)
if ret[1] == 0:
self.notes[k].update(ret[0])
local_updates[k] = True
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Synced newer note %d (%d) from server.' %
(ni, lennl)))
else:
# new note
ret = self.simplenote.get_note(k)
if ret[1] == 0:
self.notes[k] = ret[0]
local_updates[k] = True
self.notify_observers(
'progress:sync_full',
utils.KeyValueObject(
msg='Synced new note %d (%d) from server.' %
(ni, lennl)))
# in both cases, new or newer note, syncdate is now.
self.notes[k]['syncdate'] = now
# 3. for each local note not in server index, remove.
for lk in self.notes.keys():
if lk not in server_keys:
del self.notes[lk]
local_deletes[lk] = True
# sync done, now write changes to db_path
for uk in local_updates.keys():
self.helper_save_note(uk, self.notes[uk])
for dk in local_deletes.keys():
fn = self.helper_key_to_fname(dk)
if os.path.exists(fn):
os.unlink(fn)
self.notify_observers('progress:sync_full',
utils.KeyValueObject(msg='Full sync complete.'))
def set_note_content(self, key, content):
n = self.notes[key]
old_content = n.get('content')
if content != old_content:
n['content'] = content
n['modifydate'] = time.time()
self.notify_observers(
'change:note-status',
utils.KeyValueObject(what='modifydate', key=key))
def worker_save(self):
while True:
o = self.q_save.get()
if o.action == ACTION_SAVE:
# this will write the savedate into o.note
# with filename o.key.json
self.helper_save_note(o.key, o.note)
# put the whole thing back into the result q
# now we don't have to copy, because this thread
# is never going to use o again.
# somebody has to read out the queue...
self.q_save_res.put(o)
def worker_sync(self):
while True:
o = self.q_sync.get()
if o.action == ACTION_SYNC_PARTIAL_TO_SERVER:
self.waiting_for_simplenote = True
uret = self.simplenote.update_note(o.note)
self.waiting_for_simplenote = False
if uret[1] == 0:
# success!
n = uret[0]
if not n.get('content', None):
# if note has not been changed, we don't get content back
# delete our own copy too.
del o.note['content']
# syncdate was set when the note was copied into our queue
# we rely on that to determine when a returned note should
# overwrite a note in the main list.
# store the actual note back into o
# in-place update of our existing note copy
o.note.update(n)
# success!
o.error = 0
# and put it on the result queue
self.q_sync_res.put(o)
else:
o.error = 1
self.q_sync_res.put(o)
def __init__(self,username,password,num=10000):
self._note_list = []
self._handle = Simplenote(username,password)
self._note_list,_ = self._handle.get_note_list(num)
self._get_note_content()
pickleread()
last_synch_finish = d
picklewrite()
def last_synch_read(): # this is start of some future code
(last_synch_finish, notes, name_keys) = pickleread()
# local_changed = list of files mod since last_synch_finish # filename and moddate
# cloud_changed = list of notes mod since last_synch_finish # initially just key and moddate
for f in local_changed:
if f not in name_keys.keys():
push_new(f)
from simplenote import Simplenote # http://pypi.python.org/pypi/simplenote/0.2.0
simplenote = Simplenote(username, password)
def cloud_list_create():
(notes, status) = simplenote.get_note_list() # have to get the whole list
if __name__ == '__main__':
cloud_raw_list_grab()
# map_create()
# moddate_compare()
# last_synch_hack()
map_show()
# push_local_to_cloud()
# dedupe_and_map_create(True)
# map_dupe_check()
def test_simplenote_get_list_length(self):
res, status = Simplenote(self.user, self.password).get_note_list()
if status == 0:
self.assertEqual(self.initial_note_count, len(res))
else:
self.assertEqual(0, len(res))
#A simple Python utility to upload grade information to simple note in real time #Author: Eden Zik from simplenote import Simplenote from datetime import datetime from sage import get_grades EMAIL = '???@???.com' #Simplenote email PASSWORD = '******' #Simplenote password simplenote = Simplenote(EMAIL,PASSWORD) GRADES_NOTE_ID = 'bf634100f62811e4a205db0538ecac0d' #Note ID of Grades Note grades = simplenote.get_note(GRADES_NOTE_ID)[0] grades['content'] = "Grades\n" + get_grades() + "Last updated: " + str(datetime.now()) simplenote.update_note(grades)
class NotesDB(utils.SubjectMixin):
"""NotesDB will take care of the local notes database and syncing with SN.
"""
def __init__(self, config):
utils.SubjectMixin.__init__(self)
self.config = config
# create db dir if it does not exist
if not os.path.exists(config.db_path):
os.mkdir(config.db_path)
self.db_path = config.db_path
now = time.time()
# now read all .json files from disk
fnlist = glob.glob(self.helper_key_to_fname('*'))
self.notes = {}
for fn in fnlist:
n = json.load(open(fn, 'rb'))
# we always have a localkey, also when we don't have a note['key'] yet (no sync)
localkey = os.path.splitext(os.path.basename(fn))[0]
self.notes[localkey] = n
# we maintain in memory a timestamp of the last save
# these notes have just been read, so at this moment
# they're in sync with the disc.
n['savedate'] = now
# initialise the simplenote instance we're going to use
# this does not yet need network access
self.simplenote = Simplenote(config.sn_username, config.sn_password)
# we'll use this to store which notes are currently being synced by
# the background thread, so we don't add them anew if they're still
# in progress. This variable is only used by the background thread.
self.threaded_syncing_keys = {}
# reading a variable or setting this variable is atomic
# so sync thread will write to it, main thread will only
# check it sometimes.
self.waiting_for_simplenote = False
# save and sync queue
self.q_save = Queue()
self.q_save_res = Queue()
thread_save = Thread(target=self.worker_save)
thread_save.setDaemon(True)
thread_save.start()
self.q_sync = Queue()
self.q_sync_res = Queue()
thread_sync = Thread(target=self.worker_sync)
thread_sync.setDaemon(True)
thread_sync.start()
def create_note(self, title):
# need to get a key unique to this database. not really important
# what it is, as long as it's unique.
new_key = utils.generate_random_key()
while new_key in self.notes:
new_key = utils.generate_random_key()
timestamp = time.time()
# note has no internal key yet.
new_note = {
'content' : title,
'modifydate' : timestamp,
'createdate' : timestamp,
'savedate' : 0, # never been written to disc
'syncdate' : 0 # never been synced with server
}
self.notes[new_key] = new_note
return new_key
def delete_note(self, key):
n = self.notes[key]
n['deleted'] = 1
n['modifydate'] = time.time()
def filter_notes(self, search_string=None):
"""Return list of notes filtered with search_string,
a regular expression, each a tuple with (local_key, note).
"""
if search_string:
try:
sspat = re.compile(search_string)
except re.error:
sspat = None
else:
sspat = None
filtered_notes = []
for k in self.notes:
n = self.notes[k]
c = n.get('content')
if not n.get('deleted') and (not sspat or sspat.search(c)):
# we have to store our local key also
filtered_notes.append(utils.KeyValueObject(key=k, note=n))
if self.config.sort_mode == 0:
# sort alphabetically on title
filtered_notes.sort(key=lambda o: utils.get_note_title(o.note))
else:
# last modified on top
filtered_notes.sort(key=lambda o: -float(o.note.get('modifydate', 0)))
return filtered_notes
def get_note_content(self, key):
return self.notes[key].get('content')
def get_note_status(self, key):
n = self.notes[key]
o = utils.KeyValueObject(saved=False, synced=False, modified=False)
modifydate = float(n['modifydate'])
savedate = float(n['savedate'])
if savedate > modifydate:
o.saved = True
else:
o.modified = True
if float(n['syncdate']) > modifydate:
o.synced = True
return o
def get_save_queue_len(self):
return self.q_save.qsize()
def get_sync_queue_len(self):
return self.q_sync.qsize()
def helper_key_to_fname(self, k):
return os.path.join(self.db_path, k) + '.json'
def helper_save_note(self, k, note):
"""Save a single note to disc.
"""
fn = self.helper_key_to_fname(k)
json.dump(note, open(fn, 'wb'), indent=2)
# record that we saved this to disc.
note['savedate'] = time.time()
def sync_note_unthreaded(self, k):
"""Sync a single note with the server.
Update existing note in memory with the returned data.
This is a sychronous (blocking) call.
"""
note = self.notes[k]
if not note.get('key') or float(note.get('modifydate')) > float(note.get('syncdate')):
# if has no key, or it has been modified sync last sync,
# update to server
uret = self.simplenote.update_note(note)
if uret[1] == 0:
# success!
n = uret[0]
# if content was unchanged, there'll be no content sent back!
if n.get('content', None):
new_content = True
else:
new_content = False
now = time.time()
# 1. store when we've synced
n['syncdate'] = now
# update our existing note in-place!
note.update(n)
# return the key
return (k, new_content)
else:
return None
else:
# our note is synced up, but we check if server has something new for us
gret = self.simplenote.get_note(note['key'])
if gret[1] == 0:
n = gret[0]
if int(n.get('syncnum')) > int(note.get('syncnum')):
n['syncdate'] = time.time()
note.update(n)
return (k, True)
else:
return (k, False)
else:
return None
def save_threaded(self):
for k,n in self.notes.items():
savedate = float(n.get('savedate'))
if float(n.get('modifydate')) > savedate or \
float(n.get('syncdate')) > savedate:
cn = copy.deepcopy(n)
# put it on my queue as a save
o = utils.KeyValueObject(action=ACTION_SAVE, key=k, note=cn)
self.q_save.put(o)
# in this same call, we process stuff that might have been put on the result queue
nsaved = 0
something_in_queue = True
while something_in_queue:
try:
o = self.q_save_res.get_nowait()
except Empty:
something_in_queue = False
else:
# o (.action, .key, .note) is something that was written to disk
# we only record the savedate.
self.notes[o.key]['savedate'] = o.note['savedate']
self.notify_observers('change:note-status', utils.KeyValueObject(what='savedate',key=o.key))
nsaved += 1
return nsaved
def sync_to_server_threaded(self, wait_for_idle=True):
"""Only sync notes that have been changed / created locally since previous sync.
This function is called by the housekeeping handler, so once every
few seconds.
@param wait_for_idle: Usually, last modification date has to be more
than a few seconds ago before a sync to server is attempted. If
wait_for_idle is set to False, no waiting is applied. Used by exit
cleanup in controller.
"""
# this many seconds of idle time (i.e. modification this long ago)
# before we try to sync.
if wait_for_idle:
lastmod = 3
else:
lastmod = 0
now = time.time()
for k,n in self.notes.items():
# if note has been modified sinc the sync, we need to sync.
# only do so if note hasn't been touched for 3 seconds
# and if this note isn't still in the queue to be processed by the
# worker (this last one very important)
modifydate = float(n.get('modifydate', -1))
syncdate = float(n.get('syncdate', -1))
if modifydate > syncdate and \
now - modifydate > lastmod and \
k not in self.threaded_syncing_keys:
# record that we've requested a sync on this note,
# so that we don't keep on putting stuff on the queue.
self.threaded_syncing_keys[k] = True
cn = copy.deepcopy(n)
# we store the timestamp when this copy was made as the syncdate
cn['syncdate'] = time.time()
# put it on my queue as a sync
o = utils.KeyValueObject(action=ACTION_SYNC_PARTIAL_TO_SERVER, key=k, note=cn)
self.q_sync.put(o)
# in this same call, we read out the result queue
nsynced = 0
nerrored = 0
something_in_queue = True
while something_in_queue:
try:
o = self.q_sync_res.get_nowait()
except Empty:
something_in_queue = False
else:
okey = o.key
# this has come back.
del self.threaded_syncing_keys[okey]
if o.error:
nerrored += 1
else:
# o (.action, .key, .note) is something that was synced
# we only apply the changes if the syncdate is newer than
# what we already have, since the main thread could be
# running a full sync whilst the worker thread is putting
# results in the queue.
if float(o.note['syncdate']) > float(self.notes[okey]['syncdate']):
if float(o.note['syncdate']) > float(self.notes[okey]['modifydate']):
# note was synced AFTER the last modification to our local version
# do an in-place update of the existing note
# this could be with or without new content.
old_note = copy.deepcopy(self.notes[okey])
self.notes[okey].update(o.note)
# notify anyone (probably nvPY) that this note has been changed
self.notify_observers('synced:note', utils.KeyValueObject(lkey=okey, old_note=old_note))
else:
# the user has changed stuff since the version that got synced
# just record syncnum and version that we got from simplenote
# if we don't do this, merging problems start happening.
tkeys = ['syncnum', 'version', 'syncdate']
for tk in tkeys:
self.notes[okey][tk] = o.note[tk]
nsynced += 1
self.notify_observers('change:note-status', utils.KeyValueObject(what='syncdate',key=okey))
return (nsynced, nerrored)
def sync_full(self):
"""Perform a full bi-directional sync with server.
This follows the recipe in the SimpleNote 2.0 API documentation.
After this, it could be that local keys have been changed, so
reset any views that you might have.
"""
local_updates = {}
local_deletes = {}
now = time.time()
self.notify_observers('progress:sync_full', utils.KeyValueObject(msg='Starting full sync.'))
# 1. go through local notes, if anything changed or new, update to server
for ni,lk in enumerate(self.notes.keys()):
n = self.notes[lk]
if not n.get('key') or float(n.get('modifydate')) > float(n.get('syncdate')):
uret = self.simplenote.update_note(n)
if uret[1] == 0:
# replace n with uret[0]
# if this was a new note, our local key is not valid anymore
del self.notes[lk]
# in either case (new or existing note), save note at assigned key
k = uret[0].get('key')
# we merge the note we got back (content coud be empty!)
n.update(uret[0])
# and put it at the new key slot
self.notes[k] = n
# record that we just synced
uret[0]['syncdate'] = now
# whatever the case may be, k is now updated
local_updates[k] = True
if lk != k:
# if lk was a different (purely local) key, should be deleted
local_deletes[lk] = True
self.notify_observers('progress:sync_full', utils.KeyValueObject(msg='Synced modified note %d to server.' % (ni,)))
else:
raise SyncError("Sync step 1 error: Could not update note to server.")
# 2. if remote syncnum > local syncnum, update our note; if key is new, add note to local.
# this gets the FULL note list, even if multiple gets are required
self.notify_observers('progress:sync_full', utils.KeyValueObject(msg='Retrieving full note list from server, could take a while.'))
nl = self.simplenote.get_note_list()
if nl[1] == 0:
nl = nl[0]
self.notify_observers('progress:sync_full', utils.KeyValueObject(msg='Retrieved full note list from server.'))
else:
raise SyncError('Could not get note list from server.')
server_keys = {}
lennl = len(nl)
for ni,n in enumerate(nl):
k = n.get('key')
server_keys[k] = True
if k in self.notes:
# we already have this
# check if server n has a newer syncnum than mine
if int(n.get('syncnum')) > int(self.notes[k].get('syncnum', -1)):
# and the server is newer
ret = self.simplenote.get_note(k)
if ret[1] == 0:
self.notes[k].update(ret[0])
local_updates[k] = True
self.notify_observers('progress:sync_full', utils.KeyValueObject(msg='Synced newer note %d (%d) from server.' % (ni,lennl)))
else:
# new note
ret = self.simplenote.get_note(k)
if ret[1] == 0:
self.notes[k] = ret[0]
local_updates[k] = True
self.notify_observers('progress:sync_full', utils.KeyValueObject(msg='Synced new note %d (%d) from server.' % (ni,lennl)))
# in both cases, new or newer note, syncdate is now.
self.notes[k]['syncdate'] = now
# 3. for each local note not in server index, remove.
for lk in self.notes.keys():
if lk not in server_keys:
del self.notes[lk]
local_deletes[lk] = True
# sync done, now write changes to db_path
for uk in local_updates.keys():
self.helper_save_note(uk, self.notes[uk])
for dk in local_deletes.keys():
fn = self.helper_key_to_fname(dk)
if os.path.exists(fn):
os.unlink(fn)
self.notify_observers('progress:sync_full', utils.KeyValueObject(msg='Full sync complete.'))
def set_note_content(self, key, content):
n = self.notes[key]
old_content = n.get('content')
if content != old_content:
n['content'] = content
n['modifydate'] = time.time()
self.notify_observers('change:note-status', utils.KeyValueObject(what='modifydate', key=key))
def worker_save(self):
while True:
o = self.q_save.get()
if o.action == ACTION_SAVE:
# this will write the savedate into o.note
# with filename o.key.json
self.helper_save_note(o.key, o.note)
# put the whole thing back into the result q
# now we don't have to copy, because this thread
# is never going to use o again.
# somebody has to read out the queue...
self.q_save_res.put(o)
def worker_sync(self):
while True:
o = self.q_sync.get()
if o.action == ACTION_SYNC_PARTIAL_TO_SERVER:
self.waiting_for_simplenote = True
uret = self.simplenote.update_note(o.note)
self.waiting_for_simplenote = False
if uret[1] == 0:
# success!
n = uret[0]
if not n.get('content', None):
# if note has not been changed, we don't get content back
# delete our own copy too.
del o.note['content']
# syncdate was set when the note was copied into our queue
# we rely on that to determine when a returned note should
# overwrite a note in the main list.
# store the actual note back into o
# in-place update of our existing note copy
o.note.update(n)
# success!
o.error = 0
# and put it on the result queue
self.q_sync_res.put(o)
else:
o.error = 1
self.q_sync_res.put(o)
def grades_note_setup:
simplenote = Simplenote(EMAIL,PASSWORD)
simplenote.add_note("Grades")
def main(path, user, password):
s = Simplenote(user, password)
notes, status = s.get_note_list()
check_status(status)
with open(path, "w") as f:
for note in notes:
note, status = s.get_note(note["key"])
check_status(status)
if not note["deleted"]:
note["tags"] = " ".join(note["tags"])
note["createdate"] = time.strftime("%d-%m-%Y %H:%M:%S", time.localtime(float(note["createdate"])))
note["modifydate"] = time.strftime("%d-%m-%Y %H:%M:%S", time.localtime(float(note["modifydate"])))
f.write(NOTE_FORMAT.format(**note))
