def save_node(node_id, content, use_ust=True, callback=None):
"""
Save content into node (must be a file !)
Arguments:
node_id: node to update
content: content to write to file
use_ust: if True, process save in User Synchronous mode
"""
node, db_sess = _get_node(node_id)
l.info("save_node: %s" % (node.db_path, ))
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, node.user)
else:
ust_start(node.user)
try:
# First save to dropbox
u_content = force_unicode((smart_unicode(content)))
if django_settings.ASYNC_DROPBOX:
yield_key = object()
db_sess.write_file(encode_if_unicode(node.db_path), u_content,
callback=(yield gen.Callback(yield_key)))
ret_dict = yield gen.Wait(yield_key)
else:
ret_dict = db_sess.write_file(encode_if_unicode(node.db_path), u_content)
# Then to local storage
_save_and_convert(node, content)
except Exception, e:
err_msg = "save_node: can't save %s (%s)" %\
(node.db_path, e, )
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
def sync_file_from_dropbox(node_id, use_ust=True, callback=None):
"""
Sync file which was updated on dropbox
Arguments:
node_id: updated file
use_ust: if True, process remove in User Synchronous mode
"""
node, db_sess = _get_node(node_id)
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, node.user)
else:
ust_start(node.user)
l.info("sync_file_FROM_dropbox: '%s'" % (node.db_path, ))
try:
# Read file content from dropbox
if django_settings.ASYNC_DROPBOX:
yield_key = object()
db_sess.read_file(encode_if_unicode(node.db_path),
callback=(yield gen.Callback(yield_key)))
content = yield gen.Wait(yield_key)
else:
content = db_sess.read_file(encode_if_unicode(node.db_path))
# And save it to local storage
_save_and_convert(node, content)
# User may want to know that this file was synced
append_to_changes_list(node.user, node.db_path)
except Exception, e:
err_msg = "SYNC_file_from_dropbox: '%s' (%s)" %\
(node.db_path, e,)
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
def rename_node(node_id, new_filename, use_ust=True, callback=None):
"""
Rename node to new_filename
Arguments:
node_id: node to rename
new_filename: filename to remove to (can be both directory and file)
use_ust: if True, process remove in User Synchronous mode
"""
node, db_sess = _get_node(node_id)
l.info("rename_node: '%s' -> '%s'" % (node.db_path, new_filename, ))
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, node.user)
else:
ust_start(node.user)
# TODO: Dropbox exception's text contains HTML, which can't be parsed
# TODO: correctly by frontend, use Reason (part of exception dict)
try:
# First rename on dropbox
new_db_path = get_dropbox_path(os.path.split(node.db_path)[0], new_filename)
from_path = encode_if_unicode(node.db_path)
to_path = encode_if_unicode(new_db_path)
if django_settings.ASYNC_DROPBOX:
yield_key = object()
db_sess.move_file(from_path, to_path,
callback=(yield gen.Callback(yield_key)))
ret_dict = yield gen.Wait(yield_key)
else:
db_sess.db_client.file_move(from_path, to_path)
# Then on local storage
new_src = get_server_src_file_path(node.user, new_db_path)
forced_rename(node.srv_src, new_src)
new_html = get_server_html_file_path(node.user, new_db_path)
if new_html != "":
forced_rename(node.srv_html, new_html)
# And the object itself
MDBTree.objects.filter(pk=node.pk).update(
db_path = new_db_path,
srv_src = new_src,
srv_html = new_html,
)
except Exception, e:
err_msg = "rename_node: '%s' -> '%s' (%s)" %\
(node.db_path, new_filename, e,)
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
def create_new_file(parent_node_id, filename, content, use_ust=True, callback=None):
"""
Add new file TO dropbox and also to our database and to server
NOTE: filename should be just the name of the file, without any
path elements (it will be added as file in parent_node)
Arguments:
parent_node_id: dir where the file should be created
filename: name of new file
content: initial content
use_ust: if True, process remove in User Synchronous mode
"""
parent_node, db_sess = _get_node(parent_node_id)
l.info("create_new_file: '%s' in '%s' (%d bytes)" %\
(filename, parent_node.db_path, len(content), ))
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, parent_node.user)
else:
ust_start(parent_node.user)
try:
# We first create file ON dropbox and then sync it back
# to our database and server to keep the exec logic consistent
db_file_path = get_dropbox_path(parent_node.db_path, filename)
if django_settings.ASYNC_DROPBOX:
yield_key = object()
db_sess.write_file(encode_if_unicode(db_file_path), content,
callback=(yield gen.Callback(yield_key)))
ret_dict = yield gen.Wait(yield_key)
else:
ret_dict = db_sess.write_file(encode_if_unicode(db_file_path), content)
l.info("create_new_file: '%s' in '%s' DONE. Now syncing dir..." %\
(filename, parent_node.db_path, ))
if django_settings.ASYNC_DROPBOX:
yield_key = object()
add_file_from_dropbox(parent_node.id, db_file_path, datetime.utcnow(), use_ust=False,
callback=(yield gen.Callback(yield_key)))
ret_dict = yield gen.Wait(yield_key)
else:
add_file_from_dropbox(parent_node.id, db_file_path, datetime.utcnow(), use_ust=False)
except Exception, e:
err_msg = "create_new_file: '%s' in '%s' (%s)" %\
(filename, parent_node.db_path, e,)
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
def remove_node_as_file(node_id, skip_dropbox=False, use_ust=True, callback=None):
"""
Remove node (must be a file !)
Arguments:
node_id: node to remove
skip_dropbox: if True, only remove from local storage (used in sync)
use_ust: if True, process remove in User Synchronous mode
"""
node, db_sess = _get_node(node_id)
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, node.user)
else:
ust_start(node.user)
try:
# First remove from dropbox
if not skip_dropbox:
if django_settings.ASYNC_DROPBOX:
yield_key = object()
db_sess.delete_file(encode_if_unicode(node.db_path),
callback=(yield gen.Callback(yield_key)))
yield gen.Wait(yield_key)
else:
db_sess.db_client.file_delete(encode_if_unicode(node.db_path))
else:
l.info("remove_node_as_file: '%s' skipping dropbox removal" %\
(node.db_path, ))
# Then from local storage
for file in (node.srv_html, node.srv_src):
l.info("remove_node_as_file: removing '%s'" %\
(file, ))
if not os.path.exists(file):
l.info("remove_node_as_file: '%s' does not exists ?!" %\
(file, ))
else:
os.remove(file)
# Also delete treebeard node
node.delete()
except Exception, e:
err_msg = "remove_node_as_file: '%s' (%s)" %\
(node.db_path, e,)
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
def create_new_dir(parent_node_id, dir_name, use_ust=True, callback=None):
"""
Add new directory TO dropbox and also to our database and to server
NOTE: directory should be just the name of the directory, without any
path elements (it will be added in parent_node)
Arguments:
parent_node_id: dir where the new dir should be created
filename: name of new dir
use_ust: if True, process remove in User Synchronous mode
"""
parent_node, db_sess = _get_node(parent_node_id)
l.info("create_new_dir: '%s' in '%s'" %\
(dir_name, parent_node.db_path, ))
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, parent_node.user)
else:
ust_start(parent_node.user)
try:
# First save to dropbox
db_dir_path = get_dropbox_path(parent_node.db_path, dir_name)
if django_settings.ASYNC_DROPBOX:
yield_key = object()
db_sess.create_folder(encode_if_unicode(db_dir_path),
callback=(yield gen.Callback(yield_key)))
ret_dict = yield gen.Wait(yield_key)
else:
ret_dict = db_sess.db_client.file_create_folder(encode_if_unicode(db_dir_path))
l.info("create_new_dir: '%s' in '%s' DONE. Now syncing parent..." %\
(dir_name, parent_node.db_path, ))
if django_settings.ASYNC_DROPBOX:
yield_key = object()
add_dir_from_dropbox(parent_node.id, db_dir_path, use_ust=False,
callback=(yield gen.Callback(yield_key)))
yield gen.Wait(yield_key)
else:
add_dir_from_dropbox(parent_node.id, db_dir_path, use_ust=False)
except Exception, e:
err_msg = "create_new_dir: '%s' -> '%s' (%s)" %\
(dir_name, parent_node.db_path, e,)
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
def remove_node_as_dir(node_id, skip_dropbox=False, use_ust=True, callback=None):
"""
Remove node (must be a directory !)
Arguments:
node_id: node to remove
skip_dropbox: if True, only remove from local storage (used in sync)
use_ust: if True, process remove in User Synchronous mode
"""
node, db_sess = _get_node(node_id)
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, node.user)
else:
ust_start(node.user)
try:
# Remove dir from dropbox if asked
if not skip_dropbox:
# Removes dir from dropbox (recursively)
if django_settings.ASYNC_DROPBOX:
yield_key = object()
db_sess.delete_file(encode_if_unicode(node.db_path),
callback=(yield gen.Callback(yield_key)))
yield gen.Wait(yield_key)
else:
db_sess.db_client.file_delete(encode_if_unicode(node.db_path))
else:
l.info("remove_node_as_dir: '%s' skipping dropbox removal" %\
(node.db_path, ))
# Then from local storage
remove_server_dir(node.user, node.db_path)
# Go through treebeard to find and delete node's descendants
for child in node.get_children():
child.delete()
# And finally delete treebeard node
node.delete()
except Exception, e:
err_msg = "remove_node_as_dir: '%s' (%s)" %\
(node.db_path, e,)
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
def add_dir_from_dropbox(parent_node_id, db_dir_path, use_ust=True, callback=None):
"""
Add new dir from dropbox to our database and sync it to server
Arguments:
parent_node_id: dir where the new dir should be created
db_file_path: full path to dropbox dir
use_ust: if True, process remove in User Synchronous mode
"""
parent_node, db_sess = _get_node(parent_node_id, need_db_sess=False)
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, parent_node.user)
else:
ust_start(parent_node.user)
# Add new node to treebeard
node = parent_node.add_child (
db_path = db_dir_path,
is_file = False,
last_modified = datetime.utcnow(),
srv_src = get_server_dir(parent_node.user, db_dir_path),
srv_html = get_server_dir(parent_node.user, db_dir_path),
user = parent_node.user
)
try:
# And dir to local storage
create_server_dir(parent_node.user, db_dir_path)
# Sync creates the dir on server if it does not exists
if django_settings.ASYNC_DROPBOX:
yield_key = object()
sync_dir_from_dropbox(node.id, use_ust=False,
callback=(yield gen.Callback(yield_key)))
yield gen.Wait(yield_key)
else:
sync_dir_from_dropbox(node.id, use_ust=False)
except Exception, e:
l.error("add_DIR_FROM_dropbox: '%s':'%s' FAILED" %\
(node.user.username, node.db_path, ))
raise MDBException("ERROR: "+e.message)
def add_file_from_dropbox(parent_node_id, db_file_path, db_last_modified, use_ust=True, callback=None):
"""
Add new file from dropbox to our database and sync it to server
Arguments:
parent_node_id: dir where the new file should be created
db_file_path: full path to dropbox file
db_last_modified: taken from dropbox to remember last mod date for next sync
use_ust: if True, process remove in User Synchronous mode
"""
parent_node, db_sess = _get_node(parent_node_id, need_db_sess=False)
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, parent_node.user)
else:
ust_start(parent_node.user)
# Add new node to treebeard
node = parent_node.add_child (
db_path = db_file_path,
is_file = True,
last_modified = db_last_modified,
srv_src = get_server_src_file_path(parent_node.user, db_file_path),
srv_html = get_server_html_file_path(parent_node.user, db_file_path),
user = parent_node.user
)
try:
# Sync also creates the file on server if it does not exists
if django_settings.ASYNC_DROPBOX:
yield_key = object()
sync_file_from_dropbox(node.id, use_ust=False,
callback=(yield gen.Callback(yield_key)))
yield gen.Wait(yield_key)
else:
sync_file_from_dropbox(node.id, use_ust=False)
except Exception, e:
err_msg = "ADD_file_from_dropbox: '%s' (%s)" %\
(node.db_path, e,)
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
def sync_dir_from_dropbox(node_id, use_ust=True, callback=None):
"""
Sync dir which was updated on dropbox
The whole dir is checked for added/removed files and dirs and each entry
is updated using appropriate method
Arguments:
node_id: updated dir
use_ust: if True, process remove in User Synchronous mode
"""
node, db_sess = _get_node(node_id)
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, node.user)
else:
ust_start(node.user)
# We need to try/except the whole block to make sure
# that we call ust_end() before leaving the method
try:
if django_settings.ASYNC_DROPBOX:
resp = yield gen.Task(db_sess.dir_changed,
node.db_path, node.hash)
else:
resp = db_sess.dir_changed(node.db_path, node.hash)
if not resp:
l.info("sync_DIR_FROM_dropbox: '%s':'%s' skipping (same hash)" %\
(node.user.username, node.db_path, ))
else:
# User may want to know that this dir was updated
append_to_changes_list(node.user, node.db_path)
# Retrieve list of files and dirs from dropbox for given directory
if django_settings.ASYNC_DROPBOX:
file_dicts = yield gen.Task(db_sess.get_files_as_dicts,
node.db_path)
else:
file_dicts = db_sess.get_files_as_dicts(node.db_path)
if file_dicts:
yield_keys = []
# Check each file in directory for updates
# We go through OUR file list and compare each file's
# .last_modified date with the one in matching dropbox file
for child in node.get_children():
if not child.is_file:
continue
file_dict = dicts_find(file_dicts, child.db_path)
if file_dict and dict_last_modified(file_dict) > child.last_modified:
# File in dropbox was changed: remember new mod date and sync it
MDBTree.objects.filter(pk=child.pk).update(
last_modified = dict_last_modified(file_dict),
)
if django_settings.ASYNC_DROPBOX:
yield_key = object()
yield_keys.append(yield_key)
local_callback = (yield gen.Callback(yield_key))
else:
local_callback = None
sync_file_from_dropbox(child.id, use_ust=False, callback=local_callback)
# Wait for all add_file_from_dropbox() to complete
if django_settings.ASYNC_DROPBOX and yield_keys:
yield gen.WaitAll(yield_keys)
# Get only list of files as a set
# And see what's new/missing
db_files = set(dicts_to_paths(file_dicts, files_only=True))
if db_files:
# Get list of files in dir from database
known_files = set(node.get_paths('FILES'))
yield_keys = []
# Add new files from dropbox to our database
for new_file in db_files - known_files:
if new_file != '':
db_last_modified =\
dict_last_modified(dicts_find(file_dicts, new_file))
if django_settings.ASYNC_DROPBOX:
yield_key = object()
yield_keys.append(yield_key)
local_callback = (yield gen.Callback(yield_key))
else:
local_callback = None
add_file_from_dropbox(node.id, new_file, db_last_modified, use_ust=False,
callback=local_callback)
# Wait for all add_file_from_dropbox() to complete
if django_settings.ASYNC_DROPBOX and yield_keys:
yield gen.WaitAll(yield_keys)
# Remove files deleted from dropbox from our database
for missing_file in known_files - db_files:
if missing_file != '':
missing_node = get_node_by_path(node.user, missing_file)
# Only remove it from our database and server
if django_settings.ASYNC_DROPBOX:
yield_key = object()
remove_node_as_file(missing_node.id, skip_dropbox=True, use_ust=False,
callback=(yield gen.Callback(yield_key)))
yield gen.Wait(yield_key)
else:
remove_node_as_file(missing_node.id, skip_dropbox=True, use_ust=False)
# Get only list of dirs as a set
# And see what's new/missing
db_dirs = set(dicts_to_paths(file_dicts, dirs_only=True))
yield_keys = []
if db_dirs:
# Get list of dirs in dir from database
known_dirs = set(node.get_paths('DIRS'))
# Add new dirs from dropbox to our database
for new_dir in db_dirs - known_dirs:
if new_dir != '':
if django_settings.ASYNC_DROPBOX:
yield_key = object()
yield_keys.append(yield_key)
local_callback = (yield gen.Callback(yield_key))
else:
local_callback = None
add_dir_from_dropbox(node.id, new_dir, use_ust=False, callback=local_callback)
#ust_store_id(async_res.task_id, node.user)
# Wait for all add_dir_from_dropbox() to complete
if django_settings.ASYNC_DROPBOX and yield_keys:
yield gen.WaitAll(yield_keys)
# Remove dirs, which are missing from dropbox from our database
for missing_dir in known_dirs - db_dirs:
if missing_dir != '' and missing_dir != '/' :
missing_node = get_node_by_path(node.user, missing_dir)
# Only remove it from our database and server
if django_settings.ASYNC_DROPBOX:
yield_key = object()
remove_node_as_dir(missing_node.id, skip_dropbox=True, use_ust=False,
callback=(yield gen.Callback(yield_key)))
yield gen.Wait(yield_key)
else:
remove_node_as_dir(missing_node.id, skip_dropbox=True, use_ust=False)
# Save new hash for future checks
MDBTree.objects.filter(pk=node.pk).update(
hash = resp['hash'],
last_synced = datetime.utcnow(),
)
except Exception, e:
l.error("sync_DIR_FROM_dropbox: '%s':'%s' (%s)" %\
(node.user.username, node.db_path, e, ))
raise MDBException("ERROR: "+e.message)
use_ust: if True, process remove in User Synchronous mode
"""
# Get MDBUser object
try:
user = MDBUser.objects.get(pk=user_id)
except Exception, e:
err_msg = "sync_all_dirs_from_dropbox(): can't retrieve user %d from database (%s)" %\
(user_id, e, )
l.error(err_msg)
raise MDBException("ERROR: "+err_msg)
if use_ust:
if django_settings.ASYNC_DROPBOX:
yield gen.Task(async_ust_start, user)
else:
ust_start(user)
try:
yield_keys = []
# Sync root node
root_node = MDBTree.objects.get(user=user, db_path='/')
if django_settings.ASYNC_DROPBOX:
yield_key = object()
yield_keys.append(yield_key)
local_callback = (yield gen.Callback(yield_key))
else:
local_callback = None
sync_dir_from_dropbox(root_node.id, use_ust=False, callback=local_callback)
# And all subdirs
for node in root_node.get_descendants():
if node and not node.is_file: