def delete_local_node(self, local_node):
logger.debug('delete_local_node')
assert isinstance(local_node, Node)
path = local_node.path
# delete model
session.delete(local_node)
save(session)
yield from self.queue.put(DeleteFolder(path))
def delete_local_node(self, local_node):
logger.debug('delete_local_node')
assert isinstance(local_node, Node)
path = local_node.path
# delete model
session.delete(local_node)
save(session)
yield from self.queue.put(DeleteFolder(path))
def on_deleted(self, event):
"""Called when a file or directory is deleted.
:param event:
Event representing file/directory deletion.
:type event:
:class:`DirDeletedEvent` or :class:`FileDeletedEvent`
"""
try:
src_path = ProperPath(event.src_path, event.is_directory)
except Exception:
logging.exception('Exception caught: Invalid path')
AlertHandler.warn(
'invalid path specified. {} will not be synced'.format(
os.path.basename(event.src_path)))
else:
if not self._already_exists(src_path):
return
# get item
try:
item = self._get_item_by_path(src_path)
except ItemNotInDB:
logging.exception(
'Exception caught: Tried to delete item {}, but it was not found in DB'
.format(src_path.name))
else:
# put item in delete state after waiting a second and
# checking to make sure the file was actually deleted
yield from asyncio.sleep(1)
if not os.path.exists(item.path):
item.locally_deleted = True
# nodes cannot be deleted online. THUS, delete it inside database. It will be recreated locally.
if isinstance(item, Node):
session.delete(item)
try:
save(session)
except SQLAlchemyError as e:
logging.exception(
'Exception caught: Error deleting node {} from database.'
.format(item.name))
return
try:
save(session, item)
except SQLAlchemyError as e:
logging.exception(
'Exception caught: Error deleting {} {} from database.'
.format('folder' if event.is_directory else 'file',
item.name))
else:
logging.info('{} set to be deleted'.format(
src_path.full_path))
def on_deleted(self, event):
"""Called when a file or directory is deleted.
:param event:
Event representing file/directory deletion.
:type event:
:class:`DirDeletedEvent` or :class:`FileDeletedEvent`
"""
try:
src_path = ProperPath(event.src_path, event.is_directory)
except Exception:
logging.exception("Exception caught: Invalid path")
AlertHandler.warn("invalid path specified. {} will not be synced".format(os.path.basename(event.src_path)))
else:
if not self._already_exists(src_path):
return
# get item
try:
item = self._get_item_by_path(src_path)
except ItemNotInDB:
logging.exception(
"Exception caught: Tried to delete item {}, but it was not found in DB".format(src_path.name)
)
else:
# put item in delete state after waiting a second and
# checking to make sure the file was actually deleted
yield from asyncio.sleep(1)
if not os.path.exists(item.path):
item.locally_deleted = True
# nodes cannot be deleted online. THUS, delete it inside database. It will be recreated locally.
if isinstance(item, Node):
session.delete(item)
try:
save(session)
except SQLAlchemyError as e:
logging.exception(
"Exception caught: Error deleting node {} from database.".format(item.name)
)
return
try:
save(session, item)
except SQLAlchemyError as e:
logging.exception(
"Exception caught: Error deleting {} {} from database.".format(
"folder" if event.is_directory else "file", item.name
)
)
else:
logging.info("{} set to be deleted".format(src_path.full_path))
def delete_remote_file_folder(self, local_file_folder, remote_file_folder):
logger.debug('delete_remote_file_folder')
assert local_file_folder is not None
assert remote_file_folder is not None
assert local_file_folder.osf_id == self.get_id(remote_file_folder)
assert local_file_folder.locally_deleted
if local_file_folder.is_file:
yield from self.osf_query.delete_remote_file(remote_file_folder)
elif local_file_folder.is_folder:
yield from self.osf_query.delete_remote_folder(remote_file_folder)
local_file_folder.deleted = False
session.delete(local_file_folder)
save(session)
def delete_local_file_folder(self, local_file_folder):
logger.debug('delete_local_file_folder')
assert isinstance(local_file_folder, File)
path = local_file_folder.path
is_folder = local_file_folder.is_folder
# delete model
session.delete(local_file_folder)
save(session)
# delete from local
if is_folder:
yield from self.queue.put(DeleteFolder(path))
else:
yield from self.queue.put(DeleteFile(path))
def delete_remote_file_folder(self, local_file_folder, remote_file_folder):
logger.debug('delete_remote_file_folder')
assert local_file_folder is not None
assert remote_file_folder is not None
assert local_file_folder.osf_id == self.get_id(remote_file_folder)
assert local_file_folder.locally_deleted
if local_file_folder.is_file:
yield from self.osf_query.delete_remote_file(remote_file_folder)
elif local_file_folder.is_folder:
yield from self.osf_query.delete_remote_folder(remote_file_folder)
local_file_folder.deleted = False
session.delete(local_file_folder)
save(session)
def delete_local_file_folder(self, local_file_folder):
logger.debug('delete_local_file_folder')
assert isinstance(local_file_folder, File)
path = local_file_folder.path
is_folder = local_file_folder.is_folder
# delete model
session.delete(local_file_folder)
save(session)
# delete from local
if is_folder:
yield from self.queue.put(DeleteFolder(path))
else:
yield from self.queue.put(DeleteFile(path))
def on_moved(self, event):
"""Called when a file or a directory is moved or renamed.
:param event:
Event representing file/directory movement.
:type event:
:class:`DirMovedEvent` or :class:`FileMovedEvent`
"""
try:
src_path = ProperPath(event.src_path, event.is_directory)
dest_path = ProperPath(event.dest_path, event.is_directory)
except Exception:
logging.exception('Exception caught: Invalid path specified')
AlertHandler.warn('Error moving {}. {} will not be synced.'.format(
'folder' if event.is_directory else 'file',
os.path.basename(event.src_path)))
else:
# determine and get what moved
if not self._already_exists(src_path):
try:
self._get_parent_item_from_path(src_path)
except ItemNotInDB:
# This means it was put into a place on the hierarchy being watched but otherwise not attached to a
# node, so it needs to be added just like a new event rather than as a move.
new_event = DirCreatedEvent(
event.dest_path
) if event.is_directory else FileCreatedEvent(
event.dest_path)
yield from self._create_file_or_folder(new_event,
src_path=dest_path)
return
logging.warning(
'Tried to move item that does not exist: {}'.format(
src_path.name))
return
try:
item = self._get_item_by_path(src_path)
except ItemNotInDB:
logging.exception(
'Exception caught: Tried to move or rename item {}, but it could not be found in DB'
.format(src_path.name))
AlertHandler.warn(
'Could not find item to manipulate. {} will not be synced'.
format(src_path.name))
else:
if isinstance(item, Node):
AlertHandler.warn(
'Cannot manipulate components locally. {} will stop syncing'
.format(item.title))
return
# File
# rename
if item.name != dest_path.name:
item.name = dest_path.name
item.locally_renamed = True
try:
save(session, item)
except SQLAlchemyError:
logging.exception(
'Exception caught: Could not save data for {}'.
format(item))
AlertHandler.warn(
'Error renaming file. {} will stop syncing.'.
format(item.name))
else:
logging.info("renamed a file {}".format(
dest_path.full_path))
# move
elif src_path != dest_path:
# check if file already exists in this moved location. If so, delete it from db.
try:
item_to_replace = self._get_item_by_path(dest_path)
session.delete(item_to_replace)
save(session)
except ItemNotInDB:
logging.info(
'file does not already exist in moved destination: {}'
.format(dest_path.full_path))
except SQLAlchemyError:
logging.exception(
'Exception caught: Could not save data for {}'.
format(item_to_replace))
AlertHandler.warn(
'Error moving file. {} will stop syncing.'.format(
dest_path.name))
try:
new_parent_item = self._get_parent_item_from_path(
dest_path)
except ItemNotInDB:
AlertHandler.warn(
'{} {} placed into invalid containing folder. It will not be synced.'
.format('Folder' if event.is_directory else 'File',
dest_path.name))
else:
# move item
# set previous fields
item.previous_provider = item.provider
item.previous_node_osf_id = item.node.osf_id
# update parent and node fields
# NOTE: this line makes it so the file no longer exists in the database.
# NOTE: item at this point is stale. Unclear why it matters though.
# NOTE: fix is above: session.refresh(item)
item.parent = new_parent_item if isinstance(
new_parent_item, File) else None
item.node = new_parent_item if isinstance(
new_parent_item, Node) else new_parent_item.node
# basically always osfstorage. this is just meant to be extendible in the future to other providers
item.provider = new_parent_item.provider if isinstance(
new_parent_item, File) else File.DEFAULT_PROVIDER
# flags
item.locally_moved = True
try:
save(session, item)
except SQLAlchemyError:
logging.exception(
'Exception caught: Could not save data for {}'.
format(item))
AlertHandler.warn(
'Error moving file. {} will stop syncing.'.
format(item.name))
else:
logging.info('moved from {} to {}'.format(
src_path.full_path, dest_path.full_path))
def _check_file_folder(self,
local_file_folder,
remote_file_folder,
local_parent_file_folder,
local_node):
"""
VARIOUS STATES (update as neccessary):
(None, None) -> Error --
(None, remote) ->
if locally moved -> do nothing
else -> create local
(local.create, None) -> create remote --
(local.create, remote) -> ERROR --
(local.delete, None) -> ERROR --
(local.delete, remote) - > delete remote --
(local, None) ->
if locally moved -> move
else -> delete local
(local, remote) -> check modifications --
"""
assert local_file_folder or remote_file_folder # both shouldnt be None.
logger.debug('checking file_folder internal')
if local_file_folder is None:
locally_moved = yield from self.is_locally_moved(remote_file_folder)
if locally_moved:
return
else:
local_file_folder = yield from self.create_local_file_folder(
remote_file_folder,
local_parent_file_folder,
local_node
)
elif local_file_folder.locally_created and remote_file_folder is None:
if not local_file_folder.is_provider:
remote_file_folder = yield from self.create_remote_file_folder(local_file_folder, local_node)
return
elif local_file_folder.locally_created and remote_file_folder is not None:
raise ValueError('newly created local file_folder was already on server')
elif local_file_folder.locally_deleted and remote_file_folder is None:
session.delete(local_file_folder)
save(session)
logger.warning('local file_folder is to be deleted, however, it was never on the server.')
return
elif local_file_folder.locally_deleted and remote_file_folder is not None:
yield from self.delete_remote_file_folder(local_file_folder, remote_file_folder)
return
elif local_file_folder is not None and remote_file_folder is None:
if local_file_folder.locally_moved:
# todo: we are ignoring return value for now because to start going down new tree would require
# todo: us to have the new node. we currently use the head node instead of dynamically determining
# todo: node. This is problematic. And Bad. FIX IT.
remote_file_folder = yield from self.move_remote_file_folder(local_file_folder)
return
else:
logger.warning('delete_local_file_folder called on {}'.format(local_file_folder.name))
yield from self.delete_local_file_folder(local_file_folder)
return
elif local_file_folder is not None and remote_file_folder is not None:
possibly_new_remote_file_folder = yield from self.modify_file_folder_logic(local_file_folder,
remote_file_folder)
# if we do not need to modify things, remote file folder and local file folder does not change
# we do not need to get a new local file folder because it is updated internally by the db
if possibly_new_remote_file_folder:
remote_file_folder = possibly_new_remote_file_folder
else:
raise ValueError('in some weird state. figure it out.')
assert local_file_folder is not None
assert remote_file_folder is not None
# recursively handle folder's children
if local_file_folder.is_folder:
remote_children = yield from self.osf_query.get_child_files(remote_file_folder)
local_remote_file_folders = self.make_local_remote_tuple_list(local_file_folder.files, remote_children)
for local, remote in local_remote_file_folders:
yield from self._check_file_folder(
local,
remote,
local_parent_file_folder=local_file_folder,
local_node=local_node
)
def _check_file_folder(self, local_file_folder, remote_file_folder,
local_parent_file_folder, local_node):
"""
VARIOUS STATES (update as neccessary):
(None, None) -> Error --
(None, remote) ->
if locally moved -> do nothing
else -> create local
(local.create, None) -> create remote --
(local.create, remote) -> ERROR --
(local.delete, None) -> ERROR --
(local.delete, remote) - > delete remote --
(local, None) ->
if locally moved -> move
else -> delete local
(local, remote) -> check modifications --
"""
assert local_file_folder or remote_file_folder # both shouldnt be None.
logger.debug('checking file_folder internal')
if local_file_folder is None:
locally_moved = yield from self.is_locally_moved(
remote_file_folder)
if locally_moved:
return
else:
local_file_folder = yield from self.create_local_file_folder(
remote_file_folder, local_parent_file_folder, local_node)
elif local_file_folder.locally_created and remote_file_folder is None:
if not local_file_folder.is_provider:
remote_file_folder = yield from self.create_remote_file_folder(
local_file_folder, local_node)
return
elif local_file_folder.locally_created and remote_file_folder is not None:
raise ValueError(
'newly created local file_folder was already on server')
elif local_file_folder.locally_deleted and remote_file_folder is None:
session.delete(local_file_folder)
save(session)
logger.warning(
'local file_folder is to be deleted, however, it was never on the server.'
)
return
elif local_file_folder.locally_deleted and remote_file_folder is not None:
yield from self.delete_remote_file_folder(local_file_folder,
remote_file_folder)
return
elif local_file_folder is not None and remote_file_folder is None:
if local_file_folder.locally_moved:
# todo: we are ignoring return value for now because to start going down new tree would require
# todo: us to have the new node. we currently use the head node instead of dynamically determining
# todo: node. This is problematic. And Bad. FIX IT.
remote_file_folder = yield from self.move_remote_file_folder(
local_file_folder)
return
else:
logger.warning('delete_local_file_folder called on {}'.format(
local_file_folder.name))
yield from self.delete_local_file_folder(local_file_folder)
return
elif local_file_folder is not None and remote_file_folder is not None:
possibly_new_remote_file_folder = yield from self.modify_file_folder_logic(
local_file_folder, remote_file_folder)
# if we do not need to modify things, remote file folder and local file folder does not change
# we do not need to get a new local file folder because it is updated internally by the db
if possibly_new_remote_file_folder:
remote_file_folder = possibly_new_remote_file_folder
else:
raise ValueError('in some weird state. figure it out.')
assert local_file_folder is not None
assert remote_file_folder is not None
# recursively handle folder's children
if local_file_folder.is_folder:
remote_children = yield from self.osf_query.get_child_files(
remote_file_folder)
local_remote_file_folders = self.make_local_remote_tuple_list(
local_file_folder.files, remote_children)
for local, remote in local_remote_file_folders:
yield from self._check_file_folder(
local,
remote,
local_parent_file_folder=local_file_folder,
local_node=local_node)
def on_moved(self, event):
"""Called when a file or a directory is moved or renamed.
:param event:
Event representing file/directory movement.
:type event:
:class:`DirMovedEvent` or :class:`FileMovedEvent`
"""
try:
src_path = ProperPath(event.src_path, event.is_directory)
dest_path = ProperPath(event.dest_path, event.is_directory)
except Exception:
logging.exception("Exception caught: Invalid path specified")
AlertHandler.warn(
"Error moving {}. {} will not be synced.".format(
"folder" if event.is_directory else "file", os.path.basename(event.src_path)
)
)
else:
# determine and get what moved
if not self._already_exists(src_path):
try:
self._get_parent_item_from_path(src_path)
except ItemNotInDB:
# This means it was put into a place on the hierarchy being watched but otherwise not attached to a
# node, so it needs to be added just like a new event rather than as a move.
new_event = (
DirCreatedEvent(event.dest_path) if event.is_directory else FileCreatedEvent(event.dest_path)
)
yield from self._create_file_or_folder(new_event, src_path=dest_path)
return
logging.warning("Tried to move item that does not exist: {}".format(src_path.name))
return
try:
item = self._get_item_by_path(src_path)
except ItemNotInDB:
logging.exception(
"Exception caught: Tried to move or rename item {}, but it could not be found in DB".format(
src_path.name
)
)
AlertHandler.warn("Could not find item to manipulate. {} will not be synced".format(src_path.name))
else:
if isinstance(item, Node):
AlertHandler.warn("Cannot manipulate components locally. {} will stop syncing".format(item.title))
return
# File
# rename
if item.name != dest_path.name:
item.name = dest_path.name
item.locally_renamed = True
try:
save(session, item)
except SQLAlchemyError:
logging.exception("Exception caught: Could not save data for {}".format(item))
AlertHandler.warn("Error renaming file. {} will stop syncing.".format(item.name))
else:
logging.info("renamed a file {}".format(dest_path.full_path))
# move
elif src_path != dest_path:
# check if file already exists in this moved location. If so, delete it from db.
try:
item_to_replace = self._get_item_by_path(dest_path)
session.delete(item_to_replace)
save(session)
except ItemNotInDB:
logging.info("file does not already exist in moved destination: {}".format(dest_path.full_path))
except SQLAlchemyError:
logging.exception("Exception caught: Could not save data for {}".format(item_to_replace))
AlertHandler.warn("Error moving file. {} will stop syncing.".format(dest_path.name))
try:
new_parent_item = self._get_parent_item_from_path(dest_path)
except ItemNotInDB:
AlertHandler.warn(
"{} {} placed into invalid containing folder. It will not be synced.".format(
"Folder" if event.is_directory else "File", dest_path.name
)
)
else:
# move item
# set previous fields
item.previous_provider = item.provider
item.previous_node_osf_id = item.node.osf_id
# update parent and node fields
# NOTE: this line makes it so the file no longer exists in the database.
# NOTE: item at this point is stale. Unclear why it matters though.
# NOTE: fix is above: session.refresh(item)
item.parent = new_parent_item if isinstance(new_parent_item, File) else None
item.node = new_parent_item if isinstance(new_parent_item, Node) else new_parent_item.node
# basically always osfstorage. this is just meant to be extendible in the future to other providers
item.provider = (
new_parent_item.provider if isinstance(new_parent_item, File) else File.DEFAULT_PROVIDER
)
# flags
item.locally_moved = True
try:
save(session, item)
except SQLAlchemyError:
logging.exception("Exception caught: Could not save data for {}".format(item))
AlertHandler.warn("Error moving file. {} will stop syncing.".format(item.name))
else:
logging.info("moved from {} to {}".format(src_path.full_path, dest_path.full_path))