def test_is_root(self):
self.assertTrue(ProperPath('/', True).is_root)
with self.assertRaises(InvalidPathError):
resp = ProperPath('/', False).is_root
self.assertFalse(ProperPath('/this/folder', True).is_root)
self.assertFalse(ProperPath('/this/folder', False).is_root)
def test_input_isdir_determines_file(self):
self.assertFalse(ProperPath('/this/is/folder', True).is_file)
self.assertFalse(ProperPath('/this/is/folder/', True).is_file)
self.assertTrue(ProperPath('/this/is/folder', False).is_file)
with self.assertRaises(InvalidPathError):
resp = ProperPath('/this/is/folder/', False).is_file
def test_double_slash_is_invalid(self):
with self.assertRaises(InvalidPathError):
resp = ProperPath('/hi//this', True)
with self.assertRaises(InvalidPathError):
resp = ProperPath('//hi/this', True)
with self.assertRaises(InvalidPathError):
resp = ProperPath('/hi/this//', True)
def _get_item_by_path(self, path):
for node in session.query(Node):
if ProperPath(node.path, True) == path:
return node
for file_folder in session.query(File):
file_path = ProperPath(file_folder.path, file_folder.is_folder)
if file_path == path:
return file_folder
raise ItemNotInDB('item has path: {}'.format(path.full_path))
def test_cant_have_dotdot(self):
with self.assertRaises(InvalidPathError):
resp = ProperPath('..', True)
with self.assertRaises(InvalidPathError):
resp = ProperPath('/hi/../as', True)
with self.assertRaises(InvalidPathError):
resp = ProperPath('../', True)
with self.assertRaises(InvalidPathError):
resp = ProperPath('/..', True)
def _event_is_for_components_file_folder(self, event):
try:
if ProperPath(event.src_path, True).name == 'Components':
return True
try:
return ProperPath(event.dest_path, True).name == 'Components'
except AttributeError:
return False
except Exception:
pass # Similar path-related error will occur and be caught later, better error message will be displayed then
def _get_proper_path(self, item):
if isinstance(item, ProperPath):
return item
elif isinstance(item, User):
return ProperPath(item.osf_local_folder_path, True)
elif isinstance(item, File) and item.is_file:
return ProperPath(item.path, False)
elif isinstance(item, Base):
return ProperPath(item.path, True)
elif isinstance(item, str):
absolute = os.path.join(self.osf_path.full_path, item)
return ProperPath(absolute, os.path.isdir(absolute))
else:
raise InvalidItemType('LocalDBSync._get_proper_path does '
'not handle items of type '
'{item_type}'.format(item_type=type(item)))
def run(self):
try:
old_folder_path = ProperPath(self.old_path, is_dir=True)
except Exception:
# TODO: Narrow down this exception and do client side warnings
logging.exception(
'Exception caught: Invalid origin path for renamed folder.')
return
try:
new_folder_path = ProperPath(self.new_path, is_dir=True)
except Exception:
# TODO: Narrow down this exception and do client side warnings
logging.exception(
'Exception caught: Invalid target path for renamed folder.')
return
AlertHandler.info(new_folder_path.name, AlertHandler.MODIFYING)
yield from _rename(old_folder_path, new_folder_path)
def run(self):
file_to_delete = ProperPath(self.path, is_dir=False)
AlertHandler.info(file_to_delete.name, AlertHandler.DELETING)
try:
os.remove(file_to_delete.full_path)
except FileNotFoundError:
logging.warning(
'file not deleted because does not exist on local filesystem. inside delete_local_file_folder (2)'
)
def _get_children(self, item):
if item is None:
return []
# local
if isinstance(item, ProperPath):
if item.is_file:
return []
else:
children = []
for child in os.listdir(item.full_path):
child_item_path = os.path.join(item.full_path, child)
is_dir = os.path.isdir(child_item_path)
child_item = ProperPath(child_item_path, is_dir)
# handle the components folder
if child_item.name == LocalDBSync.COMPONENTS_FOLDER_NAME:
for component in os.listdir(child_item_path):
component_path = os.path.join(
child_item_path, component)
component_is_dir = os.path.isdir(component_path)
# NOTE: making a concious decision here to ignore invalid file in components folder
# NOTE: this means the user is not getting an alert in this case
if component_is_dir:
children.append(
ProperPath(component_path,
component_is_dir))
else:
children.append(child_item)
return children
# db
else:
if isinstance(item, File):
return item.files
elif isinstance(item, Node):
return item.child_nodes + item.top_level_file_folders
elif isinstance(item, User):
return item.top_level_nodes
else:
raise InvalidItemType(
'LocalDBSync._get_children does '
'not handle items of type '
'{item_type}'.format(item_type=type(item)))
def run(self):
try:
new_file_path = ProperPath(self.path, is_dir=False)
except Exception:
# TODO: Narrow down this exception and do client side warnings
logging.exception(
'Exception caught: Invalid target path for new file.')
return
AlertHandler.info(new_file_path.name, AlertHandler.DOWNLOAD)
yield from _download_file(new_file_path, self.download_url,
self.osf_query)
def __init__(self, absolute_osf_dir_path, observer, user):
if not isinstance(observer, Observer):
raise TypeError
if not isinstance(user, User):
raise TypeError
if not os.path.isdir(absolute_osf_dir_path):
raise FolderNotInFileSystem
self.osf_path = ProperPath(absolute_osf_dir_path, True)
self.observer = observer
self.user = user
def on_deleted(self, event):
"""Called when a file or directory is deleted.
:param event:
Event representing file/directory deletion.
:type event:
:class:`DirDeletedEvent` or :class:`FileDeletedEvent`
"""
try:
src_path = ProperPath(event.src_path, event.is_directory)
except Exception:
logging.exception('Exception caught: Invalid path')
AlertHandler.warn(
'invalid path specified. {} will not be synced'.format(
os.path.basename(event.src_path)))
else:
if not self._already_exists(src_path):
return
# get item
try:
item = self._get_item_by_path(src_path)
except ItemNotInDB:
logging.exception(
'Exception caught: Tried to delete item {}, but it was not found in DB'
.format(src_path.name))
else:
# put item in delete state after waiting a second and
# checking to make sure the file was actually deleted
yield from asyncio.sleep(1)
if not os.path.exists(item.path):
item.locally_deleted = True
# nodes cannot be deleted online. THUS, delete it inside database. It will be recreated locally.
if isinstance(item, Node):
session.delete(item)
try:
save(session)
except SQLAlchemyError as e:
logging.exception(
'Exception caught: Error deleting node {} from database.'
.format(item.name))
return
try:
save(session, item)
except SQLAlchemyError as e:
logging.exception(
'Exception caught: Error deleting {} {} from database.'
.format('folder' if event.is_directory else 'file',
item.name))
else:
logging.info('{} set to be deleted'.format(
src_path.full_path))
def on_modified(self, event):
"""Called when a file or directory is modified.
:param event:
Event representing file/directory modification.
:type event:
:class:`DirModifiedEvent` or :class:`FileModifiedEvent`
"""
if isinstance(event, DirModifiedEvent):
return
try:
src_path = ProperPath(event.src_path, event.is_directory)
except Exception:
logging.exception('Exception caught: Invalid path')
AlertHandler.warn(
'invalid path specified. {} will not be synced'.format(
os.path.basename(event.src_path)))
else:
# get item
try:
item = self._get_item_by_path(src_path)
except ItemNotInDB:
# todo: create file folder
logging.warning(
'file {} was modified but not already in db. create it in db.'
.format(src_path))
return # todo: remove this once above is implemented
# update hash
try:
item.update_hash()
except OSError:
logging.exception('File inaccessible during update_hash')
AlertHandler.warn(
'Error updating {}. {} inaccessible, will stop syncing.'.
format('Folder' if event.is_directory else 'File',
item.name))
return
# save
try:
save(session, item)
except SQLAlchemyError:
logging.exception(
'Exception caught: Could not save data for {}'.format(
item))
AlertHandler.warn(
'Error updating {}. {} will stop syncing.'.format(
'folder' if event.is_directory else 'file', item.name))
def run(self):
if not isinstance(self.osf_query, OSFQuery):
logging.error('Update file query is not an instance of OSFQuery')
return
if not isinstance(self.download_url, str):
logging.error('Update file download_url is not a str.')
return
try:
updated_file_path = ProperPath(self.path, is_dir=False)
except Exception:
# TODO: Narrow down this exception and do client side warnings
logging.exception(
'Exception caught: Invalid target path for updated file.')
return
AlertHandler.info(updated_file_path.name, AlertHandler.MODIFYING)
yield from _download_file(updated_file_path, self.download_url,
self.osf_query)
def run(self):
# create local node folder on filesystem
try:
folder_to_create = ProperPath(self.path, is_dir=True)
except Exception:
# TODO: Narrow down this exception and do client side warnings
logging.exception(
'Exception caught: Invalid target path for folder.')
return
if not os.path.exists(folder_to_create.full_path):
AlertHandler.info(folder_to_create.name, AlertHandler.DOWNLOAD)
try:
os.makedirs(folder_to_create.full_path)
except Exception:
# TODO: Narrow down this exception and do client side warnings
logging.exception(
'Exception caught: Problem making a directory.')
return
def run(self):
try:
folder_to_delete = ProperPath(self.path, is_dir=True)
except Exception:
# TODO: Narrow down this exception and do client side warnings
logging.exception(
'Exception caught: Invalid source path for deleted folder.')
return
AlertHandler.info(folder_to_delete.name, AlertHandler.DELETING)
try:
shutil.rmtree(
folder_to_delete.full_path,
onerror=lambda a, b, c: logging.warning(
'local node not deleted because it does not exist.'))
except Exception:
# TODO: Narrow down this exception and do client side warnings
logging.exception('Exception caught: Problem removing the tree.')
return
def on_created(self, event):
"""Called when a file or directory is created.
:param event:
Event representing file/directory creation.
:type event:
:class:`DirCreatedEvent` or :class:`FileCreatedEvent`
"""
try:
src_path = ProperPath(event.src_path, event.is_directory)
except Exception:
logging.exception('Exception caught: Invalid path')
AlertHandler.warn(
'invalid path specified. {} will not be synced.'.format(
os.path.basename(event.src_path)))
else:
# create new model
if self._already_exists(src_path):
return
yield from self._create_file_or_folder(event, src_path=src_path)
def __init__(self, osf_folder, loop):
super().__init__()
self._loop = loop or asyncio.get_event_loop()
self.osf_folder = ProperPath(osf_folder, True)
self.user = session.query(User).filter(User.logged_in).one()
def test_should_expand_tilde(self):
with self.assertRaises(InvalidPathError):
resp = ProperPath('~/mydir/', True)
with self.assertRaises(InvalidPathError):
resp = ProperPath('~/mydir', False)
def test_file_cant_end_with_slash(self):
with self.assertRaises(InvalidPathError):
resp = ProperPath('/hi/this/is/file/', False)
def test_root_cant_be_file(self):
with self.assertRaises(InvalidPathError):
resp = ProperPath('/', False)
def on_moved(self, event):
"""Called when a file or a directory is moved or renamed.
:param event:
Event representing file/directory movement.
:type event:
:class:`DirMovedEvent` or :class:`FileMovedEvent`
"""
try:
src_path = ProperPath(event.src_path, event.is_directory)
dest_path = ProperPath(event.dest_path, event.is_directory)
except Exception:
logging.exception('Exception caught: Invalid path specified')
AlertHandler.warn('Error moving {}. {} will not be synced.'.format(
'folder' if event.is_directory else 'file',
os.path.basename(event.src_path)))
else:
# determine and get what moved
if not self._already_exists(src_path):
try:
self._get_parent_item_from_path(src_path)
except ItemNotInDB:
# This means it was put into a place on the hierarchy being watched but otherwise not attached to a
# node, so it needs to be added just like a new event rather than as a move.
new_event = DirCreatedEvent(
event.dest_path
) if event.is_directory else FileCreatedEvent(
event.dest_path)
yield from self._create_file_or_folder(new_event,
src_path=dest_path)
return
logging.warning(
'Tried to move item that does not exist: {}'.format(
src_path.name))
return
try:
item = self._get_item_by_path(src_path)
except ItemNotInDB:
logging.exception(
'Exception caught: Tried to move or rename item {}, but it could not be found in DB'
.format(src_path.name))
AlertHandler.warn(
'Could not find item to manipulate. {} will not be synced'.
format(src_path.name))
else:
if isinstance(item, Node):
AlertHandler.warn(
'Cannot manipulate components locally. {} will stop syncing'
.format(item.title))
return
# File
# rename
if item.name != dest_path.name:
item.name = dest_path.name
item.locally_renamed = True
try:
save(session, item)
except SQLAlchemyError:
logging.exception(
'Exception caught: Could not save data for {}'.
format(item))
AlertHandler.warn(
'Error renaming file. {} will stop syncing.'.
format(item.name))
else:
logging.info("renamed a file {}".format(
dest_path.full_path))
# move
elif src_path != dest_path:
# check if file already exists in this moved location. If so, delete it from db.
try:
item_to_replace = self._get_item_by_path(dest_path)
session.delete(item_to_replace)
save(session)
except ItemNotInDB:
logging.info(
'file does not already exist in moved destination: {}'
.format(dest_path.full_path))
except SQLAlchemyError:
logging.exception(
'Exception caught: Could not save data for {}'.
format(item_to_replace))
AlertHandler.warn(
'Error moving file. {} will stop syncing.'.format(
dest_path.name))
try:
new_parent_item = self._get_parent_item_from_path(
dest_path)
except ItemNotInDB:
AlertHandler.warn(
'{} {} placed into invalid containing folder. It will not be synced.'
.format('Folder' if event.is_directory else 'File',
dest_path.name))
else:
# move item
# set previous fields
item.previous_provider = item.provider
item.previous_node_osf_id = item.node.osf_id
# update parent and node fields
# NOTE: this line makes it so the file no longer exists in the database.
# NOTE: item at this point is stale. Unclear why it matters though.
# NOTE: fix is above: session.refresh(item)
item.parent = new_parent_item if isinstance(
new_parent_item, File) else None
item.node = new_parent_item if isinstance(
new_parent_item, Node) else new_parent_item.node
# basically always osfstorage. this is just meant to be extendible in the future to other providers
item.provider = new_parent_item.provider if isinstance(
new_parent_item, File) else File.DEFAULT_PROVIDER
# flags
item.locally_moved = True
try:
save(session, item)
except SQLAlchemyError:
logging.exception(
'Exception caught: Could not save data for {}'.
format(item))
AlertHandler.warn(
'Error moving file. {} will stop syncing.'.
format(item.name))
else:
logging.info('moved from {} to {}'.format(
src_path.full_path, dest_path.full_path))
def test_cant_be_empty(self):
with self.assertRaises(InvalidPathError):
resp = ProperPath('', True)
def test_name_file(self):
path = ProperPath('/this/is/a/long/path', False)
self.assertEquals(path.name, 'path')
def test_name_file_with_slash(self):
with self.assertRaises(InvalidPathError):
path = ProperPath('/this/is/a/long/path/', False)
def test_parent_folder_with_slash(self):
path = ProperPath('/this/is/a/long/path/', True)
self.assertEquals(path.parent.name, 'long')
self.assertEquals(path.parent, ProperPath('/this/is/a/long', True))
def test_parent_file(self):
path = ProperPath('/this/is/a/long/path', False)
self.assertEquals(path.parent.name, 'long')
self.assertEquals(path.parent, ProperPath('/this/is/a/long', True))
def test_name_folder_with_slash(self):
path = ProperPath('/this/is/a/long/path/', True)
self.assertEquals(path.name, 'path')