def on_deleted(self, event): """Called when a file or directory is deleted. :param event: Event representing file/directory deletion. :type event: :class:`DirDeletedEvent` or :class:`FileDeletedEvent` """ try: src_path = ProperPath(event.src_path, event.is_directory) except Exception: logging.exception('Exception caught: Invalid path') AlertHandler.warn( 'invalid path specified. {} will not be synced'.format( os.path.basename(event.src_path))) else: if not self._already_exists(src_path): return # get item try: item = self._get_item_by_path(src_path) except ItemNotInDB: logging.exception( 'Exception caught: Tried to delete item {}, but it was not found in DB' .format(src_path.name)) else: # put item in delete state after waiting a second and # checking to make sure the file was actually deleted yield from asyncio.sleep(1) if not os.path.exists(item.path): item.locally_deleted = True # nodes cannot be deleted online. THUS, delete it inside database. It will be recreated locally. if isinstance(item, Node): session.delete(item) try: save(session) except SQLAlchemyError as e: logging.exception( 'Exception caught: Error deleting node {} from database.' .format(item.name)) return try: save(session, item) except SQLAlchemyError as e: logging.exception( 'Exception caught: Error deleting {} {} from database.' .format('folder' if event.is_directory else 'file', item.name)) else: logging.info('{} set to be deleted'.format( src_path.full_path))
def on_deleted(self, event): """Called when a file or directory is deleted. :param event: Event representing file/directory deletion. :type event: :class:`DirDeletedEvent` or :class:`FileDeletedEvent` """ try: src_path = ProperPath(event.src_path, event.is_directory) except Exception: logging.exception("Exception caught: Invalid path") AlertHandler.warn("invalid path specified. {} will not be synced".format(os.path.basename(event.src_path))) else: if not self._already_exists(src_path): return # get item try: item = self._get_item_by_path(src_path) except ItemNotInDB: logging.exception( "Exception caught: Tried to delete item {}, but it was not found in DB".format(src_path.name) ) else: # put item in delete state after waiting a second and # checking to make sure the file was actually deleted yield from asyncio.sleep(1) if not os.path.exists(item.path): item.locally_deleted = True # nodes cannot be deleted online. THUS, delete it inside database. It will be recreated locally. if isinstance(item, Node): session.delete(item) try: save(session) except SQLAlchemyError as e: logging.exception( "Exception caught: Error deleting node {} from database.".format(item.name) ) return try: save(session, item) except SQLAlchemyError as e: logging.exception( "Exception caught: Error deleting {} {} from database.".format( "folder" if event.is_directory else "file", item.name ) ) else: logging.info("{} set to be deleted".format(src_path.full_path))
def open_osf_folder(self): # fixme: containing folder not being updated. import logging logging.debug("containing folder is :{}".format(self.containing_folder)) if validate_containing_folder(self.containing_folder): if sys.platform == "win32": os.startfile(self.containing_folder) elif sys.platform == "darwin": subprocess.Popen(["open", self.containing_folder]) else: try: subprocess.Popen(["xdg-open", self.containing_folder]) except OSError: raise NotImplementedError else: AlertHandler.warn("osf folder is not set")
def open_osf_folder(self): # fixme: containing folder not being updated. import logging logging.debug("containing folder is :{}".format( self.containing_folder)) if validate_containing_folder(self.containing_folder): if sys.platform == 'win32': os.startfile(self.containing_folder) elif sys.platform == 'darwin': subprocess.Popen(['open', self.containing_folder]) else: try: subprocess.Popen(['xdg-open', self.containing_folder]) except OSError: raise NotImplementedError else: AlertHandler.warn('osf folder is not set')
def on_created(self, event): """Called when a file or directory is created. :param event: Event representing file/directory creation. :type event: :class:`DirCreatedEvent` or :class:`FileCreatedEvent` """ try: src_path = ProperPath(event.src_path, event.is_directory) except Exception: logging.exception("Exception caught: Invalid path") AlertHandler.warn("invalid path specified. {} will not be synced.".format(os.path.basename(event.src_path))) else: # create new model if self._already_exists(src_path): return yield from self._create_file_or_folder(event, src_path=src_path)
def start(self): logger.debug('Start in main called.') try: user = session.query(User).filter(User.logged_in).one() except MultipleResultsFound: session.query(User).delete() self.login_signal.emit() return except NoResultFound: self.login_signal.emit() return try: # Simple request to ensure user logged in with valid oauth_token user = asyncio.get_event_loop().run_until_complete( AuthClient().populate_user_data(user)) except AuthError as e: logging.exception(e.message) self.login_signal.emit() containing_folder = os.path.dirname(user.osf_local_folder_path) while not validate_containing_folder(containing_folder): logger.warning( 'Invalid containing folder: {}'.format(containing_folder)) AlertHandler.warn( 'Invalid containing folder. Please choose another.') containing_folder = os.path.abspath( self.set_containing_folder_initial()) user.osf_local_folder_path = os.path.join(containing_folder, 'OSF') save(session, user) self.tray.set_containing_folder(containing_folder) if not os.path.isdir(user.osf_local_folder_path): os.makedirs(user.osf_local_folder_path) self.start_tray_signal.emit() logger.debug('starting background worker from main.start') self.background_worker = BackgroundWorker() self.background_worker.start()
def _create_file_or_folder(self, event, src_path): # assert: whats being created is a file folder try: containing_item = self._get_parent_item_from_path(src_path) except ItemNotInDB: logging.error( "tried to create item {} for parent {} but parent does not exist".format( src_path.full_path, src_path.parent.full_path ) ) return if isinstance(containing_item, Node): node = containing_item else: # file node = containing_item.node new_item = File( name=src_path.name, type=File.FOLDER if event.is_directory else File.FILE, user=self.user, locally_created=True, provider=File.DEFAULT_PROVIDER, node=node, ) containing_item.files.append(new_item) if new_item.is_file: try: new_item.update_hash() except FileNotFoundError: # if file doesnt exist just as we create it, then file is likely temp file. thus don't put it in db. return try: save(session, new_item, containing_item) except SQLAlchemyError: logging.exception("Exception caught: Could not save data for {} in {}".format(new_item, containing_item)) AlertHandler.warn( "Error creating {}: {} will not be synced.".format( "file" if new_item.is_file else "folder", new_item.name ) ) else: logging.info("created new {} {}".format("folder" if event.is_directory else "file", src_path.full_path))
def handle_exception(self, future): # Note: The actual futures never exit, if they do an exception is raised try: raise future.exception() except (aiohttp.ClientError, asyncio.TimeoutError): logger.error('Unable to connect to the internet') AlertHandler.warn('Unable to connection to OSF, we\'ll try again later.') except asyncio.CancelledError: # Cancellations just mean this thread is exiting return # Make sure all our jobs are cancelled if future == self.poll_job: self.process_job.cancel() else: self.poll_job.cancel() logger.info('Restarting polling in 5 seconds') # Finally restart our watcher thread in 5 seconds self._loop.call_later(5, self.start)
def _download_file(path, url, osf_query): if not isinstance(path, ProperPath): logging.error("New file path is not a ProperPath.") return if not isinstance(url, str): logging.error("New file URL is not a string.") return try: resp = yield from osf_query.make_request(url) except (aiohttp.errors.ClientOSError): AlertHandler.warn("Please install operating system updates") logging.exception("SSL certificate error") return except (aiohttp.errors.ClientConnectionError, aiohttp.errors.ClientTimeoutError): # FIXME: Consolidate redundant messages AlertHandler.warn("Bad Internet Connection") logging.exception("Bad Internet Connection") return except (aiohttp.errors.HttpMethodNotAllowed, aiohttp.errors.BadHttpMessage): AlertHandler.warn("Do not have access to file.") logging.exception("Do not have access to file.") return except aiohttp.errors.HttpBadRequest: AlertHandler.warn("Problem accessing file.") logging.exception("Exception caught downloading file.") return except Exception: logging.exception("Exception caught: problem downloading file.") return try: with open(path.full_path, 'wb') as fd: while True: chunk = yield from resp.content.read(2048) if not chunk: break fd.write(chunk) resp.close() except OSError: AlertHandler.warn("unable to open file")
def on_created(self, event): """Called when a file or directory is created. :param event: Event representing file/directory creation. :type event: :class:`DirCreatedEvent` or :class:`FileCreatedEvent` """ try: src_path = ProperPath(event.src_path, event.is_directory) except Exception: logging.exception('Exception caught: Invalid path') AlertHandler.warn( 'invalid path specified. {} will not be synced.'.format( os.path.basename(event.src_path))) else: # create new model if self._already_exists(src_path): return yield from self._create_file_or_folder(event, src_path=src_path)
def handle_exception(self, future): # Note: The actual futures never exit, if they do an exception is raised try: raise future.exception() except (aiohttp.ClientError, asyncio.TimeoutError): logger.error('Unable to connect to the internet') AlertHandler.warn( 'Unable to connection to OSF, we\'ll try again later.') except asyncio.CancelledError: # Cancellations just mean this thread is exiting return # Make sure all our jobs are cancelled if future == self.poll_job: self.process_job.cancel() else: self.poll_job.cancel() logger.info('Restarting polling in 5 seconds') # Finally restart our watcher thread in 5 seconds self._loop.call_later(5, self.start)
def _create_file_or_folder(self, event, src_path): # assert: whats being created is a file folder try: containing_item = self._get_parent_item_from_path(src_path) except ItemNotInDB: logging.error( 'tried to create item {} for parent {} but parent does not exist' .format(src_path.full_path, src_path.parent.full_path)) return if isinstance(containing_item, Node): node = containing_item else: # file node = containing_item.node new_item = File(name=src_path.name, type=File.FOLDER if event.is_directory else File.FILE, user=self.user, locally_created=True, provider=File.DEFAULT_PROVIDER, node=node) containing_item.files.append(new_item) if new_item.is_file: try: new_item.update_hash() except FileNotFoundError: # if file doesnt exist just as we create it, then file is likely temp file. thus don't put it in db. return try: save(session, new_item, containing_item) except SQLAlchemyError: logging.exception( 'Exception caught: Could not save data for {} in {}'.format( new_item, containing_item)) AlertHandler.warn( 'Error creating {}: {} will not be synced.'.format( 'file' if new_item.is_file else 'folder', new_item.name)) else: logging.info("created new {} {}".format( 'folder' if event.is_directory else 'file', src_path.full_path))
def set_containing_folder(self): new_containing_folder = QFileDialog.getExistingDirectory(self, "Choose where to place OSF folder") osf_path = os.path.join(new_containing_folder, "OSF") if new_containing_folder == "": # cancel, closed, or no folder chosen return elif not os.path.exists(osf_path): os.makedirs(osf_path) elif os.path.isfile(osf_path): # FIXME: Consolidate redundant messages AlertHandler.warn( "An OSF file exists where you would like to create the OSF folder. Delete it, or choose a different location") logging.warning("An OSF file exists where you would like to create the OSF folder.") return user = session.query(User).filter(User.logged_in).one() user.osf_local_folder_path = os.path.join(osf_path) self.preferences_window.containingFolderTextEdit.setText(self._translate("Preferences", self.containing_folder)) self.open_window(tab=Preferences.GENERAL) # todo: dynamically update ui???? self.containing_folder_updated_signal.emit(new_containing_folder)
def start(self): logger.debug('Start in main called.') try: user = session.query(User).filter(User.logged_in).one() except MultipleResultsFound: session.query(User).delete() self.login_signal.emit() return except NoResultFound: self.login_signal.emit() return try: # Simple request to ensure user logged in with valid oauth_token user = asyncio.get_event_loop().run_until_complete(AuthClient().populate_user_data(user)) except AuthError as e: logging.exception(e.message) self.login_signal.emit() containing_folder = os.path.dirname(user.osf_local_folder_path) while not validate_containing_folder(containing_folder): logger.warning('Invalid containing folder: {}'.format(containing_folder)) AlertHandler.warn('Invalid containing folder. Please choose another.') containing_folder = os.path.abspath(self.set_containing_folder_initial()) user.osf_local_folder_path = os.path.join(containing_folder, 'OSF') save(session, user) self.tray.set_containing_folder(containing_folder) if not os.path.isdir(user.osf_local_folder_path): os.makedirs(user.osf_local_folder_path) self.start_tray_signal.emit() logger.debug('starting background worker from main.start') self.background_worker = BackgroundWorker() self.background_worker.start()
def check_file_folder(self, local_node, remote_node): logger.debug('checking file_folder') # todo: probably can put this step into get_child_files for nodes. # fixme: doesnt handle multiple providers right now... try: remote_node_files = yield from self.osf_query.get_child_files( remote_node) except aiohttp.errors.HttpBadRequest: AlertHandler.warn( 'could not access files for node {}. Node might have been deleted.' .format(remote_node.name)) return assert len(remote_node_files) >= 1 for node_file in remote_node_files: if node_file.name == 'osfstorage': osfstorage_folder = node_file assert osfstorage_folder try: remote_node_top_level_file_folders = yield from self.osf_query.get_child_files( osfstorage_folder) except aiohttp.errors.HttpBadRequest: AlertHandler.warn( 'could not access files for node {}. Node might have been deleted.' .format(remote_node.name)) return local_remote_files = self.make_local_remote_tuple_list( local_node.top_level_file_folders, remote_node_top_level_file_folders) for local, remote in local_remote_files: yield from self._check_file_folder(local, remote, local_parent_file_folder=None, local_node=local_node)
def check_file_folder(self, local_node, remote_node): logger.debug('checking file_folder') # todo: probably can put this step into get_child_files for nodes. # fixme: doesnt handle multiple providers right now... try: remote_node_files = yield from self.osf_query.get_child_files(remote_node) except aiohttp.errors.HttpBadRequest: AlertHandler.warn( 'could not access files for node {}. Node might have been deleted.'.format(remote_node.name)) return assert len(remote_node_files) >= 1 for node_file in remote_node_files: if node_file.name == 'osfstorage': osfstorage_folder = node_file assert osfstorage_folder try: remote_node_top_level_file_folders = yield from self.osf_query.get_child_files(osfstorage_folder) except aiohttp.errors.HttpBadRequest: AlertHandler.warn( 'could not access files for node {}. Node might have been deleted.'.format(remote_node.name)) return local_remote_files = self.make_local_remote_tuple_list( local_node.top_level_file_folders, remote_node_top_level_file_folders ) for local, remote in local_remote_files: yield from self._check_file_folder( local, remote, local_parent_file_folder=None, local_node=local_node )
def on_modified(self, event): """Called when a file or directory is modified. :param event: Event representing file/directory modification. :type event: :class:`DirModifiedEvent` or :class:`FileModifiedEvent` """ if isinstance(event, DirModifiedEvent): return try: src_path = ProperPath(event.src_path, event.is_directory) except Exception: logging.exception('Exception caught: Invalid path') AlertHandler.warn( 'invalid path specified. {} will not be synced'.format( os.path.basename(event.src_path))) else: # get item try: item = self._get_item_by_path(src_path) except ItemNotInDB: # todo: create file folder logging.warning( 'file {} was modified but not already in db. create it in db.' .format(src_path)) return # todo: remove this once above is implemented # update hash try: item.update_hash() except OSError: logging.exception('File inaccessible during update_hash') AlertHandler.warn( 'Error updating {}. {} inaccessible, will stop syncing.'. format('Folder' if event.is_directory else 'File', item.name)) return # save try: save(session, item) except SQLAlchemyError: logging.exception( 'Exception caught: Could not save data for {}'.format( item)) AlertHandler.warn( 'Error updating {}. {} will stop syncing.'.format( 'folder' if event.is_directory else 'file', item.name))
def on_modified(self, event): """Called when a file or directory is modified. :param event: Event representing file/directory modification. :type event: :class:`DirModifiedEvent` or :class:`FileModifiedEvent` """ if isinstance(event, DirModifiedEvent): return try: src_path = ProperPath(event.src_path, event.is_directory) except Exception: logging.exception("Exception caught: Invalid path") AlertHandler.warn("invalid path specified. {} will not be synced".format(os.path.basename(event.src_path))) else: # get item try: item = self._get_item_by_path(src_path) except ItemNotInDB: # todo: create file folder logging.warning("file {} was modified but not already in db. create it in db.".format(src_path)) return # todo: remove this once above is implemented # update hash try: item.update_hash() except OSError: logging.exception("File inaccessible during update_hash") AlertHandler.warn( "Error updating {}. {} inaccessible, will stop syncing.".format( "Folder" if event.is_directory else "File", item.name ) ) return # save try: save(session, item) except SQLAlchemyError: logging.exception("Exception caught: Could not save data for {}".format(item)) AlertHandler.warn( "Error updating {}. {} will stop syncing.".format( "folder" if event.is_directory else "file", item.name ) )
def on_moved(self, event): """Called when a file or a directory is moved or renamed. :param event: Event representing file/directory movement. :type event: :class:`DirMovedEvent` or :class:`FileMovedEvent` """ try: src_path = ProperPath(event.src_path, event.is_directory) dest_path = ProperPath(event.dest_path, event.is_directory) except Exception: logging.exception('Exception caught: Invalid path specified') AlertHandler.warn('Error moving {}. {} will not be synced.'.format( 'folder' if event.is_directory else 'file', os.path.basename(event.src_path))) else: # determine and get what moved if not self._already_exists(src_path): try: self._get_parent_item_from_path(src_path) except ItemNotInDB: # This means it was put into a place on the hierarchy being watched but otherwise not attached to a # node, so it needs to be added just like a new event rather than as a move. new_event = DirCreatedEvent( event.dest_path ) if event.is_directory else FileCreatedEvent( event.dest_path) yield from self._create_file_or_folder(new_event, src_path=dest_path) return logging.warning( 'Tried to move item that does not exist: {}'.format( src_path.name)) return try: item = self._get_item_by_path(src_path) except ItemNotInDB: logging.exception( 'Exception caught: Tried to move or rename item {}, but it could not be found in DB' .format(src_path.name)) AlertHandler.warn( 'Could not find item to manipulate. {} will not be synced'. format(src_path.name)) else: if isinstance(item, Node): AlertHandler.warn( 'Cannot manipulate components locally. {} will stop syncing' .format(item.title)) return # File # rename if item.name != dest_path.name: item.name = dest_path.name item.locally_renamed = True try: save(session, item) except SQLAlchemyError: logging.exception( 'Exception caught: Could not save data for {}'. format(item)) AlertHandler.warn( 'Error renaming file. {} will stop syncing.'. format(item.name)) else: logging.info("renamed a file {}".format( dest_path.full_path)) # move elif src_path != dest_path: # check if file already exists in this moved location. If so, delete it from db. try: item_to_replace = self._get_item_by_path(dest_path) session.delete(item_to_replace) save(session) except ItemNotInDB: logging.info( 'file does not already exist in moved destination: {}' .format(dest_path.full_path)) except SQLAlchemyError: logging.exception( 'Exception caught: Could not save data for {}'. format(item_to_replace)) AlertHandler.warn( 'Error moving file. {} will stop syncing.'.format( dest_path.name)) try: new_parent_item = self._get_parent_item_from_path( dest_path) except ItemNotInDB: AlertHandler.warn( '{} {} placed into invalid containing folder. It will not be synced.' .format('Folder' if event.is_directory else 'File', dest_path.name)) else: # move item # set previous fields item.previous_provider = item.provider item.previous_node_osf_id = item.node.osf_id # update parent and node fields # NOTE: this line makes it so the file no longer exists in the database. # NOTE: item at this point is stale. Unclear why it matters though. # NOTE: fix is above: session.refresh(item) item.parent = new_parent_item if isinstance( new_parent_item, File) else None item.node = new_parent_item if isinstance( new_parent_item, Node) else new_parent_item.node # basically always osfstorage. this is just meant to be extendible in the future to other providers item.provider = new_parent_item.provider if isinstance( new_parent_item, File) else File.DEFAULT_PROVIDER # flags item.locally_moved = True try: save(session, item) except SQLAlchemyError: logging.exception( 'Exception caught: Could not save data for {}'. format(item)) AlertHandler.warn( 'Error moving file. {} will stop syncing.'. format(item.name)) else: logging.info('moved from {} to {}'.format( src_path.full_path, dest_path.full_path))
def on_moved(self, event): """Called when a file or a directory is moved or renamed. :param event: Event representing file/directory movement. :type event: :class:`DirMovedEvent` or :class:`FileMovedEvent` """ try: src_path = ProperPath(event.src_path, event.is_directory) dest_path = ProperPath(event.dest_path, event.is_directory) except Exception: logging.exception("Exception caught: Invalid path specified") AlertHandler.warn( "Error moving {}. {} will not be synced.".format( "folder" if event.is_directory else "file", os.path.basename(event.src_path) ) ) else: # determine and get what moved if not self._already_exists(src_path): try: self._get_parent_item_from_path(src_path) except ItemNotInDB: # This means it was put into a place on the hierarchy being watched but otherwise not attached to a # node, so it needs to be added just like a new event rather than as a move. new_event = ( DirCreatedEvent(event.dest_path) if event.is_directory else FileCreatedEvent(event.dest_path) ) yield from self._create_file_or_folder(new_event, src_path=dest_path) return logging.warning("Tried to move item that does not exist: {}".format(src_path.name)) return try: item = self._get_item_by_path(src_path) except ItemNotInDB: logging.exception( "Exception caught: Tried to move or rename item {}, but it could not be found in DB".format( src_path.name ) ) AlertHandler.warn("Could not find item to manipulate. {} will not be synced".format(src_path.name)) else: if isinstance(item, Node): AlertHandler.warn("Cannot manipulate components locally. {} will stop syncing".format(item.title)) return # File # rename if item.name != dest_path.name: item.name = dest_path.name item.locally_renamed = True try: save(session, item) except SQLAlchemyError: logging.exception("Exception caught: Could not save data for {}".format(item)) AlertHandler.warn("Error renaming file. {} will stop syncing.".format(item.name)) else: logging.info("renamed a file {}".format(dest_path.full_path)) # move elif src_path != dest_path: # check if file already exists in this moved location. If so, delete it from db. try: item_to_replace = self._get_item_by_path(dest_path) session.delete(item_to_replace) save(session) except ItemNotInDB: logging.info("file does not already exist in moved destination: {}".format(dest_path.full_path)) except SQLAlchemyError: logging.exception("Exception caught: Could not save data for {}".format(item_to_replace)) AlertHandler.warn("Error moving file. {} will stop syncing.".format(dest_path.name)) try: new_parent_item = self._get_parent_item_from_path(dest_path) except ItemNotInDB: AlertHandler.warn( "{} {} placed into invalid containing folder. It will not be synced.".format( "Folder" if event.is_directory else "File", dest_path.name ) ) else: # move item # set previous fields item.previous_provider = item.provider item.previous_node_osf_id = item.node.osf_id # update parent and node fields # NOTE: this line makes it so the file no longer exists in the database. # NOTE: item at this point is stale. Unclear why it matters though. # NOTE: fix is above: session.refresh(item) item.parent = new_parent_item if isinstance(new_parent_item, File) else None item.node = new_parent_item if isinstance(new_parent_item, Node) else new_parent_item.node # basically always osfstorage. this is just meant to be extendible in the future to other providers item.provider = ( new_parent_item.provider if isinstance(new_parent_item, File) else File.DEFAULT_PROVIDER ) # flags item.locally_moved = True try: save(session, item) except SQLAlchemyError: logging.exception("Exception caught: Could not save data for {}".format(item)) AlertHandler.warn("Error moving file. {} will stop syncing.".format(item.name)) else: logging.info("moved from {} to {}".format(src_path.full_path, dest_path.full_path))