def _run(self): if self.db is None: # TODO: Test edge case where file is created both locally and remotely with same name within a given sync window logger.debug( 'File not yet tracked; will run create operation instead') return RemoteCreateFile(self._context).run() url = '{}/v1/resources/{}/providers/{}/{}'.format( settings.FILE_BASE, self.node.id, self.db.provider, self.db.osf_path) with open(str(self.local), 'rb') as fobj: resp = OSFClient().request('PUT', url, data=fobj) data = resp.json() if resp.status_code == http.client.FORBIDDEN: permission_error_notification('file', self.local.name, self.node.title) else: assert resp.status_code in ( http.client.OK, http.client.CREATED), '{}\n{}\n{}'.format(resp, url, data) remote = osf_client.File(None, data['data']) # WB id are <provider>/<id> remote.id = remote.id.replace(remote.provider + '/', '') remote.parent = self.db.parent DatabaseUpdateFile( OperationContext(remote=remote, db=self.db, node=self.node)).run() Notification().info('Updated File {} in {}'.format( self.db.pretty_path, self.node.title))
def _run(self): if self.db is not None: # On windows, a file update operation can sometimes jump the queue ahead of a file create # due to how watchdog fires events logger.debug( 'File already exists; will run update operation instead') return RemoteUpdateFile(self._context).run() parent = utils.local_to_db(self.local.parent, self.node) url = '{}/v1/resources/{}/providers/{}/{}'.format( settings.FILE_BASE, self.node.id, parent.provider, parent.osf_path) with self.local.open(mode='rb') as fobj: resp = OSFClient().request('PUT', url, data=fobj, params={'name': self.local.name}) data = resp.json() if resp.status_code == http.client.FORBIDDEN: permission_error_notification('file', self.local.name, self.node.title) else: assert resp.status_code == http.client.CREATED, '{}\n{}\n{}'.format( resp, url, data) remote = osf_client.File(None, data['data']) # WB id are <provider>/<id> remote.id = remote.id.replace(remote.provider + '/', '') remote.parent = parent DatabaseCreateFile(OperationContext(remote=remote, node=self.node)).run() Notification().info('Uploaded New File: {} in {}'.format( self.db.pretty_path, self.node.title))
def _run(self): parent = utils.local_to_db(self.local.parent, self.node) url = '{}/v1/resources/{}/providers/{}/{}'.format( settings.FILE_BASE, self.node.id, parent.provider, parent.osf_path) resp = OSFClient().request('PUT', url, params={ 'kind': 'folder', 'name': self.local.name }) data = resp.json() if resp.status_code == http.client.FORBIDDEN: permission_error_notification('folder', self.local.name, self.node.title) else: assert resp.status_code == http.client.CREATED, '{}\n{}\n{}'.format( resp, url, data) remote = osf_client.File(None, data['data']) # WB id are <provider>/<id>/ remote.id = remote.id.replace(remote.provider + '/', '').rstrip('/') remote.parent = parent DatabaseCreateFolder( OperationContext(remote=remote, node=self.node)).run() Notification().info('Created Folder {} in {}'.format( self.db.pretty_path, self.node.title))
def _run(self): with Session() as session: db_parent = session.query(models.File).filter( models.File.id == self.remote.parent.id).one() # TODO folder and file with same name os.mkdir(os.path.join(db_parent.path, self.remote.name)) DatabaseCreateFolder( OperationContext(remote=self.remote, node=self.node)).run() Notification().info('Downloaded Folder: {}'.format( self.db.pretty_path))
def _run(self): resp = OSFClient().request('DELETE', self.remote.raw['links']['delete']) with Session() as session: db_model = session.query( models.File).filter(models.File.id == self.remote.id).one() if resp.status_code == http.client.FORBIDDEN: permission_error_notification(db_model.kind.lower(), self.remote.name, self.node.title) else: assert resp.status_code == http.client.NO_CONTENT, resp Notification().info('Deleted {}: {} in {}'.format( db_model.kind.capitalize(), db_model.pretty_path, self.node.title)) # Always delete the database record. There are two cases: # 1. User can write, and the remote file is deleted # 2. User can not write, but has deleted a local file. Forgetting the database record means that file # will get re-synced later DatabaseDelete(OperationContext(db=db_model)).run()
def run(self): while not self.__stop.is_set(): # Note: CHECK_INTERVAL must be < 24 hours logger.info('Sleeping for {} seconds'.format(settings.REMOTE_CHECK_INTERVAL)) if self._sync_now_event.wait(timeout=settings.REMOTE_CHECK_INTERVAL): if self.__stop.is_set(): break logger.info('Sleep interrupted, syncing now') self._sync_now_event.clear() logger.info('Beginning remote sync') LocalSyncWorker().ignore.set() # Ensure selected node directories exist and db entries created with Session() as session: nodes = session.query(Node).filter(Node.sync).all() for node in nodes: try: self._preprocess_node(node, delete=False) except OSError: # TODO: If the node folder cannot be created, what further actions must be taken before attempting to sync? # TODO: Should the error be user-facing? logger.exception('Error creating node directory for sync') # Session().commit() OperationWorker().join_queue() try: self._check() except: # TODO: Add user-facing notification? msg = 'Error encountered in remote sync operation; will try again later' Notification().error(msg) logger.exception(msg) # We need to ignore modifications to the local filesystem made by the RemoteSyncWorker. # Since there can be a delay between when an operation is popped off the OperationWorker's # queue and when the event is actually captured by watchdog, this sleep tries to ensure the # watchdog observer does not capture any events triggered by the application itself. time.sleep(10) LocalSyncWorker().ignore.clear() logger.info('Finished remote sync') logger.info('Stopped RemoteSyncWorker')
def _run(self): with Session() as session: db_file = session.query( models.File).filter(models.File.id == self.remote.id).one() tmp_path = os.path.join(db_file.parent.path, '.~tmp.{}'.format(db_file.name)) resp = OSFClient().request('GET', self.remote.raw['links']['download'], stream=True) with open(tmp_path, 'wb') as fobj: for chunk in resp.iter_content(chunk_size=1024 * 64): if chunk: fobj.write(chunk) shutil.move(tmp_path, db_file.path) DatabaseUpdateFile( OperationContext(db=db_file, remote=self.remote, node=db_file.node)).run() Notification().info('Uploaded File {} to {}'.format( db_file.pretty_path, self.node.title))
def _run(self): with Session() as session: db_parent = session.query(models.File).filter( models.File.id == self.remote.parent.id).one() path = os.path.join(db_parent.path, self.remote.name) # TODO: Create temp file in target directory while downloading, and rename when done. (check that no temp file exists) resp = OSFClient().request('GET', self.remote.raw['links']['download'], stream=True) with open(path, 'wb') as fobj: for chunk in resp.iter_content(chunk_size=1024 * 64): if chunk: fobj.write(chunk) # After file is saved, create a new database object to track the file # If the task fails, the database task will be kicked off separately by the auditor on a future cycle # TODO: Handle a filename being aliased in local storage (due to OS limitations)? DatabaseCreateFile(OperationContext(remote=self.remote, node=self.node)).run() Notification().info('Downloaded File {} in {}'.format( self.db.pretty_path, self.node.title))
def run(self): logger.info('Start processing queue') while not self.__stop.is_set(): job = self._queue.get() if job is None: self._queue.task_done() continue try: job.run(dry=settings.DRY) except (NodeNotFound, ) as e: logger.warning(e) except Exception as e: logger.exception(e) file_name = job.local.name project_name = job.node.title Notification().error( 'Error while updating the file {} in project {}.'.format( file_name, project_name)) finally: self._queue.task_done() logger.debug('OperationWorker stopped')
def permission_error_notification(file_or_folder, file_name, node_title): Notification().error( 'Could not sync {} {} in project {}. Please verify you ' 'have write permission to the project.'.format(file_or_folder, file_name, node_title))