def _run(self):
if self.db is None:
# TODO: Test edge case where file is created both locally and remotely with same name within a given sync window
logger.debug(
'File not yet tracked; will run create operation instead')
return RemoteCreateFile(self._context).run()
url = '{}/v1/resources/{}/providers/{}/{}'.format(
settings.FILE_BASE, self.node.id, self.db.provider,
self.db.osf_path)
with open(str(self.local), 'rb') as fobj:
resp = OSFClient().request('PUT', url, data=fobj)
data = resp.json()
if resp.status_code == http.client.FORBIDDEN:
permission_error_notification('file', self.local.name,
self.node.title)
else:
assert resp.status_code in (
http.client.OK,
http.client.CREATED), '{}\n{}\n{}'.format(resp, url, data)
remote = osf_client.File(None, data['data'])
# WB id are <provider>/<id>
remote.id = remote.id.replace(remote.provider + '/', '')
remote.parent = self.db.parent
DatabaseUpdateFile(
OperationContext(remote=remote, db=self.db,
node=self.node)).run()
Notification().info('Updated File {} in {}'.format(
self.db.pretty_path, self.node.title))
def _run(self):
if self.db is not None:
# On windows, a file update operation can sometimes jump the queue ahead of a file create
# due to how watchdog fires events
logger.debug(
'File already exists; will run update operation instead')
return RemoteUpdateFile(self._context).run()
parent = utils.local_to_db(self.local.parent, self.node)
url = '{}/v1/resources/{}/providers/{}/{}'.format(
settings.FILE_BASE, self.node.id, parent.provider, parent.osf_path)
with self.local.open(mode='rb') as fobj:
resp = OSFClient().request('PUT',
url,
data=fobj,
params={'name': self.local.name})
data = resp.json()
if resp.status_code == http.client.FORBIDDEN:
permission_error_notification('file', self.local.name,
self.node.title)
else:
assert resp.status_code == http.client.CREATED, '{}\n{}\n{}'.format(
resp, url, data)
remote = osf_client.File(None, data['data'])
# WB id are <provider>/<id>
remote.id = remote.id.replace(remote.provider + '/', '')
remote.parent = parent
DatabaseCreateFile(OperationContext(remote=remote,
node=self.node)).run()
Notification().info('Uploaded New File: {} in {}'.format(
self.db.pretty_path, self.node.title))
def _run(self):
parent = utils.local_to_db(self.local.parent, self.node)
url = '{}/v1/resources/{}/providers/{}/{}'.format(
settings.FILE_BASE, self.node.id, parent.provider, parent.osf_path)
resp = OSFClient().request('PUT',
url,
params={
'kind': 'folder',
'name': self.local.name
})
data = resp.json()
if resp.status_code == http.client.FORBIDDEN:
permission_error_notification('folder', self.local.name,
self.node.title)
else:
assert resp.status_code == http.client.CREATED, '{}\n{}\n{}'.format(
resp, url, data)
remote = osf_client.File(None, data['data'])
# WB id are <provider>/<id>/
remote.id = remote.id.replace(remote.provider + '/',
'').rstrip('/')
remote.parent = parent
DatabaseCreateFolder(
OperationContext(remote=remote, node=self.node)).run()
Notification().info('Created Folder {} in {}'.format(
self.db.pretty_path, self.node.title))
def _run(self):
with Session() as session:
db_parent = session.query(models.File).filter(
models.File.id == self.remote.parent.id).one()
# TODO folder and file with same name
os.mkdir(os.path.join(db_parent.path, self.remote.name))
DatabaseCreateFolder(
OperationContext(remote=self.remote, node=self.node)).run()
Notification().info('Downloaded Folder: {}'.format(
self.db.pretty_path))
def _run(self):
resp = OSFClient().request('DELETE',
self.remote.raw['links']['delete'])
with Session() as session:
db_model = session.query(
models.File).filter(models.File.id == self.remote.id).one()
if resp.status_code == http.client.FORBIDDEN:
permission_error_notification(db_model.kind.lower(),
self.remote.name, self.node.title)
else:
assert resp.status_code == http.client.NO_CONTENT, resp
Notification().info('Deleted {}: {} in {}'.format(
db_model.kind.capitalize(), db_model.pretty_path,
self.node.title))
# Always delete the database record. There are two cases:
# 1. User can write, and the remote file is deleted
# 2. User can not write, but has deleted a local file. Forgetting the database record means that file
# will get re-synced later
DatabaseDelete(OperationContext(db=db_model)).run()
def run(self):
while not self.__stop.is_set():
# Note: CHECK_INTERVAL must be < 24 hours
logger.info('Sleeping for {} seconds'.format(settings.REMOTE_CHECK_INTERVAL))
if self._sync_now_event.wait(timeout=settings.REMOTE_CHECK_INTERVAL):
if self.__stop.is_set():
break
logger.info('Sleep interrupted, syncing now')
self._sync_now_event.clear()
logger.info('Beginning remote sync')
LocalSyncWorker().ignore.set()
# Ensure selected node directories exist and db entries created
with Session() as session:
nodes = session.query(Node).filter(Node.sync).all()
for node in nodes:
try:
self._preprocess_node(node, delete=False)
except OSError:
# TODO: If the node folder cannot be created, what further actions must be taken before attempting to sync?
# TODO: Should the error be user-facing?
logger.exception('Error creating node directory for sync')
# Session().commit()
OperationWorker().join_queue()
try:
self._check()
except:
# TODO: Add user-facing notification?
msg = 'Error encountered in remote sync operation; will try again later'
Notification().error(msg)
logger.exception(msg)
# We need to ignore modifications to the local filesystem made by the RemoteSyncWorker.
# Since there can be a delay between when an operation is popped off the OperationWorker's
# queue and when the event is actually captured by watchdog, this sleep tries to ensure the
# watchdog observer does not capture any events triggered by the application itself.
time.sleep(10)
LocalSyncWorker().ignore.clear()
logger.info('Finished remote sync')
logger.info('Stopped RemoteSyncWorker')
def _run(self):
with Session() as session:
db_file = session.query(
models.File).filter(models.File.id == self.remote.id).one()
tmp_path = os.path.join(db_file.parent.path,
'.~tmp.{}'.format(db_file.name))
resp = OSFClient().request('GET',
self.remote.raw['links']['download'],
stream=True)
with open(tmp_path, 'wb') as fobj:
for chunk in resp.iter_content(chunk_size=1024 * 64):
if chunk:
fobj.write(chunk)
shutil.move(tmp_path, db_file.path)
DatabaseUpdateFile(
OperationContext(db=db_file, remote=self.remote,
node=db_file.node)).run()
Notification().info('Uploaded File {} to {}'.format(
db_file.pretty_path, self.node.title))
def _run(self):
with Session() as session:
db_parent = session.query(models.File).filter(
models.File.id == self.remote.parent.id).one()
path = os.path.join(db_parent.path, self.remote.name)
# TODO: Create temp file in target directory while downloading, and rename when done. (check that no temp file exists)
resp = OSFClient().request('GET',
self.remote.raw['links']['download'],
stream=True)
with open(path, 'wb') as fobj:
for chunk in resp.iter_content(chunk_size=1024 * 64):
if chunk:
fobj.write(chunk)
# After file is saved, create a new database object to track the file
# If the task fails, the database task will be kicked off separately by the auditor on a future cycle
# TODO: Handle a filename being aliased in local storage (due to OS limitations)?
DatabaseCreateFile(OperationContext(remote=self.remote,
node=self.node)).run()
Notification().info('Downloaded File {} in {}'.format(
self.db.pretty_path, self.node.title))
def run(self):
logger.info('Start processing queue')
while not self.__stop.is_set():
job = self._queue.get()
if job is None:
self._queue.task_done()
continue
try:
job.run(dry=settings.DRY)
except (NodeNotFound, ) as e:
logger.warning(e)
except Exception as e:
logger.exception(e)
file_name = job.local.name
project_name = job.node.title
Notification().error(
'Error while updating the file {} in project {}.'.format(
file_name, project_name))
finally:
self._queue.task_done()
logger.debug('OperationWorker stopped')
def permission_error_notification(file_or_folder, file_name, node_title):
Notification().error(
'Could not sync {} {} in project {}. Please verify you '
'have write permission to the project.'.format(file_or_folder,
file_name, node_title))