def handle(self):
logging.info('Downloading file "%s" to "%s".', self.remote_item.id,
self.local_abspath)
try:
tmp_name = self.repo.path_filter.get_temp_name(
self.remote_item.name)
tmp_path = self.repo.local_root + self.parent_relpath + '/' + tmp_name
item_request = self.repo.authenticator.client.item(
drive=self.repo.drive.id, id=self.remote_item.id)
item_mtime, item_mtime_editable = get_item_modified_datetime(
self.remote_item)
item_request_call(self.repo, item_request.download, tmp_path)
hashes = self.remote_item.file.hashes
if hashes is None or hashes.sha1_hash is None or hashes.sha1_hash == sha1_value(
tmp_path):
item_size_local = os.path.getsize(tmp_path)
os.rename(tmp_path, self.local_abspath)
fix_owner_and_timestamp(self.local_abspath,
self.repo.context.user_uid,
datetime_to_timestamp(item_mtime))
self.repo.update_item(self.remote_item, self.parent_relpath,
item_size_local)
logging.info('Finished downloading item "%s".',
self.remote_item.id)
return True
else:
# We assumed server's SHA-1 value is always correct -- might not be true.
logging.error('Hash mismatch for downloaded file "%s".',
self.local_abspath)
os.remove(tmp_path)
except (onedrivesdk.error.OneDriveError, OSError) as e:
logging.error('Error when downloading file "%s": %s.',
self.remote_item.id, e)
return False
def handle(self):
if not os.path.isdir(self.local_abspath):
logging.error('Error: Local path "%s" is not a directory.' % self.local_abspath)
return
self.repo.context.watcher.rm_watch(self.repo, self.local_abspath)
try:
all_local_items = self.list_local_names()
except (IOError, OSError) as e:
logging.error('Error merging dir "%s": %s.', self.local_abspath, e)
return
all_records = self.repo.get_immediate_children_of_dir(self.rel_path)
if not self.assume_remote_unchanged or not self.parent_remote_unchanged:
try:
remote_item_page = item_request_call(self.repo, self.item_request.children.get)
all_remote_items = remote_item_page
while True:
# HACK: ChildrenCollectionPage is not guaranteed to have
# the _next_page_link attribute and
# ChildrenCollectionPage.get_next_page_request doesn't
# implement the check correctly
if not hasattr(remote_item_page, '_next_page_link'):
break
logging.debug('Paging for more items: %s', self.rel_path)
remote_item_page = item_request_call(
self.repo,
ChildrenCollectionRequest.get_next_page_request(
remote_item_page,
self.repo.authenticator.client).get)
all_remote_items = itertools.chain(
all_remote_items, remote_item_page)
except onedrivesdk.error.OneDriveError as e:
logging.error('Encountered API Error: %s. Skip directory "%s".', e, self.rel_path)
return
for remote_item in all_remote_items:
remote_is_folder = remote_item.folder is not None
all_local_items.discard(remote_item.name) # Remove remote item from untouched list.
if not self.repo.path_filter.should_ignore(self.rel_path + '/' + remote_item.name, remote_is_folder):
self._handle_remote_item(remote_item, all_local_items, all_records)
else:
logging.debug('Ignored remote path "%s/%s".', self.rel_path, remote_item.name)
for n in all_local_items:
self._handle_local_item(n, all_records)
for rec_name, rec in all_records.items():
logging.info('Record for item %s (%s/%s) is dead. Delete it it.', rec.item_id, rec.parent_path, rec_name)
self.repo.delete_item(rec_name, rec.parent_path, is_folder=rec.type == ItemRecordType.FOLDER)
self.repo.context.watcher.add_watch(self.repo, self.local_abspath)
def handle(self):
logging.info('Deleting remote item "%s".', self.rel_path)
item_request = self.get_item_request()
try:
od_api_helper.item_request_call(self.repo, item_request.delete)
self.repo.delete_item(self.item_name, self.parent_relpath, self.is_folder)
logging.info('Deleted remote item "%s".', self.rel_path)
return True
except (onedrivesdk.error.OneDriveError, OSError) as e:
logging.error('Error deleting item "%s": %s.', self.rel_path, e)
return False
def handle(self):
logging.info('Updating webhook for Drive %s.', self.repo.drive.id)
item_request = self.repo.authenticator.client.item(
drive=self.repo.drive.id, path='/')
expiration_time = datetime.utcnow() + timedelta(
seconds=self.repo.context.config['webhook_renew_interval_sec'])
try:
if self.subscription_id is None:
subscription = od_api_helper.create_subscription(
item_request, self.repo, self.webhook_worker.webhook_url,
expiration_time)
else:
subscription = onedrivesdk.Subscription()
subscription.id = self.subscription_id
subscription.notification_url = self.webhook_worker.webhook_url
subscription.expiration_date_time = expiration_time
subscription = od_api_helper.item_request_call(
self.repo,
item_request.subscriptions[self.subscription_id].update,
subscription)
self.webhook_worker.add_subscription(subscription, self.repo)
logging.info('Webhook for Drive %s updated.', self.repo.drive.id)
return subscription
except onedrivesdk.error.OneDriveError as e:
logging.error('Error: %s', e)
return None
def test_item_request_call_on_unauthorized_error(self):
account_id = 'dummy_acct'
mock_repo = mock.MagicMock(
account_id=account_id,
**{
'authenticator.refresh_session.return_value': 0,
'other.side_effect': KeyError
})
od_api_helper.item_request_call(
mock_repo, self.dummy_api_call,
error.OneDriveError(prop_dict={
'code': error.ErrorCode.Unauthenticated,
'message': 'dummy'
},
status_code=requests.codes.unauthorized))
self.assertEqual(1, len(mock_repo.mock_calls))
name, args, _ = mock_repo.method_calls[0]
self.assertEqual('authenticator.refresh_session', name)
self.assertEqual((account_id, ), args)
def handle(self):
logging.info('Uploading file "%s" to OneDrive.', self.local_abspath)
occupy_task = self.task_pool.occupy_path(self.local_abspath, self)
if occupy_task is not self:
logging.warning('Cannot upload "%s" because %s.', self.local_abspath,
"path is blacklisted" if occupy_task is None else str(occupy_task) + ' is in progress')
return False
try:
item_stat = os.stat(self.local_abspath)
if item_stat.st_size < self.PUT_FILE_SIZE_THRESHOLD_BYTES:
item_request = self.parent_dir_request.children[self.item_name]
returned_item = item_request_call(self.repo, item_request.upload, self.local_abspath)
if returned_item is None:
logging.warning('Upload API did not return metadata of remote item for "%s". '
'Make an explicit request.', self.local_abspath)
returned_item = item_request_call(self.repo, item_request.get)
else:
logging.info('Uploading large file "%s" in chunks of 10MB.', self.local_abspath)
item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, path=self.rel_path)
returned_item = item_request_call(self.repo, item_request.upload_async,
local_path=self.local_abspath, upload_status=self.update_progress)
if not isinstance(returned_item, onedrivesdk.Item):
if hasattr(returned_item, '_prop_dict'):
returned_item = onedrivesdk.Item(returned_item._prop_dict)
else:
returned_item = item_request_call(self.repo, item_request.get)
self.update_timestamp_and_record(returned_item, item_stat)
self.task_pool.release_path(self.local_abspath)
logging.info('Finished uploading file "%s".', self.local_abspath)
return True
except (onedrivesdk.error.OneDriveError, OSError) as e:
logging.error('Error uploading file "%s": %s.', self.local_abspath, e)
# TODO: what if quota is exceeded?
if (isinstance(e, onedrivesdk.error.OneDriveError) and
e.code == onedrivesdk.error.ErrorCode.MalwareDetected):
logging.warning('File "%s" was detected as malware by OneDrive. '
'Do not upload during program session.', self.local_abspath)
self.task_pool.occupy_path(self.local_abspath, None)
return False
self.task_pool.release_path(self.local_abspath)
return False
def update_timestamp_and_record(self, new_item, item_local_stat):
remote_mtime, remote_mtime_w = get_item_modified_datetime(new_item)
if not remote_mtime_w:
# last_modified_datetime attribute is not modifiable in OneDrive server. Update local mtime.
fix_owner_and_timestamp(self.local_abspath, self.repo.context.user_uid,
datetime_to_timestamp(remote_mtime))
else:
file_system_info = FileSystemInfo()
file_system_info.last_modified_date_time = datetime.utcfromtimestamp(item_local_stat.st_mtime)
updated_item = Item()
updated_item.file_system_info = file_system_info
item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, id=new_item.id)
new_item = item_request_call(self.repo, item_request.update, updated_item)
self.repo.update_item(new_item, self.parent_relpath, item_local_stat.st_size)
def handle(self):
logging.info('Updating timestamp for file "%s".', self.local_abspath)
try:
if not os.path.isfile(self.local_abspath):
logging.warning('Local path "%s" is no longer a file. Cannot update timestamp.', self.local_abspath)
return False
item = item_request_call(self.repo, self.get_item_request().get)
item_stat = os.stat(self.local_abspath)
self.update_timestamp_and_record(item, item_stat)
logging.info('Finished updating timestamp for file "%s".', self.local_abspath)
return True
except (onedrivesdk.error.OneDriveError, OSError) as e:
logging.error('Error updating timestamp for file "%s": %s.', self.local_abspath, e)
return False
def ensure_remote_path_is_dir(self, repo, rel_path):
"""
Make sure the path is a folder in remote repository. If the path does not exist, create it. If the path is a
file, rename the file and create the dir. Return False if the remote path can't be made a dir.
:param onedrive_client.od_repo.OneDriveLocalRepository repo:
:param str rel_path:
:return True | False:
"""
if rel_path == '':
# Drive root is guaranteed a directory.
return True
item_request = repo.authenticator.client.item(drive=repo.drive.id, path=rel_path)
parent_relpath, item_name = os.path.split(rel_path)
if parent_relpath == '/':
parent_relpath = ''
try:
item = item_request_call(repo, item_request.get)
# Return True if the remote path exists and is a directory.
if item.folder is not None:
return item_name == item.name
# Remote path is not a directory. Try renaming it and if renaming fails, deleting it.
new_name = get_filename_with_incremented_count(item_name)
logging.info('Remote item "%s" in Drive %s is not a directory. Try renaming it to "%s".',
rel_path, repo.drive.id, new_name)
if not move_item.MoveItemTask(repo=repo, task_pool=self.task_pool,
parent_relpath=parent_relpath, item_name=item_name,
new_name=new_name, is_folder=False).handle():
if not delete_item.DeleteRemoteItemTask(repo=repo, task_pool=self.task_pool,
parent_relpath=parent_relpath,
item_name=item_name, is_folder=False).handle():
logging.warning('Failed to rename or delete remote item "%s" in Drive %s.',
rel_path, repo.drive.id)
return False
except onedrivesdk.error.OneDriveError as e:
if e.code != onedrivesdk.error.ErrorCode.ItemNotFound:
return False
if not merge_dir.CreateFolderTask(repo=repo, task_pool=self.task_pool,
item_name=item_name, parent_relpath=parent_relpath,
upload_if_success=False, abort_if_local_gone=True).handle():
logging.critical('Failed to create remote directory "%s" on Drive %s.', rel_path, repo.drive.id)
return False
return True
def handle(self):
logging.info('Creating remote item for local dir "%s".', self.local_abspath)
try:
if self.abort_if_local_gone and not os.path.isdir(self.local_abspath):
logging.warning('Local dir "%s" is gone. Skip creating remote item for it.', self.local_abspath)
return
item = self._get_folder_pseudo_item(self.item_name)
item_request = self._get_item_request()
item = item_request_call(self.repo, item_request.children.add, item)
self.repo.update_item(item, self.parent_relpath, 0)
logging.info('Created remote item for local dir "%s".', self.local_abspath)
if self.upload_if_success:
logging.info('Adding task to merge "%s" after remote item was created.', self.local_abspath)
self.task_pool.add_task(MergeDirectoryTask(
self.repo, self.task_pool, self.parent_relpath + '/' + self.item_name,
self.repo.authenticator.client.item(drive=self.repo.drive.id, id=item.id)))
return True
except (onedrivesdk.error.OneDriveError, OSError) as e:
logging.error('Error when creating remote dir of "%s": %s.', self.local_abspath, e)
return False
def handle(self):
logging.info('Moving item "%s" to "%s".', self.rel_path,
self.new_relpath)
# The routine assumes that the directory to save the new path exists remotely.
item_request = self.get_item_request()
try:
item_stat = os.stat(self.new_local_abspath)
item = item_request_call(self.repo, item_request.update,
self._get_new_item())
# TODO: update all records or rebuild records after deletion?
# self.repo.delete_item(self.item_name, self.parent_relpath, self.is_folder)
self.repo.move_item(item_name=self.item_name,
parent_relpath=self.parent_relpath,
new_name=self.new_name,
new_parent_relpath=self.new_parent_relpath,
is_folder=self.is_folder)
self.update_timestamp_and_record(item, item_stat)
return True
except (onedrivesdk.error.OneDriveError, OSError) as e:
logging.error('Error moving item "%s" to "%s": %s.', self.rel_path,
self.new_relpath, e)
return False
def test_item_request_call_on_connection_error(self, mock_sleep):
od_api_helper.item_request_call(None, self.dummy_api_call,
requests.ConnectionError())
mock_sleep.assert_called_once_with(od_api_helper.THROTTLE_PAUSE_SEC)
def _get_remote_item(repo, relpath):
item_request = repo.authenticator.client.item(drive=repo.drive.id, path=relpath)
try:
return item_request, item_request_call(repo, item_request.get)
except onedrivesdk.error.OneDriveError:
return item_request, None