def handle(self): logging.info('Downloading file "%s" to "%s".', self.remote_item.id, self.local_abspath) try: tmp_name = self.repo.path_filter.get_temp_name( self.remote_item.name) tmp_path = self.repo.local_root + self.parent_relpath + '/' + tmp_name item_request = self.repo.authenticator.client.item( drive=self.repo.drive.id, id=self.remote_item.id) item_mtime, item_mtime_editable = get_item_modified_datetime( self.remote_item) item_request_call(self.repo, item_request.download, tmp_path) hashes = self.remote_item.file.hashes if hashes is None or hashes.sha1_hash is None or hashes.sha1_hash == sha1_value( tmp_path): item_size_local = os.path.getsize(tmp_path) os.rename(tmp_path, self.local_abspath) fix_owner_and_timestamp(self.local_abspath, self.repo.context.user_uid, datetime_to_timestamp(item_mtime)) self.repo.update_item(self.remote_item, self.parent_relpath, item_size_local) logging.info('Finished downloading item "%s".', self.remote_item.id) return True else: # We assumed server's SHA-1 value is always correct -- might not be true. logging.error('Hash mismatch for downloaded file "%s".', self.local_abspath) os.remove(tmp_path) except (onedrivesdk.error.OneDriveError, OSError) as e: logging.error('Error when downloading file "%s": %s.', self.remote_item.id, e) return False
def handle(self): if not os.path.isdir(self.local_abspath): logging.error('Error: Local path "%s" is not a directory.' % self.local_abspath) return self.repo.context.watcher.rm_watch(self.repo, self.local_abspath) try: all_local_items = self.list_local_names() except (IOError, OSError) as e: logging.error('Error merging dir "%s": %s.', self.local_abspath, e) return all_records = self.repo.get_immediate_children_of_dir(self.rel_path) if not self.assume_remote_unchanged or not self.parent_remote_unchanged: try: remote_item_page = item_request_call(self.repo, self.item_request.children.get) all_remote_items = remote_item_page while True: # HACK: ChildrenCollectionPage is not guaranteed to have # the _next_page_link attribute and # ChildrenCollectionPage.get_next_page_request doesn't # implement the check correctly if not hasattr(remote_item_page, '_next_page_link'): break logging.debug('Paging for more items: %s', self.rel_path) remote_item_page = item_request_call( self.repo, ChildrenCollectionRequest.get_next_page_request( remote_item_page, self.repo.authenticator.client).get) all_remote_items = itertools.chain( all_remote_items, remote_item_page) except onedrivesdk.error.OneDriveError as e: logging.error('Encountered API Error: %s. Skip directory "%s".', e, self.rel_path) return for remote_item in all_remote_items: remote_is_folder = remote_item.folder is not None all_local_items.discard(remote_item.name) # Remove remote item from untouched list. if not self.repo.path_filter.should_ignore(self.rel_path + '/' + remote_item.name, remote_is_folder): self._handle_remote_item(remote_item, all_local_items, all_records) else: logging.debug('Ignored remote path "%s/%s".', self.rel_path, remote_item.name) for n in all_local_items: self._handle_local_item(n, all_records) for rec_name, rec in all_records.items(): logging.info('Record for item %s (%s/%s) is dead. Delete it it.', rec.item_id, rec.parent_path, rec_name) self.repo.delete_item(rec_name, rec.parent_path, is_folder=rec.type == ItemRecordType.FOLDER) self.repo.context.watcher.add_watch(self.repo, self.local_abspath)
def handle(self): logging.info('Deleting remote item "%s".', self.rel_path) item_request = self.get_item_request() try: od_api_helper.item_request_call(self.repo, item_request.delete) self.repo.delete_item(self.item_name, self.parent_relpath, self.is_folder) logging.info('Deleted remote item "%s".', self.rel_path) return True except (onedrivesdk.error.OneDriveError, OSError) as e: logging.error('Error deleting item "%s": %s.', self.rel_path, e) return False
def handle(self): logging.info('Updating webhook for Drive %s.', self.repo.drive.id) item_request = self.repo.authenticator.client.item( drive=self.repo.drive.id, path='/') expiration_time = datetime.utcnow() + timedelta( seconds=self.repo.context.config['webhook_renew_interval_sec']) try: if self.subscription_id is None: subscription = od_api_helper.create_subscription( item_request, self.repo, self.webhook_worker.webhook_url, expiration_time) else: subscription = onedrivesdk.Subscription() subscription.id = self.subscription_id subscription.notification_url = self.webhook_worker.webhook_url subscription.expiration_date_time = expiration_time subscription = od_api_helper.item_request_call( self.repo, item_request.subscriptions[self.subscription_id].update, subscription) self.webhook_worker.add_subscription(subscription, self.repo) logging.info('Webhook for Drive %s updated.', self.repo.drive.id) return subscription except onedrivesdk.error.OneDriveError as e: logging.error('Error: %s', e) return None
def test_item_request_call_on_unauthorized_error(self): account_id = 'dummy_acct' mock_repo = mock.MagicMock( account_id=account_id, **{ 'authenticator.refresh_session.return_value': 0, 'other.side_effect': KeyError }) od_api_helper.item_request_call( mock_repo, self.dummy_api_call, error.OneDriveError(prop_dict={ 'code': error.ErrorCode.Unauthenticated, 'message': 'dummy' }, status_code=requests.codes.unauthorized)) self.assertEqual(1, len(mock_repo.mock_calls)) name, args, _ = mock_repo.method_calls[0] self.assertEqual('authenticator.refresh_session', name) self.assertEqual((account_id, ), args)
def handle(self): logging.info('Uploading file "%s" to OneDrive.', self.local_abspath) occupy_task = self.task_pool.occupy_path(self.local_abspath, self) if occupy_task is not self: logging.warning('Cannot upload "%s" because %s.', self.local_abspath, "path is blacklisted" if occupy_task is None else str(occupy_task) + ' is in progress') return False try: item_stat = os.stat(self.local_abspath) if item_stat.st_size < self.PUT_FILE_SIZE_THRESHOLD_BYTES: item_request = self.parent_dir_request.children[self.item_name] returned_item = item_request_call(self.repo, item_request.upload, self.local_abspath) if returned_item is None: logging.warning('Upload API did not return metadata of remote item for "%s". ' 'Make an explicit request.', self.local_abspath) returned_item = item_request_call(self.repo, item_request.get) else: logging.info('Uploading large file "%s" in chunks of 10MB.', self.local_abspath) item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, path=self.rel_path) returned_item = item_request_call(self.repo, item_request.upload_async, local_path=self.local_abspath, upload_status=self.update_progress) if not isinstance(returned_item, onedrivesdk.Item): if hasattr(returned_item, '_prop_dict'): returned_item = onedrivesdk.Item(returned_item._prop_dict) else: returned_item = item_request_call(self.repo, item_request.get) self.update_timestamp_and_record(returned_item, item_stat) self.task_pool.release_path(self.local_abspath) logging.info('Finished uploading file "%s".', self.local_abspath) return True except (onedrivesdk.error.OneDriveError, OSError) as e: logging.error('Error uploading file "%s": %s.', self.local_abspath, e) # TODO: what if quota is exceeded? if (isinstance(e, onedrivesdk.error.OneDriveError) and e.code == onedrivesdk.error.ErrorCode.MalwareDetected): logging.warning('File "%s" was detected as malware by OneDrive. ' 'Do not upload during program session.', self.local_abspath) self.task_pool.occupy_path(self.local_abspath, None) return False self.task_pool.release_path(self.local_abspath) return False
def update_timestamp_and_record(self, new_item, item_local_stat): remote_mtime, remote_mtime_w = get_item_modified_datetime(new_item) if not remote_mtime_w: # last_modified_datetime attribute is not modifiable in OneDrive server. Update local mtime. fix_owner_and_timestamp(self.local_abspath, self.repo.context.user_uid, datetime_to_timestamp(remote_mtime)) else: file_system_info = FileSystemInfo() file_system_info.last_modified_date_time = datetime.utcfromtimestamp(item_local_stat.st_mtime) updated_item = Item() updated_item.file_system_info = file_system_info item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, id=new_item.id) new_item = item_request_call(self.repo, item_request.update, updated_item) self.repo.update_item(new_item, self.parent_relpath, item_local_stat.st_size)
def handle(self): logging.info('Updating timestamp for file "%s".', self.local_abspath) try: if not os.path.isfile(self.local_abspath): logging.warning('Local path "%s" is no longer a file. Cannot update timestamp.', self.local_abspath) return False item = item_request_call(self.repo, self.get_item_request().get) item_stat = os.stat(self.local_abspath) self.update_timestamp_and_record(item, item_stat) logging.info('Finished updating timestamp for file "%s".', self.local_abspath) return True except (onedrivesdk.error.OneDriveError, OSError) as e: logging.error('Error updating timestamp for file "%s": %s.', self.local_abspath, e) return False
def ensure_remote_path_is_dir(self, repo, rel_path): """ Make sure the path is a folder in remote repository. If the path does not exist, create it. If the path is a file, rename the file and create the dir. Return False if the remote path can't be made a dir. :param onedrive_client.od_repo.OneDriveLocalRepository repo: :param str rel_path: :return True | False: """ if rel_path == '': # Drive root is guaranteed a directory. return True item_request = repo.authenticator.client.item(drive=repo.drive.id, path=rel_path) parent_relpath, item_name = os.path.split(rel_path) if parent_relpath == '/': parent_relpath = '' try: item = item_request_call(repo, item_request.get) # Return True if the remote path exists and is a directory. if item.folder is not None: return item_name == item.name # Remote path is not a directory. Try renaming it and if renaming fails, deleting it. new_name = get_filename_with_incremented_count(item_name) logging.info('Remote item "%s" in Drive %s is not a directory. Try renaming it to "%s".', rel_path, repo.drive.id, new_name) if not move_item.MoveItemTask(repo=repo, task_pool=self.task_pool, parent_relpath=parent_relpath, item_name=item_name, new_name=new_name, is_folder=False).handle(): if not delete_item.DeleteRemoteItemTask(repo=repo, task_pool=self.task_pool, parent_relpath=parent_relpath, item_name=item_name, is_folder=False).handle(): logging.warning('Failed to rename or delete remote item "%s" in Drive %s.', rel_path, repo.drive.id) return False except onedrivesdk.error.OneDriveError as e: if e.code != onedrivesdk.error.ErrorCode.ItemNotFound: return False if not merge_dir.CreateFolderTask(repo=repo, task_pool=self.task_pool, item_name=item_name, parent_relpath=parent_relpath, upload_if_success=False, abort_if_local_gone=True).handle(): logging.critical('Failed to create remote directory "%s" on Drive %s.', rel_path, repo.drive.id) return False return True
def handle(self): logging.info('Creating remote item for local dir "%s".', self.local_abspath) try: if self.abort_if_local_gone and not os.path.isdir(self.local_abspath): logging.warning('Local dir "%s" is gone. Skip creating remote item for it.', self.local_abspath) return item = self._get_folder_pseudo_item(self.item_name) item_request = self._get_item_request() item = item_request_call(self.repo, item_request.children.add, item) self.repo.update_item(item, self.parent_relpath, 0) logging.info('Created remote item for local dir "%s".', self.local_abspath) if self.upload_if_success: logging.info('Adding task to merge "%s" after remote item was created.', self.local_abspath) self.task_pool.add_task(MergeDirectoryTask( self.repo, self.task_pool, self.parent_relpath + '/' + self.item_name, self.repo.authenticator.client.item(drive=self.repo.drive.id, id=item.id))) return True except (onedrivesdk.error.OneDriveError, OSError) as e: logging.error('Error when creating remote dir of "%s": %s.', self.local_abspath, e) return False
def handle(self): logging.info('Moving item "%s" to "%s".', self.rel_path, self.new_relpath) # The routine assumes that the directory to save the new path exists remotely. item_request = self.get_item_request() try: item_stat = os.stat(self.new_local_abspath) item = item_request_call(self.repo, item_request.update, self._get_new_item()) # TODO: update all records or rebuild records after deletion? # self.repo.delete_item(self.item_name, self.parent_relpath, self.is_folder) self.repo.move_item(item_name=self.item_name, parent_relpath=self.parent_relpath, new_name=self.new_name, new_parent_relpath=self.new_parent_relpath, is_folder=self.is_folder) self.update_timestamp_and_record(item, item_stat) return True except (onedrivesdk.error.OneDriveError, OSError) as e: logging.error('Error moving item "%s" to "%s": %s.', self.rel_path, self.new_relpath, e) return False
def test_item_request_call_on_connection_error(self, mock_sleep): od_api_helper.item_request_call(None, self.dummy_api_call, requests.ConnectionError()) mock_sleep.assert_called_once_with(od_api_helper.THROTTLE_PAUSE_SEC)
def _get_remote_item(repo, relpath): item_request = repo.authenticator.client.item(drive=repo.drive.id, path=relpath) try: return item_request, item_request_call(repo, item_request.get) except onedrivesdk.error.OneDriveError: return item_request, None