def __exit__(self, ty, ex, tb): # pylint: disable=too-many-branches
self.__box._mutex.__exit__(ty, ex, tb)
if ex:
try:
raise ex
except (TimeoutError,):
self.__box.disconnect()
raise CloudDisconnectedError("disconnected on timeout")
except BoxOAuthException as e:
self.__box.disconnect()
raise CloudTokenError("oauth fail %s" % e)
except BoxNetworkException as e:
self.__box.disconenct()
raise CloudDisconnectedError("disconnected %s" % e)
except BoxValueError:
raise CloudFileNotFoundError()
except BoxAPIException as e:
if e.status == 400 and e.code == 'folder_not_empty':
raise CloudFileExistsError()
if e.status == 404 and e.code == 'not_found':
raise CloudFileNotFoundError()
if e.status == 404 and e.code == 'trashed':
raise CloudFileNotFoundError()
if e.status == 405 and e.code == 'method_not_allowed':
raise PermissionError()
if e.status == 409 and e.code == 'item_name_in_use':
raise CloudFileExistsError()
if e.status == 400 and e.code == 'invalid_grant':
raise CloudTokenError()
log.exception("unknown box exception: \n%s", e)
except CloudException:
raise
except Exception:
pass # this will not swallow the exception, because this is in a context manager
def connect_impl(self, creds):
log.debug('Connecting to dropbox')
with self._mutex:
if not self._client or creds != self._creds:
if creds:
self._creds = creds
api_key = creds.get('key', creds.get('access_token'))
else:
raise CloudTokenError("no creds")
if not api_key:
raise CloudTokenError("no api key/access token")
self._client = Dropbox(api_key)
self._longpoll_client = Dropbox(api_key)
try:
self.__memoize_quota.clear()
info = self.__memoize_quota()
self._long_poll_manager.start()
return info['uid']
except CloudTokenError:
self.disconnect()
raise
except Exception as e:
self.disconnect()
if isinstance(e, exceptions.AuthError):
log.debug("auth error connecting %s", e)
raise CloudTokenError(str(e))
if isinstance(e, exceptions.BadInputError):
log.debug("auth error connecting %s", e)
raise CloudTokenError(str(e))
log.exception("error connecting %s", e)
raise CloudDisconnectedError()
def create(self, path, file_like, metadata=None) -> 'OInfo':
if not metadata:
metadata = {}
if self.exists_path(path):
raise CloudFileExistsError()
ul, size = self._media_io(file_like)
fields = 'id, md5Checksum, size, modifiedTime'
# Cache is accurate, just refreshed from exists_path() call
parent_oid = self._get_parent_id(path, use_cache=True)
metadata['appProperties'] = self._prep_app_properties(parent_oid)
gdrive_info = self._prep_upload(path, metadata)
gdrive_info['parents'] = [parent_oid]
try:
def api_call():
return self._api('files',
'create',
body=gdrive_info,
media_body=ul,
fields=fields)
if self._client:
with patch.object(self._client._http.http, "follow_redirects",
False): # pylint: disable=protected-access
res = api_call()
else:
res = api_call()
except OSError as e:
self.disconnect()
raise CloudDisconnectedError("OSError in file create: %s" %
repr(e))
log.debug("response from create %s : %s", path, res)
if not res:
raise CloudTemporaryError("unknown response from drive on upload")
self._ids[path] = res['id']
log.debug("path cache %s", self._ids)
size = int(res.get("size", 0))
mtime = res.get('modifiedTime')
mtime = mtime and self._parse_time(mtime)
cache_ent = self.get_quota.get() # pylint: disable=no-member
if cache_ent:
cache_ent["used"] += size
return OInfo(otype=FILE,
oid=res['id'],
hash=res['md5Checksum'],
path=path,
size=size,
mtime=mtime)
def connect_impl(self, creds):
log.debug('Connecting to box')
if not self.__client or creds != self.__creds:
try:
if creds:
self.__creds = creds
else:
raise CloudTokenError("no creds")
jwt_token = creds.get('jwt_token')
access_token = creds.get('access_token')
refresh_token = creds.get('refresh_token')
if not jwt_token:
if not ((self._oauth_config.app_id and self._oauth_config.app_secret) and (refresh_token or access_token)):
raise CloudTokenError("require app_id/secret and either access_token or refresh token")
with self._mutex:
box_session = Session()
box_kwargs = box_session.get_constructor_kwargs()
box_kwargs["api_config"] = boxsdk.config.API
box_kwargs["default_network_request_kwargs"] = {"timeout": 60}
if jwt_token:
jwt_dict = json.loads(jwt_token)
user_id = creds.get('user_id')
auth = JWTAuth.from_settings_dictionary(jwt_dict, user=user_id,
store_tokens=self._store_refresh_token)
else:
if not refresh_token:
raise CloudTokenError("Missing refresh token")
auth = OAuth2(client_id=self._oauth_config.app_id,
client_secret=self._oauth_config.app_secret,
access_token=access_token,
refresh_token=refresh_token,
store_tokens=self._store_refresh_token)
box_session = AuthorizedSession(auth, **box_kwargs)
self.__client = Client(auth, box_session)
with self._api():
self.__access_token = auth.access_token
self._long_poll_manager.start()
except BoxNetworkException as e:
log.exception("Error during connect %s", e)
self.disconnect()
raise CloudDisconnectedError()
except (CloudTokenError, CloudDisconnectedError):
raise
except Exception as e:
log.exception("Error during connect %s", e)
self.disconnect()
raise CloudTokenError()
with self._api() as client:
return client.user(user_id='me').get().id
def upload(self, oid, file_like, metadata=None) -> 'OInfo':
if not metadata:
metadata = {}
gdrive_info = self._prep_upload(None, metadata)
ul, size = self._media_io(file_like)
fields = 'id, md5Checksum, modifiedTime'
try:
def api_call():
return self._api('files',
'update',
body=gdrive_info,
fileId=oid,
media_body=ul,
fields=fields)
if self._client:
with patch.object(self._client._http.http, "follow_redirects",
False): # pylint: disable=protected-access
res = api_call()
else:
res = api_call()
except OSError as e:
self.disconnect()
raise CloudDisconnectedError("OSError in file upload: %s" %
repr(e))
log.debug("response from upload %s", res)
if not res:
raise CloudTemporaryError("unknown response from drive on upload")
mtime = res.get('modifiedTime')
mtime = mtime and self._parse_time(mtime)
md5 = res.get(
'md5Checksum',
None) # can be none if the user tries to upload to a folder
if md5 is None:
possible_conflict = self._info_oid(oid)
if possible_conflict and possible_conflict.otype == DIRECTORY:
raise CloudFileExistsError("Can only upload to a file: %s" %
possible_conflict.path)
return OInfo(otype=FILE,
oid=res['id'],
hash=md5,
path=None,
size=size,
mtime=mtime)
def connect_impl(self, creds):
log.debug('Connecting to googledrive')
if not self._client or creds != self._creds:
if creds:
self._creds = creds
else:
raise CloudTokenError("no creds")
refresh_token = creds and creds.get('refresh_token')
if not refresh_token:
raise CloudTokenError(
"acquire a token using authenticate() first")
try:
new = self._oauth_config.refresh(self._oauth_info.token_url,
refresh_token,
scope=self._oauth_info.scopes)
google_creds = google.oauth2.credentials.Credentials(
new.access_token,
new.refresh_token,
scopes=self._oauth_info.scopes)
self._client = build('drive',
'v3',
credentials=google_creds,
cache_discovery=False)
try:
self.get_quota.clear() # pylint: disable=no-member
quota = self.get_quota()
except SSLError: # pragma: no cover
# Seeing some intermittent SSL failures that resolve on retry
log.warning('Retrying intermittent SSLError')
quota = self.get_quota()
self._creds = creds
return quota['permissionId']
except OAuthError as e:
self.disconnect()
raise CloudTokenError(repr(e))
except CloudTokenError:
self.disconnect()
raise
except Exception as e:
raise CloudDisconnectedError(repr(e))
return self.connection_id
def _api(self, *args, **kwargs) -> 'BoxProvider._BoxProviderGuard':
needs_client = kwargs.get('needs_client', True)
if needs_client and not self.__client:
raise CloudDisconnectedError("currently disconnected")
return self._BoxProviderGuard(self.__client, self)
def _real_api(self, client, mutex, method, *args, **kwargs): # pylint: disable=too-many-branches, too-many-statements
log.debug("_api: %s (%s)", method, debug_args(args, kwargs))
with mutex:
if not client:
raise CloudDisconnectedError("currently disconnected")
try:
return getattr(client, method)(*args, **kwargs)
except exceptions.AuthError:
self.disconnect()
raise CloudTokenError()
except exceptions.ApiError as e:
inside_error: Union[files.LookupError, files.WriteError]
if isinstance(e.error,
(files.ListFolderError, files.GetMetadataError,
files.ListRevisionsError)):
if e.error.is_path() and isinstance(
e.error.get_path(), files.LookupError):
inside_error = e.error.get_path()
if inside_error.is_malformed_path():
log.debug(
'Malformed path when executing %s(%s %s) : %s',
*debug_args(method, args, kwargs, e))
raise CloudFileNotFoundError(
'Malformed path when executing %s(%s)' %
debug_args(method, kwargs))
if inside_error.is_not_found():
log.debug('File not found %s(%s %s) : %s',
*debug_args(method, args, kwargs, e))
raise CloudFileNotFoundError(
'File not found when executing %s(%s)' %
debug_args(method, kwargs))
if inside_error.is_not_folder():
log.debug(
'Expected folder is actually a file when executing %s(%s %s) : %s',
*debug_args(method, args, kwargs, e))
raise CloudFileExistsError(
'Expected folder is actually a file when executing %s(%s %s)'
% debug_args(method, args, kwargs))
if isinstance(e.error, sharing.SharedFolderAccessError):
raise CloudFileNotFoundError(str(e))
if isinstance(e.error, files.UploadError):
if e.error.is_path() and isinstance(
e.error.get_path(), files.UploadWriteFailed):
inside_error = e.error.get_path()
write_error = inside_error.reason
if write_error.is_insufficient_space():
log.debug('out of space %s(%s %s) : %s',
*debug_args(method, args, kwargs, e))
raise CloudOutOfSpaceError(
'Out of space when executing %s(%s)' %
debug_args(method, kwargs))
if write_error.is_conflict():
raise CloudFileExistsError(
'Conflict when executing %s(%s)' %
debug_args(method, kwargs))
if isinstance(e.error, files.DownloadError):
if e.error.is_path() and isinstance(
e.error.get_path(), files.LookupError):
inside_error = e.error.get_path()
if inside_error.is_not_found():
raise CloudFileNotFoundError(
"Not found when executing %s(%s)" %
debug_args(method, kwargs))
if isinstance(e.error, files.DeleteError):
if e.error.is_path_lookup():
inside_error = e.error.get_path_lookup()
if inside_error.is_not_found():
log.debug('file not found %s(%s %s) : %s',
*debug_args(method, args, kwargs, e))
raise CloudFileNotFoundError(
'File not found when executing %s(%s)' %
debug_args(method, kwargs))
if isinstance(e.error, files.RelocationError):
if e.error.is_from_lookup():
inside_error = e.error.get_from_lookup()
if inside_error.is_not_found():
log.debug('file not found %s(%s %s) : %s',
*debug_args(method, args, kwargs, e))
raise CloudFileNotFoundError(
'File not found when executing %s(%s,%s)' %
debug_args(method, args, kwargs))
if e.error.is_to():
inside_error = e.error.get_to()
if inside_error.is_conflict():
raise CloudFileExistsError(
'File already exists when executing %s(%s)' %
debug_args(method, kwargs))
if e.error.is_duplicated_or_nested_paths():
raise CloudFileExistsError(
'Duplicated or nested path %s(%s)' %
debug_args(method, kwargs))
if isinstance(e.error, files.CreateFolderError):
if e.error.is_path() and isinstance(
e.error.get_path(), files.WriteError):
inside_error = e.error.get_path()
if inside_error.is_conflict():
raise CloudFileExistsError(
'File already exists when executing %s(%s)' %
debug_args(method, kwargs))
if isinstance(e.error, files.ListFolderContinueError):
# all list-folder-continue errors should cause a cursor reset
# these include the actual "is_reset" and, of course a is_path (fnf)
# and also is_other which can secretly contain a reset as well
raise CloudCursorError("Cursor reset request")
if isinstance(e.error, files.ListRevisionsError):
if e.error.is_path():
inside_error = e.error.get_path()
if inside_error.is_not_file():
raise NotAFileError(str(e))
if isinstance(e.error, files.ListFolderLongpollError):
raise CloudCursorError(
"cursor invalidated during longpoll")
raise CloudException(
"Unknown exception when executing %s(%s,%s): %s" %
debug_args(method, args, kwargs, e))
except (exceptions.InternalServerError, exceptions.RateLimitError,
requests.exceptions.ReadTimeout):
raise CloudTemporaryError()
except dropbox.stone_validators.ValidationError as e:
log.debug("f*ed up api error: %s", e)
if "never created" in str(e):
raise CloudFileNotFoundError()
if "did not match" in str(e):
log.warning("oid error %s", e)
raise CloudFileNotFoundError()
raise
except requests.exceptions.ConnectionError as e:
log.error('api error handled exception %s:%s', "dropbox",
e.__class__.__name__)
self.disconnect()
raise CloudDisconnectedError()
def _api(self, resource, method, *args, **kwargs): # pylint: disable=arguments-differ, too-many-branches, too-many-statements
if not self._client:
raise CloudDisconnectedError("currently disconnected")
with self._mutex:
try:
if resource == 'media':
res = args[0]
args = args[1:]
else:
res = getattr(self._client, resource)()
meth = getattr(res, method)(*args, **kwargs)
if resource == 'media' or (resource == 'files'
and method == 'get_media'):
ret = meth
else:
ret = meth.execute()
log.debug("api: %s (%s) -> %s", method,
debug_args(args, kwargs), ret)
return ret
except SSLError as e:
if "WRONG_VERSION" in str(e):
# httplib2 used by google's api gives this weird error for no discernable reason
raise CloudTemporaryError(str(e))
raise
except google.auth.exceptions.RefreshError:
self.disconnect()
raise CloudTokenError("refresh error")
except HttpError as e:
log.debug("api: %s (%s) -> %s", method,
debug_args(args, kwargs), e.resp.status)
if str(e.resp.status) == '416':
raise GDriveFileDoneError()
if str(e.resp.status) == '413':
raise CloudOutOfSpaceError('Payload too large')
if str(e.resp.status) == '409':
raise CloudFileExistsError('Another user is modifying')
if str(e.resp.status) == '404':
raise CloudFileNotFoundError(
'File not found when executing %s.%s(%s)' %
debug_args(resource, method, kwargs))
reason = self._get_reason_from_http_error(e)
if str(e.resp.status) == '403' and str(
reason) == 'storageQuotaExceeded':
raise CloudOutOfSpaceError("Storage storageQuotaExceeded")
if str(e.resp.status) == '401':
self.disconnect()
raise CloudTokenError("Unauthorized %s" % reason)
if str(e.resp.status) == '403' and str(
reason) == 'parentNotAFolder':
raise CloudFileExistsError("Parent Not A Folder")
if str(e.resp.status) == '403' and str(
reason) == 'insufficientFilePermissions':
raise PermissionError("PermissionError")
if (str(e.resp.status) == '403' and reason in (
'userRateLimitExceeded', 'rateLimitExceeded', 'dailyLimitExceeded')) \
or str(e.resp.status) == '429':
raise CloudTemporaryError("rate limit hit")
# At this point, _should_retry_response() returns true for error codes >=500, 429, and 403 with
# the reason 'userRateLimitExceeded' or 'rateLimitExceeded'. 403 without content, or any other
# response is not retried. We have already taken care of some of those cases above, but we call this
# below to catch the rest, and in case they improve their library with more conditions. If we called
# meth.execute() above with a num_retries argument, all this retrying would happen in the google api
# library, and we wouldn't have to think about retries.
should_retry = _should_retry_response(e.resp.status, e.content)
if should_retry:
raise CloudTemporaryError("unknown error %s" % e)
log.error("Unhandled %s error %s", e.resp.status, reason)
raise
except (TimeoutError, HttpLib2Error):
self.disconnect()
raise CloudDisconnectedError("disconnected on timeout")
except ConnectionResetError:
raise CloudTemporaryError(
"An existing connection was forcibly closed by the remote host"
)