def open(self, path, mode='r', **kwargs):
if 'w' in mode or '+' in mode or 'a' in mode:
logging.error('cannot use httpfs.open() in write mode: %s' % path)
raise UnsupportedError('open', path=path)
url = self._build_url(path)
for attempt in (1, 2, 3):
try:
response = requests.get(url)
except requests.RequestException as error:
logging.warning('open attempt %d: %s %s' %
(attempt, url, error))
else:
break
else:
raise RemoteConnectionError('getinfo', path)
if response.status_code == 200:
return StringIO(response.content)
elif response.status_code == 404:
raise ResourceNotFoundError(path)
else:
logging.warning(
'open status %d for %s assumed as connection error.' %
(response.status_code, url))
raise RemoteConnectionError('open', path)
def getpathurl(self, path, allow_none=False):
"""Returns a url that corresponds to the given path, if one exists.
If the path does not have an equivalent URL form (and allow_none is False)
then a :class:`~fs.errors.NoPathURLError` exception is thrown. Otherwise the URL will be
returns as an unicode string.
@param path: object id for which to return url path
@param allow_none: if true, this method can return None if there is no
URL form of the given path
@type allow_none: bool
@raises `fs.errors.NoPathURLError`: If no URL form exists, and allow_none is False (the default)
@rtype: unicode
"""
url = None
try:
url = self.getinfo(path)['source']
except RemoteConnectionError as e:
raise RemoteConnectionError(e)
except:
if not allow_none:
raise NoPathURLError(path=path)
return url
def xrd_ping(self):
"""Ping xrootd server.
Specific to ``XRootDPyFS``.
"""
status, dummy = self._client.ping()
if not status.ok:
raise RemoteConnectionError(opname="ping", details=status)
return True
def _getinfo(self, path):
url = self._build_url(path)
for attempt in (1, 2, 3):
try:
response = requests.head(url)
except requests.RequestException as error:
logging.warning('getinfo attempt %d: %s %s' %
(attempt, url, error))
else:
break
else:
raise RemoteConnectionError('getinfo', path)
if response.status_code == 200:
if 'content-length' in response.headers:
size_in_bytes = int(response.headers['content-length'])
if 'last-modified' in response.headers:
last_modified = parse_http_date(
response.headers['last-modified'])
else:
last_modified = None
file_time = last_modified or datetime.datetime.now()
return {
'size': size_in_bytes,
'created_time': file_time,
'accessed_time': file_time,
'modified_time': file_time,
}
logging.warning('getinfo missing required headers: %s' %
response.headers)
raise RemoteConnectionError('getinfo', path)
elif response.status_code in (301, 302, 403, 404):
# Pretend that redirects and forbiddens are 404s.
raise ResourceNotFoundError(path)
else:
logging.warning(
'getinfo status %d for %s assumed as connection error.' %
(response.status_code, url))
raise RemoteConnectionError('getinfo', path)
def file_delete(self, path):
try:
super(OneDriveClient, self).delete(path)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='file_delete', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.pop(path, None)
def exists(self, path):
"""
Checks if a the specified path exists
@param path: Id of the file/folder to check
"""
try:
return self.client.metadata(path)
except RemoteConnectionError as e:
raise RemoteConnectionError(e)
except:
return False
def file_copy(self, src, dst):
try:
metadata = super(OneDriveClient, self).copy(src, dst, False)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(
"Parent or source file don't exist")
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.set(metadata['id'], metadata, parent=dst)
return metadata['id']
def _runMethod(*args, **kwargs):
def prepare_error(exp):
return ''.join([
'run command "', name, '", args: ',
str(args), ', kwargs: ',
str(kwargs), ', smbc exception: ',
str(exp)
])
try:
method = getattr(self._ctx, name)
return method(*args, **kwargs)
except smbc.TimedOutError, e:
raise RemoteConnectionError(prepare_error(e), details=e)
def file_delete(self, path):
"""Delete a file of a give path."""
try:
super(DropboxClient, self).file_delete(path)
except rest.ErrorResponse as e:
if e.status == 404:
raise ResourceNotFoundError(path)
if e.status == 400 and 'must not be empty' in str(e):
raise DirectoryNotEmptyError(path)
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.pop(path, None)
def _retry_operation(self, method, *args):
"""Method retries an operation.
Sometimes access_token expires and we need to rebuild it using
the refresh token. This method does that and retries the
operation that failed.
"""
if self._retry < 5:
self._retry += 1
self.service = self._build_service()
return method(*args)
else:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
def media(self, path):
"""Media."""
try:
info = super(DropboxClient, self).media(path)
return info.get('url', None)
except rest.ErrorResponse as e:
if e.status == 400:
raise UnsupportedError("create a link to a folder")
if e.status == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
def put_file(self, path, f, overwrite=False):
"""Upload a file."""
try:
response = super(DropboxClient, self).put_file(path,
f,
overwrite=overwrite)
except rest.ErrorResponse as e:
raise OperationFailedError(opname='file_copy', msg=str(e))
except TypeError as e:
raise ResourceInvalidError("put_file", path)
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.pop(dirname(path), None)
return response
def file_create_folder(self, path):
"""Add newly created directory to cache."""
try:
metadata = super(DropboxClient, self).file_create_folder(path)
except rest.ErrorResponse as e:
if e.status == 403:
raise DestinationExistsError(path)
if e.status == 400:
raise OperationFailedError(opname='file_create_folder',
msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.set(path, metadata)
return metadata['path']
def put_file(self, parent_id, title, content, overwrite=False):
try:
metadata = super(OneDriveClient, self).put((title, content),
parent_id,
overwrite=overwrite)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(parent_id)
raise OperationFailedError(opname='put_copy', msg=str(e))
except TypeError as e:
raise ResourceInvalidError("put_file")
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
return metadata['id']
def update_file(self, file_id, new_file_info):
try:
metadata = super(OneDriveClient,
self).info_update(file_id, new_file_info)
except api_v5.ProtocolError as e:
if e.resp.status == 404:
raise ResourceNotFoundError(path=file_id)
raise OperationFailedError(opname='update_file', msg=e.resp.reason)
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.pop(file_id, None)
self.cache.set(metadata['id'], metadata, parent=metadata['parent_id'])
return metadata['id']
def file_create_folder(self, parent_id, title):
"Add newly created directory to cache."
try:
metadata = super(OneDriveClient, self).mkdir(title, parent_id)
except api_v5.ProtocolError as e:
if e.code == 405:
raise ResourceInvalidError(parent_id)
if e.code == 404:
raise ResourceNotFoundError(parent_id)
raise OperationFailedError(opname='file_create_folder', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.set(metadata["id"], metadata, parent=parent_id)
return metadata['id']
def children(self, path):
"""Get children of a given path."""
update = False
hash_ = None
item = self.cache.get(path)
if item:
if item.expired:
update = True
if item.metadata and item.children:
hash_ = item.metadata['hash']
else:
if not item.metadata.get('is_dir'):
raise ResourceInvalidError(path)
if not item.children:
update = True
else:
update = True
if update:
try:
metadata = super(DropboxClient,
self).metadata(path,
hash=hash_,
include_deleted=False,
list=True)
children = []
contents = metadata.pop('contents')
for child in contents:
if child.get('is_deleted', False):
continue
children.append(basename(child['path']))
self.cache[child['path']] = CacheItem(child)
item = self.cache[path] = CacheItem(metadata, children)
except rest.ErrorResponse as e:
if not item or e.status != 304:
raise OperationFailedError(opname='metadata',
path=path,
msg=str(e))
# We have an item from cache (perhaps expired), but it's
# hash is still valid (as far as Dropbox is concerned),
# so just renew it and keep using it.
item.renew()
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
return item.children
def file_copy(self, src, dst):
"""Copy a file to another location."""
try:
metadata = super(DropboxClient, self).file_copy(src, dst)
except rest.ErrorResponse as e:
if e.status == 404:
raise ResourceNotFoundError(src)
if e.status == 403:
raise DestinationExistsError(dst)
if e.status == 503:
raise OperationFailedError(opname='file_copy',
msg="User over storage quota")
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.set(dst, metadata)
return metadata['path']
def metadata(self, path):
"Gets metadata for a given path."
item = self.cache.get(path)
if not item or item.metadata is None or item.expired:
try:
metadata = super(OneDriveClient, self).info(path)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='metadata',
path=path,
msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
item = self.cache[path] = CacheItem(metadata)
# Copy the info so the caller cannot affect our cache.
return dict(item.metadata.items())
def children(self, path):
"Gets children of a given path."
update = False
item = self.cache.get(path)
if item:
if item.expired:
update = True
else:
if item.metadata["type"] != "folder" and \
not ("folder" in path):
raise ResourceInvalidError(path)
if not item.children:
update = True
else:
update = True
if update:
try:
metadata = super(OneDriveClient, self).info(path)
if metadata["type"] != "folder" and not ("folder" in path):
raise ResourceInvalidError(path)
children = []
contents = super(OneDriveClient, self).listdir(path)
for child in contents:
children.append(child['id'])
self.cache[child['id']] = CacheItem(child, parent=path)
item = self.cache[path] = CacheItem(metadata, children)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(path)
if not item or e.resp.status != 304:
raise OperationFailedError(opname='metadata',
path=path,
msg=str(e))
# We have an item from cache (perhaps expired), but it's
# hash is still valid (as far as OneDrive is concerned),
# so just renew it and keep using it.
item.renew()
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
return item.children
def get_file(self, path):
metadata = None
if not self.cache.get(path, None):
try:
metadata = super(OneDriveClient, self).info(path)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError("Source file doesn't exist")
raise OperationFailedError(opname='get_file', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.set(metadata['id'],
metadata,
parent=metadata['parent_id'])
else:
item = self.cache[path]
metadata = item.metadata
return super(OneDriveClient, self).get(path)
def metadata(self, path):
"""Get metadata for a given path."""
item = self.cache.get(path)
if not item or item.metadata is None or item.expired:
try:
metadata = super(DropboxClient,
self).metadata(path,
include_deleted=False,
list=False)
except rest.ErrorResponse as e:
if e.status == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='metadata',
path=path,
msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
if metadata.get('is_deleted', False):
raise ResourceNotFoundError(path)
item = self.cache[path] = CacheItem(metadata)
# Copy the info so the caller cannot affect our cache.
return dict(item.metadata.items())
def inner(*args, **kwargs):
try:
return outer(*args, **kwargs)
except socket.gaierror as e:
if e.errno == socket.EAI_NONAME:
raise RemoteConnectionError(str(e), details=e)
raise
except socket.error as e:
if e.errno in (errno.ECONNREFUSED, errno.EPIPE, errno.ETIMEDOUT):
raise RemoteConnectionError(str(e), details=e)
raise
except OperationFailure as e:
# Cycle through each message and map the first one to PyFilesystem
# that is not a successful status (0x00000000).
share = args[0]
path = args[1]
for msg in e.smb_messages:
# Protocol versions expose the error values differently.
if msg.protocol == 1:
msg_status = msg.status.internal_value
else:
msg_status = msg.status
if msg_status == 0x0:
continue
elif msg_status == 0x103:
# Unknown error, but message says it is not found.
raise ResourceNotFoundError(path=path)
elif msg_status == 0xc000000f:
raise ResourceNotFoundError(path=path)
elif msg_status == 0xc0000022:
raise PermissionDeniedError('access', path=path)
elif msg_status == 0xc0000033:
raise ResourceInvalidError(path=path)
elif msg_status == 0xc0000034:
raise ResourceNotFoundError(path=path)
elif msg_status == 0xc0000035:
raise DestinationExistsError(path=path)
elif msg_status == 0xc000003a:
raise ResourceNotFoundError(path=path)
elif msg_status == 0xc0000056:
raise DeletePendingError(path=path)
elif msg_status == 0xc000006f:
raise AccountTimeRestrictedError(path=path)
elif msg_status == 0xc0000071:
raise PasswordExpiredError(path=path)
elif msg_status == 0xc00000ba:
raise ResourceInvalidError(path=path)
elif msg_status == 0xc00000cc:
# Share does not exist.
raise ResourceInvalidError(path=share)
elif msg_status == 0xc00000d0:
raise ResourceInvalidError(path=path)
elif msg_status == 0xc0000101:
raise DirectoryNotEmptyError(path=path)
elif msg_status == 0xc0000103:
raise ResourceInvalidError(path=path)
elif msg_status == 0xc0000193:
raise AccountExpiredError(path=path)
elif msg_status == 0xc0000224:
raise PasswordChangeRequiredError(path=path)
elif msg_status == 0xc0000234:
raise AccountLockedError(path=path)
else:
raise Exception('Unhandled SMB error: {0}'.format(
hex(msg_status)))
raise
except NotConnectedError as e:
# Connection does not exist or was disconnected. Using the wrong
# NetBIOS name can cause this against a Windows server while Samba
# will ignore it.
raise RemoteConnectionError(details=e)
except NotReadyError as e:
# Connection has not been successfully authenticated.
raise RemoteConnectionError(details=e)
