def update_file_content(self, file_id, content):
"""Update the file's contents of the given id on google drive."""
item = self.cache.get(file_id, None)
if not item or item.metadata is None or item.expired:
try:
metadata = self.service.files().get(fileId=file_id).execute()
except errors.HttpError as e:
raise OperationFailedError(opname='update_file_content',
msg=e.resp.reason)
except:
return self._retry_operation(self.update_file_content,
file_id, content)
self.cache.set(metadata['id'], metadata)
else:
metadata = item.metadata
media_body = MediaInMemoryUpload(content)
try:
updated_file = self.service.files().update(fileId=file_id,
body=metadata,
media_body=media_body
).execute()
except errors.HttpError as e:
raise OperationFailedError(opname='update_file_content',
msg=e.resp.reason)
except TypeError as e:
raise ResourceInvalidError("update_file_content %r" % e)
except:
return self._retry_operation(self.update_file_content, file_id,
content)
self.cache.pop(file_id, None)
self._add_to_cache_from_dict(updated_file, updated_file['parents'])
return updated_file
def file_create_folder(self, parent_id, title):
"""Add newly created directory to cache."""
body = {
"title": title,
"parents": [{
"id": parent_id
}],
"mimeType": "application/vnd.google-apps.folder"
}
try:
metadata = self.service.files().insert(body=body).execute()
except errors.HttpError as e:
if e.resp.status == 405:
raise ResourceInvalidError(parent_id)
if e.resp.status == 404:
raise ResourceNotFoundError(parent_id)
raise OperationFailedError(opname='file_create_folder',
msg="%s, the reasons could be: parent "
"doesn't exist or is a file" %
(e.resp.reason, ))
except:
return self._retry_operation(self.file_create_folder, parent_id,
title)
self.cache.set(metadata["id"], metadata, parents=[parent_id])
return metadata
def remove_versions_before(self, path, version):
"""Removes snapshots before a specified version.
The specified version can be either a version (int) or a time (str)
in the following format: '%Y-%m-%dT%H:%M:%S'
"""
if not self.exists(path):
raise ResourceNotFoundError(path)
if not self.isfile(path):
raise OperationFailedError(path)
# if the version number is a string, try converting it into an int
if isinstance(version, str) or isinstance(version, unicode):
try:
if str(version).isdigit():
version = int(version)
except ValueError:
raise VersionError("Invalid version.")
if isinstance(version, int):
current_version = self.version(path)
# Versions can't be deleted before version 1 or after the current
if version > current_version or version <= 1:
raise VersionError("Invalid version.")
date_to_delete = self.list_info(path)[version]
else:
# check for an invalid timestamp string
if not is_valid_time_format(version):
raise VersionError("Invalid time format.")
date_to_delete = version
snap_dir = self.snapshot_snap_path(path)
command = [
'rdiff-backup', '--parsable-output', '--force',
'--remove-older-than',
str(date_to_delete), '--tempdir',
self.tmp.getsyspath('/'), snap_dir
]
process = Popen(command, stdout=PIPE, stderr=PIPE)
stderr = process.communicate()[1]
if len(stderr) > 0:
raise OperationFailedError(path)
def setcontents(self, path, *args, **kwargs):
if self.writefs is None:
raise OperationFailedError('setcontents',
path=path,
msg='No writeable FS set')
try:
with _autocreate_missing_writefs_directory(self, path, 'w'):
return self.writefs.setcontents(path, *args, **kwargs)
except CreatedDirectory:
return self.writefs.setcontents(path, *args, **kwargs)
def file_copy(self, src, dst):
"""Copy a file to another location."""
try:
metadata = super(DropboxClient, self).file_copy(src, dst)
except rest.ErrorResponse as e:
if e.status == 404:
raise ResourceNotFoundError(src)
if e.status == 403:
raise DestinationExistsError(dst)
if e.status == 503:
raise OperationFailedError(opname='file_copy',
msg="User over storage quota")
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.set(dst, metadata)
return metadata['path']
def file_delete(self, path):
"""Delete a file."""
try:
self.service.files().delete(fileId=path).execute()
except errors.HttpError as e:
if e.resp.status == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='file_delete', msg=e.resp.reason)
except:
return self._retry_operation(self.file_delete, path)
self.cache.pop(path, None)
def file_delete(self, path):
try:
super(OneDriveClient, self).delete(path)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='file_delete', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.pop(path, None)
def file_copy(self, src, dst):
try:
metadata = super(OneDriveClient, self).copy(src, dst, False)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(
"Parent or source file don't exist")
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.set(metadata['id'], metadata, parent=dst)
return metadata['id']
def file_delete(self, path):
"""Delete a file of a give path."""
try:
super(DropboxClient, self).file_delete(path)
except rest.ErrorResponse as e:
if e.status == 404:
raise ResourceNotFoundError(path)
if e.status == 400 and 'must not be empty' in str(e):
raise DirectoryNotEmptyError(path)
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.pop(path, None)
def update_file(self, file_id, new_file):
"""Update the file's contents of the given id."""
try:
metadata = self.service.files().update(fileId=file_id,
body=new_file).execute()
except errors.HttpError as e:
if e.resp.status == 404:
raise ResourceNotFoundError(
"Parent or source file don't exist.")
raise OperationFailedError(opname='update_file', msg=e.resp.reason)
except:
return self._retry_operation(self.update_file, file_id, new_file)
self.cache.pop(file_id, None)
self._add_to_cache_from_dict(metadata, metadata['parents'])
return metadata
def get_file(self, path):
"""Get file of a given path."""
item = self.cache.get(path)
if not item or item.metadata is None or item.expired:
try:
metadata = self.service.files().get(fileId=path).execute()
except errors.HttpError as e:
if e.resp.status == 404:
raise ResourceNotFoundError("Source file doesn't exist")
raise OperationFailedError(opname='get_file',
msg=e.resp.reason)
except:
return self._retry_operation(self.get_file, path)
self._add_to_cache_from_dict(metadata, metadata['parents'])
else:
item = self.cache[path]
metadata = item.metadata
download_url = metadata.get('downloadUrl')
resp, content = self.service._http.request(download_url)
if resp.status == 200:
return content
else:
raise OperationFailedError(opname="get_file", msg=str(resp))
def put_file(self, parent_id, title, content, overwrite=False):
try:
metadata = super(OneDriveClient, self).put((title, content),
parent_id,
overwrite=overwrite)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(parent_id)
raise OperationFailedError(opname='put_copy', msg=str(e))
except TypeError as e:
raise ResourceInvalidError("put_file")
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
return metadata['id']
def update_file(self, file_id, new_file_info):
try:
metadata = super(OneDriveClient,
self).info_update(file_id, new_file_info)
except api_v5.ProtocolError as e:
if e.resp.status == 404:
raise ResourceNotFoundError(path=file_id)
raise OperationFailedError(opname='update_file', msg=e.resp.reason)
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.pop(file_id, None)
self.cache.set(metadata['id'], metadata, parent=metadata['parent_id'])
return metadata['id']
def put_file(self, path, f, overwrite=False):
"""Upload a file."""
try:
response = super(DropboxClient, self).put_file(path,
f,
overwrite=overwrite)
except rest.ErrorResponse as e:
raise OperationFailedError(opname='file_copy', msg=str(e))
except TypeError as e:
raise ResourceInvalidError("put_file", path)
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.pop(dirname(path), None)
return response
def media(self, path):
"""Media."""
try:
info = super(DropboxClient, self).media(path)
return info.get('url', None)
except rest.ErrorResponse as e:
if e.status == 400:
raise UnsupportedError("create a link to a folder")
if e.status == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
def file_create_folder(self, path):
"""Add newly created directory to cache."""
try:
metadata = super(DropboxClient, self).file_create_folder(path)
except rest.ErrorResponse as e:
if e.status == 403:
raise DestinationExistsError(path)
if e.status == 400:
raise OperationFailedError(opname='file_create_folder',
msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.set(path, metadata)
return metadata['path']
def children(self, path):
"""Get children of a given path."""
update = False
hash_ = None
item = self.cache.get(path)
if item:
if item.expired:
update = True
if item.metadata and item.children:
hash_ = item.metadata['hash']
else:
if not item.metadata.get('is_dir'):
raise ResourceInvalidError(path)
if not item.children:
update = True
else:
update = True
if update:
try:
metadata = super(DropboxClient,
self).metadata(path,
hash=hash_,
include_deleted=False,
list=True)
children = []
contents = metadata.pop('contents')
for child in contents:
if child.get('is_deleted', False):
continue
children.append(basename(child['path']))
self.cache[child['path']] = CacheItem(child)
item = self.cache[path] = CacheItem(metadata, children)
except rest.ErrorResponse as e:
if not item or e.status != 304:
raise OperationFailedError(opname='metadata',
path=path,
msg=str(e))
# We have an item from cache (perhaps expired), but it's
# hash is still valid (as far as Dropbox is concerned),
# so just renew it and keep using it.
item.renew()
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
return item.children
def file_create_folder(self, parent_id, title):
"Add newly created directory to cache."
try:
metadata = super(OneDriveClient, self).mkdir(title, parent_id)
except api_v5.ProtocolError as e:
if e.code == 405:
raise ResourceInvalidError(parent_id)
if e.code == 404:
raise ResourceNotFoundError(parent_id)
raise OperationFailedError(opname='file_create_folder', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.set(metadata["id"], metadata, parent=parent_id)
return metadata['id']
def file_copy(self, file_id, parent_id):
"""Copy a file to a folder of a given id."""
body = {"parents": [{"id": parent_id}]}
try:
metadata = self.service.files().copy(fileId=file_id,
body=body,
).execute()
except errors.HttpError as e:
if e.resp.status == 404:
raise ResourceNotFoundError(
"Parent or source file don't exist.")
raise OperationFailedError(opname='file_copy', msg=e.resp.reason)
except:
return self._retry_operation(self.file_copy, file_id, parent_id)
self.cache.set(metadata['id'], metadata, parents=[parent_id])
return metadata
def metadata(self, path):
"""Get metadata of a given path."""
item = self.cache.get(path)
if not item or item.metadata is None or item.expired:
try:
metadata = self.service.files().get(fileId=path).execute()
except errors.HttpError as e:
if e.resp.status == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='metadata', path=path,
msg=e.resp.reason)
except:
return self._retry_operation(self.metadata, path)
if metadata.get('trashed', False):
raise ResourceNotFoundError(path)
item = self.cache[path] = CacheItem(metadata)
# Copy the info so the caller cannot affect our cache.
return dict(item.metadata.items())
def metadata(self, path):
"Gets metadata for a given path."
item = self.cache.get(path)
if not item or item.metadata is None or item.expired:
try:
metadata = super(OneDriveClient, self).info(path)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='metadata',
path=path,
msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
item = self.cache[path] = CacheItem(metadata)
# Copy the info so the caller cannot affect our cache.
return dict(item.metadata.items())
def children(self, path):
"Gets children of a given path."
update = False
item = self.cache.get(path)
if item:
if item.expired:
update = True
else:
if item.metadata["type"] != "folder" and \
not ("folder" in path):
raise ResourceInvalidError(path)
if not item.children:
update = True
else:
update = True
if update:
try:
metadata = super(OneDriveClient, self).info(path)
if metadata["type"] != "folder" and not ("folder" in path):
raise ResourceInvalidError(path)
children = []
contents = super(OneDriveClient, self).listdir(path)
for child in contents:
children.append(child['id'])
self.cache[child['id']] = CacheItem(child, parent=path)
item = self.cache[path] = CacheItem(metadata, children)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError(path)
if not item or e.resp.status != 304:
raise OperationFailedError(opname='metadata',
path=path,
msg=str(e))
# We have an item from cache (perhaps expired), but it's
# hash is still valid (as far as OneDrive is concerned),
# so just renew it and keep using it.
item.renew()
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
return item.children
def put_file(self, parent_id, title, content, description=None):
"""Add a file to folder of the given id."""
media_body = MediaInMemoryUpload(content)
body = {
'title': title,
'description': description,
'parents': [{'id': parent_id}]
}
try:
metadata = self.service.files().insert(body=body,
media_body=media_body
).execute()
except errors.HttpError as e:
raise OperationFailedError(opname='put_file', msg=e.resp.reason)
except TypeError as e:
raise ResourceInvalidError("put_file")
except:
return self._retry_operation(self.put_file, parent_id, title,
content, description)
self.cache.set(metadata['id'], metadata, parents=[parent_id])
return metadata
def children(self, path):
"""Get children of a given path."""
update = False
item = self.cache.get(path)
if item:
if item.expired:
update = True
else:
if item.metadata["mimeType"] != GD_FOLDER:
raise ResourceInvalidError(path)
if not item.children:
update = True
else:
update = True
if update:
try:
metadata = self.service.files().get(fileId=path).execute()
if metadata["mimeType"] != GD_FOLDER:
raise ResourceInvalidError(path)
param = {"q": "trashed=false and '%s' in parents" % path}
children = []
filesResource = self.service.files().list(**param).execute()
for child in filesResource['items']:
children.append(child['id'])
self.cache[child['id']] = CacheItem(child, parents=[path])
item = self.cache[path] = CacheItem(metadata, children)
except errors.HttpError as e:
if e.resp.status == 404:
raise ResourceNotFoundError(path)
if not item or e.resp.status != 304:
raise OperationFailedError(opname='metadata',
path=path,
msg=e.resp.reason)
# We have an item from cache (perhaps expired),
# but it's still valid (as far as GoogleDrive is
# concerned), so just renew it and keep using it.
item.renew()
except:
return self._retry_operation(self.children, path)
return item.children
def get_file(self, path):
metadata = None
if not self.cache.get(path, None):
try:
metadata = super(OneDriveClient, self).info(path)
except api_v5.ProtocolError as e:
if e.code == 404:
raise ResourceNotFoundError("Source file doesn't exist")
raise OperationFailedError(opname='get_file', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired "
"or user credentials are invalid.")
self.cache.set(metadata['id'],
metadata,
parent=metadata['parent_id'])
else:
item = self.cache[path]
metadata = item.metadata
return super(OneDriveClient, self).get(path)
def metadata(self, path):
"""Get metadata for a given path."""
item = self.cache.get(path)
if not item or item.metadata is None or item.expired:
try:
metadata = super(DropboxClient,
self).metadata(path,
include_deleted=False,
list=False)
except rest.ErrorResponse as e:
if e.status == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='metadata',
path=path,
msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
if metadata.get('is_deleted', False):
raise ResourceNotFoundError(path)
item = self.cache[path] = CacheItem(metadata)
# Copy the info so the caller cannot affect our cache.
return dict(item.metadata.items())
def makedir(self, path, *args, **kwargs):
if self.writefs is None:
raise OperationFailedError('makedir',
path=path,
msg='No writeable FS set')
return self.writefs.makedir(path, *args, **kwargs)
def remove(self, path):
if self.writefs is None:
raise OperationFailedError('remove',
path=path,
msg='No writeable FS set')
self.writefs.remove(path)
