def __init__(self, config):
'''*config* is a dictionary with the keys user, password, and URL. For instance::
#url can also contain an existing subfolder to access, i.e. https://webdav.mediencenter.t-online.de/myfolder
#url can also contain the port for the WebDAV server, i.e. https://webdav.mediencenter.t-online.de:443
config['url'] = 'https://webdav.mediencenter.t-online.de'
config['user'] = '******' #your account username/e-mail address
config['password'] = '******' #your account password
:param config: dictionary with key value pairs'''
super(WebdavStore, self).__init__()
self.name = 'webdav'
self._logging_handler = self.name
self.logger = logging.getLogger(self._logging_handler)
self.logger = db_logging_thread.make_logger_multiprocessingsave(
self.logger)
self.logger.info("creating %s store", self.name)
self.tinyclient = TinyDAVClient(config['url'], config['user'],
config['password'])
self.logger.info("api initialized")
def __init__(self, config):
'''*config* is a dictionary with the keys user, password, and URL. For instance::
#url can also contain an existing subfolder to access, i.e. https://webdav.mediencenter.t-online.de/myfolder
#url can also contain the port for the WebDAV server, i.e. https://webdav.mediencenter.t-online.de:443
config['url'] = 'https://webdav.mediencenter.t-online.de'
config['user'] = '******' #your account username/e-mail address
config['password'] = '******' #your account password
:param config: dictionary with key value pairs'''
super(WebdavStore, self).__init__()
self.name = 'webdav'
self._logging_handler = self.name
self.logger = logging.getLogger(self._logging_handler)
self.logger = db_logging_thread.make_logger_multiprocessingsave(self.logger)
self.logger.info("creating %s store", self.name)
self.tinyclient = TinyDAVClient(config['url'], config['user'], config['password'] )
self.logger.info("api initialized")
class WebdavStore(Store):
def __init__(self, config):
'''*config* is a dictionary with the keys user, password, and URL. For instance::
#url can also contain an existing subfolder to access, i.e. https://webdav.mediencenter.t-online.de/myfolder
#url can also contain the port for the WebDAV server, i.e. https://webdav.mediencenter.t-online.de:443
config['url'] = 'https://webdav.mediencenter.t-online.de'
config['user'] = '******' #your account username/e-mail address
config['password'] = '******' #your account password
:param config: dictionary with key value pairs'''
super(WebdavStore, self).__init__()
self.name = 'webdav'
self._logging_handler = self.name
self.logger = logging.getLogger(self._logging_handler)
self.logger = db_logging_thread.make_logger_multiprocessingsave(
self.logger)
self.logger.info("creating %s store", self.name)
self.tinyclient = TinyDAVClient(config['url'], config['user'],
config['password'])
self.logger.info("api initialized")
def __deepcopy__(self, memo):
from copy import deepcopy
cls = self.__class__
result = cls.__new__(cls)
memo[id(self)] = result
for k, v in self.__dict__.items():
if k == 'logger':
setattr(result, k, self.logger)
elif k == '_logging_handler':
setattr(result, k, self._logging_handler)
elif k == 'tinyclient':
setattr(result, k, self.tinyclient)
else:
setattr(result, k, deepcopy(v, memo))
return result
def get_name(self):
self.logger.info("getting name")
return self.name
@retry((Exception), tries=14, delay=0.1, backoff=2)
def get_file(self, path_to_file):
self.logger.info("getting file: %s", path_to_file)
self._raise_error_if_invalid_path(path_to_file)
return self.tinyclient.get_file(path_to_file)
def __get_size(self, fileobject):
pos = fileobject.tell()
fileobject.seek(0, 2)
size = fileobject.tell()
fileobject.seek(pos, 0)
return size
@retry((Exception), tries=2, delay=0)
def store_fileobject(self, fileobject, path, interrupt_event=None):
size = self.__get_size(fileobject)
self.logger.info("Storing file object of size %s to %s", size, path)
if hasattr(fileobject, 'name') and fileobject.name is not None:
file_name = fileobject.name
else:
with tempfile.NamedTemporaryFile(delete=False) as fh:
for line in fileobject:
fh.write(line)
fh.flush()
file_name = fh.name
self.tinyclient.upload(file_name, path)
return int(time.time())
# worst case: object still exists and takes up space or is appended to, by mistake
# with caching_store, the entry in cache is deleted anyways
@retry((Exception), tries=5, delay=0)
def delete(self, path, is_dir=False):
self.logger.info("deleting %s", path)
self._raise_error_if_invalid_path(path)
if is_dir:
self.tinyclient.rmdir(path)
else:
self.tinyclient.rm(path)
def account_info(self):
self.logger.debug("retrieving account info")
return "Webdav"
@retry((Exception), tries=14, delay=0.1, backoff=2)
def create_directory(self, directory):
self.logger.info("creating directory %s", directory)
self.tinyclient.mkdir(directory)
@retry((Exception), tries=1)
def duplicate(self, path_to_src, path_to_dest):
self.logger.info("duplicating %s to %s", path_to_src, path_to_dest)
self.tinyclient.copy(path_to_src, path_to_dest)
@retry((Exception), tries=1)
def move(self, path_to_src, path_to_dest):
self.logger.info("moving %s to %s", path_to_src, path_to_dest)
self.tinyclient.move(path_to_src, path_to_dest)
@retry((Exception), tries=4, delay=0.1, backoff=2)
def get_overall_space(self):
self.logger.debug("retrieving all space")
return self.tinyclient.get_overall_space()
@retry((Exception), tries=4, delay=0.1,
backoff=2) #Got HTTP error BAD REQUEST 400 from t-online once
def get_used_space(self):
self.logger.debug("retrieving used space")
return self.tinyclient.get_used_space()
@retry((Exception), tries=14, delay=0.1, backoff=2)
def get_directory_listing(self, directory):
self.logger.debug("getting directory listing for %s", directory)
return self.tinyclient.get_directory_listing(directory)
def _handle_error(self, error, stacktrace, method_name, remaining_tries,
*args, **kwargs):
if method_name == 'get_file': #box.com does not instantly see files that are written to it (eventual consistency)
if isinstance(error,
NoSuchFilesytemObjectError) and remaining_tries != 0:
return False
if isinstance(error, NoSuchFilesytemObjectError) or \
isinstance(error, AlreadyExistsError) or \
isinstance(error, StoreAccessError):
self.logger.error("Error could not be handled: \n%s", stacktrace)
raise error
if remaining_tries == 0: # throw error after last try
raise StoreAccessError(str(error), 0)
return False
@retry((Exception), tries=14, delay=0.1, backoff=2)
def get_metadata(self, path):
self.logger.debug("getting metadata for %s", path)
return self.tinyclient.get_metadata(path)
def _get_time_difference(self):
self.logger.debug("getting time difference")
return 0
def get_logging_handler(self):
return self._logging_handler
def get_max_filesize(self):
"""Return maximum number of bytes per file"""
return 1000 * 1000 * 1000 * 1000
class WebdavStore(Store):
def __init__(self, config):
'''*config* is a dictionary with the keys user, password, and URL. For instance::
#url can also contain an existing subfolder to access, i.e. https://webdav.mediencenter.t-online.de/myfolder
#url can also contain the port for the WebDAV server, i.e. https://webdav.mediencenter.t-online.de:443
config['url'] = 'https://webdav.mediencenter.t-online.de'
config['user'] = '******' #your account username/e-mail address
config['password'] = '******' #your account password
:param config: dictionary with key value pairs'''
super(WebdavStore, self).__init__()
self.name = 'webdav'
self._logging_handler = self.name
self.logger = logging.getLogger(self._logging_handler)
self.logger = db_logging_thread.make_logger_multiprocessingsave(self.logger)
self.logger.info("creating %s store", self.name)
self.tinyclient = TinyDAVClient(config['url'], config['user'], config['password'] )
self.logger.info("api initialized")
def __deepcopy__(self, memo):
from copy import deepcopy
cls = self.__class__
result = cls.__new__(cls)
memo[id(self)] = result
for k, v in self.__dict__.items():
if k == 'logger':
setattr(result, k, self.logger)
elif k == '_logging_handler':
setattr(result, k, self._logging_handler)
elif k == 'tinyclient':
setattr(result, k, self.tinyclient)
else:
setattr(result, k, deepcopy(v, memo))
return result
def get_name(self):
self.logger.info("getting name")
return self.name
@retry((Exception), tries=14, delay=0.1, backoff=2)
def get_file(self, path_to_file):
self.logger.debug("getting file: %s", path_to_file)
self._raise_error_if_invalid_path(path_to_file)
return self.tinyclient.get_file(path_to_file)
def __get_size(self, fileobject):
pos = fileobject.tell()
fileobject.seek(0,2)
size = fileobject.tell()
fileobject.seek(pos, 0)
return size
@retry((Exception), tries=2, delay=0)
def store_fileobject(self, fileobject, path, interrupt_event=None):
size = self.__get_size(fileobject)
self.logger.debug("Storing file object of size %s to %s", size, path)
if hasattr(fileobject, 'name'):
file_name = fileobject.name
else:
with tempfile.NamedTemporaryFile(delete=False) as fh:
for line in fileobject:
fh.write(line)
fh.flush()
file_name = fh.name
self.tinyclient.upload(file_name, path)
return int(time.time())
# worst case: object still exists and takes up space or is appended to, by mistake
# with caching_store, the entry in cache is deleted anyways
@retry((Exception), tries=5, delay=0)
def delete(self, path, is_dir=False):
self.logger.debug("deleting %s", path)
self._raise_error_if_invalid_path(path)
if is_dir:
self.tinyclient.rmdir(path)
else:
self.tinyclient.rm(path)
def account_info(self):
self.logger.debug("retrieving account info")
return "Webdav"
@retry((Exception), tries=14, delay=0.1, backoff=2)
def create_directory(self, directory):
self.logger.debug("creating directory %s", directory)
self.tinyclient.mkdir(directory)
@retry((Exception), tries=1)
def duplicate(self, path_to_src, path_to_dest):
self.logger.debug("duplicating %s to %s", path_to_src, path_to_dest)
self.tinyclient.copy(path_to_src, path_to_dest)
@retry((Exception), tries=1)
def move(self, path_to_src, path_to_dest):
self.logger.debug("moving %s to %s", path_to_src, path_to_dest)
self.tinyclient.move(path_to_src, path_to_dest)
@retry((Exception), tries=4, delay=0.1, backoff=2)
def get_overall_space(self):
self.logger.debug("retrieving all space")
return self.tinyclient.get_overall_space()
@retry((Exception), tries=4, delay=0.1, backoff=2) #Got HTTP error BAD REQUEST 400 from t-online once
def get_used_space(self):
self.logger.debug("retrieving used space")
return self.tinyclient.get_used_space()
@retry((Exception), tries=14, delay=0.1, backoff=2)
def get_directory_listing(self, directory):
self.logger.debug("getting directory listing for %s", directory)
return self.tinyclient.get_directory_listing(directory)
def _handle_error(self, error, method_name, remaining_tries, *args, **kwargs):
if method_name == 'get_file': #box.com does not instantly see files that are written to it (eventual consistency)
if isinstance(error, NoSuchFilesytemObjectError) and remaining_tries != 0:
return False
if isinstance(error, NoSuchFilesytemObjectError) or \
isinstance(error, AlreadyExistsError) or \
isinstance(error, StoreAccessError):
self.logger.error("Error could not be handled: %s", error)
raise error
if remaining_tries == 0: # throw error after last try
raise StoreAccessError(str(error), 0)
return False
@retry((Exception), tries=14, delay=0.1, backoff=2)
def get_metadata(self, path):
self.logger.debug("getting metadata for %s", path)
return self.tinyclient.get_metadata(path)
def _get_time_difference(self):
self.logger.debug("getting time difference")
return 0
def get_logging_handler(self):
return self._logging_handler
def get_max_filesize(self):
"""Return maximum number of bytes per file"""
return 1000*1000*1000*1000
