class WebDavConnection:
def __init__(self, host, username, password):
self.host = host
self.username = username
self.password = password
def connect(self):
options = {
'webdav_hostname': self.host,
'webdav_login': self.username,
'webdav_password': self.password
}
self.connection = WebdavClient(options)
def get_modified(self, path):
modified = self.connection.info(path)['modified']
format = '%a, %d %b %Y %X %Z'
timestamp_tzaware = parser.parse(modified)
# fromtimestamp() does not include the timezone.
# So stripping it here.
return datetime.datetime.fromtimestamp(timestamp_tzaware.timestamp())
def download(self, remote_path, local_path):
self.connection.download_file(remote_path, local_path)
def upload(self, remote_path, local_path):
self.connection.upload_file(remote_path, local_path)
class connector:
def __init__(self, config):
self.config = config
options = {
'webdav_hostname': config['Connector']['Host'],
'webdav_login': config['Connector']['Username'],
'webdav_password': config['Connector']['Password']
}
self.client = Client(options)
self.client.verfiy = False
self.base = config['Connector']['Base']
def listdir(self):
l = self.client.list(self.base)
l.pop(l.index(self.base + '/'))
print(l)
return l
def remove(self, name):
self.client.clean(self.base + '/' + name)
def check(self, name):
return self.client.check(self.base + '/' + name)
def get(self, name):
return self.client.resource(self.base + '/' + name)
def create(self, name):
self.client.upload_to(None, self.base + '/' + name)
def upload(self, name):
self.client.upload_file(self.base + '/' + name, name)
class DavSync(Sync):
"""DavSync synchronizes the changes with a webdav server.
Attributes:
path (:obj:`str`): The local directory path to synchronize.
dav_path (:obj:`str`): The remote directory path which corresponds to the local one.
dav_root (:obj:`str`): The path to access the dav server. Nextcloud, e.g., uses /remote.php/webdav/
username (:obj:`str`): Webdav username.
password (:obj:`str`): Webdav password.
hostname (:obj:`str`): Webdav server host, e.g. https://cloud.example.com/
"""
def __init__(
self,
path: str,
dav_path: str,
dav_root: str,
username: str,
password: str,
hostname: str,
):
super().__init__(path)
self.dav_path = dav_path
self.username = username
self.password = password
self.hostname = hostname
options = {
"webdav_hostname": hostname,
"webdav_login": username,
"webdav_password": password,
"root": dav_root,
}
self.client = Client(options)
self.pull()
def pull(self):
"""Download updated directory from server."""
self.client.download_sync(self.dav_path, self.os_path)
def push(self, fname, msg=""):
"""Upload a file to the server. If the directory or file does not exist
on the remote server, create it.
Args:
fname (:obj:`str`): File to upload.
msg (:obj:`str`, optional): Not used with DavSync.
"""
self.client.upload_file(path.join(self.dav_path, fname),
path.join(self.os_path, fname))
def sync_to_webdav(url, username, password):
client = Client({
"webdav_hostname": url,
"webdav_login": username,
"webdav_password": password,
"webdav_root": ROOT_DIR,
})
files = (
"/etc/certs/otp_configuration.json",
"/etc/certs/super_gluu_creds.json",
)
for file_ in files:
try:
logger.info(f"Sync {file_} to {url}{ROOT_DIR}{file_}")
client.mkdir("/etc")
client.mkdir("/etc/certs")
client.upload_file(file_, file_)
except (RemoteResourceNotFound, NoConnection,
RemoteParentNotFound) as exc:
logger.warning(
f"Unable to sync {file_} to {url}{ROOT_DIR}{file_}; reason={exc}"
)
class BaseClientTestCase(unittest.TestCase):
remote_path_file = 'test_dir/test.txt'
remote_path_file2 = 'test_dir2/test.txt'
remote_inner_path_file = 'test_dir/inner/test.txt'
remote_path_dir = 'test_dir'
remote_path_dir2 = 'test_dir2'
remote_inner_path_dir = 'test_dir/inner'
local_base_dir = 'tests/'
local_file = 'test.txt'
local_file_path = local_base_dir + 'test.txt'
local_path_dir = local_base_dir + 'res/test_dir'
options = {
'webdav_hostname': 'http://localhost:8585',
'webdav_login': '******',
'webdav_password': '******'
}
# options = {
# 'webdav_hostname': 'https://webdav.yandex.ru',
# 'webdav_login': '******',
# 'webdav_password': '******'
# }
def setUp(self):
self.client = Client(self.options)
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
def tearDown(self):
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
if self.client.check(remote_path=self.remote_path_dir):
self.client.clean(remote_path=self.remote_path_dir)
if self.client.check(remote_path=self.remote_path_dir2):
self.client.clean(remote_path=self.remote_path_dir2)
def _prepare_for_downloading(self, inner_dir=False):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not self.client.check(remote_path=self.remote_path_file):
self.client.upload_file(remote_path=self.remote_path_file,
local_path=self.local_file_path)
if not path.exists(self.local_path_dir):
os.makedirs(self.local_path_dir)
if inner_dir:
if not self.client.check(remote_path=self.remote_inner_path_dir):
self.client.mkdir(remote_path=self.remote_inner_path_dir)
if not self.client.check(remote_path=self.remote_inner_path_file):
self.client.upload_file(
remote_path=self.remote_inner_path_file,
local_path=self.local_file_path)
def _prepare_for_uploading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not path.exists(path=self.local_path_dir):
os.makedirs(self.local_path_dir)
if not path.exists(path=self.local_path_dir + os.sep +
self.local_file):
shutil.copy(src=self.local_file_path,
dst=self.local_path_dir + os.sep + self.local_file)
class BaseClientTestCase(unittest.TestCase):
remote_path_file = 'test_dir/test.txt'
remote_compressed_path_file = 'test_dir/compressed.txt'
remote_path_file2 = 'test_dir2/test.txt'
remote_inner_path_file = 'test_dir/inner/test.txt'
remote_path_dir = 'test_dir'
remote_path_dir2 = 'test_dir2'
remote_inner_path_dir = 'test_dir/inner'
inner_dir_name = 'inner'
local_base_dir = 'tests/'
local_file = 'test.txt'
local_file_path = local_base_dir + 'test.txt'
local_compressed_file_path = local_base_dir + 'compressed.txt'
local_path_dir = local_base_dir + 'res/test_dir'
options = {
'webdav_hostname': 'http://localhost:8585',
'webdav_login': '******',
'webdav_password': '******',
'webdav_timeout': 10,
'webdav_override_methods': {
'check': 'GET'
}
}
# options = {
# 'webdav_hostname': 'https://demo1.nextcloud.com/remote.php/dav/files/RCw8Y9XXFnzkLJbN/',
# 'webdav_login': '******',
# 'webdav_password': '******',
# 'webdav_override_methods': {
# 'check': 'GET'
# }
# }
# options = {
# 'webdav_hostname': 'https://webdav.yandex.ru',
# 'webdav_login': '******',
# 'webdav_password': '******'
# }
def setUp(self):
self.client = Client(self.options)
self.clean_local_dir(self.local_path_dir)
def tearDown(self):
self.clean_local_dir(self.local_path_dir)
self.clean_remote_dir(self.remote_path_dir)
self.clean_remote_dir(self.remote_path_dir2)
def clean_remote_dir(self, remote_path_dir):
if self.client.check(remote_path=remote_path_dir):
self.client.clean(remote_path=remote_path_dir)
@staticmethod
def clean_local_dir(local_path_dir):
if path.exists(path=local_path_dir):
shutil.rmtree(path=local_path_dir)
def _prepare_for_downloading(self, inner_dir=False, base_path=''):
if base_path:
self._create_remote_dir_if_needed(base_path)
self._prepare_dir_for_downloading(base_path + self.remote_path_dir,
base_path + self.remote_path_file,
self.local_file_path)
if not path.exists(self.local_path_dir):
os.makedirs(self.local_path_dir)
if inner_dir:
self._prepare_dir_for_downloading(
base_path + self.remote_inner_path_dir,
base_path + self.remote_inner_path_file, self.local_file_path)
def _prepare_dir_for_downloading(self, remote_path_dir, remote_path_file,
local_file_path):
self._create_remote_dir_if_needed(remote_path_dir)
if not self.client.check(remote_path=remote_path_file):
self.client.upload_file(remote_path=remote_path_file,
local_path=local_file_path)
def _create_remote_dir_if_needed(self, remote_dir):
if not self.client.check(remote_path=remote_dir):
self.client.mkdir(remote_path=remote_dir)
def _prepare_for_uploading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not path.exists(path=self.local_path_dir):
os.makedirs(self.local_path_dir)
if not path.exists(path=self.local_path_dir + os.sep +
self.local_file):
shutil.copy(src=self.local_file_path,
dst=self.local_path_dir + os.sep + self.local_file)
class ClientTestCase(TestCase):
remote_path_file = 'test_dir/test.txt'
remote_path_file2 = 'test_dir2/test.txt'
remote_path_dir = 'test_dir'
remote_path_dir2 = 'test_dir2'
local_base_dir = 'tests/'
local_file = 'test.txt'
local_file_path = local_base_dir + 'test.txt'
local_path_dir = local_base_dir + 'res/test_dir'
def setUp(self):
options = {
'webdav_hostname': 'https://webdav.yandex.ru',
'webdav_login': '******',
'webdav_password': '******'
}
self.client = Client(options)
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
def tearDown(self):
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
if self.client.check(remote_path=self.remote_path_dir):
self.client.clean(remote_path=self.remote_path_dir)
if self.client.check(remote_path=self.remote_path_dir2):
self.client.clean(remote_path=self.remote_path_dir2)
def test_list(self):
self._prepare_for_downloading()
file_list = self.client.list()
self.assertIsNotNone(file_list, 'List of files should not be None')
self.assertGreater(file_list.__len__(), 0,
'Expected that amount of files more then 0')
def test_free(self):
self.assertGreater(
self.client.free(), 0,
'Expected that free space on WebDAV server is more then 0 bytes')
def test_check(self):
self.assertTrue(self.client.check(),
'Expected that root directory is exist')
def test_mkdir(self):
if self.client.check(remote_path=self.remote_path_dir):
self.client.clean(remote_path=self.remote_path_dir)
self.client.mkdir(remote_path=self.remote_path_dir)
self.assertTrue(self.client.check(remote_path=self.remote_path_dir),
'Expected the directory is created.')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_to(self):
self._prepare_for_downloading()
buff = BytesIO()
self.client.download_from(buff=buff, remote_path=self.remote_path_file)
self.assertEquals(buff.getvalue(),
'test content for testing of webdav client')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download(self):
self._prepare_for_downloading()
self.client.download(local_path=self.local_path_dir,
remote_path=self.remote_path_dir)
self.assertTrue(path.exists(self.local_path_dir),
'Expected the directory is downloaded.')
self.assertTrue(path.isdir(self.local_path_dir),
'Expected this is a directory.')
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_path_file), 'Expected this is a file')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_sync(self):
self._prepare_for_downloading()
os.mkdir(self.local_path_dir)
def callback():
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep +
self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_file), 'Expected this is a file')
self.client.download_sync(local_path=self.local_path_dir +
os.path.sep + self.local_file,
remote_path=self.remote_path_file,
callback=callback)
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file has already been downloaded')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_async(self):
self._prepare_for_downloading()
os.mkdir(self.local_path_dir)
def callback():
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep +
self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_file), 'Expected this is a file')
self.client.download_async(local_path=self.local_path_dir +
os.path.sep + self.local_file,
remote_path=self.remote_path_file,
callback=callback)
self.assertFalse(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file has not been downloaded yet')
def test_upload_from(self):
self._prepare_for_uploading()
buff = StringIO(u'test content for testing of webdav client')
self.client.upload_to(buff=buff, remote_path=self.remote_path_file)
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
def test_upload(self):
self._prepare_for_uploading()
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
def test_upload_file(self):
self._prepare_for_uploading()
self.client.upload_file(remote_path=self.remote_path_file,
local_path=self.local_file_path)
self.assertTrue(self.client.check(remote_path=self.remote_path_file),
'Expected the file is uploaded.')
def test_upload_sync(self):
self._prepare_for_uploading()
def callback():
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
def test_upload_async(self):
self._prepare_for_uploading()
def callback():
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
def test_copy(self):
self._prepare_for_downloading()
self.client.mkdir(remote_path=self.remote_path_dir2)
self.client.copy(remote_path_from=self.remote_path_file,
remote_path_to=self.remote_path_file2)
self.assertTrue(self.client.check(remote_path=self.remote_path_file2))
def test_move(self):
self._prepare_for_downloading()
self.client.mkdir(remote_path=self.remote_path_dir2)
self.client.move(remote_path_from=self.remote_path_file,
remote_path_to=self.remote_path_file2)
self.assertFalse(self.client.check(remote_path=self.remote_path_file))
self.assertTrue(self.client.check(remote_path=self.remote_path_file2))
def test_clean(self):
self._prepare_for_downloading()
self.client.clean(remote_path=self.remote_path_dir)
self.assertFalse(self.client.check(remote_path=self.remote_path_file))
self.assertFalse(self.client.check(remote_path=self.remote_path_dir))
def test_info(self):
self._prepare_for_downloading()
result = self.client.info(remote_path=self.remote_path_file)
self.assertEquals(result['name'], 'test.txt')
self.assertEquals(result['size'], '41')
self.assertTrue('created' in result)
self.assertTrue('modified' in result)
def test_directory_is_dir(self):
self._prepare_for_downloading()
self.assertTrue(self.client.is_dir(self.remote_path_dir),
'Should return True for directory')
def test_file_is_not_dir(self):
self._prepare_for_downloading()
self.assertFalse(self.client.is_dir(self.remote_path_file),
'Should return False for file')
def test_get_property_of_non_exist(self):
self._prepare_for_downloading()
result = self.client.get_property(remote_path=self.remote_path_file,
option={'name': 'aProperty'})
self.assertEquals(
result, None, 'For not found property should return value as None')
def test_set_property(self):
self._prepare_for_downloading()
self.client.set_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty',
'value': 'aValue'
})
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty'
})
self.assertEquals(result, 'aValue', 'Property value should be set')
def test_set_property_batch(self):
self._prepare_for_downloading()
self.client.set_property_batch(remote_path=self.remote_path_file,
option=[{
'namespace': 'test',
'name': 'aProperty',
'value': 'aValue'
}, {
'namespace': 'test',
'name': 'aProperty2',
'value': 'aValue2'
}])
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty'
})
self.assertEquals(result, 'aValue',
'First property value should be set')
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty2'
})
self.assertEquals(result, 'aValue2',
'Second property value should be set')
def _prepare_for_downloading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not self.client.check(remote_path=self.remote_path_file):
self.client.upload_file(remote_path=self.remote_path_file,
local_path=self.local_file_path)
if not path.exists(self.local_path_dir):
os.makedirs(self.local_path_dir)
def _prepare_for_uploading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not path.exists(path=self.local_path_dir):
os.makedirs(self.local_path_dir)
if not path.exists(path=self.local_path_dir + os.sep +
self.local_file):
shutil.copy(src=self.local_file_path,
dst=self.local_path_dir + os.sep + self.local_file)
