def test_check_another_client(self):
self._prepare_for_uploading()
client = Client(self.options)
if self.client.check(self.remote_path_dir):
self.client.clean(self.remote_path_dir)
self.assertTrue(self.client.mkdir(self.remote_path_dir))
self.assertTrue(self.client.check(self.remote_path_dir))
self.client.upload_sync(remote_path=self.remote_path_file, local_path=self.local_path_dir)
self.assertTrue(self.client.check(self.remote_path_file))
self.assertTrue(client.check(self.remote_path_dir))
self.assertTrue(client.check(self.remote_path_file))
def _client(self):
from webdav3.client import Client
# Set password or ask for it
if self.ask_password and self.password is None and self.token is None:
self.password = ask_password(self.hostname, self.user)
# Setup webdav client options dictionary
options = {
"webdav_hostname": self.hostname,
"webdav_login": self.user,
"webdav_password": self.password,
"webdav_token": self.token,
"webdav_cert_path": self.cert_path,
"webdav_key_path": self.key_path,
"webdav_timeout": self.timeout,
"webdav_chunk_size": self.CHUNK_SIZE,
}
client = Client(options)
# Check whether client options are valid
if not client.valid():
raise ConfigError(
f"Configuration for WebDAV {self.hostname} is invalid.")
# Check whether connection is valid (root should always exist)
if not client.check(self.path_info.path):
raise WebDAVConnectionError(self.hostname)
return client
class connector:
def __init__(self, config):
self.config = config
options = {
'webdav_hostname': config['Connector']['Host'],
'webdav_login': config['Connector']['Username'],
'webdav_password': config['Connector']['Password']
}
self.client = Client(options)
self.client.verfiy = False
self.base = config['Connector']['Base']
def listdir(self):
l = self.client.list(self.base)
l.pop(l.index(self.base + '/'))
print(l)
return l
def remove(self, name):
self.client.clean(self.base + '/' + name)
def check(self, name):
return self.client.check(self.base + '/' + name)
def get(self, name):
return self.client.resource(self.base + '/' + name)
def create(self, name):
self.client.upload_to(None, self.base + '/' + name)
def upload(self, name):
self.client.upload_file(self.base + '/' + name, name)
def upload(options=options):
client = Client(options)
if not client.check("/server"):
client.mkdir("/server")
path="/server/"+str(int(time.time()))
client.mkdir(path)
client.upload(path,"./dat/")
print("Data backup succeeded!")
class WebDavServiceBase:
def __init__(self, conf: WebDavConfig):
url = f"http{'s' if conf.use_https else ''}://{conf.host}:{conf.port}{conf.path}{conf.root_dir}"
options = {
'webdav_hostname': url,
'webdav_login': conf.username,
'webdav_password': conf.password,
'webdav_timeout': 600
}
self.client = Client(options)
if conf.force_direct:
self.client.session.proxies = {}
self.sem = threading.Semaphore(5)
def dir_exists(self, path: str):
try:
# rclone doesn't allow you check an existing dir
return self.client.check(path)
except webdav3.exceptions.MethodNotSupported as e:
print(e)
return True
def ensure_dir(self, tag: str, path: str):
service_name = f"{SERVICE_NAME}:{tag}"
if ServiceKVStore.exists(service_name, path):
return
if self.dir_exists(path):
ServiceKVStore.put(service_name, path, {})
return
self.client.mkdir(path)
ServiceKVStore.put(service_name, path, {})
def write_file(self, tag: str, service: ServiceType, dir_name: str,
filename: str, buffer: IO):
with self.sem:
self.ensure_dir(tag, service.value)
self.ensure_dir(tag, f"{service.value}/{dir_name}")
fp = f"{service.value}/{dir_name}/{filename}"
with NamedTemporaryFile() as f:
f.write(buffer.read())
f.flush()
try:
self.client.upload(fp, f.name)
except webdav3.exceptions.RemoteParentNotFound as err:
print(err)
pp = f"{service.value}/{dir_name}"
# assert not ServiceKVStore.exists(SERVICE_NAME, pp)
# self.client.mkdir(pp)
from webdav3.urn import Urn
directory_urn = Urn(pp, directory=True)
response = self.client.execute_request(
action='mkdir', path=directory_urn.quote())
assert response in (200, 201), response
ServiceKVStore.put(SERVICE_NAME, pp, {})
self.client.upload(fp, f.name)
return fp
def fetch(options=options):
client = Client(options)
if not client.check("/server"):
client.mkdir("/server")
lst=client.list("/server")[1::]
cnt=0
for i in lst:
cnt+=1
print("Index:",cnt," Time:",parse_time(i))
index=int(input("Please select the file you want to restore:"))%len(lst)
client.download("/server/"+lst[index-1],"./dat/")
print(parse_time(lst[index-1]),"has been restored")
def fetch(options=options):
client = Client(options)
if not client.check("/blocks"):
client.mkdir("/blocks")
lst = client.list("/blocks")[1::]
cnt = 0
for i in lst:
cnt += 1
print("Index:", cnt, i)
index = int(
input("Please select the block you want to install:")) % len(lst)
client.download("/blocks/" + lst[index - 1], "./blocks/" + lst[index - 1])
# print(parse_time(lst[index-1]),"has been restored")
print(lst[index - 1], "has been installed successfully")
def upload_webdav():
options = {
'webdav_hostname': parse_config('WEBDAV', 'WEBDAV_UPLOAD_URL'),
'webdav_login': parse_config('WEBDAV', 'WEBDAV_UPLOAD_USER'),
'webdav_password': parse_config('WEBDAV', 'WEBDAV_UPLOAD_PW')
}
remotePath = parse_config('WEBDAV', 'WEBDAV_UPLOAD_PATH')
client = Client(options)
client.verify = False
today = datetime.date.today()
year, week_num, day_of_week = today.isocalendar()
path = str(year) + "_cw" + str(week_num)
if not client.check(remotePath + "/" + path):
client.mkdir(remotePath + "/" + path)
kwargs = {
'remote_path': remotePath + "/" + path + "/",
'local_path': targetPath + "/" + path + "/"
}
client.upload_async(**kwargs)
def _client(self):
from webdav3.client import Client
# Construct hostname from path_info by stripping path
http_info = HTTPURLInfo(self.path_info.url)
hostname = http_info.replace(path="").url
# Set password or ask for it
if self.ask_password and self.password is None and self.token is None:
host, user = self.path_info.host, self.path_info.user
self.password = ask_password(host, user)
# Setup webdav client options dictionary
options = {
"webdav_hostname": hostname,
"webdav_login": self.user,
"webdav_password": self.password,
"webdav_token": self.token,
"webdav_cert_path": self.cert_path,
"webdav_key_path": self.key_path,
"webdav_timeout": self.timeout,
"webdav_chunk_size": self.CHUNK_SIZE,
}
client = Client(options)
# Check whether client options are valid
if not client.valid():
raise ConfigError(
f"Configuration for WebDAV {hostname} is invalid."
)
# Check whether connection is valid (root should always exist)
if not client.check(self.path_info.path):
raise WebDAVConnectionError(hostname)
return client
class Webdav_sync(Sync):
def __init__(self, user, password, url, db_file):
super().__init__()
log.info("We are in Webdav_sync.__init__")
if user == '' or password == '' or url == '':
log.error("Webdav config incomplete. Check config.yaml")
raise SystemExit
else:
self.user = user
self.password = password
self.url = url
self.discobase = db_file
#self.backuppath = '/discodos/{}'.format(db_file)
options = {
'webdav_hostname': self.url,
'webdav_login': self.user,
'webdav_password': self.password
}
self.client = Client(options)
#print(dir(self.client))
#print('')
#print(self.client.is_dir('discodos'))
#print(self.client.check(self.discobase))
def _webdav_mtime(
self,
filename): # we currently don't need this, put to func anyway
mod_server_dt = parse(self.client.info(filename)['modified'])
mod_server_str = mod_server_dt.strftime('%Y-%m-%d_%H%M%S')
#if mod_local_str != mod_server_str:
# print('Local and server discobase.db modification time diverge.')
# print(mod_local_str)
# print(mod_server_str)
return mod_server_str
def backup(self):
# check file stats on local machine
bak_file_name = self._get_fileobj_mtime(self.discobase)
print("Uploading as {} to {}".format(bak_file_name, self.url))
existing = False
try:
if self.client.check(bak_file_name):
existing = True
else:
existing = False
except WebDavException as exception:
log.error('Webserver returned: {}'.format(exception))
raise SystemExit
if existing:
log.warning('Backup existing. Won\'t overwrite "{}" '.format(
bak_file_name))
else:
print('Backup not existing yet, uploading ...')
self.client.upload_sync(remote_path='{}'.format(bak_file_name),
local_path='{}'.format(self.discobase))
# in any case, show list of existing backups
self.show_backups()
return True
def show_backups(self, restore=False):
if not restore:
print('\nExisting backups:')
#relevant_files = self.client.list()[1:] # leave out first item, it's the containing folder
all_files = self.client.list()
all_files.sort() # sorts by name
relevant_files = []
for i, resource in enumerate(all_files):
if re.search('_(\d+)-(\d+)-(\d+)_(\d+)$', resource):
relevant_files.append(resource)
else:
log.debug('Sync: Skipping resource: {}'.format(all_files[i]))
for j, file in enumerate(relevant_files):
file = '({}) - {}'.format(j, file)
print(file)
if restore:
restore_id = ask_user('Restore backup #: ')
try:
restore_file = relevant_files[int(restore_id)]
except ValueError:
log.warning('Nothing to restore!')
raise SystemExit
except IndexError:
log.warning('Non-existent ID. Nothing to restore!')
raise SystemExit
print('Restoring backup {}...'.format(restore_file))
return restore_file
print()
def restore(self):
print('\nWhich backup would you like to restore?')
restore_filename = self.show_backups(restore=True)
overwrite = ask_user(
"Download backup and overwrite local file {} (n)? ".format(
self.discobase))
if overwrite.lower() == 'y':
self.client.download_sync(
remote_path='{}'.format(restore_filename),
local_path='{}'.format(self.discobase))
self._touch_to_backupdate(restore_filename)
class BaseClientTestCase(unittest.TestCase):
remote_path_file = 'test_dir/test.txt'
remote_compressed_path_file = 'test_dir/compressed.txt'
remote_path_file2 = 'test_dir2/test.txt'
remote_inner_path_file = 'test_dir/inner/test.txt'
remote_path_dir = 'test_dir'
remote_path_dir2 = 'test_dir2'
remote_inner_path_dir = 'test_dir/inner'
inner_dir_name = 'inner'
local_base_dir = 'tests/'
local_file = 'test.txt'
local_file_path = local_base_dir + 'test.txt'
local_compressed_file_path = local_base_dir + 'compressed.txt'
local_path_dir = local_base_dir + 'res/test_dir'
options = {
'webdav_hostname': 'http://localhost:8585',
'webdav_login': '******',
'webdav_password': '******',
'webdav_timeout': 10,
'webdav_override_methods': {
'check': 'GET'
}
}
# options = {
# 'webdav_hostname': 'https://demo1.nextcloud.com/remote.php/dav/files/RCw8Y9XXFnzkLJbN/',
# 'webdav_login': '******',
# 'webdav_password': '******',
# 'webdav_override_methods': {
# 'check': 'GET'
# }
# }
# options = {
# 'webdav_hostname': 'https://webdav.yandex.ru',
# 'webdav_login': '******',
# 'webdav_password': '******'
# }
def setUp(self):
self.client = Client(self.options)
self.clean_local_dir(self.local_path_dir)
def tearDown(self):
self.clean_local_dir(self.local_path_dir)
self.clean_remote_dir(self.remote_path_dir)
self.clean_remote_dir(self.remote_path_dir2)
def clean_remote_dir(self, remote_path_dir):
if self.client.check(remote_path=remote_path_dir):
self.client.clean(remote_path=remote_path_dir)
@staticmethod
def clean_local_dir(local_path_dir):
if path.exists(path=local_path_dir):
shutil.rmtree(path=local_path_dir)
def _prepare_for_downloading(self, inner_dir=False, base_path=''):
if base_path:
self._create_remote_dir_if_needed(base_path)
self._prepare_dir_for_downloading(base_path + self.remote_path_dir,
base_path + self.remote_path_file,
self.local_file_path)
if not path.exists(self.local_path_dir):
os.makedirs(self.local_path_dir)
if inner_dir:
self._prepare_dir_for_downloading(
base_path + self.remote_inner_path_dir,
base_path + self.remote_inner_path_file, self.local_file_path)
def _prepare_dir_for_downloading(self, remote_path_dir, remote_path_file,
local_file_path):
self._create_remote_dir_if_needed(remote_path_dir)
if not self.client.check(remote_path=remote_path_file):
self.client.upload_file(remote_path=remote_path_file,
local_path=local_file_path)
def _create_remote_dir_if_needed(self, remote_dir):
if not self.client.check(remote_path=remote_dir):
self.client.mkdir(remote_path=remote_dir)
def _prepare_for_uploading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not path.exists(path=self.local_path_dir):
os.makedirs(self.local_path_dir)
if not path.exists(path=self.local_path_dir + os.sep +
self.local_file):
shutil.copy(src=self.local_file_path,
dst=self.local_path_dir + os.sep + self.local_file)
class WebDev(object):
def __init__(self, options ):
# self.client = wc.Client(options)
self.client = Client(options)
def read(self, name):
with BytesIO() as buf:
self.client.download_from(buf, name)
return buf.getvalue()
def write(self,content, target):
data = self.read(target)
with BytesIO() as buf:
buf.write(data)
buf.write(content.encode())
self.client.upload_to(buf.getvalue(), target)
def mkdir(self, dir_name):
self.client.mkdir(dir_name)
def list(self):
data = self.client.list()
print(data)
def clear(self, target):
self.client.clean(target)
def check(self,remote_path):
data = self.client.check(remote_path)
if data:
print('{} is exists'.format(remote_path))
else:
print('{} is not exists'.format(remote_path))
def upload(self,remote_path, local_path):
self.client.upload(remote_path, local_path)
def help_dev(self):
print(dir(self.client))
def mkfile(self, remote_path):
with BytesIO() as buf:
buf.write(b'')
self.client.upload_to(buf.getvalue(), remote_path)
def download_file(self, remote_path):
self.client.download_file(remote_path, remote_path)
def rename(self, old_name, new_name):
self.client.copy(old_name, old_name)
self.client.clean(old_name)
def set_property(self,remote_path, option):
self.client.set_property(remote_path, option=[option])
class FileBrowserWebdavStorage(StorageMixin, Storage):
def __init__(self,
base_url='/',
url_as_download=True,
simple_listdir=False,
webdav_root='/'):
self.base_url = base_url
self.url_as_download = url_as_download
self.simple_listdir = simple_listdir
webdav_client_options = {
'webdav_hostname':
settings.CONTRAX_FILE_STORAGE_WEBDAV_ROOT_URL.rstrip('/'),
'webdav_login':
settings.CONTRAX_FILE_STORAGE_WEBDAV_USERNAME,
'webdav_password':
settings.CONTRAX_FILE_STORAGE_WEBDAV_PASSWORD,
}
self.client = Client(webdav_client_options)
try:
self.client.mkdir('/media')
self.client.mkdir('/media/photo')
except:
pass
self.client.webdav.root = webdav_root
self.client.root = webdav_root
def path(self, name):
"""
Return a local filesystem path where the file can be retrieved using
Python's built-in open() function. Storage systems that can't be
accessed using open() should *not* implement this method.
"""
# FIXME: this would be useful with self.location != ''
# in this case use this notation:
# 1. define self.location in __init__
# 2. rewrite path() method to be like
# return os.oath.join(self.location, name)
# 3. everywhere in other sel.methods use self.path(name) instead of name attr
return name
def isdir(self, path):
"""
Returns true if name exists and is a directory.
"""
return self.client.check(path) and self.client.is_dir(path)
def isfile(self, path):
"""
Returns true if name exists and is a regular file.
"""
return self.client.check(path) and not self.client.is_dir(path)
def move(self, old_file_name, new_file_name, allow_overwrite=False):
"""
Moves safely a file from one location to another.
If allow_ovewrite==False and new_file_name exists, raises an exception.
"""
return self.client.move(remote_path_from=old_file_name,
remote_path_to=new_file_name,
overwrite=allow_overwrite)
def makedirs(self, path):
"""
Creates all missing directories specified by name. Analogue to os.mkdirs().
"""
return self.client.mkdir(path)
def rmtree(self, path):
"""
Deletes a directory and everything it contains. Analogue to shutil.rmtree().
"""
return self.client.clean(path)
def setpermission(self, path):
"""
Sets file permission
"""
pass
def _open(self, path, mode='rb'):
tmp = io.BytesIO()
self.client.download_from(tmp, path)
tmp.seek(0)
return File(tmp)
def _save(self, path, content):
res = self.client.resource(path)
res.read_from(content)
return path
def get_valid_name(self, name):
"""
Return a filename, based on the provided filename, that's suitable for
use in the target storage system.
"""
return get_valid_filename(name)
def delete(self, path):
"""
Delete the specified file from the storage system.
"""
if self.exists(path):
self.client.clean(path)
def exists(self, path):
"""
Return True if a file referenced by the given name already exists in the
storage system, or False if the name is available for a new file.
"""
return self.client.check(path)
def listdir(self, path):
"""
List the contents of the specified path. Return a 2-tuple of lists:
the first item being directories, the second item being files.
"""
_list = self.client.list(path)
# for API: iterating over big directory take too much time
if self.simple_listdir:
return _list
# for filebrowser
directories, files = [], []
for entry in _list:
entry_path = os.path.join(path, entry)
if self.isdir(entry_path):
directories.append(entry.rstrip('/'))
else:
files.append(entry)
return directories, files
def size(self, path):
"""
Return the total size, in bytes, of the file specified by name.
"""
return self.client.info(path)['size']
def url(self, path):
"""
Return an absolute URL where the file's contents can be accessed
directly by a Web browser.
"""
url = filepath_to_uri(path)
if url is not None:
url = url.lstrip('/')
url = urljoin(self.base_url, url)
if self.url_as_download and self.isfile(path):
url += '?action=download'
return url
@staticmethod
def _datetime_from_timestamp(ts, fmt):
"""
If timezone support is enabled, make an aware datetime object in UTC;
otherwise make a naive one in the local timezone.
"""
dt = datetime.strptime(ts, fmt)
if settings.USE_TZ:
# Safe to use .replace() because UTC doesn't have DST
return dt.replace(tzinfo=timezone.utc)
else:
return dt
def get_accessed_time(self, path):
"""
Return the last accessed time (as a datetime) of the file specified by
name. The datetime will be timezone-aware if USE_TZ=True.
"""
pass
def get_created_time(self, path):
"""
Return the creation time (as a datetime) of the file specified by name.
The datetime will be timezone-aware if USE_TZ=True.
"""
return self._datetime_from_timestamp(self.client.info(path)['created'],
fmt='%Y-%m-%dT%H:%M:%SZ')
def get_modified_time(self, path):
"""
Return the last modified time (as a datetime) of the file specified by
name. The datetime will be timezone-aware if USE_TZ=True.
"""
return self._datetime_from_timestamp(
self.client.info(path)['modified'], fmt='%a, %d %b %Y %H:%M:%S %Z')
#build a list of ALL courses in /internal
courseList = client.list("internal/courses")
#this is the list of dictionaries one for each backup
theList = []
#for testing only, using islice to limit itterations
for course in islice(courseList, 0, coursesToCheck):
#construct full path to archive directory for this course
arcpath = "internal/courses/" + course + "archive"
#execute if archive directory exists
if client.check(arcpath):
#get a list of all backups in the directory
backups = client.list(arcpath)
#itterate through each backup
for backup in backups:
#filter by filename prefix
if backup.startswith(prefixes):
#push the file information into a dictionary
backupInfo = client.info(arcpath + "/" + backup)
#convert the created string to a date
createdDateTime = datetime.strptime(backupInfo.get('created'), "%Y-%m-%dT%H:%M:%SZ")
def test_webdav(self):
server_2_ip = '172.16.3.155'
error, server_id = self.mm['webdavalpine'].run("add_server", ip_addr=server_2_ip, fqdn='webdav.test')
self.assertTrue(error == None, msg=error)
time.sleep(10)
webdav_url = "https://{}/files/".format(server_2_ip)
full_path = parentdir + "/work/webdavalpine/1/webdav/admin.pass"
self.assertTrue(os.path.exists(full_path), msg="{} does not exist".format(full_path))
admin_pass = open(full_path, "r").read().strip()
resp = requests.get("https://{}/".format(server_2_ip), verify=False)
self.assertTrue(resp.status_code == 200)
options = {
'webdav_hostname': webdav_url,
'webdav_login': "******",
'webdav_password': admin_pass
}
client = Client(options)
client.verify = False
result = client.check("public")
self.assertTrue(result == True)
result = client.list()
self.assertTrue(len(result) > 0)
subprocess.check_output(["touch /tmp/testdata"], shell=True)
client.upload_sync("public/testdata", "/tmp/testdata")
result = client.check("public/testdata")
self.assertTrue(result == True)
unauth_client = Client({
'webdav_hostname': webdav_url
})
unauth_client.verify = False
public_list = unauth_client.list("public/")
self.assertTrue(len(public_list) > 0)
try:
unauth_client.upload_sync("public/testdata2", "/tmp/testdata")
self.fail()
except:
pass
error, _ = self.mm['webdavalpine'].run("stop_server", id=server_id)
self.assertTrue(error == None, msg=error)
time.sleep(2)
error, _ = self.mm['webdavalpine'].run("start_server", id=server_id)
self.assertTrue(error == None, msg=error)
time.sleep(10)
resp = requests.get("https://{}/".format(server_2_ip), verify=False)
self.assertTrue(resp.status_code == 200)
options = {
'webdav_hostname': webdav_url,
'webdav_login': "******",
'webdav_password': admin_pass
}
client2 = Client(options)
client2.verify = False
public_list = client2.list("public/")
self.assertTrue(len(public_list) > 0)
self.assertTrue(public_list[1] == "testdata")
error, _ = self.mm['webdavalpine'].run("remove_server", id=server_id)
self.assertTrue(error == None, msg=error)
#open the client
client = Client(options)
x = client.list()
#the file should have at least two columns "deleteMe" (Y/N) and "fullPath"
with open("theList.csv") as csvfile:
#this pushes the file data into a list of dictionaries
reader = csv.DictReader(csvfile)
#itterate on each dictionary in the list
for row in reader:
#if the backup doesn't exist, say so and move on
if not client.check(row.get("fullPath")):
print("Cannot find: " + row.get("fullPath"))
continue
#determine if the item is marked to be deleted
if row.get("deleteMe") == "Y":
#if the flag is True, this will remove the backup
if removeFiles:
client.clean(row.get("fullPath"))
#determine if the backup still exists or not
if client.check(row.get("fullPath")):
print("Not deleted: " + row.get("fullPath"))
else:
print("Deleted: " + row.get("fullPath"))
class ClientTestCase(TestCase):
remote_path_file = 'test_dir/test.txt'
remote_path_file2 = 'test_dir2/test.txt'
remote_path_dir = 'test_dir'
remote_path_dir2 = 'test_dir2'
local_base_dir = 'tests/'
local_file = 'test.txt'
local_file_path = local_base_dir + 'test.txt'
local_path_dir = local_base_dir + 'res/test_dir'
def setUp(self):
options = {
'webdav_hostname': 'https://webdav.yandex.ru',
'webdav_login': '******',
'webdav_password': '******'
}
self.client = Client(options)
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
def tearDown(self):
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
if self.client.check(remote_path=self.remote_path_dir):
self.client.clean(remote_path=self.remote_path_dir)
if self.client.check(remote_path=self.remote_path_dir2):
self.client.clean(remote_path=self.remote_path_dir2)
def test_list(self):
self._prepare_for_downloading()
file_list = self.client.list()
self.assertIsNotNone(file_list, 'List of files should not be None')
self.assertGreater(file_list.__len__(), 0,
'Expected that amount of files more then 0')
def test_free(self):
self.assertGreater(
self.client.free(), 0,
'Expected that free space on WebDAV server is more then 0 bytes')
def test_check(self):
self.assertTrue(self.client.check(),
'Expected that root directory is exist')
def test_mkdir(self):
if self.client.check(remote_path=self.remote_path_dir):
self.client.clean(remote_path=self.remote_path_dir)
self.client.mkdir(remote_path=self.remote_path_dir)
self.assertTrue(self.client.check(remote_path=self.remote_path_dir),
'Expected the directory is created.')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_to(self):
self._prepare_for_downloading()
buff = BytesIO()
self.client.download_from(buff=buff, remote_path=self.remote_path_file)
self.assertEquals(buff.getvalue(),
'test content for testing of webdav client')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download(self):
self._prepare_for_downloading()
self.client.download(local_path=self.local_path_dir,
remote_path=self.remote_path_dir)
self.assertTrue(path.exists(self.local_path_dir),
'Expected the directory is downloaded.')
self.assertTrue(path.isdir(self.local_path_dir),
'Expected this is a directory.')
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_path_file), 'Expected this is a file')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_sync(self):
self._prepare_for_downloading()
os.mkdir(self.local_path_dir)
def callback():
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep +
self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_file), 'Expected this is a file')
self.client.download_sync(local_path=self.local_path_dir +
os.path.sep + self.local_file,
remote_path=self.remote_path_file,
callback=callback)
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file has already been downloaded')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_async(self):
self._prepare_for_downloading()
os.mkdir(self.local_path_dir)
def callback():
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep +
self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_file), 'Expected this is a file')
self.client.download_async(local_path=self.local_path_dir +
os.path.sep + self.local_file,
remote_path=self.remote_path_file,
callback=callback)
self.assertFalse(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file has not been downloaded yet')
def test_upload_from(self):
self._prepare_for_uploading()
buff = StringIO(u'test content for testing of webdav client')
self.client.upload_to(buff=buff, remote_path=self.remote_path_file)
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
def test_upload(self):
self._prepare_for_uploading()
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
def test_upload_file(self):
self._prepare_for_uploading()
self.client.upload_file(remote_path=self.remote_path_file,
local_path=self.local_file_path)
self.assertTrue(self.client.check(remote_path=self.remote_path_file),
'Expected the file is uploaded.')
def test_upload_sync(self):
self._prepare_for_uploading()
def callback():
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
def test_upload_async(self):
self._prepare_for_uploading()
def callback():
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
def test_copy(self):
self._prepare_for_downloading()
self.client.mkdir(remote_path=self.remote_path_dir2)
self.client.copy(remote_path_from=self.remote_path_file,
remote_path_to=self.remote_path_file2)
self.assertTrue(self.client.check(remote_path=self.remote_path_file2))
def test_move(self):
self._prepare_for_downloading()
self.client.mkdir(remote_path=self.remote_path_dir2)
self.client.move(remote_path_from=self.remote_path_file,
remote_path_to=self.remote_path_file2)
self.assertFalse(self.client.check(remote_path=self.remote_path_file))
self.assertTrue(self.client.check(remote_path=self.remote_path_file2))
def test_clean(self):
self._prepare_for_downloading()
self.client.clean(remote_path=self.remote_path_dir)
self.assertFalse(self.client.check(remote_path=self.remote_path_file))
self.assertFalse(self.client.check(remote_path=self.remote_path_dir))
def test_info(self):
self._prepare_for_downloading()
result = self.client.info(remote_path=self.remote_path_file)
self.assertEquals(result['name'], 'test.txt')
self.assertEquals(result['size'], '41')
self.assertTrue('created' in result)
self.assertTrue('modified' in result)
def test_directory_is_dir(self):
self._prepare_for_downloading()
self.assertTrue(self.client.is_dir(self.remote_path_dir),
'Should return True for directory')
def test_file_is_not_dir(self):
self._prepare_for_downloading()
self.assertFalse(self.client.is_dir(self.remote_path_file),
'Should return False for file')
def test_get_property_of_non_exist(self):
self._prepare_for_downloading()
result = self.client.get_property(remote_path=self.remote_path_file,
option={'name': 'aProperty'})
self.assertEquals(
result, None, 'For not found property should return value as None')
def test_set_property(self):
self._prepare_for_downloading()
self.client.set_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty',
'value': 'aValue'
})
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty'
})
self.assertEquals(result, 'aValue', 'Property value should be set')
def test_set_property_batch(self):
self._prepare_for_downloading()
self.client.set_property_batch(remote_path=self.remote_path_file,
option=[{
'namespace': 'test',
'name': 'aProperty',
'value': 'aValue'
}, {
'namespace': 'test',
'name': 'aProperty2',
'value': 'aValue2'
}])
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty'
})
self.assertEquals(result, 'aValue',
'First property value should be set')
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty2'
})
self.assertEquals(result, 'aValue2',
'Second property value should be set')
def _prepare_for_downloading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not self.client.check(remote_path=self.remote_path_file):
self.client.upload_file(remote_path=self.remote_path_file,
local_path=self.local_file_path)
if not path.exists(self.local_path_dir):
os.makedirs(self.local_path_dir)
def _prepare_for_uploading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not path.exists(path=self.local_path_dir):
os.makedirs(self.local_path_dir)
if not path.exists(path=self.local_path_dir + os.sep +
self.local_file):
shutil.copy(src=self.local_file_path,
dst=self.local_path_dir + os.sep + self.local_file)
class BaseClientTestCase(unittest.TestCase):
remote_path_file = 'test_dir/test.txt'
remote_path_file2 = 'test_dir2/test.txt'
remote_inner_path_file = 'test_dir/inner/test.txt'
remote_path_dir = 'test_dir'
remote_path_dir2 = 'test_dir2'
remote_inner_path_dir = 'test_dir/inner'
local_base_dir = 'tests/'
local_file = 'test.txt'
local_file_path = local_base_dir + 'test.txt'
local_path_dir = local_base_dir + 'res/test_dir'
options = {
'webdav_hostname': 'http://localhost:8585',
'webdav_login': '******',
'webdav_password': '******'
}
# options = {
# 'webdav_hostname': 'https://webdav.yandex.ru',
# 'webdav_login': '******',
# 'webdav_password': '******'
# }
def setUp(self):
self.client = Client(self.options)
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
def tearDown(self):
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
if self.client.check(remote_path=self.remote_path_dir):
self.client.clean(remote_path=self.remote_path_dir)
if self.client.check(remote_path=self.remote_path_dir2):
self.client.clean(remote_path=self.remote_path_dir2)
def _prepare_for_downloading(self, inner_dir=False):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not self.client.check(remote_path=self.remote_path_file):
self.client.upload_file(remote_path=self.remote_path_file,
local_path=self.local_file_path)
if not path.exists(self.local_path_dir):
os.makedirs(self.local_path_dir)
if inner_dir:
if not self.client.check(remote_path=self.remote_inner_path_dir):
self.client.mkdir(remote_path=self.remote_inner_path_dir)
if not self.client.check(remote_path=self.remote_inner_path_file):
self.client.upload_file(
remote_path=self.remote_inner_path_file,
local_path=self.local_file_path)
def _prepare_for_uploading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not path.exists(path=self.local_path_dir):
os.makedirs(self.local_path_dir)
if not path.exists(path=self.local_path_dir + os.sep +
self.local_file):
shutil.copy(src=self.local_file_path,
dst=self.local_path_dir + os.sep + self.local_file)
def test(self,
start=True,
gedis=True,
webdav=False,
bottle=True,
websockets=False):
"""
kosmos 'j.servers.rack.test()'
kosmos 'j.servers.rack.test(gedis_ssl=True)'
kosmos 'j.servers.rack.test(ssl=False)'
kosmos 'j.servers.rack.test(start=True)'
:param manual means the server is run manually using e.g. js_shell 'j.servers.rack.start()'
"""
if start:
self._server_test_start(background=True,
gedis=gedis,
webdav=webdav,
bottle=bottle,
websockets=websockets)
namespace = "system"
secret = "1234"
cl = j.clients.gedis.get(namespace, port=8901, host="localhost")
assert cl.ping()
if webdav:
# how to use see https://github.com/ezhov-evgeny/webdav-client-python-3/blob/da46592c6f1cc9fb810ca54019763b1e7dce4583/webdav3/client.py#L197
options = {
"webdav_hostname": "http://127.0.0.1:4443",
"webdav_login": "******",
"webdav_password": "******"
}
from webdav3.client import Client
cl = Client(options)
cl.check()
assert len(cl.list("")) > 0
if websockets:
# TODO: does not work yet
from websocket import create_connection
ws = create_connection("ws://localhost:4444")
ws.send("Hello, World")
result = ws.recv()
print(result)
# ws.close()
if bottle:
import requests
# https://realpython.com/python-requests/#the-get-request
r1 = requests.get("http://localhost:4442/ping")
self._log(r1.status_code)
self._log(r1.content)
assert r1.content == b"pong"
assert r1.status_code == 200
print("tests are ok")
def on_event(self, event, payload):
if event == "plugin_backup_backup_created":
# Helper function for human readable sizes
def _convert_size(size_bytes):
if size_bytes == 0:
return "0B"
size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB",
"YB")
i = int(math.floor(math.log(size_bytes, 1024)))
p = math.pow(1024, i)
s = round(size_bytes / p, 2)
return "%s %s" % (s, size_name[i])
now = datetime.now()
davoptions = {
'webdav_hostname': self._settings.get(["server"]),
'webdav_login': self._settings.get(["username"]),
'webdav_password': self._settings.get(["password"]),
'webdav_timeout': self._settings.get(["timeout"]),
}
backup_path = payload["path"]
backup_name = payload["name"]
self._logger.info("Backup " + backup_path +
" created, will now attempt to upload to " +
davoptions["webdav_hostname"])
davclient = Client(davoptions)
davclient.verify = self._settings.get(["verify_certificate"])
check_space = self._settings.get(["check_space"])
upload_path = now.strftime(self._settings.get(["upload_path"]))
upload_path = ospath.join("/", upload_path)
if self._settings.get(["upload_name"]):
upload_name = now.strftime(self._settings.get(
["upload_name"])) + ospath.splitext(backup_path)[1]
else:
upload_name = backup_name
self._logger.debug("Filename for upload: " + upload_name)
upload_file = ospath.join("/", upload_path, upload_name)
upload_temp = ospath.join("/", upload_path, upload_name + ".tmp")
self._logger.debug("Upload location: " + upload_file)
# Check actual connection to the WebDAV server as the check command will not do this.
if check_space:
self._logger.debug("Attempting to check free space.")
try:
# If the resource was not found
dav_free = davclient.free()
if dav_free < 0:
# If we get a negative free size, this server is not returning correct value.
check_space = False
self._logger.warning(
"Free space on server: " +
_convert_size(dav_free) +
", it appears your server does not support reporting size correctly but it's still a proper way to check connectivity."
)
else:
self._logger.info("Free space on server: " +
_convert_size(dav_free))
except RemoteResourceNotFound as exception:
self._logger.error(
"Resource was not found, something is probably wrong with your settings."
)
return
except ResponseErrorCode as exception:
# Write error and exit function
status = HTTPStatus(exception.code)
error_switcher = {
400: "Bad request",
401: "Unauthorized",
403: "Forbidden",
404: "Not found",
405: "Method not allowed",
408: "Request timeout",
500: "Internal error",
501: "Not implemented",
502: "Bad gateway",
503: "Service unavailable",
504: "Gateway timeout",
508: "Loop detected",
}
if (exception.code == 401):
http_error = "HTTP error 401 encountered, your credentials are most likely wrong."
else:
http_error = "HTTP error encountered: " + str(
status.value) + " " + error_switcher.get(
exception.code, status.phrase)
self._logger.error(http_error)
return
except WebDavException as exception:
self._logger.error(
"An unexpected WebDAV error was encountered: " +
exception.args)
raise
else:
self._logger.debug(
"Not checking free space, just try to check the WebDAV root."
)
# Not as proper of a check as retrieving size, but it's something.
if davclient.check("/"):
self._logger.debug("Server returned WebDAV root.")
else:
self._logger.error(
"Server did not return WebDAV root, something is probably wronkg with your settings."
)
return
backup_size = ospath.getsize(backup_path)
self._logger.info("Backup file size: " +
_convert_size(backup_size))
if check_space and (backup_size > dav_free):
self._logger.error("Unable to upload, size is" +
_convert_size(backup_size) +
", free space is " +
_convert_size(dav_free))
return
else:
# Helper function to recursively create paths
def _recursive_create_path(path):
# Append leading / for preventing abspath issues
path = ospath.join("/", path)
if davclient.check(path):
self._logger.debug("Directory " + path + " was found.")
return True
else:
if path != "/":
self._logger.debug(
"Directory " + path +
" was not found, checking parent.")
if _recursive_create_path(
ospath.abspath(ospath.join(path, ".."))):
davclient.mkdir(path)
self._logger.debug("Directory " + path +
" has been created.")
return True
else:
self._logger.error(
"Could not find WebDAV root, something is probably wrong with your settings."
)
return False
if _recursive_create_path(upload_path):
self._logger.debug("Uploading " + backup_path + " to " +
upload_temp)
davclient.upload_sync(remote_path=upload_temp,
local_path=backup_path)
self._logger.debug("Moving " + upload_temp + " to " +
upload_file)
davclient.move(remote_path_from=upload_temp,
remote_path_to=upload_file)
self._logger.info(
"Backup has been uploaded successfully to " +
davoptions["webdav_hostname"] + " as " + upload_file)
else:
self._logger.error(
"Something went wrong trying to check/create the upload path."
)
k += 1
else:
print('\r' + ' ' * l + '\r', end='')
k = 0
print(s * (l - k) + '] ' + ' = 100%')
options = {
'webdav_hostname': token,
'webdav_login': login,
'webdav_password': password,
}
client = Client(options)
rem_path = 'tmp'
if not client.check(rem_path): # если нет, то создаём
client.mkdir(rem_path)
mymes('Directory [' + rem_path + '] created', 0)
client.info(rem_path)
video_path = './videos'
files = os.listdir(video_path)
free_space()
for file in files:
if re.fullmatch(r'Video\d\.\w{,5}', file):
client.upload_async(remote_path=rem_path + '/' + file,
local_path=video_path + '/' + file,
callback=callback)
upload_progress(file, '#', 0.3)
callback_flag = False
