def test_check_another_client(self): self._prepare_for_uploading() client = Client(self.options) if self.client.check(self.remote_path_dir): self.client.clean(self.remote_path_dir) self.assertTrue(self.client.mkdir(self.remote_path_dir)) self.assertTrue(self.client.check(self.remote_path_dir)) self.client.upload_sync(remote_path=self.remote_path_file, local_path=self.local_path_dir) self.assertTrue(self.client.check(self.remote_path_file)) self.assertTrue(client.check(self.remote_path_dir)) self.assertTrue(client.check(self.remote_path_file))
def _client(self): from webdav3.client import Client # Set password or ask for it if self.ask_password and self.password is None and self.token is None: self.password = ask_password(self.hostname, self.user) # Setup webdav client options dictionary options = { "webdav_hostname": self.hostname, "webdav_login": self.user, "webdav_password": self.password, "webdav_token": self.token, "webdav_cert_path": self.cert_path, "webdav_key_path": self.key_path, "webdav_timeout": self.timeout, "webdav_chunk_size": self.CHUNK_SIZE, } client = Client(options) # Check whether client options are valid if not client.valid(): raise ConfigError( f"Configuration for WebDAV {self.hostname} is invalid.") # Check whether connection is valid (root should always exist) if not client.check(self.path_info.path): raise WebDAVConnectionError(self.hostname) return client
class connector: def __init__(self, config): self.config = config options = { 'webdav_hostname': config['Connector']['Host'], 'webdav_login': config['Connector']['Username'], 'webdav_password': config['Connector']['Password'] } self.client = Client(options) self.client.verfiy = False self.base = config['Connector']['Base'] def listdir(self): l = self.client.list(self.base) l.pop(l.index(self.base + '/')) print(l) return l def remove(self, name): self.client.clean(self.base + '/' + name) def check(self, name): return self.client.check(self.base + '/' + name) def get(self, name): return self.client.resource(self.base + '/' + name) def create(self, name): self.client.upload_to(None, self.base + '/' + name) def upload(self, name): self.client.upload_file(self.base + '/' + name, name)
def upload(options=options): client = Client(options) if not client.check("/server"): client.mkdir("/server") path="/server/"+str(int(time.time())) client.mkdir(path) client.upload(path,"./dat/") print("Data backup succeeded!")
class WebDavServiceBase: def __init__(self, conf: WebDavConfig): url = f"http{'s' if conf.use_https else ''}://{conf.host}:{conf.port}{conf.path}{conf.root_dir}" options = { 'webdav_hostname': url, 'webdav_login': conf.username, 'webdav_password': conf.password, 'webdav_timeout': 600 } self.client = Client(options) if conf.force_direct: self.client.session.proxies = {} self.sem = threading.Semaphore(5) def dir_exists(self, path: str): try: # rclone doesn't allow you check an existing dir return self.client.check(path) except webdav3.exceptions.MethodNotSupported as e: print(e) return True def ensure_dir(self, tag: str, path: str): service_name = f"{SERVICE_NAME}:{tag}" if ServiceKVStore.exists(service_name, path): return if self.dir_exists(path): ServiceKVStore.put(service_name, path, {}) return self.client.mkdir(path) ServiceKVStore.put(service_name, path, {}) def write_file(self, tag: str, service: ServiceType, dir_name: str, filename: str, buffer: IO): with self.sem: self.ensure_dir(tag, service.value) self.ensure_dir(tag, f"{service.value}/{dir_name}") fp = f"{service.value}/{dir_name}/{filename}" with NamedTemporaryFile() as f: f.write(buffer.read()) f.flush() try: self.client.upload(fp, f.name) except webdav3.exceptions.RemoteParentNotFound as err: print(err) pp = f"{service.value}/{dir_name}" # assert not ServiceKVStore.exists(SERVICE_NAME, pp) # self.client.mkdir(pp) from webdav3.urn import Urn directory_urn = Urn(pp, directory=True) response = self.client.execute_request( action='mkdir', path=directory_urn.quote()) assert response in (200, 201), response ServiceKVStore.put(SERVICE_NAME, pp, {}) self.client.upload(fp, f.name) return fp
def fetch(options=options): client = Client(options) if not client.check("/server"): client.mkdir("/server") lst=client.list("/server")[1::] cnt=0 for i in lst: cnt+=1 print("Index:",cnt," Time:",parse_time(i)) index=int(input("Please select the file you want to restore:"))%len(lst) client.download("/server/"+lst[index-1],"./dat/") print(parse_time(lst[index-1]),"has been restored")
def fetch(options=options): client = Client(options) if not client.check("/blocks"): client.mkdir("/blocks") lst = client.list("/blocks")[1::] cnt = 0 for i in lst: cnt += 1 print("Index:", cnt, i) index = int( input("Please select the block you want to install:")) % len(lst) client.download("/blocks/" + lst[index - 1], "./blocks/" + lst[index - 1]) # print(parse_time(lst[index-1]),"has been restored") print(lst[index - 1], "has been installed successfully")
def upload_webdav(): options = { 'webdav_hostname': parse_config('WEBDAV', 'WEBDAV_UPLOAD_URL'), 'webdav_login': parse_config('WEBDAV', 'WEBDAV_UPLOAD_USER'), 'webdav_password': parse_config('WEBDAV', 'WEBDAV_UPLOAD_PW') } remotePath = parse_config('WEBDAV', 'WEBDAV_UPLOAD_PATH') client = Client(options) client.verify = False today = datetime.date.today() year, week_num, day_of_week = today.isocalendar() path = str(year) + "_cw" + str(week_num) if not client.check(remotePath + "/" + path): client.mkdir(remotePath + "/" + path) kwargs = { 'remote_path': remotePath + "/" + path + "/", 'local_path': targetPath + "/" + path + "/" } client.upload_async(**kwargs)
def _client(self): from webdav3.client import Client # Construct hostname from path_info by stripping path http_info = HTTPURLInfo(self.path_info.url) hostname = http_info.replace(path="").url # Set password or ask for it if self.ask_password and self.password is None and self.token is None: host, user = self.path_info.host, self.path_info.user self.password = ask_password(host, user) # Setup webdav client options dictionary options = { "webdav_hostname": hostname, "webdav_login": self.user, "webdav_password": self.password, "webdav_token": self.token, "webdav_cert_path": self.cert_path, "webdav_key_path": self.key_path, "webdav_timeout": self.timeout, "webdav_chunk_size": self.CHUNK_SIZE, } client = Client(options) # Check whether client options are valid if not client.valid(): raise ConfigError( f"Configuration for WebDAV {hostname} is invalid." ) # Check whether connection is valid (root should always exist) if not client.check(self.path_info.path): raise WebDAVConnectionError(hostname) return client
class Webdav_sync(Sync): def __init__(self, user, password, url, db_file): super().__init__() log.info("We are in Webdav_sync.__init__") if user == '' or password == '' or url == '': log.error("Webdav config incomplete. Check config.yaml") raise SystemExit else: self.user = user self.password = password self.url = url self.discobase = db_file #self.backuppath = '/discodos/{}'.format(db_file) options = { 'webdav_hostname': self.url, 'webdav_login': self.user, 'webdav_password': self.password } self.client = Client(options) #print(dir(self.client)) #print('') #print(self.client.is_dir('discodos')) #print(self.client.check(self.discobase)) def _webdav_mtime( self, filename): # we currently don't need this, put to func anyway mod_server_dt = parse(self.client.info(filename)['modified']) mod_server_str = mod_server_dt.strftime('%Y-%m-%d_%H%M%S') #if mod_local_str != mod_server_str: # print('Local and server discobase.db modification time diverge.') # print(mod_local_str) # print(mod_server_str) return mod_server_str def backup(self): # check file stats on local machine bak_file_name = self._get_fileobj_mtime(self.discobase) print("Uploading as {} to {}".format(bak_file_name, self.url)) existing = False try: if self.client.check(bak_file_name): existing = True else: existing = False except WebDavException as exception: log.error('Webserver returned: {}'.format(exception)) raise SystemExit if existing: log.warning('Backup existing. Won\'t overwrite "{}" '.format( bak_file_name)) else: print('Backup not existing yet, uploading ...') self.client.upload_sync(remote_path='{}'.format(bak_file_name), local_path='{}'.format(self.discobase)) # in any case, show list of existing backups self.show_backups() return True def show_backups(self, restore=False): if not restore: print('\nExisting backups:') #relevant_files = self.client.list()[1:] # leave out first item, it's the containing folder all_files = self.client.list() all_files.sort() # sorts by name relevant_files = [] for i, resource in enumerate(all_files): if re.search('_(\d+)-(\d+)-(\d+)_(\d+)$', resource): relevant_files.append(resource) else: log.debug('Sync: Skipping resource: {}'.format(all_files[i])) for j, file in enumerate(relevant_files): file = '({}) - {}'.format(j, file) print(file) if restore: restore_id = ask_user('Restore backup #: ') try: restore_file = relevant_files[int(restore_id)] except ValueError: log.warning('Nothing to restore!') raise SystemExit except IndexError: log.warning('Non-existent ID. Nothing to restore!') raise SystemExit print('Restoring backup {}...'.format(restore_file)) return restore_file print() def restore(self): print('\nWhich backup would you like to restore?') restore_filename = self.show_backups(restore=True) overwrite = ask_user( "Download backup and overwrite local file {} (n)? ".format( self.discobase)) if overwrite.lower() == 'y': self.client.download_sync( remote_path='{}'.format(restore_filename), local_path='{}'.format(self.discobase)) self._touch_to_backupdate(restore_filename)
class BaseClientTestCase(unittest.TestCase): remote_path_file = 'test_dir/test.txt' remote_compressed_path_file = 'test_dir/compressed.txt' remote_path_file2 = 'test_dir2/test.txt' remote_inner_path_file = 'test_dir/inner/test.txt' remote_path_dir = 'test_dir' remote_path_dir2 = 'test_dir2' remote_inner_path_dir = 'test_dir/inner' inner_dir_name = 'inner' local_base_dir = 'tests/' local_file = 'test.txt' local_file_path = local_base_dir + 'test.txt' local_compressed_file_path = local_base_dir + 'compressed.txt' local_path_dir = local_base_dir + 'res/test_dir' options = { 'webdav_hostname': 'http://localhost:8585', 'webdav_login': '******', 'webdav_password': '******', 'webdav_timeout': 10, 'webdav_override_methods': { 'check': 'GET' } } # options = { # 'webdav_hostname': 'https://demo1.nextcloud.com/remote.php/dav/files/RCw8Y9XXFnzkLJbN/', # 'webdav_login': '******', # 'webdav_password': '******', # 'webdav_override_methods': { # 'check': 'GET' # } # } # options = { # 'webdav_hostname': 'https://webdav.yandex.ru', # 'webdav_login': '******', # 'webdav_password': '******' # } def setUp(self): self.client = Client(self.options) self.clean_local_dir(self.local_path_dir) def tearDown(self): self.clean_local_dir(self.local_path_dir) self.clean_remote_dir(self.remote_path_dir) self.clean_remote_dir(self.remote_path_dir2) def clean_remote_dir(self, remote_path_dir): if self.client.check(remote_path=remote_path_dir): self.client.clean(remote_path=remote_path_dir) @staticmethod def clean_local_dir(local_path_dir): if path.exists(path=local_path_dir): shutil.rmtree(path=local_path_dir) def _prepare_for_downloading(self, inner_dir=False, base_path=''): if base_path: self._create_remote_dir_if_needed(base_path) self._prepare_dir_for_downloading(base_path + self.remote_path_dir, base_path + self.remote_path_file, self.local_file_path) if not path.exists(self.local_path_dir): os.makedirs(self.local_path_dir) if inner_dir: self._prepare_dir_for_downloading( base_path + self.remote_inner_path_dir, base_path + self.remote_inner_path_file, self.local_file_path) def _prepare_dir_for_downloading(self, remote_path_dir, remote_path_file, local_file_path): self._create_remote_dir_if_needed(remote_path_dir) if not self.client.check(remote_path=remote_path_file): self.client.upload_file(remote_path=remote_path_file, local_path=local_file_path) def _create_remote_dir_if_needed(self, remote_dir): if not self.client.check(remote_path=remote_dir): self.client.mkdir(remote_path=remote_dir) def _prepare_for_uploading(self): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not path.exists(path=self.local_path_dir): os.makedirs(self.local_path_dir) if not path.exists(path=self.local_path_dir + os.sep + self.local_file): shutil.copy(src=self.local_file_path, dst=self.local_path_dir + os.sep + self.local_file)
class WebDev(object): def __init__(self, options ): # self.client = wc.Client(options) self.client = Client(options) def read(self, name): with BytesIO() as buf: self.client.download_from(buf, name) return buf.getvalue() def write(self,content, target): data = self.read(target) with BytesIO() as buf: buf.write(data) buf.write(content.encode()) self.client.upload_to(buf.getvalue(), target) def mkdir(self, dir_name): self.client.mkdir(dir_name) def list(self): data = self.client.list() print(data) def clear(self, target): self.client.clean(target) def check(self,remote_path): data = self.client.check(remote_path) if data: print('{} is exists'.format(remote_path)) else: print('{} is not exists'.format(remote_path)) def upload(self,remote_path, local_path): self.client.upload(remote_path, local_path) def help_dev(self): print(dir(self.client)) def mkfile(self, remote_path): with BytesIO() as buf: buf.write(b'') self.client.upload_to(buf.getvalue(), remote_path) def download_file(self, remote_path): self.client.download_file(remote_path, remote_path) def rename(self, old_name, new_name): self.client.copy(old_name, old_name) self.client.clean(old_name) def set_property(self,remote_path, option): self.client.set_property(remote_path, option=[option])
class FileBrowserWebdavStorage(StorageMixin, Storage): def __init__(self, base_url='/', url_as_download=True, simple_listdir=False, webdav_root='/'): self.base_url = base_url self.url_as_download = url_as_download self.simple_listdir = simple_listdir webdav_client_options = { 'webdav_hostname': settings.CONTRAX_FILE_STORAGE_WEBDAV_ROOT_URL.rstrip('/'), 'webdav_login': settings.CONTRAX_FILE_STORAGE_WEBDAV_USERNAME, 'webdav_password': settings.CONTRAX_FILE_STORAGE_WEBDAV_PASSWORD, } self.client = Client(webdav_client_options) try: self.client.mkdir('/media') self.client.mkdir('/media/photo') except: pass self.client.webdav.root = webdav_root self.client.root = webdav_root def path(self, name): """ Return a local filesystem path where the file can be retrieved using Python's built-in open() function. Storage systems that can't be accessed using open() should *not* implement this method. """ # FIXME: this would be useful with self.location != '' # in this case use this notation: # 1. define self.location in __init__ # 2. rewrite path() method to be like # return os.oath.join(self.location, name) # 3. everywhere in other sel.methods use self.path(name) instead of name attr return name def isdir(self, path): """ Returns true if name exists and is a directory. """ return self.client.check(path) and self.client.is_dir(path) def isfile(self, path): """ Returns true if name exists and is a regular file. """ return self.client.check(path) and not self.client.is_dir(path) def move(self, old_file_name, new_file_name, allow_overwrite=False): """ Moves safely a file from one location to another. If allow_ovewrite==False and new_file_name exists, raises an exception. """ return self.client.move(remote_path_from=old_file_name, remote_path_to=new_file_name, overwrite=allow_overwrite) def makedirs(self, path): """ Creates all missing directories specified by name. Analogue to os.mkdirs(). """ return self.client.mkdir(path) def rmtree(self, path): """ Deletes a directory and everything it contains. Analogue to shutil.rmtree(). """ return self.client.clean(path) def setpermission(self, path): """ Sets file permission """ pass def _open(self, path, mode='rb'): tmp = io.BytesIO() self.client.download_from(tmp, path) tmp.seek(0) return File(tmp) def _save(self, path, content): res = self.client.resource(path) res.read_from(content) return path def get_valid_name(self, name): """ Return a filename, based on the provided filename, that's suitable for use in the target storage system. """ return get_valid_filename(name) def delete(self, path): """ Delete the specified file from the storage system. """ if self.exists(path): self.client.clean(path) def exists(self, path): """ Return True if a file referenced by the given name already exists in the storage system, or False if the name is available for a new file. """ return self.client.check(path) def listdir(self, path): """ List the contents of the specified path. Return a 2-tuple of lists: the first item being directories, the second item being files. """ _list = self.client.list(path) # for API: iterating over big directory take too much time if self.simple_listdir: return _list # for filebrowser directories, files = [], [] for entry in _list: entry_path = os.path.join(path, entry) if self.isdir(entry_path): directories.append(entry.rstrip('/')) else: files.append(entry) return directories, files def size(self, path): """ Return the total size, in bytes, of the file specified by name. """ return self.client.info(path)['size'] def url(self, path): """ Return an absolute URL where the file's contents can be accessed directly by a Web browser. """ url = filepath_to_uri(path) if url is not None: url = url.lstrip('/') url = urljoin(self.base_url, url) if self.url_as_download and self.isfile(path): url += '?action=download' return url @staticmethod def _datetime_from_timestamp(ts, fmt): """ If timezone support is enabled, make an aware datetime object in UTC; otherwise make a naive one in the local timezone. """ dt = datetime.strptime(ts, fmt) if settings.USE_TZ: # Safe to use .replace() because UTC doesn't have DST return dt.replace(tzinfo=timezone.utc) else: return dt def get_accessed_time(self, path): """ Return the last accessed time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ pass def get_created_time(self, path): """ Return the creation time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ return self._datetime_from_timestamp(self.client.info(path)['created'], fmt='%Y-%m-%dT%H:%M:%SZ') def get_modified_time(self, path): """ Return the last modified time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ return self._datetime_from_timestamp( self.client.info(path)['modified'], fmt='%a, %d %b %Y %H:%M:%S %Z')
#build a list of ALL courses in /internal courseList = client.list("internal/courses") #this is the list of dictionaries one for each backup theList = [] #for testing only, using islice to limit itterations for course in islice(courseList, 0, coursesToCheck): #construct full path to archive directory for this course arcpath = "internal/courses/" + course + "archive" #execute if archive directory exists if client.check(arcpath): #get a list of all backups in the directory backups = client.list(arcpath) #itterate through each backup for backup in backups: #filter by filename prefix if backup.startswith(prefixes): #push the file information into a dictionary backupInfo = client.info(arcpath + "/" + backup) #convert the created string to a date createdDateTime = datetime.strptime(backupInfo.get('created'), "%Y-%m-%dT%H:%M:%SZ")
def test_webdav(self): server_2_ip = '172.16.3.155' error, server_id = self.mm['webdavalpine'].run("add_server", ip_addr=server_2_ip, fqdn='webdav.test') self.assertTrue(error == None, msg=error) time.sleep(10) webdav_url = "https://{}/files/".format(server_2_ip) full_path = parentdir + "/work/webdavalpine/1/webdav/admin.pass" self.assertTrue(os.path.exists(full_path), msg="{} does not exist".format(full_path)) admin_pass = open(full_path, "r").read().strip() resp = requests.get("https://{}/".format(server_2_ip), verify=False) self.assertTrue(resp.status_code == 200) options = { 'webdav_hostname': webdav_url, 'webdav_login': "******", 'webdav_password': admin_pass } client = Client(options) client.verify = False result = client.check("public") self.assertTrue(result == True) result = client.list() self.assertTrue(len(result) > 0) subprocess.check_output(["touch /tmp/testdata"], shell=True) client.upload_sync("public/testdata", "/tmp/testdata") result = client.check("public/testdata") self.assertTrue(result == True) unauth_client = Client({ 'webdav_hostname': webdav_url }) unauth_client.verify = False public_list = unauth_client.list("public/") self.assertTrue(len(public_list) > 0) try: unauth_client.upload_sync("public/testdata2", "/tmp/testdata") self.fail() except: pass error, _ = self.mm['webdavalpine'].run("stop_server", id=server_id) self.assertTrue(error == None, msg=error) time.sleep(2) error, _ = self.mm['webdavalpine'].run("start_server", id=server_id) self.assertTrue(error == None, msg=error) time.sleep(10) resp = requests.get("https://{}/".format(server_2_ip), verify=False) self.assertTrue(resp.status_code == 200) options = { 'webdav_hostname': webdav_url, 'webdav_login': "******", 'webdav_password': admin_pass } client2 = Client(options) client2.verify = False public_list = client2.list("public/") self.assertTrue(len(public_list) > 0) self.assertTrue(public_list[1] == "testdata") error, _ = self.mm['webdavalpine'].run("remove_server", id=server_id) self.assertTrue(error == None, msg=error)
#open the client client = Client(options) x = client.list() #the file should have at least two columns "deleteMe" (Y/N) and "fullPath" with open("theList.csv") as csvfile: #this pushes the file data into a list of dictionaries reader = csv.DictReader(csvfile) #itterate on each dictionary in the list for row in reader: #if the backup doesn't exist, say so and move on if not client.check(row.get("fullPath")): print("Cannot find: " + row.get("fullPath")) continue #determine if the item is marked to be deleted if row.get("deleteMe") == "Y": #if the flag is True, this will remove the backup if removeFiles: client.clean(row.get("fullPath")) #determine if the backup still exists or not if client.check(row.get("fullPath")): print("Not deleted: " + row.get("fullPath")) else: print("Deleted: " + row.get("fullPath"))
class ClientTestCase(TestCase): remote_path_file = 'test_dir/test.txt' remote_path_file2 = 'test_dir2/test.txt' remote_path_dir = 'test_dir' remote_path_dir2 = 'test_dir2' local_base_dir = 'tests/' local_file = 'test.txt' local_file_path = local_base_dir + 'test.txt' local_path_dir = local_base_dir + 'res/test_dir' def setUp(self): options = { 'webdav_hostname': 'https://webdav.yandex.ru', 'webdav_login': '******', 'webdav_password': '******' } self.client = Client(options) if path.exists(path=self.local_path_dir): shutil.rmtree(path=self.local_path_dir) def tearDown(self): if path.exists(path=self.local_path_dir): shutil.rmtree(path=self.local_path_dir) if self.client.check(remote_path=self.remote_path_dir): self.client.clean(remote_path=self.remote_path_dir) if self.client.check(remote_path=self.remote_path_dir2): self.client.clean(remote_path=self.remote_path_dir2) def test_list(self): self._prepare_for_downloading() file_list = self.client.list() self.assertIsNotNone(file_list, 'List of files should not be None') self.assertGreater(file_list.__len__(), 0, 'Expected that amount of files more then 0') def test_free(self): self.assertGreater( self.client.free(), 0, 'Expected that free space on WebDAV server is more then 0 bytes') def test_check(self): self.assertTrue(self.client.check(), 'Expected that root directory is exist') def test_mkdir(self): if self.client.check(remote_path=self.remote_path_dir): self.client.clean(remote_path=self.remote_path_dir) self.client.mkdir(remote_path=self.remote_path_dir) self.assertTrue(self.client.check(remote_path=self.remote_path_dir), 'Expected the directory is created.') @unittest.skip( "Yandex brakes response for file it contains property resourcetype as collection but it should " "be empty for file") def test_download_to(self): self._prepare_for_downloading() buff = BytesIO() self.client.download_from(buff=buff, remote_path=self.remote_path_file) self.assertEquals(buff.getvalue(), 'test content for testing of webdav client') @unittest.skip( "Yandex brakes response for file it contains property resourcetype as collection but it should " "be empty for file") def test_download(self): self._prepare_for_downloading() self.client.download(local_path=self.local_path_dir, remote_path=self.remote_path_dir) self.assertTrue(path.exists(self.local_path_dir), 'Expected the directory is downloaded.') self.assertTrue(path.isdir(self.local_path_dir), 'Expected this is a directory.') self.assertTrue( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file is downloaded') self.assertTrue( path.isfile(self.local_path_dir + os.path.sep + self.local_path_file), 'Expected this is a file') @unittest.skip( "Yandex brakes response for file it contains property resourcetype as collection but it should " "be empty for file") def test_download_sync(self): self._prepare_for_downloading() os.mkdir(self.local_path_dir) def callback(): self.assertTrue( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file is downloaded') self.assertTrue( path.isfile(self.local_path_dir + os.path.sep + self.local_file), 'Expected this is a file') self.client.download_sync(local_path=self.local_path_dir + os.path.sep + self.local_file, remote_path=self.remote_path_file, callback=callback) self.assertTrue( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file has already been downloaded') @unittest.skip( "Yandex brakes response for file it contains property resourcetype as collection but it should " "be empty for file") def test_download_async(self): self._prepare_for_downloading() os.mkdir(self.local_path_dir) def callback(): self.assertTrue( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file is downloaded') self.assertTrue( path.isfile(self.local_path_dir + os.path.sep + self.local_file), 'Expected this is a file') self.client.download_async(local_path=self.local_path_dir + os.path.sep + self.local_file, remote_path=self.remote_path_file, callback=callback) self.assertFalse( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file has not been downloaded yet') def test_upload_from(self): self._prepare_for_uploading() buff = StringIO(u'test content for testing of webdav client') self.client.upload_to(buff=buff, remote_path=self.remote_path_file) self.assertTrue(self.client.check(self.remote_path_file), 'Expected the file is uploaded.') def test_upload(self): self._prepare_for_uploading() self.client.upload(remote_path=self.remote_path_file, local_path=self.local_path_dir) self.assertTrue(self.client.check(self.remote_path_dir), 'Expected the directory is created.') self.assertTrue(self.client.check(self.remote_path_file), 'Expected the file is uploaded.') def test_upload_file(self): self._prepare_for_uploading() self.client.upload_file(remote_path=self.remote_path_file, local_path=self.local_file_path) self.assertTrue(self.client.check(remote_path=self.remote_path_file), 'Expected the file is uploaded.') def test_upload_sync(self): self._prepare_for_uploading() def callback(): self.assertTrue(self.client.check(self.remote_path_dir), 'Expected the directory is created.') self.assertTrue(self.client.check(self.remote_path_file), 'Expected the file is uploaded.') self.client.upload(remote_path=self.remote_path_file, local_path=self.local_path_dir) def test_upload_async(self): self._prepare_for_uploading() def callback(): self.assertTrue(self.client.check(self.remote_path_dir), 'Expected the directory is created.') self.assertTrue(self.client.check(self.remote_path_file), 'Expected the file is uploaded.') self.client.upload(remote_path=self.remote_path_file, local_path=self.local_path_dir) def test_copy(self): self._prepare_for_downloading() self.client.mkdir(remote_path=self.remote_path_dir2) self.client.copy(remote_path_from=self.remote_path_file, remote_path_to=self.remote_path_file2) self.assertTrue(self.client.check(remote_path=self.remote_path_file2)) def test_move(self): self._prepare_for_downloading() self.client.mkdir(remote_path=self.remote_path_dir2) self.client.move(remote_path_from=self.remote_path_file, remote_path_to=self.remote_path_file2) self.assertFalse(self.client.check(remote_path=self.remote_path_file)) self.assertTrue(self.client.check(remote_path=self.remote_path_file2)) def test_clean(self): self._prepare_for_downloading() self.client.clean(remote_path=self.remote_path_dir) self.assertFalse(self.client.check(remote_path=self.remote_path_file)) self.assertFalse(self.client.check(remote_path=self.remote_path_dir)) def test_info(self): self._prepare_for_downloading() result = self.client.info(remote_path=self.remote_path_file) self.assertEquals(result['name'], 'test.txt') self.assertEquals(result['size'], '41') self.assertTrue('created' in result) self.assertTrue('modified' in result) def test_directory_is_dir(self): self._prepare_for_downloading() self.assertTrue(self.client.is_dir(self.remote_path_dir), 'Should return True for directory') def test_file_is_not_dir(self): self._prepare_for_downloading() self.assertFalse(self.client.is_dir(self.remote_path_file), 'Should return False for file') def test_get_property_of_non_exist(self): self._prepare_for_downloading() result = self.client.get_property(remote_path=self.remote_path_file, option={'name': 'aProperty'}) self.assertEquals( result, None, 'For not found property should return value as None') def test_set_property(self): self._prepare_for_downloading() self.client.set_property(remote_path=self.remote_path_file, option={ 'namespace': 'test', 'name': 'aProperty', 'value': 'aValue' }) result = self.client.get_property(remote_path=self.remote_path_file, option={ 'namespace': 'test', 'name': 'aProperty' }) self.assertEquals(result, 'aValue', 'Property value should be set') def test_set_property_batch(self): self._prepare_for_downloading() self.client.set_property_batch(remote_path=self.remote_path_file, option=[{ 'namespace': 'test', 'name': 'aProperty', 'value': 'aValue' }, { 'namespace': 'test', 'name': 'aProperty2', 'value': 'aValue2' }]) result = self.client.get_property(remote_path=self.remote_path_file, option={ 'namespace': 'test', 'name': 'aProperty' }) self.assertEquals(result, 'aValue', 'First property value should be set') result = self.client.get_property(remote_path=self.remote_path_file, option={ 'namespace': 'test', 'name': 'aProperty2' }) self.assertEquals(result, 'aValue2', 'Second property value should be set') def _prepare_for_downloading(self): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not self.client.check(remote_path=self.remote_path_file): self.client.upload_file(remote_path=self.remote_path_file, local_path=self.local_file_path) if not path.exists(self.local_path_dir): os.makedirs(self.local_path_dir) def _prepare_for_uploading(self): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not path.exists(path=self.local_path_dir): os.makedirs(self.local_path_dir) if not path.exists(path=self.local_path_dir + os.sep + self.local_file): shutil.copy(src=self.local_file_path, dst=self.local_path_dir + os.sep + self.local_file)
class BaseClientTestCase(unittest.TestCase): remote_path_file = 'test_dir/test.txt' remote_path_file2 = 'test_dir2/test.txt' remote_inner_path_file = 'test_dir/inner/test.txt' remote_path_dir = 'test_dir' remote_path_dir2 = 'test_dir2' remote_inner_path_dir = 'test_dir/inner' local_base_dir = 'tests/' local_file = 'test.txt' local_file_path = local_base_dir + 'test.txt' local_path_dir = local_base_dir + 'res/test_dir' options = { 'webdav_hostname': 'http://localhost:8585', 'webdav_login': '******', 'webdav_password': '******' } # options = { # 'webdav_hostname': 'https://webdav.yandex.ru', # 'webdav_login': '******', # 'webdav_password': '******' # } def setUp(self): self.client = Client(self.options) if path.exists(path=self.local_path_dir): shutil.rmtree(path=self.local_path_dir) def tearDown(self): if path.exists(path=self.local_path_dir): shutil.rmtree(path=self.local_path_dir) if self.client.check(remote_path=self.remote_path_dir): self.client.clean(remote_path=self.remote_path_dir) if self.client.check(remote_path=self.remote_path_dir2): self.client.clean(remote_path=self.remote_path_dir2) def _prepare_for_downloading(self, inner_dir=False): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not self.client.check(remote_path=self.remote_path_file): self.client.upload_file(remote_path=self.remote_path_file, local_path=self.local_file_path) if not path.exists(self.local_path_dir): os.makedirs(self.local_path_dir) if inner_dir: if not self.client.check(remote_path=self.remote_inner_path_dir): self.client.mkdir(remote_path=self.remote_inner_path_dir) if not self.client.check(remote_path=self.remote_inner_path_file): self.client.upload_file( remote_path=self.remote_inner_path_file, local_path=self.local_file_path) def _prepare_for_uploading(self): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not path.exists(path=self.local_path_dir): os.makedirs(self.local_path_dir) if not path.exists(path=self.local_path_dir + os.sep + self.local_file): shutil.copy(src=self.local_file_path, dst=self.local_path_dir + os.sep + self.local_file)
def test(self, start=True, gedis=True, webdav=False, bottle=True, websockets=False): """ kosmos 'j.servers.rack.test()' kosmos 'j.servers.rack.test(gedis_ssl=True)' kosmos 'j.servers.rack.test(ssl=False)' kosmos 'j.servers.rack.test(start=True)' :param manual means the server is run manually using e.g. js_shell 'j.servers.rack.start()' """ if start: self._server_test_start(background=True, gedis=gedis, webdav=webdav, bottle=bottle, websockets=websockets) namespace = "system" secret = "1234" cl = j.clients.gedis.get(namespace, port=8901, host="localhost") assert cl.ping() if webdav: # how to use see https://github.com/ezhov-evgeny/webdav-client-python-3/blob/da46592c6f1cc9fb810ca54019763b1e7dce4583/webdav3/client.py#L197 options = { "webdav_hostname": "http://127.0.0.1:4443", "webdav_login": "******", "webdav_password": "******" } from webdav3.client import Client cl = Client(options) cl.check() assert len(cl.list("")) > 0 if websockets: # TODO: does not work yet from websocket import create_connection ws = create_connection("ws://localhost:4444") ws.send("Hello, World") result = ws.recv() print(result) # ws.close() if bottle: import requests # https://realpython.com/python-requests/#the-get-request r1 = requests.get("http://localhost:4442/ping") self._log(r1.status_code) self._log(r1.content) assert r1.content == b"pong" assert r1.status_code == 200 print("tests are ok")
def on_event(self, event, payload): if event == "plugin_backup_backup_created": # Helper function for human readable sizes def _convert_size(size_bytes): if size_bytes == 0: return "0B" size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") i = int(math.floor(math.log(size_bytes, 1024))) p = math.pow(1024, i) s = round(size_bytes / p, 2) return "%s %s" % (s, size_name[i]) now = datetime.now() davoptions = { 'webdav_hostname': self._settings.get(["server"]), 'webdav_login': self._settings.get(["username"]), 'webdav_password': self._settings.get(["password"]), 'webdav_timeout': self._settings.get(["timeout"]), } backup_path = payload["path"] backup_name = payload["name"] self._logger.info("Backup " + backup_path + " created, will now attempt to upload to " + davoptions["webdav_hostname"]) davclient = Client(davoptions) davclient.verify = self._settings.get(["verify_certificate"]) check_space = self._settings.get(["check_space"]) upload_path = now.strftime(self._settings.get(["upload_path"])) upload_path = ospath.join("/", upload_path) if self._settings.get(["upload_name"]): upload_name = now.strftime(self._settings.get( ["upload_name"])) + ospath.splitext(backup_path)[1] else: upload_name = backup_name self._logger.debug("Filename for upload: " + upload_name) upload_file = ospath.join("/", upload_path, upload_name) upload_temp = ospath.join("/", upload_path, upload_name + ".tmp") self._logger.debug("Upload location: " + upload_file) # Check actual connection to the WebDAV server as the check command will not do this. if check_space: self._logger.debug("Attempting to check free space.") try: # If the resource was not found dav_free = davclient.free() if dav_free < 0: # If we get a negative free size, this server is not returning correct value. check_space = False self._logger.warning( "Free space on server: " + _convert_size(dav_free) + ", it appears your server does not support reporting size correctly but it's still a proper way to check connectivity." ) else: self._logger.info("Free space on server: " + _convert_size(dav_free)) except RemoteResourceNotFound as exception: self._logger.error( "Resource was not found, something is probably wrong with your settings." ) return except ResponseErrorCode as exception: # Write error and exit function status = HTTPStatus(exception.code) error_switcher = { 400: "Bad request", 401: "Unauthorized", 403: "Forbidden", 404: "Not found", 405: "Method not allowed", 408: "Request timeout", 500: "Internal error", 501: "Not implemented", 502: "Bad gateway", 503: "Service unavailable", 504: "Gateway timeout", 508: "Loop detected", } if (exception.code == 401): http_error = "HTTP error 401 encountered, your credentials are most likely wrong." else: http_error = "HTTP error encountered: " + str( status.value) + " " + error_switcher.get( exception.code, status.phrase) self._logger.error(http_error) return except WebDavException as exception: self._logger.error( "An unexpected WebDAV error was encountered: " + exception.args) raise else: self._logger.debug( "Not checking free space, just try to check the WebDAV root." ) # Not as proper of a check as retrieving size, but it's something. if davclient.check("/"): self._logger.debug("Server returned WebDAV root.") else: self._logger.error( "Server did not return WebDAV root, something is probably wronkg with your settings." ) return backup_size = ospath.getsize(backup_path) self._logger.info("Backup file size: " + _convert_size(backup_size)) if check_space and (backup_size > dav_free): self._logger.error("Unable to upload, size is" + _convert_size(backup_size) + ", free space is " + _convert_size(dav_free)) return else: # Helper function to recursively create paths def _recursive_create_path(path): # Append leading / for preventing abspath issues path = ospath.join("/", path) if davclient.check(path): self._logger.debug("Directory " + path + " was found.") return True else: if path != "/": self._logger.debug( "Directory " + path + " was not found, checking parent.") if _recursive_create_path( ospath.abspath(ospath.join(path, ".."))): davclient.mkdir(path) self._logger.debug("Directory " + path + " has been created.") return True else: self._logger.error( "Could not find WebDAV root, something is probably wrong with your settings." ) return False if _recursive_create_path(upload_path): self._logger.debug("Uploading " + backup_path + " to " + upload_temp) davclient.upload_sync(remote_path=upload_temp, local_path=backup_path) self._logger.debug("Moving " + upload_temp + " to " + upload_file) davclient.move(remote_path_from=upload_temp, remote_path_to=upload_file) self._logger.info( "Backup has been uploaded successfully to " + davoptions["webdav_hostname"] + " as " + upload_file) else: self._logger.error( "Something went wrong trying to check/create the upload path." )
k += 1 else: print('\r' + ' ' * l + '\r', end='') k = 0 print(s * (l - k) + '] ' + ' = 100%') options = { 'webdav_hostname': token, 'webdav_login': login, 'webdav_password': password, } client = Client(options) rem_path = 'tmp' if not client.check(rem_path): # если нет, то создаём client.mkdir(rem_path) mymes('Directory [' + rem_path + '] created', 0) client.info(rem_path) video_path = './videos' files = os.listdir(video_path) free_space() for file in files: if re.fullmatch(r'Video\d\.\w{,5}', file): client.upload_async(remote_path=rem_path + '/' + file, local_path=video_path + '/' + file, callback=callback) upload_progress(file, '#', 0.3) callback_flag = False