def upload(options=options):
client = Client(options)
if not client.check("/server"):
client.mkdir("/server")
path="/server/"+str(int(time.time()))
client.mkdir(path)
client.upload(path,"./dat/")
print("Data backup succeeded!")
class WebDavServiceBase:
def __init__(self, conf: WebDavConfig):
url = f"http{'s' if conf.use_https else ''}://{conf.host}:{conf.port}{conf.path}{conf.root_dir}"
options = {
'webdav_hostname': url,
'webdav_login': conf.username,
'webdav_password': conf.password,
'webdav_timeout': 600
}
self.client = Client(options)
if conf.force_direct:
self.client.session.proxies = {}
self.sem = threading.Semaphore(5)
def dir_exists(self, path: str):
try:
# rclone doesn't allow you check an existing dir
return self.client.check(path)
except webdav3.exceptions.MethodNotSupported as e:
print(e)
return True
def ensure_dir(self, tag: str, path: str):
service_name = f"{SERVICE_NAME}:{tag}"
if ServiceKVStore.exists(service_name, path):
return
if self.dir_exists(path):
ServiceKVStore.put(service_name, path, {})
return
self.client.mkdir(path)
ServiceKVStore.put(service_name, path, {})
def write_file(self, tag: str, service: ServiceType, dir_name: str,
filename: str, buffer: IO):
with self.sem:
self.ensure_dir(tag, service.value)
self.ensure_dir(tag, f"{service.value}/{dir_name}")
fp = f"{service.value}/{dir_name}/{filename}"
with NamedTemporaryFile() as f:
f.write(buffer.read())
f.flush()
try:
self.client.upload(fp, f.name)
except webdav3.exceptions.RemoteParentNotFound as err:
print(err)
pp = f"{service.value}/{dir_name}"
# assert not ServiceKVStore.exists(SERVICE_NAME, pp)
# self.client.mkdir(pp)
from webdav3.urn import Urn
directory_urn = Urn(pp, directory=True)
response = self.client.execute_request(
action='mkdir', path=directory_urn.quote())
assert response in (200, 201), response
ServiceKVStore.put(SERVICE_NAME, pp, {})
self.client.upload(fp, f.name)
return fp
class webDavService:
def __init__(self):
self.conf = config()
log = logger()
self.path = os.path.dirname(os.path.realpath(sys.argv[0]))
self.confwebdav = log.getlogger('webdav')
self.options = {}
self.client = None
def loadBaseInfo(self):
url = self.conf.decrypt(self.conf.getOption('webDav', 'url'))
if url.endswith("/"):
url = url[0:-1]
self.options = {
'webdav_hostname': url,
'webdav_login': self.conf.decrypt(self.conf.getOption('webDav', 'username')),
'webdav_password': self.conf.decrypt(self.conf.getOption('webDav', 'password')),
# 'webdav_root': '/dav/',
'disable_check': True,
}
self.client = Client(self.options)
def creatDir(self):
try:
self.client.mkdir("tomato_db")
except Exception as e:
self.confwebdav.error(e)
pass
def clean(self):
try:
self.client.clean("tomato_db/tomato.db")
except Exception as e:
self.confwebdav.error(e)
pass
def upload(self, file):
try:
self.loadBaseInfo()
self.creatDir()
self.clean()
self.client.upload(remote_path="tomato_db/tomato.db",
local_path=self.path + file)
except Exception as e:
self.confwebdav.error(e)
pass
def download(self, file):
try:
self.loadBaseInfo()
self.client.download(remote_path="tomato_db/tomato.db",
local_path=self.path + file)
except Exception as e:
self.confwebdav.error(e)
pass
def _setup_test_dir(webdav_url, token):
with tempfile.TemporaryDirectory() as tmpdirname:
path = pathlib.Path(tmpdirname)
root = path / 'test'
root.mkdir()
for subdir in {'testdir_1', 'testdir_2', 'empty_testdir'}:
path = root / subdir
path.mkdir()
if 'empty' in subdir:
continue
for file in {'file_1.txt', 'file_2.txt'}:
file = path / file
file.write_text(_file_content)
client = Client(dict(webdav_hostname=webdav_url, webdav_token=token))
client.upload(f'/{root.name}', root.as_posix())
def upload():
options = {
'webdav_hostname': "https://dav.jianguoyun.com/dav/backup/",
'webdav_login': "******",
'webdav_password': "******",
# 'disable_check': True, #有的网盘不支持check功能
}
client = Client(options)
# 我选择用时间戳为备份文件命名
file_name = str(math.floor(datetime.now().timestamp())) + '.bak'
try:
# 写死的路径，第一个参数是网盘地址
client.upload('backup/' + file_name,
'/home/shaobowang/programming/风格迁移前端测试/BLL/image1.jpg')
# 打印结果，之后会重定向到log
print('upload at ' + file_name)
except LocalResourceNotFound as exception:
print('An error happen: LocalResourceNotFound ---' + file_name)
if driver_prefix==None or driver_prefix=="Home":
print("请先新建Wiki页面，详情请参考项目Readme!")
exit(-1)
driver_prefix = driver_prefix[:-3]
webdav_url = os.getenv(f"{driver_prefix}_url")
webdav_username = os.getenv(f"{driver_prefix}_username")
webdav_password = os.getenv(f"{driver_prefix}_password")
if webdav_username==None or webdav_username=="" or webdav_password==None or webdav_password=="":
print(f"{driver_prefix}_username 或 {driver_prefix}_password 没有配置secret")
exit(-1)
options = {
'webdav_hostname': webdav_url,
'webdav_login': webdav_username,
'webdav_password': webdav_password,
'disable_check': True
}
client = Client(options)
os.chdir("download")
for it in os.listdir():
if os.path.isfile(it):
client.upload(remote_path=it,local_path=it)
print("✨ All file uploaded")
class ClientTestCase(TestCase):
remote_path_file = 'test_dir/test.txt'
remote_path_file2 = 'test_dir2/test.txt'
remote_path_dir = 'test_dir'
remote_path_dir2 = 'test_dir2'
local_base_dir = 'tests/'
local_file = 'test.txt'
local_file_path = local_base_dir + 'test.txt'
local_path_dir = local_base_dir + 'res/test_dir'
def setUp(self):
options = {
'webdav_hostname': 'https://webdav.yandex.ru',
'webdav_login': '******',
'webdav_password': '******'
}
self.client = Client(options)
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
def tearDown(self):
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
if self.client.check(remote_path=self.remote_path_dir):
self.client.clean(remote_path=self.remote_path_dir)
if self.client.check(remote_path=self.remote_path_dir2):
self.client.clean(remote_path=self.remote_path_dir2)
def test_list(self):
self._prepare_for_downloading()
file_list = self.client.list()
self.assertIsNotNone(file_list, 'List of files should not be None')
self.assertGreater(file_list.__len__(), 0,
'Expected that amount of files more then 0')
def test_free(self):
self.assertGreater(
self.client.free(), 0,
'Expected that free space on WebDAV server is more then 0 bytes')
def test_check(self):
self.assertTrue(self.client.check(),
'Expected that root directory is exist')
def test_mkdir(self):
if self.client.check(remote_path=self.remote_path_dir):
self.client.clean(remote_path=self.remote_path_dir)
self.client.mkdir(remote_path=self.remote_path_dir)
self.assertTrue(self.client.check(remote_path=self.remote_path_dir),
'Expected the directory is created.')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_to(self):
self._prepare_for_downloading()
buff = BytesIO()
self.client.download_from(buff=buff, remote_path=self.remote_path_file)
self.assertEquals(buff.getvalue(),
'test content for testing of webdav client')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download(self):
self._prepare_for_downloading()
self.client.download(local_path=self.local_path_dir,
remote_path=self.remote_path_dir)
self.assertTrue(path.exists(self.local_path_dir),
'Expected the directory is downloaded.')
self.assertTrue(path.isdir(self.local_path_dir),
'Expected this is a directory.')
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_path_file), 'Expected this is a file')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_sync(self):
self._prepare_for_downloading()
os.mkdir(self.local_path_dir)
def callback():
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep +
self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_file), 'Expected this is a file')
self.client.download_sync(local_path=self.local_path_dir +
os.path.sep + self.local_file,
remote_path=self.remote_path_file,
callback=callback)
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file has already been downloaded')
@unittest.skip(
"Yandex brakes response for file it contains property resourcetype as collection but it should "
"be empty for file")
def test_download_async(self):
self._prepare_for_downloading()
os.mkdir(self.local_path_dir)
def callback():
self.assertTrue(
path.exists(self.local_path_dir + os.path.sep +
self.local_file),
'Expected the file is downloaded')
self.assertTrue(
path.isfile(self.local_path_dir + os.path.sep +
self.local_file), 'Expected this is a file')
self.client.download_async(local_path=self.local_path_dir +
os.path.sep + self.local_file,
remote_path=self.remote_path_file,
callback=callback)
self.assertFalse(
path.exists(self.local_path_dir + os.path.sep + self.local_file),
'Expected the file has not been downloaded yet')
def test_upload_from(self):
self._prepare_for_uploading()
buff = StringIO(u'test content for testing of webdav client')
self.client.upload_to(buff=buff, remote_path=self.remote_path_file)
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
def test_upload(self):
self._prepare_for_uploading()
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
def test_upload_file(self):
self._prepare_for_uploading()
self.client.upload_file(remote_path=self.remote_path_file,
local_path=self.local_file_path)
self.assertTrue(self.client.check(remote_path=self.remote_path_file),
'Expected the file is uploaded.')
def test_upload_sync(self):
self._prepare_for_uploading()
def callback():
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
def test_upload_async(self):
self._prepare_for_uploading()
def callback():
self.assertTrue(self.client.check(self.remote_path_dir),
'Expected the directory is created.')
self.assertTrue(self.client.check(self.remote_path_file),
'Expected the file is uploaded.')
self.client.upload(remote_path=self.remote_path_file,
local_path=self.local_path_dir)
def test_copy(self):
self._prepare_for_downloading()
self.client.mkdir(remote_path=self.remote_path_dir2)
self.client.copy(remote_path_from=self.remote_path_file,
remote_path_to=self.remote_path_file2)
self.assertTrue(self.client.check(remote_path=self.remote_path_file2))
def test_move(self):
self._prepare_for_downloading()
self.client.mkdir(remote_path=self.remote_path_dir2)
self.client.move(remote_path_from=self.remote_path_file,
remote_path_to=self.remote_path_file2)
self.assertFalse(self.client.check(remote_path=self.remote_path_file))
self.assertTrue(self.client.check(remote_path=self.remote_path_file2))
def test_clean(self):
self._prepare_for_downloading()
self.client.clean(remote_path=self.remote_path_dir)
self.assertFalse(self.client.check(remote_path=self.remote_path_file))
self.assertFalse(self.client.check(remote_path=self.remote_path_dir))
def test_info(self):
self._prepare_for_downloading()
result = self.client.info(remote_path=self.remote_path_file)
self.assertEquals(result['name'], 'test.txt')
self.assertEquals(result['size'], '41')
self.assertTrue('created' in result)
self.assertTrue('modified' in result)
def test_directory_is_dir(self):
self._prepare_for_downloading()
self.assertTrue(self.client.is_dir(self.remote_path_dir),
'Should return True for directory')
def test_file_is_not_dir(self):
self._prepare_for_downloading()
self.assertFalse(self.client.is_dir(self.remote_path_file),
'Should return False for file')
def test_get_property_of_non_exist(self):
self._prepare_for_downloading()
result = self.client.get_property(remote_path=self.remote_path_file,
option={'name': 'aProperty'})
self.assertEquals(
result, None, 'For not found property should return value as None')
def test_set_property(self):
self._prepare_for_downloading()
self.client.set_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty',
'value': 'aValue'
})
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty'
})
self.assertEquals(result, 'aValue', 'Property value should be set')
def test_set_property_batch(self):
self._prepare_for_downloading()
self.client.set_property_batch(remote_path=self.remote_path_file,
option=[{
'namespace': 'test',
'name': 'aProperty',
'value': 'aValue'
}, {
'namespace': 'test',
'name': 'aProperty2',
'value': 'aValue2'
}])
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty'
})
self.assertEquals(result, 'aValue',
'First property value should be set')
result = self.client.get_property(remote_path=self.remote_path_file,
option={
'namespace': 'test',
'name': 'aProperty2'
})
self.assertEquals(result, 'aValue2',
'Second property value should be set')
def _prepare_for_downloading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not self.client.check(remote_path=self.remote_path_file):
self.client.upload_file(remote_path=self.remote_path_file,
local_path=self.local_file_path)
if not path.exists(self.local_path_dir):
os.makedirs(self.local_path_dir)
def _prepare_for_uploading(self):
if not self.client.check(remote_path=self.remote_path_dir):
self.client.mkdir(remote_path=self.remote_path_dir)
if not path.exists(path=self.local_path_dir):
os.makedirs(self.local_path_dir)
if not path.exists(path=self.local_path_dir + os.sep +
self.local_file):
shutil.copy(src=self.local_file_path,
dst=self.local_path_dir + os.sep + self.local_file)
class WebDev(object):
def __init__(self, options ):
# self.client = wc.Client(options)
self.client = Client(options)
def read(self, name):
with BytesIO() as buf:
self.client.download_from(buf, name)
return buf.getvalue()
def write(self,content, target):
data = self.read(target)
with BytesIO() as buf:
buf.write(data)
buf.write(content.encode())
self.client.upload_to(buf.getvalue(), target)
def mkdir(self, dir_name):
self.client.mkdir(dir_name)
def list(self):
data = self.client.list()
print(data)
def clear(self, target):
self.client.clean(target)
def check(self,remote_path):
data = self.client.check(remote_path)
if data:
print('{} is exists'.format(remote_path))
else:
print('{} is not exists'.format(remote_path))
def upload(self,remote_path, local_path):
self.client.upload(remote_path, local_path)
def help_dev(self):
print(dir(self.client))
def mkfile(self, remote_path):
with BytesIO() as buf:
buf.write(b'')
self.client.upload_to(buf.getvalue(), remote_path)
def download_file(self, remote_path):
self.client.download_file(remote_path, remote_path)
def rename(self, old_name, new_name):
self.client.copy(old_name, old_name)
self.client.clean(old_name)
def set_property(self,remote_path, option):
self.client.set_property(remote_path, option=[option])
