def upload():
hostname = config['webdav']['hostname']
option = {
'webdav_hostname': hostname,
'webdav_login': config['webdav']['username'],
'webdav_password': config['webdav']['password'],
'disable_check': True, # 有的网盘不支持check功能
}
client = Client(option)
av_min_size = 500 * 1024 * 1024 # size of av should > 500 MB
paths = list_all_paths(client, '/')
av_paths = []
for path in paths:
info = client.info(path)
if info['size'] is not None:
size = int(info['size'])
if size > av_min_size:
url = f'{hostname}/{path}'
url = url.replace('///', '/')
av_paths.append(url)
print(path)
file.write_all_lines('paths.txt', av_paths)
def certs_from_webdav():
store_type = os.environ.get("GLUU_DOCUMENT_STORE_TYPE", "LOCAL")
if store_type != "JCA":
return
url = get_jackrabbit_url()
username = os.environ.get("GLUU_JACKRABBIT_ADMIN_ID", "admin")
password = ""
password_file = os.environ.get(
"GLUU_JACKRABBIT_ADMIN_PASSWORD_FILE",
"/etc/gluu/conf/jackrabbit_admin_password",
)
with contextlib.suppress(FileNotFoundError):
with open(password_file) as f:
password = f.read().strip()
password = password or username
client = Client({
"webdav_hostname": url,
"webdav_login": username,
"webdav_password": password,
"webdav_root": "/repository/default",
})
certs = [
"/etc/certs/otp_configuration.json",
"/etc/certs/super_gluu_creds.json",
]
try:
for cert in certs:
client.download_file(cert, cert)
except (RemoteResourceNotFound, NoConnection):
pass
def webdav_client(self, webdav_options: dict = None):
# We localize the import of webdav3 here so it is an optional dependency. Only users who want to use webdav will
# need to pip install webdavclient3
from webdav3.client import Client
options = {
"webdav_hostname": self.base_url,
}
if self._api_key is not None:
options["webdav_login"] = "******"
options["webdav_password"] = f"apikey|{self._api_key}"
if webdav_options is not None:
options = {
**options,
**webdav_options,
}
client = Client(options)
if self._verify_ssl is False:
client.verify = False # Set verify to false if using localhost without HTTPS
return client
def reauth(self):
"""Checks token expiry and re-initialises the Client if a new token is needed.
"""
if self.token["expires_at"] < int(time.time()):
self.client.getToken()
self.options.update({"webdav_token": self.token["access_token"]})
self.webdav_client = Client(self.options)
def sync_from_webdav(url, username, password):
client = Client({
"webdav_hostname": url,
"webdav_login": username,
"webdav_password": password,
"webdav_root": ROOT_DIR,
})
try:
logger.info(f"Sync files from {url}{ROOT_DIR}{SYNC_DIR}")
# download files to temporary directory to avoid `/opt/gluu/jetty/oxauth/custom`
# getting deleted
client.download(SYNC_DIR, TMP_DIR)
# copy all downloaded files to /opt/gluu/jetty/oxauth/custom
for subdir, _, files in os.walk(TMP_DIR):
for file_ in files:
src = os.path.join(subdir, file_)
dest = src.replace(TMP_DIR, SYNC_DIR)
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
# logger.info(f"Copying {src} to {dest}")
shutil.copyfile(src, dest)
except (RemoteResourceNotFound, NoConnection) as exc:
logger.warning(
f"Unable to sync files from {url}{ROOT_DIR}{SYNC_DIR}; reason={exc}"
)
def main():
args = create_parser().parse_args()
webdav_url = args.webdav_url
webdav_path = args.webdav_path
pattern = args.pattern
login = args.login
password = args.password
options = {
'webdav_hostname': webdav_url,
'webdav_login': login,
'webdav_password': password
}
client = Client(options)
client.verify = False
not_matched_files = {}
for dir in client.list(webdav_path):
new_path = '/'.join([webdav_path, dir])
files = client.list(new_path)
for file in files:
if not re.match(pattern, file):
not_matched_files[''.join([new_path, file])] = 'NOT MATCHED'
print('Running on the options {}'.format({
'webdav_url': webdav_url,
'webdav_path': webdav_path,
'pattern': pattern
}))
print(
tabulate(not_matched_files.items(),
headers=['FILENAME', 'WARNING'],
tablefmt="grid"))
class WebDavConnection:
def __init__(self, host, username, password):
self.host = host
self.username = username
self.password = password
def connect(self):
options = {
'webdav_hostname': self.host,
'webdav_login': self.username,
'webdav_password': self.password
}
self.connection = WebdavClient(options)
def get_modified(self, path):
modified = self.connection.info(path)['modified']
format = '%a, %d %b %Y %X %Z'
timestamp_tzaware = parser.parse(modified)
# fromtimestamp() does not include the timezone.
# So stripping it here.
return datetime.datetime.fromtimestamp(timestamp_tzaware.timestamp())
def download(self, remote_path, local_path):
self.connection.download_file(remote_path, local_path)
def upload(self, remote_path, local_path):
self.connection.upload_file(remote_path, local_path)
def _client(self):
from webdav3.client import Client
# Set password or ask for it
if self.ask_password and self.password is None and self.token is None:
self.password = ask_password(self.hostname, self.user)
# Setup webdav client options dictionary
options = {
"webdav_hostname": self.hostname,
"webdav_login": self.user,
"webdav_password": self.password,
"webdav_token": self.token,
"webdav_cert_path": self.cert_path,
"webdav_key_path": self.key_path,
"webdav_timeout": self.timeout,
"webdav_chunk_size": self.CHUNK_SIZE,
}
client = Client(options)
# Check whether client options are valid
if not client.valid():
raise ConfigError(
f"Configuration for WebDAV {self.hostname} is invalid.")
# Check whether connection is valid (root should always exist)
if not client.check(self.path_info.path):
raise WebDAVConnectionError(self.hostname)
return client
def __init__(self,
base_url='/',
url_as_download=True,
simple_listdir=False,
webdav_root='/'):
self.base_url = base_url
self.url_as_download = url_as_download
self.simple_listdir = simple_listdir
webdav_client_options = {
'webdav_hostname':
settings.CONTRAX_FILE_STORAGE_WEBDAV_ROOT_URL.rstrip('/'),
'webdav_login':
settings.CONTRAX_FILE_STORAGE_WEBDAV_USERNAME,
'webdav_password':
settings.CONTRAX_FILE_STORAGE_WEBDAV_PASSWORD,
}
self.client = Client(webdav_client_options)
try:
self.client.mkdir('/media')
self.client.mkdir('/media/photo')
except:
pass
self.client.webdav.root = webdav_root
self.client.root = webdav_root
def setUp(self):
options = {
'webdav_hostname': 'https://webdav.yandex.ru',
'webdav_login': '******',
'webdav_password': '******'
}
self.client = Client(options)
if path.exists(path=self.local_path_dir):
shutil.rmtree(path=self.local_path_dir)
def handle(self, *args, **kwargs):
localpath = settings.DATABASES['default']['NAME']
dtstr = datetime.now().strftime(r'%Y-%m-%d')
ext = os.path.basename(localpath).split('.')[-1]
remotepath = settings.WEBDAV_BACKUP_DIR
remotepath += f'/{settings.SITE_NAME}-{settings.HOSTNAME}-{dtstr}.{ext}'.lower()
log.info('Backing up database localpath: %s; remotepath: %s', localpath, remotepath)
client = Client(settings.WEBDAV_OPTIONS)
client.upload_sync(local_path=localpath, remote_path=remotepath)
def download_to_path(options, paths):
client = Client(options)
for path in paths:
urn = Urn(path)
create_path = path if urn.is_dir() else os.path.dirname(path)
try:
os.makedirs(create_path)
except:
pass
client.download_sync(path, path)
class connector:
def __init__(self, config):
self.config = config
options = {
'webdav_hostname': config['Connector']['Host'],
'webdav_login': config['Connector']['Username'],
'webdav_password': config['Connector']['Password']
}
self.client = Client(options)
self.client.verfiy = False
self.base = config['Connector']['Base']
def listdir(self):
l = self.client.list(self.base)
l.pop(l.index(self.base + '/'))
print(l)
return l
def remove(self, name):
self.client.clean(self.base + '/' + name)
def check(self, name):
return self.client.check(self.base + '/' + name)
def get(self, name):
return self.client.resource(self.base + '/' + name)
def create(self, name):
self.client.upload_to(None, self.base + '/' + name)
def upload(self, name):
self.client.upload_file(self.base + '/' + name, name)
def __init__(self, config):
self.config = config
options = {
'webdav_hostname': config['Connector']['Host'],
'webdav_login': config['Connector']['Username'],
'webdav_password': config['Connector']['Password']
}
self.client = Client(options)
self.client.verfiy = False
self.base = config['Connector']['Base']
def connect(self):
options = {
'webdav_hostname': self.web_dav_host,
'webdav_login': self.web_dav_login,
'webdav_password': self.web_dav_password,
'webdav_override_methods': {
'check': 'GET'
}
}
self.client = Client(options)
def upload(options=options):
client = Client(options)
if not client.check("/server"):
client.mkdir("/server")
path="/server/"+str(int(time.time()))
client.mkdir(path)
client.upload(path,"./dat/")
print("Data backup succeeded!")
def put_file(filename_local, filename_remote):
from webdav3.client import Client
options = {
'webdav_hostname': WEBDAV_HOSTNAME,
'webdav_login': WEBDAV_LOGIN,
'webdav_password': WEBDAV_PASSWORD
}
print(options)
client = Client(options)
client.verify = False # To not check SSL certificates (Default = True)
client.upload_sync(remote_path="/Documents/esp_frame/%s" % filename_remote,
local_path=filename_local)
def __init__(self, conf: WebDavConfig):
url = f"http{'s' if conf.use_https else ''}://{conf.host}:{conf.port}{conf.path}{conf.root_dir}"
options = {
'webdav_hostname': url,
'webdav_login': conf.username,
'webdav_password': conf.password,
'webdav_timeout': 600
}
self.client = Client(options)
if conf.force_direct:
self.client.session.proxies = {}
self.sem = threading.Semaphore(5)
def loadBaseInfo(self):
url = self.conf.decrypt(self.conf.getOption('webDav', 'url'))
if url.endswith("/"):
url = url[0:-1]
self.options = {
'webdav_hostname': url,
'webdav_login': self.conf.decrypt(self.conf.getOption('webDav', 'username')),
'webdav_password': self.conf.decrypt(self.conf.getOption('webDav', 'password')),
# 'webdav_root': '/dav/',
'disable_check': True,
}
self.client = Client(self.options)
def test_check_another_client(self):
self._prepare_for_uploading()
client = Client(self.options)
if self.client.check(self.remote_path_dir):
self.client.clean(self.remote_path_dir)
self.assertTrue(self.client.mkdir(self.remote_path_dir))
self.assertTrue(self.client.check(self.remote_path_dir))
self.client.upload_sync(remote_path=self.remote_path_file, local_path=self.local_path_dir)
self.assertTrue(self.client.check(self.remote_path_file))
self.assertTrue(client.check(self.remote_path_dir))
self.assertTrue(client.check(self.remote_path_file))
def show_WebDAV(self, session):
cfg = session.config
hpc = cfg["hpc"]
lobcder = hpc["url"]
port = lobcder[lobcder.find(":"):]
webdav_host = lobcder.replace(port, "32223")
options = {
'webdav_hostname': webdav_host,
'webdav_login': "******",
'webdav_password': hpc["webdav_pwd"]
}
client = Client(options)
session.rw_fits = client.list("krk/LOFAR_results")
def __init__(self):
#-------------------------------------------------------
# WebWavの接続情報
options = {
'webdav_hostname': "https://seto.teracloud.jp/dav/",
'webdav_login': "******",
'webdav_password': "******"
}
#-------------------------------------------------------
# WebWavクライアント
self.client = Client(options)
self.client.verify = True # To not check SSL certificates (Default = True)
def upload_webdav(fn, url, login, pwd):
logger.debug("Uploading file to webdav")
#logger.debug("URL: "+ url)
#logger.debug("Filename: "+ fn)
options = {
'webdav_hostname': url,
'webdav_login': login,
'webdav_password': pwd
}
client = Client(options)
client.verify = False
client.upload_sync(remote_path=basename(fn), local_path=fn)
return True
def __init__(self, userId, apiKey=None):
self._user_id = userId
self._access_token = (apiKey if apiKey is not None else Util.loadToken(
userId, "port-owncloud"))
options = {
"webdav_hostname":
"{}/remote.php/webdav".format(
os.getenv("OWNCLOUD_INSTALLATION_URL",
"http://localhost:3000")),
"webdav_token":
self._access_token,
}
self.client = Client(options)
self.client.verify = os.environ.get("VERIFY_SSL", "True") == "True"
def _setup_test_dir(webdav_url, token):
with tempfile.TemporaryDirectory() as tmpdirname:
path = pathlib.Path(tmpdirname)
root = path / 'test'
root.mkdir()
for subdir in {'testdir_1', 'testdir_2', 'empty_testdir'}:
path = root / subdir
path.mkdir()
if 'empty' in subdir:
continue
for file in {'file_1.txt', 'file_2.txt'}:
file = path / file
file.write_text(_file_content)
client = Client(dict(webdav_hostname=webdav_url, webdav_token=token))
client.upload(f'/{root.name}', root.as_posix())
def __init__(self, host, user, password):
"""initializes a webdav connection
Args:
host (string): host url
user (string): dav user name
password (string): dav password
"""
self.options = {
'webdav_hostname': host,
'webdav_login': user,
'webdav_password': password
}
self.client=Client(self.options)
class DavSync(Sync):
"""DavSync synchronizes the changes with a webdav server.
Attributes:
path (:obj:`str`): The local directory path to synchronize.
dav_path (:obj:`str`): The remote directory path which corresponds to the local one.
dav_root (:obj:`str`): The path to access the dav server. Nextcloud, e.g., uses /remote.php/webdav/
username (:obj:`str`): Webdav username.
password (:obj:`str`): Webdav password.
hostname (:obj:`str`): Webdav server host, e.g. https://cloud.example.com/
"""
def __init__(
self,
path: str,
dav_path: str,
dav_root: str,
username: str,
password: str,
hostname: str,
):
super().__init__(path)
self.dav_path = dav_path
self.username = username
self.password = password
self.hostname = hostname
options = {
"webdav_hostname": hostname,
"webdav_login": username,
"webdav_password": password,
"root": dav_root,
}
self.client = Client(options)
self.pull()
def pull(self):
"""Download updated directory from server."""
self.client.download_sync(self.dav_path, self.os_path)
def push(self, fname, msg=""):
"""Upload a file to the server. If the directory or file does not exist
on the remote server, create it.
Args:
fname (:obj:`str`): File to upload.
msg (:obj:`str`, optional): Not used with DavSync.
"""
self.client.upload_file(path.join(self.dav_path, fname),
path.join(self.os_path, fname))
def test_not_enough_space(self, mock_session):
client = Client(self.options)
client.session.request.return_value.status_code = 507
self.assertRaises(NotEnoughSpace,
client.execute_request,
action='list',
path='')
def test_not_found(self, mock_session):
client = Client(self.options)
client.session.request.return_value.status_code = 404
self.assertRaises(RemoteResourceNotFound,
client.execute_request,
action='list',
path='')
def test_method_not_supported(self, mock_session):
client = Client(self.options)
client.session.request.return_value.status_code = 405
self.assertRaises(MethodNotSupported,
client.execute_request,
action='list',
path='')
