def push(self, remote_directory, local_directory):
def prune(src, exp):
return [sub(exp, "", item) for item in src]
updated = False
urn = Urn(remote_directory, directory=True)
self._validate_remote_directory(urn)
self._validate_local_directory(local_directory)
paths = self.list(urn.path())
expression = "{begin}{end}".format(begin="^", end=urn.path())
remote_resource_names = prune(paths, expression)
for local_resource_name in listdir(local_directory):
local_path = os.path.join(local_directory, local_resource_name)
remote_path = "{remote_directory}{resource_name}".format(
remote_directory=urn.path(), resource_name=local_resource_name)
if os.path.isdir(local_path):
if not self.check(remote_path=remote_path):
self.mkdir(remote_path=remote_path)
result = self.push(remote_directory=remote_path,
local_directory=local_path)
updated = updated or result
else:
if local_resource_name in remote_resource_names and not self.is_local_more_recent(
local_path, remote_path):
continue
self.upload_file(remote_path=remote_path,
local_path=local_path)
updated = True
return updated
def pull(self, remote_directory, local_directory):
def prune(src, exp):
return [sub(exp, "", item) for item in src]
updated = False
urn = Urn(remote_directory, directory=True)
self._validate_remote_directory(urn)
self._validate_local_directory(local_directory)
local_resource_names = listdir(local_directory)
paths = self.list(urn.path())
expression = "{begin}{end}".format(begin="^", end=remote_directory)
remote_resource_names = prune(paths, expression)
for remote_resource_name in remote_resource_names:
if urn.path().endswith(remote_resource_name):
continue
local_path = os.path.join(local_directory, remote_resource_name)
remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(), resource_name=remote_resource_name)
remote_urn = Urn(remote_path)
if remote_urn.path().endswith("/"):
if not os.path.exists(local_path):
updated = True
os.mkdir(local_path)
result = self.pull(remote_directory=remote_path, local_directory=local_path)
updated = updated or result
else:
if remote_resource_name in local_resource_names and self.is_local_more_recent(local_path, remote_path):
continue
self.download_file(remote_path=remote_path, local_path=local_path)
updated = True
return updated
def upload_directory(self, remote_path, local_path, progress=None):
"""Uploads directory to remote path on WebDAV server.
In case directory is exist on remote server it will delete it and then upload directory with nested files and
directories.
:param remote_path: the path to directory for uploading on WebDAV server.
:param local_path: the path to local directory for uploading.
:param progress: Progress function. Not supported now.
"""
urn = Urn(remote_path, directory=True)
if not urn.is_dir():
raise OptionNotValid(name="remote_path", value=remote_path)
if not os.path.isdir(local_path):
raise OptionNotValid(name="local_path", value=local_path)
if not os.path.exists(local_path):
raise LocalResourceNotFound(local_path)
if self.check(urn.path()):
self.clean(urn.path())
self.mkdir(remote_path)
for resource_name in listdir(local_path):
_remote_path = "{parent}{name}".format(parent=urn.path(),
name=resource_name).replace(
'\\', '')
_local_path = os.path.join(local_path, resource_name)
self.upload(local_path=_local_path,
remote_path=_remote_path,
progress=progress)
def push(self, remote_directory, local_directory):
def prune(src, exp):
return [sub(exp, "", item) for item in src]
urn = Urn(remote_directory, directory=True)
if not self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_directory)
if not os.path.isdir(local_directory):
raise OptionNotValid(name="local_path", value=local_directory)
if not os.path.exists(local_directory):
raise LocalResourceNotFound(local_directory)
paths = self.list(urn.path())
expression = "{begin}{end}".format(begin="^", end=urn.path())
remote_resource_names = prune(paths, expression)
for local_resource_name in listdir(local_directory):
local_path = os.path.join(local_directory, local_resource_name)
remote_path = "{remote_directory}{resource_name}".format(
remote_directory=urn.path(), resource_name=local_resource_name)
if os.path.isdir(local_path):
if not self.check(remote_path=remote_path):
self.mkdir(remote_path=remote_path)
self.push(remote_directory=remote_path,
local_directory=local_path)
else:
if local_resource_name in remote_resource_names:
continue
self.upload_file(remote_path=remote_path,
local_path=local_path)
def download_directory(self, remote_path, local_path, progress=None):
"""Downloads directory and downloads all nested files and directories from remote WebDAV to local.
If there is something on local path it deletes directories and files then creates new.
:param remote_path: the path to directory for downloading form WebDAV server.
:param local_path: the path to local directory for saving downloaded files and directories.
:param progress: Progress function. Not supported now.
"""
urn = Urn(remote_path, directory=True)
if not self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_path)
if os.path.exists(local_path):
shutil.rmtree(local_path)
os.makedirs(local_path)
for resource_name in self.list(urn.path()):
if urn.path().endswith(resource_name):
continue
_remote_path = "{parent}{name}".format(parent=urn.path(),
name=resource_name)
_local_path = os.path.join(local_path, resource_name)
self.download(local_path=_local_path,
remote_path=_remote_path,
progress=progress)
def download_from(self, buff, remote_path):
"""Downloads file from WebDAV and writes it in buffer.
:param buff: buffer object for writing of downloaded file content.
:param remote_path: path to file on WebDAV server.
"""
urn = Urn(remote_path)
if self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_path)
if not self.check(urn.path()):
raise RemoteResourceNotFound(urn.path())
response = self.execute_request(action='download', path=urn.quote())
shutil.copyfileobj(response.raw, buff)
def list(self, remote_path=root):
"""Returns list of nested files and directories for remote WebDAV directory by path.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
:param remote_path: path to remote directory.
:return: list of nested file or directory names.
"""
directory_urn = Urn(remote_path, directory=True)
if directory_urn.path() != Client.root and not self.check(directory_urn.path()):
raise RemoteResourceNotFound(directory_urn.path())
response = self.execute_request(action='list', path=directory_urn.quote())
urns = WebDavXmlUtils.parse_get_list_response(response.content)
path = Urn.normalize_path(self.get_full_path(directory_urn))
return [urn.filename() for urn in urns if Urn.compare_path(path, urn.path()) is False]
def upload_file(self, remote_path, local_path, progress=None):
"""Uploads file to remote path on WebDAV server. File should be 2Gb or less.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PUT
:param remote_path: the path to uploading file on WebDAV server.
:param local_path: the path to local file for uploading.
:param progress: Progress function. Not supported now.
"""
if not os.path.exists(local_path):
raise LocalResourceNotFound(local_path)
urn = Urn(remote_path)
if urn.is_dir():
raise OptionNotValid(name="remote_path", value=remote_path)
if os.path.isdir(local_path):
raise OptionNotValid(name="local_path", value=local_path)
if not self.check(urn.parent()):
raise RemoteParentNotFound(urn.path())
with open(local_path, "rb") as local_file:
self.execute_request(action='upload',
path=urn.quote(),
data=local_file)
def set_property_batch(self, remote_path, option):
"""Sets batch metadata properties of remote resource on WebDAV server in batch.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPPATCH
:param remote_path: the path to remote resource.
:param option: the property attributes as list of dictionaries with following keys:
`namespace`: (optional) the namespace for XML property which will be set,
`name`: the name of property which will be set,
`value`: (optional) the value of property which will be set. Defaults is empty string.
"""
urn = Urn(remote_path)
if not self.check(urn.path()):
raise RemoteResourceNotFound(urn.path())
data = WebDavXmlUtils.create_set_property_batch_request_content(option)
self.execute_request(action='set_property', path=urn.quote(), data=data)
def pull(self, remote_directory, local_directory):
def prune(src, exp):
return [sub(exp, "", item) for item in src]
updated = False
urn = Urn(remote_directory, directory=True)
## here we check for the current directory
if not self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_directory)
if not os.path.exists(local_directory):
raise LocalResourceNotFound(local_directory)
local_resource_names = listdir(local_directory)
## here we get all files and directories in the current directory
paths = self.list(urn.path())
expression = "{begin}{end}".format(begin="^", end=remote_directory)
remote_resource_names = prune(paths, expression)
for remote_resource_name in remote_resource_names:
local_path = os.path.join(local_directory, remote_resource_name)
remote_path = "{remote_directory}{resource_name}".format(
remote_directory=urn.path(),
resource_name=remote_resource_name)
remote_urn = Urn(remote_path)
## if the string ends with an "/" it *must* be a directory
## no need to check it here
if remote_urn.path().endswith("/"):
if not os.path.exists(local_path):
updated = True
os.mkdir(local_path)
## nevertheless, since we pull the directory, it will be checked up there
self.pull(remote_directory=remote_path,
local_directory=local_path)
else:
if remote_resource_name in local_resource_names:
continue
self.download_file(remote_path=remote_path,
local_path=local_path)
updated = True
return updated
def get_property(self, remote_path, option):
"""Gets metadata property of remote resource on WebDAV server.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
:param remote_path: the path to remote resource.
:param option: the property attribute as dictionary with following keys:
`namespace`: (optional) the namespace for XML property which will be set,
`name`: the name of property which will be set.
:return: the value of property or None if property is not found.
"""
urn = Urn(remote_path)
if not self.check(urn.path()):
raise RemoteResourceNotFound(urn.path())
data = WebDavXmlUtils.create_get_property_request_content(option)
response = self.execute_request(action='get_property', path=urn.quote(), data=data)
return WebDavXmlUtils.parse_get_property_response(response.content, option['name'])
def move(self, remote_path_from, remote_path_to, overwrite=False):
"""Moves resource from one place to another on WebDAV server.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_MOVE
:param remote_path_from: the path to resource which will be moved,
:param remote_path_to: the path where resource will be moved.
:param overwrite: (optional) the flag, overwrite file if it exists. Defaults is False
"""
urn_from = Urn(remote_path_from)
if not self.check(urn_from.path()):
raise RemoteResourceNotFound(urn_from.path())
urn_to = Urn(remote_path_to)
if not self.check(urn_to.parent()):
raise RemoteParentNotFound(urn_to.path())
header_destination = "Destination: {path}".format(path=self.get_url(urn_to.quote()))
header_overwrite = "Overwrite: {flag}".format(flag="T" if overwrite else "F")
self.execute_request(action='move', path=urn_from.quote(), headers_ext=[header_destination, header_overwrite])
def copy(self, remote_path_from, remote_path_to, depth=1):
"""Copies resource from one place to another on WebDAV server.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_COPY
:param remote_path_from: the path to resource which will be copied,
:param remote_path_to: the path where resource will be copied.
:param depth: folder depth to copy
"""
urn_from = Urn(remote_path_from)
if not self.check(urn_from.path()):
raise RemoteResourceNotFound(urn_from.path())
urn_to = Urn(remote_path_to)
if not self.check(urn_to.parent()):
raise RemoteParentNotFound(urn_to.path())
header_destination = "Destination: {path}".format(path=self.get_full_path(urn_to))
header_depth = "Depth: {depth}".format(depth=depth)
self.execute_request(action='copy', path=urn_from.quote(), headers_ext=[header_destination, header_depth])
def list(self, remote_path=root, get_info=False):
"""Returns list of nested files and directories for remote WebDAV directory by path.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
:param remote_path: path to remote directory.
:param get_info: path and element info to remote directory, like cmd 'ls -l'.
:return: if get_info=False it returns list of nested file or directory names, otherwise it returns
list of information, the information is a dictionary and it values with following keys:
`created`: date of resource creation,
`name`: name of resource,
`size`: size of resource,
`modified`: date of resource modification,
`etag`: etag of resource,
`content_type`: content type of resource,
`isdir`: type of resource,
`path`: path of resource.
"""
directory_urn = Urn(remote_path, directory=True)
if directory_urn.path() != Client.root and not self.check(
directory_urn.path()):
raise RemoteResourceNotFound(directory_urn.path())
path = Urn.normalize_path(self.get_full_path(directory_urn))
response = self.execute_request(action='list',
path=directory_urn.quote())
if get_info:
subfiles = WebDavXmlUtils.parse_get_list_info_response(
response.content)
return [
subfile for subfile in subfiles
if Urn.compare_path(path, subfile.get('path')) is False
]
urns = WebDavXmlUtils.parse_get_list_response(response.content)
return [
urn.filename() for urn in urns
if Urn.compare_path(path, urn.path()) is False
]
def pull(self, remote_directory, local_directory):
def prune(src, exp):
return [sub(exp, "", item) for item in src]
updated = False
urn = Urn(remote_directory, directory=True)
if not self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_directory)
if not os.path.exists(local_directory):
raise LocalResourceNotFound(local_directory)
local_resource_names = listdir(local_directory)
paths = self.list(urn.path())
expression = "{begin}{end}".format(begin="^", end=remote_directory)
remote_resource_names = prune(paths, expression)
for remote_resource_name in remote_resource_names:
local_path = os.path.join(local_directory, remote_resource_name)
remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(),
resource_name=remote_resource_name)
remote_urn = Urn(remote_path)
if remote_urn.path().endswith("/"):
if not os.path.exists(local_path):
updated = True
os.mkdir(local_path)
self.pull(remote_directory=remote_path, local_directory=local_path)
else:
if remote_resource_name in local_resource_names:
continue
self.download_file(remote_path=remote_path, local_path=local_path)
updated = True
return updated
def download(self, remote_path, local_path, progress=None):
"""Downloads remote resource from WebDAV and save it in local path.
More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
:param remote_path: the path to remote resource for downloading can be file and directory.
:param local_path: the path to save resource locally.
:param progress: progress function. Not supported now.
"""
urn = Urn(remote_path)
if self.is_dir(urn.path()):
self.download_directory(local_path=local_path, remote_path=remote_path, progress=progress)
else:
self.download_file(local_path=local_path, remote_path=remote_path, progress=progress)
def download_file(self, remote_path, local_path, progress=None):
"""Downloads file from WebDAV server and save it locally.
More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
:param remote_path: the path to remote file for downloading.
:param local_path: the path to save file locally.
:param progress: progress function. Not supported now.
"""
urn = Urn(remote_path)
if self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_path)
if os.path.isdir(local_path):
raise OptionNotValid(name="local_path", value=local_path)
if not self.check(urn.path()):
raise RemoteResourceNotFound(urn.path())
with open(local_path, 'wb') as local_file:
response = self.execute_request('download', urn.quote())
for block in response.iter_content(1024):
local_file.write(block)
def mkdir(self, remote_path):
"""Makes new directory on WebDAV server.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_MKCOL
:param remote_path: path to directory
:return: True if request executed with code 200 or 201 and False otherwise.
"""
directory_urn = Urn(remote_path, directory=True)
if not self.check(directory_urn.parent()):
raise RemoteParentNotFound(directory_urn.path())
response = self.execute_request(action='mkdir', path=directory_urn.quote())
return response.status_code in (200, 201)
def list(self, remote_path="/"):
def parse(directory_urn, response):
try:
response_str = response.content
tree = etree.fromstring(response_str)
hrees = [unquote(hree.text) for hree in tree.findall(".//{DAV:}href")]
etags = [unquote(etag.text) for etag in tree.findall(".//{DAV:}getetag")]
i=0
ret = []
for hree in hrees:
if(i == 0):
i = i+1
continue
urn = MyUrn(hree, etags[i], remote_path)
ret.append(urn)
i = i+1
return ret
except etree.XMLSyntaxError:
return list()
try:
directory_urn = Urn(remote_path, directory=True)
if directory_urn.path() != wc.Client.root:
if not self.client.check(directory_urn.path()):
raise RemoteResourceNotFound(directory_urn.path())
response = BytesIO()
response = self.client.execute_request(action='list', path=directory_urn.quote())
urns = parse(directory_urn, response)
return urns
except Exception:
raise NotConnection(self.client.webdav.hostname)
def upload_to(self, buff, remote_path):
"""Uploads file from buffer to remote path on WebDAV server.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PUT
:param buff: the buffer with content for file.
:param remote_path: the path to save file remotely on WebDAV server.
"""
urn = Urn(remote_path)
if urn.is_dir():
raise OptionNotValid(name="remote_path", value=remote_path)
if not self.check(urn.parent()):
raise RemoteParentNotFound(urn.path())
self.execute_request(action='upload', path=urn.quote(), data=buff)
def is_dir(self, remote_path):
"""Checks is the remote resource directory.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
:param remote_path: the path to remote resource.
:return: True in case the remote resource is directory and False otherwise.
"""
urn = Urn(remote_path)
parent_urn = Urn(urn.parent())
if not self.check(urn.path()) and not self.check(Urn(remote_path, directory=True).path()):
raise RemoteResourceNotFound(remote_path)
response = self.execute_request(action='info', path=parent_urn.quote())
path = self.get_full_path(urn)
return WebDavXmlUtils.parse_is_dir_response(content=response.content, path=path, hostname=self.webdav.hostname)
def download_file(self, remote_path,progress=None):
"""Downloads file from WebDAV server and save it locally.
More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
:param remote_path: the path to remote file for downloading.
:param local_path: the path to save file locally.
:param progress: progress function. Not supported now.
"""
urn = Urn(remote_path)
if not self.check(urn.path()):
return 0
try:
response = self.execute_request('download', urn.quote())
return response
except:
return 0
def info(self, remote_path):
"""Gets information about resource on WebDAV.
More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
:param remote_path: the path to remote resource.
:return: a dictionary of information attributes and them values with following keys:
`created`: date of resource creation,
`name`: name of resource,
`size`: size of resource,
`modified`: date of resource modification.
"""
urn = Urn(remote_path)
if not self.check(urn.path()) and not self.check(Urn(remote_path, directory=True).path()):
raise RemoteResourceNotFound(remote_path)
response = self.execute_request(action='info', path=urn.quote())
path = self.get_full_path(urn)
return WebDavXmlUtils.parse_info_response(content=response.content, path=path, hostname=self.webdav.hostname)
def resource(self, remote_path):
urn = Urn(remote_path)
return Resource(self, urn.path())
def move(self, remote_path):
new_urn = Urn(remote_path)
self.client.move(remote_path_from=self.urn.path(),
remote_path_to=new_urn.path())
self.urn = new_urn
class Resource(object):
def __init__(self, client, urn):
self.client = client
self.urn = urn
def __str__(self):
return "resource {path}".format(path=self.urn.path())
def is_dir(self):
return self.client.is_dir(self.urn.path())
def rename(self, new_name):
old_path = self.urn.path()
parent_path = self.urn.parent()
new_name = Urn(new_name).filename()
new_path = "{directory}{filename}".format(directory=parent_path,
filename=new_name)
self.client.move(remote_path_from=old_path, remote_path_to=new_path)
self.urn = Urn(new_path)
def move(self, remote_path):
new_urn = Urn(remote_path)
self.client.move(remote_path_from=self.urn.path(),
remote_path_to=new_urn.path())
self.urn = new_urn
def copy(self, remote_path):
urn = Urn(remote_path)
self.client.copy(remote_path_from=self.urn.path(),
remote_path_to=remote_path)
return Resource(self.client, urn)
def info(self, params=None):
info = self.client.info(self.urn.path())
if not params:
return info
return {key: value for (key, value) in info.items() if key in params}
def clean(self):
return self.client.clean(self.urn.path())
def check(self):
return self.client.check(self.urn.path())
def read_from(self, buff):
self.client.upload_to(buff=buff, remote_path=self.urn.path())
def read(self, local_path):
return self.client.upload_sync(local_path=local_path,
remote_path=self.urn.path())
def read_async(self, local_path, callback=None):
return self.client.upload_async(local_path=local_path,
remote_path=self.urn.path(),
callback=callback)
def write_to(self, buff):
return self.client.download_from(buff=buff,
remote_path=self.urn.path())
def write(self, local_path):
return self.client.download_sync(local_path=local_path,
remote_path=self.urn.path())
def write_async(self, local_path, callback=None):
return self.client.download_async(local_path=local_path,
remote_path=self.urn.path(),
callback=callback)
def publish(self):
return self.client.publish(self.urn.path())
def unpublish(self):
return self.client.unpublish(self.urn.path())
def get_property(self, option):
return self.client.get_property(remote_path=self.urn.path(),
option=option)
def set_property(self, option, value):
option['value'] = value.__str__()
self.client.set_property(remote_path=self.urn.path(), option=option)
def push_force(self, remote_directory, local_directory):
"""
Validate remote folder with local via put method, check bit-bit and replace remote if mismatch
:param remote_directory:
:param local_directory:
:return:
"""
def prune(src, exp):
return [sub(exp, "", item) for item in src]
urn = Urn(remote_directory, directory=True)
if not self.is_dir(urn.path()):
raise OptionNotValid(name="remote_path", value=remote_directory)
if not os.path.isdir(local_directory):
raise OptionNotValid(name="local_path", value=local_directory)
if not os.path.exists(local_directory):
raise LocalResourceNotFound(local_directory)
paths = self.list(urn.path())
paths_local = os.listdir(local_directory)
expression = "{begin}{end}".format(begin="^", end=urn.path())
remote_resource_names = prune(paths, expression)
local_resources = prune(paths_local, expression)
##clean shit
for each in remote_resource_names:
remote_path = "{remote_directory}{resource_name}".format(
remote_directory=urn.path(), resource_name=each)
if each not in local_resources:
print("removing " + remote_path)
self.execute_request("clean", remote_path)
continue
for local_resource_name in listdir(local_directory):
local_path = os.path.join(local_directory, local_resource_name)
remote_path = "{remote_directory}{resource_name}".format(
remote_directory=urn.path(), resource_name=local_resource_name)
if os.path.isdir(local_path):
if not self.check(remote_path=remote_path):
self.mkdir(remote_path=remote_path)
self.push_force(remote_directory=remote_path,
local_directory=local_path)
else:
if local_resource_name in remote_resource_names:
response = self.execute_request("check", remote_path)
if os.path.isdir(local_path):
continue
local_hash = self.getHash(file=local_path)
if (response.headers["ETag"] == local_hash):
continue
print("diff for " + local_resource_name)
self.upload_file(remote_path=remote_path,
local_path=local_path)
print("uploading " + local_path)
self.upload_file(remote_path=remote_path,
local_path=local_path)
