def push(self, remote_directory, local_directory): def prune(src, exp): return [sub(exp, "", item) for item in src] updated = False urn = Urn(remote_directory, directory=True) self._validate_remote_directory(urn) self._validate_local_directory(local_directory) paths = self.list(urn.path()) expression = "{begin}{end}".format(begin="^", end=urn.path()) remote_resource_names = prune(paths, expression) for local_resource_name in listdir(local_directory): local_path = os.path.join(local_directory, local_resource_name) remote_path = "{remote_directory}{resource_name}".format( remote_directory=urn.path(), resource_name=local_resource_name) if os.path.isdir(local_path): if not self.check(remote_path=remote_path): self.mkdir(remote_path=remote_path) result = self.push(remote_directory=remote_path, local_directory=local_path) updated = updated or result else: if local_resource_name in remote_resource_names and not self.is_local_more_recent( local_path, remote_path): continue self.upload_file(remote_path=remote_path, local_path=local_path) updated = True return updated
def pull(self, remote_directory, local_directory): def prune(src, exp): return [sub(exp, "", item) for item in src] updated = False urn = Urn(remote_directory, directory=True) self._validate_remote_directory(urn) self._validate_local_directory(local_directory) local_resource_names = listdir(local_directory) paths = self.list(urn.path()) expression = "{begin}{end}".format(begin="^", end=remote_directory) remote_resource_names = prune(paths, expression) for remote_resource_name in remote_resource_names: if urn.path().endswith(remote_resource_name): continue local_path = os.path.join(local_directory, remote_resource_name) remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(), resource_name=remote_resource_name) remote_urn = Urn(remote_path) if remote_urn.path().endswith("/"): if not os.path.exists(local_path): updated = True os.mkdir(local_path) result = self.pull(remote_directory=remote_path, local_directory=local_path) updated = updated or result else: if remote_resource_name in local_resource_names and self.is_local_more_recent(local_path, remote_path): continue self.download_file(remote_path=remote_path, local_path=local_path) updated = True return updated
def upload_directory(self, remote_path, local_path, progress=None): """Uploads directory to remote path on WebDAV server. In case directory is exist on remote server it will delete it and then upload directory with nested files and directories. :param remote_path: the path to directory for uploading on WebDAV server. :param local_path: the path to local directory for uploading. :param progress: Progress function. Not supported now. """ urn = Urn(remote_path, directory=True) if not urn.is_dir(): raise OptionNotValid(name="remote_path", value=remote_path) if not os.path.isdir(local_path): raise OptionNotValid(name="local_path", value=local_path) if not os.path.exists(local_path): raise LocalResourceNotFound(local_path) if self.check(urn.path()): self.clean(urn.path()) self.mkdir(remote_path) for resource_name in listdir(local_path): _remote_path = "{parent}{name}".format(parent=urn.path(), name=resource_name).replace( '\\', '') _local_path = os.path.join(local_path, resource_name) self.upload(local_path=_local_path, remote_path=_remote_path, progress=progress)
def push(self, remote_directory, local_directory): def prune(src, exp): return [sub(exp, "", item) for item in src] urn = Urn(remote_directory, directory=True) if not self.is_dir(urn.path()): raise OptionNotValid(name="remote_path", value=remote_directory) if not os.path.isdir(local_directory): raise OptionNotValid(name="local_path", value=local_directory) if not os.path.exists(local_directory): raise LocalResourceNotFound(local_directory) paths = self.list(urn.path()) expression = "{begin}{end}".format(begin="^", end=urn.path()) remote_resource_names = prune(paths, expression) for local_resource_name in listdir(local_directory): local_path = os.path.join(local_directory, local_resource_name) remote_path = "{remote_directory}{resource_name}".format( remote_directory=urn.path(), resource_name=local_resource_name) if os.path.isdir(local_path): if not self.check(remote_path=remote_path): self.mkdir(remote_path=remote_path) self.push(remote_directory=remote_path, local_directory=local_path) else: if local_resource_name in remote_resource_names: continue self.upload_file(remote_path=remote_path, local_path=local_path)
def download_directory(self, remote_path, local_path, progress=None): """Downloads directory and downloads all nested files and directories from remote WebDAV to local. If there is something on local path it deletes directories and files then creates new. :param remote_path: the path to directory for downloading form WebDAV server. :param local_path: the path to local directory for saving downloaded files and directories. :param progress: Progress function. Not supported now. """ urn = Urn(remote_path, directory=True) if not self.is_dir(urn.path()): raise OptionNotValid(name="remote_path", value=remote_path) if os.path.exists(local_path): shutil.rmtree(local_path) os.makedirs(local_path) for resource_name in self.list(urn.path()): if urn.path().endswith(resource_name): continue _remote_path = "{parent}{name}".format(parent=urn.path(), name=resource_name) _local_path = os.path.join(local_path, resource_name) self.download(local_path=_local_path, remote_path=_remote_path, progress=progress)
def download_from(self, buff, remote_path): """Downloads file from WebDAV and writes it in buffer. :param buff: buffer object for writing of downloaded file content. :param remote_path: path to file on WebDAV server. """ urn = Urn(remote_path) if self.is_dir(urn.path()): raise OptionNotValid(name="remote_path", value=remote_path) if not self.check(urn.path()): raise RemoteResourceNotFound(urn.path()) response = self.execute_request(action='download', path=urn.quote()) shutil.copyfileobj(response.raw, buff)
def list(self, remote_path=root): """Returns list of nested files and directories for remote WebDAV directory by path. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND :param remote_path: path to remote directory. :return: list of nested file or directory names. """ directory_urn = Urn(remote_path, directory=True) if directory_urn.path() != Client.root and not self.check(directory_urn.path()): raise RemoteResourceNotFound(directory_urn.path()) response = self.execute_request(action='list', path=directory_urn.quote()) urns = WebDavXmlUtils.parse_get_list_response(response.content) path = Urn.normalize_path(self.get_full_path(directory_urn)) return [urn.filename() for urn in urns if Urn.compare_path(path, urn.path()) is False]
def upload_file(self, remote_path, local_path, progress=None): """Uploads file to remote path on WebDAV server. File should be 2Gb or less. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PUT :param remote_path: the path to uploading file on WebDAV server. :param local_path: the path to local file for uploading. :param progress: Progress function. Not supported now. """ if not os.path.exists(local_path): raise LocalResourceNotFound(local_path) urn = Urn(remote_path) if urn.is_dir(): raise OptionNotValid(name="remote_path", value=remote_path) if os.path.isdir(local_path): raise OptionNotValid(name="local_path", value=local_path) if not self.check(urn.parent()): raise RemoteParentNotFound(urn.path()) with open(local_path, "rb") as local_file: self.execute_request(action='upload', path=urn.quote(), data=local_file)
def set_property_batch(self, remote_path, option): """Sets batch metadata properties of remote resource on WebDAV server in batch. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPPATCH :param remote_path: the path to remote resource. :param option: the property attributes as list of dictionaries with following keys: `namespace`: (optional) the namespace for XML property which will be set, `name`: the name of property which will be set, `value`: (optional) the value of property which will be set. Defaults is empty string. """ urn = Urn(remote_path) if not self.check(urn.path()): raise RemoteResourceNotFound(urn.path()) data = WebDavXmlUtils.create_set_property_batch_request_content(option) self.execute_request(action='set_property', path=urn.quote(), data=data)
def pull(self, remote_directory, local_directory): def prune(src, exp): return [sub(exp, "", item) for item in src] updated = False urn = Urn(remote_directory, directory=True) ## here we check for the current directory if not self.is_dir(urn.path()): raise OptionNotValid(name="remote_path", value=remote_directory) if not os.path.exists(local_directory): raise LocalResourceNotFound(local_directory) local_resource_names = listdir(local_directory) ## here we get all files and directories in the current directory paths = self.list(urn.path()) expression = "{begin}{end}".format(begin="^", end=remote_directory) remote_resource_names = prune(paths, expression) for remote_resource_name in remote_resource_names: local_path = os.path.join(local_directory, remote_resource_name) remote_path = "{remote_directory}{resource_name}".format( remote_directory=urn.path(), resource_name=remote_resource_name) remote_urn = Urn(remote_path) ## if the string ends with an "/" it *must* be a directory ## no need to check it here if remote_urn.path().endswith("/"): if not os.path.exists(local_path): updated = True os.mkdir(local_path) ## nevertheless, since we pull the directory, it will be checked up there self.pull(remote_directory=remote_path, local_directory=local_path) else: if remote_resource_name in local_resource_names: continue self.download_file(remote_path=remote_path, local_path=local_path) updated = True return updated
def get_property(self, remote_path, option): """Gets metadata property of remote resource on WebDAV server. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND :param remote_path: the path to remote resource. :param option: the property attribute as dictionary with following keys: `namespace`: (optional) the namespace for XML property which will be set, `name`: the name of property which will be set. :return: the value of property or None if property is not found. """ urn = Urn(remote_path) if not self.check(urn.path()): raise RemoteResourceNotFound(urn.path()) data = WebDavXmlUtils.create_get_property_request_content(option) response = self.execute_request(action='get_property', path=urn.quote(), data=data) return WebDavXmlUtils.parse_get_property_response(response.content, option['name'])
def move(self, remote_path_from, remote_path_to, overwrite=False): """Moves resource from one place to another on WebDAV server. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_MOVE :param remote_path_from: the path to resource which will be moved, :param remote_path_to: the path where resource will be moved. :param overwrite: (optional) the flag, overwrite file if it exists. Defaults is False """ urn_from = Urn(remote_path_from) if not self.check(urn_from.path()): raise RemoteResourceNotFound(urn_from.path()) urn_to = Urn(remote_path_to) if not self.check(urn_to.parent()): raise RemoteParentNotFound(urn_to.path()) header_destination = "Destination: {path}".format(path=self.get_url(urn_to.quote())) header_overwrite = "Overwrite: {flag}".format(flag="T" if overwrite else "F") self.execute_request(action='move', path=urn_from.quote(), headers_ext=[header_destination, header_overwrite])
def copy(self, remote_path_from, remote_path_to, depth=1): """Copies resource from one place to another on WebDAV server. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_COPY :param remote_path_from: the path to resource which will be copied, :param remote_path_to: the path where resource will be copied. :param depth: folder depth to copy """ urn_from = Urn(remote_path_from) if not self.check(urn_from.path()): raise RemoteResourceNotFound(urn_from.path()) urn_to = Urn(remote_path_to) if not self.check(urn_to.parent()): raise RemoteParentNotFound(urn_to.path()) header_destination = "Destination: {path}".format(path=self.get_full_path(urn_to)) header_depth = "Depth: {depth}".format(depth=depth) self.execute_request(action='copy', path=urn_from.quote(), headers_ext=[header_destination, header_depth])
def list(self, remote_path=root, get_info=False): """Returns list of nested files and directories for remote WebDAV directory by path. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND :param remote_path: path to remote directory. :param get_info: path and element info to remote directory, like cmd 'ls -l'. :return: if get_info=False it returns list of nested file or directory names, otherwise it returns list of information, the information is a dictionary and it values with following keys: `created`: date of resource creation, `name`: name of resource, `size`: size of resource, `modified`: date of resource modification, `etag`: etag of resource, `content_type`: content type of resource, `isdir`: type of resource, `path`: path of resource. """ directory_urn = Urn(remote_path, directory=True) if directory_urn.path() != Client.root and not self.check( directory_urn.path()): raise RemoteResourceNotFound(directory_urn.path()) path = Urn.normalize_path(self.get_full_path(directory_urn)) response = self.execute_request(action='list', path=directory_urn.quote()) if get_info: subfiles = WebDavXmlUtils.parse_get_list_info_response( response.content) return [ subfile for subfile in subfiles if Urn.compare_path(path, subfile.get('path')) is False ] urns = WebDavXmlUtils.parse_get_list_response(response.content) return [ urn.filename() for urn in urns if Urn.compare_path(path, urn.path()) is False ]
def pull(self, remote_directory, local_directory): def prune(src, exp): return [sub(exp, "", item) for item in src] updated = False urn = Urn(remote_directory, directory=True) if not self.is_dir(urn.path()): raise OptionNotValid(name="remote_path", value=remote_directory) if not os.path.exists(local_directory): raise LocalResourceNotFound(local_directory) local_resource_names = listdir(local_directory) paths = self.list(urn.path()) expression = "{begin}{end}".format(begin="^", end=remote_directory) remote_resource_names = prune(paths, expression) for remote_resource_name in remote_resource_names: local_path = os.path.join(local_directory, remote_resource_name) remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(), resource_name=remote_resource_name) remote_urn = Urn(remote_path) if remote_urn.path().endswith("/"): if not os.path.exists(local_path): updated = True os.mkdir(local_path) self.pull(remote_directory=remote_path, local_directory=local_path) else: if remote_resource_name in local_resource_names: continue self.download_file(remote_path=remote_path, local_path=local_path) updated = True return updated
def download(self, remote_path, local_path, progress=None): """Downloads remote resource from WebDAV and save it in local path. More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4 :param remote_path: the path to remote resource for downloading can be file and directory. :param local_path: the path to save resource locally. :param progress: progress function. Not supported now. """ urn = Urn(remote_path) if self.is_dir(urn.path()): self.download_directory(local_path=local_path, remote_path=remote_path, progress=progress) else: self.download_file(local_path=local_path, remote_path=remote_path, progress=progress)
def download_file(self, remote_path, local_path, progress=None): """Downloads file from WebDAV server and save it locally. More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4 :param remote_path: the path to remote file for downloading. :param local_path: the path to save file locally. :param progress: progress function. Not supported now. """ urn = Urn(remote_path) if self.is_dir(urn.path()): raise OptionNotValid(name="remote_path", value=remote_path) if os.path.isdir(local_path): raise OptionNotValid(name="local_path", value=local_path) if not self.check(urn.path()): raise RemoteResourceNotFound(urn.path()) with open(local_path, 'wb') as local_file: response = self.execute_request('download', urn.quote()) for block in response.iter_content(1024): local_file.write(block)
def mkdir(self, remote_path): """Makes new directory on WebDAV server. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_MKCOL :param remote_path: path to directory :return: True if request executed with code 200 or 201 and False otherwise. """ directory_urn = Urn(remote_path, directory=True) if not self.check(directory_urn.parent()): raise RemoteParentNotFound(directory_urn.path()) response = self.execute_request(action='mkdir', path=directory_urn.quote()) return response.status_code in (200, 201)
def list(self, remote_path="/"): def parse(directory_urn, response): try: response_str = response.content tree = etree.fromstring(response_str) hrees = [unquote(hree.text) for hree in tree.findall(".//{DAV:}href")] etags = [unquote(etag.text) for etag in tree.findall(".//{DAV:}getetag")] i=0 ret = [] for hree in hrees: if(i == 0): i = i+1 continue urn = MyUrn(hree, etags[i], remote_path) ret.append(urn) i = i+1 return ret except etree.XMLSyntaxError: return list() try: directory_urn = Urn(remote_path, directory=True) if directory_urn.path() != wc.Client.root: if not self.client.check(directory_urn.path()): raise RemoteResourceNotFound(directory_urn.path()) response = BytesIO() response = self.client.execute_request(action='list', path=directory_urn.quote()) urns = parse(directory_urn, response) return urns except Exception: raise NotConnection(self.client.webdav.hostname)
def upload_to(self, buff, remote_path): """Uploads file from buffer to remote path on WebDAV server. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PUT :param buff: the buffer with content for file. :param remote_path: the path to save file remotely on WebDAV server. """ urn = Urn(remote_path) if urn.is_dir(): raise OptionNotValid(name="remote_path", value=remote_path) if not self.check(urn.parent()): raise RemoteParentNotFound(urn.path()) self.execute_request(action='upload', path=urn.quote(), data=buff)
def is_dir(self, remote_path): """Checks is the remote resource directory. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND :param remote_path: the path to remote resource. :return: True in case the remote resource is directory and False otherwise. """ urn = Urn(remote_path) parent_urn = Urn(urn.parent()) if not self.check(urn.path()) and not self.check(Urn(remote_path, directory=True).path()): raise RemoteResourceNotFound(remote_path) response = self.execute_request(action='info', path=parent_urn.quote()) path = self.get_full_path(urn) return WebDavXmlUtils.parse_is_dir_response(content=response.content, path=path, hostname=self.webdav.hostname)
def download_file(self, remote_path,progress=None): """Downloads file from WebDAV server and save it locally. More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4 :param remote_path: the path to remote file for downloading. :param local_path: the path to save file locally. :param progress: progress function. Not supported now. """ urn = Urn(remote_path) if not self.check(urn.path()): return 0 try: response = self.execute_request('download', urn.quote()) return response except: return 0
def info(self, remote_path): """Gets information about resource on WebDAV. More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND :param remote_path: the path to remote resource. :return: a dictionary of information attributes and them values with following keys: `created`: date of resource creation, `name`: name of resource, `size`: size of resource, `modified`: date of resource modification. """ urn = Urn(remote_path) if not self.check(urn.path()) and not self.check(Urn(remote_path, directory=True).path()): raise RemoteResourceNotFound(remote_path) response = self.execute_request(action='info', path=urn.quote()) path = self.get_full_path(urn) return WebDavXmlUtils.parse_info_response(content=response.content, path=path, hostname=self.webdav.hostname)
def resource(self, remote_path): urn = Urn(remote_path) return Resource(self, urn.path())
def move(self, remote_path): new_urn = Urn(remote_path) self.client.move(remote_path_from=self.urn.path(), remote_path_to=new_urn.path()) self.urn = new_urn
class Resource(object): def __init__(self, client, urn): self.client = client self.urn = urn def __str__(self): return "resource {path}".format(path=self.urn.path()) def is_dir(self): return self.client.is_dir(self.urn.path()) def rename(self, new_name): old_path = self.urn.path() parent_path = self.urn.parent() new_name = Urn(new_name).filename() new_path = "{directory}{filename}".format(directory=parent_path, filename=new_name) self.client.move(remote_path_from=old_path, remote_path_to=new_path) self.urn = Urn(new_path) def move(self, remote_path): new_urn = Urn(remote_path) self.client.move(remote_path_from=self.urn.path(), remote_path_to=new_urn.path()) self.urn = new_urn def copy(self, remote_path): urn = Urn(remote_path) self.client.copy(remote_path_from=self.urn.path(), remote_path_to=remote_path) return Resource(self.client, urn) def info(self, params=None): info = self.client.info(self.urn.path()) if not params: return info return {key: value for (key, value) in info.items() if key in params} def clean(self): return self.client.clean(self.urn.path()) def check(self): return self.client.check(self.urn.path()) def read_from(self, buff): self.client.upload_to(buff=buff, remote_path=self.urn.path()) def read(self, local_path): return self.client.upload_sync(local_path=local_path, remote_path=self.urn.path()) def read_async(self, local_path, callback=None): return self.client.upload_async(local_path=local_path, remote_path=self.urn.path(), callback=callback) def write_to(self, buff): return self.client.download_from(buff=buff, remote_path=self.urn.path()) def write(self, local_path): return self.client.download_sync(local_path=local_path, remote_path=self.urn.path()) def write_async(self, local_path, callback=None): return self.client.download_async(local_path=local_path, remote_path=self.urn.path(), callback=callback) def publish(self): return self.client.publish(self.urn.path()) def unpublish(self): return self.client.unpublish(self.urn.path()) def get_property(self, option): return self.client.get_property(remote_path=self.urn.path(), option=option) def set_property(self, option, value): option['value'] = value.__str__() self.client.set_property(remote_path=self.urn.path(), option=option)
def push_force(self, remote_directory, local_directory): """ Validate remote folder with local via put method, check bit-bit and replace remote if mismatch :param remote_directory: :param local_directory: :return: """ def prune(src, exp): return [sub(exp, "", item) for item in src] urn = Urn(remote_directory, directory=True) if not self.is_dir(urn.path()): raise OptionNotValid(name="remote_path", value=remote_directory) if not os.path.isdir(local_directory): raise OptionNotValid(name="local_path", value=local_directory) if not os.path.exists(local_directory): raise LocalResourceNotFound(local_directory) paths = self.list(urn.path()) paths_local = os.listdir(local_directory) expression = "{begin}{end}".format(begin="^", end=urn.path()) remote_resource_names = prune(paths, expression) local_resources = prune(paths_local, expression) ##clean shit for each in remote_resource_names: remote_path = "{remote_directory}{resource_name}".format( remote_directory=urn.path(), resource_name=each) if each not in local_resources: print("removing " + remote_path) self.execute_request("clean", remote_path) continue for local_resource_name in listdir(local_directory): local_path = os.path.join(local_directory, local_resource_name) remote_path = "{remote_directory}{resource_name}".format( remote_directory=urn.path(), resource_name=local_resource_name) if os.path.isdir(local_path): if not self.check(remote_path=remote_path): self.mkdir(remote_path=remote_path) self.push_force(remote_directory=remote_path, local_directory=local_path) else: if local_resource_name in remote_resource_names: response = self.execute_request("check", remote_path) if os.path.isdir(local_path): continue local_hash = self.getHash(file=local_path) if (response.headers["ETag"] == local_hash): continue print("diff for " + local_resource_name) self.upload_file(remote_path=remote_path, local_path=local_path) print("uploading " + local_path) self.upload_file(remote_path=remote_path, local_path=local_path)