def info(self, remote_path):
def parse(response, path):
try:
response_str = response.getvalue()
tree = etree.fromstring(response_str)
find_attributes = {
'created': ".//{DAV:}creationdate",
'name': ".//{DAV:}displayname",
'size': ".//{DAV:}getcontentlength",
'modified': ".//{DAV:}getlastmodified"
}
resps = tree.findall("{DAV:}response")
for resp in resps:
href = resp.findtext("{DAV:}href")
urn = unquote(href)
if path[-1] == Urn.separate:
if not path == urn:
continue
else:
path_with_sep = "{path}{sep}".format(path=path,
sep=Urn.separate)
if not path == urn and not path_with_sep == urn:
continue
info = dict()
for (name, value) in find_attributes.items():
info[name] = resp.findtext(value)
return info
raise RemoteResourceNotFound(path)
except etree.XMLSyntaxError:
raise MethodNotSupported(name="info",
server=self.webdav.hostname)
try:
urn = Urn(remote_path)
response = BytesIO()
if not self.check(urn.path()) and not self.check(
Urn(remote_path, directory=True).path()):
raise RemoteResourceNotFound(remote_path)
url = {
'hostname': self.webdav.hostname,
'root': self.webdav.root,
'path': urn.quote()
}
options = {
'URL': "{hostname}{root}{path}".format(**url),
'CUSTOMREQUEST': Client.requests['info'],
'HTTPHEADER': self.get_header('info'),
'WRITEDATA': response,
'NOBODY': 0
}
request = self.Request(options=options)
request.perform()
request.close()
path = "{root}{path}".format(root=self.webdav.root,
path=urn.path())
return parse(response, path)
except pycurl.error:
raise NotConnection(self.webdav.hostname)
def is_dir(self, remote_path):
def parse(response, path):
try:
response_str = response.getvalue()
tree = etree.fromstring(response_str)
resps = tree.findall("{DAV:}response")
for resp in resps:
href = resp.findtext("{DAV:}href")
urn = unquote(href)
if path[-1] == Urn.separate:
if not path == urn:
continue
else:
path_with_sep = "{path}{sep}".format(path=path,
sep=Urn.separate)
if not path == urn and not path_with_sep == urn:
continue
type = resp.find(".//{DAV:}resourcetype")
if type is None:
raise MethodNotSupported(name="is_dir",
server=self.webdav.hostname)
dir_type = type.find("{DAV:}collection")
return True if dir_type is not None else False
raise RemoteResourceNotFound(path)
except etree.XMLSyntaxError:
raise MethodNotSupported(name="is_dir",
server=self.webdav.hostname)
try:
urn = Urn(remote_path)
parent_urn = Urn(urn.parent())
if not self.check(urn.path()) and not self.check(
Urn(remote_path, directory=True).path()):
raise RemoteResourceNotFound(remote_path)
response = BytesIO()
url = {
'hostname': self.webdav.hostname,
'root': self.webdav.root,
'path': parent_urn.quote()
}
options = {
'URL': "{hostname}{root}{path}".format(**url),
'CUSTOMREQUEST': Client.requests['info'],
'HTTPHEADER': self.get_header('info'),
'WRITEDATA': response,
'NOBODY': 0
}
request = self.Request(options=options)
request.perform()
request.close()
path = "{root}{path}".format(root=self.webdav.root,
path=urn.path())
return parse(response, path)
except pycurl.error:
raise NotConnection(self.webdav.hostname)
def copy(self, remote_path):
urn = Urn(remote_path)
self.client.copy(remote_path_from=self.urn.path(),
remote_path_to=remote_path)
return Resource(self.client, urn)
def publish(self, remote_path):
def parse(response):
try:
response_str = response.getvalue()
tree = etree.fromstring(response_str)
result = tree.xpath("//*[local-name() = 'public_url']")
public_url = result[0]
return public_url.text
except IndexError:
raise MethodNotSupported(name="publish",
server=self.webdav.hostname)
except etree.XMLSyntaxError:
return ""
def data(for_server):
root_node = etree.Element("propertyupdate", xmlns="DAV:")
set_node = etree.SubElement(root_node, "set")
prop_node = etree.SubElement(set_node, "prop")
xmlns = Client.meta_xmlns.get(for_server, "")
public_url = etree.SubElement(prop_node, "public_url", xmlns=xmlns)
public_url.text = "true"
tree = etree.ElementTree(root_node)
buff = BytesIO()
tree.write(buff)
return buff.getvalue()
try:
urn = Urn(remote_path)
if not self.check(urn.path()):
raise RemoteResourceNotFound(urn.path())
response = BytesIO()
url = {
'hostname': self.webdav.hostname,
'root': self.webdav.root,
'path': urn.quote()
}
options = {
'URL': "{hostname}{root}{path}".format(**url),
'CUSTOMREQUEST': Client.requests['publish'],
'HTTPHEADER': self.get_header('publish'),
'POSTFIELDS': data(for_server=self.webdav.hostname),
'WRITEDATA': response,
'NOBODY': 0
}
request = self.Request(options=options)
request.perform()
request.close()
return parse(response)
except pycurl.error:
raise NotConnection(self.webdav.hostname)
def move(self, remote_path):
new_urn = Urn(remote_path)
self.client.move(remote_path_from=self.urn.path(),
remote_path_to=new_urn.path())
self.urn = new_urn
def get_list(self, rootdir='/'):
with self._internal_lock2:
self.is_remote_available.wait()
def recursive_files(catalog, get_list, is_dir=None):
def slash_is_dir(path):
return path.endswith('/')
if is_dir is None:
is_dir = slash_is_dir
def recursive(catalog, get_list, is_dir, files, dirs=[]):
for item in get_list(catalog):
item = os.path.join(catalog, item)
if not is_dir(item): files.append(item)
else:
dirs.append(item)
recursive(item, get_list, is_dir, files, dirs)
files = []
recursive(catalog, get_list, is_dir, files)
return files
def list(remote_path='/'):
def parse(response):
try:
response_str = response.getvalue()
tree = etree.fromstring(response_str)
hrees = [unquote(hree.text) for hree in tree.findall(".//{DAV:}href")]
return [Urn(hree) for hree in hrees]
except etree.XMLSyntaxError:
return list()
try:
directory_urn = Urn(remote_path, directory=True)
# if directory_urn.path() != self._wc.root:
# if not self._wc.check(directory_urn.path()):
# raise RemoteResourceNotFound(directory_urn.path())
response = BytesIO()
url = {'hostname': self._wc.webdav.hostname, 'root': self._wc.webdav.root, 'path': directory_urn.quote()}
options = {
'URL': "{hostname}{root}{path}".format(**url),
'CUSTOMREQUEST': self._wc.requests['list'],
'HTTPHEADER': self._wc.get_header('list'),
'WRITEDATA': response,
'NOBODY': 0
}
request = self._wc.Request(options=options)
request.perform()
request.close()
urns = parse(response)
path = "{root}{path}".format(root=self._wc.webdav.root, path=directory_urn.path())
return [urn.filename() for urn in urns if urn.path() != path and urn.path() != path[:-1]]
except pycurl.error:
raise NotConnection(self._wc.webdav.hostname)
start_time = time.clock()
print(start_time)
# files = recursive_files(rootdir, self._wc.list)
files = recursive_files(rootdir, list)
my_list = []
for path in files:
while 1:
# info = self._wc.info(path)
info = None
try:
urn = Urn(path)
response = BytesIO()
# Disable of exist check
#
# if not self._wc.check(urn.path()) and not self._wc.check(Urn(path, directory=True).path()):
# raise RemoteResourceNotFound(path)
url = {'hostname': self._wc.webdav.hostname, 'root': self._wc.webdav.root, 'path': urn.quote()}
options = {
'URL': "{hostname}{root}{path}".format(**url),
'CUSTOMREQUEST': self._wc.requests['info'],
'HTTPHEADER': self._wc.get_header('info'),
'WRITEDATA': response,
'NOBODY': 0
}
request = self._wc.Request(options=options)
while 1:
try:
# time.sleep(0.3)
request.perform()
except pycurl.error as ex:
# raise NotConnection(self._wc.webdav.hostname)
self.logger.info(self._prefix + " pycurl.error returned: " + str(ex))
# time.sleep(self._webdav_timeout)
continue
# http://pycurl.io/docs/dev/curlobject.html
http_code = request.getinfo_raw(pycurl.HTTP_CODE)
if http_code != 207:
self.logger.debug(self._prefix + "Info request returned http-code: " + str(http_code))
time.sleep(self._webdav_timeout)
else:
request.close()
break
remote_path = "{root}{path}".format(root=self._wc.webdav.root, path=urn.path())
try:
response_str = response.getvalue()
# print(response_str)
tree = etree.fromstring(response_str)
find_attributes = {
'created': ".//{DAV:}creationdate",
'name': ".//{DAV:}displayname",
'size': ".//{DAV:}getcontentlength",
'modified': ".//{DAV:}getlastmodified"
}
resps = tree.findall("{DAV:}response")
for resp in resps:
href = resp.findtext("{DAV:}href")
urn = unquote(href)
if path[-1] == Urn.separate:
if not remote_path == urn:
continue
else:
path_with_sep = "{path}{sep}".format(path=path, sep=Urn.separate)
if not remote_path == urn and not path_with_sep == urn:
continue
my_info = dict()
for (name, value) in find_attributes.items():
my_info[name] = resp.findtext(value)
info = my_info
break
# raise RemoteResourceNotFound(remote_path)
except etree.XMLSyntaxError:
raise MethodNotSupported(name="info", server=self._wc.webdav.hostname)
size = int(info["size"])
created = info["created"]
modified = info["modified"]
my_list.append({"path": path, "size": size, "hash": "", "created": created, "modified": modified})
break
except Exception as ex:
exc = str(ex)
if exc.startswith("Remote resource: ") and exc.endswith(" not found"):
raise Exception(self._prefix + "Can't get info of file " + str(path) + " on webdav server: " + exc)
# elif exc.startswith("Method info not supported for https://webdav.yandex.ru"):
# self.logger.debug(self._prefix + "Can't get 2 info of file " + str(path) + " on webdav server: " + exc)
# time.sleep(self._webdav_timeout)
else:
raise Exception(self._prefix + "Can't get info of file " + str(path) + " on webdav server: " + exc)
end_time = time.clock()
self.logger.debug(self._prefix + "Time for get list of {:g} elements from server: {:g} s".format(len(files), end_time - start_time))
return my_list
def resource(self, remote_path):
urn = Urn(remote_path)
return Resource(self, urn.path())
def upload_file(self,
remote_path,
local_path,
progress=None,
matrix_params=None,
query_params=None):
try:
if not os.path.exists(local_path):
raise LocalResourceNotFound(local_path)
urn = Urn(remote_path,
matrix_params=matrix_params,
query_params=query_params)
if urn.is_dir():
raise OptionNotValid(name="remote_path", value=remote_path)
if os.path.isdir(local_path):
raise OptionNotValid(name="local_path", value=local_path)
if not os.path.exists(local_path):
raise LocalResourceNotFound(local_path)
if not self.check(urn.parent()):
raise RemoteParentNotFound(urn.path())
with open(local_path, "rb") as local_file:
url = {
'hostname': self.webdav.hostname,
'root': self.webdav.root,
'path': urn.quote()
}
options = {
'URL': "{hostname}{root}{path}".format(**url),
'HTTPHEADER': self.get_header('upload_file'),
'UPLOAD': 1,
'READFUNCTION': local_file.read,
'NOPROGRESS': 0 if progress else 1
}
if progress:
options["PROGRESSFUNCTION"] = progress
file_size = os.path.getsize(local_path)
if file_size > self.large_size:
options['INFILESIZE_LARGE'] = file_size
else:
options['INFILESIZE'] = file_size
request = self.Request(options=options)
request.perform()
code = request.getinfo(pycurl.HTTP_CODE)
if code == "507":
raise NotEnoughSpace()
request.close()
except pycurl.error:
raise NotConnection(self.webdav.hostname)
def check(self, remote_path=root, matrix_params=None, query_params=None):
def parse(response, path):
try:
response_str = response.getvalue()
tree = etree.fromstring(response_str)
resps = tree.findall("{DAV:}response")
for resp in resps:
href = resp.findtext("{DAV:}href")
urn = unquote(href)
if path.endswith(Urn.separate):
if path == urn or path[:-1] == urn:
return True
else:
if path == urn:
return True
return False
except etree.XMLSyntaxError:
raise MethodNotSupported(name="check",
server=self.webdav.hostname)
try:
urn = Urn(remote_path,
matrix_params=matrix_params,
query_params=query_params)
matrix_params = matrix_params or {}
parent_matrix_params = matrix_params.copy()
parent_matrix_params.pop(remote_path, None)
parent_urn = Urn(urn.parent(),
matrix_params=parent_matrix_params,
query_params=query_params)
response = BytesIO()
url = {
'hostname': self.webdav.hostname,
'root': self.webdav.root,
'path': parent_urn.quote()
}
options = {
'URL': "{hostname}{root}{path}".format(**url),
'CUSTOMREQUEST': Client.requests['info'],
'HTTPHEADER': self.get_header('info'),
'WRITEDATA': response,
'NOBODY': 0
}
request = self.Request(options=options)
request.perform()
request.close()
path = "{root}{path}".format(root=self.webdav.root,
path=urn.path())
return parse(response, path)
except pycurl.error:
raise NotConnection(self.webdav.hostname)
