def get_display_name(self):
doc = self.doc
if doc.get("_title"):
return doc["_title"].encode("utf8")
elif doc.get("title"):
return doc["title"].encode("utf8")
elif doc.get("_id"):
return compat.to_native(doc["_id"])
return compat.to_native(doc["key"])
def __init__(self, path, environ, file_path):
super(FileResource, self).__init__(path, environ)
self._file_path = file_path
self.file_stat = os.stat(self._file_path)
# Setting the name from the file path should fix the case on Windows
self.name = os.path.basename(self._file_path)
self.name = compat.to_native(self.name)
def get_member_names(self):
"""Return list of direct collection member names (utf-8 encoded).
See DAVCollection.get_member_names()
"""
# On Windows NT/2k/XP and Unix, if path is a Unicode object, the result
# will be a list of Unicode objects.
# Undecodable filenames will still be returned as string objects
# If we don't request unicode, for example Vista may return a '?'
# instead of a special character. The name would then be unusable to
# build a distinct URL that references this resource.
nameList = []
# self._file_path is unicode, so os.listdir returns unicode as well
assert compat.is_unicode(self._file_path)
for name in os.listdir(self._file_path):
if not compat.is_unicode(name):
name = name.decode(sys.getfilesystemencoding())
assert compat.is_unicode(name)
# Skip non files (links and mount points)
fp = os.path.join(self._file_path, name)
if not os.path.isdir(fp) and not os.path.isfile(fp):
_logger.debug("Skipping non-file {!r}".format(fp))
continue
# name = name.encode("utf8")
name = compat.to_native(name)
nameList.append(name)
return nameList
def get_member_names(self):
assert self.is_collection
cache = self.environ["wsgidav.hg.cache"][compat.to_native(self.rev)]
dirinfos = cache["dirinfos"]
if self.localHgPath not in dirinfos:
return []
return dirinfos[self.localHgPath][0] + dirinfos[self.localHgPath][1]
def send_multi_status_response(environ, start_response, multistatusEL):
# If logging of the body is desired, then this is the place to do it
# pretty:
if environ.get("wsgidav.dump_response_body"):
xml = "{} XML response body:\n{}".format(
environ["REQUEST_METHOD"],
compat.to_native(xml_to_bytes(multistatusEL, pretty_print=True)),
)
environ["wsgidav.dump_response_body"] = xml
# Hotfix for Windows XP
# PROPFIND XML response is not recognized, when pretty_print = True!
# (Vista and others would accept this).
xml_data = xml_to_bytes(multistatusEL, pretty_print=False)
# If not, Content-Length is wrong!
assert compat.is_bytes(xml_data), xml_data
headers = [
("Content-Type", "application/xml"),
("Date", get_rfc1123_time()),
("Content-Length", str(len(xml_data))),
]
# if 'keep-alive' in environ.get('HTTP_CONNECTION', '').lower():
# headers += [
# ('Connection', 'keep-alive'),
# ]
start_response("207 Multi-Status", headers)
return [xml_data]
def read(self, size=0):
"""Read a chunk of bytes from queue.
size = 0: Read next chunk (arbitrary length)
> 0: Read one chunk of `size` bytes (or less if stream was closed)
< 0: Read all bytes as single chunk (i.e. blocks until stream is closed)
This method blocks until the requested size become available.
However, if close() was called, '' is returned immediately.
"""
res = self.unread
self.unread = ""
# Get next chunk, cumulating requested size as needed
while res == "" or size < 0 or (size > 0 and len(res) < size):
try:
# Read pending data, blocking if neccessary
# (but handle the case that close() is called while waiting)
res += compat.to_native(self.queue.get(True, 0.1))
except compat.queue.Empty:
# There was no pending data: wait for more, unless close() was called
if self.is_closed:
break
# Deliver `size` bytes from buffer
if size > 0 and len(res) > size:
self.unread = res[size:]
res = res[:size]
# print("FileLikeQueue.read({}) => {} bytes".format(size, len(res)))
return res
def testGetPut(self):
"""Read and write file contents."""
app = self.app
# Prepare file content
data1 = b"this is a file\nwith two lines"
data2 = b"this is another file\nwith three lines\nsee?"
# Big file with 10 MB
lines = []
line = "." * (1000 - 6 - len("\n"))
for i in compat.xrange(10 * 1000):
lines.append("%04i: %s\n" % (i, line))
data3 = "".join(lines)
data3 = compat.to_bytes(data3)
# Remove old test files
app.delete("/file1.txt", expect_errors=True)
app.delete("/file2.txt", expect_errors=True)
app.delete("/file3.txt", expect_errors=True)
# Access unmapped resource (expect '404 Not Found')
app.delete("/file1.txt", status=404)
app.get("/file1.txt", status=404)
# PUT a small file (expect '201 Created')
app.put("/file1.txt", params=data1, status=201)
res = app.get("/file1.txt", status=200)
assert res.body == data1, "GET file content different from PUT"
# PUT overwrites a small file (expect '204 No Content')
app.put("/file1.txt", params=data2, status=204)
res = app.get("/file1.txt", status=200)
assert res.body == data2, "GET file content different from PUT"
# PUT writes a big file (expect '201 Created')
app.put("/file2.txt", params=data3, status=201)
res = app.get("/file2.txt", status=200)
assert res.body == data3, "GET file content different from PUT"
# Request must not contain a body (expect '415 Media Type Not
# Supported')
app.request(
"/file1.txt",
method="GET",
headers={"Content-Length": compat.to_native(len(data1))},
body=data1,
status=415,
)
# Delete existing resource (expect '204 No Content')
app.delete("/file1.txt", status=204)
# Get deleted resource (expect '404 Not Found')
app.get("/file1.txt", status=404)
# PUT a small file (expect '201 Created')
app.put("/file1.txt", params=data1, status=201)
def __init__(self, path, environ, file_path):
super(FolderResource, self).__init__(path, environ)
self._file_path = file_path
# self._dict = None
self.file_stat = os.stat(self._file_path)
# Setting the name from the file path should fix the case on Windows
self.name = os.path.basename(self._file_path)
self.name = compat.to_native(self.name) # .encode("utf8")
def get_etag(self):
return (
md5(self.path).hexdigest()
+ "-"
+ compat.to_native(self.get_last_modified())
+ "-"
+ str(self.get_content_length())
)
def get_property_value(self, name):
"""Return the value of a property.
See get_property_value()
"""
# Supported custom live properties
if name == "{hg:}branch":
return self.fctx.branch()
elif name == "{hg:}date":
# (secs, tz-ofs)
return compat.to_native(self.fctx.date()[0])
elif name == "{hg:}description":
return self.fctx.description()
elif name == "{hg:}filerev":
return compat.to_native(self.fctx.filerev())
elif name == "{hg:}rev":
return compat.to_native(self.fctx.rev())
elif name == "{hg:}user":
return compat.to_native(self.fctx.user())
# Let base class implementation report live and dead properties
return super(HgResource, self).get_property_value(name)
def get_resource_inst(self, path, environ):
"""Return HgResource object for path.
See DAVProvider.get_resource_inst()
"""
self._count_get_resource_inst += 1
# HG expects the resource paths without leading '/'
localHgPath = path.strip("/")
rev = None
cmd, rest = util.pop_path(path)
if cmd == "":
return VirtualCollection(
path, environ, "root", ["edit", "released", "archive"]
)
elif cmd == "edit":
localHgPath = rest.strip("/")
rev = None
elif cmd == "released":
localHgPath = rest.strip("/")
rev = "tip"
elif cmd == "archive":
if rest == "/":
# Browse /archive: return a list of revision folders:
loglist = self._get_log(limit=10)
members = [compat.to_native(l["local_id"]) for l in loglist]
return VirtualCollection(path, environ, "Revisions", members)
revid, rest = util.pop_path(rest)
try:
int(revid)
except Exception:
# Tried to access /archive/anyname
return None
# Access /archive/19
rev = revid
localHgPath = rest.strip("/")
else:
return None
# read mercurial repo into request cache
cache = self._get_repo_info(environ, rev)
if localHgPath in cache["filedict"]:
# It is a version controlled file
return HgResource(path, False, environ, rev, localHgPath)
if localHgPath in cache["dirinfos"] or localHgPath == "":
# It is an existing folder
return HgResource(path, True, environ, rev, localHgPath)
return None
def handle_basic_auth_request(self, environ, start_response):
realm = self.domain_controller.get_domain_realm(environ["PATH_INFO"], environ)
auth_header = environ["HTTP_AUTHORIZATION"]
auth_value = ""
try:
auth_value = auth_header[len("Basic ") :].strip()
except Exception:
auth_value = ""
auth_value = compat.base64_decodebytes(compat.to_bytes(auth_value))
auth_value = compat.to_native(auth_value)
user_name, password = auth_value.split(":", 1)
if self.domain_controller.basic_auth_user(realm, user_name, password, environ):
environ["wsgidav.auth.realm"] = realm
environ["wsgidav.auth.user_name"] = user_name
return self.next_app(environ, start_response)
_logger.warning(
"Authentication (basic) failed for user '{}', realm '{}'.".format(
user_name, realm
)
)
return self.send_basic_auth_response(environ, start_response)
def get_display_name(self):
return compat.to_native(self.title)
def normalizeLockRoot(path):
# Normalize root: /foo/bar
assert path
path = compat.to_native(path)
path = "/" + path.strip("/")
return path
def generateLockToken():
return "opaquelocktoken:" + compat.to_native(hex(random.getrandbits(256)))
def calc_base64(s):
"""Return base64 encoded binarystring."""
s = compat.to_bytes(s)
s = compat.base64_encodebytes(s).strip() # return bytestring
return compat.to_native(s)
def parse_xml_body(environ, allow_empty=False):
"""Read request body XML into an etree.Element.
Return None, if no request body was sent.
Raise HTTP_BAD_REQUEST, if something else went wrong.
TODO: this is a very relaxed interpretation: should we raise HTTP_BAD_REQUEST
instead, if CONTENT_LENGTH is missing, invalid, or 0?
RFC: For compatibility with HTTP/1.0 applications, HTTP/1.1 requests containing
a message-body MUST include a valid Content-Length header field unless the
server is known to be HTTP/1.1 compliant.
If a request contains a message-body and a Content-Length is not given, the
server SHOULD respond with 400 (bad request) if it cannot determine the
length of the message, or with 411 (length required) if it wishes to insist
on receiving a valid Content-Length."
So I'd say, we should accept a missing CONTENT_LENGTH, and try to read the
content anyway.
But WSGI doesn't guarantee to support input.read() without length(?).
At least it locked, when I tried it with a request that had a missing
content-type and no body.
Current approach: if CONTENT_LENGTH is
- valid and >0:
read body (exactly <CONTENT_LENGTH> bytes) and parse the result.
- 0:
Assume empty body and return None or raise exception.
- invalid (negative or not a number:
raise HTTP_BAD_REQUEST
- missing:
NOT: Try to read body until end and parse the result.
BUT: assume '0'
- empty string:
WSGI allows it to be empty or absent: treated like 'missing'.
"""
#
clHeader = environ.get("CONTENT_LENGTH", "").strip()
# content_length = -1 # read all of stream
if clHeader == "":
# No Content-Length given: read to end of stream
# TODO: etree.parse() locks, if input is invalid?
# pfroot = etree.parse(environ["wsgi.input"]).getroot()
# requestbody = environ["wsgi.input"].read() # TODO: read() should be
# called in a loop?
requestbody = ""
else:
try:
content_length = int(clHeader)
if content_length < 0:
raise DAVError(HTTP_BAD_REQUEST, "Negative content-length.")
except ValueError:
raise DAVError(HTTP_BAD_REQUEST, "content-length is not numeric.")
if content_length == 0:
requestbody = ""
else:
requestbody = environ["wsgi.input"].read(content_length)
environ["wsgidav.all_input_read"] = 1
if requestbody == "":
if allow_empty:
return None
else:
raise DAVError(HTTP_BAD_REQUEST, "Body must not be empty.")
try:
rootEL = etree.fromstring(requestbody)
except Exception as e:
raise DAVError(HTTP_BAD_REQUEST, "Invalid XML format.", src_exception=e)
# If dumps of the body are desired, then this is the place to do it pretty:
if environ.get("wsgidav.dump_request_body"):
_logger.info(
"{} XML request body:\n{}".format(
environ["REQUEST_METHOD"],
compat.to_native(xml_to_bytes(rootEL, pretty_print=True)),
)
)
environ["wsgidav.dump_request_body"] = False
return rootEL
def normalize_lock_root(path):
# Normalize root: /foo/bar
assert path
path = compat.to_native(path)
path = "/" + path.strip("/")
return path
def generate_lock_token():
return "opaquelocktoken:" + compat.to_native(hex(random.getrandbits(256)))
def _get_repo_info(self, environ, rev, reload=False):
"""Return a dictionary containing all files under source control.
dirinfos:
Dictionary containing direct members for every collection.
{folderpath: (collectionlist, filelist), ...}
files:
Sorted list of all file paths in the manifest.
filedict:
Dictionary containing all files under source control.
::
{'dirinfos': {'': (['wsgidav',
'tools',
'WsgiDAV.egg-info',
'tests'],
['index.rst',
'wsgidav MAKE_DAILY_BUILD.launch',
'wsgidav run_server.py DEBUG.launch',
'wsgidav-paste.conf',
...
'setup.py']),
'wsgidav': (['addons', 'samples', 'server', 'interfaces'],
['__init__.pyc',
'dav_error.pyc',
'dav_provider.pyc',
...
'wsgidav_app.py']),
},
'files': ['.hgignore',
'ADDONS.txt',
'wsgidav/samples/mysql_dav_provider.py',
...
],
'filedict': {'.hgignore': True,
'README.txt': True,
'WsgiDAV.egg-info/PKG-INFO': True,
}
}
"""
caches = environ.setdefault("wsgidav.hg.cache", {})
if caches.get(compat.to_native(rev)) is not None:
_logger.debug("_get_repo_info(%s): cache hit." % rev)
return caches[compat.to_native(rev)]
start_time = time.time()
self.ui.pushbuffer()
commands.manifest(self.ui, self.repo, rev)
res = self.ui.popbuffer()
files = []
dirinfos = {}
filedict = {}
for file in res.split("\n"):
if file.strip() == "":
continue
file = file.replace("\\", "/")
# add all parent directories to 'dirinfos'
parents = file.split("/")
if len(parents) >= 1:
p1 = ""
for i in range(0, len(parents) - 1):
p2 = parents[i]
dir = dirinfos.setdefault(p1, ([], []))
if p2 not in dir[0]:
dir[0].append(p2)
if p1 == "":
p1 = p2
else:
p1 = "%s/%s" % (p1, p2)
dirinfos.setdefault(p1, ([], []))[1].append(parents[-1])
filedict[file] = True
files.sort()
cache = {"files": files, "dirinfos": dirinfos, "filedict": filedict}
caches[compat.to_native(rev)] = cache
_logger.info("_getRepoInfo(%s) took %.3f" % (rev, time.time() - start_time))
return cache
def _getRepoInfo(self, environ, rev, reload=False):
"""Return a dictionary containing all files under source control.
dirinfos:
Dictionary containing direct members for every collection.
{folderpath: (collectionlist, filelist), ...}
files:
Sorted list of all file paths in the manifest.
filedict:
Dictionary containing all files under source control.
::
{'dirinfos': {'': (['wsgidav',
'tools',
'WsgiDAV.egg-info',
'tests'],
['index.rst',
'wsgidav MAKE_DAILY_BUILD.launch',
'wsgidav run_server.py DEBUG.launch',
'wsgidav-paste.conf',
...
'setup.py']),
'wsgidav': (['addons', 'samples', 'server', 'interfaces'],
['__init__.pyc',
'dav_error.pyc',
'dav_provider.pyc',
...
'wsgidav_app.py']),
},
'files': ['.hgignore',
'ADDONS.txt',
'wsgidav/addons/mysql_dav_provider.py',
...
],
'filedict': {'.hgignore': True,
'README.txt': True,
'WsgiDAV.egg-info/PKG-INFO': True,
}
}
"""
caches = environ.setdefault("wsgidav.hg.cache", {})
if caches.get(compat.to_native(rev)) is not None:
util.debug("_getRepoInfo(%s): cache hit." % rev)
return caches[compat.to_native(rev)]
start_time = time.time()
self.ui.pushbuffer()
commands.manifest(self.ui, self.repo, rev)
res = self.ui.popbuffer()
files = []
dirinfos = {}
filedict = {}
for file in res.split("\n"):
if file.strip() == "":
continue
file = file.replace("\\", "/")
# add all parent directories to 'dirinfos'
parents = file.split("/")
if len(parents) >= 1:
p1 = ""
for i in range(0, len(parents) - 1):
p2 = parents[i]
dir = dirinfos.setdefault(p1, ([], []))
if not p2 in dir[0]:
dir[0].append(p2)
if p1 == "":
p1 = p2
else:
p1 = "%s/%s" % (p1, p2)
dirinfos.setdefault(p1, ([], []))[1].append(parents[-1])
filedict[file] = True
files.sort()
cache = {
"files": files,
"dirinfos": dirinfos,
"filedict": filedict,
}
caches[compat.to_native(rev)] = cache
util.note("_getRepoInfo(%s) took %.3f" %
(rev, time.time() - start_time)
# , var=cache
)
return cache
def calc_base64(s):
"""Return base64 encoded binarystring."""
s = compat.to_bytes(s)
s = compat.base64_encodebytes(s).strip() # return bytestring
return compat.to_native(s)
def get_etag(self):
return (md5(self.path).hexdigest() + "-" +
compat.to_native(self.get_last_modified()) + "-" +
str(self.get_content_length()))
def getMemberNames(self):
res = []
for doc in self.coll.find():
res.append(compat.to_native(doc["_id"]))
return res
def get_member_names(self):
res = []
for doc in self.coll.find():
res.append(compat.to_native(doc["_id"]))
return res
def __call__(self, environ, start_response):
# util.log("SCRIPT_NAME='%s', PATH_INFO='%s'" % (
# environ.get("SCRIPT_NAME"), environ.get("PATH_INFO")))
path = environ["PATH_INFO"]
# (#73) Failed on processing non-iso-8859-1 characters on Python 3
#
# Note: we encode using UTF-8 here (falling back to ISO-8859-1)!
# This seems to be wrong, since per PEP 3333 PATH_INFO is always ISO-8859-1 encoded
# (see https://www.python.org/dev/peps/pep-3333/#unicode-issues).
# But also seems to resolve errors when accessing resources with Chinese characters, for
# example.
# This is done by default for Python 3, but can be turned off in settings.
re_encode_path_info = self.config.get("re_encode_path_info")
if re_encode_path_info is None:
re_encode_path_info = compat.PY3
if re_encode_path_info:
b = compat.wsgi_to_bytes(path).decode()
path = environ["PATH_INFO"] = b
# We optionally unquote PATH_INFO here, although this should already be
# done by the server (#8).
if self.config.get("unquote_path_info", False):
path = compat.unquote(environ["PATH_INFO"])
# GC issue 22: Pylons sends root as u'/'
# if isinstance(path, unicode):
if not compat.is_native(path):
util.log("Got non-native PATH_INFO: %r" % path)
# path = path.encode("utf8")
path = compat.to_native(path)
# Always adding these values to environ:
environ["wsgidav.config"] = self.config
environ["wsgidav.provider"] = None
environ["wsgidav.verbose"] = self._verbose
# Find DAV provider that matches the share
# sorting share list by reverse length
# shareList = self.providerMap.keys()
# shareList.sort(key=len, reverse=True)
shareList = sorted(self.providerMap.keys(), key=len, reverse=True)
share = None
for r in shareList:
# @@: Case sensitivity should be an option of some sort here;
# os.path.normpath might give the preferred case for a filename.
if r == "/":
share = r
break
elif path.upper() == r.upper() or path.upper().startswith(r.upper() + "/"):
share = r
break
# Note: we call the next app, even if provider is None, because OPTIONS
# must still be handled.
# All other requests will result in '404 Not Found'
if share is not None:
share_data = self.providerMap.get(share)
environ["wsgidav.provider"] = share_data['provider']
# TODO: test with multi-level realms: 'aa/bb'
# TODO: test security: url contains '..'
# Transform SCRIPT_NAME and PATH_INFO
# (Since path and share are unquoted, this also fixes quoted values.)
if share == "/" or not share:
environ["PATH_INFO"] = path
else:
environ["SCRIPT_NAME"] += share
environ["PATH_INFO"] = path[len(share):]
# util.log("--> SCRIPT_NAME='%s', PATH_INFO='%s'" % (environ.get("SCRIPT_NAME"), environ.get("PATH_INFO")))
# assert isinstance(path, str)
assert compat.is_native(path)
# See http://mail.python.org/pipermail/web-sig/2007-January/002475.html
# for some clarification about SCRIPT_NAME/PATH_INFO format
# SCRIPT_NAME starts with '/' or is empty
assert environ["SCRIPT_NAME"] == "" or environ[
"SCRIPT_NAME"].startswith("/")
# SCRIPT_NAME must not have a trailing '/'
assert environ["SCRIPT_NAME"] in (
"", "/") or not environ["SCRIPT_NAME"].endswith("/")
# PATH_INFO starts with '/'
assert environ["PATH_INFO"] == "" or environ[
"PATH_INFO"].startswith("/")
start_time = time.time()
def _start_response_wrapper(status, response_headers, exc_info=None):
# Postprocess response headers
headerDict = {}
for header, value in response_headers:
if header.lower() in headerDict:
util.warn("Duplicate header in response: %s" % header)
headerDict[header.lower()] = value
# Check if we should close the connection after this request.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
forceCloseConnection = False
currentContentLength = headerDict.get("content-length")
statusCode = int(status.split(" ", 1)[0])
contentLengthRequired = (environ["REQUEST_METHOD"] != "HEAD"
and statusCode >= 200
and not statusCode in (204, 304))
# print(environ["REQUEST_METHOD"], statusCode, contentLengthRequired)
if contentLengthRequired and currentContentLength in (None, ""):
# A typical case: a GET request on a virtual resource, for which
# the provider doesn't know the length
util.warn(
"Missing required Content-Length header in %s-response: closing connection" %
statusCode)
forceCloseConnection = True
elif not type(currentContentLength) is str:
util.warn("Invalid Content-Length header in response (%r): closing connection" %
headerDict.get("content-length"))
forceCloseConnection = True
# HOTFIX for Vista and Windows 7 (GC issue 13, issue 23)
# It seems that we must read *all* of the request body, otherwise
# clients may miss the response.
# For example Vista MiniRedir didn't understand a 401 response,
# when trying an anonymous PUT of big files. As a consequence, it
# doesn't retry with credentials and the file copy fails.
# (XP is fine however).
util.readAndDiscardInput(environ)
# Make sure the socket is not reused, unless we are 100% sure all
# current input was consumed
if(util.getContentLength(environ) != 0
and not environ.get("wsgidav.all_input_read")):
util.warn(
"Input stream not completely consumed: closing connection")
forceCloseConnection = True
if forceCloseConnection and headerDict.get("connection") != "close":
util.warn("Adding 'Connection: close' header")
response_headers.append(("Connection", "close"))
# Log request
if self._verbose >= 1:
userInfo = environ.get("http_authenticator.username")
if not userInfo:
userInfo = "(anonymous)"
threadInfo = ""
if self._verbose >= 1:
threadInfo = "<%s> " % threading.currentThread().ident
extra = []
if "HTTP_DESTINATION" in environ:
extra.append('dest="%s"' % environ.get("HTTP_DESTINATION"))
if environ.get("CONTENT_LENGTH", "") != "":
extra.append("length=%s" % environ.get("CONTENT_LENGTH"))
if "HTTP_DEPTH" in environ:
extra.append("depth=%s" % environ.get("HTTP_DEPTH"))
if "HTTP_RANGE" in environ:
extra.append("range=%s" % environ.get("HTTP_RANGE"))
if "HTTP_OVERWRITE" in environ:
extra.append("overwrite=%s" %
environ.get("HTTP_OVERWRITE"))
if self._verbose >= 1 and "HTTP_EXPECT" in environ:
extra.append('expect="%s"' % environ.get("HTTP_EXPECT"))
if self._verbose >= 2 and "HTTP_CONNECTION" in environ:
extra.append('connection="%s"' %
environ.get("HTTP_CONNECTION"))
if self._verbose >= 2 and "HTTP_USER_AGENT" in environ:
extra.append('agent="%s"' % environ.get("HTTP_USER_AGENT"))
if self._verbose >= 2 and "HTTP_TRANSFER_ENCODING" in environ:
extra.append('transfer-enc=%s' %
environ.get("HTTP_TRANSFER_ENCODING"))
if self._verbose >= 1:
extra.append('elap=%.3fsec' % (time.time() - start_time))
extra = ", ".join(extra)
# This is the CherryPy format:
# 127.0.0.1 - - [08/Jul/2009:17:25:23] "GET /loginPrompt?redirect=/renderActionList%3Frelation%3Dpersonal%26key%3D%26filter%3DprivateSchedule&reason=0 HTTP/1.1" 200 1944 "http://127.0.0.1:8002/command?id=CMD_Schedule" "Mozilla/5.0 (Windows; U; Windows NT 6.0; de; rv:1.9.1) Gecko/20090624 Firefox/3.5"
# print >>sys.stderr, '%s - %s - [%s] "%s" %s -> %s' % (
print('%s - %s - [%s] "%s" %s -> %s' % (
threadInfo + environ.get("REMOTE_ADDR", ""),
userInfo,
util.getLogTime(),
environ.get("REQUEST_METHOD") + " " +
safeReEncode(environ.get("PATH_INFO", ""), sys.stdout.encoding or "ASCII"),
extra,
status,
# response_headers.get(""), # response Content-Length
# referer
), file=sys.stdout)
return start_response(status, response_headers, exc_info)
# Call next middleware
app_iter = self._application(environ, _start_response_wrapper)
for v in app_iter:
yield v
if hasattr(app_iter, "close"):
app_iter.close()
return
def __call__(self, environ, start_response):
# util.log("SCRIPT_NAME='{}', PATH_INFO='{}'".format(
# environ.get("SCRIPT_NAME"), environ.get("PATH_INFO")))
path = environ["PATH_INFO"]
# (#73) Failed on processing non-iso-8859-1 characters on Python 3
#
# Note: we encode using UTF-8 here (falling back to ISO-8859-1)!
# This seems to be wrong, since per PEP 3333 PATH_INFO is always ISO-8859-1 encoded
# (see https://www.python.org/dev/peps/pep-3333/#unicode-issues).
# But also seems to resolve errors when accessing resources with Chinese characters, for
# example.
# This is done by default for Python 3, but can be turned off in settings.
if self.re_encode_path_info:
path = environ["PATH_INFO"] = compat.wsgi_to_bytes(path).decode()
# We optionally unquote PATH_INFO here, although this should already be
# done by the server (#8).
if self.unquote_path_info:
path = compat.unquote(environ["PATH_INFO"])
# GC issue 22: Pylons sends root as u'/'
if not compat.is_native(path):
_logger.warning("Got non-native PATH_INFO: {!r}".format(path))
# path = path.encode("utf8")
path = compat.to_native(path)
# Always adding these values to environ:
environ["wsgidav.config"] = self.config
environ["wsgidav.provider"] = None
environ["wsgidav.verbose"] = self.verbose
# Find DAV provider that matches the share
share, provider = self.resolve_provider(path)
# share = None
# lower_path = path.lower()
# for r in self.sorted_share_list:
# # @@: Case sensitivity should be an option of some sort here;
# # os.path.normpath might give the preferred case for a filename.
# if r == "/":
# share = r
# break
# elif lower_path == r or lower_path.startswith(r + "/"):
# share = r
# break
# Note: we call the next app, even if provider is None, because OPTIONS
# must still be handled.
# All other requests will result in '404 Not Found'
# if share is not None:
# share_data = self.provider_map.get(share)
# environ["wsgidav.provider"] = share_data["provider"]
environ["wsgidav.provider"] = provider
# TODO: test with multi-level realms: 'aa/bb'
# TODO: test security: url contains '..'
# Transform SCRIPT_NAME and PATH_INFO
# (Since path and share are unquoted, this also fixes quoted values.)
if share == "/" or not share:
environ["PATH_INFO"] = path
else:
environ["SCRIPT_NAME"] += share
environ["PATH_INFO"] = path[len(share) :]
# assert isinstance(path, str)
assert compat.is_native(path)
# See http://mail.python.org/pipermail/web-sig/2007-January/002475.html
# for some clarification about SCRIPT_NAME/PATH_INFO format
# SCRIPT_NAME starts with '/' or is empty
assert environ["SCRIPT_NAME"] == "" or environ["SCRIPT_NAME"].startswith("/")
# SCRIPT_NAME must not have a trailing '/'
assert environ["SCRIPT_NAME"] in ("", "/") or not environ[
"SCRIPT_NAME"
].endswith("/")
# PATH_INFO starts with '/'
assert environ["PATH_INFO"] == "" or environ["PATH_INFO"].startswith("/")
start_time = time.time()
def _start_response_wrapper(status, response_headers, exc_info=None):
# Postprocess response headers
headerDict = {}
for header, value in response_headers:
if header.lower() in headerDict:
_logger.error("Duplicate header in response: {}".format(header))
headerDict[header.lower()] = value
# Check if we should close the connection after this request.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
forceCloseConnection = False
currentContentLength = headerDict.get("content-length")
statusCode = int(status.split(" ", 1)[0])
contentLengthRequired = (
environ["REQUEST_METHOD"] != "HEAD"
and statusCode >= 200
and statusCode not in (204, 304)
)
# _logger.info(environ["REQUEST_METHOD"], statusCode, contentLengthRequired)
if contentLengthRequired and currentContentLength in (None, ""):
# A typical case: a GET request on a virtual resource, for which
# the provider doesn't know the length
_logger.error(
"Missing required Content-Length header in {}-response: closing connection".format(
statusCode
)
)
forceCloseConnection = True
elif not type(currentContentLength) is str:
_logger.error(
"Invalid Content-Length header in response ({!r}): closing connection".format(
headerDict.get("content-length")
)
)
forceCloseConnection = True
# HOTFIX for Vista and Windows 7 (GC issue 13, issue 23)
# It seems that we must read *all* of the request body, otherwise
# clients may miss the response.
# For example Vista MiniRedir didn't understand a 401 response,
# when trying an anonymous PUT of big files. As a consequence, it
# doesn't retry with credentials and the file copy fails.
# (XP is fine however).
util.read_and_discard_input(environ)
# Make sure the socket is not reused, unless we are 100% sure all
# current input was consumed
if util.get_content_length(environ) != 0 and not environ.get(
"wsgidav.all_input_read"
):
_logger.warning(
"Input stream not completely consumed: closing connection."
)
forceCloseConnection = True
if forceCloseConnection and headerDict.get("connection") != "close":
_logger.warning("Adding 'Connection: close' header.")
response_headers.append(("Connection", "close"))
# Log request
if self.verbose >= 3:
userInfo = environ.get("wsgidav.auth.user_name")
if not userInfo:
userInfo = "(anonymous)"
extra = []
if "HTTP_DESTINATION" in environ:
extra.append('dest="{}"'.format(environ.get("HTTP_DESTINATION")))
if environ.get("CONTENT_LENGTH", "") != "":
extra.append("length={}".format(environ.get("CONTENT_LENGTH")))
if "HTTP_DEPTH" in environ:
extra.append("depth={}".format(environ.get("HTTP_DEPTH")))
if "HTTP_RANGE" in environ:
extra.append("range={}".format(environ.get("HTTP_RANGE")))
if "HTTP_OVERWRITE" in environ:
extra.append("overwrite={}".format(environ.get("HTTP_OVERWRITE")))
if self.verbose >= 3 and "HTTP_EXPECT" in environ:
extra.append('expect="{}"'.format(environ.get("HTTP_EXPECT")))
if self.verbose >= 4 and "HTTP_CONNECTION" in environ:
extra.append(
'connection="{}"'.format(environ.get("HTTP_CONNECTION"))
)
if self.verbose >= 4 and "HTTP_USER_AGENT" in environ:
extra.append('agent="{}"'.format(environ.get("HTTP_USER_AGENT")))
if self.verbose >= 4 and "HTTP_TRANSFER_ENCODING" in environ:
extra.append(
"transfer-enc={}".format(environ.get("HTTP_TRANSFER_ENCODING"))
)
if self.verbose >= 3:
extra.append("elap={:.3f}sec".format(time.time() - start_time))
extra = ", ".join(extra)
# This is the CherryPy format:
# 127.0.0.1 - - [08/Jul/2009:17:25:23] "GET /loginPrompt?redirect=/renderActionList%3Frelation%3Dpersonal%26key%3D%26filter%3DprivateSchedule&reason=0 HTTP/1.1" 200 1944 "http://127.0.0.1:8002/command?id=CMD_Schedule" "Mozilla/5.0 (Windows; U; Windows NT 6.0; de; rv:1.9.1) Gecko/20090624 Firefox/3.5" # noqa
_logger.info(
'{addr} - {user} - [{time}] "{method} {path}" {extra} -> {status}'.format(
addr=environ.get("REMOTE_ADDR", ""),
user=userInfo,
time=util.get_log_time(),
method=environ.get("REQUEST_METHOD"),
path=safe_re_encode(
environ.get("PATH_INFO", ""),
sys.stdout.encoding if sys.stdout.encoding else "utf-8",
),
extra=extra,
status=status,
# response_headers.get(""), # response Content-Length
# referer
)
)
return start_response(status, response_headers, exc_info)
# Call first middleware
app_iter = self.application(environ, _start_response_wrapper)
try:
for v in app_iter:
yield v
finally:
if hasattr(app_iter, "close"):
app_iter.close()
return
def runWSGIApp(self, application, script_name, path_info, query):
# logging.info ("Running application with SCRIPT_NAME {} PATH_INFO {}".format(
# script_name, path_info))
if self.command == "PUT":
pass # breakpoint
env = {
"wsgi.version": (1, 0),
"wsgi.url_scheme": "http",
"wsgi.input": self.rfile,
"wsgi.errors": sys.stderr,
"wsgi.multithread": 1,
"wsgi.multiprocess": 0,
"wsgi.run_once": 0,
"REQUEST_METHOD": self.command,
"SCRIPT_NAME": script_name,
"PATH_INFO": path_info,
"QUERY_STRING": query,
"CONTENT_TYPE": self.headers.get("Content-Type", ""),
"CONTENT_LENGTH": self.headers.get("Content-Length", ""),
"REMOTE_ADDR": self.client_address[0],
"SERVER_NAME": self.server.server_address[0],
"SERVER_PORT": compat.to_native(self.server.server_address[1]),
"SERVER_PROTOCOL": self.request_version,
}
for httpHeader, httpValue in self.headers.items():
if not httpHeader.lower() in ("content-type", "content-length"):
env["HTTP_{}".format(httpHeader.replace(
"-", "_").upper())] = httpValue
# Setup the state
self.wsgiSentHeaders = 0
self.wsgiHeaders = []
try:
# We have there environment, now invoke the application
_logger.debug("runWSGIApp application()...")
result = application(env, self.wsgiStartResponse)
try:
for data in result:
if data:
self.wsgiWriteData(data)
else:
_logger.debug("runWSGIApp empty data")
finally:
_logger.debug("runWSGIApp finally.")
if hasattr(result, "close"):
result.close()
except Exception:
_logger.debug("runWSGIApp caught exception...")
errorMsg = compat.StringIO()
traceback.print_exc(file=errorMsg)
logging.error(errorMsg.getvalue())
if not self.wsgiSentHeaders:
self.wsgiStartResponse("500 Server Error",
[("Content-type", "text/html")])
self.wsgiWriteData(SERVER_ERROR)
if not self.wsgiSentHeaders:
# GC issue 29 sending one byte, when content-length is '0' seems wrong
# We must write out something!
# self.wsgiWriteData (" ")
self.wsgiWriteData(b"")
return
def as_string(self):
return compat.to_native(xml_tools.xml_to_bytes(self.as_xml(), True))
def getEtag(self):
return (md5(self.path).hexdigest() + "-" +
compat.to_native(self.getLastModified()) + "-" +
str(self.getContentLength()))
def as_string(self):
return compat.to_native(xml_tools.xml_to_bytes(self.as_xml(), True))
def parse_xml_body(environ, allow_empty=False):
"""Read request body XML into an etree.Element.
Return None, if no request body was sent.
Raise HTTP_BAD_REQUEST, if something else went wrong.
TODO: this is a very relaxed interpretation: should we raise HTTP_BAD_REQUEST
instead, if CONTENT_LENGTH is missing, invalid, or 0?
RFC: For compatibility with HTTP/1.0 applications, HTTP/1.1 requests containing
a message-body MUST include a valid Content-Length header field unless the
server is known to be HTTP/1.1 compliant.
If a request contains a message-body and a Content-Length is not given, the
server SHOULD respond with 400 (bad request) if it cannot determine the
length of the message, or with 411 (length required) if it wishes to insist
on receiving a valid Content-Length."
So I'd say, we should accept a missing CONTENT_LENGTH, and try to read the
content anyway.
But WSGI doesn't guarantee to support input.read() without length(?).
At least it locked, when I tried it with a request that had a missing
content-type and no body.
Current approach: if CONTENT_LENGTH is
- valid and >0:
read body (exactly <CONTENT_LENGTH> bytes) and parse the result.
- 0:
Assume empty body and return None or raise exception.
- invalid (negative or not a number:
raise HTTP_BAD_REQUEST
- missing:
NOT: Try to read body until end and parse the result.
BUT: assume '0'
- empty string:
WSGI allows it to be empty or absent: treated like 'missing'.
"""
#
clHeader = environ.get("CONTENT_LENGTH", "").strip()
# content_length = -1 # read all of stream
if clHeader == "":
# No Content-Length given: read to end of stream
# TODO: etree.parse() locks, if input is invalid?
# pfroot = etree.parse(environ["wsgi.input"]).getroot()
# requestbody = environ["wsgi.input"].read() # TODO: read() should be
# called in a loop?
requestbody = ""
else:
try:
content_length = int(clHeader)
if content_length < 0:
raise DAVError(HTTP_BAD_REQUEST, "Negative content-length.")
except ValueError:
raise DAVError(HTTP_BAD_REQUEST, "content-length is not numeric.")
if content_length == 0:
requestbody = ""
else:
requestbody = environ["wsgi.input"].read(content_length)
environ["wsgidav.all_input_read"] = 1
if requestbody == "":
if allow_empty:
return None
else:
raise DAVError(HTTP_BAD_REQUEST, "Body must not be empty.")
try:
rootEL = etree.fromstring(requestbody)
except Exception as e:
raise DAVError(HTTP_BAD_REQUEST,
"Invalid XML format.",
src_exception=e)
# If dumps of the body are desired, then this is the place to do it pretty:
if environ.get("wsgidav.dump_request_body"):
_logger.info("{} XML request body:\n{}".format(
environ["REQUEST_METHOD"],
compat.to_native(xml_to_bytes(rootEL, pretty_print=True)),
))
environ["wsgidav.dump_request_body"] = False
return rootEL
def __call__(self, environ, start_response):
# util.log("SCRIPT_NAME='{}', PATH_INFO='{}'".format(
# environ.get("SCRIPT_NAME"), environ.get("PATH_INFO")))
path = environ["PATH_INFO"]
# (#73) Failed on processing non-iso-8859-1 characters on Python 3
#
# Note: we encode using UTF-8 here (falling back to ISO-8859-1)!
# This seems to be wrong, since per PEP 3333 PATH_INFO is always ISO-8859-1 encoded
# (see https://www.python.org/dev/peps/pep-3333/#unicode-issues).
# But also seems to resolve errors when accessing resources with Chinese characters, for
# example.
# This is done by default for Python 3, but can be turned off in settings.
if self.re_encode_path_info:
path = environ["PATH_INFO"] = compat.wsgi_to_bytes(path).decode()
# We optionally unquote PATH_INFO here, although this should already be
# done by the server (#8).
if self.unquote_path_info:
path = compat.unquote(environ["PATH_INFO"])
# GC issue 22: Pylons sends root as u'/'
if not compat.is_native(path):
_logger.warning("Got non-native PATH_INFO: {!r}".format(path))
# path = path.encode("utf8")
path = compat.to_native(path)
# Always adding these values to environ:
environ["wsgidav.config"] = self.config
environ["wsgidav.provider"] = None
environ["wsgidav.verbose"] = self.verbose
# Find DAV provider that matches the share
share, provider = self.resolve_provider(path)
# share = None
# lower_path = path.lower()
# for r in self.sorted_share_list:
# # @@: Case sensitivity should be an option of some sort here;
# # os.path.normpath might give the preferred case for a filename.
# if r == "/":
# share = r
# break
# elif lower_path == r or lower_path.startswith(r + "/"):
# share = r
# break
# Note: we call the next app, even if provider is None, because OPTIONS
# must still be handled.
# All other requests will result in '404 Not Found'
# if share is not None:
# share_data = self.provider_map.get(share)
# environ["wsgidav.provider"] = share_data["provider"]
environ["wsgidav.provider"] = provider
# TODO: test with multi-level realms: 'aa/bb'
# TODO: test security: url contains '..'
# Transform SCRIPT_NAME and PATH_INFO
# (Since path and share are unquoted, this also fixes quoted values.)
if share == "/" or not share:
environ["PATH_INFO"] = path
else:
environ["SCRIPT_NAME"] += share
environ["PATH_INFO"] = path[len(share):]
# assert isinstance(path, str)
assert compat.is_native(path)
# See http://mail.python.org/pipermail/web-sig/2007-January/002475.html
# for some clarification about SCRIPT_NAME/PATH_INFO format
# SCRIPT_NAME starts with '/' or is empty
assert environ["SCRIPT_NAME"] == "" or environ[
"SCRIPT_NAME"].startswith("/")
# SCRIPT_NAME must not have a trailing '/'
assert environ["SCRIPT_NAME"] in (
"", "/") or not environ["SCRIPT_NAME"].endswith("/")
# PATH_INFO starts with '/'
assert environ["PATH_INFO"] == "" or environ["PATH_INFO"].startswith(
"/")
start_time = time.time()
def _start_response_wrapper(status, response_headers, exc_info=None):
# Postprocess response headers
headerDict = {}
for header, value in response_headers:
if header.lower() in headerDict:
_logger.error(
"Duplicate header in response: {}".format(header))
headerDict[header.lower()] = value
# Check if we should close the connection after this request.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.4
forceCloseConnection = False
currentContentLength = headerDict.get("content-length")
statusCode = int(status.split(" ", 1)[0])
contentLengthRequired = (environ["REQUEST_METHOD"] != "HEAD"
and statusCode >= 200
and statusCode not in (204, 304))
# _logger.info(environ["REQUEST_METHOD"], statusCode, contentLengthRequired)
if contentLengthRequired and currentContentLength in (None, ""):
# A typical case: a GET request on a virtual resource, for which
# the provider doesn't know the length
_logger.error(
"Missing required Content-Length header in {}-response: closing connection"
.format(statusCode))
forceCloseConnection = True
elif not type(currentContentLength) is str:
_logger.error(
"Invalid Content-Length header in response ({!r}): closing connection"
.format(headerDict.get("content-length")))
forceCloseConnection = True
# HOTFIX for Vista and Windows 7 (GC issue 13, issue 23)
# It seems that we must read *all* of the request body, otherwise
# clients may miss the response.
# For example Vista MiniRedir didn't understand a 401 response,
# when trying an anonymous PUT of big files. As a consequence, it
# doesn't retry with credentials and the file copy fails.
# (XP is fine however).
util.read_and_discard_input(environ)
# Make sure the socket is not reused, unless we are 100% sure all
# current input was consumed
if util.get_content_length(environ) != 0 and not environ.get(
"wsgidav.all_input_read"):
_logger.warning(
"Input stream not completely consumed: closing connection."
)
forceCloseConnection = True
if forceCloseConnection and headerDict.get(
"connection") != "close":
_logger.warning("Adding 'Connection: close' header.")
response_headers.append(("Connection", "close"))
# Log request
if self.verbose >= 3:
userInfo = environ.get("wsgidav.auth.user_name")
if not userInfo:
userInfo = "(anonymous)"
extra = []
if "HTTP_DESTINATION" in environ:
extra.append('dest="{}"'.format(
environ.get("HTTP_DESTINATION")))
if environ.get("CONTENT_LENGTH", "") != "":
extra.append("length={}".format(
environ.get("CONTENT_LENGTH")))
if "HTTP_DEPTH" in environ:
extra.append("depth={}".format(environ.get("HTTP_DEPTH")))
if "HTTP_RANGE" in environ:
extra.append("range={}".format(environ.get("HTTP_RANGE")))
if "HTTP_OVERWRITE" in environ:
extra.append("overwrite={}".format(
environ.get("HTTP_OVERWRITE")))
if self.verbose >= 3 and "HTTP_EXPECT" in environ:
extra.append('expect="{}"'.format(
environ.get("HTTP_EXPECT")))
if self.verbose >= 4 and "HTTP_CONNECTION" in environ:
extra.append('connection="{}"'.format(
environ.get("HTTP_CONNECTION")))
if self.verbose >= 4 and "HTTP_USER_AGENT" in environ:
extra.append('agent="{}"'.format(
environ.get("HTTP_USER_AGENT")))
if self.verbose >= 4 and "HTTP_TRANSFER_ENCODING" in environ:
extra.append("transfer-enc={}".format(
environ.get("HTTP_TRANSFER_ENCODING")))
if self.verbose >= 3:
extra.append("elap={:.3f}sec".format(time.time() -
start_time))
extra = ", ".join(extra)
# This is the CherryPy format:
# 127.0.0.1 - - [08/Jul/2009:17:25:23] "GET /loginPrompt?redirect=/renderActionList%3Frelation%3Dpersonal%26key%3D%26filter%3DprivateSchedule&reason=0 HTTP/1.1" 200 1944 "http://127.0.0.1:8002/command?id=CMD_Schedule" "Mozilla/5.0 (Windows; U; Windows NT 6.0; de; rv:1.9.1) Gecko/20090624 Firefox/3.5" # noqa
_logger.info(
'{addr} - {user} - [{time}] "{method} {path}" {extra} -> {status}'
.format(
addr=environ.get("REMOTE_ADDR", ""),
user=userInfo,
time=util.get_log_time(),
method=environ.get("REQUEST_METHOD"),
path=safe_re_encode(
environ.get("PATH_INFO", ""),
sys.stdout.encoding
if sys.stdout.encoding else "utf-8",
),
extra=extra,
status=status,
# response_headers.get(""), # response Content-Length
# referer
))
return start_response(status, response_headers, exc_info)
# Call first middleware
app_iter = self.application(environ, _start_response_wrapper)
try:
for v in app_iter:
yield v
finally:
if hasattr(app_iter, "close"):
app_iter.close()
return
