def options(self, res_id):
"""
Proxy OPTIONS request to daemon.
"""
logging.info("[%s] OPTIONS", web.client_address(self.request))
allow = {"GET", "PUT", "PATCH", "OPTIONS"}
features = {"zero", "flush"}
# Reporting the meta-capabilities for all images
if res_id == "*":
return web.response(payload={"features": list(features)},
allow=','.join(allow))
ticket = auth.authorize_request(res_id, self.request)
try:
res = self.make_imaged_request(
"OPTIONS",
self.get_imaged_url(ticket),
self.request.headers,
None,
False,
connection_timeout=self.config.imaged_connection_timeout_sec,
read_timeout=self.config.imaged_read_timeout_sec)
except exc.HTTPMethodNotAllowed:
# An old daemon - we estimate its methods. We also assume that
# the ticket is readable and writable, since only the daemon
# knows about that.
logging.info("The daemon does not support OPTIONS, "
"returning an estimation")
return web.response(payload={"features": []},
allow="OPTIONS,GET,PUT")
if res.status_code != httplib.OK:
raise exc.HTTPInternalServerError(
"Got unexpected response from host: %d %s" %
(res.status_code, res.content))
try:
daemon_allow = set(res.headers.get("Allow").split(","))
except KeyError:
raise exc.HTTPInternalServerError(
"Got invalid response from host: missing Allow header")
try:
daemon_features = set(res.json()["features"])
except (ValueError, KeyError):
raise exc.HTTPInternalServerError(
"Got invalid response from host: "
"invalid JSON or missing 'features'")
allow = allow.intersection(daemon_allow)
features = features.intersection(daemon_features)
return web.response(payload={"features": list(features)},
allow=','.join(allow))
def options(self, res_id):
"""
Proxy OPTIONS request to daemon.
"""
allow = {"GET", "PUT", "PATCH", "OPTIONS"}
features = {"zero", "flush"}
# Reporting the meta-capabilities for all images
if res_id == "*":
return web.response(payload={"features": list(features)},
allow=','.join(allow))
ticket = auth.authorize_request(res_id, self.request)
try:
res = self.make_imaged_request(
"OPTIONS",
self.get_imaged_url(ticket),
self.request.headers,
None,
False,
connection_timeout=self.config.imaged_connection_timeout_sec,
read_timeout=self.config.imaged_read_timeout_sec)
except exc.HTTPMethodNotAllowed:
# An old daemon - we estimate its methods. We also assume that
# the ticket is readable and writable, since only the daemon
# knows about that.
logging.info("The daemon does not support OPTIONS, "
"returning an estimation")
return web.response(payload={"features": []},
allow="OPTIONS,GET,PUT")
if res.status_code != httplib.OK:
raise exc.HTTPInternalServerError(
"Got unexpected response from host: %d %s" %
(res.status_code, res.content))
try:
daemon_allow = set(res.headers.get("Allow").split(","))
except KeyError:
raise exc.HTTPInternalServerError(
"Got invalid response from host: missing Allow header")
try:
daemon_features = set(res.json()["features"])
except (ValueError, KeyError):
raise exc.HTTPInternalServerError(
"Got invalid response from host: "
"invalid JSON or missing 'features'")
allow = allow.intersection(daemon_allow)
features = features.intersection(daemon_features)
return web.response(payload={"features": list(features)},
allow=','.join(allow))
def patch(self, res_id):
"""
Proxy PATCH request to daemon.
"""
if not self.request.content_length:
raise exc.HTTPBadRequest("Content-Length is required")
logging.info("[%s] PATCH ticket=%s",
web.client_address(self.request), self.ticket.id)
# Notes:
# - PATCH response is not cachable, no need for cache-control.
# - We cannot have read_timeout since PATCH can take unpredictable
# time, depending on size of the modified byte range, and the storage
# capabillties.
res = self.make_imaged_request(
"PATCH",
self.get_imaged_url(self.ticket),
self.request.headers,
web.CappedStream(self.request.body_file,
self.request.content_length),
False,
connection_timeout=self.config.imaged_connection_timeout_sec)
# TODO: We expect empty response from the daemon. If we start to return
# non-empty response, this must be changed to stream the daemon
# response to the caller.
return web.response(res.status_code)
def options(self, ticket_id):
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
if ticket_id == "*":
# Reporting the meta-capabilities for all images.
allow = ["OPTIONS", "GET", "PUT", "PATCH"]
features = ["zero", "flush"]
else:
# Reporting real image capabilities per ticket.
try:
ticket = tickets.get(ticket_id)
except KeyError:
raise HTTPForbidden("No such ticket %r" % ticket_id)
# Accessing ticket options considered as client activity.
ticket.touch()
allow = ["OPTIONS"]
features = []
if ticket.may("read"):
allow.append("GET")
if ticket.may("write"):
allow.extend(("PUT", "PATCH"))
features = ["zero", "flush"]
return web.response(payload={
"features": features,
"unix_socket": self.config.images.socket,
},
allow=",".join(allow))
def patch(self, res_id):
"""
Proxy PATCH request to daemon.
"""
if not self.request.content_length:
raise exc.HTTPBadRequest("Content-Length is required")
# Notes:
# - PATCH response is not cachable, no need for cache-control.
# - We cannot have read_timeout since PATCH can take unpredictable
# time, depending on size of the modified byte range, and the storage
# capabillties.
res = self.make_imaged_request(
"PATCH",
self.get_imaged_url(self.ticket),
self.request.headers,
web.CappedStream(self.request.body_file,
self.request.content_length),
False,
connection_timeout=self.config.imaged_connection_timeout_sec)
# TODO: We expect empty response from the daemon. If we start to return
# non-empty response, this must be changed to stream the daemon
# response to the caller.
return web.response(res.status_code)
def put(self, ticket_id):
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
size = self.request.content_length
if size is None:
raise HTTPBadRequest("Content-Length header is required")
if size < 0:
raise HTTPBadRequest("Invalid Content-Length header: %r" % size)
content_range = web.content_range(self.request)
offset = content_range.start or 0
# For backward compatibility, we flush by default.
flush = validate.enum(self.request.params,
"flush", ("y", "n"),
default="y")
flush = (flush == "y")
ticket = tickets.authorize(ticket_id, "write", offset, size)
# TODO: cancel copy if ticket expired or revoked
self.log.info(
"Writing %d bytes at offset %d flush %s to %s for ticket %s", size,
offset, flush, ticket.url.path, ticket_id)
op = directio.Receive(ticket.url.path,
self.request.body_file_raw,
size,
offset=offset,
flush=flush,
buffersize=self.config.daemon.buffer_size,
clock=self.clock)
try:
ticket.run(op)
except errors.PartialContent as e:
raise HTTPBadRequest(str(e))
return web.response()
def patch(self, ticket_id):
# TODO: restart expire timer
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
try:
patch = self.request.json
except ValueError as e:
raise HTTPBadRequest("Invalid patch: %s" % e)
try:
timeout = patch["timeout"]
except KeyError:
raise HTTPBadRequest("Missing timeout key")
try:
timeout = int(timeout)
except ValueError as e:
raise HTTPBadRequest("Invalid timeout value: %s" % e)
try:
ticket = tickets.get(ticket_id)
except KeyError:
raise HTTPNotFound("No such ticket: %s" % ticket_id)
self.log.info("[%s] EXTEND timeout=%s ticket=%s",
web.client_address(self.request), timeout, ticket_id)
ticket.extend(timeout)
return web.response()
def options(self, ticket_id):
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
self.log.info("[%s] OPTIONS ticket=%s",
web.client_address(self.request), ticket_id)
if ticket_id == "*":
# Reporting the meta-capabilities for all images.
allow = ["OPTIONS", "GET", "PUT", "PATCH"]
features = ["zero", "flush"]
else:
# Reporting real image capabilities per ticket.
# This check will fail if the ticket has expired.
ticket = tickets.authorize(ticket_id, "read", 0, 0)
# Accessing ticket options considered as client activity.
ticket.touch()
allow = ["OPTIONS"]
features = []
if ticket.may("read"):
allow.append("GET")
if ticket.may("write"):
allow.extend(("PUT", "PATCH"))
features = ["zero", "flush"]
return web.response(payload={
"features": features,
"unix_socket": self.config.images.socket,
},
allow=",".join(allow))
def _flush(self, ticket_id, msg):
ticket = tickets.authorize(ticket_id, "write", 0, 0)
self.log.info("[%s] FLUSH ticket=%s", web.client_address(self.request),
ticket_id)
op = directio.Flush(ticket.url.path, clock=self.clock)
ticket.run(op)
return web.response()
def get(self, ticket_id):
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
try:
ticket = tickets.get(ticket_id)
except KeyError:
raise HTTPNotFound("No such ticket %r" % ticket_id)
self.log.info("Retrieving ticket %s", ticket_id)
return web.response(payload=ticket.info())
def put(self, ticket_id=None):
"""
Verify and add a signed_ticket, allowing transfer to /images/ticket_id.
"""
try:
auth.add_signed_ticket(self.request.body)
except auth.Error as e:
raise exc.HTTPForbidden("Error verifying signed ticket: %s" % e)
return web.response()
def get(self, ticket_id):
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
try:
ticket = tickets.get(ticket_id)
except KeyError:
raise HTTPNotFound("No such ticket %r" % ticket_id)
ticket_info = ticket.info()
self.log.debug("[%s] GET ticket=%s", web.client_address(self.request),
ticket_info)
return web.response(payload=ticket_info)
def delete(self, ticket_id=None):
"""
Delete ticket by a specified ticket_id
"""
if not ticket_id:
raise exc.HTTPBadRequest("Missing ticket ID")
try:
auth.delete_ticket(ticket_id)
except auth.NoSuchTicket as e:
raise exc.HTTPNotFound("Ticket not found: %s" % e)
return web.response(http_client.NO_CONTENT)
def post(self):
"""
Start of stop the profiler.
"""
if yappi is None:
raise HTTPNotFound("yappi is not installed")
run = validate.enum(self.request.params, "run", ("y", "n"))
if run == "y":
clock = validate.enum(self.request.params,
"clock", ("cpu", "wall"),
default="cpu")
self._start_profiling(clock)
else:
self._stop_profiling()
return web.response()
def delete(self, ticket_id):
"""
Delete a ticket if exists.
Note that DELETE is idempotent; the client can issue multiple DELETE
requests in case of network failures. See
https://tools.ietf.org/html/rfc7231#section-4.2.2.
"""
# TODO: cancel requests using deleted tickets
if ticket_id:
try:
tickets.remove(ticket_id)
except KeyError:
log.debug("Ticket %s does not exists", ticket_id)
else:
tickets.clear()
return web.response(status=204)
def put(self, ticket_id):
# TODO
# - reject invalid or expired ticket
# - start expire timer
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
try:
ticket_dict = self.request.json
except ValueError as e:
raise HTTPBadRequest("Ticket is not in a json format: %s" % e)
try:
tickets.add(ticket_dict)
except errors.InvalidTicket as e:
raise HTTPBadRequest("Invalid ticket: %s" % e)
return web.response()
def _zero(self, ticket_id, msg):
size = validate.integer(msg, "size", minval=0)
offset = validate.integer(msg, "offset", minval=0, default=0)
flush = validate.boolean(msg, "flush", default=False)
ticket = tickets.authorize(ticket_id, "write", offset, size)
self.log.info(
"Zeroing %d bytes at offset %d flush %s to %s for ticket %s", size,
offset, flush, ticket.url.path, ticket_id)
op = directio.Zero(ticket.url.path,
size,
offset=offset,
flush=flush,
buffersize=self.config.daemon.buffer_size,
clock=self.clock)
try:
ticket.run(op)
except errors.PartialContent as e:
raise HTTPBadRequest(str(e))
return web.response()
def put(self, res_id):
""" Handles sending data to host for PUT or PATCH.
:param request: http request object
:type request: webob.Request
:return: http response object
:rtype: webob.Response
"""
imaged_url = self.get_imaged_url(self.ticket)
if "flush" in self.request.params:
imaged_url += "?flush=" + self.request.params["flush"]
headers = self.get_default_headers(res_id)
if 'Content-Length' not in self.request.headers:
raise exc.HTTPBadRequest("Content-Length header is required")
headers['Content-Length'] = self.request.headers['Content-Length']
if 'Content-Range' in self.request.headers:
headers['Content-Range'] = self.request.headers['Content-Range']
max_transfer_bytes = int(headers['Content-Length'])
body = web.CappedStream(self.request.body_file, max_transfer_bytes)
stream = False
content_range = web.content_range(self.request)
offset = content_range.start or 0
flush = self.request.params.get("flush", "y") == "y"
logging.info("[%s] WRITE size=%d offset=%d flush=%s ticket=%s",
web.client_address(self.request), max_transfer_bytes, offset,
flush, self.ticket.id)
imaged_response = self.make_imaged_request(
self.request.method, imaged_url, headers, body, stream,
connection_timeout=self.config.imaged_connection_timeout_sec,
read_timeout=self.config.imaged_read_timeout_sec)
response = web.response(imaged_response.status_code)
response.headers['Cache-Control'] = 'no-cache, no-store'
return response
def get(self, res_id):
imaged_url = self.get_imaged_url(self.ticket)
headers = self.get_default_headers(res_id)
# Note that webob request.headers is case-insensitive.
if 'Range' in self.request.headers:
headers['Range'] = self.request.headers['Range']
body = ""
stream = True # Don't let Requests read entire body into memory
logging.info("[%s] READ ticket=%s",
web.client_address(self.request), self.ticket.id)
imaged_response = self.make_imaged_request(
self.request.method, imaged_url, headers, body, stream,
connection_timeout=self.config.imaged_connection_timeout_sec,
read_timeout=self.config.imaged_read_timeout_sec)
response = web.response(imaged_response.status_code)
response.headers['Cache-Control'] = 'no-cache, no-store'
response.headers['Content-Range'] = \
imaged_response.headers.get('Content-Range', '')
disposition = imaged_response.headers.get('Content-Disposition')
if disposition is not None:
response.headers['Content-Disposition'] = disposition
max_transfer_bytes = int(imaged_response.headers.get('Content-Length'))
response.body_file = web.CappedStream(RequestStreamAdapter(
imaged_response.iter_content(4096, False)),
max_transfer_bytes)
response.headers['Content-Length'] = str(max_transfer_bytes)
logging.debug("Resource %s: transferring %d bytes from host",
res_id, max_transfer_bytes)
return response
def get(self, res_id):
imaged_url = self.get_imaged_url(self.ticket)
headers = self.get_default_headers(res_id)
# Note that webob request.headers is case-insensitive.
if 'Range' in self.request.headers:
headers['Range'] = self.request.headers['Range']
body = ""
stream = True # Don't let Requests read entire body into memory
imaged_response = self.make_imaged_request(
self.request.method,
imaged_url,
headers,
body,
stream,
connection_timeout=self.config.imaged_connection_timeout_sec,
read_timeout=self.config.imaged_read_timeout_sec)
response = web.response(imaged_response.status_code)
response.headers['Cache-Control'] = 'no-cache, no-store'
response.headers['Content-Range'] = \
imaged_response.headers.get('Content-Range', '')
disposition = imaged_response.headers.get('Content-Disposition')
if disposition is not None:
response.headers['Content-Disposition'] = disposition
max_transfer_bytes = int(imaged_response.headers.get('Content-Length'))
response.body_file = web.CappedStream(
RequestStreamAdapter(imaged_response.iter_content(4096, False)),
max_transfer_bytes)
response.headers['Content-Length'] = str(max_transfer_bytes)
logging.debug("Resource %s: transferring %d bytes from host", res_id,
max_transfer_bytes)
return response
def put(self, res_id):
""" Handles sending data to host for PUT or PATCH.
:param request: http request object
:type request: webob.Request
:return: http response object
:rtype: webob.Response
"""
imaged_url = self.get_imaged_url(self.ticket)
if "flush" in self.request.params:
imaged_url += "?flush=" + self.request.params["flush"]
headers = self.get_default_headers(res_id)
if 'Content-Length' not in self.request.headers:
raise exc.HTTPBadRequest("Content-Length header is required")
headers['Content-Length'] = self.request.headers['Content-Length']
if 'Content-Range' in self.request.headers:
headers['Content-Range'] = self.request.headers['Content-Range']
max_transfer_bytes = int(headers['Content-Length'])
body = web.CappedStream(self.request.body_file, max_transfer_bytes)
stream = False
logging.debug("Resource %s: transferring %d bytes to host", res_id,
max_transfer_bytes)
imaged_response = self.make_imaged_request(
self.request.method,
imaged_url,
headers,
body,
stream,
connection_timeout=self.config.imaged_connection_timeout_sec,
read_timeout=self.config.imaged_read_timeout_sec)
response = web.response(imaged_response.status_code)
response.headers['Cache-Control'] = 'no-cache, no-store'
return response
def options(self, res_id):
return web.response(httplib.NO_CONTENT)
def get(self):
return web.response(payload={'version': version.string})
def _flush(self, ticket_id, msg):
ticket = tickets.authorize(ticket_id, "write", 0, 0)
self.log.info("Flushing %s for ticket %s", ticket.url.path, ticket_id)
op = directio.Flush(ticket.url.path, clock=self.clock)
ticket.run(op)
return web.response()
def get(self):
return web.response(payload={'version': version.string})
def get(self):
if yappi is None:
raise HTTPNotFound("yappi is not installed")
return web.response(payload={"running": yappi.is_running()})
