def finish(self):
self.headers['X-Cocaine-Application'] = self.name
fill_response_in(
self.request, self.code,
httplib.responses.get(self.code, httplib.OK),
''.join(self.messages), self.headers)
def process(self, request):
try:
payload = json.loads(request.body)
except ValueError:
JSONRPC._send_400_error(
request, -32700,
'Parse error: Invalid JSON was received by the server.')
return
if not all(k in payload for k in REQUEST_FIELDS):
JSONRPC._send_400_error(
request, -32600,
'The JSON sent is not a valid Request object.')
return
name, method = payload['method'].split('.', 2)
args = payload['params']
chunks = payload.get('chunks', [])
headers = {}
if 'authorization' in request.headers:
headers['authorization'] = request.headers['authorization']
try:
service = yield self.proxy.get_service(name, request)
if service is None:
JSONRPC._send_500_error(request, payload, 'Service not found.')
return
except Exception as err:
JSONRPC._send_500_error(request, payload, err)
return
api = dict((data[0], data) for data in service.api.itervalues())
if method not in api:
JSONRPC._send_400_error(request, -32601, 'Method not found.')
return
name, tx_tree, rx_tree = api[method]
try:
for match, handle in self._protocols:
if match(tx_tree, rx_tree):
result = yield handle(service, method, args, chunks,
**headers)
break
else:
JSONRPC._send_400_error(request, -32000,
'Protocol type is not supported.')
return
except Exception as err:
JSONRPC._send_500_error(request, payload, err)
return
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/json-rpc'})
body = {
'jsonrpc': '2.0',
'result': result,
'id': payload['id'],
}
fill_response_in(request, 200, 'OK', json.dumps(body), headers)
def wrapper(self, request):
self.requests_in_progress += 1
self.requests_total += 1
traceid = None
try:
generated_traceid = self.get_request_id(request)
if generated_traceid is not None:
# assume we have hexdigest form of number
# get only 16 digits
traceid = generated_traceid[:16]
adaptor = ContextAdapter(self.access_log, {"trace_id": traceid})
request.logger = adaptor
# verify user input: request_header must be valid hexdigest
try:
int(traceid, 16)
except ValueError:
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
"Request-Id `%s` is not a hexdigest" % traceid,
proxy_error_headers())
return
else:
request.logger = NULLLOGGER # pylint: disable=R0204
request.traceid = traceid
request.tracebit = True
request.logger.info("start request: %s %s %s", request.host, request.remote_ip, request.uri)
yield func(self, request)
finally:
self.requests_in_progress -= 1
def wrapper(self, request):
self.requests_in_progress += 1
self.requests_total += 1
traceid = None
try:
generated_traceid = self.get_request_id(request)
if generated_traceid is not None:
# assume we have hexdigest form of number
# get only 16 digits
traceid = generated_traceid[:16]
adaptor = ContextAdapter(self.access_log,
{"trace_id": traceid})
request.logger = adaptor
# verify user input: request_header must be valid hexdigest
try:
int(traceid, 16)
except ValueError:
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
"Request-Id `%s` is not a hexdigest" % traceid,
proxy_error_headers())
return
else:
request.logger = NULLLOGGER # pylint: disable=R0204
request.traceid = traceid
request.logger.info("start request: %s %s %s", request.host,
request.remote_ip, request.uri)
yield func(self, request)
finally:
self.requests_in_progress -= 1
def on_error(app, err, extra_msg, code=httplib.INTERNAL_SERVER_ERROR):
if len(extra_msg) > 0 and not extra_msg.endswith(' '):
extra_msg += ' '
request.logger.error("%s: %s%s", app.id, extra_msg, err)
message = "UID %s: application `%s` error: %s" % (request.traceid, app.name, str(err))
fill_response_in(request, code, httplib.responses[code], message, proxy_error_headers(app.name))
def ping(self, request):
if self.locator_status:
fill_response_in(request, httplib.OK, "OK", "OK")
return
fill_response_in(request, httplib.SERVICE_UNAVAILABLE,
httplib.responses[httplib.SERVICE_UNAVAILABLE],
"Failed", proxy_error_headers())
def ping(self, request):
if self.locator_status:
fill_response_in(request, httplib.OK, "OK", "OK")
return
fill_response_in(request, httplib.SERVICE_UNAVAILABLE,
httplib.responses[httplib.SERVICE_UNAVAILABLE],
"Failed", proxy_error_headers())
def on_error(app, err, extra_msg, code=httplib.INTERNAL_SERVER_ERROR):
if len(extra_msg) > 0 and not extra_msg.endswith(' '):
extra_msg += ' '
request.logger.error("%s: %s%s", app.id, extra_msg, err)
message = "UID %s: application `%s` error: %s" % (
request.traceid, app.name, str(err))
fill_response_in(request, code, httplib.responses[code], message,
proxy_error_headers(app.name))
def __init__(self, request, name, code, headers):
super(ChunkedBodyProcessor, self).__init__(
request, name, code, headers)
self.headers.add('Transfer-Encoding', 'chunked')
self.headers['X-Cocaine-Application'] = self.name
fill_response_in(self.request, self.code,
httplib.responses.get(self.code, httplib.OK),
''.join(self.messages), self.headers, chunked=True)
def _send_500_error(request, payload, err):
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/json-rpc'})
body = {
'jsonrpc': '2.0',
'error': str(err),
'id': payload['id'],
}
fill_response_in(request, 500, 'Internal Server Error',
json.dumps(body), headers)
def __init__(self, request, name, code, headers):
super(ChunkedBodyProcessor, self).__init__(request, name, code,
headers)
self.headers.add('Transfer-Encoding', 'chunked')
self.headers['X-Cocaine-Application'] = self.name
fill_response_in(self.request,
self.code,
httplib.responses.get(self.code, httplib.OK),
''.join(self.messages),
self.headers,
chunked=True)
def process(self, request):
name, event = extract_app_and_event(request)
timeout = self.proxy.get_timeout(name, event)
# as MDS proxy bypasses the mechanism of routing groups
# the proxy is responsible to provide this feature
name = self.proxy.resolve_group_to_version(name)
headers = request.headers
namespace = headers["X-Srw-Namespace"]
key = headers["X-Srw-Key"]
mds_request_headers = httputil.HTTPHeaders()
if "Authorization" in request.headers:
mds_request_headers["Authorization"] = request.headers["Authorization"]
traceid = getattr(request, "traceid", None)
if traceid is not None:
mds_request_headers["X-Request-Id"] = traceid
srw_request = HTTPRequest("%s/exec-%s/%s/%s/stid/%s?timeout=%d" % (self.srw_host, namespace, name, event, key, timeout),
method="POST",
headers=mds_request_headers,
body=msgpack.packb(pack_httprequest(request)),
allow_ipv6=True,
request_timeout=timeout)
try:
# NOTE: we can do it in a streaming way
resp = yield self.srw_httpclient.fetch(srw_request)
code, reply_headers, body = decode_chunked_encoded_reply(resp)
fill_response_in(request, code,
httplib.responses.get(code, httplib.OK),
body, reply_headers)
except HTTPError as err:
if err.code == 404:
raise PluginNoSuchApplication("worker was not found")
if err.code == 500:
raise PluginApplicationError(42, 42, "worker replied with error")
if err.code == 401:
fill_response_in(request, err.code,
httplib.responses.get(err.code, httplib.OK),
err.response.body, err.response.headers)
return
raise err
def fetch_mds_endpoints(self, request, srw_request):
try:
# NOTE: we can do it in a streaming way
resp = yield self.srw_httpclient.fetch(srw_request)
body = resp.buffer.read(None)
if self.is_stid_request(request):
raise gen.Return(self.decode_mulca_dist_info(body))
else:
raise gen.Return(self.decode_mds_dist_info(body))
except HTTPError as err:
if err.code == 404:
raise PluginNoSuchApplication("404")
if err.code == 500:
raise PluginApplicationError(42, 42, "500")
if err.code == 401:
fill_response_in(request, err.code,
httplib.responses.get(err.code, httplib.OK),
err.response.body, err.response.headers)
return
raise err
def fetch_mds_endpoints(self, request, srw_request):
try:
# NOTE: we can do it in a streaming way
resp = yield self.srw_httpclient.fetch(srw_request)
body = resp.buffer.read(None)
if self.is_stid_request(request):
raise gen.Return(self.decode_mulca_dist_info(body))
else:
raise gen.Return(self.decode_mds_dist_info(body))
except HTTPError as err:
if err.code == 404:
raise PluginNoSuchApplication("404")
if err.code == 500:
raise PluginApplicationError(42, 42, "500")
if err.code == 401:
fill_response_in(request, err.code,
httplib.responses.get(err.code, httplib.OK),
err.response.body, err.response.headers)
return
raise err
def __call__(self, request):
for plugin in self.plugins:
if plugin.match(request):
request.logger.info('processed by %s plugin', plugin.name())
try:
yield plugin.process(request)
except PluginNoSuchApplication as err:
fill_response_in(request, NO_SUCH_APP, "No such application",
str(err), proxy_error_headers())
except PluginApplicationError:
message = "application error"
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers())
except ProxyInvalidRequest:
if request.path == "/ping":
self.ping(request)
else:
fill_response_in(request, httplib.NOT_FOUND, httplib.responses[httplib.NOT_FOUND],
"Invalid url", proxy_error_headers())
except Exception as err:
request.logger.exception('plugin %s returned error: %s', plugin.name(), err)
message = "unknown error"
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers())
return
try:
name, event = extract_app_and_event(request)
except ProxyInvalidRequest:
if request.path == "/ping":
self.ping(request)
else:
fill_response_in(request, httplib.NOT_FOUND, httplib.responses[httplib.NOT_FOUND],
"Invalid url", proxy_error_headers())
return
self.setup_tracing(request, name)
if self.sticky_header in request.headers:
seed = request.headers.get(self.sticky_header)
seed_value = header_to_seed(seed)
request.logger.info('sticky_header has been found: name %s, value %s, seed %d', name, seed, seed_value)
name = self.resolve_group_to_version(name, seed_value)
app = yield self.get_service(name, request)
if app is None:
message = "current application %s is unavailable" % name
fill_response_in(request, NO_SUCH_APP, "No Such Application",
message, proxy_error_headers(name))
return
try:
# TODO: attempts should be configurable
yield self.process(request, name, app, event, pack_httprequest(request), self.reelect_app, 2)
except Exception as err:
request.logger.exception("error during processing request %s", err)
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
"UID %s: %s" % (request.traceid, str(err)), proxy_error_headers(name))
request.logger.info("exit from process")
def _send_400_error(request, code, message):
headers = httputil.HTTPHeaders(
{'Content-Type': 'application/json-rpc'})
body = {'code': code, 'message': message}
fill_response_in(request, 400, 'Bad JSON-RPC request',
json.dumps(body), headers)
def __call__(self, request):
for plugin in self.plugins:
if plugin.match(request):
request.logger.info('processed by %s plugin', plugin.name())
try:
yield plugin.process(request)
except PluginNoSuchApplication as err:
fill_response_in(request, NO_SUCH_APP, "No such application",
str(err), proxy_error_headers())
except PluginApplicationError:
message = "application error"
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers())
except ProxyInvalidRequest:
if request.path == "/ping":
self.ping(request)
else:
fill_response_in(request, httplib.NOT_FOUND, httplib.responses[httplib.NOT_FOUND],
"Invalid url", proxy_error_headers())
except Exception as err:
request.logger.exception('plugin %s returned error: %s', plugin.name(), err)
message = "unknown error"
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers())
return
try:
name, event = extract_app_and_event(request)
except ProxyInvalidRequest:
if request.path == "/ping":
self.ping(request)
else:
fill_response_in(request, httplib.NOT_FOUND, httplib.responses[httplib.NOT_FOUND],
"Invalid url", proxy_error_headers())
return
if getattr(request, "traceid", None) is not None:
tracing_chance = self.sampled_apps.get(name, self.default_tracing_chance)
rolled_dice = random.uniform(0, 100)
request.logger.debug("tracing_chance %f, rolled dice %f", tracing_chance, rolled_dice)
if tracing_chance < rolled_dice:
request.logger.info('stop tracing the request')
request.logger = NULLLOGGER
request.traceid = None
if self.sticky_header in request.headers:
seed = request.headers.get(self.sticky_header)
seed_value = header_to_seed(seed)
request.logger.info('sticky_header has been found: name %s, value %s, seed %d', name, seed, seed_value)
name = self.resolve_group_to_version(name, seed_value)
app = yield self.get_service(name, request)
if app is None:
message = "current application %s is unavailable" % name
fill_response_in(request, NO_SUCH_APP, "No Such Application",
message, proxy_error_headers(name))
return
try:
yield self.process(request, name, app, event, pack_httprequest(request), self.reelect_app)
except Exception as err:
request.logger.exception("error during processing request %s", err)
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
"UID %s: %s" % (request.traceid, str(err)), proxy_error_headers(name))
request.logger.info("exit from process")
def process(self,
request,
name,
app,
event,
data,
reelect_app_fn,
timeout=None):
if timeout is None:
timeout = self.get_timeout(name, event)
request.logger.info(
"start processing event `%s` for an app `%s` (appid: %s) after %.3f ms with timeout %f",
event, app.name, app.id,
request.request_time() * 1000, timeout)
# allow to reconnect this amount of times.
attempts = 2 # make it configurable
parentid = 0
if request.traceid is not None:
traceid = int(request.traceid, 16)
trace = Trace(traceid=traceid, spanid=traceid, parentid=parentid)
else:
trace = None
headers = {}
if 'authorization' in request.headers:
headers['authorization'] = request.headers['authorization']
while attempts > 0:
body_parts = []
attempts -= 1
try:
request.logger.debug("%s: enqueue event (attempt %d)", app.id,
attempts)
channel = yield app.enqueue(event, trace=trace, **headers)
request.logger.debug("%s: send event data (attempt %d)",
app.id, attempts)
yield channel.tx.write(msgpack.packb(data), trace=trace)
yield channel.tx.close(trace=trace)
request.logger.debug(
"%s: waiting for a code and headers (attempt %d)", app.id,
attempts)
code_and_headers = yield channel.rx.get(timeout=timeout)
request.logger.debug(
"%s: code and headers have been received (attempt %d)",
app.id, attempts)
code, raw_headers = msgpack.unpackb(code_and_headers)
headers = httputil.HTTPHeaders(raw_headers)
cocaine_http_proto_version = headers.get(
X_COCAINE_HTTP_PROTO_VERSION)
if cocaine_http_proto_version is None or cocaine_http_proto_version == "1.0":
cocaine_http_proto_version = "1.0"
def stop_condition(body):
return isinstance(body, EmptyResponse)
elif cocaine_http_proto_version == "1.1":
def stop_condition(body):
return isinstance(body,
EmptyResponse) or len(body) == 0
else:
raise Exception(
"unsupported X-Cocaine-HTTP-Proto-Version: %s" %
cocaine_http_proto_version)
while True:
body = yield channel.rx.get(timeout=timeout)
if stop_condition(body):
request.logger.info("%s: body finished (attempt %d)",
app.id, attempts)
break
request.logger.debug(
"%s: received %d bytes as a body chunk (attempt %d)",
app.id, len(body), attempts)
body_parts.append(body)
except gen.TimeoutError as err:
request.logger.error("%s %s: %s", app.id, name, err)
message = "UID %s: application `%s` error: TimeoutError" % (
request.traceid, name)
fill_response_in(request, httplib.GATEWAY_TIMEOUT,
httplib.responses[httplib.GATEWAY_TIMEOUT],
message, proxy_error_headers(name))
except (DisconnectionError, StreamClosedError) as err:
self.requests_disconnections += 1
# Probably it's dangerous to retry requests all the time.
# I must find the way to determine whether it failed during writing
# or reading a reply. And retry only writing fails.
request.logger.error("%s: %s", app.id, err)
if attempts <= 0:
request.logger.error("%s: no more attempts", app.id)
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
"UID %s: Connection problem" % request.traceid,
proxy_error_headers(name))
return
# Seems on_close callback is not called in case of connecting through IPVS
# We detect disconnection here to avoid unnecessary errors.
# Try to reconnect here and give the request a go
try:
start_time = time.time()
reconn_timeout = timeout - request.request_time()
request.logger.info("%s: connecting with timeout %.fms",
app.id, reconn_timeout * 1000)
yield gen.with_timeout(start_time + reconn_timeout,
app.connect(request.traceid))
reconn_time = time.time() - start_time
request.logger.info("%s: connecting took %.3fms", app.id,
reconn_time * 1000)
except Exception as err:
if attempts <= 0:
# we have no attempts more, so quit here
request.logger.error("%s: %s (no attempts left)",
app.id, err)
message = "UID %s: application `%s` error: %s" % (
request.traceid, name, str(err))
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers(name))
return
request.logger.error(
"%s: unable to reconnect: %s (%d attempts left)", err,
attempts)
# We have an attempt to process request again.
# Jump to the begining of `while attempts > 0`, either we connected successfully
# or we were failed to connect
continue
except ServiceError as err:
# if the application has been restarted, we get broken pipe code
# and system category
if err.category in SYSTEMCATEGORY and err.code == EAPPSTOPPED:
request.logger.error(
"%s: the application has been restarted", app.id)
app.disconnect()
continue
elif err.category in OVERSEERCATEGORY and err.code == EQUEUEISFULL:
request.logger.error(
"%s: queue is full. Pick another application instance",
app.id)
app = yield reelect_app_fn(request, app)
continue
request.logger.error("%s: service error: [%d, %d] %s", app.id,
err.category, err.code, err.reason)
message = "UID %s: application `%s` error: %s" % (
request.traceid, name, str(err))
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR], message,
proxy_error_headers(name))
except Exception as err:
request.logger.exception("%s: %s", app.id, err)
message = "UID %s: unknown `%s` error: %s" % (request.traceid,
name, str(err))
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR], message,
proxy_error_headers(name))
else:
message = ''.join(body_parts)
headers['X-Cocaine-Application'] = name
fill_response_in(request, code,
httplib.responses.get(code, httplib.OK),
message, headers)
# to return from all errors except Disconnection
# or receiving a good reply
return
def __call__(self, request):
for plugin in self.plugins:
if plugin.match(request):
request.logger.info('processed by %s plugin', plugin.name())
try:
yield plugin.process(request)
except PluginNoSuchApplication as err:
fill_response_in(request, NO_SUCH_APP,
"No such application", str(err),
proxy_error_headers())
except PluginApplicationError:
message = "application error"
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers())
except ProxyInvalidRequest:
if request.path == "/ping":
self.ping(request)
else:
fill_response_in(request, httplib.NOT_FOUND,
httplib.responses[httplib.NOT_FOUND],
"Invalid url", proxy_error_headers())
except Exception as err:
request.logger.exception('plugin %s returned error: %s',
plugin.name(), err)
message = "unknown error"
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers())
return
try:
name, event = extract_app_and_event(request)
except ProxyInvalidRequest:
if request.path == "/ping":
self.ping(request)
else:
fill_response_in(request, httplib.NOT_FOUND,
httplib.responses[httplib.NOT_FOUND],
"Invalid url", proxy_error_headers())
return
if getattr(request, "traceid", None) is not None:
tracing_chance = self.sampled_apps.get(name,
self.default_tracing_chance)
rolled_dice = random.uniform(0, 100)
request.logger.debug("tracing_chance %f, rolled dice %f",
tracing_chance, rolled_dice)
if tracing_chance < rolled_dice:
request.logger.info('stop tracing the request')
request.logger = NULLLOGGER
request.traceid = None
if self.sticky_header in request.headers:
seed = request.headers.get(self.sticky_header)
seed_value = header_to_seed(seed)
request.logger.info(
'sticky_header has been found: name %s, value %s, seed %d',
name, seed, seed_value)
name = self.resolve_group_to_version(name, seed_value)
app = yield self.get_service(name, request)
if app is None:
message = "current application %s is unavailable" % name
fill_response_in(request, NO_SUCH_APP, "No Such Application",
message, proxy_error_headers(name))
return
try:
yield self.process(request, name, app, event,
pack_httprequest(request), self.reelect_app)
except Exception as err:
request.logger.exception("error during processing request %s", err)
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
"UID %s: %s" % (request.traceid, str(err)),
proxy_error_headers(name))
request.logger.info("exit from process")
def process(self, request, name, app, event, data, reelect_app_fn, timeout=None):
if timeout is None:
timeout = self.get_timeout(name, event)
request.logger.info("start processing event `%s` for an app `%s` (appid: %s) after %.3f ms with timeout %f",
event, app.name, app.id, request.request_time() * 1000, timeout)
# allow to reconnect this amount of times.
attempts = 2 # make it configurable
parentid = 0
if request.traceid is not None:
traceid = int(request.traceid, 16)
trace = Trace(traceid=traceid, spanid=traceid, parentid=parentid)
else:
trace = None
headers = {}
if 'authorization' in request.headers:
headers['authorization'] = request.headers['authorization']
while attempts > 0:
body_parts = []
attempts -= 1
try:
request.logger.debug("%s: enqueue event (attempt %d)", app.id, attempts)
channel = yield app.enqueue(event, trace=trace, **headers)
request.logger.debug("%s: send event data (attempt %d)", app.id, attempts)
yield channel.tx.write(msgpack.packb(data), trace=trace)
yield channel.tx.close(trace=trace)
request.logger.debug("%s: waiting for a code and headers (attempt %d)",
app.id, attempts)
code_and_headers = yield channel.rx.get(timeout=timeout)
request.logger.debug("%s: code and headers have been received (attempt %d)",
app.id, attempts)
code, raw_headers = msgpack.unpackb(code_and_headers)
headers = httputil.HTTPHeaders(raw_headers)
cocaine_http_proto_version = headers.get(X_COCAINE_HTTP_PROTO_VERSION)
if cocaine_http_proto_version is None or cocaine_http_proto_version == "1.0":
cocaine_http_proto_version = "1.0"
def stop_condition(body):
return isinstance(body, EmptyResponse)
elif cocaine_http_proto_version == "1.1":
def stop_condition(body):
return isinstance(body, EmptyResponse) or len(body) == 0
else:
raise Exception("unsupported X-Cocaine-HTTP-Proto-Version: %s" % cocaine_http_proto_version)
while True:
body = yield channel.rx.get(timeout=timeout)
if stop_condition(body):
request.logger.info("%s: body finished (attempt %d)", app.id, attempts)
break
request.logger.debug("%s: received %d bytes as a body chunk (attempt %d)",
app.id, len(body), attempts)
body_parts.append(body)
except gen.TimeoutError as err:
request.logger.error("%s %s: %s", app.id, name, err)
message = "UID %s: application `%s` error: TimeoutError" % (request.traceid, name)
fill_response_in(request, httplib.GATEWAY_TIMEOUT,
httplib.responses[httplib.GATEWAY_TIMEOUT],
message, proxy_error_headers(name))
except (DisconnectionError, StreamClosedError) as err:
self.requests_disconnections += 1
# Probably it's dangerous to retry requests all the time.
# I must find the way to determine whether it failed during writing
# or reading a reply. And retry only writing fails.
request.logger.error("%s: %s", app.id, err)
if attempts <= 0:
request.logger.error("%s: no more attempts", app.id)
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
"UID %s: Connection problem" % request.traceid,
proxy_error_headers(name))
return
# Seems on_close callback is not called in case of connecting through IPVS
# We detect disconnection here to avoid unnecessary errors.
# Try to reconnect here and give the request a go
try:
start_time = time.time()
reconn_timeout = timeout - request.request_time()
request.logger.info("%s: connecting with timeout %.fms", app.id, reconn_timeout * 1000)
yield gen.with_timeout(start_time + reconn_timeout, app.connect(request.traceid))
reconn_time = time.time() - start_time
request.logger.info("%s: connecting took %.3fms", app.id, reconn_time * 1000)
except Exception as err:
if attempts <= 0:
# we have no attempts more, so quit here
request.logger.error("%s: %s (no attempts left)", app.id, err)
message = "UID %s: application `%s` error: %s" % (request.traceid, name, str(err))
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers(name))
return
request.logger.error("%s: unable to reconnect: %s (%d attempts left)",
err, attempts)
# We have an attempt to process request again.
# Jump to the begining of `while attempts > 0`, either we connected successfully
# or we were failed to connect
continue
except ServiceError as err:
# if the application has been restarted, we get broken pipe code
# and system category
if err.category in SYSTEMCATEGORY and err.code == EAPPSTOPPED:
request.logger.error("%s: the application has been restarted", app.id)
app.disconnect()
continue
elif err.category in OVERSEERCATEGORY and err.code == EQUEUEISFULL:
request.logger.error("%s: queue is full. Pick another application instance", app.id)
app = yield reelect_app_fn(request, app)
continue
request.logger.error("%s: service error: [%d, %d] %s", app.id, err.category, err.code, err.reason)
message = "UID %s: application `%s` error: %s" % (request.traceid, name, str(err))
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers(name))
except Exception as err:
request.logger.exception("%s: %s", app.id, err)
message = "UID %s: unknown `%s` error: %s" % (request.traceid, name, str(err))
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers(name))
else:
message = ''.join(body_parts)
headers['X-Cocaine-Application'] = name
fill_response_in(request, code,
httplib.responses.get(code, httplib.OK),
message, headers)
# to return from all errors except Disconnection
# or receiving a good reply
return
def __call__(self, request):
for plugin in self.plugins:
if plugin.match(request):
request.logger.info('processed by %s plugin', plugin.name())
try:
yield plugin.process(request)
except PluginNoSuchApplication as err:
fill_response_in(request, NO_SUCH_APP,
"No such application", str(err),
proxy_error_headers())
except PluginApplicationError:
message = "application error"
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers())
except ProxyInvalidRequest:
if request.path == "/ping":
self.ping(request)
else:
fill_response_in(request, httplib.NOT_FOUND,
httplib.responses[httplib.NOT_FOUND],
"Invalid url", proxy_error_headers())
except Exception as err:
request.logger.exception('plugin %s returned error: %s',
plugin.name(), err)
message = "unknown error"
fill_response_in(
request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
message, proxy_error_headers())
return
try:
name, event = extract_app_and_event(request)
except ProxyInvalidRequest:
if request.path == "/ping":
self.ping(request)
else:
fill_response_in(request, httplib.NOT_FOUND,
httplib.responses[httplib.NOT_FOUND],
"Invalid url", proxy_error_headers())
return
self.setup_tracing(request, name)
if self.sticky_header in request.headers:
seed = request.headers.get(self.sticky_header)
seed_value = header_to_seed(seed)
request.logger.info(
'sticky_header has been found: name %s, value %s, seed %d',
name, seed, seed_value)
name = self.resolve_group_to_version(name, seed_value)
app = yield self.get_service(name, request)
if app is None:
message = "current application %s is unavailable" % name
fill_response_in(request, NO_SUCH_APP, "No Such Application",
message, proxy_error_headers(name))
return
try:
# TODO: attempts should be configurable
yield self.process(request, name, app, event,
pack_httprequest(request), self.reelect_app, 2)
except Exception as err:
request.logger.exception("error during processing request %s", err)
fill_response_in(request, httplib.INTERNAL_SERVER_ERROR,
httplib.responses[httplib.INTERNAL_SERVER_ERROR],
"UID %s: %s" % (request.traceid, str(err)),
proxy_error_headers(name))
request.logger.info("exit from process")
def finish(self):
self.headers['X-Cocaine-Application'] = self.name
fill_response_in(self.request, self.code,
httplib.responses.get(self.code, httplib.OK),
''.join(self.messages), self.headers)
