def load_clients(self, path=None, apis=[]):
"""Generate client libraries for the given apis, without starting an
api server"""
if not path:
raise Exception("Missing path to api swagger files")
if type(apis) is not list:
raise Exception("'apis' should be a list of api names")
if len(apis) == 0:
raise Exception(
"'apis' is an empty list - Expected at least one api name")
for api_name in apis:
api_path = os.path.join(path, '%s.yaml' % api_name)
if not os.path.isfile(api_path):
raise Exception("Cannot find swagger specification at %s" %
api_path)
log.info("Loading api %s from %s" % (api_name, api_path))
ApiPool.add(
api_name,
yaml_path=api_path,
timeout=self.timeout,
error_callback=self.error_callback,
formats=self.formats,
do_persist=False,
local=False,
)
return self
def do_version():
"""Return version details of the running server api"""
v = ApiPool.ping.model.Version(
name=ApiPool().current_server_name,
version=ApiPool().current_server_api.get_version(),
container=get_container_version(),
)
log.info("/version: " + pprint.pformat(v))
return v
def test_apipool_current_server_name_api():
assert ApiPool().current_server_name == ''
assert ApiPool().current_server_api is None
app = MagicMock()
api = ApiPool().add('foo', yaml_str=yaml_foo)
api.spawn_api(app)
assert ApiPool().current_server_name == 'foo'
assert ApiPool().current_server_api == api
def to_model(self):
"""Return a bravado-core Error instance"""
e = ApiPool().current_server_api.model.Error(
status=self.status,
error=self.code.upper(),
error_description=str(self),
)
if self.error_id:
e.error_id = self.error_id
if self.user_message:
e.user_message = self.user_message
if self.error_caught:
e.error_caught = pformat(self.error_caught)
return e
def test__cmp_models():
assert ApiPool._cmp_models(
{'a': 1, 'b': 2},
{'a': 1, 'b': 2}
) == 0
assert ApiPool._cmp_models(
{'a': 1},
{'a': 1, 'b': 2}
) != 0
assert ApiPool._cmp_models(
{'a': 1, 'b': 2, 'x-model': 12},
{'a': 1, 'b': 2}
) == 0
assert ApiPool._cmp_models(
{'a': 1, 'b': 2, 'x-model': 12, 'properties': {'foo': {'$ref': 'a'}}},
{'a': 1, 'b': 2, 'properties': {'foo': {'$ref': 'a', 'x-scope': [1, 2]}}},
) == 0
def start(self, serve=[]):
"""Load all apis, either as local apis served by the flask app, or as
remote apis to be called from whithin the app's endpoints, then start
the app server"""
# Check arguments
if type(serve) is str:
serve = [serve]
elif type(serve) is list:
pass
else:
raise Exception(
"'serve' should be an api name or a list of api names")
if len(serve) == 0:
raise Exception("You must specify at least one api to serve")
for api_name in serve:
if api_name not in self.apis:
raise Exception(
"Can't find %s.yaml (swagger file) in the api directory %s"
% (api_name, self.path_apis))
app = self.app
app.secret_key = os.urandom(24)
# Initialize JWT config
conf = get_config()
if hasattr(conf, 'jwt_secret'):
log.info(
"Set JWT parameters to issuer=%s audience=%s secret=%s***" % (
conf.jwt_issuer,
conf.jwt_audience,
conf.jwt_secret[0:8],
))
# Always serve the ping api
serve.append('ping')
# Let's compress returned data when possible
compress = Compress()
compress.init_app(app)
# All apis that are not served locally are not persistent
not_persistent = []
for api_name in self.apis.keys():
if api_name in serve:
pass
else:
not_persistent.append(api_name)
# Now load those apis into the ApiPool
for api_name, api_path in self.apis.items():
host = None
port = None
if api_name in serve:
# We are serving this api locally: override the host:port specified in the swagger spec
host = self.host
port = self.port
do_persist = True if api_name not in not_persistent else False
local = True if api_name in serve else False
log.info("Loading api %s from %s (persist: %s)" %
(api_name, api_path, do_persist))
ApiPool.add(
api_name,
yaml_path=api_path,
timeout=self.timeout,
error_callback=self.error_callback,
formats=self.formats,
do_persist=do_persist,
host=host,
port=port,
local=local,
)
ApiPool.merge()
# Now spawn flask routes for all endpoints
for api_name in self.apis.keys():
if api_name in serve:
log.info("Spawning api %s" % api_name)
api = getattr(ApiPool, api_name)
# Spawn api and wrap every endpoint in a crash handler that
# catches replies and reports errors
api.spawn_api(app,
decorator=generate_crash_handler_decorator(
self.error_decorator))
log.debug("Argv is [%s]" % ' '.join(sys.argv))
if 'celery' in sys.argv[0].lower():
# This code is loading in a celery server - Don't start the actual flask app.
log.info("Running in a Celery worker - Not starting the Flask app")
return
if os.path.basename(sys.argv[0]) == 'gunicorn':
# Gunicorn takes care of spawning workers
log.info("Running in Gunicorn - Not starting the Flask app")
return
# Debug mode is the default when not running via gunicorn
app.debug = self.debug
app.run(host='0.0.0.0', port=self.port)
def report_error(title=None, data={}, caught=None, is_fatal=False):
"""Format a crash report and send it somewhere relevant. There are two
types of crashes: fatal crashes (backend errors) or non-fatal ones (just
reporting a glitch, but the api call did not fail)"""
# Don't report errors if NO_ERROR_REPORTING set to 1 (set by run_acceptance_tests)
if os.environ.get('DO_REPORT_ERROR', None):
# Force error reporting
pass
elif os.environ.get('NO_ERROR_REPORTING', '') == '1':
log.info("NO_ERROR_REPORTING is set: not reporting error!")
return
elif 'is_ec2_instance' in data:
if not data['is_ec2_instance']:
# Not running on amazon: no reporting
log.info("DATA[is_ec2_instance] is False: not reporting error!")
return
elif not is_ec2_instance():
log.info("Not running on an EC2 instance: not reporting error!")
return
# Fill error report with tons of usefull data
if 'user' not in data:
populate_error_report(data)
# Add the message
data['title'] = title
data['is_fatal_error'] = is_fatal
# Add the error caught, if any:
if caught:
data['error_caught'] = "%s" % caught
# Add a trace - Formatting traceback may raise a UnicodeDecodeError...
data['stack'] = []
try:
data['stack'] = [l for l in traceback.format_stack()]
except Exception as ee:
data['stack'] = 'Skipped trace - contained non-ascii chars'
# inspect may raise a UnicodeDecodeError...
fname = ''
try:
fname = inspect.stack()[1][3]
except Exception as e:
fname = 'unknown-method'
# Format the error's title
status, code = 'unknown_status', 'unknown_error_code'
if 'response' in data:
status = data['response'].get('status', status)
code = data['response'].get('error_code', code)
title_details = "%s %s %s" % (ApiPool().current_server_name, status, code)
else:
title_details = "%s %s()" % (ApiPool().current_server_name, fname)
if is_fatal:
title_details = 'FATAL ERROR %s' % title_details
else:
title_details = 'NON-FATAL ERROR %s' % title_details
if title:
title = "%s: %s" % (title_details, title)
else:
title = title_details
global error_reporter
log.info("Reporting crash...")
try:
error_reporter(title, json.dumps(data, sort_keys=True, indent=4))
except Exception as e:
# Don't block on replying to api caller
log.error("Failed to send email report: %s" % str(e))
def populate_error_report(data):
"""Add generic stats to the error report"""
# Did klue-client-server set a call_id and call_path?
call_id, call_path = '', ''
if hasattr(stack.top, 'call_id'):
call_id = stack.top.call_id
if hasattr(stack.top, 'call_path'):
call_path = stack.top.call_path
# Unique ID associated to all responses associated to a given
# call to klue-api, across all micro-services
data['call_id'] = call_id
data['call_path'] = call_path
# Are we in aws?
data['is_ec2_instance'] = is_ec2_instance()
# If user is authenticated, get her id
user_data = {
'id': '',
'is_auth': 0,
'ip': '',
}
if stack.top:
# We are in a request context
user_data['ip'] = request.remote_addr
if 'X-Forwarded-For' in request.headers:
user_data['forwarded_ip'] = request.headers.get('X-Forwarded-For', '')
if 'User-Agent' in request.headers:
user_data['user_agent'] = request.headers.get('User-Agent', '')
if hasattr(stack.top, 'current_user'):
user_data['is_auth'] = 1
user_data['id'] = stack.top.current_user.get('sub', '')
for k in ('name', 'email', 'is_expert', 'is_admin', 'is_support', 'is_tester', 'language'):
v = stack.top.current_user.get(k, None)
if v:
user_data[k] = v
data['user'] = user_data
# Is the current code running as a server?
if ApiPool().current_server_api:
# Server info
server = request.base_url
server = server.replace('http://', '')
server = server.replace('https://', '')
server = server.split('/')[0]
parts = server.split(':')
fqdn = parts[0]
port = parts[1] if len(parts) == 2 else ''
data['server'] = {
'fqdn': fqdn,
'port': port,
'api_name': ApiPool().current_server_name,
'api_version': ApiPool().current_server_api.get_version(),
}
# Endpoint data
data['endpoint'] = {
'id': "%s %s %s" % (ApiPool().current_server_name, request.method, request.path),
'url': request.url,
'base_url': request.base_url,
'path': request.path,
'method': request.method
}
