def huskar_audit_log(action_type, **extra):
if not switch.is_switched_on(SWITCH_ENABLE_AUDIT_LOG):
yield
return
action = action_creator.make_action(action_type, **extra)
yield
try:
if switch.is_switched_on(SWITCH_ENABLE_MINIMAL_MODE, False):
action_name = action_types[action_type]
fallback_audit_logger.info('arch.huskar_api %s %r', action_name,
action.action_data)
else:
user = User.get_by_name('arch.huskar_api')
user_id = user.id if user else 0
AuditLog.create(user_id, settings.LOCAL_REMOTE_ADDR, action)
except AuditLogTooLongError:
logger.info('Audit log is too long. %s arch.huskar_api',
action_types[action_type])
return
except AuditLogLostError:
action_name = action_types[action_type]
fallback_audit_logger.info('arch.huskar %s %r', action_name,
action.action_data)
except Exception:
logger.exception('Unexpected error of audit log')
def clean(self):
if not (switch.is_switched_on(SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK)
and switch.is_switched_on(
SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN, False)):
return
if self._is_time_to_clean():
self._clean()
def test_switch_change_on_fly(zk):
switch_name = 'test_switch_change_on_fly'
path = '/huskar/switch/arch.huskar_api/overall/%s' % switch_name
zk.ensure_path(path)
zk.set(path, '0')
sleep(1)
assert switch.is_switched_on(switch_name, default=None) is False
# update
zk.set(path, '100')
sleep(1)
assert switch.is_switched_on(switch_name, default=None) is True
def handle_event_for_extra_type(self, event_type, path):
body = {}
if not switch.is_switched_on(SWITCH_ENABLE_META_MESSAGE_CANARY):
return body
extra_types = subdomain_map.get_extra_types(path.type_name)
for extra_type in extra_types:
type_data = body.setdefault(extra_type, {})
application_names = set()
if path.application_name:
if path.application_name in self.watch_map[extra_type]:
application_names.add(path.application_name)
else:
application_names = self.watch_map[extra_type]
for application_name in application_names:
app_data = type_data.setdefault(application_name, {})
handler = extra_handlers[extra_type, event_type]
data = handler(
self,
Path.make(path.type_name, application_name,
path.cluster_name, path.data_name))
if data:
app_data.update(data)
return {
type_name: type_data
for type_name, type_data in body.items() if any(type_data.values())
}
def __init__(self, huskar_client, from_application_name, from_cluster_name,
remote_addr, route_mode, request_domain):
self.huskar_client = huskar_client
self.from_application_name = from_application_name
self.from_cluster_name = from_cluster_name
self.remote_addr = remote_addr
self.route_mode = route_mode
if switch.is_switched_on(SWITCH_ENABLE_ROUTE_HIJACK):
ezone = _get_ezone(request_domain, from_application_name,
self.from_cluster_name, remote_addr)
default_hijack_mode = settings.ROUTE_EZONE_DEFAULT_HIJACK_MODE.get(
ezone, self.Mode.disabled.value)
route_hijack_list = _get_route_hijack_list(
from_application_name, ezone)
try:
self.hijack_mode = self.Mode(
route_hijack_list.get(
self.from_application_name, default_hijack_mode))
except ValueError:
logger.warning(
'Invalid hijack mode: %s', self.from_application_name)
self.hijack_mode = self.Mode.disabled
else:
self.hijack_mode = self.Mode.disabled
self.hijack_map = {}
self._force_enable_dest_apps = set()
def _get_ezone(request_domain, application_name, cluster_name, request_addr):
ezone = settings.ROUTE_DOMAIN_EZONE_MAP.get(request_domain, '')
if ezone not in settings.ROUTE_EZONE_DEFAULT_HIJACK_MODE:
logger.info(
'unknown domain: %s %s %s %r', request_addr, request_domain,
application_name, cluster_name)
monitor_client.increment('route_hijack.unknown_domain', tags={
'domain': request_domain,
'from_application_name': application_name,
'appid': application_name,
})
if not cluster_name:
logger.info('unknown domain and cluster: %s %s %s',
request_addr, request_domain, application_name)
monitor_client.increment('route_hijack.unknown_cluster', tags={
'domain': request_domain,
'from_application_name': application_name,
'appid': application_name,
})
return settings.EZONE or 'default'
if not switch.is_switched_on(
SWITCH_ENABLE_ROUTE_HIJACK_WITH_LOCAL_EZONE, False):
return settings.EZONE or 'default'
ezone = try_to_extract_ezone(cluster_name, default='')
if not ezone:
return settings.ROUTE_OVERALL_EZONE
return ezone
def audit_log(action_type, **extra):
if not switch.is_switched_on(SWITCH_ENABLE_AUDIT_LOG):
yield
return
action = action_creator.make_action(action_type, **extra)
yield
try:
if g.auth.is_minimal_mode:
action_name = action_types[action_type]
fallback_audit_logger.info('%s %s %r', g.auth.username,
action_name, action.action_data)
else:
user_id = g.auth.id if g.auth else 0
AuditLog.create(user_id, request.remote_addr, action)
except AuditLogTooLongError:
logger.info('Audit log is too long. %s %s %s',
action_types[action_type], g.auth.username,
request.remote_addr)
return
except AuditLogLostError:
action_name = action_types[action_type]
fallback_audit_logger.info('%s %s %r', g.auth.username, action_name,
action.action_data)
sentry.captureException(level=logging.WARNING)
except Exception:
logger.exception('Unexpected error of audit log')
sentry.captureException()
def capture_exception(*args, **kwargs):
if not switch.is_switched_on(SWITCH_ENABLE_SENTRY_EXCEPTION):
return
try:
if raven_client:
raven_client.captureException(*args, **kwargs)
else:
logger.warn('Ignored capture_exception with %r %r', args, kwargs)
except Exception as e:
logger.warn('Failed to send event to sentry: %r', e, exc_info=True)
def track(self, application_name, type_name):
if not switch.is_switched_on(SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK):
return
name = '{}:{}'.format(application_name, type_name)
score = time.time()
try:
redis_client.zadd(REDIS_KEY, **{name: score})
except Exception as e:
logger.warning('tree holder cleaner track item failed: %s', e)
def allow_update_api(username, endpoint):
action = 'update'
if not switch.is_switched_on(SWITCH_DISABLE_UPDATE_VIA_API, False):
return True
if is_allow(username, endpoint, settings.ALLOW_UPDATE_VIA_API_USERS,
action):
return True
return False
def allow_fetch_api(username, endpoint):
action = 'fetch'
if not switch.is_switched_on(SWITCH_DISABLE_FETCH_VIA_API, False):
return True
if is_allow(username, endpoint, settings.ALLOW_FETCH_VIA_API_USERS,
action):
return True
return False
def declare_upstream_from_request(self, request_data):
if not g.auth.is_application or not g.cluster_name:
return
if not switch.is_switched_on(SWITCH_ENABLE_DECLARE_UPSTREAM):
return
route_management = RouteManagement(huskar_client, g.auth.username,
g.cluster_name)
application_names = frozenset(request_data.get(SERVICE_SUBDOMAIN, []))
try:
route_management.declare_upstream(application_names)
except Exception:
capture_exception(level=logging.WARNING)
def get_life_span(old_life_span):
if not switch.is_switched_on(SWITCH_ENABLE_LONG_POLLING_MAX_LIFE_SPAN):
return old_life_span
if g.auth.username in settings.LONG_POLLING_MAX_LIFE_SPAN_EXCLUDE:
return old_life_span
max_life_span = settings.LONG_POLLING_MAX_LIFE_SPAN
life_span_jitter = settings.LONG_POLLING_LIFE_SPAN_JITTER
if 0 < old_life_span < life_span_jitter:
return old_life_span
new_life_span = min(old_life_span or max_life_span,
max_life_span) + random.random() * life_span_jitter
return new_life_span
def load_user(self, username=None):
username = username or self._name
if username is None:
return
if switch.is_switched_on(SWITCH_ENABLE_MINIMAL_MODE, False):
self.enter_minimal_mode(MM_REASON_SWITCH)
return
try:
self._user = User.get_by_name(username)
except (SQLAlchemyError, RedisError, socket.error):
logger.exception('Enter minimal mode')
self.enter_minimal_mode(MM_REASON_AUTH)
session_load_user_failed.send(self)
def check_rate_limit():
if not switch.is_switched_on(SWITCH_ENABLE_RATE_LIMITER):
return
remote_addr = request.remote_addr
config = get_limiter_config(settings.RATE_LIMITER_SETTINGS, remote_addr)
if not config:
return
rate, capacity = config['rate'], config['capacity']
try:
check_new_request(remote_addr, rate, capacity)
except RateExceededError:
abort(429, 'Too Many Requests, the rate limit is {}/s'.format(rate))
def check_rate_limit():
if not switch.is_switched_on(SWITCH_ENABLE_RATE_LIMITER):
return
if not g.get('auth'):
return
username = g.auth.username
config = get_limiter_config(settings.RATE_LIMITER_SETTINGS, username)
if not config:
return
rate, capacity = config['rate'], config['capacity']
try:
check_new_request(username, rate, capacity)
except RateExceededError:
abort(429, 'Too Many Requests, the rate limit is {}/s'.format(rate))
def check_instance_key_in_creation(cls, subdomain, application_name,
cluster_name, key):
if subdomain != CONFIG_SUBDOMAIN:
return
if not switch.is_switched_on(SWITCH_ENABLE_CONFIG_PREFIX_BLACKLIST,
False):
return
if config_facade.exists(application_name, cluster_name, key=key):
return
for prefix in settings.CONFIG_PREFIX_BLACKLIST:
if key.startswith(prefix):
abort(
400,
'The key {key} starts with {prefix} is denied.'.format(
key=key, prefix=prefix))
def validate_fields(schema, data, optional_fields=(), partial=True):
"""validate fields value but which field name in `optional_fields`
and the value is None.
"""
if not switch.is_switched_on(SWITCH_VALIDATE_SCHEMA, True):
return
fields = set(data)
if not fields.issubset(schema.fields):
raise ValidationError('The set of fields "%s" is not a subset of %s' %
(fields, schema))
data = {
k: v
for k, v in data.items() if not (k in optional_fields and v is None)
}
schema.validate(data, partial=partial)
def _detect_bad_route(self, body):
if not switch.is_switched_on(SWITCH_DETECT_BAD_ROUTE):
return
if self.from_application_name in settings.LEGACY_APPLICATION_LIST:
return
from_cluster_blacklist = settings.ROUTE_FROM_CLUSTER_BLACKLIST.get(
self.from_application_name, [])
if self.from_cluster_name in from_cluster_blacklist:
return
type_name = SERVICE_SUBDOMAIN
type_body = body[type_name]
flat_cluster_names = (
(application_name, cluster_name, cluster_body)
for application_name, application_body in type_body.iteritems()
for cluster_name, cluster_body in application_body.iteritems())
for application_name, cluster_name, cluster_body in flat_cluster_names:
if application_name in settings.LEGACY_APPLICATION_LIST:
continue
if cluster_name in settings.ROUTE_DEST_CLUSTER_BLACKLIST.get(
application_name, []):
continue
cluster_map = self.cluster_maps[application_name, type_name]
resolved_name = cluster_map.cluster_names.get(cluster_name)
if cluster_body or not resolved_name:
continue
monitor_client.increment(
'tree_watcher.bad_route',
1,
tags=dict(
from_application_name=self.from_application_name,
from_cluster_name=self.from_cluster_name,
dest_application_name=application_name,
appid=application_name,
dest_cluster_name=cluster_name,
dest_resolved_cluster_name=resolved_name,
))
logger.info('Bad route detected: %s %s %s %s -> %s (%r)',
self.from_application_name, self.from_cluster_name,
application_name, cluster_name, resolved_name,
dict(cluster_map.cluster_names))
def check_config_and_switch_read_only():
method = request.method
view_args = request.view_args
appid = view_args and view_args.get('application_name')
response = api_response(message='Config and switch write inhibit',
status="Forbidden")
response.status_code = 403
if method in READ_METHOD_SET:
return
if request.endpoint not in config_and_switch_readonly_endpoints:
return
if appid and appid in settings.CONFIG_AND_SWITCH_READONLY_BLACKLIST:
return response
if switch.is_switched_on(SWITCH_ENABLE_CONFIG_AND_SWITCH_WRITE, True):
return
if appid and appid in settings.CONFIG_AND_SWITCH_READONLY_WHITELIST:
return
return response
def prepare(self, tree_watcher, request_data):
"""Reads data sources."""
if (self.from_application_name in settings.LEGACY_APPLICATION_LIST or
not self.from_cluster_name):
logger.info('Skip: %s %s %s', self.from_application_name,
self.remote_addr, self.from_cluster_name)
self.hijack_mode = self.Mode.disabled
if (switch.is_switched_on(SWITCH_ENABLE_ROUTE_FORCE_CLUSTERS,
default=False) and
self.from_cluster_name in settings.FORCE_ROUTING_CLUSTERS):
self.hijack_mode = self.Mode.standalone
self._force_enable_dest_apps = set(
_get_force_enable_dest_apps(
self.from_application_name, request_data))
if (self.hijack_mode in (self.Mode.enabled, self.Mode.standalone) or
self._force_enable_dest_apps):
tree_watcher.from_application_name = self.from_application_name
tree_watcher.from_cluster_name = self.from_cluster_name
def resolve(self,
cluster_name,
from_application_name=None,
intent=None,
force_route_cluster_name=None):
"""Resolves the cluster name and returns the name of physical cluster.
There are two steps to resolve a cluster. First, the route table will
be checked if ``from_application_name`` is provided. Then, the resolved
cluster will be resolved again, but uses the symlink configuration.
There is an optional ``intent`` parameter which indicates the route
intent in the first step, once ``from_application_name`` is provided.
:param cluster_name: The original cluster name.
:param from_application_name: Optional. The name of caller application.
:param intent: Optional. The route intent of caller.
:param force_route_cluster_name: Optional. The name of caller cluster
:returns: The physical cluster name or ``None``.
"""
cluster_info = None
resolved_name = cluster_name
if from_application_name:
resolve_via_default = False
try:
cluster_info = self.get_cluster_info(cluster_name)
except MalformedDataError as e:
logger.warning('Failed to parse route "%s"', e.info.path)
resolve_via_default = True
else:
route = cluster_info.get_route()
route_key = make_route_key(from_application_name, intent)
resolved_name = route.get(route_key)
if resolved_name is None:
resolve_via_default = True
if resolve_via_default:
resolved_name = self.resolve_via_default(cluster_name, intent)
if resolved_name:
try:
if not cluster_info or resolved_name != cluster_name:
cluster_info = self.get_cluster_info(resolved_name)
except MalformedDataError as e:
logger.warning('Failed to parse symlink "%s"', e.info.path)
else:
resolved_name = cluster_info.get_link() or resolved_name
if force_route_cluster_name in settings.FORCE_ROUTING_CLUSTERS and \
switch.is_switched_on(SWITCH_ENABLE_ROUTE_FORCE_CLUSTERS,
default=False):
cluster_key = _make_force_route_cluster_key(
force_route_cluster_name, intent)
# for route mode
if (from_application_name and intent
and cluster_key in settings.FORCE_ROUTING_CLUSTERS):
resolved_name = settings.FORCE_ROUTING_CLUSTERS.get(
cluster_key)
else:
# case: cluster_name is spec cluster which is dest cluster
# ignore this cluster's link
dest_clusters = settings.FORCE_ROUTING_CLUSTERS.values()
if ((not from_application_name)
and cluster_name in dest_clusters):
resolved_name = cluster_name
else:
resolved_name = settings.FORCE_ROUTING_CLUSTERS.get(
force_route_cluster_name)
if resolved_name != cluster_name:
return resolved_name
def _add(self, func, *args, **kwargs):
if not switch.is_switched_on(SWITCH_ENABLE_WEBHOOK_NOTIFY, True):
return
sender = functools.partial(func, *args, **kwargs)
self.hook_queue.put(sender)