def huskar_audit_log(action_type, **extra): if not switch.is_switched_on(SWITCH_ENABLE_AUDIT_LOG): yield return action = action_creator.make_action(action_type, **extra) yield try: if switch.is_switched_on(SWITCH_ENABLE_MINIMAL_MODE, False): action_name = action_types[action_type] fallback_audit_logger.info('arch.huskar_api %s %r', action_name, action.action_data) else: user = User.get_by_name('arch.huskar_api') user_id = user.id if user else 0 AuditLog.create(user_id, settings.LOCAL_REMOTE_ADDR, action) except AuditLogTooLongError: logger.info('Audit log is too long. %s arch.huskar_api', action_types[action_type]) return except AuditLogLostError: action_name = action_types[action_type] fallback_audit_logger.info('arch.huskar %s %r', action_name, action.action_data) except Exception: logger.exception('Unexpected error of audit log')
def clean(self): if not (switch.is_switched_on(SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK) and switch.is_switched_on( SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN, False)): return if self._is_time_to_clean(): self._clean()
def test_switch_change_on_fly(zk): switch_name = 'test_switch_change_on_fly' path = '/huskar/switch/arch.huskar_api/overall/%s' % switch_name zk.ensure_path(path) zk.set(path, '0') sleep(1) assert switch.is_switched_on(switch_name, default=None) is False # update zk.set(path, '100') sleep(1) assert switch.is_switched_on(switch_name, default=None) is True
def handle_event_for_extra_type(self, event_type, path): body = {} if not switch.is_switched_on(SWITCH_ENABLE_META_MESSAGE_CANARY): return body extra_types = subdomain_map.get_extra_types(path.type_name) for extra_type in extra_types: type_data = body.setdefault(extra_type, {}) application_names = set() if path.application_name: if path.application_name in self.watch_map[extra_type]: application_names.add(path.application_name) else: application_names = self.watch_map[extra_type] for application_name in application_names: app_data = type_data.setdefault(application_name, {}) handler = extra_handlers[extra_type, event_type] data = handler( self, Path.make(path.type_name, application_name, path.cluster_name, path.data_name)) if data: app_data.update(data) return { type_name: type_data for type_name, type_data in body.items() if any(type_data.values()) }
def __init__(self, huskar_client, from_application_name, from_cluster_name, remote_addr, route_mode, request_domain): self.huskar_client = huskar_client self.from_application_name = from_application_name self.from_cluster_name = from_cluster_name self.remote_addr = remote_addr self.route_mode = route_mode if switch.is_switched_on(SWITCH_ENABLE_ROUTE_HIJACK): ezone = _get_ezone(request_domain, from_application_name, self.from_cluster_name, remote_addr) default_hijack_mode = settings.ROUTE_EZONE_DEFAULT_HIJACK_MODE.get( ezone, self.Mode.disabled.value) route_hijack_list = _get_route_hijack_list( from_application_name, ezone) try: self.hijack_mode = self.Mode( route_hijack_list.get( self.from_application_name, default_hijack_mode)) except ValueError: logger.warning( 'Invalid hijack mode: %s', self.from_application_name) self.hijack_mode = self.Mode.disabled else: self.hijack_mode = self.Mode.disabled self.hijack_map = {} self._force_enable_dest_apps = set()
def _get_ezone(request_domain, application_name, cluster_name, request_addr): ezone = settings.ROUTE_DOMAIN_EZONE_MAP.get(request_domain, '') if ezone not in settings.ROUTE_EZONE_DEFAULT_HIJACK_MODE: logger.info( 'unknown domain: %s %s %s %r', request_addr, request_domain, application_name, cluster_name) monitor_client.increment('route_hijack.unknown_domain', tags={ 'domain': request_domain, 'from_application_name': application_name, 'appid': application_name, }) if not cluster_name: logger.info('unknown domain and cluster: %s %s %s', request_addr, request_domain, application_name) monitor_client.increment('route_hijack.unknown_cluster', tags={ 'domain': request_domain, 'from_application_name': application_name, 'appid': application_name, }) return settings.EZONE or 'default' if not switch.is_switched_on( SWITCH_ENABLE_ROUTE_HIJACK_WITH_LOCAL_EZONE, False): return settings.EZONE or 'default' ezone = try_to_extract_ezone(cluster_name, default='') if not ezone: return settings.ROUTE_OVERALL_EZONE return ezone
def audit_log(action_type, **extra): if not switch.is_switched_on(SWITCH_ENABLE_AUDIT_LOG): yield return action = action_creator.make_action(action_type, **extra) yield try: if g.auth.is_minimal_mode: action_name = action_types[action_type] fallback_audit_logger.info('%s %s %r', g.auth.username, action_name, action.action_data) else: user_id = g.auth.id if g.auth else 0 AuditLog.create(user_id, request.remote_addr, action) except AuditLogTooLongError: logger.info('Audit log is too long. %s %s %s', action_types[action_type], g.auth.username, request.remote_addr) return except AuditLogLostError: action_name = action_types[action_type] fallback_audit_logger.info('%s %s %r', g.auth.username, action_name, action.action_data) sentry.captureException(level=logging.WARNING) except Exception: logger.exception('Unexpected error of audit log') sentry.captureException()
def capture_exception(*args, **kwargs): if not switch.is_switched_on(SWITCH_ENABLE_SENTRY_EXCEPTION): return try: if raven_client: raven_client.captureException(*args, **kwargs) else: logger.warn('Ignored capture_exception with %r %r', args, kwargs) except Exception as e: logger.warn('Failed to send event to sentry: %r', e, exc_info=True)
def track(self, application_name, type_name): if not switch.is_switched_on(SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK): return name = '{}:{}'.format(application_name, type_name) score = time.time() try: redis_client.zadd(REDIS_KEY, **{name: score}) except Exception as e: logger.warning('tree holder cleaner track item failed: %s', e)
def allow_update_api(username, endpoint): action = 'update' if not switch.is_switched_on(SWITCH_DISABLE_UPDATE_VIA_API, False): return True if is_allow(username, endpoint, settings.ALLOW_UPDATE_VIA_API_USERS, action): return True return False
def allow_fetch_api(username, endpoint): action = 'fetch' if not switch.is_switched_on(SWITCH_DISABLE_FETCH_VIA_API, False): return True if is_allow(username, endpoint, settings.ALLOW_FETCH_VIA_API_USERS, action): return True return False
def declare_upstream_from_request(self, request_data): if not g.auth.is_application or not g.cluster_name: return if not switch.is_switched_on(SWITCH_ENABLE_DECLARE_UPSTREAM): return route_management = RouteManagement(huskar_client, g.auth.username, g.cluster_name) application_names = frozenset(request_data.get(SERVICE_SUBDOMAIN, [])) try: route_management.declare_upstream(application_names) except Exception: capture_exception(level=logging.WARNING)
def get_life_span(old_life_span): if not switch.is_switched_on(SWITCH_ENABLE_LONG_POLLING_MAX_LIFE_SPAN): return old_life_span if g.auth.username in settings.LONG_POLLING_MAX_LIFE_SPAN_EXCLUDE: return old_life_span max_life_span = settings.LONG_POLLING_MAX_LIFE_SPAN life_span_jitter = settings.LONG_POLLING_LIFE_SPAN_JITTER if 0 < old_life_span < life_span_jitter: return old_life_span new_life_span = min(old_life_span or max_life_span, max_life_span) + random.random() * life_span_jitter return new_life_span
def load_user(self, username=None): username = username or self._name if username is None: return if switch.is_switched_on(SWITCH_ENABLE_MINIMAL_MODE, False): self.enter_minimal_mode(MM_REASON_SWITCH) return try: self._user = User.get_by_name(username) except (SQLAlchemyError, RedisError, socket.error): logger.exception('Enter minimal mode') self.enter_minimal_mode(MM_REASON_AUTH) session_load_user_failed.send(self)
def check_rate_limit(): if not switch.is_switched_on(SWITCH_ENABLE_RATE_LIMITER): return remote_addr = request.remote_addr config = get_limiter_config(settings.RATE_LIMITER_SETTINGS, remote_addr) if not config: return rate, capacity = config['rate'], config['capacity'] try: check_new_request(remote_addr, rate, capacity) except RateExceededError: abort(429, 'Too Many Requests, the rate limit is {}/s'.format(rate))
def check_rate_limit(): if not switch.is_switched_on(SWITCH_ENABLE_RATE_LIMITER): return if not g.get('auth'): return username = g.auth.username config = get_limiter_config(settings.RATE_LIMITER_SETTINGS, username) if not config: return rate, capacity = config['rate'], config['capacity'] try: check_new_request(username, rate, capacity) except RateExceededError: abort(429, 'Too Many Requests, the rate limit is {}/s'.format(rate))
def check_instance_key_in_creation(cls, subdomain, application_name, cluster_name, key): if subdomain != CONFIG_SUBDOMAIN: return if not switch.is_switched_on(SWITCH_ENABLE_CONFIG_PREFIX_BLACKLIST, False): return if config_facade.exists(application_name, cluster_name, key=key): return for prefix in settings.CONFIG_PREFIX_BLACKLIST: if key.startswith(prefix): abort( 400, 'The key {key} starts with {prefix} is denied.'.format( key=key, prefix=prefix))
def validate_fields(schema, data, optional_fields=(), partial=True): """validate fields value but which field name in `optional_fields` and the value is None. """ if not switch.is_switched_on(SWITCH_VALIDATE_SCHEMA, True): return fields = set(data) if not fields.issubset(schema.fields): raise ValidationError('The set of fields "%s" is not a subset of %s' % (fields, schema)) data = { k: v for k, v in data.items() if not (k in optional_fields and v is None) } schema.validate(data, partial=partial)
def _detect_bad_route(self, body): if not switch.is_switched_on(SWITCH_DETECT_BAD_ROUTE): return if self.from_application_name in settings.LEGACY_APPLICATION_LIST: return from_cluster_blacklist = settings.ROUTE_FROM_CLUSTER_BLACKLIST.get( self.from_application_name, []) if self.from_cluster_name in from_cluster_blacklist: return type_name = SERVICE_SUBDOMAIN type_body = body[type_name] flat_cluster_names = ( (application_name, cluster_name, cluster_body) for application_name, application_body in type_body.iteritems() for cluster_name, cluster_body in application_body.iteritems()) for application_name, cluster_name, cluster_body in flat_cluster_names: if application_name in settings.LEGACY_APPLICATION_LIST: continue if cluster_name in settings.ROUTE_DEST_CLUSTER_BLACKLIST.get( application_name, []): continue cluster_map = self.cluster_maps[application_name, type_name] resolved_name = cluster_map.cluster_names.get(cluster_name) if cluster_body or not resolved_name: continue monitor_client.increment( 'tree_watcher.bad_route', 1, tags=dict( from_application_name=self.from_application_name, from_cluster_name=self.from_cluster_name, dest_application_name=application_name, appid=application_name, dest_cluster_name=cluster_name, dest_resolved_cluster_name=resolved_name, )) logger.info('Bad route detected: %s %s %s %s -> %s (%r)', self.from_application_name, self.from_cluster_name, application_name, cluster_name, resolved_name, dict(cluster_map.cluster_names))
def check_config_and_switch_read_only(): method = request.method view_args = request.view_args appid = view_args and view_args.get('application_name') response = api_response(message='Config and switch write inhibit', status="Forbidden") response.status_code = 403 if method in READ_METHOD_SET: return if request.endpoint not in config_and_switch_readonly_endpoints: return if appid and appid in settings.CONFIG_AND_SWITCH_READONLY_BLACKLIST: return response if switch.is_switched_on(SWITCH_ENABLE_CONFIG_AND_SWITCH_WRITE, True): return if appid and appid in settings.CONFIG_AND_SWITCH_READONLY_WHITELIST: return return response
def prepare(self, tree_watcher, request_data): """Reads data sources.""" if (self.from_application_name in settings.LEGACY_APPLICATION_LIST or not self.from_cluster_name): logger.info('Skip: %s %s %s', self.from_application_name, self.remote_addr, self.from_cluster_name) self.hijack_mode = self.Mode.disabled if (switch.is_switched_on(SWITCH_ENABLE_ROUTE_FORCE_CLUSTERS, default=False) and self.from_cluster_name in settings.FORCE_ROUTING_CLUSTERS): self.hijack_mode = self.Mode.standalone self._force_enable_dest_apps = set( _get_force_enable_dest_apps( self.from_application_name, request_data)) if (self.hijack_mode in (self.Mode.enabled, self.Mode.standalone) or self._force_enable_dest_apps): tree_watcher.from_application_name = self.from_application_name tree_watcher.from_cluster_name = self.from_cluster_name
def resolve(self, cluster_name, from_application_name=None, intent=None, force_route_cluster_name=None): """Resolves the cluster name and returns the name of physical cluster. There are two steps to resolve a cluster. First, the route table will be checked if ``from_application_name`` is provided. Then, the resolved cluster will be resolved again, but uses the symlink configuration. There is an optional ``intent`` parameter which indicates the route intent in the first step, once ``from_application_name`` is provided. :param cluster_name: The original cluster name. :param from_application_name: Optional. The name of caller application. :param intent: Optional. The route intent of caller. :param force_route_cluster_name: Optional. The name of caller cluster :returns: The physical cluster name or ``None``. """ cluster_info = None resolved_name = cluster_name if from_application_name: resolve_via_default = False try: cluster_info = self.get_cluster_info(cluster_name) except MalformedDataError as e: logger.warning('Failed to parse route "%s"', e.info.path) resolve_via_default = True else: route = cluster_info.get_route() route_key = make_route_key(from_application_name, intent) resolved_name = route.get(route_key) if resolved_name is None: resolve_via_default = True if resolve_via_default: resolved_name = self.resolve_via_default(cluster_name, intent) if resolved_name: try: if not cluster_info or resolved_name != cluster_name: cluster_info = self.get_cluster_info(resolved_name) except MalformedDataError as e: logger.warning('Failed to parse symlink "%s"', e.info.path) else: resolved_name = cluster_info.get_link() or resolved_name if force_route_cluster_name in settings.FORCE_ROUTING_CLUSTERS and \ switch.is_switched_on(SWITCH_ENABLE_ROUTE_FORCE_CLUSTERS, default=False): cluster_key = _make_force_route_cluster_key( force_route_cluster_name, intent) # for route mode if (from_application_name and intent and cluster_key in settings.FORCE_ROUTING_CLUSTERS): resolved_name = settings.FORCE_ROUTING_CLUSTERS.get( cluster_key) else: # case: cluster_name is spec cluster which is dest cluster # ignore this cluster's link dest_clusters = settings.FORCE_ROUTING_CLUSTERS.values() if ((not from_application_name) and cluster_name in dest_clusters): resolved_name = cluster_name else: resolved_name = settings.FORCE_ROUTING_CLUSTERS.get( force_route_cluster_name) if resolved_name != cluster_name: return resolved_name
def _add(self, func, *args, **kwargs): if not switch.is_switched_on(SWITCH_ENABLE_WEBHOOK_NOTIFY, True): return sender = functools.partial(func, *args, **kwargs) self.hook_queue.put(sender)