async def http_request(self, dump_body=None, timeout=None, **kwargs): """ :param dump_body: :param timeout: :param kwargs: :return: """ if dump_body is None: dump_body = self._dump_body else: dump_body = self._normalize_dump_flag(dump_body) if timeout is None: timeout = self._timeout elif not isinstance(timeout, aiohttp.ClientTimeout): timeout = aiohttp.ClientTimeout(sock_read=timeout, sock_connect=timeout) try: self._logger.info( "%s", self.dump_request(ObjectDict(**kwargs), dump_body[0])) async with aiohttp.ClientSession( timeout=timeout) as session, session.request( **kwargs) as resp: try: body = await resp.json() except (aiohttp.ContentTypeError, ValueError, TypeError): body = await resp.text() try: response = ObjectDict( body=body, status=resp.status, headers=dict(resp.headers.items()), ) log_resp = response log_resp.text = response.body log_resp = self.dump_response(log_resp, dump_body[1]) resp.raise_for_status() self._logger.info("%s", log_resp) except aiohttp.ClientResponseError as exc: self._logger.warning("%s", log_resp) if self._raise_on_exc is True: raise # pragma: no cover response.exception = exc return response except ( aiohttp.ClientError, aiohttp.ServerTimeoutError, asyncio.TimeoutError, ) as exc: self._logger.exception(exc) if self._raise_on_exc is True: raise # pragma: no cover return ObjectDict(body={}, status=httpcode.SERVICE_UNAVAILABLE, headers={}, exception=exc)
def create( self, identity, refresh=True, expires_access=None, expires_refresh=None, scope=None, ): """ :param identity: user identifier, generally the username :param refresh: enable refresh token :param expires_access: in seconds :param expires_refresh: in seconds :param scope: :return: """ identity = self.prepare_identity(identity) access_token = self.get_access(identity=identity, expires=expires_access) decoded = self.decode(access_token) resp = ObjectDict( access_token=access_token, expires_in=decoded.exp, issued_at=decoded.iat, token_type=cap.config.JWT_DEFAULT_TOKEN_TYPE, scope=scope or cap.config.JWT_DEFAULT_SCOPE, ) if refresh: resp.refresh_token = self.get_refresh( identity=identity, expires=expires_refresh ) return resp
def health_glances(conf=None, **__): """ :param conf: :return: """ conf = conf() if callable(conf) else (conf or {}) default_conf = ObjectDict( SYSTEM_ENDPOINT="http://localhost:4000/api/2", SYSTEM_CPU_THRESHOLD=90, SYSTEM_MEM_THRESHOLD=90, SYSTEM_FS_THRESHOLD=85, SYSTEM_MEM_INCLUDE_SWAP=True, SYSTEM_FS_MOUNT_POINTS=("/", ), SYSTEM_DUMP_ALL=False, ) conf = ObjectDict(**{**default_conf, **conf}) resp = ObjectDict(errors=[]) units = ["cpu", "mem", "fs"] if conf.SYSTEM_MEM_INCLUDE_SWAP: units.append("memswap") th_mem = conf.SYSTEM_MEM_THRESHOLD th_cpu = conf.SYSTEM_CPU_THRESHOLD th_fs = conf.SYSTEM_FS_THRESHOLD responses = HTTPBatch().request( [dict(url=f"{conf.SYSTEM_ENDPOINT}/{u}") for u in units]) for i, r in enumerate(responses): if r.exception: return False, str(r.exception) resp[units[i]] = r.body if resp.cpu.total > th_cpu: resp.errors.append( f"high CPU usage: {resp.cpu.total}, threshold: {th_cpu}") if resp.mem.percent > th_mem: resp.errors.append( f"high RAM usage: {resp.mem.percent}, threshold: {th_mem}") if conf.SYSTEM_MEM_INCLUDE_SWAP and resp.memswap.percent > th_mem: resp.errors.append( f"high SWAP usage: {resp.memswap.percent}, threshold: {th_mem}") for f in resp.fs: if f.mnt_point in conf.SYSTEM_FS_MOUNT_POINTS and f.percent > th_fs: resp.errors.append( f"high DISK usage on {f.mnt_point}: {f.percent}, threshold: {th_fs}" ) output = resp if conf.SYSTEM_DUMP_ALL else (resp.errors or None) return bool(not resp.errors), dict(messages=output)
def block(self, ips, permanent=False, timestamp=None, url="block"): """ add a list of ip addresses to the block list :param ips: list of ip addresses to block :param permanent: (optional) True=do not allow entries to expire :param timestamp: use this timestamp instead of now() :param url: url or reason to block :returns number of entries in the block list """ timestamp = timestamp or datetime.now() for ip in ips: entry = self._ip_banned.get(ip) if entry: entry.timestamp = timestamp entry.count = cap.config.IPBAN_COUNT * 2 # retain permanent on extra blocks entry.permanent = entry.permanent or permanent cap.logger.warning("%s added to ban list", ip) else: self._ip_banned[ip] = ObjectDict( timestamp=timestamp, count=cap.config.IPBAN_COUNT * 2, permanent=permanent, url=url, ) cap.logger.info("%s updated in ban list", ip) return len(self._ip_banned)
def to_dict(self): """ :return: """ if self._cached: return self._cached # pragma: no cover self._cached = ObjectDict( raw=self.ua_string, browser=dict( family=self.browser.family, version=dict(number=self.browser.version, string=self.browser.version_string), ), os=dict( family=self.os.family, version=dict(number=self.os.version, string=self.os.version_string), ), device=dict( family=self.device.family, brand=self.device.brand, model=self.device.model, type=dict( mobile=self.is_mobile, tablet=self.is_tablet, pc=self.is_pc, bot=self.is_bot, email_client=self.is_email_client, touch_capable=self.is_touch_capable, ), ), ) return self._cached
def __init__(self, app=None, **kwargs): self._ip_banned = {} self._url_blocked = {} self._ip_whitelist = {"127.0.0.1": True} self._url_whitelist = { "^/.well-known/": ObjectDict( pattern=re.compile(r"^/.well-known"), match_type="regex" ), "/favicon.ico": ObjectDict(pattern=re.compile(""), match_type="string"), "/robots.txt": ObjectDict(pattern=re.compile(""), match_type="string"), "/ads.txt": ObjectDict(pattern=re.compile(""), match_type="string"), } if app: self.init_app(app, **kwargs) # pragma: no cover
def prepare_response(body=None, status=httpcode.SUCCESS, headers=None, exception=None): return ObjectDict(body=body or {}, status=status, headers=headers or {}, exception=exception)
def dump(self, token_type=None, scope=None): """ :param token_type: :param scope: :return: """ return ObjectDict( token_type=token_type or cap.config.JWT_DEFAULT_TOKEN_TYPE, scope=scope or cap.config.JWT_DEFAULT_SCOPE, **self.get_raw(), )
def add_url_block(self, url, match_type="regex"): """ add or replace the pattern to the list of url patterns to block :param match_type: regex or string - determines the match strategy to use :param url: regex pattern to match with requested url :return: length of the blocked list """ self._url_blocked[url] = ObjectDict( pattern=re.compile(url), match_type=match_type ) return len(self._url_blocked)
def refresh(self, expires=None): """ :param expires: in seconds :return: """ access_token = self.get_access(expires=expires) decoded = self.decode(access_token) return ObjectDict( access_token=access_token, expires_in=decoded.exp, issued_at=decoded.iat, token_type=cap.config.JWT_DEFAULT_TOKEN_TYPE, scope=cap.config.JWT_DEFAULT_SCOPE, )
def dispatch_request(self, *_, **__): tasks = [] responses = [] try: payload = self._validate_payload() except rpc.RPCError as ex: return ( ObjectDict( jsonrpc=self.version, id=getattr(ex, "req_id", None), error=ex.as_dict(), ), httpcode.BAD_REQUEST, ) for d in payload if isinstance(payload, list) else [payload]: resp = ObjectDict(jsonrpc=self.version, id=None) try: if "id" not in d: tasks.append((self._get_action(d["method"]), { **(d.get("params") or {}) })) else: resp.id = d.get("id") # pylint: disable=invalid-name action = self._get_action(d["method"]) resp.result = action(**(d.get("params") or {})) except rpc.RPCError as ex: resp.error = ex.as_dict() except Exception as ex: # pylint: disable=broad-except cap.logger.exception(ex) mess = str(ex) if cap.debug is True else None resp.error = rpc.RPCInternalError(message=mess).as_dict() if "id" in d: responses.append(resp) self._batch_executor(tasks=tasks, **self._batch_args).run() if not responses: res = Response.no_content() return None, res.status_code, res.headers if isinstance(payload, (list, tuple)): if len(responses) > 1: return responses, httpcode.MULTI_STATUS return responses return responses[0]
def object( cls, required=(), not_required=(), properties=None, all_required=True, additional=False, **kwargs, ): properties = properties or {} if not required and all_required is True: required = [i for i in properties.keys() if i not in not_required] return ObjectDict( type="object", additionalProperties=additional, required=required, properties=properties, **kwargs, )
def _request(self, method, params=None, **kwargs): """ :param method: :param params: :param kwargs: :return: """ kwargs.setdefault("raise_on_exc", True) resp = super().request( self._uri, method=HttpMethod.POST, json=dict( jsonrpc=self._version, method=method, params=params or {}, id=self._request_id, ), **kwargs, ) return resp.body or ObjectDict()
def add(self, ip=None, url=None, timestamp=None): """ increment ban count ip of the current request in the banned list :return: :param ip: optional ip to add (ip ban will by default use current ip) :param url: optional url to display/store :param timestamp: entry time to set :return True if entry added/updated """ ip = ip or self.get_ip() url = url or self.get_url() if self._is_excluded(ip=ip, url=url): return False entry = self._ip_banned.get(ip) # check url block list if no existing entry or existing entry has expired if ( not entry or (entry and (entry.count or 0) < cap.config.IPBAN_COUNT) and self._test_blocked(url, ip=ip) ): self.block([ip], url=url) return True if not timestamp or (timestamp and timestamp > datetime.now()): timestamp = datetime.now() if entry: entry.timestamp = timestamp count = entry.count = entry.count + 1 else: count = 1 self._ip_banned[ip] = ObjectDict(timestamp=timestamp, count=count, url=url) cap.logger.info("%s %s added/updated ban list. Count: %d", ip, url, count) return True
def init_app(self, app, *args, uri=None, ext_name="default", **kwargs): """ :param app: :param uri: :param ext_name: :param args: :param kwargs: """ assert PyMongo is not object, "you must install 'flask_pymongo'" app.config.setdefault("MONGO_URI", "mongodb://localhost") app.config.setdefault("MONGO_OPTS", {}) app.config["MONGO_OPTS"].setdefault("connectTimeoutMS", Seconds.millis) app.config["MONGO_OPTS"].setdefault("serverSelectionTimeoutMS", Seconds.millis) app.config["MONGO_OPTS"].update(**kwargs) # noinspection PyUnresolvedReferences super().init_app(app, uri, *args, **kwargs) setattr(app, "extensions", getattr(app, "extensions", {})) app.extensions["mongo"] = ObjectDict() app.extensions["mongo"][ext_name] = self
def to_dict(self, restricted: bool = False) -> dict: _ = restricted # noinspection PyUnresolvedReferences cols = self.columns() # type: ignore return ObjectDict(**{c: getattr(self, c, None) for c in cols})
def array_object(cls, min_items=0, **kwargs): return ObjectDict(type="array", minItems=min_items, items=cls.object(**kwargs))
def array(cls, items, min_items=0, **kwargs): return ObjectDict(type="array", minItems=min_items, items=items, **kwargs)
def ref(cls, path, **kwargs): return ObjectDict(**{"$ref": f"#{path}", **kwargs})
def anyof(cls, *args, **kwargs): return ObjectDict(anyOf=args if len(args) > 1 else (*args, cls.null), **kwargs)
def decode(cls, token): return ObjectDict(**jwt.decode_token(token))
class Opt: integer = ObjectDict(type=["integer", "null"]) string = ObjectDict(type=["string", "null"]) number = ObjectDict(type=["number", "null"]) boolean = ObjectDict(type=["boolean", "null"])
class Fields: schema = ObjectDict( **{"$schema": "http://json-schema.org/draft-07/schema#"}) null = ObjectDict(type="null") integer = ObjectDict(type="integer") string = ObjectDict(type="string") number = ObjectDict(type="number") boolean = ObjectDict(type="boolean") datetime = ObjectDict(type="string", format="date-time") any_object = ObjectDict(type="object", additionalProperties=True) any = ObjectDict(type=[ "integer", "string", "number", "boolean", "array", "object", "null" ]) class Opt: integer = ObjectDict(type=["integer", "null"]) string = ObjectDict(type=["string", "null"]) number = ObjectDict(type=["number", "null"]) boolean = ObjectDict(type=["boolean", "null"]) @classmethod def oneof(cls, *args, **kwargs): return ObjectDict(oneOf=args if len(args) > 1 else (*args, cls.null), **kwargs) @classmethod def anyof(cls, *args, **kwargs): return ObjectDict(anyOf=args if len(args) > 1 else (*args, cls.null), **kwargs) @classmethod def ref(cls, path, **kwargs): return ObjectDict(**{"$ref": f"#{path}", **kwargs}) @classmethod def enum(cls, *args, **kwargs): return {"enum": args, **kwargs} @classmethod def type(cls, *args, **kwargs): return {"type": args, **kwargs} @classmethod def object( cls, required=(), not_required=(), properties=None, all_required=True, additional=False, **kwargs, ): properties = properties or {} if not required and all_required is True: required = [i for i in properties.keys() if i not in not_required] return ObjectDict( type="object", additionalProperties=additional, required=required, properties=properties, **kwargs, ) @classmethod def array(cls, items, min_items=0, **kwargs): return ObjectDict(type="array", minItems=min_items, items=items, **kwargs) @classmethod def array_object(cls, min_items=0, **kwargs): return ObjectDict(type="array", minItems=min_items, items=cls.object(**kwargs))
def health_system(conf=None, **__): """ :param conf: :return: """ conf = conf() if callable(conf) else (conf or {}) default_conf = dict( SYSTEM_FS_MOUNT_POINTS=("/", ), SYSTEM_CPU_THRESHOLD=90, SYSTEM_MEM_THRESHOLD=90, SYSTEM_FS_THRESHOLD=85, SYSTEM_MEM_INCLUDE_SWAP=True, SYSTEM_DUMP_ALL=False, ) conf = ObjectDict(**{**default_conf, **conf}) resp = ObjectDict(errors=[]) th_mem = conf.SYSTEM_MEM_THRESHOLD th_cpu = conf.SYSTEM_CPU_THRESHOLD th_fs = conf.SYSTEM_FS_THRESHOLD resp.mem = ObjectDict(**psutil.virtual_memory()._asdict()) if resp.mem.percent > th_mem: resp.errors.append( f"high RAM usage: {resp.mem.percent}, threshold: {th_mem}") if conf.SYSTEM_MEM_INCLUDE_SWAP: resp.swap = ObjectDict(**psutil.swap_memory()._asdict()) if resp.swap.percent > th_mem: resp.errors.append( f"high SWAP usage: {resp.swap.percent}, threshold: {th_mem}") resp.cpu = psutil.cpu_percent() if resp.cpu > th_cpu: resp.errors.append(f"high CPU usage: {resp.cpu}, threshold: {th_cpu}") resp.disk = ObjectDict() for f in conf.SYSTEM_FS_MOUNT_POINTS: resp.disk[f] = ObjectDict(**psutil.disk_usage(f)._asdict()) percent = resp.disk[f].percent if percent > th_mem: resp.errors.append( f"high DISK usage on '{f}': {percent}, threshold: {th_fs}") output = resp if conf.SYSTEM_DUMP_ALL else (resp.errors or None) return bool(not resp.errors), dict(messages=output)
def as_dict(self): """ :return: """ return ObjectDict(code=self.code, message=self.message, data=self.data)
def identity(cls): identity = jwt.get_jwt_identity() if isinstance(identity, dict): identity = ObjectDict(**identity) return identity
def oneof(cls, *args, **kwargs): return ObjectDict(oneOf=args if len(args) > 1 else (*args, cls.null), **kwargs)
def request( self, uri, method=HttpMethod.GET, raise_on_exc=False, dump_body=None, chunk_size=None, decode_unicode=False, **kwargs, ): """ :param uri: :param method: :param raise_on_exc: :param dump_body: :param chunk_size: :param decode_unicode: :param kwargs: :return: """ kwargs["auth"] = self.get_auth() if dump_body is None: dump_body = self._dump_body else: dump_body = self._normalize_dump_flag(dump_body) if kwargs.get("stream") is True: # if stream not dump response body dump_body = (dump_body[0], False) try: kwargs.setdefault("timeout", self._timeout) url = self.normalize_url(uri) req = ObjectDict(method=method, url=url, **kwargs) self._logger.info("%s", self.dump_request(req, dump_body[0])) response = send_request(method, self.normalize_url(uri), **kwargs) except NetworkError as exc: self._logger.exception(exc) if raise_on_exc or self._raise_on_exc: raise # pragma: no cover return self.prepare_response(status=httpcode.SERVICE_UNAVAILABLE, exception=exc) log_resp = self.dump_response(response, dump_body[1]) try: response.raise_for_status() self._logger.info("%s", log_resp) except HTTPStatusError as exc: self._logger.warning("%s", log_resp) response = exc.response if raise_on_exc or self._raise_on_exc: raise if kwargs.get("stream") is True: body = response.iter_content(chunk_size, decode_unicode) elif "json" in (response.headers.get("Content-Type") or ""): body = response.json() else: body = response.text return self.prepare_response(body=body, status=response.status_code, headers=dict(response.headers))
def get_raw(cls): return ObjectDict(**jwt.get_jwt())