async def http_request(self, dump_body=None, timeout=None, **kwargs):
"""
:param dump_body:
:param timeout:
:param kwargs:
:return:
"""
if dump_body is None:
dump_body = self._dump_body
else:
dump_body = self._normalize_dump_flag(dump_body)
if timeout is None:
timeout = self._timeout
elif not isinstance(timeout, aiohttp.ClientTimeout):
timeout = aiohttp.ClientTimeout(sock_read=timeout,
sock_connect=timeout)
try:
self._logger.info(
"%s", self.dump_request(ObjectDict(**kwargs), dump_body[0]))
async with aiohttp.ClientSession(
timeout=timeout) as session, session.request(
**kwargs) as resp:
try:
body = await resp.json()
except (aiohttp.ContentTypeError, ValueError, TypeError):
body = await resp.text()
try:
response = ObjectDict(
body=body,
status=resp.status,
headers=dict(resp.headers.items()),
)
log_resp = response
log_resp.text = response.body
log_resp = self.dump_response(log_resp, dump_body[1])
resp.raise_for_status()
self._logger.info("%s", log_resp)
except aiohttp.ClientResponseError as exc:
self._logger.warning("%s", log_resp)
if self._raise_on_exc is True:
raise # pragma: no cover
response.exception = exc
return response
except (
aiohttp.ClientError,
aiohttp.ServerTimeoutError,
asyncio.TimeoutError,
) as exc:
self._logger.exception(exc)
if self._raise_on_exc is True:
raise # pragma: no cover
return ObjectDict(body={},
status=httpcode.SERVICE_UNAVAILABLE,
headers={},
exception=exc)
def create(
self,
identity,
refresh=True,
expires_access=None,
expires_refresh=None,
scope=None,
):
"""
:param identity: user identifier, generally the username
:param refresh: enable refresh token
:param expires_access: in seconds
:param expires_refresh: in seconds
:param scope:
:return:
"""
identity = self.prepare_identity(identity)
access_token = self.get_access(identity=identity, expires=expires_access)
decoded = self.decode(access_token)
resp = ObjectDict(
access_token=access_token,
expires_in=decoded.exp,
issued_at=decoded.iat,
token_type=cap.config.JWT_DEFAULT_TOKEN_TYPE,
scope=scope or cap.config.JWT_DEFAULT_SCOPE,
)
if refresh:
resp.refresh_token = self.get_refresh(
identity=identity, expires=expires_refresh
)
return resp
def health_glances(conf=None, **__):
"""
:param conf:
:return:
"""
conf = conf() if callable(conf) else (conf or {})
default_conf = ObjectDict(
SYSTEM_ENDPOINT="http://localhost:4000/api/2",
SYSTEM_CPU_THRESHOLD=90,
SYSTEM_MEM_THRESHOLD=90,
SYSTEM_FS_THRESHOLD=85,
SYSTEM_MEM_INCLUDE_SWAP=True,
SYSTEM_FS_MOUNT_POINTS=("/", ),
SYSTEM_DUMP_ALL=False,
)
conf = ObjectDict(**{**default_conf, **conf})
resp = ObjectDict(errors=[])
units = ["cpu", "mem", "fs"]
if conf.SYSTEM_MEM_INCLUDE_SWAP:
units.append("memswap")
th_mem = conf.SYSTEM_MEM_THRESHOLD
th_cpu = conf.SYSTEM_CPU_THRESHOLD
th_fs = conf.SYSTEM_FS_THRESHOLD
responses = HTTPBatch().request(
[dict(url=f"{conf.SYSTEM_ENDPOINT}/{u}") for u in units])
for i, r in enumerate(responses):
if r.exception:
return False, str(r.exception)
resp[units[i]] = r.body
if resp.cpu.total > th_cpu:
resp.errors.append(
f"high CPU usage: {resp.cpu.total}, threshold: {th_cpu}")
if resp.mem.percent > th_mem:
resp.errors.append(
f"high RAM usage: {resp.mem.percent}, threshold: {th_mem}")
if conf.SYSTEM_MEM_INCLUDE_SWAP and resp.memswap.percent > th_mem:
resp.errors.append(
f"high SWAP usage: {resp.memswap.percent}, threshold: {th_mem}")
for f in resp.fs:
if f.mnt_point in conf.SYSTEM_FS_MOUNT_POINTS and f.percent > th_fs:
resp.errors.append(
f"high DISK usage on {f.mnt_point}: {f.percent}, threshold: {th_fs}"
)
output = resp if conf.SYSTEM_DUMP_ALL else (resp.errors or None)
return bool(not resp.errors), dict(messages=output)
def block(self, ips, permanent=False, timestamp=None, url="block"):
"""
add a list of ip addresses to the block list
:param ips: list of ip addresses to block
:param permanent: (optional) True=do not allow entries to expire
:param timestamp: use this timestamp instead of now()
:param url: url or reason to block
:returns number of entries in the block list
"""
timestamp = timestamp or datetime.now()
for ip in ips:
entry = self._ip_banned.get(ip)
if entry:
entry.timestamp = timestamp
entry.count = cap.config.IPBAN_COUNT * 2
# retain permanent on extra blocks
entry.permanent = entry.permanent or permanent
cap.logger.warning("%s added to ban list", ip)
else:
self._ip_banned[ip] = ObjectDict(
timestamp=timestamp,
count=cap.config.IPBAN_COUNT * 2,
permanent=permanent,
url=url,
)
cap.logger.info("%s updated in ban list", ip)
return len(self._ip_banned)
def to_dict(self):
"""
:return:
"""
if self._cached:
return self._cached # pragma: no cover
self._cached = ObjectDict(
raw=self.ua_string,
browser=dict(
family=self.browser.family,
version=dict(number=self.browser.version,
string=self.browser.version_string),
),
os=dict(
family=self.os.family,
version=dict(number=self.os.version,
string=self.os.version_string),
),
device=dict(
family=self.device.family,
brand=self.device.brand,
model=self.device.model,
type=dict(
mobile=self.is_mobile,
tablet=self.is_tablet,
pc=self.is_pc,
bot=self.is_bot,
email_client=self.is_email_client,
touch_capable=self.is_touch_capable,
),
),
)
return self._cached
def __init__(self, app=None, **kwargs):
self._ip_banned = {}
self._url_blocked = {}
self._ip_whitelist = {"127.0.0.1": True}
self._url_whitelist = {
"^/.well-known/": ObjectDict(
pattern=re.compile(r"^/.well-known"), match_type="regex"
),
"/favicon.ico": ObjectDict(pattern=re.compile(""), match_type="string"),
"/robots.txt": ObjectDict(pattern=re.compile(""), match_type="string"),
"/ads.txt": ObjectDict(pattern=re.compile(""), match_type="string"),
}
if app:
self.init_app(app, **kwargs) # pragma: no cover
def prepare_response(body=None,
status=httpcode.SUCCESS,
headers=None,
exception=None):
return ObjectDict(body=body or {},
status=status,
headers=headers or {},
exception=exception)
def dump(self, token_type=None, scope=None):
"""
:param token_type:
:param scope:
:return:
"""
return ObjectDict(
token_type=token_type or cap.config.JWT_DEFAULT_TOKEN_TYPE,
scope=scope or cap.config.JWT_DEFAULT_SCOPE,
**self.get_raw(),
)
def add_url_block(self, url, match_type="regex"):
"""
add or replace the pattern to the list of url patterns to block
:param match_type: regex or string - determines the match strategy to use
:param url: regex pattern to match with requested url
:return: length of the blocked list
"""
self._url_blocked[url] = ObjectDict(
pattern=re.compile(url), match_type=match_type
)
return len(self._url_blocked)
def refresh(self, expires=None):
"""
:param expires: in seconds
:return:
"""
access_token = self.get_access(expires=expires)
decoded = self.decode(access_token)
return ObjectDict(
access_token=access_token,
expires_in=decoded.exp,
issued_at=decoded.iat,
token_type=cap.config.JWT_DEFAULT_TOKEN_TYPE,
scope=cap.config.JWT_DEFAULT_SCOPE,
)
def dispatch_request(self, *_, **__):
tasks = []
responses = []
try:
payload = self._validate_payload()
except rpc.RPCError as ex:
return (
ObjectDict(
jsonrpc=self.version,
id=getattr(ex, "req_id", None),
error=ex.as_dict(),
),
httpcode.BAD_REQUEST,
)
for d in payload if isinstance(payload, list) else [payload]:
resp = ObjectDict(jsonrpc=self.version, id=None)
try:
if "id" not in d:
tasks.append((self._get_action(d["method"]), {
**(d.get("params") or {})
}))
else:
resp.id = d.get("id") # pylint: disable=invalid-name
action = self._get_action(d["method"])
resp.result = action(**(d.get("params") or {}))
except rpc.RPCError as ex:
resp.error = ex.as_dict()
except Exception as ex: # pylint: disable=broad-except
cap.logger.exception(ex)
mess = str(ex) if cap.debug is True else None
resp.error = rpc.RPCInternalError(message=mess).as_dict()
if "id" in d:
responses.append(resp)
self._batch_executor(tasks=tasks, **self._batch_args).run()
if not responses:
res = Response.no_content()
return None, res.status_code, res.headers
if isinstance(payload, (list, tuple)):
if len(responses) > 1:
return responses, httpcode.MULTI_STATUS
return responses
return responses[0]
def object(
cls,
required=(),
not_required=(),
properties=None,
all_required=True,
additional=False,
**kwargs,
):
properties = properties or {}
if not required and all_required is True:
required = [i for i in properties.keys() if i not in not_required]
return ObjectDict(
type="object",
additionalProperties=additional,
required=required,
properties=properties,
**kwargs,
)
def _request(self, method, params=None, **kwargs):
"""
:param method:
:param params:
:param kwargs:
:return:
"""
kwargs.setdefault("raise_on_exc", True)
resp = super().request(
self._uri,
method=HttpMethod.POST,
json=dict(
jsonrpc=self._version,
method=method,
params=params or {},
id=self._request_id,
),
**kwargs,
)
return resp.body or ObjectDict()
def add(self, ip=None, url=None, timestamp=None):
"""
increment ban count ip of the current request in the banned list
:return:
:param ip: optional ip to add (ip ban will by default use current ip)
:param url: optional url to display/store
:param timestamp: entry time to set
:return True if entry added/updated
"""
ip = ip or self.get_ip()
url = url or self.get_url()
if self._is_excluded(ip=ip, url=url):
return False
entry = self._ip_banned.get(ip)
# check url block list if no existing entry or existing entry has expired
if (
not entry
or (entry and (entry.count or 0) < cap.config.IPBAN_COUNT)
and self._test_blocked(url, ip=ip)
):
self.block([ip], url=url)
return True
if not timestamp or (timestamp and timestamp > datetime.now()):
timestamp = datetime.now()
if entry:
entry.timestamp = timestamp
count = entry.count = entry.count + 1
else:
count = 1
self._ip_banned[ip] = ObjectDict(timestamp=timestamp, count=count, url=url)
cap.logger.info("%s %s added/updated ban list. Count: %d", ip, url, count)
return True
def init_app(self, app, *args, uri=None, ext_name="default", **kwargs):
"""
:param app:
:param uri:
:param ext_name:
:param args:
:param kwargs:
"""
assert PyMongo is not object, "you must install 'flask_pymongo'"
app.config.setdefault("MONGO_URI", "mongodb://localhost")
app.config.setdefault("MONGO_OPTS", {})
app.config["MONGO_OPTS"].setdefault("connectTimeoutMS", Seconds.millis)
app.config["MONGO_OPTS"].setdefault("serverSelectionTimeoutMS",
Seconds.millis)
app.config["MONGO_OPTS"].update(**kwargs)
# noinspection PyUnresolvedReferences
super().init_app(app, uri, *args, **kwargs)
setattr(app, "extensions", getattr(app, "extensions", {}))
app.extensions["mongo"] = ObjectDict()
app.extensions["mongo"][ext_name] = self
def to_dict(self, restricted: bool = False) -> dict:
_ = restricted
# noinspection PyUnresolvedReferences
cols = self.columns() # type: ignore
return ObjectDict(**{c: getattr(self, c, None) for c in cols})
def array_object(cls, min_items=0, **kwargs):
return ObjectDict(type="array",
minItems=min_items,
items=cls.object(**kwargs))
def array(cls, items, min_items=0, **kwargs):
return ObjectDict(type="array",
minItems=min_items,
items=items,
**kwargs)
def ref(cls, path, **kwargs):
return ObjectDict(**{"$ref": f"#{path}", **kwargs})
def anyof(cls, *args, **kwargs):
return ObjectDict(anyOf=args if len(args) > 1 else (*args, cls.null),
**kwargs)
def decode(cls, token):
return ObjectDict(**jwt.decode_token(token))
class Opt:
integer = ObjectDict(type=["integer", "null"])
string = ObjectDict(type=["string", "null"])
number = ObjectDict(type=["number", "null"])
boolean = ObjectDict(type=["boolean", "null"])
class Fields:
schema = ObjectDict(
**{"$schema": "http://json-schema.org/draft-07/schema#"})
null = ObjectDict(type="null")
integer = ObjectDict(type="integer")
string = ObjectDict(type="string")
number = ObjectDict(type="number")
boolean = ObjectDict(type="boolean")
datetime = ObjectDict(type="string", format="date-time")
any_object = ObjectDict(type="object", additionalProperties=True)
any = ObjectDict(type=[
"integer", "string", "number", "boolean", "array", "object", "null"
])
class Opt:
integer = ObjectDict(type=["integer", "null"])
string = ObjectDict(type=["string", "null"])
number = ObjectDict(type=["number", "null"])
boolean = ObjectDict(type=["boolean", "null"])
@classmethod
def oneof(cls, *args, **kwargs):
return ObjectDict(oneOf=args if len(args) > 1 else (*args, cls.null),
**kwargs)
@classmethod
def anyof(cls, *args, **kwargs):
return ObjectDict(anyOf=args if len(args) > 1 else (*args, cls.null),
**kwargs)
@classmethod
def ref(cls, path, **kwargs):
return ObjectDict(**{"$ref": f"#{path}", **kwargs})
@classmethod
def enum(cls, *args, **kwargs):
return {"enum": args, **kwargs}
@classmethod
def type(cls, *args, **kwargs):
return {"type": args, **kwargs}
@classmethod
def object(
cls,
required=(),
not_required=(),
properties=None,
all_required=True,
additional=False,
**kwargs,
):
properties = properties or {}
if not required and all_required is True:
required = [i for i in properties.keys() if i not in not_required]
return ObjectDict(
type="object",
additionalProperties=additional,
required=required,
properties=properties,
**kwargs,
)
@classmethod
def array(cls, items, min_items=0, **kwargs):
return ObjectDict(type="array",
minItems=min_items,
items=items,
**kwargs)
@classmethod
def array_object(cls, min_items=0, **kwargs):
return ObjectDict(type="array",
minItems=min_items,
items=cls.object(**kwargs))
def health_system(conf=None, **__):
"""
:param conf:
:return:
"""
conf = conf() if callable(conf) else (conf or {})
default_conf = dict(
SYSTEM_FS_MOUNT_POINTS=("/", ),
SYSTEM_CPU_THRESHOLD=90,
SYSTEM_MEM_THRESHOLD=90,
SYSTEM_FS_THRESHOLD=85,
SYSTEM_MEM_INCLUDE_SWAP=True,
SYSTEM_DUMP_ALL=False,
)
conf = ObjectDict(**{**default_conf, **conf})
resp = ObjectDict(errors=[])
th_mem = conf.SYSTEM_MEM_THRESHOLD
th_cpu = conf.SYSTEM_CPU_THRESHOLD
th_fs = conf.SYSTEM_FS_THRESHOLD
resp.mem = ObjectDict(**psutil.virtual_memory()._asdict())
if resp.mem.percent > th_mem:
resp.errors.append(
f"high RAM usage: {resp.mem.percent}, threshold: {th_mem}")
if conf.SYSTEM_MEM_INCLUDE_SWAP:
resp.swap = ObjectDict(**psutil.swap_memory()._asdict())
if resp.swap.percent > th_mem:
resp.errors.append(
f"high SWAP usage: {resp.swap.percent}, threshold: {th_mem}")
resp.cpu = psutil.cpu_percent()
if resp.cpu > th_cpu:
resp.errors.append(f"high CPU usage: {resp.cpu}, threshold: {th_cpu}")
resp.disk = ObjectDict()
for f in conf.SYSTEM_FS_MOUNT_POINTS:
resp.disk[f] = ObjectDict(**psutil.disk_usage(f)._asdict())
percent = resp.disk[f].percent
if percent > th_mem:
resp.errors.append(
f"high DISK usage on '{f}': {percent}, threshold: {th_fs}")
output = resp if conf.SYSTEM_DUMP_ALL else (resp.errors or None)
return bool(not resp.errors), dict(messages=output)
def as_dict(self):
"""
:return:
"""
return ObjectDict(code=self.code, message=self.message, data=self.data)
def identity(cls):
identity = jwt.get_jwt_identity()
if isinstance(identity, dict):
identity = ObjectDict(**identity)
return identity
def oneof(cls, *args, **kwargs):
return ObjectDict(oneOf=args if len(args) > 1 else (*args, cls.null),
**kwargs)
def request(
self,
uri,
method=HttpMethod.GET,
raise_on_exc=False,
dump_body=None,
chunk_size=None,
decode_unicode=False,
**kwargs,
):
"""
:param uri:
:param method:
:param raise_on_exc:
:param dump_body:
:param chunk_size:
:param decode_unicode:
:param kwargs:
:return:
"""
kwargs["auth"] = self.get_auth()
if dump_body is None:
dump_body = self._dump_body
else:
dump_body = self._normalize_dump_flag(dump_body)
if kwargs.get("stream") is True: # if stream not dump response body
dump_body = (dump_body[0], False)
try:
kwargs.setdefault("timeout", self._timeout)
url = self.normalize_url(uri)
req = ObjectDict(method=method, url=url, **kwargs)
self._logger.info("%s", self.dump_request(req, dump_body[0]))
response = send_request(method, self.normalize_url(uri), **kwargs)
except NetworkError as exc:
self._logger.exception(exc)
if raise_on_exc or self._raise_on_exc:
raise # pragma: no cover
return self.prepare_response(status=httpcode.SERVICE_UNAVAILABLE,
exception=exc)
log_resp = self.dump_response(response, dump_body[1])
try:
response.raise_for_status()
self._logger.info("%s", log_resp)
except HTTPStatusError as exc:
self._logger.warning("%s", log_resp)
response = exc.response
if raise_on_exc or self._raise_on_exc:
raise
if kwargs.get("stream") is True:
body = response.iter_content(chunk_size, decode_unicode)
elif "json" in (response.headers.get("Content-Type") or ""):
body = response.json()
else:
body = response.text
return self.prepare_response(body=body,
status=response.status_code,
headers=dict(response.headers))
def get_raw(cls):
return ObjectDict(**jwt.get_jwt())
