class ConsoleLoggerInterface(LoggerInterface):
def __init__(self, name, level=None, style=None):
super().__init__(name, level=level)
self._json = JsonEncoder()
self._style = style or 'json'
def process_message(self, level: Text, message: Text, data: Dict) -> Text:
when = TimeUtils.utc_now().strftime('%m/%d/%Y %H:%M:%S')
level = level.upper()[0]
if data:
data = self._json.decode(self._json.encode(data))
if self._style == 'json':
dumped_data = self._to_json(data)
elif self._style == 'yaml':
dumped_data = self._to_yaml(data)
else:
raise ValueError(f'unrcognized log style: {self.style}')
else:
dumped_data = None
thread = StringUtils.dash(current_thread().name)
display_string = (f'{when} ({level}) {self._name}' f' - {message}')
if dumped_data:
display_string += f'\n\n{dumped_data}\n'
return display_string
def _to_json(self, data):
return json.dumps(data, indent=2, sort_keys=True)
def _to_yaml(self, data):
lines = yaml.dump(data, default_flow_style=False).split('\n')
return '\n'.join(' ' + line for line in lines)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.encoder = JsonEncoder()
self.field_2_index_type = {
fields.String: StringIndex,
fields.Int: NumericIndex,
fields.Float: NumericIndex,
fields.DateTime: NumericIndex,
fields.Timestamp: NumericIndex,
fields.Uuid: NumericIndex,
fields.Bool: NumericIndex,
}
def read_body_json_object(self):
body = {}
content_len = int(self.headers.get('content-length', 0))
if content_len:
body_bytes = self.rfile.read(content_len)
if body_bytes:
body = JsonEncoder.decode(body_bytes.decode())
return body
class AES:
def __init__(self, key: Text = None, iv: bytes = None):
self.json = JsonEncoder()
self.iv = iv or get_random_bytes(16)
self.key = (key or random_uuid().hex).encode('utf-8')
def encrypt_CBC(self, value: object) -> Text:
json_str = self.json.encode(value).encode('utf-8')
cipher = AES_Cipher.new(self.key, AES_Cipher.MODE_CBC, self.iv)
enc_bytes = cipher.encrypt(pad(json_str, AES_Cipher.block_size))
return b64encode(enc_bytes).decode('utf-8')
def decrypt_CBC(self, encrypted_value: Text) -> object:
cipher = AES_Cipher.new(self.key, AES_Cipher.MODE_CBC, self.iv)
enc_bytes = b64decode(encrypted_value.encode('utf-8'))
json_str = unpad(cipher.decrypt(enc_bytes), AES_Cipher.block_size)
return self.json.decode(json_str)
def fetch_many(self, _ids: List, fields: Set[Text] = None) -> Dict:
pipe = self.redis.pipeline()
self.records.get_many(_ids, pipe=pipe)
self.revs.get_many(_ids, pipe=pipe)
json_records, rev_strs = pipe.execute()
fields = fields if isinstance(fields, set) else set(fields or [])
records = {}
if fields:
for record_json, _rev in zip(json_records, rev_strs):
record = JsonEncoder.decode(record_json)
record = {k: record.get(k) for k in fields}
record[REV] = int(_rev) - 1
records[record[ID]] = record
else:
for record_json, _rev in zip(json_records, rev_strs):
record = JsonEncoder.decode(record_json)
record[REV] = int(_rev) - 1
records[record[ID]] = record
return records
def query(self, predicate: 'Predicate', **kwargs):
if predicate is None:
_ids = self.records.keys()
else:
_ids = self.query_ids(predicate)
records = []
if _ids:
_id_field = self.resource_type.Schema.fields[ID]
json_records = self.records.get_many(_ids)
rev_strs = self.revs.get_many(_ids)
for json_record, rev_str in zip(json_records, rev_strs):
record = JsonEncoder.decode(json_record)
record[REV] = int(rev_str.decode())
records.append(record)
return records
def fetch(self, _id, fields: Set[Text] = None) -> Dict:
fields = fields if isinstance(fields, set) else set(fields or [])
pipe = self.redis.pipeline()
self.records.get(_id)
self.revs.get(_id)
record_json, rev_str = pipe.execute()
if not record_json:
return None
full_record = JsonEncoder.decode(record_json)
full_record[REV] = int(rev_str) - 1
if fields:
return {k: full_record.get(k) for k in fields}
else:
return full_record
def __init__(
self,
middleware: List['Middleware'] = None,
manifest: Manifest = None,
mode: Mode = Mode.normal,
):
# thread-local storage
self.local = local()
self.local.is_bootstrapped = False
self.local.is_started = False
self.local.manifest = Manifest(manifest) if manifest else None
self.local.bootstrapper_thread_id = None
self.local.thread_executor = None
self.local.process_executor = None
self.local.tx_manager = None
self.shared = DictObject() # storage shared by threads
self.shared.manifest_data = None
self.shared.app_counter = 1
self.env = Environment()
self._mode = mode
self._json = JsonEncoder()
self._binder = ResourceBinder()
self._namespace = {}
self._arg_loader = None
self._logger = None
self._root_pid = os.getpid()
self._actions = DictObject()
self._middleware = deque([
m for m in (middleware or [])
if isinstance(self, m.app_types)
])
# set default main thread name
current_thread().name = (
f'{get_class_name(self)}MainThread'
)
class RedisStore(Store):
redis = None
host = 'localhost'
port = 6379
db = 0
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.encoder = JsonEncoder()
self.field_2_index_type = {
fields.String: StringIndex,
fields.Int: NumericIndex,
fields.Float: NumericIndex,
fields.DateTime: NumericIndex,
fields.Timestamp: NumericIndex,
fields.Uuid: NumericIndex,
fields.Bool: NumericIndex,
}
@classmethod
def on_bootstrap(cls, host=None, port=None, db=None):
cls.host = host or cls.env.REDIS_HOST or cls.host
cls.port = port or cls.env.REDIS_PORT or cls.port
cls.db = db if db is not None else (cls.env.REDIS_DB or cls.db)
cls.redis = RedisClient(host=cls.host, port=cls.port, db=cls.db)
def on_bind(self, resource_type: Type['Resource'], **kwargs):
self.type_name = StringUtils.snake(resource_type.__name__).lower()
self.records = HashSet(self.redis, self.type_name)
self.revs = HashSet(self.redis, f'{self.type_name}_revisions')
self.indexes = self._bind_indexes()
def _bind_indexes(self):
indexes = {}
for k, field in self.resource_type.Schema.fields.items():
index_name = f'{self.type_name}:{k.lower()}'
index_type = self.field_2_index_type.get(field.__class__)
if index_type is None:
index_type = StringIndex
indexes[k] = index_type(self.redis, index_name)
return indexes
def exists(self, _id) -> bool:
return (_id in self.records)
def exists_many(self, _ids: Set) -> Dict[object, bool]:
return {
_id: (_id in self.records)
for _id in _ids
}
def count(self) -> int:
return len(self.records)
def fetch(self, _id, fields: Set[Text] = None) -> Dict:
fields = fields if isinstance(fields, set) else set(fields or [])
pipe = self.redis.pipeline()
self.records.get(_id)
self.revs.get(_id)
record_json, rev_str = pipe.execute()
if not record_json:
return None
full_record = JsonEncoder.decode(record_json)
full_record[REV] = int(rev_str) - 1
if fields:
return {k: full_record.get(k) for k in fields}
else:
return full_record
def fetch_many(self, _ids: List, fields: Set[Text] = None) -> Dict:
pipe = self.redis.pipeline()
self.records.get_many(_ids, pipe=pipe)
self.revs.get_many(_ids, pipe=pipe)
json_records, rev_strs = pipe.execute()
fields = fields if isinstance(fields, set) else set(fields or [])
records = {}
if fields:
for record_json, _rev in zip(json_records, rev_strs):
record = JsonEncoder.decode(record_json)
record = {k: record.get(k) for k in fields}
record[REV] = int(_rev) - 1
records[record[ID]] = record
else:
for record_json, _rev in zip(json_records, rev_strs):
record = JsonEncoder.decode(record_json)
record[REV] = int(_rev) - 1
records[record[ID]] = record
return records
def fetch_all(self, fields: Set[Text] = None) -> Dict:
return self.fetch_many(list(self.records.keys()), fields=fields)
def upsert(self, record: Dict, pipe=None) -> Dict:
is_creating = record.get(ID) is None
_id = self.create_id(record)
# prepare the record for upsert
upserted_record = record.copy()
upserted_record[ID] = _id
# json encode and store record JSON
self.records[_id] = self.encoder.encode(upserted_record)
# update indexes
for k, v in record.items():
self.indexes[k].upsert(_id, v)
# add rev to record AFTER insert to avoid storing _rev in records hset
if is_creating:
upserted_record[REV] = 0
else:
upserted_record[REV] = self.revs.increment(_id)
return upserted_record
def create(self, data: Dict) -> Dict:
pipe = self.redis.pipeline()
created_record = self.upsert(data)
pipe.execute()
return created_record
def update(self, _id, record: Dict) -> Dict:
pipe = self.redis.pipeline()
updated_record = self.upsert(record, pipe=pipe)
pipe.execute()
return updated_record
def create_many(self, records: List[Dict]) -> Dict:
pipe = self.redis.pipeline()
created_records = [self.upsert(rec, pipe=pipe) for rec in records]
pipe.execute()
return created_records
def update_many(self, _ids: List, data: Dict = None) -> Dict:
pipe = self.redis.pipeline()
updated_records = [self.upsert(rec, pipe=pipe) for rec in records]
pipe.execute()
return updated_records
def delete(self, _id) -> None:
pipe = self.redis.pipeline()
if self.records.delete(_id, pipe=pipe):
for index in self.indexes.values():
index.delete(_id, pipe=pipe)
pipe.execute()
def delete_many(self, _ids: List) -> None:
pipe = self.redis.pipeline()
if self.records.delete_many(_ids, pipe=pipe):
self.revs.delete_many(_ids)
for index in self.indexes.values():
index.delete_many(_ids, pipe=pipe)
pipe.execute()
def delete_all(self):
if self.count():
self.delete_many(self.records.keys())
def query(self, predicate: 'Predicate', **kwargs):
if predicate is None:
_ids = self.records.keys()
else:
_ids = self.query_ids(predicate)
records = []
if _ids:
_id_field = self.resource_type.Schema.fields[ID]
json_records = self.records.get_many(_ids)
rev_strs = self.revs.get_many(_ids)
for json_record, rev_str in zip(json_records, rev_strs):
record = JsonEncoder.decode(json_record)
record[REV] = int(rev_str.decode())
records.append(record)
return records
def query_ids(self, predicate, pipeline=None):
# TODO: use a redis pipeline object
if isinstance(predicate, ConditionalPredicate):
index = self.indexes.get(predicate.field.name)
if index is None:
# TODO: raise custom exception
raise Exception('no index')
ids = set()
if predicate.op == OP_CODE.EQ:
ids = set(index.search(
upper=predicate.value,
lower=predicate.value,
include_upper=True,
include_lower=True
))
elif predicate.op == OP_CODE.NEQ:
ids = set(index.search(
upper=predicate.value,
include_upper=False,
include_lower=True
))
ids |= set(index.search(
lower=predicate.value,
include_upper=True,
include_lower=False
))
elif predicate.op == OP_CODE.GEQ:
ids = set(index.search(
lower=predicate.value,
include_lower=True
))
elif predicate.op == OP_CODE.LEQ:
ids = set(index.search(
upper=predicate.value,
include_upper=True
))
elif predicate.op == OP_CODE.LT:
ids = set(index.search(
upper=predicate.value,
include_upper=False
))
elif predicate.op == OP_CODE.GT:
ids = set(index.search(
lower=predicate.value,
include_lower=False
))
elif predicate.op == OP_CODE.INCLUDING:
ids = set()
for v in predicate.value:
ids |= set(index.search(
upper=v,
lower=v,
include_upper=True,
include_lower=True
))
elif predicate.op == OP_CODE.EXCLUDING:
ids = set()
for v in predicate.value:
ids |= set(index.search(
upper=v,
include_upper=False,
include_lower=True
))
ids |= set(index.search(
lower=v,
include_upper=True,
include_lower=False
))
else:
raise ValueError('unrecognized op: {}'.format(predicate.op))
elif isinstance(predicate, BooleanPredicate):
ids_lhs = self.query_ids(predicate.lhs)
if ids_lhs:
ids_rhs = self.query_ids(predicate.rhs)
else:
ids_rhs = set()
if predicate.op == OP_CODE.OR:
ids = ids_lhs | ids_rhs
elif predicate.op == OP_CODE.AND:
ids = ids_lhs & ids_rhs
else:
raise ValueError('unrecognized op: {}'.format(predicate.op))
return ids
def __init__(self, name, level=None, style=None):
super().__init__(name, level=level)
self._json = JsonEncoder()
self._style = style or 'json'
def __init__(self, hostname: Text = None):
self.hostname = (hostname or '').rstrip('/')
self.json = JsonEncoder()
class JsonSchemaGenerator:
def __init__(self, hostname: Text = None):
self.hostname = (hostname or '').rstrip('/')
self.json = JsonEncoder()
def from_schema(self, schema: 'Schema', encode=False) -> Dict:
schema_class_name = self.get_schema_name(schema)
json_schema = {
'$schema': 'http://json-schema.org/schema#',
'$id': f'{self.hostname}/schemas/{schema_class_name}.json',
'properties': self._derive_properties(schema),
'required': list(schema.required_fields.keys()),
}
if encode:
return self.json.encode(json_schema)
return json_schema
def from_resource(self, resource_type: Type['Resource'], encode=False) -> Dict:
json_schema = self.from_schema(resource_type.ravel.schema)
for resolver in resource_type.ravel.resolvers.values():
if resolver.private:
continue
if resolver.name not in resource_type.ravel.schema.fields:
target_schema = resolver.target.ravel.schema
field_dict = self._derive_property(target_schema)
if resolver.many:
json_schema['properties'][resolver.name] = {
'type': 'array',
'default': [],
'items': field_dict
}
else:
json_schema[resolver.name] = field_dict
if encode:
return self.json.encode(json_schema)
return json_schema
def _derive_properties(self, schema):
properties = {}
for field in schema.fields.values():
if not field.meta.get('private', False):
prop = self._derive_property(field)
properties[field.name] = prop
return properties
def _derive_property(self, field: 'Field'):
field_dict = {}
if isinstance(field, field_types.String):
field_dict['type'] = 'string'
format_str = STRING_FIELD_TYPE_2_FORMAT.get(type(field))
if format_str is not None:
field_dict['format'] = format_str
elif isinstance(field, field_types.Bool):
field_dict['type'] = 'boolean'
elif isinstance(field, field_types.Int):
field_dict['type'] = 'integer'
elif isinstance(field, field_types.Float):
field_dict['type'] = 'number'
elif isinstance(field, field_types.DateTime):
field_dict['type'] = 'integer'
field_dict['format'] = 'date-time'
elif isinstance(field, field_types.DateTimeString):
field_dict['type'] = 'string'
field_dict['format'] = 'date-time'
elif isinstance(field, field_types.Dict):
field_dict['type'] = 'object'
elif isinstance(field, (field_types.List, field_types.Set)):
field_dict['type'] = 'array'
field_dict['items'] = self._derive_property(field.nested)
field_dict['default'] = []
elif isinstance(field, field_types.Enum):
inner_field_dict = self._derive_property(field.nested)
field_dict['type'] = inner_field_dict['type']
field_dict['enum'] = list(field.values)
elif isinstance(field, Schema):
defined_name = self.get_schema_name(field)
field_dict['$ref'] = defined_name
elif isinstance(field, field_types.Nested):
defined_name = self.get_schema_name(field.schema_type)
field_dict['$ref'] = defined_name
elif isinstance(field, field_types.Uuid):
field_dict['type'] = 'string'
field_dict['format'] = 'uuid'
else:
field_dict['type'] = 'null'
if field.nullable:
field_dict['type'] = [field_dict['type'], 'null']
return field_dict
@staticmethod
def get_schema_name(schema: 'Schema') -> Text:
name = get_class_name(schema)
if name.endswith('Schema'):
name = name[:-6]
return name
def __init__(self, json_encode=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.handler = type('Handler', (self.Handler, ), {'service': self})
self.json_encode = json_encode or JsonEncoder().encode
self.server = None
from typing import Text, Type
import sqlalchemy as sa
from sqlalchemy import ForeignKey
from appyratus.utils.string_utils import StringUtils
from ravel.constants import REV, ID
from ravel.util.loggers import console
from ravel.util.json_encoder import JsonEncoder
from ravel.util.misc_functions import get_class_name
from ravel.schema import fields, Id
json_encoder = JsonEncoder()
class SqlalchemyTableBuilder(object):
"""
The role of SqlalchemyTableBuilder is to derive and instantiate a
Sqlalchemy Table object from a Resource object's schema. This process
occurs in the Sqlalchemy's on_bootstrap method and assumes that the
column names in the table map onto each schema's field.source attribute.
"""
def __init__(self, store: 'SqlalchemyStore'):
self._dialect = store.dialect
self._resource_type = store.resource_type
self._adapters = store.adapters
self._metadata = store.get_metadata()
@property
def __init__(self, key: Text = None, iv: bytes = None):
self.json = JsonEncoder()
self.iv = iv or get_random_bytes(16)
self.key = (key or random_uuid().hex).encode('utf-8')
from __future__ import absolute_import
from ravel.util.json_encoder import JsonEncoder
from falcon import HTTPError, status_codes
json = JsonEncoder()
class RavelFalconError(HTTPError):
http_status = status_codes.HTTP_400
api_status = 'bad-request'
default_message = 'bad request'
def __init__(self, message=None, data=None):
self.data = data or {}
self.message = message or self.default_message
super().__init__(status=self.http_status, description=self.message)
def __repr__(self):
return '<FalconError({})>'.format(self.__class__.__name__)
def to_dict(self):
return {
'status': self.http_status,
'message': self.message,
'data': self.data,
}
def to_json(self):
def __init__(self, router=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self._json_encoder = JsonEncoder()
self._route_2_resource = {}
self._router = router