def build(self, source) -> 'Query':
query = self.right_loader_property.resolver.owner.select()
if is_resource(source):
source_value = getattr(source, self.left_field.name)
if not source_value:
console.debug(
message=(f'relationship {get_class_name(source)}.'
f'{self.relationship.name} aborting execution'
f'because {self.left_field.name} is None'),
data={
'resource': source._id,
})
# NOTE: ^ if you don't want this, then clear the field from
# source using source.clean(field_name)
return None
query.where(self.right_loader_property == source_value)
else:
assert is_batch(source)
left_field_name = self.left_field.name
source_values = {getattr(res, left_field_name) for res in source}
if not source_values:
console.warning(
message=(f'relationship {get_class_name(source)}.'
f'{self.relationship.name} aborting query '
f'because {self.left_field.name} is empty'),
data={
'resources': source._id,
})
# NOTE: ^ if you don't want this, then clear the field from
# source using source.clean(field_name)
return None
query.where(self.right_loader_property.including(source_values))
return query
def create_many(self, records: List[Dict]) -> Dict:
prepared_records = []
nullable_fields = self.resource_type.Schema.nullable_fields
for record in records:
record[self.id_column_name] = self.create_id(record)
prepared_record = self.prepare(record, serialize=True)
prepared_records.append(prepared_record)
for nullable_field in nullable_fields.values():
if nullable_field.name not in prepared_record:
prepared_record[nullable_field.name] = None
try:
self.conn.execute(self.table.insert(), prepared_records)
except Exception:
console.error(f'failed to insert records')
raise
n = len(prepared_records)
id_list_str = (', '.join(
str(x['_id'])[:7] for x in prepared_records if x.get('_id')))
console.debug(f'SQL: INSERT {id_list_str} INTO {self.table} ' +
(f'(count: {n})' if n > 1 else ''))
if self.supports_returning:
# TODO: use implicit returning if possible
pass
return self.fetch_many((rec[self.id_column_name] for rec in records),
as_list=True)
def commit(cls, rollback=True, **kwargs):
"""
Call commit on the thread-local database transaction. "Begin" must be
called to start a new transaction at this point, if a new transaction
is desired.
"""
def perform_sqlalchemy_commit():
tx = getattr(cls.ravel.local, 'sqla_tx', None)
if tx is not None:
cls.ravel.local.sqla_tx.commit()
cls.ravel.local.sqla_tx = None
# try to commit the transaction.
console.debug(f'committing sqlalchemy transaction')
try:
perform_sqlalchemy_commit()
except Exception:
if rollback:
# if the commit fails, rollback the transaction
console.critical(f'rolling back sqlalchemy transaction')
cls.rollback()
else:
console.exception(f'sqlalchemy transaction failed commit')
finally:
# ensure we close the connection either way
cls.close()
def inherit_base_manifest(
cls,
base: Text = None,
visited: Set = None,
data: Dict = None,
):
"""
If there is a base manifest file specified by the loaded manifest
data, then load it here and merge the current manifest into the base
manifest, recusively back to the root base.
"""
visited = set() if visited is None else visited
data = data if data is not None else {}
base = os.path.abspath(os.path.expandvars(os.path.expanduser(base)))
if base in visited:
# avoid infinite loop if there is a bad inheritence cycle
raise ManifestInheritanceError(
f'manifest inheritance loop detected: {base}')
else:
visited.add(base)
console.debug(f'inheriting manifest from {base}')
if os.path.isfile(base):
# merge current data dict into new base dict
base_data = cls._read_file(base)
data = DictUtils.merge(base_data.copy(), data)
# recurse on base's base manifest...
nested_base = base_data.get('base')
if nested_base:
cls.inherit_base_manifest(nested_base, visited, data)
return data
def update_many(self, _ids: List, data: List[Dict] = None) -> None:
assert data
prepared_ids = []
prepared_records = []
for _id, record in zip(_ids, data):
prepared_id = self.adapt_id(_id)
prepared_record = self.prepare(record, serialize=True)
if prepared_record:
prepared_ids.append(prepared_id)
prepared_records.append(prepared_record)
prepared_record[ID] = prepared_id
if prepared_records:
n = len(prepared_records)
console.debug(f'SQL: UPDATE {self.table} ' +
(f'({n}x)' if n > 1 else ''))
values = {k: bindparam(k) for k in prepared_records[0].keys()}
update_stmt = (self.table.update().where(
self._id_column == bindparam(self.id_column_name)).values(
**values))
self.conn.execute(update_stmt, prepared_records)
if self._options.get('fetch_on_update', True):
if self.supports_returning:
# TODO: use implicit returning if possible
return self.fetch_many(_ids)
else:
return self.fetch_many(_ids)
return
def add_action(self, action: 'Action', overwrite=False, *args, **kwargs):
"""
Add an action to this app.
"""
if action.name not in self._actions or overwrite:
console.debug(f'registered action {action.name}')
if isinstance(action, Action):
if action.app is not self:
if args or kwargs:
console.warning(
'*args and **kwargs ignored by add_action'
)
decorator = self.action(
*action.decorator.args,
**action.decorator.kwargs
)
decorator(action.target)
else:
self._actions[action.name] = action
else:
assert callable(action)
decorator = self.action(*args, **kwargs)
decorator(action)
else:
raise ApplicationError(
message=f'action already registered: {action.name}',
data={'action': action}
)
def get_store_instance(
self,
resource_type: Type['Resource'],
bind=True,
rebind=False,
) -> 'Store':
if isinstance(resource_type, str):
binding = self._bindings.get(resource_type)
else:
binding = self._bindings.get(get_class_name(resource_type))
if binding is None:
# lazily register a new binding
base_store_type = resource_type.__store__()
console.debug(
f'calling {get_class_name(resource_type)}.__store__()')
binding = self.register(resource_type, base_store_type)
# call bind only if it already hasn't been called
if rebind or ((not binding.is_bound) and bind):
console.debug(message=(
f'setting {get_class_name(binding.resource_type)}.ravel.store '
f'= {get_class_name(binding.store_instance)}()'))
binding.bind(binder=self)
return binding.store_instance
def on_request(self, action, request, *args, **kwargs):
"""
Take the attributes on the incoming protobuf Message object and
map them to the args and kwargs expected by the action target.
"""
console.debug(message=f'RPC method {action.name} requested', )
# get field data to process into args and kwargs
def deserialize_value(field, value):
if isinstance(field, fields.Dict):
if not value:
value = None if field.nullable else {}
else:
value = self.json.decode(value)
return value
all_arguments = {}
for field in action.schemas.request.fields.values():
value = getattr(request, field.name, None)
all_arguments[field.name] = deserialize_value(field, value)
# add a dummy request object so that partition_arguments can
# run below; then remove it from the resulting args tuple.
all_arguments['request'] = None
# process field_data into args and kwargs
args, kwargs = FuncUtils.partition_arguments(action.signature,
all_arguments)
args = args[1:]
return (args, kwargs)
def bootstrap(self, app: 'Application'):
self._app = app
console.debug(
f'bootstrapping {get_class_name(self)} middleware singleton')
self.on_bootstrap()
self._is_bootstrapped = True
def _scan_filesystem(self) -> ManifestScanner:
"""
Use venusian simply to scan the action packages/modules, causing the
action callables to register themselves with the Application instance.
"""
t1 = datetime.now()
package = self.package
executor = ThreadPoolExecutor(
max_workers=6,
thread_name_prefix=(f'{StringUtils.camel(self.package or "")}'
f'ManifestWorker'.strip()))
scanner = ManifestScanner(self)
futures = []
def scan(package, verbose=False):
console.debug(f'manifest scanning {package}')
try:
scanner.scan(package)
except:
console.exception('scan failed')
def async_scan(package, verbose=False):
future = executor.submit(scan, package, verbose)
futures.append(future)
async_scan('ravel.resource')
async_scan('ravel.store')
installed_pkg_names = {pkg.key for pkg in pkg_resources.working_set}
# scan extension directories if the package installation requirements
# are met, like sqlalchemy and redis.
pgk_name_2_ravel_scan_path = {
'sqlalchemy': 'ravel.ext.sqlalchemy',
'redis': 'ravel.ext.redis',
'pygame': 'ravel.ext.gaming.pygame',
'celery': 'ravel.ext.celery',
'numpy': 'ravel.ext.np'
}
for pkg_name, scan_path in pgk_name_2_ravel_scan_path.items():
if pkg_name in installed_pkg_names:
async_scan(scan_path)
# scan the app project package
if package:
async_scan(package, verbose=True)
completed_scans, incomplete_scans = (concurrent.futures.wait(futures))
if incomplete_scans:
raise FilesystemScanTimeout(message='filesystem scan timed out', )
t2 = datetime.now()
secs = (t2 - t1).total_seconds()
console.debug(f'scanned filesystem in {secs:.2f}s')
return scanner
def close(cls):
"""
Return the thread-local database connection to the sqlalchemy
connection pool (AKA the "engine").
"""
sqla_conn = getattr(cls.ravel.local, 'sqla_conn', None)
if sqla_conn is not None:
console.debug('closing sqlalchemy connection')
sqla_conn.close()
cls.ravel.local.sqla_conn = None
else:
console.warning('sqlalchemy has no connection to close')
def bind(self, resource_type: Type['Resource'], **kwargs):
t1 = datetime.now()
self._resource_type = resource_type
self.on_bind(resource_type, **kwargs)
t2 = datetime.now()
secs = (t2 - t1).total_seconds()
console.debug(f'bound {TypeUtils.get_class_name(resource_type)} to '
f'{TypeUtils.get_class_name(self)} '
f'in {secs:.2f}s')
self._is_bound = True
def bind(cls, store: 'Store', **kwargs):
t1 = datetime.now()
cls.ravel.local.store = store
for resolver in cls.ravel.resolvers.values():
resolver.bind()
cls.on_bind()
cls.ravel.is_bound = True
t2 = datetime.now()
secs = (t2 - t1).total_seconds()
console.debug(f'bound {TypeUtils.get_class_name(store)} to '
f'{TypeUtils.get_class_name(cls)} '
f'in {secs:.2f}s')
def bootstrap(cls, app: 'Application', **kwargs):
t1 = datetime.now()
cls.ravel.app = app
# bootstrap all resolvers owned by this class
for resolver in cls.ravel.resolvers.values():
if not resolver.is_bootstrapped():
resolver.bootstrap(cls.ravel.app)
# lastly perform custom developer logic
cls.on_bootstrap(app, **kwargs)
cls.ravel.local.is_bootstrapped = True
t2 = datetime.now()
secs = (t2 - t1).total_seconds()
console.debug(f'bootstrapped {TypeUtils.get_class_name(cls)} '
f'in {secs:.2f}s')
def post_request(
self,
action: 'Action',
request: 'Request',
result,
):
"""
Create and set a new token if the current one is expired or
non-existent.
"""
http_request, http_response = request.raw_args[:2]
csrf_data = request.context.get('csrf')
session = request.session
regenerate_token = action.decorator.kwargs.get('refresh_csrf_token',
False)
if (not csrf_data) or regenerate_token:
console.debug(f'generating CSRF token for session {session._id}')
self.create_and_set_token(http_request, http_response, session)
def update(self, _id, data: Dict) -> Dict:
prepared_id = self.adapt_id(_id)
prepared_data = self.prepare(data, serialize=True)
if prepared_data:
update_stmt = (self.table.update().values(**prepared_data).where(
self._id_column == prepared_id))
else:
return prepared_data
if self.supports_returning:
update_stmt = update_stmt.return_defaults()
console.debug(f'SQL: UPDATE {self.table}')
result = self.conn.execute(update_stmt)
return dict(data, **(result.returned_defaults or {}))
else:
self.conn.execute(update_stmt)
if self._options.get('fetch_on_update', True):
return self.fetch(_id)
return data
def begin(cls, auto_connect=True, **kwargs):
"""
Initialize a thread-local transaction. An exception is raised if
there's already a pending transaction.
"""
conn = cls.get_active_connection()
if conn is None:
if auto_connect:
conn = cls.connect()
else:
raise Exception('no active sqlalchemy connection')
existing_tx = getattr(cls.ravel.local, 'sqla_tx', None)
if existing_tx is not None:
console.debug('there is already an open transaction')
else:
new_tx = cls.ravel.local.sqla_conn.begin()
cls.ravel.local.sqla_tx = new_tx
def bootstrap(cls, app: 'Application' = None, **kwargs):
"""
Perform class-level initialization, like getting
a connectio pool, for example.
"""
t1 = datetime.now()
cls.ravel.app = app
cls.on_bootstrap(**kwargs)
cls.ravel.local.is_bootstrapped = True
t2 = datetime.now()
secs = (t2 - t1).total_seconds()
console.debug(f'bootstrapped {TypeUtils.get_class_name(cls)} '
f'in {secs:.2f}s')
return cls
def _load_manifest_data(cls, source):
schema = cls.Schema(allow_additional=True)
filepath = None
data = {}
# if source is callable, it means that the manifest is being returned
# by a function, allowing for dynamic manifest properties
if callable(source):
source = source()
# load the manifest data dict differently, depending
# on its source -- e.g. from a file path, a dict, another manifest
if isinstance(source, Manifest):
data = deepcopy(source.data)
filepath = source.filepath
elif isinstance(source, str):
filepath = os.path.abspath(
os.path.expandvars(os.path.expanduser(source)))
if os.path.isfile(source):
console.debug(f'reading manifest from {source}')
data = cls._read_file(source)
else:
raise ManifestFileNotFound(
f'manifest file {filepath} not found. '
f'yaml and json manifest file types are supported.')
elif isinstance(source, dict):
data = source
filepath = None
# merge data dict into recursively inherited data dict
base_filepath = data.get('base')
if base_filepath:
inherited_data = cls.inherit_base_manifest(base_filepath)
data = DictUtils.merge(inherited_data, data)
data = cls._expand_vars(data)
# validate final computed data dict
validated_data, errors = schema.process(data)
if errors:
raise ManifestValidationError(
f'manifest validation error/s: {errors}')
return (filepath, validated_data)
def _scan_namespace(self, namespace: Dict):
"""
Non-recursively detect Store and Resource class objects contained in
the given namespace dict, making them available to the bootstrapping
app.
"""
from ravel.store import Store
from ravel.resource import Resource
for k, v in (namespace or {}).items():
if TypeUtils.is_proper_subclass(v, Resource):
if not v.ravel.is_abstract:
self.resource_classes[k] = v
console.debug(f'detected Resource class in '
f'namespace dict: {get_class_name(v)}')
elif TypeUtils.is_proper_subclass(v, Store):
self.store_classes[k] = v
console.debug(f'detected Store class in namespace '
f'dict: {get_class_name(v)}')
def _bootstrap_app_actions(self):
for action in self.app.actions.values():
action.bootstrap()
on_parts = []
pre_parts = []
post_parts = deque()
def is_virtual(func):
return getattr(func, 'is_virtual', None)
# middleware bootstrapping...
for idx, mware in enumerate(self.app.middleware):
mware.bootstrap(app=self.app)
# everything below is for generating the log message
# containing the "action execution diagram"
name = StringUtils.snake(get_class_name(mware))
if not is_virtual(mware.pre_request):
pre_parts.append(f"↪ {name}.pre_request(raw_args, raw_kwargs)")
if not is_virtual(mware.on_request):
on_parts.append(f"↪ {name}.on_request(args, kwargs)")
if not is_virtual(mware.post_request):
if is_virtual(mware.post_bad_request):
post_parts.appendleft(f"↪ {name}.post_request(result)")
else:
post_parts.appendleft(
f"↪ {name}.post_[bad_]request(result|error)")
elif not is_virtual(mware.post_bad_request):
post_parts.appendleft(f"↪ {name}.post_bad_request(error)")
parts = []
parts.append('➥ app.on_request(action, *raw_args, **raw_kwargs)')
parts.extend(pre_parts)
parts.append('➥ args, kwargs = action.marshal(raw_args, raw_kwargs)')
parts.extend(on_parts)
parts.append('➥ raw_result = action(*args, **kwargs)')
parts.extend(post_parts)
parts.append('➥ return app.on_response(action, raw_result)')
diagram = '\n'.join(f'{" " * (i+1)}{s}' for i, s in enumerate(parts))
console.debug(message=(f"action execution diagram...\n\n {diagram}\n"))
def fget(self, resource: 'Resource'):
"""
Returns resource state data associated with the resolver. For example,
if the name of the resolver were "email", corresponding to an email
field, we would return `resource.state["email"]`. However, if "email" is
missing from the state dict, then we lazy load it through the resolver,
memoizing it in the state dict. This method also runs the resolver's
on_get callback.
"""
resolver = self.resolver
# if value not loaded, lazily resolve it
if resolver.name not in resource.internal.state:
console.debug(f'lazy loading {resource}.{resolver.name}')
request = Request(resolver)
value = resolver.resolve(resource, request)
# if resolver is for a field, we know that the resolved
# field has been loaded ON the returned resource (value)
if resolver.name in resource.ravel.resolvers.fields:
value = resource.internal.state.get(resolver.name, DNE)
if value is DNE:
console.debug(message=(f'no value returned for '
f'{resource}.{resolver.name}'))
return
if (value is not None) or resolver.nullable:
resource.internal.state[resolver.name] = value
elif (value is None) and (not resolver.nullable):
if resource.internal.state.get(resolver.name) is None:
resource.internal.state.pop(resolver.name, None)
console.warning(message=(f'resolver returned bad value'),
data={
'resource': resource._id,
'class': resource.class_name,
'resolver': self.resolver.name,
'reason': 'resolver not nullable',
})
value = resource.internal.state.get(resolver.name)
resolver.on_get(resource, value)
return value
def create(self, record: dict) -> dict:
record[self.id_column_name] = self.create_id(record)
prepared_record = self.prepare(record, serialize=True)
insert_stmt = self.table.insert().values(**prepared_record)
_id = prepared_record.get('_id', '')
console.debug(f'SQL: INSERT {str(_id)[:7] + " " if _id else ""}'
f'INTO {self.table}')
try:
if self.supports_returning:
insert_stmt = insert_stmt.return_defaults()
result = self.conn.execute(insert_stmt)
return dict(record, **(result.returned_defaults or {}))
else:
result = self.conn.execute(insert_stmt)
return self.fetch(_id=record[self.id_column_name])
except Exception:
console.error(message=f'failed to insert record',
data={
'record': record,
'resource': get_class_name(self.resource_type),
})
raise
def bootstrap(self,
app: 'Application',
namespace: Dict = None) -> 'Manifest':
console.debug(message='computed manifest...', data=self.data)
self.app = app
self.bootstraps = self._initialize_bootstraps()
self.bindings = self._initialize_bindings()
self.scanner = self._scan_filesystem()
if namespace:
self._scan_namespace_dict(namespace)
self._post_process_values_dict()
self._resolve_bindings()
self._resolve_resource_id_fields()
self._bootstrap_store_classes()
self._bootstrap_resource_classes()
self._bind_resources()
self._inject_classes_to_actions()
self._bootstrap_app_actions()
return self
def on_decorate(self, endpoint: 'Endpoint'):
endpoint.routes.append(f'/{StringUtils.dash(endpoint.name).lower()}')
for route in endpoint.routes:
if route not in self.route_2_endpoint:
console.debug(f'routing {route} to {endpoint.name} action')
self.route_2_endpoint[route][endpoint.method] = endpoint
def bootstrap(self):
console.debug(f'bootstrapping {self.name} action')
self._prepare_middleware()
self.on_bootstrap()
self._is_bootstrapped = True
def scan(package, verbose=False):
console.debug(f'manifest scanning {package}')
try:
scanner.scan(package)
except:
console.exception('scan failed')
def on_draw(self, tick):
console.debug(f'Tick {self.state.tick}')
def __init__(self, event_type, **kwargs):
console.debug(
f'Event {event_type} {PYGAME_EVENT_ID_2_NAME[event_type]}', data=kwargs
)
self.__dict__['kwargs'] = kwargs or {}
self.event_type = event_type
def bootstrap(
self,
manifest: Union[Dict, Manifest, Text, Callable, 'Application'] = None,
namespace: Dict = None,
values: Dict = None,
middleware: List = None,
mode: Mode = Mode.normal,
*args,
**kwargs
) -> 'Application':
"""
Bootstrap the data, business, and service layers, wiring them up.
"""
def create_logger(level):
"""
Setup root application logger.
"""
if self.manifest.package:
name = (
f'{self.manifest.package}:'
f'{os.getpid()}:'
f'{get_ident()}'
)
else:
class_name = get_class_name(self)
name = (
f'{StringUtils.snake(class_name)}:'
f'{os.getpid()}:'
f'{get_ident()}'
)
return ConsoleLoggerInterface(name, level)
# warn about already being bootstrapped...
if (
self.is_bootstrapped and
self.local.thread_id != get_ident()
):
console.warning(
message=f'{get_class_name(self)} already bootstrapped.',
data={
'pid': os.getpid(),
'thread': threading.get_ident(),
}
)
return self
console.debug(f'bootstrapping {get_class_name(self)} app')
# override the application mode set in constructor
self._mode = Mode(mode or self.mode or Mode.normal)
# merge additional namespace data into namespace dict
self._namespace = DictUtils.merge(self._namespace, namespace or {})
# register additional middleware targing this app subclass
if middleware:
self._middleware.extend(
m for m in middleware if isinstance(self, m.app_types)
)
# build final manifest, used to bootstrap program components
if manifest is not None:
if isinstance(manifest, Application):
source_app = manifest
self.local.manifest = Manifest(source_app.shared.manifest_data)
else:
self.local.manifest = Manifest(manifest)
elif self.shared.manifest_data:
self.local.manifest = Manifest(self.shared.manifest_data)
else:
self.local.manifest = Manifest({})
assert self.local.manifest is not None
self.manifest.bootstrap(self)
if values:
self.manifest.values.update(values)
if not self.shared.manifest_data:
self.shared.manifest_data = self.local.manifest.data
# set up main thread name
if self.manifest.package:
current_thread().name = (
f'{StringUtils.camel(self.manifest.package)}'
)
else:
# update default main thread name
current_thread().name = (
f'{get_class_name(self)}'
)
self._logger = create_logger(self.manifest.logging['level'])
self._arg_loader = ArgumentLoader(self)
# if we're in a new process, unset the executors so that
# the spawn method lazily triggers the instantiation of
# new ones at runtime.
if not self._root_pid != os.getpid():
self.local.thread_executor = None
self.local.process_executor = None
self.local.thread_id = get_ident()
self.local.tx_manager = TransactionManager(self)
# execute custom lifecycle hook provided by this subclass
self.on_bootstrap(*args, **kwargs)
self.local.is_bootstrapped = True
console.debug(f'finished bootstrapping {get_class_name(self)} app')
return self
