def build(self, source) -> 'Query': query = self.right_loader_property.resolver.owner.select() if is_resource(source): source_value = getattr(source, self.left_field.name) if not source_value: console.debug( message=(f'relationship {get_class_name(source)}.' f'{self.relationship.name} aborting execution' f'because {self.left_field.name} is None'), data={ 'resource': source._id, }) # NOTE: ^ if you don't want this, then clear the field from # source using source.clean(field_name) return None query.where(self.right_loader_property == source_value) else: assert is_batch(source) left_field_name = self.left_field.name source_values = {getattr(res, left_field_name) for res in source} if not source_values: console.warning( message=(f'relationship {get_class_name(source)}.' f'{self.relationship.name} aborting query ' f'because {self.left_field.name} is empty'), data={ 'resources': source._id, }) # NOTE: ^ if you don't want this, then clear the field from # source using source.clean(field_name) return None query.where(self.right_loader_property.including(source_values)) return query
def create_many(self, records: List[Dict]) -> Dict: prepared_records = [] nullable_fields = self.resource_type.Schema.nullable_fields for record in records: record[self.id_column_name] = self.create_id(record) prepared_record = self.prepare(record, serialize=True) prepared_records.append(prepared_record) for nullable_field in nullable_fields.values(): if nullable_field.name not in prepared_record: prepared_record[nullable_field.name] = None try: self.conn.execute(self.table.insert(), prepared_records) except Exception: console.error(f'failed to insert records') raise n = len(prepared_records) id_list_str = (', '.join( str(x['_id'])[:7] for x in prepared_records if x.get('_id'))) console.debug(f'SQL: INSERT {id_list_str} INTO {self.table} ' + (f'(count: {n})' if n > 1 else '')) if self.supports_returning: # TODO: use implicit returning if possible pass return self.fetch_many((rec[self.id_column_name] for rec in records), as_list=True)
def commit(cls, rollback=True, **kwargs): """ Call commit on the thread-local database transaction. "Begin" must be called to start a new transaction at this point, if a new transaction is desired. """ def perform_sqlalchemy_commit(): tx = getattr(cls.ravel.local, 'sqla_tx', None) if tx is not None: cls.ravel.local.sqla_tx.commit() cls.ravel.local.sqla_tx = None # try to commit the transaction. console.debug(f'committing sqlalchemy transaction') try: perform_sqlalchemy_commit() except Exception: if rollback: # if the commit fails, rollback the transaction console.critical(f'rolling back sqlalchemy transaction') cls.rollback() else: console.exception(f'sqlalchemy transaction failed commit') finally: # ensure we close the connection either way cls.close()
def inherit_base_manifest( cls, base: Text = None, visited: Set = None, data: Dict = None, ): """ If there is a base manifest file specified by the loaded manifest data, then load it here and merge the current manifest into the base manifest, recusively back to the root base. """ visited = set() if visited is None else visited data = data if data is not None else {} base = os.path.abspath(os.path.expandvars(os.path.expanduser(base))) if base in visited: # avoid infinite loop if there is a bad inheritence cycle raise ManifestInheritanceError( f'manifest inheritance loop detected: {base}') else: visited.add(base) console.debug(f'inheriting manifest from {base}') if os.path.isfile(base): # merge current data dict into new base dict base_data = cls._read_file(base) data = DictUtils.merge(base_data.copy(), data) # recurse on base's base manifest... nested_base = base_data.get('base') if nested_base: cls.inherit_base_manifest(nested_base, visited, data) return data
def update_many(self, _ids: List, data: List[Dict] = None) -> None: assert data prepared_ids = [] prepared_records = [] for _id, record in zip(_ids, data): prepared_id = self.adapt_id(_id) prepared_record = self.prepare(record, serialize=True) if prepared_record: prepared_ids.append(prepared_id) prepared_records.append(prepared_record) prepared_record[ID] = prepared_id if prepared_records: n = len(prepared_records) console.debug(f'SQL: UPDATE {self.table} ' + (f'({n}x)' if n > 1 else '')) values = {k: bindparam(k) for k in prepared_records[0].keys()} update_stmt = (self.table.update().where( self._id_column == bindparam(self.id_column_name)).values( **values)) self.conn.execute(update_stmt, prepared_records) if self._options.get('fetch_on_update', True): if self.supports_returning: # TODO: use implicit returning if possible return self.fetch_many(_ids) else: return self.fetch_many(_ids) return
def add_action(self, action: 'Action', overwrite=False, *args, **kwargs): """ Add an action to this app. """ if action.name not in self._actions or overwrite: console.debug(f'registered action {action.name}') if isinstance(action, Action): if action.app is not self: if args or kwargs: console.warning( '*args and **kwargs ignored by add_action' ) decorator = self.action( *action.decorator.args, **action.decorator.kwargs ) decorator(action.target) else: self._actions[action.name] = action else: assert callable(action) decorator = self.action(*args, **kwargs) decorator(action) else: raise ApplicationError( message=f'action already registered: {action.name}', data={'action': action} )
def get_store_instance( self, resource_type: Type['Resource'], bind=True, rebind=False, ) -> 'Store': if isinstance(resource_type, str): binding = self._bindings.get(resource_type) else: binding = self._bindings.get(get_class_name(resource_type)) if binding is None: # lazily register a new binding base_store_type = resource_type.__store__() console.debug( f'calling {get_class_name(resource_type)}.__store__()') binding = self.register(resource_type, base_store_type) # call bind only if it already hasn't been called if rebind or ((not binding.is_bound) and bind): console.debug(message=( f'setting {get_class_name(binding.resource_type)}.ravel.store ' f'= {get_class_name(binding.store_instance)}()')) binding.bind(binder=self) return binding.store_instance
def on_request(self, action, request, *args, **kwargs): """ Take the attributes on the incoming protobuf Message object and map them to the args and kwargs expected by the action target. """ console.debug(message=f'RPC method {action.name} requested', ) # get field data to process into args and kwargs def deserialize_value(field, value): if isinstance(field, fields.Dict): if not value: value = None if field.nullable else {} else: value = self.json.decode(value) return value all_arguments = {} for field in action.schemas.request.fields.values(): value = getattr(request, field.name, None) all_arguments[field.name] = deserialize_value(field, value) # add a dummy request object so that partition_arguments can # run below; then remove it from the resulting args tuple. all_arguments['request'] = None # process field_data into args and kwargs args, kwargs = FuncUtils.partition_arguments(action.signature, all_arguments) args = args[1:] return (args, kwargs)
def bootstrap(self, app: 'Application'): self._app = app console.debug( f'bootstrapping {get_class_name(self)} middleware singleton') self.on_bootstrap() self._is_bootstrapped = True
def _scan_filesystem(self) -> ManifestScanner: """ Use venusian simply to scan the action packages/modules, causing the action callables to register themselves with the Application instance. """ t1 = datetime.now() package = self.package executor = ThreadPoolExecutor( max_workers=6, thread_name_prefix=(f'{StringUtils.camel(self.package or "")}' f'ManifestWorker'.strip())) scanner = ManifestScanner(self) futures = [] def scan(package, verbose=False): console.debug(f'manifest scanning {package}') try: scanner.scan(package) except: console.exception('scan failed') def async_scan(package, verbose=False): future = executor.submit(scan, package, verbose) futures.append(future) async_scan('ravel.resource') async_scan('ravel.store') installed_pkg_names = {pkg.key for pkg in pkg_resources.working_set} # scan extension directories if the package installation requirements # are met, like sqlalchemy and redis. pgk_name_2_ravel_scan_path = { 'sqlalchemy': 'ravel.ext.sqlalchemy', 'redis': 'ravel.ext.redis', 'pygame': 'ravel.ext.gaming.pygame', 'celery': 'ravel.ext.celery', 'numpy': 'ravel.ext.np' } for pkg_name, scan_path in pgk_name_2_ravel_scan_path.items(): if pkg_name in installed_pkg_names: async_scan(scan_path) # scan the app project package if package: async_scan(package, verbose=True) completed_scans, incomplete_scans = (concurrent.futures.wait(futures)) if incomplete_scans: raise FilesystemScanTimeout(message='filesystem scan timed out', ) t2 = datetime.now() secs = (t2 - t1).total_seconds() console.debug(f'scanned filesystem in {secs:.2f}s') return scanner
def close(cls): """ Return the thread-local database connection to the sqlalchemy connection pool (AKA the "engine"). """ sqla_conn = getattr(cls.ravel.local, 'sqla_conn', None) if sqla_conn is not None: console.debug('closing sqlalchemy connection') sqla_conn.close() cls.ravel.local.sqla_conn = None else: console.warning('sqlalchemy has no connection to close')
def bind(self, resource_type: Type['Resource'], **kwargs): t1 = datetime.now() self._resource_type = resource_type self.on_bind(resource_type, **kwargs) t2 = datetime.now() secs = (t2 - t1).total_seconds() console.debug(f'bound {TypeUtils.get_class_name(resource_type)} to ' f'{TypeUtils.get_class_name(self)} ' f'in {secs:.2f}s') self._is_bound = True
def bind(cls, store: 'Store', **kwargs): t1 = datetime.now() cls.ravel.local.store = store for resolver in cls.ravel.resolvers.values(): resolver.bind() cls.on_bind() cls.ravel.is_bound = True t2 = datetime.now() secs = (t2 - t1).total_seconds() console.debug(f'bound {TypeUtils.get_class_name(store)} to ' f'{TypeUtils.get_class_name(cls)} ' f'in {secs:.2f}s')
def bootstrap(cls, app: 'Application', **kwargs): t1 = datetime.now() cls.ravel.app = app # bootstrap all resolvers owned by this class for resolver in cls.ravel.resolvers.values(): if not resolver.is_bootstrapped(): resolver.bootstrap(cls.ravel.app) # lastly perform custom developer logic cls.on_bootstrap(app, **kwargs) cls.ravel.local.is_bootstrapped = True t2 = datetime.now() secs = (t2 - t1).total_seconds() console.debug(f'bootstrapped {TypeUtils.get_class_name(cls)} ' f'in {secs:.2f}s')
def post_request( self, action: 'Action', request: 'Request', result, ): """ Create and set a new token if the current one is expired or non-existent. """ http_request, http_response = request.raw_args[:2] csrf_data = request.context.get('csrf') session = request.session regenerate_token = action.decorator.kwargs.get('refresh_csrf_token', False) if (not csrf_data) or regenerate_token: console.debug(f'generating CSRF token for session {session._id}') self.create_and_set_token(http_request, http_response, session)
def update(self, _id, data: Dict) -> Dict: prepared_id = self.adapt_id(_id) prepared_data = self.prepare(data, serialize=True) if prepared_data: update_stmt = (self.table.update().values(**prepared_data).where( self._id_column == prepared_id)) else: return prepared_data if self.supports_returning: update_stmt = update_stmt.return_defaults() console.debug(f'SQL: UPDATE {self.table}') result = self.conn.execute(update_stmt) return dict(data, **(result.returned_defaults or {})) else: self.conn.execute(update_stmt) if self._options.get('fetch_on_update', True): return self.fetch(_id) return data
def begin(cls, auto_connect=True, **kwargs): """ Initialize a thread-local transaction. An exception is raised if there's already a pending transaction. """ conn = cls.get_active_connection() if conn is None: if auto_connect: conn = cls.connect() else: raise Exception('no active sqlalchemy connection') existing_tx = getattr(cls.ravel.local, 'sqla_tx', None) if existing_tx is not None: console.debug('there is already an open transaction') else: new_tx = cls.ravel.local.sqla_conn.begin() cls.ravel.local.sqla_tx = new_tx
def bootstrap(cls, app: 'Application' = None, **kwargs): """ Perform class-level initialization, like getting a connectio pool, for example. """ t1 = datetime.now() cls.ravel.app = app cls.on_bootstrap(**kwargs) cls.ravel.local.is_bootstrapped = True t2 = datetime.now() secs = (t2 - t1).total_seconds() console.debug(f'bootstrapped {TypeUtils.get_class_name(cls)} ' f'in {secs:.2f}s') return cls
def _load_manifest_data(cls, source): schema = cls.Schema(allow_additional=True) filepath = None data = {} # if source is callable, it means that the manifest is being returned # by a function, allowing for dynamic manifest properties if callable(source): source = source() # load the manifest data dict differently, depending # on its source -- e.g. from a file path, a dict, another manifest if isinstance(source, Manifest): data = deepcopy(source.data) filepath = source.filepath elif isinstance(source, str): filepath = os.path.abspath( os.path.expandvars(os.path.expanduser(source))) if os.path.isfile(source): console.debug(f'reading manifest from {source}') data = cls._read_file(source) else: raise ManifestFileNotFound( f'manifest file {filepath} not found. ' f'yaml and json manifest file types are supported.') elif isinstance(source, dict): data = source filepath = None # merge data dict into recursively inherited data dict base_filepath = data.get('base') if base_filepath: inherited_data = cls.inherit_base_manifest(base_filepath) data = DictUtils.merge(inherited_data, data) data = cls._expand_vars(data) # validate final computed data dict validated_data, errors = schema.process(data) if errors: raise ManifestValidationError( f'manifest validation error/s: {errors}') return (filepath, validated_data)
def _scan_namespace(self, namespace: Dict): """ Non-recursively detect Store and Resource class objects contained in the given namespace dict, making them available to the bootstrapping app. """ from ravel.store import Store from ravel.resource import Resource for k, v in (namespace or {}).items(): if TypeUtils.is_proper_subclass(v, Resource): if not v.ravel.is_abstract: self.resource_classes[k] = v console.debug(f'detected Resource class in ' f'namespace dict: {get_class_name(v)}') elif TypeUtils.is_proper_subclass(v, Store): self.store_classes[k] = v console.debug(f'detected Store class in namespace ' f'dict: {get_class_name(v)}')
def _bootstrap_app_actions(self): for action in self.app.actions.values(): action.bootstrap() on_parts = [] pre_parts = [] post_parts = deque() def is_virtual(func): return getattr(func, 'is_virtual', None) # middleware bootstrapping... for idx, mware in enumerate(self.app.middleware): mware.bootstrap(app=self.app) # everything below is for generating the log message # containing the "action execution diagram" name = StringUtils.snake(get_class_name(mware)) if not is_virtual(mware.pre_request): pre_parts.append(f"↪ {name}.pre_request(raw_args, raw_kwargs)") if not is_virtual(mware.on_request): on_parts.append(f"↪ {name}.on_request(args, kwargs)") if not is_virtual(mware.post_request): if is_virtual(mware.post_bad_request): post_parts.appendleft(f"↪ {name}.post_request(result)") else: post_parts.appendleft( f"↪ {name}.post_[bad_]request(result|error)") elif not is_virtual(mware.post_bad_request): post_parts.appendleft(f"↪ {name}.post_bad_request(error)") parts = [] parts.append('➥ app.on_request(action, *raw_args, **raw_kwargs)') parts.extend(pre_parts) parts.append('➥ args, kwargs = action.marshal(raw_args, raw_kwargs)') parts.extend(on_parts) parts.append('➥ raw_result = action(*args, **kwargs)') parts.extend(post_parts) parts.append('➥ return app.on_response(action, raw_result)') diagram = '\n'.join(f'{" " * (i+1)}{s}' for i, s in enumerate(parts)) console.debug(message=(f"action execution diagram...\n\n {diagram}\n"))
def fget(self, resource: 'Resource'): """ Returns resource state data associated with the resolver. For example, if the name of the resolver were "email", corresponding to an email field, we would return `resource.state["email"]`. However, if "email" is missing from the state dict, then we lazy load it through the resolver, memoizing it in the state dict. This method also runs the resolver's on_get callback. """ resolver = self.resolver # if value not loaded, lazily resolve it if resolver.name not in resource.internal.state: console.debug(f'lazy loading {resource}.{resolver.name}') request = Request(resolver) value = resolver.resolve(resource, request) # if resolver is for a field, we know that the resolved # field has been loaded ON the returned resource (value) if resolver.name in resource.ravel.resolvers.fields: value = resource.internal.state.get(resolver.name, DNE) if value is DNE: console.debug(message=(f'no value returned for ' f'{resource}.{resolver.name}')) return if (value is not None) or resolver.nullable: resource.internal.state[resolver.name] = value elif (value is None) and (not resolver.nullable): if resource.internal.state.get(resolver.name) is None: resource.internal.state.pop(resolver.name, None) console.warning(message=(f'resolver returned bad value'), data={ 'resource': resource._id, 'class': resource.class_name, 'resolver': self.resolver.name, 'reason': 'resolver not nullable', }) value = resource.internal.state.get(resolver.name) resolver.on_get(resource, value) return value
def create(self, record: dict) -> dict: record[self.id_column_name] = self.create_id(record) prepared_record = self.prepare(record, serialize=True) insert_stmt = self.table.insert().values(**prepared_record) _id = prepared_record.get('_id', '') console.debug(f'SQL: INSERT {str(_id)[:7] + " " if _id else ""}' f'INTO {self.table}') try: if self.supports_returning: insert_stmt = insert_stmt.return_defaults() result = self.conn.execute(insert_stmt) return dict(record, **(result.returned_defaults or {})) else: result = self.conn.execute(insert_stmt) return self.fetch(_id=record[self.id_column_name]) except Exception: console.error(message=f'failed to insert record', data={ 'record': record, 'resource': get_class_name(self.resource_type), }) raise
def bootstrap(self, app: 'Application', namespace: Dict = None) -> 'Manifest': console.debug(message='computed manifest...', data=self.data) self.app = app self.bootstraps = self._initialize_bootstraps() self.bindings = self._initialize_bindings() self.scanner = self._scan_filesystem() if namespace: self._scan_namespace_dict(namespace) self._post_process_values_dict() self._resolve_bindings() self._resolve_resource_id_fields() self._bootstrap_store_classes() self._bootstrap_resource_classes() self._bind_resources() self._inject_classes_to_actions() self._bootstrap_app_actions() return self
def on_decorate(self, endpoint: 'Endpoint'): endpoint.routes.append(f'/{StringUtils.dash(endpoint.name).lower()}') for route in endpoint.routes: if route not in self.route_2_endpoint: console.debug(f'routing {route} to {endpoint.name} action') self.route_2_endpoint[route][endpoint.method] = endpoint
def bootstrap(self): console.debug(f'bootstrapping {self.name} action') self._prepare_middleware() self.on_bootstrap() self._is_bootstrapped = True
def scan(package, verbose=False): console.debug(f'manifest scanning {package}') try: scanner.scan(package) except: console.exception('scan failed')
def on_draw(self, tick): console.debug(f'Tick {self.state.tick}')
def __init__(self, event_type, **kwargs): console.debug( f'Event {event_type} {PYGAME_EVENT_ID_2_NAME[event_type]}', data=kwargs ) self.__dict__['kwargs'] = kwargs or {} self.event_type = event_type
def bootstrap( self, manifest: Union[Dict, Manifest, Text, Callable, 'Application'] = None, namespace: Dict = None, values: Dict = None, middleware: List = None, mode: Mode = Mode.normal, *args, **kwargs ) -> 'Application': """ Bootstrap the data, business, and service layers, wiring them up. """ def create_logger(level): """ Setup root application logger. """ if self.manifest.package: name = ( f'{self.manifest.package}:' f'{os.getpid()}:' f'{get_ident()}' ) else: class_name = get_class_name(self) name = ( f'{StringUtils.snake(class_name)}:' f'{os.getpid()}:' f'{get_ident()}' ) return ConsoleLoggerInterface(name, level) # warn about already being bootstrapped... if ( self.is_bootstrapped and self.local.thread_id != get_ident() ): console.warning( message=f'{get_class_name(self)} already bootstrapped.', data={ 'pid': os.getpid(), 'thread': threading.get_ident(), } ) return self console.debug(f'bootstrapping {get_class_name(self)} app') # override the application mode set in constructor self._mode = Mode(mode or self.mode or Mode.normal) # merge additional namespace data into namespace dict self._namespace = DictUtils.merge(self._namespace, namespace or {}) # register additional middleware targing this app subclass if middleware: self._middleware.extend( m for m in middleware if isinstance(self, m.app_types) ) # build final manifest, used to bootstrap program components if manifest is not None: if isinstance(manifest, Application): source_app = manifest self.local.manifest = Manifest(source_app.shared.manifest_data) else: self.local.manifest = Manifest(manifest) elif self.shared.manifest_data: self.local.manifest = Manifest(self.shared.manifest_data) else: self.local.manifest = Manifest({}) assert self.local.manifest is not None self.manifest.bootstrap(self) if values: self.manifest.values.update(values) if not self.shared.manifest_data: self.shared.manifest_data = self.local.manifest.data # set up main thread name if self.manifest.package: current_thread().name = ( f'{StringUtils.camel(self.manifest.package)}' ) else: # update default main thread name current_thread().name = ( f'{get_class_name(self)}' ) self._logger = create_logger(self.manifest.logging['level']) self._arg_loader = ArgumentLoader(self) # if we're in a new process, unset the executors so that # the spawn method lazily triggers the instantiation of # new ones at runtime. if not self._root_pid != os.getpid(): self.local.thread_executor = None self.local.process_executor = None self.local.thread_id = get_ident() self.local.tx_manager = TransactionManager(self) # execute custom lifecycle hook provided by this subclass self.on_bootstrap(*args, **kwargs) self.local.is_bootstrapped = True console.debug(f'finished bootstrapping {get_class_name(self)} app') return self