def _infer_field(self, annotation, name=None): many, type_name = extract_res_info_from_annotation(annotation) field = None if type_name in self.app.manifest.resource_classes: if many: field = field_types.List(self.app[type_name].Schema()) else: field = field_types.Nested(self.app[type_name].Schema()) elif annotation is not None: field_type = self._py_type_2_field_type.get(annotation) if field_type: if many: field = field_types.List(field_type()) else: field = field_type() elif annotation is not inspect._empty: console.error( message=f'cannot infer protobuf field from annotation', data={ 'annotation': str(annotation), 'name': name, } ) if field is not None and name: field.name = name return field
def validate(self, resolvers: Set[Text] = None, strict=False) -> Dict: """ Validate an object's loaded state data. If you need to check if some state data is loaded or not and raise an exception in case absent, use self.require. """ errors = {} resolver_names_to_validate = (resolvers or set(self.ravel.resolvers.keys())) for name in resolver_names_to_validate: resolver = self.ravel.resolvers[name] if name not in self.internal.state: console.warning(message=f'skipping {name} validation', data={'reason': 'not loaded'}) else: value = self.internal.state.get(name) if value is None and not resolver.nullable: errors[name] = 'not nullable' if name in self.ravel.schema.fields: field = self.ravel.schema.fields[name] _value, error = field.process(value) if error is not None: errors[name] = error if strict and errors: console.error(message='validation error', data={'errors': errors}) raise ValidationError('see error log for details') return errors
def create_many(self, records: List[Dict]) -> Dict: prepared_records = [] nullable_fields = self.resource_type.Schema.nullable_fields for record in records: record[self.id_column_name] = self.create_id(record) prepared_record = self.prepare(record, serialize=True) prepared_records.append(prepared_record) for nullable_field in nullable_fields.values(): if nullable_field.name not in prepared_record: prepared_record[nullable_field.name] = None try: self.conn.execute(self.table.insert(), prepared_records) except Exception: console.error(f'failed to insert records') raise n = len(prepared_records) id_list_str = (', '.join( str(x['_id'])[:7] for x in prepared_records if x.get('_id'))) console.debug(f'SQL: INSERT {id_list_str} INTO {self.table} ' + (f'(count: {n})' if n > 1 else '')) if self.supports_returning: # TODO: use implicit returning if possible pass return self.fetch_many((rec[self.id_column_name] for rec in records), as_list=True)
def on_match_error(self, exc: Exception, module, context, name, value): """ If there's a problem in self.on_match, we come here. """ exc_str = traceback.format_exc() console.error(message=f'error while scanning {name} ({type(value)})', data={'traceback': exc_str.split('\n')})
def __call__(self, *raw_args, **raw_kwargs): state = ExecutionState(self, raw_args, raw_kwargs) request = Request(state) for func in ( self._apply_middleware_pre_request, self._apply_app_on_request, self._apply_middleware_on_request, self._apply_target_callable, self._apply_app_on_response, ): error = func(request) if (error is not None) or request.is_complete: break if state.exc is None: self._apply_middleware_post_request(request) else: self._apply_middleware_post_bad_request(request) if state.errors: console.error( message=f'error/s occured in action: {self.name}', data={'errors': [error.to_dict() for error in state.errors]}) raise BadRequest(self, state) return state.result
def prepare(self, record: Dict, serialize=True) -> Dict: """ When inserting or updating data, the some raw values in the record dict must be transformed before their corresponding sqlalchemy column type will accept the data. """ cb_name = 'on_encode' if serialize else 'on_decode' prepared_record = {} for k, v in record.items(): if k in (REV): prepared_record[k] = v adapter = self._adapters.get(k) if adapter: callback = getattr(adapter, cb_name, None) if callback: try: prepared_record[k] = callback(v) continue except Exception: console.error( message=f'failed to adapt column value: {k}', data={ 'value': v, 'field': adapter.field_class }) raise prepared_record[k] = v return prepared_record
def on_import_error(self, exc: Exception, module_name: Text, context): """ If the scanner fails to import a module while it walks the file system, it comes here to handle and report the problem. """ exc_str = traceback.format_exc() console.error(message=f'manifest could not scan module: {module_name}', data={'traceback': exc_str.split('\n')})
def validate(data): data, errors = schema.process(data) if errors: console.error(message=f'response validation errors', data={ 'data': pprint.pformat(data), 'errors': errors, }) return data
def __getattr__(self, method: Text) -> Callable: method = self.methods.get(method) if method is None: console.error( message=f'celery client does not recognize task name', data={'task': method} ) raise ValueError(method) return method
def create_tables(cls, overwrite=False): """ Create all tables for all SqlalchemyStores used in the host app. """ if not cls.is_bootstrapped(): console.error(f'{get_class_name(cls)} cannot create ' f'tables unless bootstrapped') return meta = cls.get_metadata() engine = cls.get_engine() if overwrite: console.info('dropping Resource SQL tables...') meta.drop_all(engine) # create all tables console.info('creating Resource SQL tables...') meta.create_all(engine)
def fset(self, owner: 'Resource', value): field = self.resolver.field if value is None: super().fset(owner, None) return processed_value, errors = field.process(value) if errors: console.error(message=f'cannot set {self}', data={ 'resolver': str(self.resolver), 'field': field, 'schema': owner.ravel.schema, 'errors': errors, 'value': value, }) raise Exception(str(errors)) super().fset(owner, processed_value)
def on_bind(self, resource_type: Type['Resource'], table: Text = None, schema: 'Schema' = None, **kwargs): """ Initialize SQLAlchemy data strutures used for constructing SQL expressions used to manage the bound resource type. """ # map each of the resource's schema fields to a corresponding adapter, # which is used to prepare values upon insert and update. table = (table or SqlalchemyTableBuilder.derive_table_name(resource_type)) field_class_2_adapter = { adapter.field_class: adapter for adapter in self.get_default_adapters(self.dialect, table) + self._custom_adapters } self._adapters = { field_name: field_class_2_adapter[type(field)] for field_name, field in self.resource_type.Schema.fields.items() if (type(field) in field_class_2_adapter and field.meta.get('ravel_on_resolve') is None) } # build the Sqlalchemy Table object for the bound resource type. self._builder = SqlalchemyTableBuilder(self) try: self._table = self._builder.build_table(name=table, schema=schema) except Exception: console.error(f'failed to build sa.Table: {table}') raise self._id_column = getattr(self._table.c, self.id_column_name) # remember which column is the _id column self._id_column_names[self._table.name] = self.id_column_name # set SqlalchemyStore options here, using bootstrap-level # options as base/default options. self._options = dict(self.ravel.kwargs, **kwargs)
def create(self, record: dict) -> dict: record[self.id_column_name] = self.create_id(record) prepared_record = self.prepare(record, serialize=True) insert_stmt = self.table.insert().values(**prepared_record) _id = prepared_record.get('_id', '') console.debug(f'SQL: INSERT {str(_id)[:7] + " " if _id else ""}' f'INTO {self.table}') try: if self.supports_returning: insert_stmt = insert_stmt.return_defaults() result = self.conn.execute(insert_stmt) return dict(record, **(result.returned_defaults or {})) else: result = self.conn.execute(insert_stmt) return self.fetch(_id=record[self.id_column_name]) except Exception: console.error(message=f'failed to insert record', data={ 'record': record, 'resource': get_class_name(self.resource_type), }) raise
def update(self, data: Dict = None, **more_data) -> 'Resource': data = dict(data or {}, **more_data) if data: self.merge(data) raw_record = self.dirty.copy() raw_record.pop(REV, None) raw_record.pop(ID, None) errors = {} prepared_record = {} for k, v in raw_record.items(): field = self.Schema.fields.get(k) if field is not None: if field.name not in self.ravel.virtual_fields: if v is None and field.nullable: prepared_record[k] = None else: prepared_record[k], error = field.process(v) if error: console.error( f'{self} failed validation for {k}: {v}') errors[k] = error self.on_update(prepared_record) if errors: raise ValidationError(f'update failed for {self}: {errors}') updated_record = self.ravel.local.store.dispatch( 'update', (self._id, prepared_record)) if updated_record: self.merge(updated_record) self.clean(prepared_record.keys() | updated_record.keys()) self.post_update() return self
def require( self, resolvers: Set[Text] = None, use_defaults=True, strict=False, overwrite=False, ) -> Set[Text]: """ Checks if all specified resolvers are present. If they are required but not present, an exception will be raised for `strict` mode; otherwise, a set of the missing resolver names is returned. """ if isinstance(resolvers, str): resolvers = {resolvers} required_resolver_names = (resolvers or set(k for k in self.ravel.resolvers.keys() if self.ravel.resolvers[k].required)) missing_resolver_names = set() defaults = self.ravel.defaults for name in required_resolver_names: resolver = self.ravel.resolvers[name] if overwrite or name not in self.internal.state: if use_defaults and name in defaults: value = defaults[name]() if value is None and not resolver.nullable: raise ValidationError(f'{name} not nullable') self[name] = value else: missing_resolver_names.add(name) if strict and missing_resolver_names: console.error(message=f'{self.class_name} missing required data', data={'missing': missing_resolver_names}) raise ValidationError('see error log for details') return missing_resolver_names
def pre_request(self, action: 'Action', request: 'Request', raw_args: Tuple, raw_kwargs: Dict): # NOTE: set csrf_protected=False in your Action decorators # in order to skip this middleware... csrf_protected = action.decorator.kwargs.get('csrf_protected') if csrf_protected is None: csrf_protected = True if not csrf_protected: return http_request = raw_args[0] token = http_request.headers.get('CSRF-TOKEN') session = request.session # bail if token missing or mismatching if (not (token and session)) or (session.csrf_token != token): raise Exception('CSRF token missing or unrecognized') now = datetime.now() iv = session.csrf_aes_cbc_iv # cipher mode initialization vector # decode & decrypt the AES encrypted CSRF token try: aes_cipher = AES.new(self.signing_key, AES.MODE_CBC, iv) json_str = unpad(aes_cipher.decrypt(b64decode(token)), AES.block_size) token_components = self.app.json.decode(json_str) expires_at_isoformat = token_components[0] # convert expiration date string to a datetime object if expires_at_isoformat is not None: expires_at = datetime.fromisoformat(expires_at_isoformat) else: expires_at = None except Exception: console.error(message='failed to decode CSRF token', data={ 'token': token, 'session_id': session._id }) raise # save crsf data to request for use in post_request request.context.csrf = { 'session_id': token_components[1], 'expires_at': token_components[0], 'csrf_token': token, } # abort request if token is expired (provided it has an expiry) if (expires_at is not None) and (now >= expires_at): console.error(message='received expired CSRF token', data={ 'csrf_token': token, 'expires_at': expires_at, 'session_id': session._id, }) raise Exception('invalid CSRF token') if request.context.csrf['session_id'] != session._id: console.error(message='CSRF token session ID mismatch', data={ 'csrf_session_id': request.context.csrf['session_id'], 'request_session_id': session._id, }) raise Exception('invalid CSRF token')
def build_column(self, field: 'Field') -> sa.Column: """ Derive a Sqlalchemy column from a Field object. It looks for Sqlalchemy-related column kwargs in the field's meta dict. """ name = field.source adapter = self.adapters.get(field.name) if adapter is None: console.warning( 'no sqlalchemy field adapter registered ' f'for {field}. using default adapter.' ) try: dtype = adapter.on_adapt(field) except Exception: console.error(f'could not adapt field {field}') raise primary_key = field.meta.get('primary_key', False) unique = field.meta.get('unique', False) if field.source == REV: indexed = True server_default = '0' else: indexed = field.meta.get('index', False) server_default = None if 'server_default' in field.meta: server_default = field.meta['server_default'] elif field.has_constant_default: defaults = self._resource_type.ravel.defaults server_default = defaults[field.name]() if server_default is not None: if not isinstance(server_default, str): if adapter is not None: server_default = adapter.encode(server_default) if isinstance(field, fields.Bool): server_default = 'true' if server_default else 'false' elif isinstance(field, (fields.Int, fields.Float)): server_default = str(server_default) else: try: server_default = json_encoder.encode( server_default ) except: server_default = str(server_default) # prepare positional arguments for Column ctor args = [ name, dtype() if isinstance(dtype, type) else dtype, ] # foreign key string path, like 'user._id' foreign_key_dotted_path = field.meta.get('foreign_key') if foreign_key_dotted_path: args.append(ForeignKey(foreign_key_dotted_path)) # prepare keyword arguments for Column ctor kwargs = dict( index=indexed, primary_key=primary_key, unique=unique, server_default=server_default ) try: column = sa.Column(*args, **kwargs) except Exception: console.error( message=f'failed to build sa.Column: {name}', data={ 'args': args, 'kwargs': kwargs } ) raise if field.nullable is not None: column.nullable = field.nullable return column