Esempio n. 1
0
    def _infer_field(self, annotation, name=None):
        many, type_name = extract_res_info_from_annotation(annotation)
        field = None

        if type_name in self.app.manifest.resource_classes:
            if many:
                field = field_types.List(self.app[type_name].Schema())
            else:
                field = field_types.Nested(self.app[type_name].Schema())
        elif annotation is not None:
            field_type = self._py_type_2_field_type.get(annotation)
            if field_type:
                if many:
                    field = field_types.List(field_type())
                else:
                    field = field_type()
            elif annotation is not inspect._empty:
                console.error(
                    message=f'cannot infer protobuf field from annotation',
                    data={
                        'annotation': str(annotation),
                        'name': name,
                    }
                )

        if field is not None and name:
            field.name = name

        return field
Esempio n. 2
0
    def validate(self, resolvers: Set[Text] = None, strict=False) -> Dict:
        """
        Validate an object's loaded state data. If you need to check if some
        state data is loaded or not and raise an exception in case absent,
        use self.require.
        """
        errors = {}
        resolver_names_to_validate = (resolvers
                                      or set(self.ravel.resolvers.keys()))
        for name in resolver_names_to_validate:
            resolver = self.ravel.resolvers[name]
            if name not in self.internal.state:
                console.warning(message=f'skipping {name} validation',
                                data={'reason': 'not loaded'})
            else:
                value = self.internal.state.get(name)
                if value is None and not resolver.nullable:
                    errors[name] = 'not nullable'
                if name in self.ravel.schema.fields:
                    field = self.ravel.schema.fields[name]
                    _value, error = field.process(value)
                    if error is not None:
                        errors[name] = error

        if strict and errors:
            console.error(message='validation error', data={'errors': errors})
            raise ValidationError('see error log for details')

        return errors
Esempio n. 3
0
    def create_many(self, records: List[Dict]) -> Dict:
        prepared_records = []
        nullable_fields = self.resource_type.Schema.nullable_fields
        for record in records:
            record[self.id_column_name] = self.create_id(record)
            prepared_record = self.prepare(record, serialize=True)
            prepared_records.append(prepared_record)
            for nullable_field in nullable_fields.values():
                if nullable_field.name not in prepared_record:
                    prepared_record[nullable_field.name] = None

        try:
            self.conn.execute(self.table.insert(), prepared_records)
        except Exception:
            console.error(f'failed to insert records')
            raise

        n = len(prepared_records)
        id_list_str = (', '.join(
            str(x['_id'])[:7] for x in prepared_records if x.get('_id')))
        console.debug(f'SQL: INSERT {id_list_str} INTO {self.table} ' +
                      (f'(count: {n})' if n > 1 else ''))

        if self.supports_returning:
            # TODO: use implicit returning if possible
            pass

        return self.fetch_many((rec[self.id_column_name] for rec in records),
                               as_list=True)
Esempio n. 4
0
 def on_match_error(self, exc: Exception, module, context, name, value):
     """
     If there's a problem in self.on_match, we come here.
     """
     exc_str = traceback.format_exc()
     console.error(message=f'error while scanning {name} ({type(value)})',
                   data={'traceback': exc_str.split('\n')})
Esempio n. 5
0
    def __call__(self, *raw_args, **raw_kwargs):
        state = ExecutionState(self, raw_args, raw_kwargs)
        request = Request(state)

        for func in (
                self._apply_middleware_pre_request,
                self._apply_app_on_request,
                self._apply_middleware_on_request,
                self._apply_target_callable,
                self._apply_app_on_response,
        ):
            error = func(request)
            if (error is not None) or request.is_complete:
                break

        if state.exc is None:
            self._apply_middleware_post_request(request)
        else:
            self._apply_middleware_post_bad_request(request)

        if state.errors:
            console.error(
                message=f'error/s occured in action: {self.name}',
                data={'errors': [error.to_dict() for error in state.errors]})
            raise BadRequest(self, state)

        return state.result
Esempio n. 6
0
 def prepare(self, record: Dict, serialize=True) -> Dict:
     """
     When inserting or updating data, the some raw values in the record
     dict must be transformed before their corresponding sqlalchemy column
     type will accept the data.
     """
     cb_name = 'on_encode' if serialize else 'on_decode'
     prepared_record = {}
     for k, v in record.items():
         if k in (REV):
             prepared_record[k] = v
         adapter = self._adapters.get(k)
         if adapter:
             callback = getattr(adapter, cb_name, None)
             if callback:
                 try:
                     prepared_record[k] = callback(v)
                     continue
                 except Exception:
                     console.error(
                         message=f'failed to adapt column value: {k}',
                         data={
                             'value': v,
                             'field': adapter.field_class
                         })
                     raise
         prepared_record[k] = v
     return prepared_record
Esempio n. 7
0
 def on_import_error(self, exc: Exception, module_name: Text, context):
     """
     If the scanner fails to import a module while it walks the file
     system, it comes here to handle and report the problem.
     """
     exc_str = traceback.format_exc()
     console.error(message=f'manifest could not scan module: {module_name}',
                   data={'traceback': exc_str.split('\n')})
Esempio n. 8
0
 def validate(data):
     data, errors = schema.process(data)
     if errors:
         console.error(message=f'response validation errors',
                       data={
                           'data': pprint.pformat(data),
                           'errors': errors,
                       })
     return data
Esempio n. 9
0
 def __getattr__(self, method: Text) -> Callable:
     method = self.methods.get(method)
     if method is None:
         console.error(
             message=f'celery client does not recognize task name',
             data={'task': method}
         )
         raise ValueError(method)
     return method
Esempio n. 10
0
    def create_tables(cls, overwrite=False):
        """
        Create all tables for all SqlalchemyStores used in the host app.
        """
        if not cls.is_bootstrapped():
            console.error(f'{get_class_name(cls)} cannot create '
                          f'tables unless bootstrapped')
            return

        meta = cls.get_metadata()
        engine = cls.get_engine()

        if overwrite:
            console.info('dropping Resource SQL tables...')
            meta.drop_all(engine)

        # create all tables
        console.info('creating Resource SQL tables...')
        meta.create_all(engine)
Esempio n. 11
0
    def fset(self, owner: 'Resource', value):
        field = self.resolver.field
        if value is None:
            super().fset(owner, None)
            return

        processed_value, errors = field.process(value)
        if errors:
            console.error(message=f'cannot set {self}',
                          data={
                              'resolver': str(self.resolver),
                              'field': field,
                              'schema': owner.ravel.schema,
                              'errors': errors,
                              'value': value,
                          })
            raise Exception(str(errors))

        super().fset(owner, processed_value)
Esempio n. 12
0
    def on_bind(self,
                resource_type: Type['Resource'],
                table: Text = None,
                schema: 'Schema' = None,
                **kwargs):
        """
        Initialize SQLAlchemy data strutures used for constructing SQL
        expressions used to manage the bound resource type.
        """
        # map each of the resource's schema fields to a corresponding adapter,
        # which is used to prepare values upon insert and update.
        table = (table
                 or SqlalchemyTableBuilder.derive_table_name(resource_type))
        field_class_2_adapter = {
            adapter.field_class: adapter
            for adapter in self.get_default_adapters(self.dialect, table) +
            self._custom_adapters
        }
        self._adapters = {
            field_name: field_class_2_adapter[type(field)]
            for field_name, field in self.resource_type.Schema.fields.items()
            if (type(field) in field_class_2_adapter
                and field.meta.get('ravel_on_resolve') is None)
        }

        # build the Sqlalchemy Table object for the bound resource type.
        self._builder = SqlalchemyTableBuilder(self)

        try:
            self._table = self._builder.build_table(name=table, schema=schema)
        except Exception:
            console.error(f'failed to build sa.Table: {table}')
            raise

        self._id_column = getattr(self._table.c, self.id_column_name)

        # remember which column is the _id column
        self._id_column_names[self._table.name] = self.id_column_name

        # set SqlalchemyStore options here, using bootstrap-level
        # options as base/default options.
        self._options = dict(self.ravel.kwargs, **kwargs)
Esempio n. 13
0
 def create(self, record: dict) -> dict:
     record[self.id_column_name] = self.create_id(record)
     prepared_record = self.prepare(record, serialize=True)
     insert_stmt = self.table.insert().values(**prepared_record)
     _id = prepared_record.get('_id', '')
     console.debug(f'SQL: INSERT {str(_id)[:7] + " " if _id else ""}'
                   f'INTO {self.table}')
     try:
         if self.supports_returning:
             insert_stmt = insert_stmt.return_defaults()
             result = self.conn.execute(insert_stmt)
             return dict(record, **(result.returned_defaults or {}))
         else:
             result = self.conn.execute(insert_stmt)
             return self.fetch(_id=record[self.id_column_name])
     except Exception:
         console.error(message=f'failed to insert record',
                       data={
                           'record': record,
                           'resource': get_class_name(self.resource_type),
                       })
         raise
Esempio n. 14
0
    def update(self, data: Dict = None, **more_data) -> 'Resource':
        data = dict(data or {}, **more_data)
        if data:
            self.merge(data)

        raw_record = self.dirty.copy()
        raw_record.pop(REV, None)
        raw_record.pop(ID, None)

        errors = {}
        prepared_record = {}
        for k, v in raw_record.items():
            field = self.Schema.fields.get(k)
            if field is not None:
                if field.name not in self.ravel.virtual_fields:
                    if v is None and field.nullable:
                        prepared_record[k] = None
                    else:
                        prepared_record[k], error = field.process(v)
                        if error:
                            console.error(
                                f'{self} failed validation for {k}: {v}')
                            errors[k] = error

        self.on_update(prepared_record)

        if errors:
            raise ValidationError(f'update failed for {self}: {errors}')

        updated_record = self.ravel.local.store.dispatch(
            'update', (self._id, prepared_record))

        if updated_record:
            self.merge(updated_record)

        self.clean(prepared_record.keys() | updated_record.keys())
        self.post_update()

        return self
Esempio n. 15
0
    def require(
        self,
        resolvers: Set[Text] = None,
        use_defaults=True,
        strict=False,
        overwrite=False,
    ) -> Set[Text]:
        """
        Checks if all specified resolvers are present. If they are required
        but not present, an exception will be raised for `strict` mode;
        otherwise, a set of the missing resolver names is returned.
        """
        if isinstance(resolvers, str):
            resolvers = {resolvers}
        required_resolver_names = (resolvers
                                   or set(k
                                          for k in self.ravel.resolvers.keys()
                                          if self.ravel.resolvers[k].required))
        missing_resolver_names = set()
        defaults = self.ravel.defaults
        for name in required_resolver_names:
            resolver = self.ravel.resolvers[name]
            if overwrite or name not in self.internal.state:
                if use_defaults and name in defaults:
                    value = defaults[name]()
                    if value is None and not resolver.nullable:
                        raise ValidationError(f'{name} not nullable')
                    self[name] = value
                else:
                    missing_resolver_names.add(name)

        if strict and missing_resolver_names:
            console.error(message=f'{self.class_name} missing required data',
                          data={'missing': missing_resolver_names})
            raise ValidationError('see error log for details')

        return missing_resolver_names
Esempio n. 16
0
    def pre_request(self, action: 'Action', request: 'Request',
                    raw_args: Tuple, raw_kwargs: Dict):
        # NOTE: set csrf_protected=False in your Action decorators
        # in order to skip this middleware...
        csrf_protected = action.decorator.kwargs.get('csrf_protected')
        if csrf_protected is None:
            csrf_protected = True

        if not csrf_protected:
            return

        http_request = raw_args[0]
        token = http_request.headers.get('CSRF-TOKEN')
        session = request.session

        # bail if token missing or mismatching
        if (not (token and session)) or (session.csrf_token != token):
            raise Exception('CSRF token missing or unrecognized')

        now = datetime.now()
        iv = session.csrf_aes_cbc_iv  # cipher mode initialization vector

        # decode & decrypt the AES encrypted CSRF token
        try:
            aes_cipher = AES.new(self.signing_key, AES.MODE_CBC, iv)
            json_str = unpad(aes_cipher.decrypt(b64decode(token)),
                             AES.block_size)
            token_components = self.app.json.decode(json_str)
            expires_at_isoformat = token_components[0]

            # convert expiration date string to a datetime object
            if expires_at_isoformat is not None:
                expires_at = datetime.fromisoformat(expires_at_isoformat)
            else:
                expires_at = None

        except Exception:
            console.error(message='failed to decode CSRF token',
                          data={
                              'token': token,
                              'session_id': session._id
                          })
            raise

        # save crsf data to request for use in post_request
        request.context.csrf = {
            'session_id': token_components[1],
            'expires_at': token_components[0],
            'csrf_token': token,
        }

        # abort request if token is expired (provided it has an expiry)
        if (expires_at is not None) and (now >= expires_at):
            console.error(message='received expired CSRF token',
                          data={
                              'csrf_token': token,
                              'expires_at': expires_at,
                              'session_id': session._id,
                          })
            raise Exception('invalid CSRF token')

        if request.context.csrf['session_id'] != session._id:
            console.error(message='CSRF token session ID mismatch',
                          data={
                              'csrf_session_id':
                              request.context.csrf['session_id'],
                              'request_session_id': session._id,
                          })
            raise Exception('invalid CSRF token')
Esempio n. 17
0
    def build_column(self, field: 'Field') -> sa.Column:
        """
        Derive a Sqlalchemy column from a Field object. It looks for
        Sqlalchemy-related column kwargs in the field's meta dict.
        """
        name = field.source
        adapter = self.adapters.get(field.name)

        if adapter is None:
            console.warning(
                'no sqlalchemy field adapter registered '
                f'for {field}. using default adapter.'
            )

        try:
            dtype = adapter.on_adapt(field)
        except Exception:
            console.error(f'could not adapt field {field}')
            raise

        primary_key = field.meta.get('primary_key', False)
        unique = field.meta.get('unique', False)

        if field.source == REV:
            indexed = True
            server_default = '0'
        else:
            indexed = field.meta.get('index', False)
            server_default = None
            if 'server_default' in field.meta:
                server_default = field.meta['server_default']
            elif field.has_constant_default:
                defaults = self._resource_type.ravel.defaults
                server_default = defaults[field.name]()
            if server_default is not None:
                if not isinstance(server_default, str):
                    if adapter is not None:
                        server_default = adapter.encode(server_default)
                    if isinstance(field, fields.Bool):
                        server_default = 'true' if server_default else 'false'
                    elif isinstance(field, (fields.Int, fields.Float)):
                        server_default = str(server_default)
                    else:
                        try:
                            server_default = json_encoder.encode(
                                server_default
                            )
                        except:
                            server_default = str(server_default)

        # prepare positional arguments for Column ctor
        args = [
            name,
            dtype() if isinstance(dtype, type) else dtype,
        ]
        # foreign key string path, like 'user._id'
        foreign_key_dotted_path = field.meta.get('foreign_key')
        if foreign_key_dotted_path:
            args.append(ForeignKey(foreign_key_dotted_path))

        # prepare keyword arguments for Column ctor
        kwargs = dict(
            index=indexed,
            primary_key=primary_key,
            unique=unique,
            server_default=server_default
        )
        try:
            column = sa.Column(*args, **kwargs)
        except Exception:
            console.error(
                message=f'failed to build sa.Column: {name}',
                data={
                    'args': args,
                    'kwargs': kwargs
                }
            )
            raise

        if field.nullable is not None:
            column.nullable = field.nullable

        return column