def _infer_field(self, annotation, name=None):
many, type_name = extract_res_info_from_annotation(annotation)
field = None
if type_name in self.app.manifest.resource_classes:
if many:
field = field_types.List(self.app[type_name].Schema())
else:
field = field_types.Nested(self.app[type_name].Schema())
elif annotation is not None:
field_type = self._py_type_2_field_type.get(annotation)
if field_type:
if many:
field = field_types.List(field_type())
else:
field = field_type()
elif annotation is not inspect._empty:
console.error(
message=f'cannot infer protobuf field from annotation',
data={
'annotation': str(annotation),
'name': name,
}
)
if field is not None and name:
field.name = name
return field
def validate(self, resolvers: Set[Text] = None, strict=False) -> Dict:
"""
Validate an object's loaded state data. If you need to check if some
state data is loaded or not and raise an exception in case absent,
use self.require.
"""
errors = {}
resolver_names_to_validate = (resolvers
or set(self.ravel.resolvers.keys()))
for name in resolver_names_to_validate:
resolver = self.ravel.resolvers[name]
if name not in self.internal.state:
console.warning(message=f'skipping {name} validation',
data={'reason': 'not loaded'})
else:
value = self.internal.state.get(name)
if value is None and not resolver.nullable:
errors[name] = 'not nullable'
if name in self.ravel.schema.fields:
field = self.ravel.schema.fields[name]
_value, error = field.process(value)
if error is not None:
errors[name] = error
if strict and errors:
console.error(message='validation error', data={'errors': errors})
raise ValidationError('see error log for details')
return errors
def create_many(self, records: List[Dict]) -> Dict:
prepared_records = []
nullable_fields = self.resource_type.Schema.nullable_fields
for record in records:
record[self.id_column_name] = self.create_id(record)
prepared_record = self.prepare(record, serialize=True)
prepared_records.append(prepared_record)
for nullable_field in nullable_fields.values():
if nullable_field.name not in prepared_record:
prepared_record[nullable_field.name] = None
try:
self.conn.execute(self.table.insert(), prepared_records)
except Exception:
console.error(f'failed to insert records')
raise
n = len(prepared_records)
id_list_str = (', '.join(
str(x['_id'])[:7] for x in prepared_records if x.get('_id')))
console.debug(f'SQL: INSERT {id_list_str} INTO {self.table} ' +
(f'(count: {n})' if n > 1 else ''))
if self.supports_returning:
# TODO: use implicit returning if possible
pass
return self.fetch_many((rec[self.id_column_name] for rec in records),
as_list=True)
def on_match_error(self, exc: Exception, module, context, name, value):
"""
If there's a problem in self.on_match, we come here.
"""
exc_str = traceback.format_exc()
console.error(message=f'error while scanning {name} ({type(value)})',
data={'traceback': exc_str.split('\n')})
def __call__(self, *raw_args, **raw_kwargs):
state = ExecutionState(self, raw_args, raw_kwargs)
request = Request(state)
for func in (
self._apply_middleware_pre_request,
self._apply_app_on_request,
self._apply_middleware_on_request,
self._apply_target_callable,
self._apply_app_on_response,
):
error = func(request)
if (error is not None) or request.is_complete:
break
if state.exc is None:
self._apply_middleware_post_request(request)
else:
self._apply_middleware_post_bad_request(request)
if state.errors:
console.error(
message=f'error/s occured in action: {self.name}',
data={'errors': [error.to_dict() for error in state.errors]})
raise BadRequest(self, state)
return state.result
def prepare(self, record: Dict, serialize=True) -> Dict:
"""
When inserting or updating data, the some raw values in the record
dict must be transformed before their corresponding sqlalchemy column
type will accept the data.
"""
cb_name = 'on_encode' if serialize else 'on_decode'
prepared_record = {}
for k, v in record.items():
if k in (REV):
prepared_record[k] = v
adapter = self._adapters.get(k)
if adapter:
callback = getattr(adapter, cb_name, None)
if callback:
try:
prepared_record[k] = callback(v)
continue
except Exception:
console.error(
message=f'failed to adapt column value: {k}',
data={
'value': v,
'field': adapter.field_class
})
raise
prepared_record[k] = v
return prepared_record
def on_import_error(self, exc: Exception, module_name: Text, context):
"""
If the scanner fails to import a module while it walks the file
system, it comes here to handle and report the problem.
"""
exc_str = traceback.format_exc()
console.error(message=f'manifest could not scan module: {module_name}',
data={'traceback': exc_str.split('\n')})
def validate(data):
data, errors = schema.process(data)
if errors:
console.error(message=f'response validation errors',
data={
'data': pprint.pformat(data),
'errors': errors,
})
return data
def __getattr__(self, method: Text) -> Callable:
method = self.methods.get(method)
if method is None:
console.error(
message=f'celery client does not recognize task name',
data={'task': method}
)
raise ValueError(method)
return method
def create_tables(cls, overwrite=False):
"""
Create all tables for all SqlalchemyStores used in the host app.
"""
if not cls.is_bootstrapped():
console.error(f'{get_class_name(cls)} cannot create '
f'tables unless bootstrapped')
return
meta = cls.get_metadata()
engine = cls.get_engine()
if overwrite:
console.info('dropping Resource SQL tables...')
meta.drop_all(engine)
# create all tables
console.info('creating Resource SQL tables...')
meta.create_all(engine)
def fset(self, owner: 'Resource', value):
field = self.resolver.field
if value is None:
super().fset(owner, None)
return
processed_value, errors = field.process(value)
if errors:
console.error(message=f'cannot set {self}',
data={
'resolver': str(self.resolver),
'field': field,
'schema': owner.ravel.schema,
'errors': errors,
'value': value,
})
raise Exception(str(errors))
super().fset(owner, processed_value)
def on_bind(self,
resource_type: Type['Resource'],
table: Text = None,
schema: 'Schema' = None,
**kwargs):
"""
Initialize SQLAlchemy data strutures used for constructing SQL
expressions used to manage the bound resource type.
"""
# map each of the resource's schema fields to a corresponding adapter,
# which is used to prepare values upon insert and update.
table = (table
or SqlalchemyTableBuilder.derive_table_name(resource_type))
field_class_2_adapter = {
adapter.field_class: adapter
for adapter in self.get_default_adapters(self.dialect, table) +
self._custom_adapters
}
self._adapters = {
field_name: field_class_2_adapter[type(field)]
for field_name, field in self.resource_type.Schema.fields.items()
if (type(field) in field_class_2_adapter
and field.meta.get('ravel_on_resolve') is None)
}
# build the Sqlalchemy Table object for the bound resource type.
self._builder = SqlalchemyTableBuilder(self)
try:
self._table = self._builder.build_table(name=table, schema=schema)
except Exception:
console.error(f'failed to build sa.Table: {table}')
raise
self._id_column = getattr(self._table.c, self.id_column_name)
# remember which column is the _id column
self._id_column_names[self._table.name] = self.id_column_name
# set SqlalchemyStore options here, using bootstrap-level
# options as base/default options.
self._options = dict(self.ravel.kwargs, **kwargs)
def create(self, record: dict) -> dict:
record[self.id_column_name] = self.create_id(record)
prepared_record = self.prepare(record, serialize=True)
insert_stmt = self.table.insert().values(**prepared_record)
_id = prepared_record.get('_id', '')
console.debug(f'SQL: INSERT {str(_id)[:7] + " " if _id else ""}'
f'INTO {self.table}')
try:
if self.supports_returning:
insert_stmt = insert_stmt.return_defaults()
result = self.conn.execute(insert_stmt)
return dict(record, **(result.returned_defaults or {}))
else:
result = self.conn.execute(insert_stmt)
return self.fetch(_id=record[self.id_column_name])
except Exception:
console.error(message=f'failed to insert record',
data={
'record': record,
'resource': get_class_name(self.resource_type),
})
raise
def update(self, data: Dict = None, **more_data) -> 'Resource':
data = dict(data or {}, **more_data)
if data:
self.merge(data)
raw_record = self.dirty.copy()
raw_record.pop(REV, None)
raw_record.pop(ID, None)
errors = {}
prepared_record = {}
for k, v in raw_record.items():
field = self.Schema.fields.get(k)
if field is not None:
if field.name not in self.ravel.virtual_fields:
if v is None and field.nullable:
prepared_record[k] = None
else:
prepared_record[k], error = field.process(v)
if error:
console.error(
f'{self} failed validation for {k}: {v}')
errors[k] = error
self.on_update(prepared_record)
if errors:
raise ValidationError(f'update failed for {self}: {errors}')
updated_record = self.ravel.local.store.dispatch(
'update', (self._id, prepared_record))
if updated_record:
self.merge(updated_record)
self.clean(prepared_record.keys() | updated_record.keys())
self.post_update()
return self
def require(
self,
resolvers: Set[Text] = None,
use_defaults=True,
strict=False,
overwrite=False,
) -> Set[Text]:
"""
Checks if all specified resolvers are present. If they are required
but not present, an exception will be raised for `strict` mode;
otherwise, a set of the missing resolver names is returned.
"""
if isinstance(resolvers, str):
resolvers = {resolvers}
required_resolver_names = (resolvers
or set(k
for k in self.ravel.resolvers.keys()
if self.ravel.resolvers[k].required))
missing_resolver_names = set()
defaults = self.ravel.defaults
for name in required_resolver_names:
resolver = self.ravel.resolvers[name]
if overwrite or name not in self.internal.state:
if use_defaults and name in defaults:
value = defaults[name]()
if value is None and not resolver.nullable:
raise ValidationError(f'{name} not nullable')
self[name] = value
else:
missing_resolver_names.add(name)
if strict and missing_resolver_names:
console.error(message=f'{self.class_name} missing required data',
data={'missing': missing_resolver_names})
raise ValidationError('see error log for details')
return missing_resolver_names
def pre_request(self, action: 'Action', request: 'Request',
raw_args: Tuple, raw_kwargs: Dict):
# NOTE: set csrf_protected=False in your Action decorators
# in order to skip this middleware...
csrf_protected = action.decorator.kwargs.get('csrf_protected')
if csrf_protected is None:
csrf_protected = True
if not csrf_protected:
return
http_request = raw_args[0]
token = http_request.headers.get('CSRF-TOKEN')
session = request.session
# bail if token missing or mismatching
if (not (token and session)) or (session.csrf_token != token):
raise Exception('CSRF token missing or unrecognized')
now = datetime.now()
iv = session.csrf_aes_cbc_iv # cipher mode initialization vector
# decode & decrypt the AES encrypted CSRF token
try:
aes_cipher = AES.new(self.signing_key, AES.MODE_CBC, iv)
json_str = unpad(aes_cipher.decrypt(b64decode(token)),
AES.block_size)
token_components = self.app.json.decode(json_str)
expires_at_isoformat = token_components[0]
# convert expiration date string to a datetime object
if expires_at_isoformat is not None:
expires_at = datetime.fromisoformat(expires_at_isoformat)
else:
expires_at = None
except Exception:
console.error(message='failed to decode CSRF token',
data={
'token': token,
'session_id': session._id
})
raise
# save crsf data to request for use in post_request
request.context.csrf = {
'session_id': token_components[1],
'expires_at': token_components[0],
'csrf_token': token,
}
# abort request if token is expired (provided it has an expiry)
if (expires_at is not None) and (now >= expires_at):
console.error(message='received expired CSRF token',
data={
'csrf_token': token,
'expires_at': expires_at,
'session_id': session._id,
})
raise Exception('invalid CSRF token')
if request.context.csrf['session_id'] != session._id:
console.error(message='CSRF token session ID mismatch',
data={
'csrf_session_id':
request.context.csrf['session_id'],
'request_session_id': session._id,
})
raise Exception('invalid CSRF token')
def build_column(self, field: 'Field') -> sa.Column:
"""
Derive a Sqlalchemy column from a Field object. It looks for
Sqlalchemy-related column kwargs in the field's meta dict.
"""
name = field.source
adapter = self.adapters.get(field.name)
if adapter is None:
console.warning(
'no sqlalchemy field adapter registered '
f'for {field}. using default adapter.'
)
try:
dtype = adapter.on_adapt(field)
except Exception:
console.error(f'could not adapt field {field}')
raise
primary_key = field.meta.get('primary_key', False)
unique = field.meta.get('unique', False)
if field.source == REV:
indexed = True
server_default = '0'
else:
indexed = field.meta.get('index', False)
server_default = None
if 'server_default' in field.meta:
server_default = field.meta['server_default']
elif field.has_constant_default:
defaults = self._resource_type.ravel.defaults
server_default = defaults[field.name]()
if server_default is not None:
if not isinstance(server_default, str):
if adapter is not None:
server_default = adapter.encode(server_default)
if isinstance(field, fields.Bool):
server_default = 'true' if server_default else 'false'
elif isinstance(field, (fields.Int, fields.Float)):
server_default = str(server_default)
else:
try:
server_default = json_encoder.encode(
server_default
)
except:
server_default = str(server_default)
# prepare positional arguments for Column ctor
args = [
name,
dtype() if isinstance(dtype, type) else dtype,
]
# foreign key string path, like 'user._id'
foreign_key_dotted_path = field.meta.get('foreign_key')
if foreign_key_dotted_path:
args.append(ForeignKey(foreign_key_dotted_path))
# prepare keyword arguments for Column ctor
kwargs = dict(
index=indexed,
primary_key=primary_key,
unique=unique,
server_default=server_default
)
try:
column = sa.Column(*args, **kwargs)
except Exception:
console.error(
message=f'failed to build sa.Column: {name}',
data={
'args': args,
'kwargs': kwargs
}
)
raise
if field.nullable is not None:
column.nullable = field.nullable
return column