def __init__(
self,
target: Union[Type['Resource'], Callable] = None,
sources: List['Resource'] = None,
parent: 'Query' = None,
request: 'Request' = None,
eager: bool = True,
):
self.sources = sources or []
self.target = target
self.parent = parent
self.options = DictObject()
self.from_request = request
self.eager = eager
self.requests = {}
self.callbacks = []
self.parameters = DictObject(data={
'where': None,
'order_by': None,
'offset': None,
'limit': None,
},
default=None)
if request:
self.merge(request, in_place=True)
if self.target is not None:
self.select(self.target.ravel.schema.required_fields.keys())
self.select(self.target.ravel.foreign_keys.keys())
def __init__(self, middleware=None, *args, **kwargs):
super().__init__(middleware=middleware, *args, **kwargs)
self.grpc = DictObject()
self.grpc.options = DictObject()
self.grpc.worker_process_count = mp.cpu_count()
self.grpc.worker_thread_pool_size = 1
self.grpc.pb2 = None
self.grpc.pb2_grpc = None
self.grpc.server = None
self.grpc.servicer = None
self._client = None
def __init__(
self,
predicate: Callable = None,
callback: Callable = None,
):
self.log = ConsoleLoggerInterface('scanner')
self.context = DictObject()
if predicate:
self.predicate = predicate
if callback:
self.on_match = callback
def __init__(self, modules: List['PythonModule'] = None, *args, **kwargs):
super().__init__(*args, **kwargs)
modules = sorted(modules, key=lambda x: x['module'])
self._modules = DictObject({
m['module'].replace('.', '_'): PythonModule(name=m['module'], **m)
for m in modules
})
def __init__(self, state=None, **more_state):
# initialize internal state data dict
self.internal = DictObject()
self.internal.state = DirtyDict()
self.merge(state, **more_state)
# eagerly generate default ID if none provided
if ID not in self.internal.state:
id_func = self.ravel.defaults.get(ID)
self.internal.state[ID] = id_func() if id_func else None
def __init__(
self,
resolver: 'Resolver',
query: 'Query' = None,
parent: 'Request' = None,
):
self.resolver = resolver
self.parameters = DictObject({'select': []})
self.query = query
self.result = None
self.parent = parent
def build_nodes(self,
node_class: 'PythonNode',
data: List,
key: Text = None):
if not key:
key = 'name'
if not data:
return []
sorted_data = sorted(data, key=lambda x: (x.get(key) or ''))
return DictObject.from_list(key,
[node_class(**d) for d in sorted_data])
def __init__(self, resources: List = None, indexed=True):
self.internal = DictObject()
self.internal.resources = deque(resources or [])
self.internal.indexed = indexed
self.internal.indexes = defaultdict(BTree)
if indexed:
self.internal.indexes.update({
k: BTree()
for k, field in self.ravel.owner.Schema.fields.items()
if isinstance(field, self.ravel.indexed_field_types)
})
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.key_event_handlers = defaultdict(lambda: defaultdict(list))
self.any_key_event_handlers = defaultdict(lambda: list)
self.mouse_event_handlers = defaultdict(lambda: list)
self.window_event_handlers = defaultdict(
list, {EVENT_TYPE.QUIT: [lambda event: {
'is_running': False
}]}
)
self.local.pygame_state = DictObject()
def factory(cls, owner: Type['Resource'], type_name=None):
"""
Thie factory method returns new Batch subtypes and is used internally at
runtime by the Resource metaclass. For example, if you have a Resource
type called `User`, then this method is invoked to create `User.Batch`.
"""
type_name = type_name or 'Batch' # name of new class
# start with inherited ravel object
ravel = DictObject()
ravel.owner = owner
ravel.indexed_field_types = (fields.String, fields.Int, fields.Id,
fields.Bool, fields.Float)
ravel.properties = {}
for k, resolver in owner.ravel.resolvers.items():
ravel.properties[k] = BatchResolverProperty(resolver)
derived_batch_type = type(type_name, (cls, ),
dict(ravel=ravel, **ravel.properties))
# IS_BATCH is a type flag used in is_batch utility methods
setattr(derived_batch_type, IS_BATCH, True)
return derived_batch_type
def __init__(
self,
middleware: List['Middleware'] = None,
manifest: Manifest = None,
mode: Mode = Mode.normal,
):
# thread-local storage
self.local = local()
self.local.is_bootstrapped = False
self.local.is_started = False
self.local.manifest = Manifest(manifest) if manifest else None
self.local.bootstrapper_thread_id = None
self.local.thread_executor = None
self.local.process_executor = None
self.local.tx_manager = None
self.shared = DictObject() # storage shared by threads
self.shared.manifest_data = None
self.shared.app_counter = 1
self.env = Environment()
self._mode = mode
self._json = JsonEncoder()
self._binder = ResourceBinder()
self._namespace = {}
self._arg_loader = None
self._logger = None
self._root_pid = os.getpid()
self._actions = DictObject()
self._middleware = deque([
m for m in (middleware or [])
if isinstance(self, m.app_types)
])
# set default main thread name
current_thread().name = (
f'{get_class_name(self)}MainThread'
)
def __init__(
self,
predicate: Callable = lambda obj: True,
callback: Callable = lambda obj: None,
log: Optional[Logger] = None,
):
self.static_context: DictObject = DictObject()
self.context = self.static_context.copy()
# replace class-level logger
if log is not None:
self.log = log
self._predicate = predicate
self._callback = callback
def scan(self, package_name, context: Dict = None, verbose=False):
context = dict(self.context.to_dict(), **(context or {}))
context = DictObject(context)
root_module = importlib.import_module(package_name)
root_filename = os.path.basename(root_module.__file__)
if root_filename != '__init__.py':
self.scan_module(root_module, context)
else:
package_dir = os.path.split(root_module.__file__)[0]
if re.match(f'\./', package_dir):
# ensure we use an absolute path for the package dir
# to prevent strange string truncation results below
package_dir = os.path.realpath(package_dir)
package_path_len = package_name.count('.') + 1
package_parent_dir = '/' + '/'.join(
package_dir.strip('/').split('/')[:-package_path_len])
for dir_name, sub_dirs, file_names in os.walk(package_dir):
file_names = set(file_names)
if '.ravel' in file_names:
dot_file_path = os.path.join(dir_name, '.ravel')
dot_data = Json.read(dot_file_path) or {}
ignore = dot_data.get('scanner', {}).get('ignore', False)
if ignore:
self.log.debug(f'scanner ignoring {dir_name}')
sub_dirs.clear()
continue
if '__init__.py' in file_names:
dir_name_offset = len(package_parent_dir)
pkg_path = dir_name[dir_name_offset + 1:].replace("/", ".")
for file_name in file_names:
if file_name.endswith('.py'):
mod_path = f'{pkg_path}.{splitext(file_name)[0]}'
try:
module = importlib.import_module(mod_path)
except Exception as exc:
self.on_import_error(exc, mod_path, context)
continue
self.scan_module(module, context)
self.context = context
return context
def _initialize_class_state(resource_type):
setattr(resource_type, IS_RESOURCE, True)
resource_type.ravel = DictObject()
resource_type.ravel.local = local()
resource_type.ravel.app = None
resource_type.ravel.resolvers = ResolverManager()
resource_type.ravel.foreign_keys = {}
resource_type.ravel.is_abstract = resource_type._compute_is_abstract()
resource_type.ravel.is_bootstrapped = False
resource_type.ravel.is_bound = False
resource_type.ravel.schema = None
resource_type.ravel.defaults = {}
resource_type.ravel.protected_field_names = set()
resource_type.ravel.predicate_parser = PredicateParser(resource_type)
resource_type.ravel.virtual_fields = {}
return resource_type._process_fields()
def _on_bootstrap_configure_grpc(self, options: Dict):
"""
Compute the gRPC options dict and other values needed to run this
application as a gRPC service.
"""
grpc = self.grpc
# get the python package containing this Application
pkg_path = self.manifest.package
pkg = import_module(self.manifest.package)
# compute some file and module paths
grpc.pkg_dir = os.path.dirname(pkg.__file__)
grpc.build_dir = os.path.join(grpc.pkg_dir, 'grpc')
grpc.proto_file = os.path.join(grpc.build_dir, REGISTRY_PROTO_FILE)
# compute grpc options dict from options kwarg and manifest data
kwarg_options = options or {}
manifest_options = self.manifest.data.get('grpc', {})
computed_options = DictUtils.merge(kwarg_options, manifest_options)
# generate default values into gRPC options data
schema = GrpcOptionsSchema()
options, errors = schema.process(computed_options)
if errors:
raise Exception(f'invalid gRPC options: {errors}')
grpc.options = DictObject(options)
grpc.options.port = port = options['port']
grpc.options.server_host = server_host = options['server_host']
grpc.options.client_host = client_host = options['client_host']
grpc.options.server_address = (f'{server_host}:{port}')
grpc.options.client_address = (f'{client_host}:{port}')
grpc.pb2_mod_path = PB2_MODULE_PATH_FSTR.format(pkg_path)
grpc.pb2_grpc_mod_path = PB2_GRPC_MODULE_PATH_FSTR.format(pkg_path)
# add build directory to PYTHONPATH; otherwise, gRPC won't know where
# the pb2 modules are when it tries to import them.
sys.path.append(grpc.build_dir)
def __init__(
self,
ftype: Text = None,
extension: Text = None,
):
super().__init__()
self._paths = DictObject()
self._cache_store = SimulationStore()
# convert the ftype string arg into a File class ref
if not ftype:
self._ftype = Yaml
elif not isinstance(ftype, BaseFile):
self._ftype = import_object(ftype)
assert issubclass(self.ftype, BaseFile)
known_extension = self._ftype.has_extension(extension)
if known_extension:
self._extension = known_extension
if extension is None:
self._extension = self._ftype.default_extension()
def __init__(self, target, decorator):
super().__init__(target, decorator)
self._msg_gen = MessageGenerator()
self._returns_stream = False
self.schemas = DictObject()
def process(
self,
source: Dict,
context: Dict = None,
strict=False,
ignore_required=False,
ignore_nullable=False,
before=None,
after=None,
):
"""
Marshal each value in the "source" dict into a new "dest" dict.
"""
errors = {}
context = DictObject(context or {})
context.schema = self
context.source = source
if self.allow_additional:
dest = source.copy()
else:
dest = {}
after_fields = []
def generate_default(field):
# generate default val from either
# the supplied constant or callable.
if callable(field.default):
source_val = field.default()
else:
source_val = deepcopy(field.default)
return source_val
self.before(source, context)
for field in self.fields.values():
# is key simply present in source?
field_key = field.source or field.name
exists_key = source is not None and field_key in source
# do we ultimately call field.process?
skip_field = not exists_key
# get source value, None is handled below
source_val = source.get(field.source) if isinstance(source,
dict) else None
# pre-process some fields, first by the schema if provided, then by
# the field itself if provided
if field.before:
source_val, source_err = field.before(field,
source_val,
context=context)
if source_err:
errors[field.name] = source_err
if not exists_key:
# source key not present but required
# try to generate default value if possible
# or error.
if field.default is not None:
skip_field = False
source_val = generate_default(field)
elif field.required and not ignore_required:
errors[field.name] = f'{field.name} is required'
continue
else:
continue
if (source_val is None):
if field.default is not None:
# field has a default specified, attempt to generate
# the default from it
skip_field = False
source_val = generate_default(field)
if not field.nullable:
# field cannot be null
if source_val is not None:
# source val is now populated with a default, set it
dest[field.name] = source_val
elif not ignore_nullable:
# but None not allowed!
errors[field.name] = f'{field.name} not nullable'
continue
else:
# when it is nullable then set to None
dest[field.name] = None
continue
if skip_field:
# the key isn't in source, but that's ok,
# as this means that the field isn't required
# and has no default value.
continue
# apply field to the source value
dest_val, field_err = field.process(source_val)
if not field_err:
dest[field.name] = dest_val
else:
errors[field.name] = field_err
if field.after is not None:
after_fields.append(field)
# call all post-process callbacks
for field in after_fields:
dest_val = dest.pop(field.name, None)
field_val, field_err = field.after(field,
dest_val,
dest,
context=context)
# now recheck nullity of the post-processed field value
if ((dest_val is None)
and (not field.nullable and not ignore_nullable)):
errors[field.name] = f'{field.name} not nullable'
elif not field_err:
dest[field.name] = field_val
else:
errors[field.name] = field_err
# "strict" means we raise an exception
# or return just the processed dict
if strict:
if errors:
raise ValidationError(self, errors)
else:
return dest
self.after(dest, context)
results = self.tuple_factory(dest, errors)
return results
class Scanner:
"""
The Scanner recursively walks the filesystem, rooted at a Python package
or module filepath, and matches each object contained in each Python
source file against a logical predicate. If the predicate matches, the
object is passed into the overriden on_match instance method.
"""
def __init__(
self,
predicate: Callable = None,
callback: Callable = None,
):
self.log = ConsoleLoggerInterface('scanner')
self.context = DictObject()
if predicate:
self.predicate = predicate
if callback:
self.on_match = callback
def scan(self, package_name, context: Dict = None, verbose=False):
context = dict(self.context.to_dict(), **(context or {}))
context = DictObject(context)
root_module = importlib.import_module(package_name)
root_filename = os.path.basename(root_module.__file__)
if root_filename != '__init__.py':
self.scan_module(root_module, context)
else:
package_dir = os.path.split(root_module.__file__)[0]
if re.match(f'\./', package_dir):
# ensure we use an absolute path for the package dir
# to prevent strange string truncation results below
package_dir = os.path.realpath(package_dir)
package_path_len = package_name.count('.') + 1
package_parent_dir = '/' + '/'.join(
package_dir.strip('/').split('/')[:-package_path_len])
for dir_name, sub_dirs, file_names in os.walk(package_dir):
file_names = set(file_names)
if '.ravel' in file_names:
dot_file_path = os.path.join(dir_name, '.ravel')
dot_data = Json.read(dot_file_path) or {}
ignore = dot_data.get('scanner', {}).get('ignore', False)
if ignore:
self.log.debug(f'scanner ignoring {dir_name}')
sub_dirs.clear()
continue
if '__init__.py' in file_names:
dir_name_offset = len(package_parent_dir)
pkg_path = dir_name[dir_name_offset + 1:].replace("/", ".")
for file_name in file_names:
if file_name.endswith('.py'):
mod_path = f'{pkg_path}.{splitext(file_name)[0]}'
try:
module = importlib.import_module(mod_path)
except Exception as exc:
self.on_import_error(exc, mod_path, context)
continue
self.scan_module(module, context)
self.context = context
return context
def scan_module(self, module, context):
if None in module.__dict__:
# XXX: why is this happenings?
del module.__dict__[None]
for k, v in inspect.getmembers(module, predicate=self.predicate):
# if verbose:
# console.debug(
# f'scanner matched "{k}" '
# f'{str(v)[:40] + "..." if len(str(v)) > 40 else v}'
# )
try:
self.on_match(k, v, context)
except Exception as exc:
self.on_match_error(exc, module, context, k, v)
def predicate(self, value) -> bool:
return True
def on_match(self, name, value, context):
context[name] = value
def on_import_error(self, exc, module_path, context):
self.log.exception(message='scanner encountered an import error',
data={
'module': module_path,
})
def on_match_error(self, exc, module, context, name, value):
self.log.exception(
message=f'scanner encountered an error while scanning object',
data={
'module': module.__name__,
'object': name,
'type': type(value),
})
def interpret(self, graphql_query_string: Text, context=None) -> Query:
context = context or DictObject()
root_node = self._parse_graphql_query_ast(graphql_query_string)
root_node.source = graphql_query_string
return self._build_query(root_node, self._root_resource_type, context)
def scan(
self,
package: Text,
context: Optional[Union[Dict, DictObject]] = None,
):
"""
Walk the filesystem relative to a python package, specified as a
dotted path. For each object in each module therein, apply a
predicate and, if True, execute a callback, like setting a value in
self.context.
"""
# prepare the initial runtime context dict by merging any new context
# into a copy of the scanner's static context.
raw_context = self.static_context.to_dict()
if context:
raw_context.update(context)
# computed runtime context to pass into self.process:
runtime_context: DictObject = DictObject(raw_context)
root_module = importlib.import_module(package)
root_filename = os.path.basename(root_module.__file__)
# if we're inside a package with an __init__.py file, scan
# it as a module. Otherwise, scan the files in the directory
# containing the file.
if root_filename != INIT_MODULE_NAME:
self.scan_module(root_module, runtime_context)
else:
# scan the directory...
# get information regarding our location in the filesystem
package_dir = os.path.split(root_module.__file__)[0]
if re.match(r'\./', package_dir):
# ensure we use an absolute path for the package dir
# to prevent strange string truncation results below
package_dir = os.path.realpath(package_dir)
package_path_len = package.count('.') + 1
package_parent_dir = '/' + '/'.join(
package_dir.strip('/').split('/')[:-package_path_len])
# walk the filesystem relative to our CWD
for dir_name, sub_dirs, file_names in os.walk(package_dir):
file_names = set(file_names)
# check local .ravel file to see if we should skip this
# directory.
if DOT_FILE_NAME in file_names:
dot_file_path = os.path.join(dir_name, DOT_FILE_NAME)
dot_data = Json.read(dot_file_path) or {}
ignore = dot_data.get('ignore', False)
if ignore:
self.on_ignore_directory(os.path.realpath(dir_name))
sub_dirs.clear()
continue
# scan files in the package directory
if INIT_MODULE_NAME in file_names:
dir_name_offset = len(package_parent_dir)
# compute the dotted package path, derived from the filepath
pkg_path = dir_name[dir_name_offset + 1:].replace("/", ".")
for file_name in file_names:
if not file_name.endswith('.' + PY_EXTENSION):
continue
# compute dotted module path
mod_path = f'{pkg_path}.{splitext(file_name)[0]}'
try:
module = importlib.import_module(mod_path)
except Exception as exc:
self.on_import_error(exc, mod_path,
runtime_context)
continue
# scan the module in the package directory
self.scan_module(module, runtime_context)
# memoize the final context, which is the result of merging runtime
# context generated into a copy of the static context
self.context = runtime_context
return self.context.copy()
class Batch(Entity):
"""
A Batch is a collection of Resources. At runtime, Ravel creates a new Batch
subclass for each Resource type in the app. For example, a User resource
will have User.Batch. An Account resource will have Account.Batch. Each
Batch subclass has non-scalar versions of the ResolverProperties owned by
the corresponding Resource type.
## Accessing Resource State in Batch
Resource and Batch types both implement the same property-based system for
accessing and writing state, allowing both `user.name` to access the name of
a single User and `users.name`, where `users` is a User.Batch instance, to
access a list of all names in the batch. For example:
```python
users = User.Batch([User(name='Augustus'), User(name='Caligula')])
assert users.name == ['Augustus', 'Caligula']
```
## CRUD
Batches implement the same CRUD interface as Resource types, performing the
batch version of each operation. For example, `users.create()` will call
`User.create_many(users)` under the hood.
## Filtering
It is possible to filter batches with a "where" method. For this to work,
a batch must be indexed. Indexing is enabled by default but can be toggled
through a constructor argument. Filtering is O(log N). The "where" method
returns a new batch containing the filtered results. This looks like:
```python
filered_users = users.where(User.name == 'Caligula')
```
## Random Access
It is possible to access specific resources by indexing a batch like an
array. For example `users[1:]`. Note that `users["name"]` will not work; for
that, do `users.name`.
"""
ravel = DictObject()
ravel.owner = None
ravel.properties = {}
ravel.indexed_field_types = (fields.String, fields.Int, fields.Id,
fields.Bool, fields.Float)
def __init__(self, resources: List = None, indexed=True):
self.internal = DictObject()
self.internal.resources = deque(resources or [])
self.internal.indexed = indexed
self.internal.indexes = defaultdict(BTree)
if indexed:
self.internal.indexes.update({
k: BTree()
for k, field in self.ravel.owner.Schema.fields.items()
if isinstance(field, self.ravel.indexed_field_types)
})
def __len__(self):
return len(self.internal.resources)
def __bool__(self):
return bool(self.internal.resources)
def __iter__(self):
return iter(self.internal.resources)
def __getitem__(self, index):
"""
Return a single resource or a new batch, for the given positional index
or slice.
"""
if isinstance(index, slice):
return type(self)(islice(self.internal.resources, index.start,
index.stop, index.step))
else:
assert isinstance(index, int)
return self.internal.resources[index]
def __setitem__(self, index: int, resource: 'Resource'):
"""
Write a resource to a specified index. The resource being written to the
batch must belong to the same type as all other elements. That type is
stored in `Batch.ravel.owner`.
"""
owner = self.ravel.owner
if owner and not isinstance(resource, owner):
raise ValueError(f'wrong Resource type')
self.internal.resources[index] = value
def __repr__(self):
"""
Show the name of the batch, its owner resource type, and its size.
"""
return (f'{get_class_name(self.ravel.owner)}.Batch('
f'size={len(self)})')
def __add__(self, other):
"""
Create and return a copy, containing the concatenated data lists.
"""
clone = type(self)(self.internal.resources, indexed=self.indexed)
if isinstance(other, (list, tuple, set)):
clone.internal.resources.extend(other)
elif isinstance(other, Batch):
assert other.ravel.owner is self.ravel.owner
clone.internal.resources.extend(other.internal.resources)
else:
raise ValueError(str(other))
return clone
@classmethod
def factory(cls, owner: Type['Resource'], type_name=None):
"""
Thie factory method returns new Batch subtypes and is used internally at
runtime by the Resource metaclass. For example, if you have a Resource
type called `User`, then this method is invoked to create `User.Batch`.
"""
type_name = type_name or 'Batch' # name of new class
# start with inherited ravel object
ravel = DictObject()
ravel.owner = owner
ravel.indexed_field_types = (fields.String, fields.Int, fields.Id,
fields.Bool, fields.Float)
ravel.properties = {}
for k, resolver in owner.ravel.resolvers.items():
ravel.properties[k] = BatchResolverProperty(resolver)
derived_batch_type = type(type_name, (cls, ),
dict(ravel=ravel, **ravel.properties))
# IS_BATCH is a type flag used in is_batch utility methods
setattr(derived_batch_type, IS_BATCH, True)
return derived_batch_type
@classmethod
def generate(
cls,
resolvers: Set[Text] = None,
values: Dict = None,
count: int = None,
):
"""
Generate a Batch of fixtures, generating random values for the
specified resolvers.
## Arguments
- `resolvers`: list of resolver names for which to generate values.
- `values`: dict of static values to use.
- `count`: number of resources to generate. defaults between [1, 10].
"""
owner = cls.ravel.owner
# if no count, randomize between [1, 10]
count = max(count, 1) if count is not None else randint(1, 10)
if owner is None:
# this batch isn't associated with any Resource type
# and therefore we don't know what to generate.
raise Exception('unbound Batch type')
if not resolvers:
resolvers = set(owner.ravel.resolvers.fields.keys())
else:
resolvers = set(resolvers)
# create and return the new batch
return cls(
owner.generate(resolvers, values=values) for _ in range(count))
def foreach(self, callback: Callable) -> 'Batch':
for i, x in enumerate(self.internal.resources):
callback(i, x)
return self
def sort(self, order_by) -> 'Batch':
self.internal.resources = OrderBy.sort(self.internal.resources,
order_by)
return self
def set(self, other: Union[Dict, 'Resource'] = None, **values) -> 'Batch':
return self.merge(other=other, **values)
def merge(self,
other: Union[Dict, 'Resource'] = None,
**values) -> 'Batch':
"""
Batch merge the given dict or resource into all resources contained in
the batch.
"""
for resource in self.internal.resources:
resource.merge(other)
resource.merge(values)
return self
def insert(self, index, resource):
"""
Insert a resource at the specified index.
"""
self.internal.resources.insert(index, resource)
if self.internal.indexed:
self._update_indexes(resource)
return self
def remove(self, resource):
"""
Remove a resource at the specified index.
"""
try:
self.internal.resources.remove(resource)
self._prune_indexes(resource)
except ValueError:
raise
return self
def append(self, resource):
"""
Add a resource at the end of the batch.
"""
self.insert(-1, resource)
return self
def appendleft(self, resource):
"""
Add a resource at the front of the batch (index: 0)
"""
self.insert(0, resource)
return self
def extend(self, resources):
"""
Insert a collection of resource at the end of the batch.
"""
self.internal.resources.extend(resources)
for resource in resources:
self._update_indexes(resource)
def extendleft(self, resources):
"""
Insert a collection of resource at the front of the batch.
"""
self.internal.resources.extendleft(resources)
for resource in resources:
self._update_indexes(resource)
def pop(self, index=-1):
"""
Remove and return the resource at the end of the batch.
"""
resource = self.internal.resources.pop(index)
self._prune_indexes(resource)
return resource
def popleft(self, index=-1):
"""
Remove and return the resource at the front of the batch.
"""
resource = self.internal.resources.popleft(index)
self._prune_indexes(resource)
def rotate(self, n=1):
"""
Shift ever element to the right so that the element at index -1 wraps
around, back to index 0. The direction can be reversed using a negative
value of `n`.
"""
self.internal.resources.rotate(n)
return self
def where(self, *predicates: Tuple['Predicate'], indexed=False) -> 'Batch':
"""
Return a new batch, containing the resources whose attributes match the
given predicates, like `User.created_at >= cutoff_date`, for instance.
The new batch is *not* indexed by default, so make sure to set
`indexed=True` if you intend to filter the filtered batch further.
"""
predicate = Predicate.reduce_and(flatten_sequence(predicates))
resources = self._apply_indexes(predicate)
return type(self)(resources, indexed=indexed)
def create(self):
"""
Insert all resources into the store.
"""
if self:
self.ravel.owner.create_many(self.internal.resources)
return self
def update(self, state: Dict = None, **more_state):
"""
Apply an update to all resources in the batch, writing it to the store.
"""
if self:
self.ravel.owner.update_many(self, data=state, **more_state)
return self
def delete(self):
"""
Delete all resources in the batch fro mthe store.
"""
if self:
self.ravel.owner.delete_many(self.internal.resources)
return self
def save(self, resolvers: Union[Text, Set[Text]] = None):
"""
Save all resources in the batch, effectively creating some and updating
others.
"""
if not self:
return self
# batch save resource resolver targets
if resolvers is not None:
owner_resource_type = self.ravel.owner
fields_to_save = set()
if isinstance(resolvers, str):
resolvers = {resolvers}
elif not isinstance(resolvers, set):
resolvers = set(resolvers)
for name in resolvers:
resolver = owner_resource_type.ravel.resolvers[name]
if name not in owner_resource_type.ravel.schema.fields:
visited_ids = set()
unique_targets = resolver.target.Batch()
if resolver.many:
for batch in getattr(self, name):
if batch:
unique_targets.extend(batch)
else:
for resource in getattr(self, name):
if resource:
unique_targets.append(resource)
if unique_targets:
unique_targets.save()
else:
fields_to_save.add(name)
else:
fields_to_save = None
# now save fields
self.ravel.owner.save_many(
self.internal.resources,
resolvers=fields_to_save,
)
return self
def clean(self, resolvers: Set[Text] = None):
"""
Mark all resources in the batch as clean, meaning that no state elements
are considered dirty and in need up saving to the store.
"""
for resource in self.internal.resources:
# TODO: renamed fields kwarg to resolvers
resource.clean(fields=resolvers)
return self
def mark(self, resolvers: Set[Text] = None):
"""
Mark the given resolvers as dirty, for all resources in the batch.
"""
for resource in self.internal.resources:
resource.mark(fields=resolvers)
return self
def dump(
self,
resolvers: Set[Text] = None,
style: 'DumpStyle' = None,
) -> List[Dict]:
"""
Return a list with the dump of each resource in the batch.
"""
return [
resource.dump(resolvers=resolvers, style=style)
for resource in self.internal.resources
]
def resolve(self, resolvers: Union[Text, Set[Text]] = None) -> 'Batch':
"""
Execute each of the resolvers, specified by name, storing the results in
`self.internal.state`.
"""
if self._id is None:
return self
if isinstance(resolvers, str):
resolvers = {resolvers}
elif not resolvers:
resolvers = set(self.ravel.owner.ravel.schema.fields.keys())
# execute all requested resolvers
for k in resolvers:
resolver = self.ravel.owner.ravel.resolvers.get(k)
if resolver is not None:
resolver.resolve_batch(self)
# clean the resolved values so they arent't accidently saved on
# update/create, as we just fetched them from the store.
self.clean(resolvers)
return self
def unload(self, resolvers: Set[Text] = None) -> 'Batch':
"""
Clear resolved state data from all resources in the batch.
"""
if not resolvers:
resolvers = set(self.owner.Schema.fields.keys())
resolvers.discard(ID)
resolvers.discard(REV)
for resource in self.internal.resources:
resource.unload(resolvers)
return self
def _update_indexes(self, resource):
"""
Insert a Resource from the index B-trees.
"""
for k, index in self.internal.indexes.items():
if k in resource.internal.state:
value = resource[k]
if value not in index:
index[value] = set()
index[value].add(resource)
def _prune_indexes(self, resource):
"""
Remove a Resource from the index B-trees.
"""
for k, index in self.internal.indexes.items():
if k in resource.internal.state:
value = resource[k]
if value in index:
index[value].remove(resource)
def _apply_indexes(self, predicate):
if predicate is None:
return self.records.keys()
op = predicate.op
empty = set()
_ids = set()
if isinstance(predicate, ConditionalPredicate):
k = predicate.field.source
v = predicate.value
indexes = self.internal.indexes
index = indexes[k]
if op == OP_CODE.EQ:
_ids = indexes[k].get(v, empty)
elif op == OP_CODE.NEQ:
_ids = union([
_id_set for v_idx, _id_set in index.items() if v_idx != v
])
elif op == OP_CODE.INCLUDING:
v = v if isinstance(v, set) else set(v)
_ids = union([index.get(k_idx, empty) for k_idx in v])
elif op == OP_CODE.EXCLUDING:
v = v if isinstance(v, set) else set(v)
_ids = union([
_id_set for v_idx, _id_set in index.items()
if v_idx not in v
])
else:
keys = np.array(index.keys(), dtype=object)
if op == OP_CODE.GEQ:
offset = bisect.bisect_left(keys, v)
interval = slice(offset, None, 1)
elif op == OP_CODE.GT:
offset = bisect.bisect(keys, v)
interval = slice(offset, None, 1)
elif op == OP_CODE.LT:
offset = bisect.bisect_left(keys, v)
interval = slice(0, offset, 1)
elif op == OP_CODE.LEQ:
offset = bisect.bisect(keys, v)
interval = slice(0, offset, 1)
else:
# XXX: raise StoreError
raise Exception('unrecognized op')
_ids = union(
[index[k] for k in keys[interval] if k is not None])
elif isinstance(predicate, BooleanPredicate):
lhs = predicate.lhs
rhs = predicate.rhs
if op == OP_CODE.AND:
lhs_result = self._apply_indexes(lhs)
if lhs_result:
rhs_result = self._apply_indexes(rhs)
_ids = set.intersection(lhs_result, rhs_result)
elif op == OP_CODE.OR:
lhs_result = self._apply_indexes(lhs)
rhs_result = self._apply_indexes(rhs)
_ids = set.union(lhs_result, rhs_result)
else:
# XXX: raise StoreError
raise Exception('unrecognized boolean predicate')
return _ids
def __init__(cls, name, bases, dict_):
cls.ravel = DictObject()
cls.ravel.local = local()
cls.ravel.local.is_bootstrapped = False
class Query(object):
def __init__(
self,
target: Union[Type['Resource'], Callable] = None,
sources: List['Resource'] = None,
parent: 'Query' = None,
request: 'Request' = None,
eager: bool = True,
):
self.sources = sources or []
self.target = target
self.parent = parent
self.options = DictObject()
self.from_request = request
self.eager = eager
self.requests = {}
self.callbacks = []
self.parameters = DictObject(data={
'where': None,
'order_by': None,
'offset': None,
'limit': None,
},
default=None)
if request:
self.merge(request, in_place=True)
if self.target is not None:
self.select(self.target.ravel.schema.required_fields.keys())
self.select(self.target.ravel.foreign_keys.keys())
def __getattr__(self, parameter_name: str):
return ParameterAssignment(self, parameter_name)
def __getitem__(self, resolver: Text):
return self.requests.get(resolver)
def __call__(self, *args, **kwargs):
return self.execute(*args, **kwargs)
def __len__(self) -> int:
return len(self.requests)
def __contains__(self, resolver: Text) -> bool:
return resolver in self.requests
def __iter__(self):
return iter(self.execute())
def __repr__(self):
offset = self.parameters.offest
limit = self.parameters.limit
return (f'Query('
f'target={get_class_name(self.target)}['
f'{offset if offset is not None else ""}'
f':'
f'{limit if limit is not None else ""}'
f'])')
def merge(self,
other: Union['Query', 'Request'],
in_place: bool = False) -> 'Query':
if in_place:
if isinstance(other, Query):
self.parameters.update(deepcopy(other.parameters.to_dict()))
self.options.update(deepcopy(other.options.to_dict()))
self.select(other.requests.values())
else:
assert isinstance(other, Request)
self._merge_request(other)
return self
else:
merged_query = type(self)(
target=self.target,
parent=self.parent,
)
merged_query.merge(self, in_place=True)
if isinstance(other, Query):
merged_query.merge(other, in_place=True)
else:
assert isinstance(other, Request)
merged_query._merge_request(other)
return merged_query
def _merge_request(self, request: 'Request'):
# set the target type or ensure that the request's resolver
# target type is the same as this query's, or bail.
if request.resolver and request.resolver.target:
if self.target is None:
self.target = request.resolver.target
elif self.target is not request.resolver.target:
raise Exception(
'cannot merge two queries with different target types')
# marshal in raw query parameters from the request
if request.parameters:
params = request.parameters
if 'select' in params:
self.select(params.select)
if 'where' in params:
self.where(params.where)
if 'order_by' in params:
self.order_by(params.order_by)
if 'limit' in params:
self.limit(params.limit)
if 'offset' in params:
self.offset(params.offset)
def configure(self, options: Dict = None, **more_options) -> 'Query':
options = dict(options or {}, **more_options)
self.options.update(options)
return self
def execute(
self,
first=None,
simulate=False,
) -> Union['Resource', 'Batch']:
"""
Execute the query, returning a single Resource ora a Batch.
"""
if self.eager:
# "eager" here means that we always fetch at least ALL field
# values by adding them to the "select" set...
self.select(self.target.ravel.resolvers.fields.keys())
executor = Executor(simulate=simulate)
batch = executor.execute(self, sources=self.sources)
if first:
result = batch[0] if batch else None
else:
result = batch
if self.callbacks:
for func in self.callbacks:
func(self, result)
return result
def exists(self):
self.requests.clear()
self.select(self.target._id)
return bool(self.execute(first=True))
def deselect(self, *args):
"""
Remove the given arguments from the query's requests dict.
"""
args = flatten_sequence(args)
keys = set()
for obj in args:
if isinstance(obj, str):
# if obj is str, replace it with the corresponding resolver
# property from the target Resource class.
_obj = getattr(self.target, obj, None)
if _obj is None:
raise ValueError(f'unknown resolver: {obj}')
obj = _obj
elif is_resource(obj):
keys.update(obj.internal.state.keys())
continue
elif is_resource_type(obj):
keys.update(obj.ravel.resolvers.keys())
continue
# build a resolver request
request = None
if isinstance(obj, LoaderProperty) and (obj.decorator is None):
resolver_property = obj
keys.add(resolver_property.resolver.name)
elif isinstance(obj, ResolverProperty):
resolver_property = obj
keys.add(resolver_property.resolver.name)
elif isinstance(obj, Request):
request = obj
keys.add(request.resolver.name)
for key in keys:
if key in self.requests:
del self.requests[key]
return self
def select(self, *args):
args = flatten_sequence(args)
for obj in args:
if isinstance(obj, str):
# if obj is str, replace it with the corresponding resolver
# property from the target Resource class.
_obj = getattr(self.target, obj, None)
if _obj is None:
raise ValueError(f'unknown resolver: {obj}')
obj = _obj
elif is_resource(obj):
self.select(obj.internal.state.keys())
continue
elif is_resource_type(obj):
self.select(obj.ravel.resolvers.keys())
continue
# build a resolver request
request = None
if isinstance(obj, LoaderProperty) and (obj.decorator is None):
resolver_property = obj
request = Request(resolver_property.resolver)
elif isinstance(obj, ResolverProperty):
resolver_property = obj
request = Request(resolver_property.resolver)
elif isinstance(obj, Request):
request = obj
# finally set the request if one was generated
if request:
resolver_name = request.resolver.name
if resolver_name not in self.target.ravel.virtual_fields:
if self.from_request is not None:
request.parent = self.from_request
self.requests[request.resolver.name] = request
return self
def where(self, *predicates, **equality_checks):
predicates = flatten_sequence(predicates)
for field_name, value in equality_checks.items():
resolver_prop = getattr(self.target, field_name)
pred = (resolver_prop == value)
predicates.append(pred)
if predicates:
if self.parameters.where:
self.parameters.where &= Predicate.reduce_and(predicates)
else:
self.parameters.where = Predicate.reduce_and(predicates)
return self
def order_by(self, *order_by):
order_by = flatten_sequence(order_by)
if order_by:
self.parameters.order_by = []
for obj in order_by:
if isinstance(obj, OrderBy):
self.parameters.order_by.append(obj)
elif isinstance(obj, str):
if obj.lower().endswith(' desc'):
order_by_obj = OrderBy(obj.split()[0], desc=True)
else:
order_by_obj = OrderBy(obj.split()[0], desc=False)
if order_by_obj.key not in self.target.ravel.resolvers:
raise ValueError(
f'uncognized resolver: {order_by_obj.key}')
self.parameters.order_by.append(order_by_obj)
else:
self.parameters.order_by = None
return self
def offset(self, offset=None):
if offset is not None:
self.parameters.offset = max(0, int(offset))
else:
self.parameters.offset = None
return self
def limit(self, limit):
if limit is not None:
self.parameters.limit = max(1, int(limit))
else:
self.parameters.limit = None
return self
def objectify(data):
copy = data.copy()
for k, v in data.items():
if isinstance(v, dict):
copy[k] = objectify(v)
return DictObject(copy)
def store_classes(self) -> Dict[Text, Type['Store']]:
if self.scanner is not None:
return self.scanner.context.store_classes
return DictObject()
def resource_classes(self) -> Dict[Text, Type['Resource']]:
if self.scanner is not None:
return self.scanner.context.resource_classes
return DictObject()
def __init__(self, state):
self.__dict__['internal'] = state
self.__dict__['context'] = DictObject()