def __init__(self, source: Union['Manifest', Dict, Text, Callable]): filepath, data = self._load_manifest_data(source) self.env = Environment() self.filepath = filepath self.data = data self.bindings = [] self.bootstraps = [] self.default_bootstrap = None self.values = None self.scanner = None self.app = None
def __init__( self, middleware: List['Middleware'] = None, manifest: Manifest = None, mode: Mode = Mode.normal, ): # thread-local storage self.local = local() self.local.is_bootstrapped = False self.local.is_started = False self.local.manifest = Manifest(manifest) if manifest else None self.local.bootstrapper_thread_id = None self.local.thread_executor = None self.local.process_executor = None self.local.tx_manager = None self.shared = DictObject() # storage shared by threads self.shared.manifest_data = None self.shared.app_counter = 1 self.env = Environment() self._mode = mode self._json = JsonEncoder() self._binder = ResourceBinder() self._namespace = {} self._arg_loader = None self._logger = None self._root_pid = os.getpid() self._actions = DictObject() self._middleware = deque([ m for m in (middleware or []) if isinstance(self, m.app_types) ]) # set default main thread name current_thread().name = ( f'{get_class_name(self)}MainThread' )
def __init__(self, store_class_name: Text = None, *args, **kwargs): super().__init__(*args, **kwargs) self.env = Environment() self.store_class_name = store_class_name or 'SqlalchemyStore'
class Store(object, metaclass=StoreMeta): env = Environment() ravel = None def __init__(self, *args, **kwargs): self._history = StoreHistory(store=self) self._is_bound = False self._resource_type = None def __repr__(self): if self.is_bound: return (f'<{get_class_name(self)}' f'(resource_type={get_class_name(self.resource_type)})>') else: return (f'<{get_class_name(self)}>') def dispatch(self, method_name: Text, args: Tuple = None, kwargs: Dict = None): """ Delegate a Store call to the named method, performing any side-effects, like creating a StoreHistory event if need be. This is used internally by Resource to call into store methods. """ # call the requested Store method func = getattr(self, method_name) exc = None try: result = func(*(args or tuple()), **(kwargs or {})) except Exception as exc: raise exc # create and store the Store call in a history event if self.history.is_recording_method(method_name): event = StoreEvent(method_name, args, kwargs, result, exc) self._history.append(event) # finally faise the exception if one was generated if exc is not None: data = {'method': method_name, 'args': args, 'kwargs': kwargs} if not isinstance(exc, RavelError): raise RavelError(data=data, wrapped_exception=exc) else: exc.data.update(data) raise exc return result @property def is_bound(self) -> bool: return self._is_bound @property def resource_type(self) -> Type['Resource']: return self._resource_type @property def schema(self) -> 'Schema': if self._resource_type is not None: return self._resource_type.ravel.schema return None @property def app(self) -> 'Application': return self.ravel.app @property def history(self) -> 'StoreHistory': return self._history def replay(self, history: StoreHistory = None, reads=True, writes=True) -> Dict[StoreEvent, object]: """ Replay events (interface calls) from this or another store's history, returning an ordered mapping from the event object to the return value of the corresponding store method. Args: - `history`: the history to replay in this store. - `reads`: replay "read" events, like query, get, get_many. - `writes`: replay "write" events, like create, update, etc. """ results = OrderedDict() # use this store's own history if none specified if history is None: history = self.history for event in history: is_read = event.method in self.history.read_method_names is_write = event.method in self.history.write_method_names if (is_read and reads) or (is_write and writes): func = getattr(self, event.method) result = func(*event.args, **event.kwargs) results[event] = result return results def bind(self, resource_type: Type['Resource'], **kwargs): t1 = datetime.now() self._resource_type = resource_type self.on_bind(resource_type, **kwargs) t2 = datetime.now() secs = (t2 - t1).total_seconds() console.debug(f'bound {TypeUtils.get_class_name(resource_type)} to ' f'{TypeUtils.get_class_name(self)} ' f'in {secs:.2f}s') self._is_bound = True @classmethod def bootstrap(cls, app: 'Application' = None, **kwargs): """ Perform class-level initialization, like getting a connectio pool, for example. """ t1 = datetime.now() cls.ravel.app = app cls.on_bootstrap(**kwargs) cls.ravel.local.is_bootstrapped = True t2 = datetime.now() secs = (t2 - t1).total_seconds() console.debug(f'bootstrapped {TypeUtils.get_class_name(cls)} ' f'in {secs:.2f}s') return cls @classmethod def on_bootstrap(cls, **kwargs): pass def on_bind(cls, resource_type: Type['Resource'], **kwargs): pass @classmethod def is_bootstrapped(cls): return getattr(cls.ravel.local, 'is_bootstrapped', False) @classmethod def has_transaction(cls): raise NotImplementedError() @classmethod def begin(cls, **kwargs): raise NotImplementedError() @classmethod def commit(cls, **kwargs): raise NotImplementedError() @classmethod def rollback(cls, **kwargs): raise NotImplementedError() def create_id(self, record: Dict) -> object: """ Generate and return a new ID for the given not-yet-created record. If this method returns None, the backend database/storage technology must create and return it instead. """ new_id = record.get(ID) if new_id is None: new_id = self.resource_type.ravel.defaults[ID]() return new_id def increment_rev(self, rev: Text = None) -> Text: time_ms = int(1000000 * time.time()) if rev: rev_no = int(rev.split('-')[1], 36) + 1 else: rev_no = 1 return f'{base36.dumps(time_ms)}-{base36.dumps(rev_no)}' @abstractmethod def exists(self, _id) -> bool: """ Return True if the record with the given _id exists. """ @abstractmethod def exists_many(self, _ids: Set) -> Dict[object, bool]: """ Return a mapping from _id to a boolean, indicating if the specified resource exists. """ @abstractmethod def count(self) -> int: """ Return the total number of stored records. """ @abstractmethod def query(self, predicate: 'Predicate', fields: Set[Text] = None, **kwargs) -> List[Dict]: """ TODO: rename to "select" Return all records whose fields match a logical predicate. """ @abstractmethod def fetch(self, _id, fields: Dict = None) -> Dict: """ Read a single record by _id, selecting only the designated fields (or all by default). """ @abstractmethod def fetch_many(self, _ids: List, fields: Dict = None) -> Dict: """ Read multiple records by _id, selecting only the designated fields (or all by default). """ @abstractmethod def fetch_all(self, fields: Set[Text] = None) -> Dict: """ Return all records managed by this Store. """ @abstractmethod def create(self, data: Dict) -> Dict: """ Create a new record with the _id. If the _id is contained is not contained in the data dict nor provided as the _id argument, it is the responsibility of the Store class to generate the _id. """ @abstractmethod def create_many(self, records: List[Dict]) -> List[Dict]: """ Create a new record. It is the responsibility of the Store class to generate the _id. """ @abstractmethod def update(self, _id, data: Dict) -> Dict: """ Update a record with the data passed in. """ @abstractmethod def update_many(self, _ids: List, data: List[Dict] = None) -> List[Dict]: """ Update multiple records. If a single data dict is passed in, then try to apply the same update to all records; otherwise, if a list of data dicts is passed in, try to zip the _ids with the data dicts and apply each unique update or each group of identical updates individually. """ @abstractmethod def delete(self, _id) -> None: """ Delete a single record. """ @abstractmethod def delete_many(self, _ids: List) -> None: """ Delete multiple records. """ @abstractmethod def delete_all(self) -> None: """
class FilesystemStore(Store): env = Environment() root = None paths = None def __init__( self, ftype: Text = None, extension: Text = None, ): super().__init__() self._paths = DictObject() self._cache_store = SimulationStore() # convert the ftype string arg into a File class ref if not ftype: self._ftype = Yaml elif not isinstance(ftype, BaseFile): self._ftype = import_object(ftype) assert issubclass(self.ftype, BaseFile) known_extension = self._ftype.has_extension(extension) if known_extension: self._extension = known_extension if extension is None: self._extension = self._ftype.default_extension() @property def paths(self): return self._paths @property def extension(self): return self._extension @property def ftype(self): return self._ftype @classmethod def on_bootstrap( cls, ftype: Text = None, root: Text = None, use_recursive_merge=True, store_primitives=False, prefetch: bool = True, yaml_loader_class: Text = 'FullLoader' ): cls.ftype = import_object(ftype) if ftype else Yaml cls.root = root or cls.root cls.use_recursive_merge = use_recursive_merge cls.store_primitives = store_primitives cls.do_prefetch = prefetch if 'yaml' in cls.ftype.extensions(): cls.yaml_loader_class = getattr(yaml, yaml_loader_class, None) else: cls.yaml_loader_class = None if not cls.root: raise MissingBootstrapParameterError('missing parameter: root') def on_bind( self, resource_type, root: Text = None, ftype: BaseFile = None, store_primitives=None, prefetch: bool = None, yaml_loader_class: Text = None, ): """ Ensure the data dir exists for this Resource type. """ if isinstance(ftype, str): self.ftype = import_object(ftype) if store_primitives is not None: self.store_primitives = store_primitives if prefetch is not None: self.do_prefetch = prefetch if yaml_loader_class is not None: if self.ftype.lower() == 'yaml': self.yaml_loader_class = getattr( yaml, yaml_loader_class, None ) else: self.yaml_loader_class = None self.paths.root = root or self.root self.paths.records = os.path.join( self.paths.root, StringUtils.snake(resource_type.__name__) ) os.makedirs(self.paths.records, exist_ok=True) # bootstrap, bind, and backfill the in-memory cache if self.do_prefetch: self.bust_cache(self.do_prefetch) def bust_cache(self, prefetch=False): self._cache_store = SimulationStore() if not self._cache_store.is_bootstrapped(): self._cache_store.bootstrap(self.resource_type.ravel.app) self._cache_store.bind(self.resource_type) if prefetch: self._cache_store.create_many( record for record in self.fetch_all(ignore_cache=True).values() if record ) @classmethod def has_transaction(cls): return False @classmethod def begin(cls, **kwargs): pass @classmethod def commit(cls, **kwargs): pass @classmethod def rollback(cls, **kwargs): pass def create_id(self, record): return record.get(ID, UuidString.next_id()) def exists(self, _id: Text) -> bool: return BaseFile.exists(self.mkpath(_id)) def exists_many(self, _ids: Set) -> Dict[object, bool]: return {_id: self.exists(_id) for _id in _ids} def create(self, record: Dict) -> Dict: _id = self.create_id(record) record = self.update(_id, record) record[ID] = _id self._cache_store.create(record) return record def create_many(self, records): created_records = [] for record in records: created_records.append(self.create(record)) self._cache_store.create_many(created_records) return created_records def count(self) -> int: fnames = glob.glob(f'{self.paths.records}/*.{self.extension}') return len(fnames) def fetch(self, _id, fields=None) -> Dict: records = self.fetch_many([_id], fields=fields) record = records.get(_id) if records else {} return record def fetch_many( self, _ids: List = None, fields: Set[Text] = None, ignore_cache=False ) -> Dict: """ """ if not _ids: _ids = [] # reduce _ids to its unique members by making it a set if not isinstance(_ids, set): all_ids = set(_ids) else: all_ids = _ids ids_to_fetch_from_fs = set() # we do not want to ignore the cache here if not ignore_cache: cached_records = self._cache_store.fetch_many(all_ids, fields=fields) for record_id, record in cached_records.items(): if record is None: ids_to_fetch_from_fs.add(record_id) # otherwise we will go straight to the filesystem else: cached_records = {} ids_to_fetch_from_fs = self._fetch_all_ids() # if there are any remaining ID's not returned from cache, # fetch them from file system if ids_to_fetch_from_fs: # prepare the set of field names to fetch fields = fields if isinstance(fields, set) else set(fields or []) if not fields: fields = set(self.resource_type.Schema.fields.keys()) fields |= {ID, REV} records = {} non_null_records = [] for _id in ids_to_fetch_from_fs: fpath = self.mkpath(_id) try: record = self.ftype.read( fpath, loader_class=self.yaml_loader_class ) except FileNotFoundError: records[_id] = None console.debug( message='file not found by filesystem store', data={'filepath': fpath} ) continue if record: record, errors = self.schema.process(record) if errors: raise Exception( f'validation error while loading ' f'{_id}.{self.extension}' ) record.setdefault(ID, _id) records[_id] = {k: record.get(k) for k in fields} non_null_records.append(record) # if for some reason a file was created manually # with a _rev, we create one here and save it if REV not in record: record[REV] = self.increment_rev() self.update(_id, record) else: records[ID] = None self._cache_store.create_many(non_null_records) cached_records.update(records) return cached_records def fetch_all(self, fields: Set[Text] = None, ignore_cache=False) -> Dict: return self.fetch_many(None, fields=fields, ignore_cache=ignore_cache) def update(self, _id, data: Dict) -> Dict: fpath = self.mkpath(_id) base_record = self.ftype.read( fpath, loader_class=self.yaml_loader_class ) schema = self.resource_type.ravel.schema base_record, errors = schema.process(base_record) if base_record: # this is an upsert if self.use_recursive_merge: record = DictUtils.merge(base_record, data) else: record = dict(base_record, **data) else: record = data record[REV] = self.increment_rev(record.get(REV)) if ID not in record: record[ID] = _id if self.store_primitives: json = self.app.json self.ftype.write( path=fpath, data=json.decode(json.encode(record)) ) else: self.ftype.write(path=fpath, data=record) self._cache_store.update(_id, record) return record def update_many(self, _ids: List, updates: List = None) -> Dict: return { _id: self.update(_id, data) for _id, data in zip(_ids, updates) } def delete(self, _id) -> None: self._cache_store.delete(_id) fpath = self.mkpath(_id) os.remove(fpath) def delete_many(self, _ids: List) -> None: for _id in _ids: self.delete(_id) def delete_all(self): _ids = self._fetch_all_ids() self.delete_many(_ids) def query(self, *args, **kwargs): if self._cache_store is not None and not self._cache_store.count(): self.fetch_all(ignore_cache=True) return self._cache_store.query(*args, **kwargs) def mkpath(self, fname: Text) -> Text: fname = self.ftype.format_file_name(fname) return os.path.join(self.paths.records, fname) def _fetch_all_ids(self): _ids = set() for fpath in glob.glob(f'{self.paths.records}/*.{self.extension}'): fname = fpath.split('/')[-1] basename = os.path.splitext(fname)[0] _ids.add(basename) return _ids
class FalconService(AbstractWsgiService): env = Environment() def __init__(self, router=None, *args, **kwargs): super().__init__(*args, **kwargs) self._json_encoder = JsonEncoder() self._route_2_resource = {} self._router = router @property def falcon_middleware(self): return [] @property def falcon_resources(self): return self._route_2_resource @property def request_type(self): return Request @property def response_type(self): return Response @staticmethod def handle_error(exc, request, response, params): response.status = falcon.HTTP_500 traceback.print_exc() def on_start(self, *args, **kwargs): console.info( message='starting Falcon web service', data={'endpoints': sorted(self.actions.keys())} ) return self.entrypoint(*args, **kwargs) def entrypoint( self, environ=None, start_response=None, *args, **kwargs ): middleware = self.falcon_middleware for m in middleware: if isinstance(m, Middleware): m.bind(self) falcon_app = falcon.App( middleware=middleware, request_type=self.request_type, response_type=self.response_type, router=self._router ) falcon_app.req_options = Request.Options() falcon_app.resp_options = Response.Options() falcon_app.add_error_handler(Exception, self.handle_error) for route, resource in self._route_2_resource.items(): falcon_app.add_route(route, resource) return falcon_app def on_decorate(self, endpoint): super().on_decorate(endpoint) for route in endpoint.routes: resource = self._route_2_resource.get(route) if resource is None: resource = FalconResource(route) self._route_2_resource[route] = resource resource.add_endpoint(endpoint) else: resource.add_endpoint(endpoint) def on_request(self, endpoint, request, response, *args, **kwargs): if request.content_length: app_kwargs = dict(request.media or {}, **kwargs) else: app_kwargs = dict(kwargs) app_kwargs.update(request.params) # append URL path variables route = request.path.strip('/').split('/') route_template = request.uri_template.strip('/').split('/') for k, v in zip(route_template, route): if k and (k[0] == '{' and k[-1] == '}'): app_kwargs[k[1:-1]] = v return (tuple(), app_kwargs) def on_response( self, action, result, raw_args=None, raw_kwargs=None, *args, **kwargs ): request, response = raw_args response.media = result return result
class SqlalchemyStore(Store): """ A SQLAlchemy-based store, which keeps a single connection pool (AKA Engine) shared by all threads; however, each thread keeps singleton thread-local database connection and transaction objects, managed through connect()/close() and begin()/end(). """ env = Environment( SQLALCHEMY_STORE_ECHO=fields.Bool(default=False), SQLALCHEMY_STORE_SHOW_QUERIES=fields.Bool(default=False), SQLALCHEMY_STORE_DIALECT=fields.Enum(fields.String(), Dialect.values(), default=Dialect.sqlite), SQLALCHEMY_STORE_PROTOCOL=fields.String(default='sqlite'), SQLALCHEMY_STORE_USER=fields.String(), SQLALCHEMY_STORE_HOST=fields.String(), SQLALCHEMY_STORE_PORT=fields.String(), SQLALCHEMY_STORE_NAME=fields.String(), ) # id_column_names is a mapping from table name to its _id column name _id_column_names = {} # only one thread needs to bootstrap the SqlalchemyStore. This lock is # used to ensure that this is what happens when the host app bootstraps. _bootstrap_lock = RLock() @classmethod def get_default_adapters(cls, dialect: Dialect, table_name) -> List[Field.Adapter]: # TODO: Move this into the adapters file adapters = [ fields.Field.adapt( on_adapt=lambda field: sa.Text, on_encode=lambda x: cls.ravel.app.json.encode(x), on_decode=lambda x: cls.ravel.app.json.decode(x), ), fields.Email.adapt(on_adapt=lambda field: sa.Text), fields.Bytes.adapt(on_adapt=lambda field: sa.LargeBinary), fields.BcryptString.adapt(on_adapt=lambda field: sa.Text), fields.Float.adapt(on_adapt=lambda field: sa.Float), fields.DateTime.adapt(on_adapt=lambda field: UtcDateTime), fields.Timestamp.adapt(on_adapt=lambda field: UtcDateTime), fields.Bool.adapt(on_adapt=lambda field: sa.Boolean), fields.TimeDelta.adapt(on_adapt=lambda field: sa.Interval), fields.Enum.adapt( on_adapt=lambda field: { fields.String: sa.Text, fields.Int: sa.Integer, fields.Float: sa.Float, }[type(field.nested)]), ] adapters.extend( field_class.adapt(on_adapt=lambda field: sa.Text) for field_class in { fields.String, fields.FormatString, fields.UuidString, fields.DateTimeString }) adapters.extend( field_class.adapt(on_adapt=lambda field: sa.BigInteger) for field_class in { fields.Int, fields.Uint32, fields.Uint64, fields.Uint, fields.Int32, }) if dialect == Dialect.postgresql: adapters.extend(cls.get_postgresql_default_adapters(table_name)) elif dialect == Dialect.mysql: adapters.extend(cls.get_mysql_default_adapters(table_name)) elif dialect == Dialect.sqlite: adapters.extend(cls.get_sqlite_default_adapters(table_name)) return adapters @classmethod def get_postgresql_default_adapters(cls, table_name) -> List[Field.Adapter]: pg_types = sa.dialects.postgresql def on_adapt_list(field): if isinstance(field.nested, fields.Enum): name = f'{table_name}__{field.name}' return ArrayOfEnum( pg_types.ENUM(*field.nested.values, name=name)) return pg_types.ARRAY({ fields.String: sa.Text, fields.Email: sa.Text, fields.Uuid: pg_types.UUID, fields.Int: sa.Integer, fields.Bool: sa.Boolean, fields.Float: sa.Float, fields.DateTime: UtcDateTime, fields.Timestamp: UtcDateTime, fields.Dict: pg_types.JSONB, fields.Field: pg_types.JSONB, fields.Nested: pg_types.JSONB, }.get(type(field.nested), sa.Text)) return [ Point.adapt( on_adapt=lambda field: GeoalchemyGeometry(field.geo_type), on_encode=lambda x: x.to_EWKT_string(), on_decode=lambda x: (PointGeometry(x['geometry']['coordinates']) if x else None)), Polygon.adapt( on_adapt=lambda field: GeoalchemyGeometry(field.geo_type), on_encode=lambda x: x.to_EWKT_string(), on_decode=lambda x: PolygonGeometry(x['geometry'][ 'coordinates'] if x else None)), fields.Field.adapt(on_adapt=lambda field: pg_types.JSONB), fields.Uuid.adapt(on_adapt=lambda field: pg_types.UUID), fields.Dict.adapt(on_adapt=lambda field: pg_types.JSONB), fields.Nested.adapt(on_adapt=lambda field: pg_types.JSONB, ), fields.Set.adapt(on_adapt=lambda field: pg_types.JSONB, on_encode=lambda x: list(x), on_decode=lambda x: set(x)), fields.UuidString.adapt( on_adapt=lambda field: pg_types.UUID, on_decode=lambda x: x.replace('-', '') if x else x, ), fields.List.adapt(on_adapt=on_adapt_list) ] @classmethod def get_mysql_default_adapters(cls, table_name) -> List[Field.Adapter]: return [ fields.Dict.adapt(on_adapt=lambda field: sa.JSON), fields.Nested.adapt(on_adapt=lambda field: sa.JSON), fields.List.adapt(on_adapt=lambda field: sa.JSON), fields.Set.adapt( on_adapt=lambda field: sa.JSON, on_encode=lambda x: cls.ravel.app.json.encode(x), on_decode=lambda x: set(cls.ravel.app.json.decode(x))), ] @classmethod def get_sqlite_default_adapters(cls, table_name) -> List[Field.Adapter]: adapters = [ field_class.adapt( on_adapt=lambda field: sa.Text, on_encode=lambda x: cls.ravel.app.json.encode(x), on_decode=lambda x: cls.ravel.app.json.decode(x), ) for field_class in {fields.Dict, fields.List, fields.Nested} ] adapters.append( fields.Set.adapt( on_adapt=lambda field: sa.Text, on_encode=lambda x: cls.ravel.app.json.encode(x), on_decode=lambda x: set(cls.ravel.app.json.decode(x)))) return adapters def __init__(self, adapters: List[Field.Adapter] = None): super().__init__() self._custom_adapters = adapters or [] self._table = None self._builder = None self._adapters = None self._id_column = None self._options = {} @property def adapters(self): return self._adapters @property def id_column_name(self): return self.resource_type.Schema.fields[ID].source def prepare(self, record: Dict, serialize=True) -> Dict: """ When inserting or updating data, the some raw values in the record dict must be transformed before their corresponding sqlalchemy column type will accept the data. """ cb_name = 'on_encode' if serialize else 'on_decode' prepared_record = {} for k, v in record.items(): if k in (REV): prepared_record[k] = v adapter = self._adapters.get(k) if adapter: callback = getattr(adapter, cb_name, None) if callback: try: prepared_record[k] = callback(v) continue except Exception: console.error( message=f'failed to adapt column value: {k}', data={ 'value': v, 'field': adapter.field_class }) raise prepared_record[k] = v return prepared_record def adapt_id(self, _id, serialize=True): cb_name = 'on_encode' if serialize else 'on_decode' adapter = self._adapters.get(self.id_column_name) if adapter: callback = getattr(adapter, cb_name) if callback: return callback(_id) return _id @classmethod def on_bootstrap(cls, url=None, dialect=None, echo=False, db=None, **kwargs): """ Initialize the SQLAlchemy connection pool (AKA Engine). """ with cls._bootstrap_lock: cls.ravel.kwargs = kwargs # construct the URL to the DB server # url can be a string or a dict if isinstance(url, dict): url_parts = url cls.ravel.app.shared.sqla_url = ( '{protocol}://{user}@{host}:{port}/{db}'.format( **url_parts)) elif isinstance(url, str): cls.ravel.app.shared.sqla_url = url else: url_parts = dict( protocol=cls.env.SQLALCHEMY_STORE_PROTOCOL, user=cls.env.SQLALCHEMY_STORE_USER or '', host=('@' + cls.env.SQLALCHEMY_STORE_HOST if cls.env.SQLALCHEMY_STORE_HOST else ''), port=(':' + cls.env.SQLALCHEMY_STORE_PORT if cls.env.SQLALCHEMY_STORE_PORT else ''), db=('/' + (db or cls.env.SQLALCHEMY_STORE_NAME or ''))) cls.ravel.app.shared.sqla_url = url or ( '{protocol}://{user}{host}{port}{db}'.format(**url_parts)) cls.dialect = dialect or cls.env.SQLALCHEMY_STORE_DIALECT from sqlalchemy.dialects import postgresql, sqlite, mysql cls.sa_dialect = None if cls.dialect == Dialect.postgresql: cls.sa_dialect = postgresql elif cls.dialect == Dialect.sqlite: cls.sa_dialect = sqlite elif cls.dialect == Dialect.mysql: cls.sa_dialect = mysql console.debug(message='creating sqlalchemy engine', data={ 'echo': cls.env.SQLALCHEMY_STORE_ECHO, 'dialect': cls.dialect, 'url': cls.ravel.app.shared.sqla_url, }) cls.ravel.local.sqla_tx = None cls.ravel.local.sqla_conn = None cls.ravel.local.sqla_metadata = sa.MetaData() cls.ravel.local.sqla_metadata.bind = sa.create_engine( name_or_url=cls.ravel.app.shared.sqla_url, echo=bool(echo or cls.env.SQLALCHEMY_STORE_ECHO), strategy='threadlocal') # set global thread-local sqlalchemy store method aliases cls.ravel.app.local.create_tables = cls.create_tables def on_bind(self, resource_type: Type['Resource'], table: Text = None, schema: 'Schema' = None, **kwargs): """ Initialize SQLAlchemy data strutures used for constructing SQL expressions used to manage the bound resource type. """ # map each of the resource's schema fields to a corresponding adapter, # which is used to prepare values upon insert and update. table = (table or SqlalchemyTableBuilder.derive_table_name(resource_type)) field_class_2_adapter = { adapter.field_class: adapter for adapter in self.get_default_adapters(self.dialect, table) + self._custom_adapters } self._adapters = { field_name: field_class_2_adapter[type(field)] for field_name, field in self.resource_type.Schema.fields.items() if (type(field) in field_class_2_adapter and field.meta.get('ravel_on_resolve') is None) } # build the Sqlalchemy Table object for the bound resource type. self._builder = SqlalchemyTableBuilder(self) try: self._table = self._builder.build_table(name=table, schema=schema) except Exception: console.error(f'failed to build sa.Table: {table}') raise self._id_column = getattr(self._table.c, self.id_column_name) # remember which column is the _id column self._id_column_names[self._table.name] = self.id_column_name # set SqlalchemyStore options here, using bootstrap-level # options as base/default options. self._options = dict(self.ravel.kwargs, **kwargs) def query( self, predicate: 'Predicate', fields: Set[Text] = None, limit: int = None, offset: int = None, order_by: Tuple = None, **kwargs, ): fields = fields or {k: None for k in self._adapters} fields.update({ self.id_column_name: None, self.resource_type.Schema.fields[REV].source: None, }) columns = [] table_alias = self.table.alias(''.join( s.strip('_')[0] for s in self.table.name.split('_'))) for k in fields: col = getattr(table_alias.c, k) if isinstance(col.type, GeoalchemyGeometry): columns.append(sa.func.ST_AsGeoJSON(col).label(k)) else: columns.append(col) predicate = Predicate.deserialize(predicate) filters = self._prepare_predicate(table_alias, predicate) # build the query object query = sa.select(columns).where(filters) if order_by: sa_order_by = [ sa.desc(getattr(table_alias.c, x.key)) if x.desc else sa.asc( getattr(table_alias.c, x.key)) for x in order_by ] query = query.order_by(*sa_order_by) if limit is not None: query = query.limit(max(0, limit)) if offset is not None: query = query.offset(max(0, limit)) console.debug( message=(f'SQL: SELECT FROM {self.table}' + (f' OFFSET {offset}' if offset is not None else '') + (f' LIMIT {limit}' if limit else '') + (f' ORDER BY {", ".join(x.to_sql() for x in order_by)}' if order_by else '')), data={ 'stack': traceback.format_stack(), 'statement': str(query.compile( compile_kwargs={'literal_binds': True})).split('\n') } if self.env.SQLALCHEMY_STORE_SHOW_QUERIES else None) # execute query, aggregating resulting records cursor = self.conn.execute(query) records = [] while True: page = [ self.prepare(dict(row.items()), serialize=False) for row in cursor.fetchmany(512) ] if page: records.extend(page) else: break return records def _prepare_predicate(self, table, pred, empty=set()): if isinstance(pred, ConditionalPredicate): if not pred.ignore_field_adapter: adapter = self._adapters.get(pred.field.source) if adapter and adapter.on_encode: pred.value = adapter.on_encode(pred.value) col = getattr(table.c, pred.field.source) if pred.op == OP_CODE.EQ: return col == pred.value elif pred.op == OP_CODE.NEQ: return col != pred.value if pred.op == OP_CODE.GEQ: return col >= pred.value elif pred.op == OP_CODE.GT: return col > pred.value elif pred.op == OP_CODE.LT: return col < pred.value elif pred.op == OP_CODE.LEQ: return col <= pred.value elif pred.op == OP_CODE.INCLUDING: return col.in_(pred.value) elif pred.op == OP_CODE.EXCLUDING: return ~col.in_(pred.value) elif pred.op == POSTGIS_OP_CODE.CONTAINS: if isinstance(pred.value, GeometryObject): EWKT_str = pred.value.to_EWKT_string() else: EWKT_str = pred.value return sa.func.ST_Contains( col, sa.func.ST_GeomFromEWKT(EWKT_str), ) elif pred.op == POSTGIS_OP_CODE.CONTAINED_BY: if isinstance(pred.value, GeometryObject): EWKT_str = pred.value.to_EWKT_string() else: EWKT_str = pred.value return sa.func.ST_Contains(sa.func.ST_GeomFromEWKT(EWKT_str), col) elif pred.op == POSTGIS_OP_CODE.WITHIN_RADIUS: center = pred.value['center'] radius = pred.value['radius'] return sa.func.ST_PointInsideCircle(col, center[0], center[1], radius) else: raise Exception('unrecognized conditional predicate') elif isinstance(pred, BooleanPredicate): if pred.op == OP_CODE.AND: lhs_result = self._prepare_predicate(table, pred.lhs) rhs_result = self._prepare_predicate(table, pred.rhs) return sa.and_(lhs_result, rhs_result) elif pred.op == OP_CODE.OR: lhs_result = self._prepare_predicate(table, pred.lhs) rhs_result = self._prepare_predicate(table, pred.rhs) return sa.or_(lhs_result, rhs_result) else: raise Exception('unrecognized boolean predicate') else: raise Exception('unrecognized predicate type') def exists(self, _id) -> bool: columns = [sa.func.count(self._id_column)] query = (sa.select(columns).where( self._id_column == self.adapt_id(_id))) result = self.conn.execute(query) return bool(result.scalar()) def exists_many(self, _ids: Set) -> Dict[object, bool]: columns = [self._id_column, sa.func.count(self._id_column)] query = (sa.select(columns).where( self._id_column.in_([self.adapt_id(_id) for _id in _ids]))) return {row[0]: row[1] for row in self.conn.execute(query)} def count(self) -> int: query = sa.select([sa.func.count(self._id_column)]) result = self.conn.execute(query) return result.scalar() def fetch(self, _id, fields=None) -> Dict: records = self.fetch_many(_ids=[_id], fields=fields) return records[_id] if records else None def fetch_many(self, _ids: List, fields=None, as_list=False) -> Dict: prepared_ids = [self.adapt_id(_id, serialize=True) for _id in _ids] if fields: if not isinstance(fields, set): fields = set(fields) else: fields = { f.source for f in self.resource_type.Schema.fields.values() if f.name in self._adapters } fields.update({ self.id_column_name, self.resource_type.Schema.fields[REV].source, }) columns = [] for k in fields: col = getattr(self.table.c, k) if isinstance(col.type, GeoalchemyGeometry): columns.append(sa.func.ST_AsGeoJSON(col).label(k)) else: columns.append(col) select_stmt = sa.select(columns) id_col = getattr(self.table.c, self.id_column_name) if prepared_ids: select_stmt = select_stmt.where(id_col.in_(prepared_ids)) cursor = self.conn.execute(select_stmt) records = {} if not as_list else [] while True: page = cursor.fetchmany(512) if page: for row in page: raw_record = dict(row.items()) record = self.prepare(raw_record, serialize=False) _id = self.adapt_id(row[self.id_column_name], serialize=False) if as_list: records.append(record) else: records[_id] = record else: break return records def fetch_all(self, fields: Set[Text] = None) -> Dict: return self.fetch_many([], fields=fields) def create(self, record: dict) -> dict: record[self.id_column_name] = self.create_id(record) prepared_record = self.prepare(record, serialize=True) insert_stmt = self.table.insert().values(**prepared_record) _id = prepared_record.get('_id', '') console.debug(f'SQL: INSERT {str(_id)[:7] + " " if _id else ""}' f'INTO {self.table}') try: if self.supports_returning: insert_stmt = insert_stmt.return_defaults() result = self.conn.execute(insert_stmt) return dict(record, **(result.returned_defaults or {})) else: result = self.conn.execute(insert_stmt) return self.fetch(_id=record[self.id_column_name]) except Exception: console.error(message=f'failed to insert record', data={ 'record': record, 'resource': get_class_name(self.resource_type), }) raise def create_many(self, records: List[Dict]) -> Dict: prepared_records = [] nullable_fields = self.resource_type.Schema.nullable_fields for record in records: record[self.id_column_name] = self.create_id(record) prepared_record = self.prepare(record, serialize=True) prepared_records.append(prepared_record) for nullable_field in nullable_fields.values(): if nullable_field.name not in prepared_record: prepared_record[nullable_field.name] = None try: self.conn.execute(self.table.insert(), prepared_records) except Exception: console.error(f'failed to insert records') raise n = len(prepared_records) id_list_str = (', '.join( str(x['_id'])[:7] for x in prepared_records if x.get('_id'))) console.debug(f'SQL: INSERT {id_list_str} INTO {self.table} ' + (f'(count: {n})' if n > 1 else '')) if self.supports_returning: # TODO: use implicit returning if possible pass return self.fetch_many((rec[self.id_column_name] for rec in records), as_list=True) def update(self, _id, data: Dict) -> Dict: prepared_id = self.adapt_id(_id) prepared_data = self.prepare(data, serialize=True) if prepared_data: update_stmt = (self.table.update().values(**prepared_data).where( self._id_column == prepared_id)) else: return prepared_data if self.supports_returning: update_stmt = update_stmt.return_defaults() console.debug(f'SQL: UPDATE {self.table}') result = self.conn.execute(update_stmt) return dict(data, **(result.returned_defaults or {})) else: self.conn.execute(update_stmt) if self._options.get('fetch_on_update', True): return self.fetch(_id) return data def update_many(self, _ids: List, data: List[Dict] = None) -> None: assert data prepared_ids = [] prepared_records = [] for _id, record in zip(_ids, data): prepared_id = self.adapt_id(_id) prepared_record = self.prepare(record, serialize=True) if prepared_record: prepared_ids.append(prepared_id) prepared_records.append(prepared_record) prepared_record[ID] = prepared_id if prepared_records: n = len(prepared_records) console.debug(f'SQL: UPDATE {self.table} ' + (f'({n}x)' if n > 1 else '')) values = {k: bindparam(k) for k in prepared_records[0].keys()} update_stmt = (self.table.update().where( self._id_column == bindparam(self.id_column_name)).values( **values)) self.conn.execute(update_stmt, prepared_records) if self._options.get('fetch_on_update', True): if self.supports_returning: # TODO: use implicit returning if possible return self.fetch_many(_ids) else: return self.fetch_many(_ids) return def delete(self, _id) -> None: prepared_id = self.adapt_id(_id) delete_stmt = self.table.delete().where(self._id_column == prepared_id) self.conn.execute(delete_stmt) def delete_many(self, _ids: list) -> None: prepared_ids = [self.adapt_id(_id) for _id in _ids] delete_stmt = self.table.delete().where( self._id_column.in_(prepared_ids)) self.conn.execute(delete_stmt) def delete_all(self): delete_stmt = self.table.delete() self.conn.execute(delete_stmt) @property def table(self): return self._table @property def conn(self): sqla_conn = getattr(self.ravel.local, 'sqla_conn', None) if sqla_conn is None: # lazily initialize a connection for this thread self.connect() return self.ravel.local.sqla_conn @property def supports_returning(self): if not self.is_bootstrapped(): return False metadata = self.get_metadata() return metadata.bind.dialect.implicit_returning @classmethod def create_tables(cls, overwrite=False): """ Create all tables for all SqlalchemyStores used in the host app. """ if not cls.is_bootstrapped(): console.error(f'{get_class_name(cls)} cannot create ' f'tables unless bootstrapped') return meta = cls.get_metadata() engine = cls.get_engine() if overwrite: console.info('dropping Resource SQL tables...') meta.drop_all(engine) # create all tables console.info('creating Resource SQL tables...') meta.create_all(engine) @classmethod def get_active_connection(cls): return getattr(cls.ravel.local, 'sqla_conn', None) @classmethod def connect(cls, refresh=True): """ Create a singleton thread-local SQLAlchemy connection, shared across all Resources backed by a SQLAlchemy store. When working with multiple threads or processes, make sure to """ sqla_conn = getattr(cls.ravel.local, 'sqla_conn', None) metadata = cls.ravel.local.sqla_metadata if sqla_conn is not None: console.warning( message='sqlalchemy store already has connection', ) if refresh: cls.close() cls.ravel.local.sqla_conn = metadata.bind.connect() else: cls.ravel.local.sqla_conn = metadata.bind.connect() return cls.ravel.local.sqla_conn @classmethod def close(cls): """ Return the thread-local database connection to the sqlalchemy connection pool (AKA the "engine"). """ sqla_conn = getattr(cls.ravel.local, 'sqla_conn', None) if sqla_conn is not None: console.debug('closing sqlalchemy connection') sqla_conn.close() cls.ravel.local.sqla_conn = None else: console.warning('sqlalchemy has no connection to close') @classmethod def begin(cls, auto_connect=True, **kwargs): """ Initialize a thread-local transaction. An exception is raised if there's already a pending transaction. """ conn = cls.get_active_connection() if conn is None: if auto_connect: conn = cls.connect() else: raise Exception('no active sqlalchemy connection') existing_tx = getattr(cls.ravel.local, 'sqla_tx', None) if existing_tx is not None: console.debug('there is already an open transaction') else: new_tx = cls.ravel.local.sqla_conn.begin() cls.ravel.local.sqla_tx = new_tx @classmethod def commit(cls, rollback=True, **kwargs): """ Call commit on the thread-local database transaction. "Begin" must be called to start a new transaction at this point, if a new transaction is desired. """ def perform_sqlalchemy_commit(): tx = getattr(cls.ravel.local, 'sqla_tx', None) if tx is not None: cls.ravel.local.sqla_tx.commit() cls.ravel.local.sqla_tx = None # try to commit the transaction. console.debug(f'committing sqlalchemy transaction') try: perform_sqlalchemy_commit() except Exception: if rollback: # if the commit fails, rollback the transaction console.critical(f'rolling back sqlalchemy transaction') cls.rollback() else: console.exception(f'sqlalchemy transaction failed commit') finally: # ensure we close the connection either way cls.close() @classmethod def rollback(cls, **kwargs): tx = getattr(cls.ravel.local, 'sqla_tx', None) if tx is not None: cls.ravel.local.sqla_tx = None try: tx.rollback() except: console.exception(f'sqlalchemy transaction failed to rollback') @classmethod def has_transaction(cls) -> bool: return cls.ravel.local.sqla_tx is not None @classmethod def get_metadata(cls): return cls.ravel.local.sqla_metadata @classmethod def get_engine(cls): return cls.get_metadata().bind @classmethod def dispose(cls): meta = cls.get_metadata() if not meta: cls.ravel.local.sqla_metadata = None return engine = meta.bind engine.dispose()