async def get_database(db_id, root=None) -> IDatabase: """ Get configured database :param db_id: configured database id """ if root is None: root = get_utility(IApplication, name="root") if db_id in root: db = root[db_id] if IDatabase.providedBy(db): return db for _, config in list_or_dict_items(app_settings["storages"]): ctype = config.get("type", config["storage"]) factory = get_adapter(root, IDatabaseManager, name=ctype, args=[config]) databases = await factory.get_names() if db_id in databases: return await factory.get_database(db_id) raise DatabaseNotFound(db_id)
async def _clear_dbs(root): # make sure to completely clear db before carrying on... for _, db in root: if not IDatabase.providedBy(db): continue storage = db.storage if IPostgresStorage.providedBy(storage) or ICockroachStorage.providedBy(storage): async with storage.pool.acquire() as conn: await conn.execute( """ DELETE from {} WHERE zoid != '{}' AND zoid != '{}' """.format( storage._objects_table_name, ROOT_ID, TRASHED_ID ) ) await conn.execute( """ SELECT 'DROP INDEX ' || string_agg(indexrelid::regclass::text, ', ') FROM pg_index i LEFT JOIN pg_depend d ON d.objid = i.indexrelid AND d.deptype = 'i' WHERE i.indrelid = '{}'::regclass AND d.objid IS NULL """.format( storage._objects_table_name ) )
async def get_containers(request): root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): tm = request._tm = db.get_transaction_manager() request._db_id = _id async with tm.lock: # reset _txn to make sure to create a new ob request._txn = None txn = await tm.begin(request) items = {} async for c_id, container in db.async_items(): items[c_id] = container await tm.abort(txn=txn) for _, container in items.items(): request._txn = txn = await tm.begin(request) container._p_jar = request._txn request.container = container request._container_id = container.id if hasattr(request, 'container_settings'): del request.container_settings yield txn, tm, container try: # do not rely on consumer of object to always close it. # there is no harm in aborting twice await tm.abort(txn=txn) except Exception: logger.warn('Error aborting transaction', exc_info=True)
async def get_containers(): root = get_utility(IApplication, name="root") for _id, db in root: if not IDatabase.providedBy(db): continue tm = db.get_transaction_manager() async with tm: task_vars.db.set(db) async with tm.lock: txn = await tm.begin() try: items = {} async for c_id, container in db.async_items(): items[c_id] = container finally: try: await tm.abort(txn=txn) except Exception: logger.warn("Error aborting transaction", exc_info=True) for _, container in items.items(): with await tm.begin() as txn: container.__txn__ = txn task_vars.container.set(container) yield txn, tm, container try: # do not rely on consumer of object to always close it. # there is no harm in aborting twice await tm.abort(txn=txn) except Exception: logger.warn("Error aborting transaction", exc_info=True)
async def run(self, arguments, settings, app): login() self.arguments = arguments root = get_utility(IApplication, name="root") for _id, db in root: if IDatabase.providedBy(db): await self.generate_test_data(db)
async def iter_databases(root=None): if root is None: root = get_utility(IApplication, name="root") loaded = [] for _, db in root: if IDatabase.providedBy(db): yield db loaded.append(db.id) last_checked = None while last_checked is None or set(last_checked) != set(loaded): # we need to continue checking until we're sure there aren't any # new storage objects that have been added since we started last_checked = loaded[:] # from all dynamic storages for _, config in list_or_dict_items(app_settings["storages"]): ctype = config.get("type", config["storage"]) factory = get_adapter(root, IDatabaseManager, name=ctype, args=[config]) for db_name in await factory.get_names(): if db_name in loaded: continue db = await factory.get_database(db_name) loaded.append(db.id) yield db
async def iter_databases(root=None): if root is None: root = get_utility(IApplication, name='root') loaded = [] for _, db in root: if IDatabase.providedBy(db): yield db loaded.append(db.id) last_checked = None while last_checked is None or set(last_checked) != set(loaded): # we need to continue checking until we're sure there aren't any # new storage objects that have been added since we started last_checked = loaded[:] # from all dynamic storages for _, config in list_or_dict_items(app_settings['storages']): ctype = config.get('type', config['storage']) factory = get_adapter(root, IDatabaseManager, name=ctype, args=[config]) for db_name in await factory.get_names(): if db_name in loaded: continue db = await factory.get_database(db_name) loaded.append(db.id) yield db
async def get_containers(request): root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): tm = request._tm = db.get_transaction_manager() request._db_id = _id async with tm.lock: # reset _txn to make sure to create a new ob request._txn = None txn = await tm.begin(request) items = {} async for c_id, container in db.async_items(): items[c_id] = container await tm.abort(txn=txn) for _, container in items.items(): request._txn = txn = await tm.begin(request) container._p_jar = request._txn request.container = container request._container_id = container.id if hasattr(request, 'container_settings'): del request.container_settings yield txn, tm, container try: # do not rely on consumer of object to always close it. # there is no harm in aborting twice await tm.abort(txn=txn) except Exception: logger.warn('Error aborting transaction', exc_info=True)
async def close_utilities(app): for utility in getAllUtilitiesRegisteredFor(IAsyncUtility): if hasattr(utility, 'finalize'): asyncio.ensure_future(lazy_apply(utility.finalize, app=app), loop=app.loop) for db in app.router._root: if IDatabase.providedBy(db[1]): await db[1]._db.finalize()
async def traverse( request: IRequest, parent: IBaseObject, path: Tuple[str, ...]) -> Tuple[IBaseObject, Tuple[str, ...]]: """Do not use outside the main router function.""" if IApplication.providedBy(parent): request.application = parent if len(path) == 0: return parent, path assert request is not None # could be used for permissions, etc if not ITraversable.providedBy(parent): # not a traversable context return parent, path try: if path[0][0] == "_" or path[0] in (".", ".."): raise HTTPUnauthorized() if path[0][0] == "@": # shortcut return parent, path if IAsyncContainer.providedBy(parent): context = await parent.async_get(path[0], suppress_events=True) if context is None: return parent, path else: context = parent[path[0]] except (TypeError, KeyError, AttributeError): return parent, path if IDatabase.providedBy(context): task_vars.db.set(context) # Add a transaction Manager to request tm = context.get_transaction_manager() task_vars.tm.set(tm) # Start a transaction txn = await tm.begin( read_only=not app_settings["check_writable_request"](request)) # Get the root of the tree context = await tm.get_root(txn=txn) if IContainer.providedBy(context): task_vars.container.set(context) # make sure to unset before we attempt to load in case # there is an existing registry object set on task_vars task_vars.registry.set(None) registry = await get_registry(context) layers = registry.get(ACTIVE_LAYERS_KEY, []) for layer in layers: try: alsoProvides(request, import_class(layer)) except ModuleNotFoundError: logger.error("Can not apply layer " + layer, request=request) return await traverse(request, context, path[1:])
def get_full_content_path(ob) -> str: parts = [] while ob is not None and not IApplication.providedBy(ob): if IDatabase.providedBy(ob): parts.append(ob.__db_id__) break else: parts.append(ob.__name__) ob = getattr(ob, '__parent__', None) return '/' + '/'.join(reversed(parts))
async def get_dbs(self): root = getUtility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): tm = db.get_transaction_manager() tm.request = self.request await tm.begin(self.request) async for s_id, container in db.async_items(): tm.request.container = container yield tm, container
async def get_containers(request): root = getUtility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): db._db._storage._transaction_strategy = 'none' tm = db.get_transaction_manager() tm.request = request txn = await tm.begin(request) async for s_id, container in db.async_items(): tm.request.container = container yield txn, tm, container
async def traverse(request, parent, path): """Do not use outside the main router function.""" if IApplication.providedBy(parent): request.application = parent if not path: return parent, path assert request is not None # could be used for permissions, etc if not ITraversable.providedBy(parent): # not a traversable context return parent, path try: if path[0][0] == '_' or path[0] in ('.', '..'): raise HTTPUnauthorized() if path[0][0] == '@': # shortcut return parent, path if IAsyncContainer.providedBy(parent): context = await parent.async_get(path[0], suppress_events=True) if context is None: return parent, path else: context = parent[path[0]] except (TypeError, KeyError, AttributeError): return parent, path if IDatabase.providedBy(context): request._db_write_enabled = app_settings['check_writable_request']( request) request._db_id = context.id # Add a transaction Manager to request tm = request._tm = context.get_transaction_manager() # Start a transaction txn = await tm.begin(request=request) # Get the root of the tree context = await tm.get_root(txn=txn) if IContainer.providedBy(context): request._container_id = context.id request.container = context annotations_container = IAnnotations(request.container) request.container_settings = await annotations_container.async_get( REGISTRY_DATA_KEY) layers = request.container_settings.get(ACTIVE_LAYERS_KEY, []) for layer in layers: try: alsoProvides(request, import_class(layer)) except ModuleNotFoundError: logger.error('Can not apply layer ' + layer, request=request) return await traverse(request, context, path[1:])
async def close_utilities(app): root = get_utility(IApplication, name='root') for utility in get_all_utilities_registered_for(IAsyncUtility): try: root.cancel_async_utility(utility) except KeyError: pass if hasattr(utility, 'finalize'): await lazy_apply(utility.finalize, app=app) for db in root: if IDatabase.providedBy(db[1]): await db[1]._db.finalize()
async def __call__(self): result = [] context = self.context while context is not None and not IDatabase.providedBy(context): result.append({ 'title': context.title, '@id': IAbsoluteURL(context, self.request)() }) context = getattr(context, '__parent__', None) result.reverse() return {"@id": self.request.url.human_repr(), "items": result}
async def traverse(request, parent, path): """Do not use outside the main router function.""" if IApplication.providedBy(parent): request.application = parent if not path: return parent, path assert request is not None # could be used for permissions, etc if not ITraversable.providedBy(parent): # not a traversable context return parent, path try: if path[0][0] == '_' or path[0] in ('.', '..'): raise HTTPUnauthorized() if path[0][0] == '@': # shortcut return parent, path if IAsyncContainer.providedBy(parent): context = await parent.async_get(path[0], suppress_events=True) if context is None: return parent, path else: context = parent[path[0]] except (TypeError, KeyError, AttributeError): return parent, path if IDatabase.providedBy(context): request._db_write_enabled = app_settings['check_writable_request'](request) request._db_id = context.id # Add a transaction Manager to request tm = request._tm = context.get_transaction_manager() # Start a transaction txn = await tm.begin(request=request) # Get the root of the tree context = await tm.get_root(txn=txn) if IContainer.providedBy(context): request._container_id = context.id request.container = context annotations_container = IAnnotations(request.container) request.container_settings = await annotations_container.async_get(REGISTRY_DATA_KEY) layers = request.container_settings.get(ACTIVE_LAYERS_KEY, []) for layer in layers: try: alsoProvides(request, import_class(layer)) except ModuleNotFoundError: logger.error('Can not apply layer ' + layer, request=request) return await traverse(request, context, path[1:])
async def __call__(self): result = {'databases': [], 'static_file': [], 'static_directory': []} allowed = IInteraction(self.request).check_permission( 'guillotina.GetDatabases', self.application) for x in self.application._dbs.keys(): if IDatabase.providedBy(self.application._dbs[x]) and allowed: result['databases'].append(x) if IStaticFile.providedBy(self.application._dbs[x]): result['static_file'].append(x) if IStaticDirectory.providedBy(self.application._dbs[x]): result['static_directory'].append(x) return result
async def get_containers(request, transaction_strategy='none'): root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): if transaction_strategy is not None: db._db._storage._transaction_strategy = transaction_strategy tm = request._tm = db.get_transaction_manager() tm.request = request request._db_id = _id request._txn = txn = await tm.begin(request) async for s_id, container in db.async_items(): tm.request.container = container tm.request._container_id = container.id yield txn, tm, container
def get_full_content_path(ob) -> str: """ Generate full path of resource object from root :param content: object to get path from """ parts = [] while ob is not None and not IApplication.providedBy(ob): if IDatabase.providedBy(ob): parts.append(ob.__db_id__) break else: parts.append(ob.__name__) ob = getattr(ob, '__parent__', None) return '/' + '/'.join(reversed(parts))
async def traverse(request, parent, path): """Do not use outside the main router function.""" if IApplication.providedBy(parent): request.application = parent if not path: return parent, path assert request is not None # could be used for permissions, etc if not ITraversable.providedBy(parent): # not a traversable context return parent, path try: if path[0].startswith('_'): raise HTTPUnauthorized() if IAsyncContainer.providedBy(parent): context = await parent.async_get(path[0]) else: context = parent[path[0]] except (TypeError, KeyError, AttributeError): return parent, path if IDatabase.providedBy(context): request._db_write_enabled = False request._db_id = context.id # Create a transaction Manager request._tm = context.new_transaction_manager() # Start a transaction try: await request._tm.begin(request=request) except asyncpg.exceptions.UndefinedTableError: pass # Get the root of the tree context = await request._tm.root() if ISite.providedBy(context): request._site_id = context.id request.site = context annotations_container = IAnnotations(request.site) request.site_settings = await annotations_container.async_get( REGISTRY_DATA_KEY) layers = request.site_settings.get(ACTIVE_LAYERS_KEY, []) for layer in layers: alsoProvides(request, import_class(layer)) return await traverse(request, context, path[1:])
async def run(self, arguments, settings, app): aiotask_context.set('request', self.request) if arguments.task_id is not None: task = arguments.task_id payload_config = False else: if hasattr(arguments, 'payload') and arguments.payload is not None: task = arguments.payload task = json.loads(task) else: task = os.environ.get('PAYLOAD', '{}') logger.warning(f"Task to do: \n {task}") task = json.loads(task) payload_config = True tags = [] if arguments.tags: tags = json.loads(arguments.tags) logger.warning("Tasks available: \n") for k, v in settings.get('hive_tasks', {}).items(): logger.warning(f"{k}: {v}") task_obj = None root_obj = get_utility(IApplication, name='root') if payload_config is False: elements = task.split('/')[1:] db_obj = await root_obj.async_get(elements[0]) if IDatabase.providedBy(db_obj): tm = self.request._tm = db_obj.get_transaction_manager() tm.request = self.request self.request._db_id = elements[0] self.request._txn = txn = await tm.begin(self.request) container_obj = await db_obj.async_get(elements[1]) if IContainer.providedBy(container_obj): guillotina_execution = await navigate_to( container_obj, '/'.join(elements[2:])) if IExecution.providedBy(guillotina_execution): task_obj = TaskObject( data=guillotina_execution.get_task_payload()) await tm.abort(txn=txn) elif payload_config is True: task_obj = TaskObject(data=task) if task_obj is None: raise NoTaskFunctionDefinedError() logger.warning("Ready to run") return await task_obj.run(self.request, tags, root_obj)
async def close_utilities(app): root = get_utility(IApplication, name='root') for utility in get_all_utilities_registered_for(IAsyncUtility): try: root.cancel_async_utility(utility) except KeyError: # attempt to delete by the provider registration try: iface = [i for i in utility.__providedBy__][-1] root.cancel_async_utility(iface.__identifier__) except (AttributeError, IndexError, KeyError): pass if hasattr(utility, 'finalize'): await lazy_apply(utility.finalize, app=app) for db in root: if IDatabase.providedBy(db[1]): await db[1].finalize()
async def get_database(db_id, root=None): if root is None: root = get_utility(IApplication, name='root') if db_id in root: db = root[db_id] if IDatabase.providedBy(db): return db for _, config in list_or_dict_items(app_settings['storages']): ctype = config.get('type', config['storage']) factory = get_adapter(root, IDatabaseManager, name=ctype, args=[config]) databases = await factory.get_names() if db_id in databases: return await factory.get_database(db_id) return None
async def __call__(self): result = { 'databases': [], 'static_file': [], 'static_directory': [], '@type': 'Application' } allowed = get_security_policy().check_permission( 'guillotina.GetDatabases', self.application) for x in self.application._items.keys(): if IDatabase.providedBy(self.application._items[x]) and allowed: result['databases'].append(x) if IStaticFile.providedBy(self.application._items[x]): result['static_file'].append(x) if IStaticDirectory.providedBy(self.application._items[x]): result['static_directory'].append(x) return result
async def __call__(self): result = { "databases": [], "static_file": [], "static_directory": [], "@type": "Application" } allowed = get_security_policy().check_permission( "guillotina.GetDatabases", self.application) for x in self.application._items.keys(): if IDatabase.providedBy(self.application._items[x]) and allowed: result["databases"].append(x) if IStaticFile.providedBy(self.application._items[x]): result["static_file"].append(x) if IStaticDirectory.providedBy(self.application._items[x]): result["static_directory"].append(x) return result
async def __call__(self): result = { 'databases': [], 'static_file': [], 'static_directory': [], '@type': 'Application' } allowed = IInteraction(self.request).check_permission( 'guillotina.GetDatabases', self.application) for x in self.application._items.keys(): if IDatabase.providedBy(self.application._items[x]) and allowed: result['databases'].append(x) if IStaticFile.providedBy(self.application._items[x]): result['static_file'].append(x) if IStaticDirectory.providedBy(self.application._items[x]): result['static_directory'].append(x) return result
async def initialize(self): from guillotina import app_settings if not app_settings['store_json']: return root = get_utility(IApplication, name='root') for _id, db in root: if not IDatabase.providedBy(db): continue tm = db.get_transaction_manager() if not IPostgresStorage.providedBy(tm.storage): continue async with tm.storage.pool.acquire() as conn: for func in PG_FUNCTIONS: await conn.execute(func) for index in [BasicJsonIndex('container_id') ] + [v for v in get_pg_indexes().values()]: sqls = index.get_index_sql(tm.storage) for sql in sqls: logger.debug(f'Creating index:\n {sql}') await conn.execute(sql)
async def get_database(db_id, root=None): if root is None: root = get_utility(IApplication, name='root') if db_id in root: db = root[db_id] if IDatabase.providedBy(db): return db for _, config in list_or_dict_items(app_settings['storages']): ctype = config.get('type', config['storage']) factory = get_adapter(root, IDatabaseManager, name=ctype, args=[config]) databases = await factory.get_names() if db_id in databases: return await factory.get_database(db_id) return None
async def get_containers(request, transaction_strategy='none'): root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): tm = request._tm = db.get_transaction_manager() tm.request = request request._db_id = _id request._txn = txn = await tm.begin(request) items = {(k, v) async for k, v in db.async_items()} await tm.abort(txn=txn) for s_id, container in items: request._txn = txn = await tm.begin(request) tm.request.container = container tm.request._container_id = container.id if hasattr(request, 'container_settings'): del request.container_settings yield txn, tm, container try: # do not rely on consumer of object to always close it. # there is no harm in aborting twice await tm.abort(txn=txn) except Exception: logger.warn('Error aborting transaction', exc_info=True)
async def close_dbs(app): root = get_utility(IApplication, name="root") for db in root: if IDatabase.providedBy(db[1]): await db[1].finalize()
async def run(self, arguments, settings, app): self.arguments = arguments root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): await self.generate_test_data(db)
async def run(self, arguments, settings, app): root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): print(f'Initializing database: {_id}') await db._db._storage.create()
async def run(self, arguments, settings, app): arguments.container = 'urls' root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): await self.check_links(db)
async def close_dbs(app): root = get_utility(IApplication, name='root') for db in root: if IDatabase.providedBy(db[1]): await db[1].finalize()
async def run(self, arguments, settings, app): root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): print(f'Initializing database: {_id}') await db._storage.create()
def change_transaction_strategy(strategy='none'): root = get_utility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): db._storage._transaction_strategy = strategy
async def run(self, arguments, settings, app): root = getUtility(IApplication, name='root') for _id, db in root: if IDatabase.providedBy(db): await self.generate_test_data(db)