async def get_containers(request):
root = get_utility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
tm = request._tm = db.get_transaction_manager()
request._db_id = _id
async with tm.lock:
# reset _txn to make sure to create a new ob
request._txn = None
txn = await tm.begin(request)
items = {}
async for c_id, container in db.async_items():
items[c_id] = container
await tm.abort(txn=txn)
for _, container in items.items():
request._txn = txn = await tm.begin(request)
container._p_jar = request._txn
request.container = container
request._container_id = container.id
if hasattr(request, 'container_settings'):
del request.container_settings
yield txn, tm, container
try:
# do not rely on consumer of object to always close it.
# there is no harm in aborting twice
await tm.abort(txn=txn)
except Exception:
logger.warn('Error aborting transaction', exc_info=True)
async def iter_databases(root=None):
if root is None:
root = get_utility(IApplication, name='root')
loaded = []
for _, db in root:
if IDatabase.providedBy(db):
yield db
loaded.append(db.id)
last_checked = None
while last_checked is None or set(last_checked) != set(loaded):
# we need to continue checking until we're sure there aren't any
# new storage objects that have been added since we started
last_checked = loaded[:]
# from all dynamic storages
for _, config in list_or_dict_items(app_settings['storages']):
ctype = config.get('type', config['storage'])
factory = get_adapter(root, IDatabaseManager, name=ctype, args=[config])
for db_name in await factory.get_names():
if db_name in loaded:
continue
db = await factory.get_database(db_name)
loaded.append(db.id)
yield db
async def traverse(request, parent, path):
"""Do not use outside the main router function."""
if IApplication.providedBy(parent):
request.application = parent
if not path:
return parent, path
assert request is not None # could be used for permissions, etc
if not ITraversable.providedBy(parent):
# not a traversable context
return parent, path
try:
if path[0][0] == '_' or path[0] in ('.', '..'):
raise HTTPUnauthorized()
if path[0][0] == '@':
# shortcut
return parent, path
if IAsyncContainer.providedBy(parent):
context = await parent.async_get(path[0], suppress_events=True)
if context is None:
return parent, path
else:
context = parent[path[0]]
except (TypeError, KeyError, AttributeError):
return parent, path
if IDatabase.providedBy(context):
request._db_write_enabled = app_settings['check_writable_request'](request)
request._db_id = context.id
# Add a transaction Manager to request
tm = request._tm = context.get_transaction_manager()
# Start a transaction
txn = await tm.begin(request=request)
# Get the root of the tree
context = await tm.get_root(txn=txn)
if IContainer.providedBy(context):
request._container_id = context.id
request.container = context
annotations_container = IAnnotations(request.container)
request.container_settings = await annotations_container.async_get(REGISTRY_DATA_KEY)
layers = request.container_settings.get(ACTIVE_LAYERS_KEY, [])
for layer in layers:
try:
alsoProvides(request, import_class(layer))
except ModuleNotFoundError:
logger.error('Can not apply layer ' + layer, request=request)
return await traverse(request, context, path[1:])
async def get_containers(request, transaction_strategy='none'):
root = get_utility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
if transaction_strategy is not None:
db._db._storage._transaction_strategy = transaction_strategy
tm = request._tm = db.get_transaction_manager()
tm.request = request
request._db_id = _id
request._txn = txn = await tm.begin(request)
async for s_id, container in db.async_items():
tm.request.container = container
tm.request._container_id = container.id
yield txn, tm, container
async def traverse(request, parent, path):
"""Do not use outside the main router function."""
if IApplication.providedBy(parent):
request.application = parent
if not path:
return parent, path
assert request is not None # could be used for permissions, etc
if not ITraversable.providedBy(parent):
# not a traversable context
return parent, path
try:
if path[0].startswith('_') or path[0] in ('.', '..'):
raise HTTPUnauthorized()
if IAsyncContainer.providedBy(parent):
context = await parent.async_get(path[0], suppress_events=True)
if context is None:
return parent, path
else:
context = parent[path[0]]
except (TypeError, KeyError, AttributeError):
return parent, path
if IDatabase.providedBy(context):
request._db_write_enabled = app_settings['check_writable_request'](
request)
request._db_id = context.id
# Add a transaction Manager to request
tm = request._tm = context.get_transaction_manager()
# Start a transaction
txn = await tm.begin(request=request)
# Get the root of the tree
context = await tm.get_root(txn=txn)
if IContainer.providedBy(context):
request._container_id = context.id
request.container = context
annotations_container = IAnnotations(request.container)
request.container_settings = await annotations_container.async_get(
REGISTRY_DATA_KEY)
layers = request.container_settings.get(ACTIVE_LAYERS_KEY, [])
for layer in layers:
try:
alsoProvides(request, import_class(layer))
except ModuleNotFoundError:
logger.error('Can not apply layer ' + layer, request=request)
return await traverse(request, context, path[1:])
def get_full_content_path(ob) -> str:
"""
Generate full path of resource object from root
:param content: object to get path from
"""
parts = []
while ob is not None and not IApplication.providedBy(ob):
if IDatabase.providedBy(ob):
parts.append(ob.__db_id__)
break
else:
parts.append(ob.__name__)
ob = getattr(ob, '__parent__', None)
return '/' + '/'.join(reversed(parts))
async def traverse(request, parent, path):
"""Do not use outside the main router function."""
if IApplication.providedBy(parent):
request.application = parent
if not path:
return parent, path
assert request is not None # could be used for permissions, etc
if not ITraversable.providedBy(parent):
# not a traversable context
return parent, path
try:
if path[0].startswith('_'):
raise HTTPUnauthorized()
if IAsyncContainer.providedBy(parent):
context = await parent.async_get(path[0])
else:
context = parent[path[0]]
except (TypeError, KeyError, AttributeError):
return parent, path
if IDatabase.providedBy(context):
request._db_write_enabled = False
request._db_id = context.id
# Create a transaction Manager
request._tm = context.new_transaction_manager()
# Start a transaction
try:
await request._tm.begin(request=request)
except asyncpg.exceptions.UndefinedTableError:
pass
# Get the root of the tree
context = await request._tm.root()
if ISite.providedBy(context):
request._site_id = context.id
request.site = context
annotations_container = IAnnotations(request.site)
request.site_settings = await annotations_container.async_get(
REGISTRY_DATA_KEY)
layers = request.site_settings.get(ACTIVE_LAYERS_KEY, [])
for layer in layers:
alsoProvides(request, import_class(layer))
return await traverse(request, context, path[1:])
async def get_database(db_id, root=None):
if root is None:
root = get_utility(IApplication, name='root')
if db_id in root:
db = root[db_id]
if IDatabase.providedBy(db):
return db
for _, config in list_or_dict_items(app_settings['storages']):
ctype = config.get('type', config['storage'])
factory = get_adapter(root, IDatabaseManager, name=ctype, args=[config])
databases = await factory.get_names()
if db_id in databases:
return await factory.get_database(db_id)
return None
async def close_utilities(app):
root = get_utility(IApplication, name='root')
for utility in get_all_utilities_registered_for(IAsyncUtility):
try:
root.cancel_async_utility(utility)
except KeyError:
# attempt to delete by the provider registration
try:
iface = [i for i in utility.__providedBy__][-1]
root.cancel_async_utility(iface.__identifier__)
except (AttributeError, IndexError, KeyError):
pass
if hasattr(utility, 'finalize'):
await lazy_apply(utility.finalize, app=app)
for db in root:
if IDatabase.providedBy(db[1]):
await db[1].finalize()
async def run(self, arguments, settings, app):
aiotask_context.set('request', self.request)
if arguments.task_id is not None:
task = arguments.task_id
payload_config = False
else:
if hasattr(arguments, 'payload') and arguments.payload is not None:
task = arguments.payload
task = json.loads(task)
else:
task = os.environ.get('PAYLOAD', '{}')
logger.warning(f"Task to do: \n {task}")
task = json.loads(task)
payload_config = True
tags = []
if arguments.tags:
tags = json.loads(arguments.tags)
logger.warning("Tasks available: \n")
for k, v in settings.get('hive_tasks', {}).items():
logger.warning(f"{k}: {v}")
task_obj = None
root_obj = get_utility(IApplication, name='root')
if payload_config is False:
elements = task.split('/')[1:]
db_obj = await root_obj.async_get(elements[0])
if IDatabase.providedBy(db_obj):
tm = self.request._tm = db_obj.get_transaction_manager()
tm.request = self.request
self.request._db_id = elements[0]
self.request._txn = txn = await tm.begin(self.request)
container_obj = await db_obj.async_get(elements[1])
if IContainer.providedBy(container_obj):
guillotina_execution = await navigate_to(
container_obj, '/'.join(elements[2:]))
if IExecution.providedBy(guillotina_execution):
task_obj = TaskObject(
data=guillotina_execution.get_task_payload())
await tm.abort(txn=txn)
elif payload_config is True:
task_obj = TaskObject(data=task)
if task_obj is None:
raise NoTaskFunctionDefinedError()
logger.warning("Ready to run")
return await task_obj.run(self.request, tags, root_obj)
async def __call__(self):
result = {
"databases": [],
"static_file": [],
"static_directory": [],
"@type": "Application"
}
allowed = get_security_policy().check_permission(
"guillotina.GetDatabases", self.application)
for x in self.application._items.keys():
if IDatabase.providedBy(self.application._items[x]) and allowed:
result["databases"].append(x)
if IStaticFile.providedBy(self.application._items[x]):
result["static_file"].append(x)
if IStaticDirectory.providedBy(self.application._items[x]):
result["static_directory"].append(x)
return result
async def __call__(self):
result = {
'databases': [],
'static_file': [],
'static_directory': [],
'@type': 'Application'
}
allowed = IInteraction(self.request).check_permission(
'guillotina.GetDatabases', self.application)
for x in self.application._items.keys():
if IDatabase.providedBy(self.application._items[x]) and allowed:
result['databases'].append(x)
if IStaticFile.providedBy(self.application._items[x]):
result['static_file'].append(x)
if IStaticDirectory.providedBy(self.application._items[x]):
result['static_directory'].append(x)
return result
async def initialize(self):
from guillotina import app_settings
if not app_settings['store_json']:
return
root = get_utility(IApplication, name='root')
for _id, db in root:
if not IDatabase.providedBy(db):
continue
tm = db.get_transaction_manager()
if not IPostgresStorage.providedBy(tm.storage):
continue
async with tm.storage.pool.acquire() as conn:
for func in PG_FUNCTIONS:
await conn.execute(func)
for index in [BasicJsonIndex('container_id')
] + [v for v in get_pg_indexes().values()]:
sqls = index.get_index_sql(tm.storage)
for sql in sqls:
logger.debug(f'Creating index:\n {sql}')
await conn.execute(sql)
async def get_database(db_id, root=None):
if root is None:
root = get_utility(IApplication, name='root')
if db_id in root:
db = root[db_id]
if IDatabase.providedBy(db):
return db
for _, config in list_or_dict_items(app_settings['storages']):
ctype = config.get('type', config['storage'])
factory = get_adapter(root,
IDatabaseManager,
name=ctype,
args=[config])
databases = await factory.get_names()
if db_id in databases:
return await factory.get_database(db_id)
return None
async def _clear_dbs(root):
# make sure to completely clear db before carrying on...
for _, db in root:
if not IDatabase.providedBy(db):
continue
storage = db.storage
if IPostgresStorage.providedBy(storage) or ICockroachStorage.providedBy(storage):
async with storage.pool.acquire() as conn:
await conn.execute('''
DELETE from {}
WHERE zoid != '{}' AND zoid != '{}'
'''.format(storage._objects_table_name, ROOT_ID, TRASHED_ID))
await conn.execute('''
SELECT 'DROP INDEX ' || string_agg(indexrelid::regclass::text, ', ')
FROM pg_index i
LEFT JOIN pg_depend d ON d.objid = i.indexrelid
AND d.deptype = 'i'
WHERE i.indrelid = '{}'::regclass
AND d.objid IS NULL
'''.format(storage._objects_table_name))
async def get_containers(request, transaction_strategy='none'):
root = get_utility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
tm = request._tm = db.get_transaction_manager()
tm.request = request
request._db_id = _id
request._txn = txn = await tm.begin(request)
items = {(k, v) async for k, v in db.async_items()}
await tm.abort(txn=txn)
for s_id, container in items:
request._txn = txn = await tm.begin(request)
tm.request.container = container
tm.request._container_id = container.id
if hasattr(request, 'container_settings'):
del request.container_settings
yield txn, tm, container
try:
# do not rely on consumer of object to always close it.
# there is no harm in aborting twice
await tm.abort(txn=txn)
except Exception:
logger.warn('Error aborting transaction', exc_info=True)
async def close_dbs(app):
root = get_utility(IApplication, name="root")
for db in root:
if IDatabase.providedBy(db[1]):
await db[1].finalize()
async def run(self, arguments, settings, app):
root = get_utility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
print(f'Initializing database: {_id}')
await db._db._storage.create()
async def run(self, arguments, settings, app):
self.arguments = arguments
root = get_utility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
await self.generate_test_data(db)
async def close_dbs(app):
root = get_utility(IApplication, name='root')
for db in root:
if IDatabase.providedBy(db[1]):
await db[1].finalize()
async def run(self, arguments, settings, app):
root = getUtility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
await self.generate_test_data(db)
def change_transaction_strategy(strategy='none'):
root = get_utility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
db._storage._transaction_strategy = strategy
async def close_utilities(app):
for utility in getAllUtilitiesRegisteredFor(IAsyncUtility):
asyncio.ensure_future(utility.finalize(app=app), loop=app.loop)
for db in app.router._root:
if IDatabase.providedBy(db[1]):
await db[1]._db.finalize()
async def run(self, arguments, settings, app):
arguments.container = 'urls'
root = get_utility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
await self.check_links(db)
async def run(self, arguments, settings, app):
root = get_utility(IApplication, name='root')
for _id, db in root:
if IDatabase.providedBy(db):
print(f'Initializing database: {_id}')
await db._storage.create()
