def get_container_id(self):
ob = self.content
while getattr(ob, '__parent__',
None) and not IContainer.providedBy(ob):
ob = ob.__parent__
if IContainer.providedBy(ob):
return ob.__name__
async def do_traverse(request, parent, path):
"""Traverse for the code API."""
if not path:
return parent, path
assert request is not None # could be used for permissions, etc
if IContainer.providedBy(parent) and \
path[0] != request._db_id:
# Tried to access a container outside the request
raise HTTPUnauthorized()
if IApplication.providedBy(parent) and \
path[0] != request._container_id:
# Tried to access a container outside the request
raise HTTPUnauthorized()
try:
if path[0].startswith('_') or path[0] in ('.', '..'):
raise HTTPUnauthorized()
context = parent[path[0]]
except TypeError:
return parent, path
except KeyError:
return parent, path
context._v_parent = parent
return await traverse(request, context, path[1:])
async def add_object(obj, event):
uid = getattr(obj, 'uuid', None)
if uid is None:
return
type_name = getattr(obj, 'type_name', None)
if type_name is None or IContainer.providedBy(obj):
return
fut = get_future()
if fut is None:
return
search = queryUtility(ICatalogUtility)
if search:
if IObjectModifiedEvent.providedBy(event):
indexes = []
if event.payload and len(event.payload) > 0:
# get a list of potential indexes
for field_name in event.payload.keys():
if '.' in field_name:
for behavior_field_name in event.payload[field_name].keys():
indexes.append(behavior_field_name)
else:
indexes.append(field_name)
fut.update[uid] = await search.get_data(obj, indexes)
else:
fut.index[uid] = await search.get_data(obj)
async def index_object(obj, indexes=None, modified=False, security=False):
uid = getattr(obj, 'uuid', None)
if uid is None:
return
type_name = getattr(obj, 'type_name', None)
if type_name is None or IContainer.providedBy(obj):
return
search = query_utility(ICatalogUtility)
if search is None:
return
fut = get_future()
if fut is None:
return
if modified:
data = {}
if security:
adapter = query_adapter(obj, ISecurityInfo)
if adapter is not None:
data = await apply_coroutine(adapter)
else:
if indexes is not None and len(indexes) > 0:
data = await search.get_data(obj, indexes)
if len(data) > 0:
if uid in fut.update:
fut.update[uid].update(data)
else:
fut.update[uid] = data
else:
fut.index[uid] = await search.get_data(obj)
async def process_object(self, ob):
'''
- check if doc does not exist
- record it
- do complete index
- if doc exist
- if diff mapping exists
- update fields in diff on doc
- else, do nothing
- remove for list of existing doc ids
'''
clear_conn_statement_cache(await ob._p_jar.get_connection())
full = False
if ob.uuid not in self.existing:
self.missing.append(ob.uuid)
full = True
else:
self.existing.remove(ob.uuid)
await self.index_object(ob, full=full)
self.processed += 1
if IFolder.providedBy(ob):
await self.process_folder(ob)
if not IContainer.providedBy(ob):
del ob.__annotations__
del ob
async def process_object(self, ob):
'''
- check if doc does not exist
- record it
- do complete index
- if doc exist
- if diff mapping exists
- update fields in diff on doc
- else, do nothing
- remove for list of existing doc ids
'''
full = False
if ob.uuid not in self.existing:
self.missing.append(ob.uuid)
full = True
else:
self.existing.remove(ob.uuid)
await self.index_object(ob, full=full)
self.processed += 1
if IIndexActive.providedBy(ob):
self.sub_indexes.append(ob)
else:
if IFolder.providedBy(ob):
await self.process_folder(ob)
if not IContainer.providedBy(ob):
try:
del self.container.__gannotations__
except AttributeError:
del self.container.__annotations__
del ob
async def index_object(obj, indexes=None, modified=False, security=False):
uid = getattr(obj, 'uuid', None)
if uid is None:
return
type_name = getattr(obj, 'type_name', None)
if type_name is None or IContainer.providedBy(obj):
return
search = query_utility(ICatalogUtility)
if search is None:
return
fut = get_future()
if fut is None:
return
if modified:
data = {}
if security:
adapter = query_adapter(obj, ISecurityInfo)
if adapter is not None:
data = await apply_coroutine(adapter)
else:
if indexes is not None and len(indexes) > 0:
data = await search.get_data(obj, indexes)
if len(data) > 0:
if uid in fut.update:
fut.update[uid].update(data)
else:
fut.update[uid] = data
else:
fut.index[uid] = await search.get_data(obj)
def find_container(context=None) -> typing.Optional[IContainer]:
container = task_vars.container.get()
if container is None:
while context is not None:
if IContainer.providedBy(context):
container = context
break
context = getattr(context, '__parent__', None)
return container
async def traverse(
request: IRequest, parent: IBaseObject,
path: Tuple[str, ...]) -> Tuple[IBaseObject, Tuple[str, ...]]:
"""Do not use outside the main router function."""
if IApplication.providedBy(parent):
request.application = parent
if len(path) == 0:
return parent, path
assert request is not None # could be used for permissions, etc
if not ITraversable.providedBy(parent):
# not a traversable context
return parent, path
try:
if path[0][0] == "_" or path[0] in (".", ".."):
raise HTTPUnauthorized()
if path[0][0] == "@":
# shortcut
return parent, path
if IAsyncContainer.providedBy(parent):
context = await parent.async_get(path[0], suppress_events=True)
if context is None:
return parent, path
else:
context = parent[path[0]]
except (TypeError, KeyError, AttributeError):
return parent, path
if IDatabase.providedBy(context):
task_vars.db.set(context)
# Add a transaction Manager to request
tm = context.get_transaction_manager()
task_vars.tm.set(tm)
# Start a transaction
txn = await tm.begin(
read_only=not app_settings["check_writable_request"](request))
# Get the root of the tree
context = await tm.get_root(txn=txn)
if IContainer.providedBy(context):
task_vars.container.set(context)
# make sure to unset before we attempt to load in case
# there is an existing registry object set on task_vars
task_vars.registry.set(None)
registry = await get_registry(context)
layers = registry.get(ACTIVE_LAYERS_KEY, [])
for layer in layers:
try:
alsoProvides(request, import_class(layer))
except ModuleNotFoundError:
logger.error("Can not apply layer " + layer, request=request)
return await traverse(request, context, path[1:])
async def traverse(request, parent, path):
"""Do not use outside the main router function."""
if IApplication.providedBy(parent):
request.application = parent
if not path:
return parent, path
assert request is not None # could be used for permissions, etc
if not ITraversable.providedBy(parent):
# not a traversable context
return parent, path
try:
if path[0][0] == '_' or path[0] in ('.', '..'):
raise HTTPUnauthorized()
if path[0][0] == '@':
# shortcut
return parent, path
if IAsyncContainer.providedBy(parent):
context = await parent.async_get(path[0], suppress_events=True)
if context is None:
return parent, path
else:
context = parent[path[0]]
except (TypeError, KeyError, AttributeError):
return parent, path
if IDatabase.providedBy(context):
request._db_write_enabled = app_settings['check_writable_request'](
request)
request._db_id = context.id
# Add a transaction Manager to request
tm = request._tm = context.get_transaction_manager()
# Start a transaction
txn = await tm.begin(request=request)
# Get the root of the tree
context = await tm.get_root(txn=txn)
if IContainer.providedBy(context):
request._container_id = context.id
request.container = context
annotations_container = IAnnotations(request.container)
request.container_settings = await annotations_container.async_get(
REGISTRY_DATA_KEY)
layers = request.container_settings.get(ACTIVE_LAYERS_KEY, [])
for layer in layers:
try:
alsoProvides(request, import_class(layer))
except ModuleNotFoundError:
logger.error('Can not apply layer ' + layer, request=request)
return await traverse(request, context, path[1:])
async def __call__(self):
result = []
context = self.context
while context is not None and not IContainer.providedBy(context):
result.append({
"title": context.title,
"@id": IAbsoluteURL(context, self.request)()
})
context = getattr(context, "__parent__", None)
result.reverse()
return {"@id": self.request.url, "items": result}
def get_content_path(content: IResource) -> str:
"""
Generate full path of resource object
"""
parts = []
parent = getattr(content, '__parent__', None)
while content is not None and content.__name__ is not None and\
parent is not None and not IContainer.providedBy(content):
parts.append(content.__name__)
content = parent
parent = getattr(content, '__parent__', None)
return '/' + '/'.join(reversed(parts))
async def remove_object(obj, event):
uid = getattr(obj, "uuid", None)
if uid is None:
return
type_name = getattr(obj, "type_name", None)
if type_name is None or IContainer.providedBy(obj):
return
fut = get_indexer(obj)
if fut is None:
return
await fut.remove_object(obj)
def __init__(self, ob, request=None):
if request is None:
request = get_current_request()
self.request = get_current_request()
self.context = ob
if hasattr(self.request, 'container'):
self.container = self.request.container
else:
if IContainer.providedBy(ob):
self.container = ob
else:
raise Exception('Could not location container object')
async def traverse(request, parent, path):
"""Do not use outside the main router function."""
if IApplication.providedBy(parent):
request.application = parent
if not path:
return parent, path
assert request is not None # could be used for permissions, etc
if not ITraversable.providedBy(parent):
# not a traversable context
return parent, path
try:
if path[0][0] == '_' or path[0] in ('.', '..'):
raise HTTPUnauthorized()
if path[0][0] == '@':
# shortcut
return parent, path
if IAsyncContainer.providedBy(parent):
context = await parent.async_get(path[0], suppress_events=True)
if context is None:
return parent, path
else:
context = parent[path[0]]
except (TypeError, KeyError, AttributeError):
return parent, path
if IDatabase.providedBy(context):
request._db_write_enabled = app_settings['check_writable_request'](request)
request._db_id = context.id
# Add a transaction Manager to request
tm = request._tm = context.get_transaction_manager()
# Start a transaction
txn = await tm.begin(request=request)
# Get the root of the tree
context = await tm.get_root(txn=txn)
if IContainer.providedBy(context):
request._container_id = context.id
request.container = context
annotations_container = IAnnotations(request.container)
request.container_settings = await annotations_container.async_get(REGISTRY_DATA_KEY)
layers = request.container_settings.get(ACTIVE_LAYERS_KEY, [])
for layer in layers:
try:
alsoProvides(request, import_class(layer))
except ModuleNotFoundError:
logger.error('Can not apply layer ' + layer, request=request)
return await traverse(request, context, path[1:])
async def index_object(obj, indexes=None, modified=False, security=False):
uid = getattr(obj, "uuid", None)
if uid is None:
return
type_name = getattr(obj, "type_name", None)
if type_name is None or IContainer.providedBy(obj):
return
fut = get_indexer(obj)
if fut is None:
return
await fut.add_object(obj, indexes, modified, security)
def find_container(context=None) -> typing.Optional[IContainer]:
"""
Find container based on contextvar or by looking up the
container from the provided context parameter
"""
container = task_vars.container.get()
if container is None:
while context is not None:
if IContainer.providedBy(context):
container = context
break
context = getattr(context, "__parent__", None)
return container
def get_content_path(content: IResource) -> str:
"""
Generate full path of resource object
:param content: object to get path from
"""
parts = []
parent = getattr(content, '__parent__', None)
while content is not None and content.__name__ is not None and\
parent is not None and not IContainer.providedBy(content):
parts.append(content.__name__)
content = parent
parent = getattr(content, '__parent__', None)
return '/' + '/'.join(reversed(parts))
def get_content_path(content: IResource) -> str:
"""
Generate path of resource object from the container
:param content: object to get path from
"""
parts = []
parent = getattr(content, "__parent__", None)
while (content is not None and content.__name__ is not None
and parent is not None and not IContainer.providedBy(content)):
parts.append(content.__name__)
content = parent
parent = getattr(content, "__parent__", None)
return "/" + "/".join(reversed(parts))
async def add_object(obj, event):
uid = getattr(obj, 'uuid', None)
if uid is None:
return
type_name = getattr(obj, 'type_name', None)
if type_name is None or IContainer.providedBy(obj):
return
hook = get_hook()
if hook is None:
return
search = queryUtility(ICatalogUtility)
if search:
hook.index[uid] = await search.get_data(obj)
def remove_object(obj, event):
uid = getattr(obj, 'uuid', None)
if uid is None:
return
type_name = getattr(obj, 'type_name', None)
if type_name is None or IContainer.providedBy(obj):
return
content_path = get_content_path(obj)
hook = get_hook()
if hook is None:
return
hook.remove.append((uid, type_name, content_path))
if uid in hook.index:
del hook.index[uid]
def remove_object(obj, event):
uid = getattr(obj, 'uuid', None)
if uid is None:
return
type_name = getattr(obj, 'type_name', None)
if type_name is None or IContainer.providedBy(obj):
return
fut = get_future()
if fut is None:
return
fut.remove.append(obj)
if uid in fut.index:
del fut.index[uid]
if uid in fut.update:
del fut.update[uid]
def remove_object(obj, event):
uid = getattr(obj, 'uuid', None)
if uid is None:
return
type_name = getattr(obj, 'type_name', None)
if type_name is None or IContainer.providedBy(obj):
return
fut = get_future()
if fut is None:
return
fut.remove.append(obj)
if uid in fut.index:
del fut.index[uid]
if uid in fut.update:
del fut.update[uid]
async def run(self, arguments, settings, app):
aiotask_context.set('request', self.request)
if arguments.task_id is not None:
task = arguments.task_id
payload_config = False
else:
if hasattr(arguments, 'payload') and arguments.payload is not None:
task = arguments.payload
task = json.loads(task)
else:
task = os.environ.get('PAYLOAD', '{}')
logger.warning(f"Task to do: \n {task}")
task = json.loads(task)
payload_config = True
tags = []
if arguments.tags:
tags = json.loads(arguments.tags)
logger.warning("Tasks available: \n")
for k, v in settings.get('hive_tasks', {}).items():
logger.warning(f"{k}: {v}")
task_obj = None
root_obj = get_utility(IApplication, name='root')
if payload_config is False:
elements = task.split('/')[1:]
db_obj = await root_obj.async_get(elements[0])
if IDatabase.providedBy(db_obj):
tm = self.request._tm = db_obj.get_transaction_manager()
tm.request = self.request
self.request._db_id = elements[0]
self.request._txn = txn = await tm.begin(self.request)
container_obj = await db_obj.async_get(elements[1])
if IContainer.providedBy(container_obj):
guillotina_execution = await navigate_to(
container_obj, '/'.join(elements[2:]))
if IExecution.providedBy(guillotina_execution):
task_obj = TaskObject(
data=guillotina_execution.get_task_payload())
await tm.abort(txn=txn)
elif payload_config is True:
task_obj = TaskObject(data=task)
if task_obj is None:
raise NoTaskFunctionDefinedError()
logger.warning("Ready to run")
return await task_obj.run(self.request, tags, root_obj)
async def traverse(self, request: IRequest) -> IResource:
'''custom traversal here...'''
resource, tail = await super().traverse(request)
if len(tail) > 0 and tail[0] in (
'liches', 'lms') and IContainer.providedBy(resource):
segment = LichesRouteSegment(tail[0], resource)
if tail[0] == 'liches':
alsoProvides(request, ILichesLayer)
elif tail[0] == 'lms':
#segment = LMSRouteSegment(tail[0], resource)
alsoProvides(request, ILMSLayer)
if len(tail) > 1:
# finish traversal from here
return await traverse(request, segment, tail[1:])
else:
resource = segment
tail = tail[1:]
return resource, tail
async def get_inherited_aliases(ob) -> list:
storage = get_storage()
if storage is None:
return []
ob_path = get_content_path(ob)
data = []
context = ob.__parent__
while context is not None and not IContainer.providedBy(context):
context_path = get_content_path(context)
for alias in await get_aliases(context):
if not alias["moved"]:
continue
path = alias["path"]
current_sub_path = ob_path[len(context_path):] # noqa
path = os.path.join(path, current_sub_path.strip("/"))
alias["context_path"] = context_path
alias["path"] = path
data.append(alias)
context = context.__parent__
return data
async def run_migration(self):
alias_index_name = await self.index_manager.get_index_name()
existing_index = await self.index_manager.get_real_index_name()
await self.setup_next_index()
self.mapping_diff = await self.calculate_mapping_diff()
diff = json.dumps(self.mapping_diff,
sort_keys=True,
indent=4,
separators=(',', ': '))
self.response.write(f'Caculated mapping diff: {diff}')
if not self.full:
# if full, we're reindexing everything does not matter what
# anyways, so skip
self.response.write(f'Copying initial index {existing_index} '
f'into {self.work_index_name}')
try:
await self.copy_to_next_index()
self.response.write('Copying initial index data finished')
except elasticsearch.exceptions.NotFoundError:
self.response.write('No initial index to copy to')
if not self.mapping_only:
try:
self.existing = await self.get_all_uids()
except elasticsearch.exceptions.NotFoundError:
pass
self.index_start_time = time.time()
if self.children_only or IContainer.providedBy(self.context):
await self.process_folder(self.context) # this is recursive
else:
await self.process_object(self.context) # this is recursive
await self.check_existing()
await self.flush()
await self.join_futures()
async with get_migration_lock(await
self.index_manager.get_index_name()):
self.response.write('Activating new index')
async with managed_transaction(self.request,
write=True,
adopt_parent_txn=True):
await self.index_manager.finish_migration()
self.status = 'done'
self.response.write(f'''Update alias({alias_index_name}):
{existing_index} -> {self.work_index_name}
''')
try:
await self.conn.indices.update_aliases({
"actions": [{
"remove": {
"alias": alias_index_name,
"index": existing_index
}
}, {
"add": {
"alias": alias_index_name,
"index": self.work_index_name
}
}]
})
except elasticsearch.exceptions.NotFoundError:
await self.conn.indices.update_aliases({
"actions": [{
"add": {
"alias": alias_index_name,
"index": self.work_index_name
}
}]
})
try:
await self.conn.indices.close(existing_index)
await self.conn.indices.delete(existing_index)
self.response.write('Old index deleted')
except elasticsearch.exceptions.NotFoundError:
pass
if len(self.sub_indexes) > 0:
self.response.write(
f'Migrating sub indexes: {len(self.sub_indexes)}')
for ob in self.sub_indexes:
im = get_adapter(ob, IIndexManager)
migrator = Migrator(self.utility,
ob,
response=self.response,
force=self.force,
log_details=self.log_details,
memory_tracking=self.memory_tracking,
request=self.request,
bulk_size=self.bulk_size,
full=self.full,
reindex_security=self.reindex_security,
mapping_only=self.mapping_only,
index_manager=im,
children_only=True)
self.response.write(f'Migrating index for: {ob}')
await migrator.run_migration()
def get_container_id(self):
ob = self.content
while getattr(ob, '__parent__', None) and not IContainer.providedBy(ob):
ob = ob.__parent__
if IContainer.providedBy(ob):
return ob.__name__