def get_container_id(self): ob = self.content while getattr(ob, '__parent__', None) and not IContainer.providedBy(ob): ob = ob.__parent__ if IContainer.providedBy(ob): return ob.__name__
async def do_traverse(request, parent, path): """Traverse for the code API.""" if not path: return parent, path assert request is not None # could be used for permissions, etc if IContainer.providedBy(parent) and \ path[0] != request._db_id: # Tried to access a container outside the request raise HTTPUnauthorized() if IApplication.providedBy(parent) and \ path[0] != request._container_id: # Tried to access a container outside the request raise HTTPUnauthorized() try: if path[0].startswith('_') or path[0] in ('.', '..'): raise HTTPUnauthorized() context = parent[path[0]] except TypeError: return parent, path except KeyError: return parent, path context._v_parent = parent return await traverse(request, context, path[1:])
async def add_object(obj, event): uid = getattr(obj, 'uuid', None) if uid is None: return type_name = getattr(obj, 'type_name', None) if type_name is None or IContainer.providedBy(obj): return fut = get_future() if fut is None: return search = queryUtility(ICatalogUtility) if search: if IObjectModifiedEvent.providedBy(event): indexes = [] if event.payload and len(event.payload) > 0: # get a list of potential indexes for field_name in event.payload.keys(): if '.' in field_name: for behavior_field_name in event.payload[field_name].keys(): indexes.append(behavior_field_name) else: indexes.append(field_name) fut.update[uid] = await search.get_data(obj, indexes) else: fut.index[uid] = await search.get_data(obj)
async def index_object(obj, indexes=None, modified=False, security=False): uid = getattr(obj, 'uuid', None) if uid is None: return type_name = getattr(obj, 'type_name', None) if type_name is None or IContainer.providedBy(obj): return search = query_utility(ICatalogUtility) if search is None: return fut = get_future() if fut is None: return if modified: data = {} if security: adapter = query_adapter(obj, ISecurityInfo) if adapter is not None: data = await apply_coroutine(adapter) else: if indexes is not None and len(indexes) > 0: data = await search.get_data(obj, indexes) if len(data) > 0: if uid in fut.update: fut.update[uid].update(data) else: fut.update[uid] = data else: fut.index[uid] = await search.get_data(obj)
async def process_object(self, ob): ''' - check if doc does not exist - record it - do complete index - if doc exist - if diff mapping exists - update fields in diff on doc - else, do nothing - remove for list of existing doc ids ''' clear_conn_statement_cache(await ob._p_jar.get_connection()) full = False if ob.uuid not in self.existing: self.missing.append(ob.uuid) full = True else: self.existing.remove(ob.uuid) await self.index_object(ob, full=full) self.processed += 1 if IFolder.providedBy(ob): await self.process_folder(ob) if not IContainer.providedBy(ob): del ob.__annotations__ del ob
async def process_object(self, ob): ''' - check if doc does not exist - record it - do complete index - if doc exist - if diff mapping exists - update fields in diff on doc - else, do nothing - remove for list of existing doc ids ''' full = False if ob.uuid not in self.existing: self.missing.append(ob.uuid) full = True else: self.existing.remove(ob.uuid) await self.index_object(ob, full=full) self.processed += 1 if IIndexActive.providedBy(ob): self.sub_indexes.append(ob) else: if IFolder.providedBy(ob): await self.process_folder(ob) if not IContainer.providedBy(ob): try: del self.container.__gannotations__ except AttributeError: del self.container.__annotations__ del ob
async def index_object(obj, indexes=None, modified=False, security=False): uid = getattr(obj, 'uuid', None) if uid is None: return type_name = getattr(obj, 'type_name', None) if type_name is None or IContainer.providedBy(obj): return search = query_utility(ICatalogUtility) if search is None: return fut = get_future() if fut is None: return if modified: data = {} if security: adapter = query_adapter(obj, ISecurityInfo) if adapter is not None: data = await apply_coroutine(adapter) else: if indexes is not None and len(indexes) > 0: data = await search.get_data(obj, indexes) if len(data) > 0: if uid in fut.update: fut.update[uid].update(data) else: fut.update[uid] = data else: fut.index[uid] = await search.get_data(obj)
def find_container(context=None) -> typing.Optional[IContainer]: container = task_vars.container.get() if container is None: while context is not None: if IContainer.providedBy(context): container = context break context = getattr(context, '__parent__', None) return container
async def traverse( request: IRequest, parent: IBaseObject, path: Tuple[str, ...]) -> Tuple[IBaseObject, Tuple[str, ...]]: """Do not use outside the main router function.""" if IApplication.providedBy(parent): request.application = parent if len(path) == 0: return parent, path assert request is not None # could be used for permissions, etc if not ITraversable.providedBy(parent): # not a traversable context return parent, path try: if path[0][0] == "_" or path[0] in (".", ".."): raise HTTPUnauthorized() if path[0][0] == "@": # shortcut return parent, path if IAsyncContainer.providedBy(parent): context = await parent.async_get(path[0], suppress_events=True) if context is None: return parent, path else: context = parent[path[0]] except (TypeError, KeyError, AttributeError): return parent, path if IDatabase.providedBy(context): task_vars.db.set(context) # Add a transaction Manager to request tm = context.get_transaction_manager() task_vars.tm.set(tm) # Start a transaction txn = await tm.begin( read_only=not app_settings["check_writable_request"](request)) # Get the root of the tree context = await tm.get_root(txn=txn) if IContainer.providedBy(context): task_vars.container.set(context) # make sure to unset before we attempt to load in case # there is an existing registry object set on task_vars task_vars.registry.set(None) registry = await get_registry(context) layers = registry.get(ACTIVE_LAYERS_KEY, []) for layer in layers: try: alsoProvides(request, import_class(layer)) except ModuleNotFoundError: logger.error("Can not apply layer " + layer, request=request) return await traverse(request, context, path[1:])
async def traverse(request, parent, path): """Do not use outside the main router function.""" if IApplication.providedBy(parent): request.application = parent if not path: return parent, path assert request is not None # could be used for permissions, etc if not ITraversable.providedBy(parent): # not a traversable context return parent, path try: if path[0][0] == '_' or path[0] in ('.', '..'): raise HTTPUnauthorized() if path[0][0] == '@': # shortcut return parent, path if IAsyncContainer.providedBy(parent): context = await parent.async_get(path[0], suppress_events=True) if context is None: return parent, path else: context = parent[path[0]] except (TypeError, KeyError, AttributeError): return parent, path if IDatabase.providedBy(context): request._db_write_enabled = app_settings['check_writable_request']( request) request._db_id = context.id # Add a transaction Manager to request tm = request._tm = context.get_transaction_manager() # Start a transaction txn = await tm.begin(request=request) # Get the root of the tree context = await tm.get_root(txn=txn) if IContainer.providedBy(context): request._container_id = context.id request.container = context annotations_container = IAnnotations(request.container) request.container_settings = await annotations_container.async_get( REGISTRY_DATA_KEY) layers = request.container_settings.get(ACTIVE_LAYERS_KEY, []) for layer in layers: try: alsoProvides(request, import_class(layer)) except ModuleNotFoundError: logger.error('Can not apply layer ' + layer, request=request) return await traverse(request, context, path[1:])
async def __call__(self): result = [] context = self.context while context is not None and not IContainer.providedBy(context): result.append({ "title": context.title, "@id": IAbsoluteURL(context, self.request)() }) context = getattr(context, "__parent__", None) result.reverse() return {"@id": self.request.url, "items": result}
def get_content_path(content: IResource) -> str: """ Generate full path of resource object """ parts = [] parent = getattr(content, '__parent__', None) while content is not None and content.__name__ is not None and\ parent is not None and not IContainer.providedBy(content): parts.append(content.__name__) content = parent parent = getattr(content, '__parent__', None) return '/' + '/'.join(reversed(parts))
async def remove_object(obj, event): uid = getattr(obj, "uuid", None) if uid is None: return type_name = getattr(obj, "type_name", None) if type_name is None or IContainer.providedBy(obj): return fut = get_indexer(obj) if fut is None: return await fut.remove_object(obj)
def __init__(self, ob, request=None): if request is None: request = get_current_request() self.request = get_current_request() self.context = ob if hasattr(self.request, 'container'): self.container = self.request.container else: if IContainer.providedBy(ob): self.container = ob else: raise Exception('Could not location container object')
async def traverse(request, parent, path): """Do not use outside the main router function.""" if IApplication.providedBy(parent): request.application = parent if not path: return parent, path assert request is not None # could be used for permissions, etc if not ITraversable.providedBy(parent): # not a traversable context return parent, path try: if path[0][0] == '_' or path[0] in ('.', '..'): raise HTTPUnauthorized() if path[0][0] == '@': # shortcut return parent, path if IAsyncContainer.providedBy(parent): context = await parent.async_get(path[0], suppress_events=True) if context is None: return parent, path else: context = parent[path[0]] except (TypeError, KeyError, AttributeError): return parent, path if IDatabase.providedBy(context): request._db_write_enabled = app_settings['check_writable_request'](request) request._db_id = context.id # Add a transaction Manager to request tm = request._tm = context.get_transaction_manager() # Start a transaction txn = await tm.begin(request=request) # Get the root of the tree context = await tm.get_root(txn=txn) if IContainer.providedBy(context): request._container_id = context.id request.container = context annotations_container = IAnnotations(request.container) request.container_settings = await annotations_container.async_get(REGISTRY_DATA_KEY) layers = request.container_settings.get(ACTIVE_LAYERS_KEY, []) for layer in layers: try: alsoProvides(request, import_class(layer)) except ModuleNotFoundError: logger.error('Can not apply layer ' + layer, request=request) return await traverse(request, context, path[1:])
async def index_object(obj, indexes=None, modified=False, security=False): uid = getattr(obj, "uuid", None) if uid is None: return type_name = getattr(obj, "type_name", None) if type_name is None or IContainer.providedBy(obj): return fut = get_indexer(obj) if fut is None: return await fut.add_object(obj, indexes, modified, security)
def find_container(context=None) -> typing.Optional[IContainer]: """ Find container based on contextvar or by looking up the container from the provided context parameter """ container = task_vars.container.get() if container is None: while context is not None: if IContainer.providedBy(context): container = context break context = getattr(context, "__parent__", None) return container
def get_content_path(content: IResource) -> str: """ Generate full path of resource object :param content: object to get path from """ parts = [] parent = getattr(content, '__parent__', None) while content is not None and content.__name__ is not None and\ parent is not None and not IContainer.providedBy(content): parts.append(content.__name__) content = parent parent = getattr(content, '__parent__', None) return '/' + '/'.join(reversed(parts))
def get_content_path(content: IResource) -> str: """ Generate path of resource object from the container :param content: object to get path from """ parts = [] parent = getattr(content, "__parent__", None) while (content is not None and content.__name__ is not None and parent is not None and not IContainer.providedBy(content)): parts.append(content.__name__) content = parent parent = getattr(content, "__parent__", None) return "/" + "/".join(reversed(parts))
async def add_object(obj, event): uid = getattr(obj, 'uuid', None) if uid is None: return type_name = getattr(obj, 'type_name', None) if type_name is None or IContainer.providedBy(obj): return hook = get_hook() if hook is None: return search = queryUtility(ICatalogUtility) if search: hook.index[uid] = await search.get_data(obj)
def remove_object(obj, event): uid = getattr(obj, 'uuid', None) if uid is None: return type_name = getattr(obj, 'type_name', None) if type_name is None or IContainer.providedBy(obj): return content_path = get_content_path(obj) hook = get_hook() if hook is None: return hook.remove.append((uid, type_name, content_path)) if uid in hook.index: del hook.index[uid]
def remove_object(obj, event): uid = getattr(obj, 'uuid', None) if uid is None: return type_name = getattr(obj, 'type_name', None) if type_name is None or IContainer.providedBy(obj): return fut = get_future() if fut is None: return fut.remove.append(obj) if uid in fut.index: del fut.index[uid] if uid in fut.update: del fut.update[uid]
def remove_object(obj, event): uid = getattr(obj, 'uuid', None) if uid is None: return type_name = getattr(obj, 'type_name', None) if type_name is None or IContainer.providedBy(obj): return fut = get_future() if fut is None: return fut.remove.append(obj) if uid in fut.index: del fut.index[uid] if uid in fut.update: del fut.update[uid]
async def run(self, arguments, settings, app): aiotask_context.set('request', self.request) if arguments.task_id is not None: task = arguments.task_id payload_config = False else: if hasattr(arguments, 'payload') and arguments.payload is not None: task = arguments.payload task = json.loads(task) else: task = os.environ.get('PAYLOAD', '{}') logger.warning(f"Task to do: \n {task}") task = json.loads(task) payload_config = True tags = [] if arguments.tags: tags = json.loads(arguments.tags) logger.warning("Tasks available: \n") for k, v in settings.get('hive_tasks', {}).items(): logger.warning(f"{k}: {v}") task_obj = None root_obj = get_utility(IApplication, name='root') if payload_config is False: elements = task.split('/')[1:] db_obj = await root_obj.async_get(elements[0]) if IDatabase.providedBy(db_obj): tm = self.request._tm = db_obj.get_transaction_manager() tm.request = self.request self.request._db_id = elements[0] self.request._txn = txn = await tm.begin(self.request) container_obj = await db_obj.async_get(elements[1]) if IContainer.providedBy(container_obj): guillotina_execution = await navigate_to( container_obj, '/'.join(elements[2:])) if IExecution.providedBy(guillotina_execution): task_obj = TaskObject( data=guillotina_execution.get_task_payload()) await tm.abort(txn=txn) elif payload_config is True: task_obj = TaskObject(data=task) if task_obj is None: raise NoTaskFunctionDefinedError() logger.warning("Ready to run") return await task_obj.run(self.request, tags, root_obj)
async def traverse(self, request: IRequest) -> IResource: '''custom traversal here...''' resource, tail = await super().traverse(request) if len(tail) > 0 and tail[0] in ( 'liches', 'lms') and IContainer.providedBy(resource): segment = LichesRouteSegment(tail[0], resource) if tail[0] == 'liches': alsoProvides(request, ILichesLayer) elif tail[0] == 'lms': #segment = LMSRouteSegment(tail[0], resource) alsoProvides(request, ILMSLayer) if len(tail) > 1: # finish traversal from here return await traverse(request, segment, tail[1:]) else: resource = segment tail = tail[1:] return resource, tail
async def get_inherited_aliases(ob) -> list: storage = get_storage() if storage is None: return [] ob_path = get_content_path(ob) data = [] context = ob.__parent__ while context is not None and not IContainer.providedBy(context): context_path = get_content_path(context) for alias in await get_aliases(context): if not alias["moved"]: continue path = alias["path"] current_sub_path = ob_path[len(context_path):] # noqa path = os.path.join(path, current_sub_path.strip("/")) alias["context_path"] = context_path alias["path"] = path data.append(alias) context = context.__parent__ return data
async def run_migration(self): alias_index_name = await self.index_manager.get_index_name() existing_index = await self.index_manager.get_real_index_name() await self.setup_next_index() self.mapping_diff = await self.calculate_mapping_diff() diff = json.dumps(self.mapping_diff, sort_keys=True, indent=4, separators=(',', ': ')) self.response.write(f'Caculated mapping diff: {diff}') if not self.full: # if full, we're reindexing everything does not matter what # anyways, so skip self.response.write(f'Copying initial index {existing_index} ' f'into {self.work_index_name}') try: await self.copy_to_next_index() self.response.write('Copying initial index data finished') except elasticsearch.exceptions.NotFoundError: self.response.write('No initial index to copy to') if not self.mapping_only: try: self.existing = await self.get_all_uids() except elasticsearch.exceptions.NotFoundError: pass self.index_start_time = time.time() if self.children_only or IContainer.providedBy(self.context): await self.process_folder(self.context) # this is recursive else: await self.process_object(self.context) # this is recursive await self.check_existing() await self.flush() await self.join_futures() async with get_migration_lock(await self.index_manager.get_index_name()): self.response.write('Activating new index') async with managed_transaction(self.request, write=True, adopt_parent_txn=True): await self.index_manager.finish_migration() self.status = 'done' self.response.write(f'''Update alias({alias_index_name}): {existing_index} -> {self.work_index_name} ''') try: await self.conn.indices.update_aliases({ "actions": [{ "remove": { "alias": alias_index_name, "index": existing_index } }, { "add": { "alias": alias_index_name, "index": self.work_index_name } }] }) except elasticsearch.exceptions.NotFoundError: await self.conn.indices.update_aliases({ "actions": [{ "add": { "alias": alias_index_name, "index": self.work_index_name } }] }) try: await self.conn.indices.close(existing_index) await self.conn.indices.delete(existing_index) self.response.write('Old index deleted') except elasticsearch.exceptions.NotFoundError: pass if len(self.sub_indexes) > 0: self.response.write( f'Migrating sub indexes: {len(self.sub_indexes)}') for ob in self.sub_indexes: im = get_adapter(ob, IIndexManager) migrator = Migrator(self.utility, ob, response=self.response, force=self.force, log_details=self.log_details, memory_tracking=self.memory_tracking, request=self.request, bulk_size=self.bulk_size, full=self.full, reindex_security=self.reindex_security, mapping_only=self.mapping_only, index_manager=im, children_only=True) self.response.write(f'Migrating index for: {ob}') await migrator.run_migration()
def get_container_id(self): ob = self.content while getattr(ob, '__parent__', None) and not IContainer.providedBy(ob): ob = ob.__parent__ if IContainer.providedBy(ob): return ob.__name__