Example #1
0
 def get_container_id(self):
     ob = self.content
     while getattr(ob, '__parent__',
                   None) and not IContainer.providedBy(ob):
         ob = ob.__parent__
     if IContainer.providedBy(ob):
         return ob.__name__
Example #2
0
async def do_traverse(request, parent, path):
    """Traverse for the code API."""
    if not path:
        return parent, path

    assert request is not None  # could be used for permissions, etc

    if IContainer.providedBy(parent) and \
       path[0] != request._db_id:
        # Tried to access a container outside the request
        raise HTTPUnauthorized()

    if IApplication.providedBy(parent) and \
       path[0] != request._container_id:
        # Tried to access a container outside the request
        raise HTTPUnauthorized()

    try:
        if path[0].startswith('_') or path[0] in ('.', '..'):
            raise HTTPUnauthorized()
        context = parent[path[0]]
    except TypeError:
        return parent, path
    except KeyError:
        return parent, path

    context._v_parent = parent

    return await traverse(request, context, path[1:])
Example #3
0
async def add_object(obj, event):
    uid = getattr(obj, 'uuid', None)
    if uid is None:
        return
    type_name = getattr(obj, 'type_name', None)
    if type_name is None or IContainer.providedBy(obj):
        return

    fut = get_future()
    if fut is None:
        return
    search = queryUtility(ICatalogUtility)
    if search:
        if IObjectModifiedEvent.providedBy(event):
            indexes = []
            if event.payload and len(event.payload) > 0:
                # get a list of potential indexes
                for field_name in event.payload.keys():
                    if '.' in field_name:
                        for behavior_field_name in event.payload[field_name].keys():
                            indexes.append(behavior_field_name)
                    else:
                        indexes.append(field_name)
                fut.update[uid] = await search.get_data(obj, indexes)
        else:
            fut.index[uid] = await search.get_data(obj)
Example #4
0
async def index_object(obj, indexes=None, modified=False, security=False):
    uid = getattr(obj, 'uuid', None)
    if uid is None:
        return
    type_name = getattr(obj, 'type_name', None)
    if type_name is None or IContainer.providedBy(obj):
        return

    search = query_utility(ICatalogUtility)
    if search is None:
        return

    fut = get_future()
    if fut is None:
        return

    if modified:
        data = {}
        if security:
            adapter = query_adapter(obj, ISecurityInfo)
            if adapter is not None:
                data = await apply_coroutine(adapter)
        else:
            if indexes is not None and len(indexes) > 0:
                data = await search.get_data(obj, indexes)
        if len(data) > 0:
            if uid in fut.update:
                fut.update[uid].update(data)
            else:
                fut.update[uid] = data
    else:
        fut.index[uid] = await search.get_data(obj)
Example #5
0
    async def process_object(self, ob):
        '''
        - check if doc does not exist
            - record it
            - do complete index
        - if doc exist
            - if diff mapping exists
                - update fields in diff on doc
            - else, do nothing
            - remove for list of existing doc ids
        '''
        clear_conn_statement_cache(await ob._p_jar.get_connection())
        full = False
        if ob.uuid not in self.existing:
            self.missing.append(ob.uuid)
            full = True
        else:
            self.existing.remove(ob.uuid)
        await self.index_object(ob, full=full)
        self.processed += 1

        if IFolder.providedBy(ob):
            await self.process_folder(ob)

        if not IContainer.providedBy(ob):
            del ob.__annotations__
        del ob
    async def process_object(self, ob):
        '''
        - check if doc does not exist
            - record it
            - do complete index
        - if doc exist
            - if diff mapping exists
                - update fields in diff on doc
            - else, do nothing
            - remove for list of existing doc ids
        '''
        full = False
        if ob.uuid not in self.existing:
            self.missing.append(ob.uuid)
            full = True
        else:
            self.existing.remove(ob.uuid)
        await self.index_object(ob, full=full)
        self.processed += 1

        if IIndexActive.providedBy(ob):
            self.sub_indexes.append(ob)
        else:
            if IFolder.providedBy(ob):
                await self.process_folder(ob)

            if not IContainer.providedBy(ob):
                try:
                    del self.container.__gannotations__
                except AttributeError:
                    del self.container.__annotations__
            del ob
Example #7
0
async def index_object(obj, indexes=None, modified=False, security=False):
    uid = getattr(obj, 'uuid', None)
    if uid is None:
        return
    type_name = getattr(obj, 'type_name', None)
    if type_name is None or IContainer.providedBy(obj):
        return

    search = query_utility(ICatalogUtility)
    if search is None:
        return

    fut = get_future()
    if fut is None:
        return

    if modified:
        data = {}
        if security:
            adapter = query_adapter(obj, ISecurityInfo)
            if adapter is not None:
                data = await apply_coroutine(adapter)
        else:
            if indexes is not None and len(indexes) > 0:
                data = await search.get_data(obj, indexes)
        if len(data) > 0:
            if uid in fut.update:
                fut.update[uid].update(data)
            else:
                fut.update[uid] = data
    else:
        fut.index[uid] = await search.get_data(obj)
Example #8
0
def find_container(context=None) -> typing.Optional[IContainer]:
    container = task_vars.container.get()
    if container is None:
        while context is not None:
            if IContainer.providedBy(context):
                container = context
                break
            context = getattr(context, '__parent__', None)
    return container
Example #9
0
async def traverse(
        request: IRequest, parent: IBaseObject,
        path: Tuple[str, ...]) -> Tuple[IBaseObject, Tuple[str, ...]]:
    """Do not use outside the main router function."""
    if IApplication.providedBy(parent):
        request.application = parent

    if len(path) == 0:
        return parent, path

    assert request is not None  # could be used for permissions, etc

    if not ITraversable.providedBy(parent):
        # not a traversable context
        return parent, path
    try:
        if path[0][0] == "_" or path[0] in (".", ".."):
            raise HTTPUnauthorized()
        if path[0][0] == "@":
            # shortcut
            return parent, path

        if IAsyncContainer.providedBy(parent):
            context = await parent.async_get(path[0], suppress_events=True)
            if context is None:
                return parent, path
        else:
            context = parent[path[0]]
    except (TypeError, KeyError, AttributeError):
        return parent, path

    if IDatabase.providedBy(context):
        task_vars.db.set(context)
        # Add a transaction Manager to request
        tm = context.get_transaction_manager()
        task_vars.tm.set(tm)
        # Start a transaction
        txn = await tm.begin(
            read_only=not app_settings["check_writable_request"](request))
        # Get the root of the tree
        context = await tm.get_root(txn=txn)

    if IContainer.providedBy(context):
        task_vars.container.set(context)
        # make sure to unset before we attempt to load in case
        # there is an existing registry object set on task_vars
        task_vars.registry.set(None)
        registry = await get_registry(context)
        layers = registry.get(ACTIVE_LAYERS_KEY, [])
        for layer in layers:
            try:
                alsoProvides(request, import_class(layer))
            except ModuleNotFoundError:
                logger.error("Can not apply layer " + layer, request=request)

    return await traverse(request, context, path[1:])
Example #10
0
async def traverse(request, parent, path):
    """Do not use outside the main router function."""
    if IApplication.providedBy(parent):
        request.application = parent

    if not path:
        return parent, path

    assert request is not None  # could be used for permissions, etc

    if not ITraversable.providedBy(parent):
        # not a traversable context
        return parent, path
    try:
        if path[0][0] == '_' or path[0] in ('.', '..'):
            raise HTTPUnauthorized()
        if path[0][0] == '@':
            # shortcut
            return parent, path

        if IAsyncContainer.providedBy(parent):
            context = await parent.async_get(path[0], suppress_events=True)
            if context is None:
                return parent, path
        else:
            context = parent[path[0]]
    except (TypeError, KeyError, AttributeError):
        return parent, path

    if IDatabase.providedBy(context):
        request._db_write_enabled = app_settings['check_writable_request'](
            request)
        request._db_id = context.id
        # Add a transaction Manager to request
        tm = request._tm = context.get_transaction_manager()
        # Start a transaction
        txn = await tm.begin(request=request)
        # Get the root of the tree
        context = await tm.get_root(txn=txn)

    if IContainer.providedBy(context):
        request._container_id = context.id
        request.container = context
        annotations_container = IAnnotations(request.container)
        request.container_settings = await annotations_container.async_get(
            REGISTRY_DATA_KEY)
        layers = request.container_settings.get(ACTIVE_LAYERS_KEY, [])
        for layer in layers:
            try:
                alsoProvides(request, import_class(layer))
            except ModuleNotFoundError:
                logger.error('Can not apply layer ' + layer, request=request)

    return await traverse(request, context, path[1:])
Example #11
0
    async def __call__(self):
        result = []
        context = self.context
        while context is not None and not IContainer.providedBy(context):
            result.append({
                "title": context.title,
                "@id": IAbsoluteURL(context, self.request)()
            })
            context = getattr(context, "__parent__", None)
        result.reverse()

        return {"@id": self.request.url, "items": result}
Example #12
0
def get_content_path(content: IResource) -> str:
    """
    Generate full path of resource object
    """
    parts = []
    parent = getattr(content, '__parent__', None)
    while content is not None and content.__name__ is not None and\
            parent is not None and not IContainer.providedBy(content):
        parts.append(content.__name__)
        content = parent
        parent = getattr(content, '__parent__', None)
    return '/' + '/'.join(reversed(parts))
Example #13
0
async def remove_object(obj, event):
    uid = getattr(obj, "uuid", None)
    if uid is None:
        return
    type_name = getattr(obj, "type_name", None)
    if type_name is None or IContainer.providedBy(obj):
        return

    fut = get_indexer(obj)
    if fut is None:
        return
    await fut.remove_object(obj)
Example #14
0
 def __init__(self, ob, request=None):
     if request is None:
         request = get_current_request()
     self.request = get_current_request()
     self.context = ob
     if hasattr(self.request, 'container'):
         self.container = self.request.container
     else:
         if IContainer.providedBy(ob):
             self.container = ob
         else:
             raise Exception('Could not location container object')
Example #15
0
async def traverse(request, parent, path):
    """Do not use outside the main router function."""
    if IApplication.providedBy(parent):
        request.application = parent

    if not path:
        return parent, path

    assert request is not None  # could be used for permissions, etc

    if not ITraversable.providedBy(parent):
        # not a traversable context
        return parent, path
    try:
        if path[0][0] == '_' or path[0] in ('.', '..'):
            raise HTTPUnauthorized()
        if path[0][0] == '@':
            # shortcut
            return parent, path

        if IAsyncContainer.providedBy(parent):
            context = await parent.async_get(path[0], suppress_events=True)
            if context is None:
                return parent, path
        else:
            context = parent[path[0]]
    except (TypeError, KeyError, AttributeError):
        return parent, path

    if IDatabase.providedBy(context):
        request._db_write_enabled = app_settings['check_writable_request'](request)
        request._db_id = context.id
        # Add a transaction Manager to request
        tm = request._tm = context.get_transaction_manager()
        # Start a transaction
        txn = await tm.begin(request=request)
        # Get the root of the tree
        context = await tm.get_root(txn=txn)

    if IContainer.providedBy(context):
        request._container_id = context.id
        request.container = context
        annotations_container = IAnnotations(request.container)
        request.container_settings = await annotations_container.async_get(REGISTRY_DATA_KEY)
        layers = request.container_settings.get(ACTIVE_LAYERS_KEY, [])
        for layer in layers:
            try:
                alsoProvides(request, import_class(layer))
            except ModuleNotFoundError:
                logger.error('Can not apply layer ' + layer, request=request)

    return await traverse(request, context, path[1:])
Example #16
0
async def index_object(obj, indexes=None, modified=False, security=False):
    uid = getattr(obj, "uuid", None)
    if uid is None:
        return
    type_name = getattr(obj, "type_name", None)
    if type_name is None or IContainer.providedBy(obj):
        return

    fut = get_indexer(obj)
    if fut is None:
        return

    await fut.add_object(obj, indexes, modified, security)
Example #17
0
def find_container(context=None) -> typing.Optional[IContainer]:
    """
    Find container based on contextvar or by looking up the
    container from the provided context parameter
    """
    container = task_vars.container.get()
    if container is None:
        while context is not None:
            if IContainer.providedBy(context):
                container = context
                break
            context = getattr(context, "__parent__", None)
    return container
Example #18
0
def get_content_path(content: IResource) -> str:
    """
    Generate full path of resource object

    :param content: object to get path from
    """
    parts = []
    parent = getattr(content, '__parent__', None)
    while content is not None and content.__name__ is not None and\
            parent is not None and not IContainer.providedBy(content):
        parts.append(content.__name__)
        content = parent
        parent = getattr(content, '__parent__', None)
    return '/' + '/'.join(reversed(parts))
Example #19
0
def get_content_path(content: IResource) -> str:
    """
    Generate path of resource object from the container

    :param content: object to get path from
    """
    parts = []
    parent = getattr(content, "__parent__", None)
    while (content is not None and content.__name__ is not None
           and parent is not None and not IContainer.providedBy(content)):
        parts.append(content.__name__)
        content = parent
        parent = getattr(content, "__parent__", None)
    return "/" + "/".join(reversed(parts))
Example #20
0
async def add_object(obj, event):
    uid = getattr(obj, 'uuid', None)
    if uid is None:
        return
    type_name = getattr(obj, 'type_name', None)
    if type_name is None or IContainer.providedBy(obj):
        return

    hook = get_hook()
    if hook is None:
        return
    search = queryUtility(ICatalogUtility)
    if search:
        hook.index[uid] = await search.get_data(obj)
Example #21
0
def remove_object(obj, event):
    uid = getattr(obj, 'uuid', None)
    if uid is None:
        return
    type_name = getattr(obj, 'type_name', None)
    if type_name is None or IContainer.providedBy(obj):
        return

    content_path = get_content_path(obj)

    hook = get_hook()
    if hook is None:
        return
    hook.remove.append((uid, type_name, content_path))
    if uid in hook.index:
        del hook.index[uid]
Example #22
0
def remove_object(obj, event):
    uid = getattr(obj, 'uuid', None)
    if uid is None:
        return
    type_name = getattr(obj, 'type_name', None)
    if type_name is None or IContainer.providedBy(obj):
        return

    fut = get_future()
    if fut is None:
        return

    fut.remove.append(obj)
    if uid in fut.index:
        del fut.index[uid]
    if uid in fut.update:
        del fut.update[uid]
Example #23
0
def remove_object(obj, event):
    uid = getattr(obj, 'uuid', None)
    if uid is None:
        return
    type_name = getattr(obj, 'type_name', None)
    if type_name is None or IContainer.providedBy(obj):
        return

    fut = get_future()
    if fut is None:
        return

    fut.remove.append(obj)
    if uid in fut.index:
        del fut.index[uid]
    if uid in fut.update:
        del fut.update[uid]
Example #24
0
    async def run(self, arguments, settings, app):
        aiotask_context.set('request', self.request)
        if arguments.task_id is not None:
            task = arguments.task_id
            payload_config = False
        else:
            if hasattr(arguments, 'payload') and arguments.payload is not None:
                task = arguments.payload
                task = json.loads(task)
            else:
                task = os.environ.get('PAYLOAD', '{}')
                logger.warning(f"Task to do: \n {task}")
                task = json.loads(task)
            payload_config = True
        tags = []
        if arguments.tags:
            tags = json.loads(arguments.tags)

        logger.warning("Tasks available: \n")
        for k, v in settings.get('hive_tasks', {}).items():
            logger.warning(f"{k}: {v}")
        task_obj = None
        root_obj = get_utility(IApplication, name='root')
        if payload_config is False:
            elements = task.split('/')[1:]
            db_obj = await root_obj.async_get(elements[0])
            if IDatabase.providedBy(db_obj):
                tm = self.request._tm = db_obj.get_transaction_manager()
                tm.request = self.request
                self.request._db_id = elements[0]
                self.request._txn = txn = await tm.begin(self.request)
                container_obj = await db_obj.async_get(elements[1])
                if IContainer.providedBy(container_obj):
                    guillotina_execution = await navigate_to(
                        container_obj, '/'.join(elements[2:]))
                    if IExecution.providedBy(guillotina_execution):
                        task_obj = TaskObject(
                            data=guillotina_execution.get_task_payload())
                await tm.abort(txn=txn)
        elif payload_config is True:
            task_obj = TaskObject(data=task)

        if task_obj is None:
            raise NoTaskFunctionDefinedError()
        logger.warning("Ready to run")
        return await task_obj.run(self.request, tags, root_obj)
Example #25
0
 async def traverse(self, request: IRequest) -> IResource:
     '''custom traversal here...'''
     resource, tail = await super().traverse(request)
     if len(tail) > 0 and tail[0] in (
             'liches', 'lms') and IContainer.providedBy(resource):
         segment = LichesRouteSegment(tail[0], resource)
         if tail[0] == 'liches':
             alsoProvides(request, ILichesLayer)
         elif tail[0] == 'lms':
             #segment = LMSRouteSegment(tail[0], resource)
             alsoProvides(request, ILMSLayer)
         if len(tail) > 1:
             # finish traversal from here
             return await traverse(request, segment, tail[1:])
         else:
             resource = segment
             tail = tail[1:]
     return resource, tail
Example #26
0
async def get_inherited_aliases(ob) -> list:
    storage = get_storage()
    if storage is None:
        return []

    ob_path = get_content_path(ob)
    data = []
    context = ob.__parent__
    while context is not None and not IContainer.providedBy(context):
        context_path = get_content_path(context)
        for alias in await get_aliases(context):
            if not alias["moved"]:
                continue
            path = alias["path"]
            current_sub_path = ob_path[len(context_path):]  # noqa
            path = os.path.join(path, current_sub_path.strip("/"))
            alias["context_path"] = context_path
            alias["path"] = path
            data.append(alias)
        context = context.__parent__

    return data
    async def run_migration(self):
        alias_index_name = await self.index_manager.get_index_name()
        existing_index = await self.index_manager.get_real_index_name()

        await self.setup_next_index()

        self.mapping_diff = await self.calculate_mapping_diff()
        diff = json.dumps(self.mapping_diff,
                          sort_keys=True,
                          indent=4,
                          separators=(',', ': '))
        self.response.write(f'Caculated mapping diff: {diff}')

        if not self.full:
            # if full, we're reindexing everything does not matter what
            # anyways, so skip
            self.response.write(f'Copying initial index {existing_index} '
                                f'into {self.work_index_name}')
            try:
                await self.copy_to_next_index()
                self.response.write('Copying initial index data finished')
            except elasticsearch.exceptions.NotFoundError:
                self.response.write('No initial index to copy to')
        if not self.mapping_only:
            try:
                self.existing = await self.get_all_uids()
            except elasticsearch.exceptions.NotFoundError:
                pass

            self.index_start_time = time.time()
            if self.children_only or IContainer.providedBy(self.context):
                await self.process_folder(self.context)  # this is recursive
            else:
                await self.process_object(self.context)  # this is recursive

            await self.check_existing()

            await self.flush()
            await self.join_futures()

        async with get_migration_lock(await
                                      self.index_manager.get_index_name()):
            self.response.write('Activating new index')
            async with managed_transaction(self.request,
                                           write=True,
                                           adopt_parent_txn=True):
                await self.index_manager.finish_migration()
            self.status = 'done'

            self.response.write(f'''Update alias({alias_index_name}):
{existing_index} -> {self.work_index_name}
''')

            try:
                await self.conn.indices.update_aliases({
                    "actions": [{
                        "remove": {
                            "alias": alias_index_name,
                            "index": existing_index
                        }
                    }, {
                        "add": {
                            "alias": alias_index_name,
                            "index": self.work_index_name
                        }
                    }]
                })
            except elasticsearch.exceptions.NotFoundError:
                await self.conn.indices.update_aliases({
                    "actions": [{
                        "add": {
                            "alias": alias_index_name,
                            "index": self.work_index_name
                        }
                    }]
                })

        try:
            await self.conn.indices.close(existing_index)
            await self.conn.indices.delete(existing_index)
            self.response.write('Old index deleted')
        except elasticsearch.exceptions.NotFoundError:
            pass

        if len(self.sub_indexes) > 0:
            self.response.write(
                f'Migrating sub indexes: {len(self.sub_indexes)}')
            for ob in self.sub_indexes:
                im = get_adapter(ob, IIndexManager)
                migrator = Migrator(self.utility,
                                    ob,
                                    response=self.response,
                                    force=self.force,
                                    log_details=self.log_details,
                                    memory_tracking=self.memory_tracking,
                                    request=self.request,
                                    bulk_size=self.bulk_size,
                                    full=self.full,
                                    reindex_security=self.reindex_security,
                                    mapping_only=self.mapping_only,
                                    index_manager=im,
                                    children_only=True)
                self.response.write(f'Migrating index for: {ob}')
                await migrator.run_migration()
Example #28
0
 def get_container_id(self):
     ob = self.content
     while getattr(ob, '__parent__', None) and not IContainer.providedBy(ob):
         ob = ob.__parent__
     if IContainer.providedBy(ob):
         return ob.__name__