def get_access_where_clauses(self): users = [] roles = [] request = get_current_request() interaction = IInteraction(request) for user in interaction.participations: users.append(user.principal.id) users.extend(user.principal.groups) roles_dict = interaction.global_principal_roles( user.principal.id, user.principal.groups) roles.extend([key for key, value in roles_dict.items() if value]) clauses = [] if len(users) > 0: clauses.append("json->'access_users' ?| array['{}']".format( "','".join(users) )) if len(roles) > 0: clauses.append("json->'access_roles' ?| array['{}']".format( "','".join(roles) )) return '({})'.format( ' OR '.join(clauses) )
async def _build_security_query( self, container, query, doc_type=None, size=10, request=None, scroll=None): if query is None: query = {} q = {} # The users who has plone.AccessContent permission by prinperm # The roles who has plone.AccessContent permission by roleperm users = [] roles = [] if request is None: request = get_current_request() interaction = IInteraction(request) for user in interaction.participations: # pylint: disable=E1133 users.append(user.principal.id) users.extend(user.principal.groups) roles_dict = interaction.global_principal_roles( user.principal.id, user.principal.groups) roles.extend([key for key, value in roles_dict.items() if value]) # We got all users and roles # users: users and groups should_list = [{'match': {'access_roles': x}} for x in roles] should_list.extend([{'match': {'access_users': x}} for x in users]) permission_query = { 'query': { 'bool': { 'filter': { 'bool': { 'should': should_list, 'minimum_should_match': 1 } } } } } query = merge_dicts(query, permission_query) # query.update(permission_query) q['body'] = query q['size'] = size if scroll: q['scroll'] = scroll logger.debug(q) return q
async def resolve_uid(context, request): uid = request.matchdict['uid'] ob = await get_object_by_oid(uid) if ob is None: return HTTPNotFound(content={'reason': f'Could not find uid: {uid}'}) interaction = IInteraction(request) if interaction.check_permission('guillotina.AccessContent', ob): return HTTPMovedPermanently(get_object_url(ob, request)) else: # if a user doesn't have access to it, they shouldn't know anything about it return HTTPNotFound(content={'reason': f'Could not find uid: {uid}'})
async def can_i_do(context, request): if 'permission' not in request.query and 'permissions' not in request.query: raise PreconditionFailed(context, 'No permission param') interaction = IInteraction(request) if 'permissions' in request.GET: results = {} for perm in request.GET['permissions'].split(','): results[perm] = interaction.check_permission(perm, context) return results else: return interaction.check_permission(request.GET['permission'], context)
def __init__(self, utility, context, response=noop_response, force=False, log_details=False, memory_tracking=False, request=None, bulk_size=40, full=False, reindex_security=False, mapping_only=False): self.utility = utility self.conn = utility.conn self.context = context self.response = response self.force = force self.full = full self.log_details = log_details self.memory_tracking = memory_tracking self.bulk_size = bulk_size self.reindex_security = reindex_security if mapping_only and full: raise Exception( 'Can not do a full reindex and a mapping only migration') self.mapping_only = mapping_only if request is None: self.request = get_current_request() self.request._db_write_enabled = False else: self.request = request # make sure that we don't cache requests... self.request._txn._cache = DummyCache(self.request._txn) self.container = self.request.container self.interaction = IInteraction(self.request) self.indexer = Indexer() self.batch = {} self.indexed = 0 self.processed = 0 self.missing = [] self.orphaned = [] self.existing = [] self.errors = [] self.mapping_diff = {} self.start_time = self.index_start_time = time.time() self.reindex_futures = [] self.status = 'started' self.active_task_id = None self.copied_docs = 0 self.work_index_name = None
async def available_actions(self, request): security = IInteraction(request) for action_name, action in self.actions.items(): add = False if 'check_permission' in action and security.check_permission( action['check_permission'], self.context): add = True elif 'check_permission' not in action: add = True if add: yield action_name, action
def check_permission(self, permission_name): if permission_name is None: return True if permission_name not in self.permission_cache: permission = query_utility(IPermission, name=permission_name) if permission is None: self.permission_cache[permission_name] = True else: security = IInteraction(self.request) self.permission_cache[permission_name] = bool( security.check_permission(permission.id, self.context)) return self.permission_cache[permission_name]
def check_permission(self, permission_name): if permission_name is None: return True if permission_name not in self.permission_cache: permission = query_utility(IPermission, name=permission_name) if permission is None: self.permission_cache[permission_name] = True else: security = IInteraction(self.request) self.permission_cache[permission_name] = bool( security.check_permission(permission.id, self.context)) return self.permission_cache[permission_name]
def get_principals_with_access_content(obj, request=None): if obj is None: return {} if request is None: request = get_current_request() interaction = IInteraction(request) roles = interaction.cached_roles(obj, 'guillotina.AccessContent', 'o') result = [] all_roles = role.global_roles() + role.local_roles() for r in roles.keys(): if r in all_roles: result.append(r) users = interaction.cached_principals(obj, result, 'guillotina.AccessContent', 'o') return list(users.keys())
def get_roles_with_access_content(obj, request=None): """ Return the roles that has access to the content that are global roles""" if obj is None: return [] if request is None: request = get_current_request() interaction = IInteraction(request) roles = interaction.cached_roles(obj, 'guillotina.AccessContent', 'o') result = [] all_roles = role.global_roles() + role.local_roles() for r in roles.keys(): if r in all_roles: result.append(r) return result
def get_principals_with_access_content(obj, request=None): if obj is None: return {} if request is None: request = get_current_request() interaction = IInteraction(request) roles = interaction.cached_roles(obj, 'guillotina.AccessContent', 'o') result = [] all_roles = role.global_roles() + role.local_roles() for r in roles.keys(): if r in all_roles: result.append(r) users = interaction.cached_principals(obj, result, 'guillotina.AccessContent', 'o') return list(users.keys())
def get_roles_with_access_content(obj, request=None): """ Return the roles that has access to the content that are global roles""" if obj is None: return [] if request is None: request = get_current_request() interaction = IInteraction(request) roles = interaction.cached_roles(obj, 'guillotina.AccessContent', 'o') result = [] all_roles = role.global_roles() + role.local_roles() for r in roles.keys(): if r in all_roles: result.append(r) return result
async def __call__(self): self.interaction = IInteraction(self.request) definition = copy.deepcopy( app_settings["swagger"]["base_configuration"]) vhm = self.request.headers.get("X-VirtualHost-Monster") if vhm: parsed_url = urlparse(vhm) definition["host"] = parsed_url.netloc definition["schemes"] = [parsed_url.scheme] definition["basePath"] = parsed_url.path else: definition["host"] = self.request.host definition["schemes"] = [get_scheme(self.request)] if 'version' not in definition['info']: definition["info"]["version"] = pkg_resources.get_distribution( "guillotina").version api_defs = app_settings["api_definition"] path = get_full_content_path(self.request, self.context) for dotted_iface in api_defs.keys(): iface = resolve_dotted_name(dotted_iface) if iface.providedBy(self.context): iface_conf = api_defs[dotted_iface] self.get_endpoints(iface_conf, path, definition["paths"]) definition["definitions"] = app_settings["json_schema_definitions"] return definition
async def get_all_types(context, request): result = [] base_url = IAbsoluteURL(context, request)() constrains = IConstrainTypes(context, None) for id, factory in FACTORY_CACHE.items(): add = True if constrains is not None: if not constrains.is_type_allowed(id): add = False if factory.add_permission: if factory.add_permission in PERMISSIONS_CACHE: permission = PERMISSIONS_CACHE[factory.add_permission] else: permission = query_utility(IPermission, name=factory.add_permission) PERMISSIONS_CACHE[factory.add_permission] = permission if permission is not None and \ not IInteraction(request).check_permission( permission.id, context): add = False if add: result.append({ '@id': base_url + '/@types/' + id, 'addable': True, 'title': id }) return result
async def create_content_in_container(parent: Folder, type_: str, id_: str, request: IRequest = None, check_security=True, **kw) -> Resource: """Utility to create a content. This method is the one to use to create content. id_ can be None :param parent: where to create content inside of :param type_: content type to create :param id_: id to give content in parent object :param request: <optional> :param check_security: be able to disable security checks """ factory = get_cached_factory(type_) if check_security and factory.add_permission: if factory.add_permission in PERMISSIONS_CACHE: permission = PERMISSIONS_CACHE[factory.add_permission] else: permission = query_utility(IPermission, name=factory.add_permission) PERMISSIONS_CACHE[factory.add_permission] = permission if request is None: request = get_current_request() if permission is not None and \ not IInteraction(request).check_permission(permission.id, parent): raise NoPermissionToAdd(str(parent), type_) constrains = IConstrainTypes(parent, None) if constrains is not None: if not constrains.is_type_allowed(type_): raise NotAllowedContentType(str(parent), type_) # We create the object with at least the ID obj = factory(id=id_, parent=parent) for key, value in kw.items(): if key == 'id': # the factory sets id continue setattr(obj, key, value) txn = getattr(parent, '_p_jar', None) or get_transaction() if txn is None or not txn.storage.supports_unique_constraints: # need to manually check unique constraints if await parent.async_contains(obj.id): raise ConflictIdOnContainer(f'Duplicate ID: {parent} -> {obj.id}') obj.__new_marker__ = True await notify(BeforeObjectAddedEvent(obj, parent, id_)) await parent.async_set(obj.id, obj) return obj
async def handle_ws_request(self, ws, message): method = app_settings['http_methods']['GET'] path = tuple(p for p in message['value'].split('/') if p) # avoid circular import from guillotina.traversal import do_traverse obj, tail = await do_traverse(self.request, self.request.container, path) traverse_to = None if tail and len(tail) == 1: view_name = tail[0] elif tail is None or len(tail) == 0: view_name = '' else: view_name = tail[0] traverse_to = tail[1:] permission = getUtility(IPermission, name='guillotina.AccessContent') allowed = IInteraction(self.request).check_permission( permission.id, obj) if not allowed: response = {'error': 'Not allowed'} ws.send_str(ujson.dumps(response)) try: view = queryMultiAdapter((obj, self.request), method, name=view_name) except AttributeError: view = None if traverse_to is not None: if view is None or not ITraversableView.providedBy(view): response = {'error': 'Not found'} ws.send_str(ujson.dumps(response)) else: try: view = await view.publish_traverse(traverse_to) except Exception as e: logger.error("Exception on view execution", exc_info=e) response = {'error': 'Not found'} ws.send_str(ujson.dumps(response)) view_result = await view() if isinstance(view_result, Response): view_result = view_result.response # Return the value ws.send_str(ujson.dumps(view_result)) # Wait for possible value futures_to_wait = self.request._futures.values() if futures_to_wait: await asyncio.gather(*list(futures_to_wait)) self.request._futures = {}
async def move(context, request): try: data = await request.json() except Exception: data = {} destination = data.get('destination') if destination is None: destination_ob = context.__parent__ else: try: destination_ob = await navigate_to(request.container, destination) except KeyError: destination_ob = None if destination_ob is None: raise PreconditionFailed(context, 'Could not find destination object') old_id = context.id if 'new_id' in data: new_id = data['new_id'] context.id = context.__name__ = new_id else: new_id = context.id security = IInteraction(request) if not security.check_permission('guillotina.AddContent', destination_ob): raise PreconditionFailed( context, 'You do not have permission to add content to the ' 'destination object') if await destination_ob.async_contains(new_id): raise PreconditionFailed( context, f'Destination already has object with the id {new_id}') original_parent = context.__parent__ txn = get_transaction(request) cache_keys = txn._cache.get_cache_keys(context, 'deleted') data['id'] = new_id await notify( BeforeObjectMovedEvent(context, original_parent, old_id, destination_ob, new_id, payload=data)) context.__parent__ = destination_ob context._p_register() await notify( ObjectMovedEvent(context, original_parent, old_id, destination_ob, new_id, payload=data)) cache_keys += txn._cache.get_cache_keys(context, 'added') await txn._cache.delete_all(cache_keys) absolute_url = query_multi_adapter((context, request), IAbsoluteURL) return { '@url': absolute_url() }
async def do_action(self, request, action, comments): available_actions = self.actions if action not in available_actions: raise KeyError('Unavailable action') action_def = available_actions[action] security = IInteraction(request) if 'check_permission' in action_def and not security.check_permission( action_def['check_permission'], self.context): raise HTTPUnauthorized() # Change permission new_state = action_def['to'] if 'set_permission' in self.states[new_state]: await apply_sharing(self.context, self.states[new_state]['set_permission']) # Write history user = get_authenticated_user_id(request) history = { 'actor': user, 'comments': comments, 'time': datetime.datetime.now(), 'title': action_def['title'], 'type': 'workflow', 'data': { 'action': action, 'review_state': new_state, } } cms_behavior = ICMSBehavior(self.context) await cms_behavior.load() cms_behavior.review_state = new_state cms_behavior.history.append(history) cms_behavior._p_register() await notify( WorkflowChangedEvent(self.context, self, action, comments)) return history
async def __call__(self): result = await super(SerializeFolderToJson, self).__call__() security = IInteraction(self.request) length = await self.context.async_len() if length > MAX_ALLOWED or length == 0: result['items'] = [] else: result['items'] = [] async for ident, member in self.context.async_items(): if not ident.startswith('_') and bool( security.check_permission('guillotina.AccessContent', member)): result['items'].append(await getMultiAdapter( (member, self.request), IResourceSerializeToJsonSummary)()) result['length'] = length return result
def get_current_interaction(request): """ Cache IInteraction on the request object because the request object is where we start adding principals """ interaction = getattr(request, 'security', None) if IInteraction.providedBy(interaction): return interaction interaction = Interaction(request) request.security = interaction return interaction
async def __call__(self, include=[], omit=[]): result = await super(SerializeFolderToJson, self).__call__(include=include, omit=omit) security = IInteraction(self.request) length = await self.context.async_len() if length > MAX_ALLOWED or length == 0: result['items'] = [] else: result['items'] = [] async for ident, member in self.context.async_items(suppress_events=True): if not ident.startswith('_') and bool( security.check_permission( 'guillotina.AccessContent', member)): result['items'].append( await get_multi_adapter( (member, self.request), IResourceSerializeToJsonSummary)()) result['length'] = length return result
async def __call__(self): result = {'databases': [], 'static_file': [], 'static_directory': []} allowed = IInteraction(self.request).check_permission( 'guillotina.GetDatabases', self.application) for x in self.application._dbs.keys(): if IDatabase.providedBy(self.application._dbs[x]) and allowed: result['databases'].append(x) if IStaticFile.providedBy(self.application._dbs[x]): result['static_file'].append(x) if IStaticDirectory.providedBy(self.application._dbs[x]): result['static_directory'].append(x) return result
def _invalidated_interaction_cache(self): # Invalidate this threads interaction cache try: request = get_current_request() except RequestNotFound: return interaction = IInteraction(request) if interaction is not None: try: invalidate_cache = interaction.invalidate_cache except AttributeError: pass else: invalidate_cache()
async def create_content_in_container(container, type_, id_, request=None, check_security=True, **kw): """Utility to create a content. This method is the one to use to create content. id_ can be None """ factory = get_cached_factory(type_) if check_security and factory.add_permission: if factory.add_permission in PERMISSIONS_CACHE: permission = PERMISSIONS_CACHE[factory.add_permission] else: permission = query_utility(IPermission, name=factory.add_permission) PERMISSIONS_CACHE[factory.add_permission] = permission if request is None: request = get_current_request() if permission is not None and \ not IInteraction(request).check_permission(permission.id, container): raise NoPermissionToAdd(str(container), type_) constrains = IConstrainTypes(container, None) if constrains is not None: if not constrains.is_type_allowed(type_): raise NotAllowedContentType(str(container), type_) # We create the object with at least the ID obj = factory(id=id_, parent=container) for key, value in kw.items(): setattr(obj, key, value) txn = getattr(container, '_p_jar', None) or get_transaction() if txn is None or not txn.storage.supports_unique_constraints: # need to manually check unique constraints if await container.async_contains(obj.id): raise ConflictIdOnContainer( f'Duplicate ID: {container} -> {obj.id}') obj.__new_marker__ = True await notify(BeforeObjectAddedEvent(obj, container, id_)) await container.async_set(obj.id, obj) return obj
async def build_security_query(container, request=None): # The users who has plone.AccessContent permission by prinperm # The roles who has plone.AccessContent permission by roleperm users = [] roles = [] if request is None: request = get_current_request() interaction = IInteraction(request) for user in interaction.participations: # pylint: disable=E1133 users.append(user.principal.id) users.extend(user.principal.groups) roles_dict = interaction.global_principal_roles( user.principal.id, user.principal.groups) roles.extend([key for key, value in roles_dict.items() if value]) # We got all users and roles # users: users and groups should_list = [{'match': {'access_roles': x}} for x in roles] should_list.extend([{'match': {'access_users': x}} for x in users]) return { 'query': { 'bool': { 'filter': { 'bool': { 'should': should_list, 'minimum_should_match': 1 } } } } }
async def create_content_in_container(container, type_, id_, request=None, **kw): """Utility to create a content. This method is the one to use to create content. id_ can be None """ factory = get_cached_factory(type_) if factory.add_permission: if factory.add_permission in PERMISSIONS_CACHE: permission = PERMISSIONS_CACHE[factory.add_permission] else: permission = query_utility(IPermission, name=factory.add_permission) PERMISSIONS_CACHE[factory.add_permission] = permission if request is None: request = get_current_request() if permission is not None and \ not IInteraction(request).check_permission(permission.id, container): raise NoPermissionToAdd(str(container), type_) constrains = IConstrainTypes(container, None) if constrains is not None: if not constrains.is_type_allowed(type_): raise NotAllowedContentType(str(container), type_) # We create the object with at least the ID obj = factory(id=id_) obj.__parent__ = container obj.__name__ = obj.id for key, value in kw.items(): setattr(obj, key, value) if request is None or 'OVERWRITE' not in request.headers: if await container.async_contains(obj.id): raise ConflictIdOnContainer(str(container), obj.id) obj.__new_marker__ = True await notify(BeforeObjectAddedEvent(obj, container, id_)) await container.async_set(obj.id, obj) return obj
def check_permission(self, permission_name): if permission_name is None: return True if permission_name not in self.permission_cache: permission = query_utility(IPermission, name=permission_name) if permission is None: self.permission_cache[permission_name] = True else: try: self.permission_cache[permission_name] = bool( IInteraction(self.request).check_permission( permission.id, self.context)) except NoInteraction: # not authenticated return False return self.permission_cache[permission_name]
async def items(context, request): try: page_size = int(request.query['page_size']) except Exception: page_size = 20 try: page = int(request.query['page']) except Exception: page = 1 txn = get_transaction(request) include = omit = [] if request.query.get('include'): include = request.query.get('include').split(',') if request.query.get('omit'): omit = request.query.get('omit').split(',') security = IInteraction(request) results = [] for key in await txn.get_page_of_keys(context._p_oid, page=page, page_size=page_size): ob = await context.async_get(key) if not security.check_permission('guillotina.ViewContent', ob): continue serializer = get_multi_adapter((ob, request), IResourceSerializeToJson) try: results.append(await serializer(include=include, omit=omit)) except TypeError: results.append(await serializer()) return { 'items': results, 'total': await context.async_len(), 'page': page, 'page_size': page_size }
async def get_user_info(context, request): """Return information about the logged in user. """ result = {} groups = set() principal = IInteraction(request).principal result[principal.id] = { 'roles': principal.roles, 'groups': principal.groups, 'properties': principal.properties } groups.update(principal.groups) group_search = getUtility(IGroups) result['groups'] = {} for group in groups: group_object = group_search.get_principal(group) result['groups'][group_object.id] = { 'roles': group_object.roles, 'groups': group_object.groups } return result
class Migrator: ''' Reindex/Migration... Reindex is going to behave much the same as migration would so we're using this for both cases most of the time... In order to do a *live* reindex, we need to follow these steps... 1. Create a next index - if already exists, fail - unless, "force" option provided 2. Put new mapping on next index 3. Mark that there is a new next index on the container object 4. All new index/delete operations are done on... - new index - and existing index - ... existing queries stay on current index 5. Copy existing index data over to the next index 6. Get a list of all existing doc ids on index 7. Take diff of existing mapping to new mapping 8. Crawl content - check if doc does not exist - make sure it wasn't added in the mean time - if it was, do an update with diff instead - record it - do complete index - if doc exist - if diff mapping exists - update fields in diff on doc - else, do nothing - remove for list of existing doc ids 9. Go through list of existing doc ids - double check not on container(query db) - delete doc if not in container - record it 10. Refresh db container ob 11. Point alias at next index 12. Delete old index ''' def __init__( self, utility, context, response=noop_response, force=False, log_details=False, memory_tracking=False, request=None, bulk_size=40, full=False, reindex_security=False, mapping_only=False, # noqa index_manager=None, children_only=False, lookup_index=False, cache=True): self.utility = utility self.conn = utility.conn self.context = context self.response = response self.force = force self.full = full self.log_details = log_details self.memory_tracking = memory_tracking self.bulk_size = bulk_size self.reindex_security = reindex_security self.children_only = children_only self.lookup_index = lookup_index if mapping_only and full: raise Exception( 'Can not do a full reindex and a mapping only migration') self.mapping_only = mapping_only if request is None: self.request = get_current_request() else: self.request = request if not cache: # make sure that we don't cache requests... self.request._txn._cache = DummyCache(self.request._txn) self.container = self.request.container if index_manager is None: self.index_manager = get_adapter(self.container, IIndexManager) else: self.index_manager = index_manager self.interaction = IInteraction(self.request) self.indexer = Indexer() self.batch = {} self.indexed = 0 self.processed = 0 self.missing = [] self.orphaned = [] self.existing = [] self.errors = [] self.mapping_diff = {} self.start_time = self.index_start_time = time.time() self.reindex_futures = [] self.status = 'started' self.active_task_id = None self.copied_docs = 0 self.work_index_name = None self.sub_indexes = [] def per_sec(self): return self.processed / (time.time() - self.index_start_time) async def create_next_index(self): async with managed_transaction(self.request, write=True, adopt_parent_txn=True) as txn: await txn.refresh(await self.index_manager.get_registry()) next_index_name = await self.index_manager.start_migration() if await self.conn.indices.exists(next_index_name): if self.force: # delete and recreate self.response.write('Clearing index') resp = await self.conn.indices.delete(next_index_name) assert resp['acknowledged'] await self.utility.create_index(next_index_name, self.index_manager) return next_index_name async def copy_to_next_index(self): conn_es = await self.conn.transport.get_connection() real_index_name = await self.index_manager.get_index_name() async with conn_es.session.post( join(str(conn_es.base_url), '_reindex'), params={'wait_for_completion': 'false'}, headers={'Content-Type': 'application/json'}, data=json.dumps({ "source": { "index": real_index_name, "size": 100 }, "dest": { "index": self.work_index_name } })) as resp: data = await resp.json() self.active_task_id = task_id = data['task'] while True: await asyncio.sleep(10) async with conn_es.session.get( join(str(conn_es.base_url), '_tasks', task_id), headers={'Content-Type': 'application/json'}) as resp: if resp.status in (400, 404): break data = await resp.json() if data['completed']: break status = data["task"]["status"] self.response.write( f'{status["created"]}/{status["total"]} - ' f'Copying data to new index. task id: {task_id}') self.copied_docs = status["created"] self.active_task_id = None response = data['response'] failures = response['failures'] if len(failures) > 0: failures = json.dumps(failures, sort_keys=True, indent=4, separators=(',', ': ')) self.response.write( f'Reindex encountered failures: {failures}') else: self.response.write( f'Finished copying to new index: {self.copied_docs}') async def get_all_uids(self): self.response.write('Retrieving existing doc ids') page_size = 3000 ids = [] index_name = await self.index_manager.get_index_name() result = await self.conn.search(index=index_name, scroll='2m', size=page_size, stored_fields='', _source=False, body={"sort": ["_doc"]}) ids.extend([r['_id'] for r in result['hits']['hits']]) scroll_id = result['_scroll_id'] while scroll_id: result = await self.utility.conn.scroll(scroll_id=scroll_id, scroll='2m') if len(result['hits']['hits']) == 0: break ids.extend([r['_id'] for r in result['hits']['hits']]) self.response.write(f'Retrieved {len(ids)} doc ids') scroll_id = result['_scroll_id'] self.response.write( f'Retrieved {len(ids)}. Copied {self.copied_docs} docs') return ids async def calculate_mapping_diff(self): ''' all we care about is new fields... Missing ones are ignored and we don't care about it. ''' next_mappings = await self.conn.indices.get_mapping( self.work_index_name) next_mappings = next_mappings[self.work_index_name]['mappings'] next_mappings = next_mappings[DOC_TYPE]['properties'] existing_index_name = await self.index_manager.get_real_index_name() try: existing_mappings = await self.conn.indices.get_mapping( existing_index_name) except elasticsearch.exceptions.NotFoundError: # allows us to upgrade when no index is present yet return next_mappings existing_mappings = existing_mappings[existing_index_name]['mappings'] existing_mappings = existing_mappings[DOC_TYPE]['properties'] new_definitions = {} for field_name, definition in next_mappings.items(): definition = _clean_mapping(definition) if (field_name not in existing_mappings or definition != _clean_mapping(existing_mappings[field_name])): # noqa new_definitions[field_name] = definition return new_definitions async def process_folder(self, ob): for key in await ob.async_keys(): try: item = await ob._p_jar.get_child(ob, key) except (KeyError, ModuleNotFoundError): continue if item is None: continue await self.process_object(item) del item del ob async def process_object(self, ob): ''' - check if doc does not exist - record it - do complete index - if doc exist - if diff mapping exists - update fields in diff on doc - else, do nothing - remove for list of existing doc ids ''' full = False if ob.uuid not in self.existing: self.missing.append(ob.uuid) full = True else: self.existing.remove(ob.uuid) await self.index_object(ob, full=full) self.processed += 1 if IIndexActive.providedBy(ob): self.sub_indexes.append(ob) else: if IFolder.providedBy(ob): await self.process_folder(ob) if not IContainer.providedBy(ob): try: del self.container.__gannotations__ except AttributeError: del self.container.__annotations__ del ob async def index_object(self, ob, full=False): batch_type = 'update' if self.reindex_security: try: data = ISecurityInfo(ob)() except TypeError: self.response.write(f'Could not index {ob}') return elif full or self.full: try: data = await ICatalogDataAdapter(ob)() except TypeError: self.response.write(f'Could not index {ob}') return batch_type = 'index' else: data = { # always need these... 'type_name': ob.type_name } for index_name in self.mapping_diff.keys(): val = await self.indexer.get_value(ob, index_name) if val is not None: data[index_name] = val if ob._p_serial: data['tid'] = ob._p_serial self.indexed += 1 self.batch[ob.uuid] = {'action': batch_type, 'data': data} if self.lookup_index: im = find_index_manager(ob) if im: self.batch[ob.uuid]['__index__'] = await im.get_index_name() if self.log_details: self.response.write( f'({self.processed} {int(self.per_sec())}) ' f'Object: {get_content_path(ob)}, ' f'Type: {batch_type}, Buffer: {len(self.batch)}') await self.attempt_flush() async def attempt_flush(self): if self.processed % 500 == 0: self.interaction.invalidate_cache() num, _, _ = gc.get_count() gc.collect() if self.memory_tracking: total_memory = round( resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0, 1) # noqa self.response.write( b'Memory usage: % 2.2f MB, cleaned: %d, total in-memory obs: %d' % ( # noqa total_memory, num, len(gc.get_objects()))) self.response.write(b'Indexing new batch, totals: (%d %d/sec)\n' % ( # noqa self.indexed, int(self.per_sec()), )) if len(self.batch) >= self.bulk_size: await notify( IndexProgress(self.request, self.context, self.processed, (len(self.existing) + len(self.missing)))) await self.flush() async def join_futures(self): for future in self.reindex_futures: if not future.done(): await asyncio.wait_for(future, None) self.reindex_futures = [] @backoff.on_exception( backoff.constant, (asyncio.TimeoutError, elasticsearch.exceptions.ConnectionTimeout), interval=1, max_tries=5) async def _index_batch(self, batch): bulk_data = [] for _id, payload in batch.items(): index = payload.pop('__index__', self.work_index_name) action_data = {'_index': index, '_id': _id} data = payload['data'] if payload['action'] == 'update': data = {'doc': data} action_data['_retry_on_conflict'] = 3 bulk_data.append({payload['action']: action_data}) if payload['action'] != 'delete': bulk_data.append(data) results = await self.utility.conn.bulk(index=self.work_index_name, doc_type=DOC_TYPE, body=bulk_data) if results['errors']: errors = [] for result in results['items']: for key, value in result.items(): if not isinstance(value, dict): continue if 'status' in value and value['status'] != 200: _id = value.get('_id') # retry conflict errors and thread pool exceeded errors if value['status'] in (409, 429): self.batch[_id] = batch[_id] elif value['status'] == 404: self.batch[_id] = batch[_id] self.batch[_id]['action'] = 'index' else: errors.append(f'{_id}: {value["status"]}') if len(errors) > 0: logger.warning(f'Error bulk putting: {errors}') async def flush(self): if len(self.batch) == 0: # nothing to flush return future = asyncio.ensure_future(self._index_batch(self.batch)) self.batch = {} self.reindex_futures.append(future) if len(self.reindex_futures) > 7: await self.join_futures() async def check_existing(self): ''' Go through self.existing and see why it wasn't processed ''' for uuid in self.existing: try: ob = await self.context._p_jar.get(uuid) except KeyError: ob = None if ob is None: try: self.batch[uuid] = {'action': 'delete', 'data': {}} await self.attempt_flush() # no longer present on db, this was orphaned self.orphaned.append(uuid) except aioelasticsearch.exceptions.NotFoundError: # it was deleted in the meantime so we're actually okay self.orphaned.append(uuid) else: # XXX this should not happen so log it. Maybe we'll try # doing something about it another time... self.errors.append({'type': 'unprocessed', 'uuid': uuid}) async def setup_next_index(self): self.response.write(b'Creating new index') async with get_migration_lock(await self.index_manager.get_index_name()): self.work_index_name = await self.create_next_index() return self.work_index_name async def cancel_migration(self): # canceling the migration, clearing index self.response.write('Canceling migration') async with managed_transaction(self.request, write=True, adopt_parent_txn=True): await self.index_manager.cancel_migration() self.response.write('Next index disabled') if self.active_task_id is not None: self.response.write('Canceling copy of index task') conn_es = await self.conn.transport.get_connection() async with conn_es.session.post( join(str(conn_es.base_url), '_tasks', self.active_task_id, '_cancel'), headers={'Content-Type': 'application/json'}): await asyncio.sleep(5) if self.work_index_name: self.response.write('Deleting new index') await self.conn.indices.delete(self.work_index_name) self.response.write('Migration canceled') async def run_migration(self): alias_index_name = await self.index_manager.get_index_name() existing_index = await self.index_manager.get_real_index_name() await self.setup_next_index() self.mapping_diff = await self.calculate_mapping_diff() diff = json.dumps(self.mapping_diff, sort_keys=True, indent=4, separators=(',', ': ')) self.response.write(f'Caculated mapping diff: {diff}') if not self.full: # if full, we're reindexing everything does not matter what # anyways, so skip self.response.write(f'Copying initial index {existing_index} ' f'into {self.work_index_name}') try: await self.copy_to_next_index() self.response.write('Copying initial index data finished') except elasticsearch.exceptions.NotFoundError: self.response.write('No initial index to copy to') if not self.mapping_only: try: self.existing = await self.get_all_uids() except elasticsearch.exceptions.NotFoundError: pass self.index_start_time = time.time() if self.children_only or IContainer.providedBy(self.context): await self.process_folder(self.context) # this is recursive else: await self.process_object(self.context) # this is recursive await self.check_existing() await self.flush() await self.join_futures() async with get_migration_lock(await self.index_manager.get_index_name()): self.response.write('Activating new index') async with managed_transaction(self.request, write=True, adopt_parent_txn=True): await self.index_manager.finish_migration() self.status = 'done' self.response.write(f'''Update alias({alias_index_name}): {existing_index} -> {self.work_index_name} ''') try: await self.conn.indices.update_aliases({ "actions": [{ "remove": { "alias": alias_index_name, "index": existing_index } }, { "add": { "alias": alias_index_name, "index": self.work_index_name } }] }) except elasticsearch.exceptions.NotFoundError: await self.conn.indices.update_aliases({ "actions": [{ "add": { "alias": alias_index_name, "index": self.work_index_name } }] }) try: await self.conn.indices.close(existing_index) await self.conn.indices.delete(existing_index) self.response.write('Old index deleted') except elasticsearch.exceptions.NotFoundError: pass if len(self.sub_indexes) > 0: self.response.write( f'Migrating sub indexes: {len(self.sub_indexes)}') for ob in self.sub_indexes: im = get_adapter(ob, IIndexManager) migrator = Migrator(self.utility, ob, response=self.response, force=self.force, log_details=self.log_details, memory_tracking=self.memory_tracking, request=self.request, bulk_size=self.bulk_size, full=self.full, reindex_security=self.reindex_security, mapping_only=self.mapping_only, index_manager=im, children_only=True) self.response.write(f'Migrating index for: {ob}') await migrator.run_migration()
def get_current_interaction(request): interaction = getattr(request, 'security', None) if IInteraction.providedBy(interaction): return interaction return Interaction(request)
async def duplicate(context, request): try: data = await request.json() except Exception: data = {} destination = data.get('destination') if destination is not None: destination_ob = await navigate_to(request.container, destination) if destination_ob is None: raise PreconditionFailed( context, 'Could not find destination object', ) else: destination_ob = context.__parent__ security = IInteraction(request) if not security.check_permission('guillotina.AddContent', destination_ob): raise PreconditionFailed( context, 'You do not have permission to add content to ' 'the destination object', ) if 'new_id' in data: new_id = data['new_id'] if await destination_ob.async_contains(new_id): raise PreconditionFailed( context, f'Destination already has object with the id {new_id}') else: count = 1 new_id = f'{context.id}-duplicate-{count}' while await destination_ob.async_contains(new_id): count += 1 new_id = f'{context.id}-duplicate-{count}' new_obj = await create_content_in_container( destination_ob, context.type_name, new_id, id=new_id, creators=context.creators, contributors=context.contributors) for key in context.__dict__.keys(): if key.startswith('__') or key.startswith('_BaseObject'): continue if key in ('id', ): continue new_obj.__dict__[key] = context.__dict__[key] new_obj.__acl__ = context.__acl__ for behavior in context.__behaviors__: new_obj.add_behavior(behavior) # need to copy annotation data as well... # load all annotations for context [b for b in await get_all_behaviors(context, load=True)] annotations_container = IAnnotations(new_obj) for anno_id, anno_data in context.__gannotations__.items(): new_anno_data = AnnotationData() for key, value in anno_data.items(): new_anno_data[key] = value await annotations_container.async_set(anno_id, new_anno_data) data['id'] = new_id await notify( ObjectDuplicatedEvent(new_obj, context, destination_ob, new_id, payload=data)) get = DefaultGET(new_obj, request) return await get()
async def __call__(self): ws = web.WebSocketResponse() await ws.prepare(self.request) async for msg in ws: if msg.tp == aiohttp.MsgType.text: message = ujson.loads(msg.data) if message['op'] == 'close': await ws.close() elif message['op'] == 'GET': method = app_settings['http_methods']['GET'] path = tuple(p for p in message['value'].split('/') if p) # avoid circular import from guillotina.traversal import do_traverse obj, tail = await do_traverse(self.request, self.request.site, path) traverse_to = None if tail and len(tail) == 1: view_name = tail[0] elif tail is None or len(tail) == 0: view_name = '' else: view_name = tail[0] traverse_to = tail[1:] permission = getUtility(IPermission, name='guillotina.AccessContent') allowed = IInteraction(self.request).check_permission( permission.id, obj) if not allowed: response = {'error': 'Not allowed'} ws.send_str(ujson.dumps(response)) try: view = queryMultiAdapter((obj, self.request), method, name=view_name) except AttributeError: view = None if traverse_to is not None: if view is None or not ITraversableView.providedBy( view): response = {'error': 'Not found'} ws.send_str(ujson.dumps(response)) else: try: view = await view.publish_traverse(traverse_to) except Exception as e: logger.error("Exception on view execution", exc_info=e) response = {'error': 'Not found'} ws.send_str(ujson.dumps(response)) view_result = await view() if isinstance(view_result, Response): view_result = view_result.response # Return the value ws.send_str(ujson.dumps(view_result)) # Wait for possible value futures_to_wait = self.request._futures.values() if futures_to_wait: await asyncio.gather(futures_to_wait) self.request._futures = {} else: await ws.close() elif msg.tp == aiohttp.MsgType.error: logger.debug( 'ws connection closed with exception {0:s}'.format( ws.exception())) logger.debug('websocket connection closed') return {}
async def can_i_do(context, request): if 'permission' not in request.query: raise TypeError('No permission param') permission = request.query['permission'] return IInteraction(request).check_permission(permission, context)