def get_access_where_clauses(self):
users = []
roles = []
request = get_current_request()
interaction = IInteraction(request)
for user in interaction.participations:
users.append(user.principal.id)
users.extend(user.principal.groups)
roles_dict = interaction.global_principal_roles(
user.principal.id,
user.principal.groups)
roles.extend([key for key, value in roles_dict.items()
if value])
clauses = []
if len(users) > 0:
clauses.append("json->'access_users' ?| array['{}']".format(
"','".join(users)
))
if len(roles) > 0:
clauses.append("json->'access_roles' ?| array['{}']".format(
"','".join(roles)
))
return '({})'.format(
' OR '.join(clauses)
)
async def _build_security_query(
self,
container,
query,
doc_type=None,
size=10,
request=None,
scroll=None):
if query is None:
query = {}
q = {}
# The users who has plone.AccessContent permission by prinperm
# The roles who has plone.AccessContent permission by roleperm
users = []
roles = []
if request is None:
request = get_current_request()
interaction = IInteraction(request)
for user in interaction.participations: # pylint: disable=E1133
users.append(user.principal.id)
users.extend(user.principal.groups)
roles_dict = interaction.global_principal_roles(
user.principal.id,
user.principal.groups)
roles.extend([key for key, value in roles_dict.items()
if value])
# We got all users and roles
# users: users and groups
should_list = [{'match': {'access_roles': x}} for x in roles]
should_list.extend([{'match': {'access_users': x}} for x in users])
permission_query = {
'query': {
'bool': {
'filter': {
'bool': {
'should': should_list,
'minimum_should_match': 1
}
}
}
}
}
query = merge_dicts(query, permission_query)
# query.update(permission_query)
q['body'] = query
q['size'] = size
if scroll:
q['scroll'] = scroll
logger.debug(q)
return q
async def resolve_uid(context, request):
uid = request.matchdict['uid']
ob = await get_object_by_oid(uid)
if ob is None:
return HTTPNotFound(content={'reason': f'Could not find uid: {uid}'})
interaction = IInteraction(request)
if interaction.check_permission('guillotina.AccessContent', ob):
return HTTPMovedPermanently(get_object_url(ob, request))
else:
# if a user doesn't have access to it, they shouldn't know anything about it
return HTTPNotFound(content={'reason': f'Could not find uid: {uid}'})
async def can_i_do(context, request):
if 'permission' not in request.query and 'permissions' not in request.query:
raise PreconditionFailed(context, 'No permission param')
interaction = IInteraction(request)
if 'permissions' in request.GET:
results = {}
for perm in request.GET['permissions'].split(','):
results[perm] = interaction.check_permission(perm, context)
return results
else:
return interaction.check_permission(request.GET['permission'], context)
def __init__(self,
utility,
context,
response=noop_response,
force=False,
log_details=False,
memory_tracking=False,
request=None,
bulk_size=40,
full=False,
reindex_security=False,
mapping_only=False):
self.utility = utility
self.conn = utility.conn
self.context = context
self.response = response
self.force = force
self.full = full
self.log_details = log_details
self.memory_tracking = memory_tracking
self.bulk_size = bulk_size
self.reindex_security = reindex_security
if mapping_only and full:
raise Exception(
'Can not do a full reindex and a mapping only migration')
self.mapping_only = mapping_only
if request is None:
self.request = get_current_request()
self.request._db_write_enabled = False
else:
self.request = request
# make sure that we don't cache requests...
self.request._txn._cache = DummyCache(self.request._txn)
self.container = self.request.container
self.interaction = IInteraction(self.request)
self.indexer = Indexer()
self.batch = {}
self.indexed = 0
self.processed = 0
self.missing = []
self.orphaned = []
self.existing = []
self.errors = []
self.mapping_diff = {}
self.start_time = self.index_start_time = time.time()
self.reindex_futures = []
self.status = 'started'
self.active_task_id = None
self.copied_docs = 0
self.work_index_name = None
async def available_actions(self, request):
security = IInteraction(request)
for action_name, action in self.actions.items():
add = False
if 'check_permission' in action and security.check_permission(
action['check_permission'], self.context):
add = True
elif 'check_permission' not in action:
add = True
if add:
yield action_name, action
def check_permission(self, permission_name):
if permission_name is None:
return True
if permission_name not in self.permission_cache:
permission = query_utility(IPermission, name=permission_name)
if permission is None:
self.permission_cache[permission_name] = True
else:
security = IInteraction(self.request)
self.permission_cache[permission_name] = bool(
security.check_permission(permission.id, self.context))
return self.permission_cache[permission_name]
def check_permission(self, permission_name):
if permission_name is None:
return True
if permission_name not in self.permission_cache:
permission = query_utility(IPermission,
name=permission_name)
if permission is None:
self.permission_cache[permission_name] = True
else:
security = IInteraction(self.request)
self.permission_cache[permission_name] = bool(
security.check_permission(permission.id, self.context))
return self.permission_cache[permission_name]
def get_principals_with_access_content(obj, request=None):
if obj is None:
return {}
if request is None:
request = get_current_request()
interaction = IInteraction(request)
roles = interaction.cached_roles(obj, 'guillotina.AccessContent', 'o')
result = []
all_roles = role.global_roles() + role.local_roles()
for r in roles.keys():
if r in all_roles:
result.append(r)
users = interaction.cached_principals(obj, result, 'guillotina.AccessContent', 'o')
return list(users.keys())
def get_roles_with_access_content(obj, request=None):
""" Return the roles that has access to the content that are global roles"""
if obj is None:
return []
if request is None:
request = get_current_request()
interaction = IInteraction(request)
roles = interaction.cached_roles(obj, 'guillotina.AccessContent', 'o')
result = []
all_roles = role.global_roles() + role.local_roles()
for r in roles.keys():
if r in all_roles:
result.append(r)
return result
def get_principals_with_access_content(obj, request=None):
if obj is None:
return {}
if request is None:
request = get_current_request()
interaction = IInteraction(request)
roles = interaction.cached_roles(obj, 'guillotina.AccessContent', 'o')
result = []
all_roles = role.global_roles() + role.local_roles()
for r in roles.keys():
if r in all_roles:
result.append(r)
users = interaction.cached_principals(obj, result, 'guillotina.AccessContent', 'o')
return list(users.keys())
def get_roles_with_access_content(obj, request=None):
""" Return the roles that has access to the content that are global roles"""
if obj is None:
return []
if request is None:
request = get_current_request()
interaction = IInteraction(request)
roles = interaction.cached_roles(obj, 'guillotina.AccessContent', 'o')
result = []
all_roles = role.global_roles() + role.local_roles()
for r in roles.keys():
if r in all_roles:
result.append(r)
return result
async def __call__(self):
self.interaction = IInteraction(self.request)
definition = copy.deepcopy(
app_settings["swagger"]["base_configuration"])
vhm = self.request.headers.get("X-VirtualHost-Monster")
if vhm:
parsed_url = urlparse(vhm)
definition["host"] = parsed_url.netloc
definition["schemes"] = [parsed_url.scheme]
definition["basePath"] = parsed_url.path
else:
definition["host"] = self.request.host
definition["schemes"] = [get_scheme(self.request)]
if 'version' not in definition['info']:
definition["info"]["version"] = pkg_resources.get_distribution(
"guillotina").version
api_defs = app_settings["api_definition"]
path = get_full_content_path(self.request, self.context)
for dotted_iface in api_defs.keys():
iface = resolve_dotted_name(dotted_iface)
if iface.providedBy(self.context):
iface_conf = api_defs[dotted_iface]
self.get_endpoints(iface_conf, path, definition["paths"])
definition["definitions"] = app_settings["json_schema_definitions"]
return definition
async def get_all_types(context, request):
result = []
base_url = IAbsoluteURL(context, request)()
constrains = IConstrainTypes(context, None)
for id, factory in FACTORY_CACHE.items():
add = True
if constrains is not None:
if not constrains.is_type_allowed(id):
add = False
if factory.add_permission:
if factory.add_permission in PERMISSIONS_CACHE:
permission = PERMISSIONS_CACHE[factory.add_permission]
else:
permission = query_utility(IPermission,
name=factory.add_permission)
PERMISSIONS_CACHE[factory.add_permission] = permission
if permission is not None and \
not IInteraction(request).check_permission(
permission.id, context):
add = False
if add:
result.append({
'@id': base_url + '/@types/' + id,
'addable': True,
'title': id
})
return result
async def create_content_in_container(parent: Folder,
type_: str,
id_: str,
request: IRequest = None,
check_security=True,
**kw) -> Resource:
"""Utility to create a content.
This method is the one to use to create content.
id_ can be None
:param parent: where to create content inside of
:param type_: content type to create
:param id_: id to give content in parent object
:param request: <optional>
:param check_security: be able to disable security checks
"""
factory = get_cached_factory(type_)
if check_security and factory.add_permission:
if factory.add_permission in PERMISSIONS_CACHE:
permission = PERMISSIONS_CACHE[factory.add_permission]
else:
permission = query_utility(IPermission,
name=factory.add_permission)
PERMISSIONS_CACHE[factory.add_permission] = permission
if request is None:
request = get_current_request()
if permission is not None and \
not IInteraction(request).check_permission(permission.id, parent):
raise NoPermissionToAdd(str(parent), type_)
constrains = IConstrainTypes(parent, None)
if constrains is not None:
if not constrains.is_type_allowed(type_):
raise NotAllowedContentType(str(parent), type_)
# We create the object with at least the ID
obj = factory(id=id_, parent=parent)
for key, value in kw.items():
if key == 'id':
# the factory sets id
continue
setattr(obj, key, value)
txn = getattr(parent, '_p_jar', None) or get_transaction()
if txn is None or not txn.storage.supports_unique_constraints:
# need to manually check unique constraints
if await parent.async_contains(obj.id):
raise ConflictIdOnContainer(f'Duplicate ID: {parent} -> {obj.id}')
obj.__new_marker__ = True
await notify(BeforeObjectAddedEvent(obj, parent, id_))
await parent.async_set(obj.id, obj)
return obj
async def handle_ws_request(self, ws, message):
method = app_settings['http_methods']['GET']
path = tuple(p for p in message['value'].split('/') if p)
# avoid circular import
from guillotina.traversal import do_traverse
obj, tail = await do_traverse(self.request, self.request.container,
path)
traverse_to = None
if tail and len(tail) == 1:
view_name = tail[0]
elif tail is None or len(tail) == 0:
view_name = ''
else:
view_name = tail[0]
traverse_to = tail[1:]
permission = getUtility(IPermission, name='guillotina.AccessContent')
allowed = IInteraction(self.request).check_permission(
permission.id, obj)
if not allowed:
response = {'error': 'Not allowed'}
ws.send_str(ujson.dumps(response))
try:
view = queryMultiAdapter((obj, self.request),
method,
name=view_name)
except AttributeError:
view = None
if traverse_to is not None:
if view is None or not ITraversableView.providedBy(view):
response = {'error': 'Not found'}
ws.send_str(ujson.dumps(response))
else:
try:
view = await view.publish_traverse(traverse_to)
except Exception as e:
logger.error("Exception on view execution", exc_info=e)
response = {'error': 'Not found'}
ws.send_str(ujson.dumps(response))
view_result = await view()
if isinstance(view_result, Response):
view_result = view_result.response
# Return the value
ws.send_str(ujson.dumps(view_result))
# Wait for possible value
futures_to_wait = self.request._futures.values()
if futures_to_wait:
await asyncio.gather(*list(futures_to_wait))
self.request._futures = {}
async def move(context, request):
try:
data = await request.json()
except Exception:
data = {}
destination = data.get('destination')
if destination is None:
destination_ob = context.__parent__
else:
try:
destination_ob = await navigate_to(request.container, destination)
except KeyError:
destination_ob = None
if destination_ob is None:
raise PreconditionFailed(context, 'Could not find destination object')
old_id = context.id
if 'new_id' in data:
new_id = data['new_id']
context.id = context.__name__ = new_id
else:
new_id = context.id
security = IInteraction(request)
if not security.check_permission('guillotina.AddContent', destination_ob):
raise PreconditionFailed(
context, 'You do not have permission to add content to the '
'destination object')
if await destination_ob.async_contains(new_id):
raise PreconditionFailed(
context, f'Destination already has object with the id {new_id}')
original_parent = context.__parent__
txn = get_transaction(request)
cache_keys = txn._cache.get_cache_keys(context, 'deleted')
data['id'] = new_id
await notify(
BeforeObjectMovedEvent(context, original_parent, old_id, destination_ob,
new_id, payload=data))
context.__parent__ = destination_ob
context._p_register()
await notify(
ObjectMovedEvent(context, original_parent, old_id, destination_ob,
new_id, payload=data))
cache_keys += txn._cache.get_cache_keys(context, 'added')
await txn._cache.delete_all(cache_keys)
absolute_url = query_multi_adapter((context, request), IAbsoluteURL)
return {
'@url': absolute_url()
}
async def do_action(self, request, action, comments):
available_actions = self.actions
if action not in available_actions:
raise KeyError('Unavailable action')
action_def = available_actions[action]
security = IInteraction(request)
if 'check_permission' in action_def and not security.check_permission(
action_def['check_permission'], self.context):
raise HTTPUnauthorized()
# Change permission
new_state = action_def['to']
if 'set_permission' in self.states[new_state]:
await apply_sharing(self.context,
self.states[new_state]['set_permission'])
# Write history
user = get_authenticated_user_id(request)
history = {
'actor': user,
'comments': comments,
'time': datetime.datetime.now(),
'title': action_def['title'],
'type': 'workflow',
'data': {
'action': action,
'review_state': new_state,
}
}
cms_behavior = ICMSBehavior(self.context)
await cms_behavior.load()
cms_behavior.review_state = new_state
cms_behavior.history.append(history)
cms_behavior._p_register()
await notify(
WorkflowChangedEvent(self.context, self, action, comments))
return history
async def __call__(self):
result = await super(SerializeFolderToJson, self).__call__()
security = IInteraction(self.request)
length = await self.context.async_len()
if length > MAX_ALLOWED or length == 0:
result['items'] = []
else:
result['items'] = []
async for ident, member in self.context.async_items():
if not ident.startswith('_') and bool(
security.check_permission('guillotina.AccessContent',
member)):
result['items'].append(await getMultiAdapter(
(member, self.request),
IResourceSerializeToJsonSummary)())
result['length'] = length
return result
def get_current_interaction(request):
"""
Cache IInteraction on the request object because the request object
is where we start adding principals
"""
interaction = getattr(request, 'security', None)
if IInteraction.providedBy(interaction):
return interaction
interaction = Interaction(request)
request.security = interaction
return interaction
async def __call__(self, include=[], omit=[]):
result = await super(SerializeFolderToJson, self).__call__(include=include, omit=omit)
security = IInteraction(self.request)
length = await self.context.async_len()
if length > MAX_ALLOWED or length == 0:
result['items'] = []
else:
result['items'] = []
async for ident, member in self.context.async_items(suppress_events=True):
if not ident.startswith('_') and bool(
security.check_permission(
'guillotina.AccessContent', member)):
result['items'].append(
await get_multi_adapter(
(member, self.request),
IResourceSerializeToJsonSummary)())
result['length'] = length
return result
async def __call__(self):
result = {'databases': [], 'static_file': [], 'static_directory': []}
allowed = IInteraction(self.request).check_permission(
'guillotina.GetDatabases', self.application)
for x in self.application._dbs.keys():
if IDatabase.providedBy(self.application._dbs[x]) and allowed:
result['databases'].append(x)
if IStaticFile.providedBy(self.application._dbs[x]):
result['static_file'].append(x)
if IStaticDirectory.providedBy(self.application._dbs[x]):
result['static_directory'].append(x)
return result
def _invalidated_interaction_cache(self):
# Invalidate this threads interaction cache
try:
request = get_current_request()
except RequestNotFound:
return
interaction = IInteraction(request)
if interaction is not None:
try:
invalidate_cache = interaction.invalidate_cache
except AttributeError:
pass
else:
invalidate_cache()
async def create_content_in_container(container,
type_,
id_,
request=None,
check_security=True,
**kw):
"""Utility to create a content.
This method is the one to use to create content.
id_ can be None
"""
factory = get_cached_factory(type_)
if check_security and factory.add_permission:
if factory.add_permission in PERMISSIONS_CACHE:
permission = PERMISSIONS_CACHE[factory.add_permission]
else:
permission = query_utility(IPermission,
name=factory.add_permission)
PERMISSIONS_CACHE[factory.add_permission] = permission
if request is None:
request = get_current_request()
if permission is not None and \
not IInteraction(request).check_permission(permission.id, container):
raise NoPermissionToAdd(str(container), type_)
constrains = IConstrainTypes(container, None)
if constrains is not None:
if not constrains.is_type_allowed(type_):
raise NotAllowedContentType(str(container), type_)
# We create the object with at least the ID
obj = factory(id=id_, parent=container)
for key, value in kw.items():
setattr(obj, key, value)
txn = getattr(container, '_p_jar', None) or get_transaction()
if txn is None or not txn.storage.supports_unique_constraints:
# need to manually check unique constraints
if await container.async_contains(obj.id):
raise ConflictIdOnContainer(
f'Duplicate ID: {container} -> {obj.id}')
obj.__new_marker__ = True
await notify(BeforeObjectAddedEvent(obj, container, id_))
await container.async_set(obj.id, obj)
return obj
async def build_security_query(container, request=None):
# The users who has plone.AccessContent permission by prinperm
# The roles who has plone.AccessContent permission by roleperm
users = []
roles = []
if request is None:
request = get_current_request()
interaction = IInteraction(request)
for user in interaction.participations: # pylint: disable=E1133
users.append(user.principal.id)
users.extend(user.principal.groups)
roles_dict = interaction.global_principal_roles(
user.principal.id,
user.principal.groups)
roles.extend([key for key, value in roles_dict.items()
if value])
# We got all users and roles
# users: users and groups
should_list = [{'match': {'access_roles': x}} for x in roles]
should_list.extend([{'match': {'access_users': x}} for x in users])
return {
'query': {
'bool': {
'filter': {
'bool': {
'should': should_list,
'minimum_should_match': 1
}
}
}
}
}
async def create_content_in_container(container,
type_,
id_,
request=None,
**kw):
"""Utility to create a content.
This method is the one to use to create content.
id_ can be None
"""
factory = get_cached_factory(type_)
if factory.add_permission:
if factory.add_permission in PERMISSIONS_CACHE:
permission = PERMISSIONS_CACHE[factory.add_permission]
else:
permission = query_utility(IPermission,
name=factory.add_permission)
PERMISSIONS_CACHE[factory.add_permission] = permission
if request is None:
request = get_current_request()
if permission is not None and \
not IInteraction(request).check_permission(permission.id, container):
raise NoPermissionToAdd(str(container), type_)
constrains = IConstrainTypes(container, None)
if constrains is not None:
if not constrains.is_type_allowed(type_):
raise NotAllowedContentType(str(container), type_)
# We create the object with at least the ID
obj = factory(id=id_)
obj.__parent__ = container
obj.__name__ = obj.id
for key, value in kw.items():
setattr(obj, key, value)
if request is None or 'OVERWRITE' not in request.headers:
if await container.async_contains(obj.id):
raise ConflictIdOnContainer(str(container), obj.id)
obj.__new_marker__ = True
await notify(BeforeObjectAddedEvent(obj, container, id_))
await container.async_set(obj.id, obj)
return obj
def check_permission(self, permission_name):
if permission_name is None:
return True
if permission_name not in self.permission_cache:
permission = query_utility(IPermission, name=permission_name)
if permission is None:
self.permission_cache[permission_name] = True
else:
try:
self.permission_cache[permission_name] = bool(
IInteraction(self.request).check_permission(
permission.id, self.context))
except NoInteraction:
# not authenticated
return False
return self.permission_cache[permission_name]
async def items(context, request):
try:
page_size = int(request.query['page_size'])
except Exception:
page_size = 20
try:
page = int(request.query['page'])
except Exception:
page = 1
txn = get_transaction(request)
include = omit = []
if request.query.get('include'):
include = request.query.get('include').split(',')
if request.query.get('omit'):
omit = request.query.get('omit').split(',')
security = IInteraction(request)
results = []
for key in await txn.get_page_of_keys(context._p_oid,
page=page,
page_size=page_size):
ob = await context.async_get(key)
if not security.check_permission('guillotina.ViewContent', ob):
continue
serializer = get_multi_adapter((ob, request), IResourceSerializeToJson)
try:
results.append(await serializer(include=include, omit=omit))
except TypeError:
results.append(await serializer())
return {
'items': results,
'total': await context.async_len(),
'page': page,
'page_size': page_size
}
async def get_user_info(context, request):
"""Return information about the logged in user.
"""
result = {}
groups = set()
principal = IInteraction(request).principal
result[principal.id] = {
'roles': principal.roles,
'groups': principal.groups,
'properties': principal.properties
}
groups.update(principal.groups)
group_search = getUtility(IGroups)
result['groups'] = {}
for group in groups:
group_object = group_search.get_principal(group)
result['groups'][group_object.id] = {
'roles': group_object.roles,
'groups': group_object.groups
}
return result
class Migrator:
'''
Reindex/Migration...
Reindex is going to behave much the same as migration would so we're using
this for both cases most of the time...
In order to do a *live* reindex, we need to follow these steps...
1. Create a next index
- if already exists, fail
- unless, "force" option provided
2. Put new mapping on next index
3. Mark that there is a new next index on the container object
4. All new index/delete operations are done on...
- new index
- and existing index
- ... existing queries stay on current index
5. Copy existing index data over to the next index
6. Get a list of all existing doc ids on index
7. Take diff of existing mapping to new mapping
8. Crawl content
- check if doc does not exist
- make sure it wasn't added in the mean time
- if it was, do an update with diff instead
- record it
- do complete index
- if doc exist
- if diff mapping exists
- update fields in diff on doc
- else, do nothing
- remove for list of existing doc ids
9. Go through list of existing doc ids
- double check not on container(query db)
- delete doc if not in container
- record it
10. Refresh db container ob
11. Point alias at next index
12. Delete old index
'''
def __init__(
self,
utility,
context,
response=noop_response,
force=False,
log_details=False,
memory_tracking=False,
request=None,
bulk_size=40,
full=False,
reindex_security=False,
mapping_only=False, # noqa
index_manager=None,
children_only=False,
lookup_index=False,
cache=True):
self.utility = utility
self.conn = utility.conn
self.context = context
self.response = response
self.force = force
self.full = full
self.log_details = log_details
self.memory_tracking = memory_tracking
self.bulk_size = bulk_size
self.reindex_security = reindex_security
self.children_only = children_only
self.lookup_index = lookup_index
if mapping_only and full:
raise Exception(
'Can not do a full reindex and a mapping only migration')
self.mapping_only = mapping_only
if request is None:
self.request = get_current_request()
else:
self.request = request
if not cache:
# make sure that we don't cache requests...
self.request._txn._cache = DummyCache(self.request._txn)
self.container = self.request.container
if index_manager is None:
self.index_manager = get_adapter(self.container, IIndexManager)
else:
self.index_manager = index_manager
self.interaction = IInteraction(self.request)
self.indexer = Indexer()
self.batch = {}
self.indexed = 0
self.processed = 0
self.missing = []
self.orphaned = []
self.existing = []
self.errors = []
self.mapping_diff = {}
self.start_time = self.index_start_time = time.time()
self.reindex_futures = []
self.status = 'started'
self.active_task_id = None
self.copied_docs = 0
self.work_index_name = None
self.sub_indexes = []
def per_sec(self):
return self.processed / (time.time() - self.index_start_time)
async def create_next_index(self):
async with managed_transaction(self.request,
write=True,
adopt_parent_txn=True) as txn:
await txn.refresh(await self.index_manager.get_registry())
next_index_name = await self.index_manager.start_migration()
if await self.conn.indices.exists(next_index_name):
if self.force:
# delete and recreate
self.response.write('Clearing index')
resp = await self.conn.indices.delete(next_index_name)
assert resp['acknowledged']
await self.utility.create_index(next_index_name, self.index_manager)
return next_index_name
async def copy_to_next_index(self):
conn_es = await self.conn.transport.get_connection()
real_index_name = await self.index_manager.get_index_name()
async with conn_es.session.post(
join(str(conn_es.base_url), '_reindex'),
params={'wait_for_completion': 'false'},
headers={'Content-Type': 'application/json'},
data=json.dumps({
"source": {
"index": real_index_name,
"size": 100
},
"dest": {
"index": self.work_index_name
}
})) as resp:
data = await resp.json()
self.active_task_id = task_id = data['task']
while True:
await asyncio.sleep(10)
async with conn_es.session.get(
join(str(conn_es.base_url), '_tasks', task_id),
headers={'Content-Type': 'application/json'}) as resp:
if resp.status in (400, 404):
break
data = await resp.json()
if data['completed']:
break
status = data["task"]["status"]
self.response.write(
f'{status["created"]}/{status["total"]} - '
f'Copying data to new index. task id: {task_id}')
self.copied_docs = status["created"]
self.active_task_id = None
response = data['response']
failures = response['failures']
if len(failures) > 0:
failures = json.dumps(failures,
sort_keys=True,
indent=4,
separators=(',', ': '))
self.response.write(
f'Reindex encountered failures: {failures}')
else:
self.response.write(
f'Finished copying to new index: {self.copied_docs}')
async def get_all_uids(self):
self.response.write('Retrieving existing doc ids')
page_size = 3000
ids = []
index_name = await self.index_manager.get_index_name()
result = await self.conn.search(index=index_name,
scroll='2m',
size=page_size,
stored_fields='',
_source=False,
body={"sort": ["_doc"]})
ids.extend([r['_id'] for r in result['hits']['hits']])
scroll_id = result['_scroll_id']
while scroll_id:
result = await self.utility.conn.scroll(scroll_id=scroll_id,
scroll='2m')
if len(result['hits']['hits']) == 0:
break
ids.extend([r['_id'] for r in result['hits']['hits']])
self.response.write(f'Retrieved {len(ids)} doc ids')
scroll_id = result['_scroll_id']
self.response.write(
f'Retrieved {len(ids)}. Copied {self.copied_docs} docs')
return ids
async def calculate_mapping_diff(self):
'''
all we care about is new fields...
Missing ones are ignored and we don't care about it.
'''
next_mappings = await self.conn.indices.get_mapping(
self.work_index_name)
next_mappings = next_mappings[self.work_index_name]['mappings']
next_mappings = next_mappings[DOC_TYPE]['properties']
existing_index_name = await self.index_manager.get_real_index_name()
try:
existing_mappings = await self.conn.indices.get_mapping(
existing_index_name)
except elasticsearch.exceptions.NotFoundError:
# allows us to upgrade when no index is present yet
return next_mappings
existing_mappings = existing_mappings[existing_index_name]['mappings']
existing_mappings = existing_mappings[DOC_TYPE]['properties']
new_definitions = {}
for field_name, definition in next_mappings.items():
definition = _clean_mapping(definition)
if (field_name not in existing_mappings or definition !=
_clean_mapping(existing_mappings[field_name])): # noqa
new_definitions[field_name] = definition
return new_definitions
async def process_folder(self, ob):
for key in await ob.async_keys():
try:
item = await ob._p_jar.get_child(ob, key)
except (KeyError, ModuleNotFoundError):
continue
if item is None:
continue
await self.process_object(item)
del item
del ob
async def process_object(self, ob):
'''
- check if doc does not exist
- record it
- do complete index
- if doc exist
- if diff mapping exists
- update fields in diff on doc
- else, do nothing
- remove for list of existing doc ids
'''
full = False
if ob.uuid not in self.existing:
self.missing.append(ob.uuid)
full = True
else:
self.existing.remove(ob.uuid)
await self.index_object(ob, full=full)
self.processed += 1
if IIndexActive.providedBy(ob):
self.sub_indexes.append(ob)
else:
if IFolder.providedBy(ob):
await self.process_folder(ob)
if not IContainer.providedBy(ob):
try:
del self.container.__gannotations__
except AttributeError:
del self.container.__annotations__
del ob
async def index_object(self, ob, full=False):
batch_type = 'update'
if self.reindex_security:
try:
data = ISecurityInfo(ob)()
except TypeError:
self.response.write(f'Could not index {ob}')
return
elif full or self.full:
try:
data = await ICatalogDataAdapter(ob)()
except TypeError:
self.response.write(f'Could not index {ob}')
return
batch_type = 'index'
else:
data = {
# always need these...
'type_name': ob.type_name
}
for index_name in self.mapping_diff.keys():
val = await self.indexer.get_value(ob, index_name)
if val is not None:
data[index_name] = val
if ob._p_serial:
data['tid'] = ob._p_serial
self.indexed += 1
self.batch[ob.uuid] = {'action': batch_type, 'data': data}
if self.lookup_index:
im = find_index_manager(ob)
if im:
self.batch[ob.uuid]['__index__'] = await im.get_index_name()
if self.log_details:
self.response.write(
f'({self.processed} {int(self.per_sec())}) '
f'Object: {get_content_path(ob)}, '
f'Type: {batch_type}, Buffer: {len(self.batch)}')
await self.attempt_flush()
async def attempt_flush(self):
if self.processed % 500 == 0:
self.interaction.invalidate_cache()
num, _, _ = gc.get_count()
gc.collect()
if self.memory_tracking:
total_memory = round(
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss /
1024.0, 1) # noqa
self.response.write(
b'Memory usage: % 2.2f MB, cleaned: %d, total in-memory obs: %d'
% ( # noqa
total_memory, num, len(gc.get_objects())))
self.response.write(b'Indexing new batch, totals: (%d %d/sec)\n' %
( # noqa
self.indexed,
int(self.per_sec()),
))
if len(self.batch) >= self.bulk_size:
await notify(
IndexProgress(self.request, self.context, self.processed,
(len(self.existing) + len(self.missing))))
await self.flush()
async def join_futures(self):
for future in self.reindex_futures:
if not future.done():
await asyncio.wait_for(future, None)
self.reindex_futures = []
@backoff.on_exception(
backoff.constant,
(asyncio.TimeoutError, elasticsearch.exceptions.ConnectionTimeout),
interval=1,
max_tries=5)
async def _index_batch(self, batch):
bulk_data = []
for _id, payload in batch.items():
index = payload.pop('__index__', self.work_index_name)
action_data = {'_index': index, '_id': _id}
data = payload['data']
if payload['action'] == 'update':
data = {'doc': data}
action_data['_retry_on_conflict'] = 3
bulk_data.append({payload['action']: action_data})
if payload['action'] != 'delete':
bulk_data.append(data)
results = await self.utility.conn.bulk(index=self.work_index_name,
doc_type=DOC_TYPE,
body=bulk_data)
if results['errors']:
errors = []
for result in results['items']:
for key, value in result.items():
if not isinstance(value, dict):
continue
if 'status' in value and value['status'] != 200:
_id = value.get('_id')
# retry conflict errors and thread pool exceeded errors
if value['status'] in (409, 429):
self.batch[_id] = batch[_id]
elif value['status'] == 404:
self.batch[_id] = batch[_id]
self.batch[_id]['action'] = 'index'
else:
errors.append(f'{_id}: {value["status"]}')
if len(errors) > 0:
logger.warning(f'Error bulk putting: {errors}')
async def flush(self):
if len(self.batch) == 0:
# nothing to flush
return
future = asyncio.ensure_future(self._index_batch(self.batch))
self.batch = {}
self.reindex_futures.append(future)
if len(self.reindex_futures) > 7:
await self.join_futures()
async def check_existing(self):
'''
Go through self.existing and see why it wasn't processed
'''
for uuid in self.existing:
try:
ob = await self.context._p_jar.get(uuid)
except KeyError:
ob = None
if ob is None:
try:
self.batch[uuid] = {'action': 'delete', 'data': {}}
await self.attempt_flush()
# no longer present on db, this was orphaned
self.orphaned.append(uuid)
except aioelasticsearch.exceptions.NotFoundError:
# it was deleted in the meantime so we're actually okay
self.orphaned.append(uuid)
else:
# XXX this should not happen so log it. Maybe we'll try
# doing something about it another time...
self.errors.append({'type': 'unprocessed', 'uuid': uuid})
async def setup_next_index(self):
self.response.write(b'Creating new index')
async with get_migration_lock(await
self.index_manager.get_index_name()):
self.work_index_name = await self.create_next_index()
return self.work_index_name
async def cancel_migration(self):
# canceling the migration, clearing index
self.response.write('Canceling migration')
async with managed_transaction(self.request,
write=True,
adopt_parent_txn=True):
await self.index_manager.cancel_migration()
self.response.write('Next index disabled')
if self.active_task_id is not None:
self.response.write('Canceling copy of index task')
conn_es = await self.conn.transport.get_connection()
async with conn_es.session.post(
join(str(conn_es.base_url), '_tasks', self.active_task_id,
'_cancel'),
headers={'Content-Type': 'application/json'}):
await asyncio.sleep(5)
if self.work_index_name:
self.response.write('Deleting new index')
await self.conn.indices.delete(self.work_index_name)
self.response.write('Migration canceled')
async def run_migration(self):
alias_index_name = await self.index_manager.get_index_name()
existing_index = await self.index_manager.get_real_index_name()
await self.setup_next_index()
self.mapping_diff = await self.calculate_mapping_diff()
diff = json.dumps(self.mapping_diff,
sort_keys=True,
indent=4,
separators=(',', ': '))
self.response.write(f'Caculated mapping diff: {diff}')
if not self.full:
# if full, we're reindexing everything does not matter what
# anyways, so skip
self.response.write(f'Copying initial index {existing_index} '
f'into {self.work_index_name}')
try:
await self.copy_to_next_index()
self.response.write('Copying initial index data finished')
except elasticsearch.exceptions.NotFoundError:
self.response.write('No initial index to copy to')
if not self.mapping_only:
try:
self.existing = await self.get_all_uids()
except elasticsearch.exceptions.NotFoundError:
pass
self.index_start_time = time.time()
if self.children_only or IContainer.providedBy(self.context):
await self.process_folder(self.context) # this is recursive
else:
await self.process_object(self.context) # this is recursive
await self.check_existing()
await self.flush()
await self.join_futures()
async with get_migration_lock(await
self.index_manager.get_index_name()):
self.response.write('Activating new index')
async with managed_transaction(self.request,
write=True,
adopt_parent_txn=True):
await self.index_manager.finish_migration()
self.status = 'done'
self.response.write(f'''Update alias({alias_index_name}):
{existing_index} -> {self.work_index_name}
''')
try:
await self.conn.indices.update_aliases({
"actions": [{
"remove": {
"alias": alias_index_name,
"index": existing_index
}
}, {
"add": {
"alias": alias_index_name,
"index": self.work_index_name
}
}]
})
except elasticsearch.exceptions.NotFoundError:
await self.conn.indices.update_aliases({
"actions": [{
"add": {
"alias": alias_index_name,
"index": self.work_index_name
}
}]
})
try:
await self.conn.indices.close(existing_index)
await self.conn.indices.delete(existing_index)
self.response.write('Old index deleted')
except elasticsearch.exceptions.NotFoundError:
pass
if len(self.sub_indexes) > 0:
self.response.write(
f'Migrating sub indexes: {len(self.sub_indexes)}')
for ob in self.sub_indexes:
im = get_adapter(ob, IIndexManager)
migrator = Migrator(self.utility,
ob,
response=self.response,
force=self.force,
log_details=self.log_details,
memory_tracking=self.memory_tracking,
request=self.request,
bulk_size=self.bulk_size,
full=self.full,
reindex_security=self.reindex_security,
mapping_only=self.mapping_only,
index_manager=im,
children_only=True)
self.response.write(f'Migrating index for: {ob}')
await migrator.run_migration()
def get_current_interaction(request):
interaction = getattr(request, 'security', None)
if IInteraction.providedBy(interaction):
return interaction
return Interaction(request)
async def duplicate(context, request):
try:
data = await request.json()
except Exception:
data = {}
destination = data.get('destination')
if destination is not None:
destination_ob = await navigate_to(request.container, destination)
if destination_ob is None:
raise PreconditionFailed(
context,
'Could not find destination object',
)
else:
destination_ob = context.__parent__
security = IInteraction(request)
if not security.check_permission('guillotina.AddContent', destination_ob):
raise PreconditionFailed(
context,
'You do not have permission to add content to '
'the destination object',
)
if 'new_id' in data:
new_id = data['new_id']
if await destination_ob.async_contains(new_id):
raise PreconditionFailed(
context,
f'Destination already has object with the id {new_id}')
else:
count = 1
new_id = f'{context.id}-duplicate-{count}'
while await destination_ob.async_contains(new_id):
count += 1
new_id = f'{context.id}-duplicate-{count}'
new_obj = await create_content_in_container(
destination_ob,
context.type_name,
new_id,
id=new_id,
creators=context.creators,
contributors=context.contributors)
for key in context.__dict__.keys():
if key.startswith('__') or key.startswith('_BaseObject'):
continue
if key in ('id', ):
continue
new_obj.__dict__[key] = context.__dict__[key]
new_obj.__acl__ = context.__acl__
for behavior in context.__behaviors__:
new_obj.add_behavior(behavior)
# need to copy annotation data as well...
# load all annotations for context
[b for b in await get_all_behaviors(context, load=True)]
annotations_container = IAnnotations(new_obj)
for anno_id, anno_data in context.__gannotations__.items():
new_anno_data = AnnotationData()
for key, value in anno_data.items():
new_anno_data[key] = value
await annotations_container.async_set(anno_id, new_anno_data)
data['id'] = new_id
await notify(
ObjectDuplicatedEvent(new_obj,
context,
destination_ob,
new_id,
payload=data))
get = DefaultGET(new_obj, request)
return await get()
async def __call__(self):
ws = web.WebSocketResponse()
await ws.prepare(self.request)
async for msg in ws:
if msg.tp == aiohttp.MsgType.text:
message = ujson.loads(msg.data)
if message['op'] == 'close':
await ws.close()
elif message['op'] == 'GET':
method = app_settings['http_methods']['GET']
path = tuple(p for p in message['value'].split('/') if p)
# avoid circular import
from guillotina.traversal import do_traverse
obj, tail = await do_traverse(self.request,
self.request.site, path)
traverse_to = None
if tail and len(tail) == 1:
view_name = tail[0]
elif tail is None or len(tail) == 0:
view_name = ''
else:
view_name = tail[0]
traverse_to = tail[1:]
permission = getUtility(IPermission,
name='guillotina.AccessContent')
allowed = IInteraction(self.request).check_permission(
permission.id, obj)
if not allowed:
response = {'error': 'Not allowed'}
ws.send_str(ujson.dumps(response))
try:
view = queryMultiAdapter((obj, self.request),
method,
name=view_name)
except AttributeError:
view = None
if traverse_to is not None:
if view is None or not ITraversableView.providedBy(
view):
response = {'error': 'Not found'}
ws.send_str(ujson.dumps(response))
else:
try:
view = await view.publish_traverse(traverse_to)
except Exception as e:
logger.error("Exception on view execution",
exc_info=e)
response = {'error': 'Not found'}
ws.send_str(ujson.dumps(response))
view_result = await view()
if isinstance(view_result, Response):
view_result = view_result.response
# Return the value
ws.send_str(ujson.dumps(view_result))
# Wait for possible value
futures_to_wait = self.request._futures.values()
if futures_to_wait:
await asyncio.gather(futures_to_wait)
self.request._futures = {}
else:
await ws.close()
elif msg.tp == aiohttp.MsgType.error:
logger.debug(
'ws connection closed with exception {0:s}'.format(
ws.exception()))
logger.debug('websocket connection closed')
return {}
async def can_i_do(context, request):
if 'permission' not in request.query:
raise TypeError('No permission param')
permission = request.query['permission']
return IInteraction(request).check_permission(permission, context)