def get_principal(self, ident):
request = get_current_request()
if not hasattr(request, '_cache_groups'):
request._cache_groups = {}
if ident not in request._cache_groups.keys():
request._cache_groups[ident] = PloneGroup(request, ident)
return request._cache_groups[ident]
async def add_security_query(self, query, request=None):
users = []
roles = []
if request is None:
request = get_current_request()
interaction = IInteraction(request)
for user in interaction.participations:
users.append(user.principal.id)
users.extend(user.principal.groups)
roles_dict = interaction.global_principal_roles(
user.principal.id, user.principal.groups)
roles.extend([key for key, value in roles_dict.items() if value])
# We got all users and roles
# users: users and groups
should_list = [{'match': {'access_roles': x}} for x in roles]
should_list.extend([{'match': {'access_users': x}} for x in users])
if 'query' not in query:
query['query'] = {}
if 'bool' not in query['query']:
query['query']['bool'] = {}
if 'filter' not in query['query']['bool']:
query['query']['bool']['filter'] = {}
query['query']['bool']['filter'] = {
'bool': {
'should': should_list,
'minimum_number_should_match': 1
}
}
return query
async def unindex_all_childs(self, resource, response=None, future=True):
if type(resource) is str:
path = resource
depth = path.count('/') + 1
else:
path = get_content_path(resource)
depth = get_content_depth(resource)
depth += 1
if response is not None:
response.write(b'Removing all childs of %s' % path.encode('utf-8'))
request = get_current_request()
index_name = self.get_index_name(request.site)
path_query = {
'query': {
'bool': {
'must': [
]
}
}
}
if path != '/':
path_query['query']['bool']['must'].append({
'term':
{'path': path}
})
path_query['query']['bool']['must'].append({
'range':
{'depth': {'gte': depth}}
})
logger.warn('Going to delete all in %s %d' % (path, depth))
if future:
_id = 'unindex_all_childs-' + uuid.uuid4().hex
request._futures.update({_id: self.call_unindex_all_childs(index_name, path_query)})
else:
await self.call_unindex_all_childs(index_name, path_query)
async def query(
self, site, query,
doc_type=None, size=10, request=None):
"""
transform into query...
right now, it's just passing through into elasticsearch
"""
t1 = time.time()
if request is None:
request = get_current_request()
q = await self._build_security_query(
site, query, doc_type, size, request)
result = await self.conn.search(**q)
items = []
site_url = IAbsoluteURL(site, request)()
for item in result['hits']['hits']:
data = item['_source']
data.update({
'@absolute_url': site_url + data.get('path', ''),
'@type': data.get('portal_type'),
})
items.append(data)
final = {
'items_count': result['hits']['total'],
'member': items
}
if 'aggregations' in result:
final['aggregations'] = result['aggregations']
if 'suggest' in result:
final['suggest'] = result['suggest']
tdif = t1 - time.time()
print('Time ELASTIC %f' % tdif)
await notify(SearchDoneEvent(
query, result['hits']['total'], request, tdif))
return final
def getPrincipal(self, ident):
request = get_current_request()
if not hasattr(request, '_cache_groups'):
request._cache_groups = {}
if ident not in request._cache_groups.keys():
request._cache_groups[ident] = PloneGroup(request, ident)
return request._cache_groups[ident]
def get_hook():
request = get_current_request()
try:
site = request.site
search = queryUtility(ICatalogUtility)
except (AttributeError, KeyError):
return
if not search:
return # no search configured
try:
trns = tm(request).get()
except RequestNotFound:
trns = transaction.get()
hook = None
for _hook in trns._after_commit:
if isinstance(_hook[0], CommitHook):
hook = _hook[0]
break
if hook is None:
hook = CommitHook(site, request)
trns.addAfterCommitHook(hook)
return hook
async def initUpload(self, context):
"""Init an upload.
self._uload_file_id : temporal url to image beeing uploaded
self._resumable_uri : uri to resumable upload
self._uri : finished uploaded image
"""
util = getUtility(IS3BlobStore)
request = get_current_request()
if hasattr(self, '_upload_file_id'
) and self._upload_file_id is not None: # noqa
util._s3client.abort_multipart_upload(
Bucket=self._bucket_name,
Key=self._upload_file_id,
UploadId=self._mpu['UploadId'])
self._mpu = None
self._upload_file_id = None
bucket = util.bucket
self._bucket_name = bucket._name
self._upload_file_id = request._site_id + '/' + uuid.uuid4().hex
self._multipart = {'Parts': []}
self._mpu = util._s3client.create_multipart_upload(
Bucket=self._bucket_name, Key=self._upload_file_id)
self._current_upload = 0
self._block = 1
self._resumable_uri_date = datetime.now(tz=tzlocal())
await notify(InitialS3Upload(context))
def get_hook():
request = get_current_request()
try:
site = request.site
search = queryUtility(ICatalogUtility)
except (AttributeError, KeyError):
return
if not search:
return # no search configured
try:
trns = tm(request).get()
except RequestNotFound:
trns = transaction.get()
hook = None
for _hook in trns._after_commit:
if isinstance(_hook[0], CommitHook):
hook = _hook[0]
break
if hook is None:
hook = CommitHook(site, request)
trns.addAfterCommitHook(hook)
return hook
def __init__(self, request=None):
ZopeSecurityPolicy.__init__(self)
if request is not None:
self.request = request
else:
# Try magic request lookup if request not given
self.request = get_current_request()
def test_gcr_memory(self):
self.request = FakeRequest()
count = 0
current = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 / 1024.0
while True:
count += 1
get_current_request()
if count % 1000000 == 0:
break
if count % 100000 == 0:
gc.collect()
new = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 / 1024.0
if new - current > 10: # memory leak, this shouldn't happen
assert new == current
async def _build_security_query(
self,
site,
query,
doc_type=None,
size=10,
request=None):
if query is None:
query = {}
q = {
'index': self.get_index_name(site)
}
if doc_type is not None:
q['doc_type'] = doc_type
# The users who has plone.AccessContent permission by prinperm
# The roles who has plone.AccessContent permission by roleperm
users = []
roles = []
if request is None:
request = get_current_request()
interaction = IInteraction(request)
for user in interaction.participations:
users.append(user.principal.id)
users.extend(user.principal.groups)
roles_dict = interaction.global_principal_roles(
user.principal.id,
user.principal.groups)
roles.extend([key for key, value in roles_dict.items()
if value])
# We got all users and roles
# users: users and groups
should_list = [{'match': {'access_roles': x}} for x in roles]
should_list.extend([{'match': {'access_users': x}} for x in users])
permission_query = {
'query': {
'bool': {
'filter': {
'bool': {
'should': should_list,
'minimum_should_match': 1
}
}
}
}
}
query = rec_merge(query, permission_query)
# query.update(permission_query)
q['body'] = query
q['size'] = size
return q
def __init__(self, request=None):
self.participations = []
self._cache = {}
self.principal = None
if request is not None:
self.request = request
else:
# Try magic request lookup if request not given
self.request = get_current_request()
async def reindex_all_content(
self, obj, security=False, response=None, clean=True):
""" We can reindex content or security for an object or
a specific query
"""
if security is False and clean is True:
await self.unindex_all_childs(obj, response=None, future=False)
# count_objects = await self.count_operation(obj)
loads = {}
total_elements = 1
request = get_current_request()
request._db_write_enabled = False
site = request.site
PENDING.clear()
await self.add_object(
obj=obj,
site=site,
loads=loads,
security=security,
response=response)
await self.index_sub_elements(
obj=obj,
site=site,
loads=loads,
security=security,
response=response)
while len(PENDING) > 0:
obj_id = PENDING.pop()
try:
obj = site._p_jar.get(obj_id)
except:
obj = None
if obj is not None:
total_elements += 1
await self.index_sub_elements(
obj=obj,
site=site,
loads=loads,
security=security,
response=response)
if total_elements > 50:
response.write(b'Size pending %d' % PENDING.__sizeof__())
await abort(request._txn, request)
request.conn.transaction_manager.begin(request)
total_elements = 1
if len(loads):
await self.reindex_bunk(site, loads, security, response=response)
def bucket(self):
request = get_current_request()
if '.' in self._bucket:
char_delimiter = '.'
else:
char_delimiter = '_'
bucket_name = request._site_id.lower() + char_delimiter + self._bucket
try:
bucket = self._client.get_bucket(bucket_name) # noqa
except NotFound:
bucket = self._client.create_bucket(bucket_name) # noqa
log.warn('We needed to create bucket ' + bucket_name)
return bucket_name
def get_principals_with_access_content(obj, request=None):
if obj is None:
return {}
if request is None:
request = get_current_request()
interaction = IInteraction(request)
roles = interaction.cached_roles(obj, 'plone.AccessContent', 'o')
result = []
for r in roles.keys():
lroles = role.local_roles()
if r in lroles:
result.append(r)
users = interaction.cached_principals(obj, result, 'plone.AccessContent', 'o')
return list(users.keys())
def get_roles_with_access_content(obj, request=None):
""" Return the roles that has access to the content that are global roles"""
if obj is None:
return []
if request is None:
request = get_current_request()
interaction = IInteraction(request)
roles = interaction.cached_roles(obj, 'plone.AccessContent', 'o')
result = []
for r in roles.keys():
lroles = role.global_roles()
if r in lroles:
result.append(r)
return result
def _invalidated_interaction_cache(self):
# Invalidate this threads interaction cache
try:
request = get_current_request()
except RequestNotFound:
return
interaction = IInteraction(request, None)
if interaction is not None:
try:
invalidate_cache = interaction.invalidate_cache
except AttributeError:
pass
else:
invalidate_cache()
async def initUpload(self, context):
"""Init an upload.
self._uload_file_id : temporal url to image beeing uploaded
self._resumable_uri : uri to resumable upload
self._uri : finished uploaded image
"""
util = getUtility(IGCloudBlobStore)
request = get_current_request()
if hasattr(self, '_upload_file_id') and self._upload_file_id is not None: # noqa
req = util._service.objects().delete(
bucket=util.bucket, object=self._upload_file_id)
try:
req.execute()
except errors.HttpError:
pass
self._upload_file_id = request._site_id + '/' + uuid.uuid4().hex
init_url = UPLOAD_URL.format(bucket=util.bucket) + '&name=' +\
self._upload_file_id
session = aiohttp.ClientSession()
creator = ','.join([x.principal.id for x
in request.security.participations])
metadata = json.dumps({
'CREATOR': creator,
'REQUEST': str(request),
'NAME': self.filename
})
call_size = len(metadata)
async with session.post(
init_url,
headers={
'AUTHORIZATION': 'Bearer %s' % util.access_token,
'X-Upload-Content-Type': self.contentType,
'X-Upload-Content-Length': str(self._size),
'Content-Type': 'application/json; charset=UTF-8',
'Content-Length': str(call_size)
},
data=metadata) as call:
if call.status != 200:
text = await call.text()
raise GoogleCloudException(text)
self._resumable_uri = call.headers['Location']
session.close()
self._current_upload = 0
self._resumable_uri_date = datetime.now(tz=tzlocal())
await notify(InitialGCloudUpload(context))
def check(self, obj, name):
permission = self.get_permissions.get(name)
if permission is not None:
if permission is CheckerPublic:
return # Public
request = get_current_request()
if IInteraction(request).checkPermission(permission, obj):
return
else:
__traceback_supplement__ = (TracebackSupplement, obj)
raise Unauthorized(obj, name, permission)
elif name in _available_by_default:
return
if name != '__iter__' or hasattr(obj, name):
__traceback_supplement__ = (TracebackSupplement, obj)
raise ForbiddenAttribute(name, obj)
def bucket(self):
request = get_current_request()
bucket_name = request._site_id.lower() + '.' + self._bucket_name
bucket = self._s3.Bucket(bucket_name)
exists = True
try:
self._s3.meta.client.head_bucket(Bucket=bucket_name)
except botocore.exceptions.ClientError as e:
# If a client error is thrown, then check that it was a 404 error.
# If it was a 404 error, then the bucket does not exist.
error_code = int(e.response['Error']['Code'])
if error_code == 404:
exists = False
self._s3.create_bucket(Bucket=bucket_name)
bucket = s3.Bucket(bucket_name)
return bucket
def check(self, obj, name):
permission = self.get_permissions.get(name)
if permission is not None:
if permission is CheckerPublic:
return # Public
request = get_current_request()
if IInteraction(request).check_permission(permission, obj):
return
else:
__traceback_supplement__ = (TracebackSupplement, obj)
raise Unauthorized(obj, name, permission)
elif name in _available_by_default:
return
if name != '__iter__' or hasattr(obj, name):
__traceback_supplement__ = (TracebackSupplement, obj)
raise ForbiddenAttribute(name, obj)
async def reindex_all_content(self, obj, security=False, loop=None):
""" We can reindex content or security for an object or
a specific query
"""
if security is False:
await self.unindex_all_childs(obj)
# count_objects = await self.count_operation(obj)
loads = {}
if loop is None:
loop = asyncio.get_event_loop()
request = get_current_request()
executor = getUtility(IApplication, name='root').executor
site = request.site
await self.add_object(obj, site, loads, security)
await self.reindex_recursive(obj, site, loads, security, loop,
executor)
if len(loads):
await self.reindex_bunk(site, loads, security)
def check_setattr(self, obj, name):
if self.set_permissions:
permission = self.set_permissions.get(name)
else:
permission = None
if permission is not None:
if permission is CheckerPublic:
return # Public
request = get_current_request()
if IInteraction(request).checkPermission(permission, obj):
return # allowed
else:
__traceback_supplement__ = (TracebackSupplement, obj)
raise Unauthorized(obj, name, permission)
__traceback_supplement__ = (TracebackSupplement, obj)
raise ForbiddenAttribute(name, obj)
def check_setattr(self, obj, name):
if self.set_permissions:
permission = self.set_permissions.get(name)
else:
permission = None
if permission is not None:
if permission is CheckerPublic:
return # Public
request = get_current_request()
if IInteraction(request).check_permission(permission, obj):
return # allowed
else:
__traceback_supplement__ = (TracebackSupplement, obj)
raise Unauthorized(obj, name, permission)
__traceback_supplement__ = (TracebackSupplement, obj)
raise ForbiddenAttribute(name, obj)
async def oneShotUpload(self, context, data):
util = getUtility(IS3BlobStore)
if hasattr(self, '_upload_file_id'
) and self._upload_file_id is not None: # noqa
util._s3client.abort_multipart_upload(
Bucket=self._bucket_name,
Key=self._upload_file_id,
UploadId=self._mpu['UploadId'])
self._mpu = None
self._upload_file_id = None
file_data = BytesIO(data)
bucket = util.bucket
self._bucket_name = bucket._name
request = get_current_request()
self._upload_file_id = request._site_id + '/' + uuid.uuid4().hex
response = util._s3client.upload_fileobj(file_data, self._bucket_name,
self._upload_file_id)
self._block += 1
self._current_upload += len(data)
return response
def create_content_in_container(container, type_, id_, request=None, **kw):
"""Utility to create a content.
This method is the one to use to create content.
id_ can be None
"""
factory = get_cached_factory(type_)
if factory.add_permission:
if factory.add_permission in PERMISSIONS_CACHE:
permission = PERMISSIONS_CACHE[factory.add_permission]
else:
permission = queryUtility(IPermission, name=factory.add_permission)
PERMISSIONS_CACHE[factory.add_permission] = permission
if request is None:
request = get_current_request()
if permission is not None and \
not request.security.checkPermission(permission.id, container):
raise NoPermissionToAdd(str(container), type_)
constrains = IConstrainTypes(container, None)
if constrains is not None:
if not constrains.is_type_allowed(type_):
raise NotAllowedContentType(str(container), type_)
# We create the object with at least the ID
obj = factory(id=id_)
obj.__parent__ = container
for key, value in kw.items():
setattr(obj, key, value)
if request is None or 'OVERWRITE' not in request.headers:
if obj.id in container:
raise ConflictIdOnContainer(str(container), obj.id)
container[obj.id] = obj
return obj
def create_content_in_container(container, type_, id_, request=None, **kw):
"""Utility to create a content.
This method is the one to use to create content.
id_ can be None
"""
factory = get_cached_factory(type_)
if factory.add_permission:
if factory.add_permission in PERMISSIONS_CACHE:
permission = PERMISSIONS_CACHE[factory.add_permission]
else:
permission = queryUtility(IPermission, name=factory.add_permission)
PERMISSIONS_CACHE[factory.add_permission] = permission
if request is None:
request = get_current_request()
if permission is not None and \
not IInteraction(request).check_permission(permission.id, container):
raise NoPermissionToAdd(str(container), type_)
constrains = IConstrainTypes(container, None)
if constrains is not None:
if not constrains.is_type_allowed(type_):
raise NotAllowedContentType(str(container), type_)
# We create the object with at least the ID
obj = factory(id=id_)
obj.__parent__ = container
for key, value in kw.items():
setattr(obj, key, value)
if request is None or 'OVERWRITE' not in request.headers:
if obj.id in container:
raise ConflictIdOnContainer(str(container), obj.id)
container[obj.id] = obj
return obj
async def unindex_all_childs(self, resource):
if type(resource) is str:
path = resource
depth = path.count('/') + 1
else:
path = get_content_path(resource)
depth = get_content_depth(resource)
depth += 1
request = get_current_request()
index_name = self.get_index_name(request.site)
path_query = {
'query': {
'bool': {
'must': [{
'term': {
'path': path
}
}, {
'range': {
'depth': {
'gte': depth
}
}
}]
}
}
}
conn_es = await self.conn.transport.get_connection()
async with conn_es._session.post(conn_es._base_url + index_name +
'/_delete_by_query',
data=json.dumps(path_query)) as resp:
result = await resp.json()
if 'deleted' in result:
logger.warn('Deleted %d childs' % result['deleted'])
else:
logger.warn('Wrong deletion of childs' + json.dumps(result))
def __init__(self, context):
request = get_current_request()
super(Absolute_URL_ObtainRequest, self).__init__(context, request)
async def security_changed(obj, event):
# We need to reindex the objects below
request = get_current_request()
request._futures.update({obj.id: AsyncCatalogReindex(obj, request, security=True)()})
def __init__(self, context):
request = get_current_request()
super(Absolute_URL_ObtainRequest, self).__init__(context, request)
