async def process_object(self, ob):
'''
- check if doc does not exist
- record it
- do complete index
- if doc exist
- if diff mapping exists
- update fields in diff on doc
- else, do nothing
- remove for list of existing doc ids
'''
clear_conn_statement_cache(await ob._p_jar.get_connection())
full = False
if ob.uuid not in self.existing:
self.missing.append(ob.uuid)
full = True
else:
self.existing.remove(ob.uuid)
await self.index_object(ob, full=full)
self.processed += 1
if IFolder.providedBy(ob):
await self.process_folder(ob)
if not IContainer.providedBy(ob):
del ob.__annotations__
del ob
async def process_object(self, ob):
'''
- check if doc does not exist
- record it
- do complete index
- if doc exist
- if diff mapping exists
- update fields in diff on doc
- else, do nothing
- remove for list of existing doc ids
'''
full = False
if ob.uuid not in self.existing:
self.missing.append(ob.uuid)
full = True
else:
self.existing.remove(ob.uuid)
await self.index_object(ob, full=full)
self.processed += 1
if IIndexActive.providedBy(ob):
self.sub_indexes.append(ob)
else:
if IFolder.providedBy(ob):
await self.process_folder(ob)
if not IContainer.providedBy(ob):
try:
del self.container.__gannotations__
except AttributeError:
del self.container.__annotations__
del ob
async def deep_walk(self, actual):
async for obj in actual.async_values():
if IFolder.providedBy(obj):
async for children in self.deep_walk(obj):
yield children
elif IItem.providedBy(obj):
yield obj
async def __call__(self, include=[], omit=[]):
self.include = include
self.omit = omit
parent = self.context.__parent__
if parent is not None:
# We render the summary of the parent
try:
parent_summary = await get_multi_adapter(
(parent, self.request), IResourceSerializeToJsonSummary)()
except ComponentLookupError:
parent_summary = {}
else:
parent_summary = {}
factory = get_cached_factory(self.context.type_name)
behaviors = []
for behavior_schema in factory.behaviors or ():
behaviors.append(behavior_schema.__identifier__)
result = {
'@id': IAbsoluteURL(self.context, self.request)(),
'@type': self.context.type_name,
'@name': self.context.__name__,
'@uid': self.context.uuid,
'@static_behaviors': behaviors,
'parent': parent_summary, # should be @parent
'is_folderish': IFolder.providedBy(self.context), # eek, should be @folderish?
'creation_date': json_compatible(self.context.creation_date),
'modification_date': json_compatible(self.context.modification_date),
'UID': self.context.uuid, # should be removed
}
main_schema = factory.schema
await self.get_schema(main_schema, self.context, result, False)
# include can be one of:
# - <field name> on content schema
# - namespace.IBehavior
# - namespace.IBehavior.field_name
included_ifaces = [name for name in self.include if '.' in name]
included_ifaces.extend([name.rsplit('.', 1)[0] for name in self.include
if '.' in name])
for behavior_schema, behavior in await get_all_behaviors(self.context, load=False):
if '*' not in self.include:
dotted_name = behavior_schema.__identifier__
if (dotted_name in self.omit or
(len(included_ifaces) > 0 and dotted_name not in included_ifaces)):
# make sure the schema isn't filtered
continue
if (not getattr(behavior, 'auto_serialize', True) and
dotted_name not in included_ifaces):
continue
if IAsyncBehavior.implementedBy(behavior.__class__):
# providedBy not working here?
await behavior.load(create=False)
await self.get_schema(behavior_schema, behavior, result, True)
return result
async def _apply_recursive(ob, function, count=0):
await apply_coroutine(function, ob)
if IFolder.providedBy(ob):
keys = await ob.async_keys()
for key in keys:
item = await ob._p_jar.get_child(ob, key)
if item is not None:
count += await _apply_recursive(item, function)
return count + 1
async def __call__(self, include=[], omit=[]):
self.include = include
self.omit = omit
parent = self.context.__parent__
if parent is not None:
# We render the summary of the parent
try:
parent_summary = await get_multi_adapter(
(parent, self.request), IResourceSerializeToJsonSummary)()
except ComponentLookupError:
parent_summary = {}
else:
parent_summary = {}
result = {
'@id': IAbsoluteURL(self.context, self.request)(),
'@type': self.context.type_name,
'@name': self.context.__name__,
'@uid': self.context.uuid,
'parent': parent_summary,
'is_folderish': IFolder.providedBy(self.context),
'creation_date': json_compatible(self.context.creation_date),
'modification_date':
json_compatible(self.context.modification_date),
'UID': self.context.uuid,
}
factory = get_cached_factory(self.context.type_name)
main_schema = factory.schema
await self.get_schema(main_schema, self.context, result, False)
# include can be one of:
# - <field name> on content schema
# - namespace.IBehavior
# - namespace.IBehavior.field_name
included_ifaces = [name for name in self.include if '.' in name]
included_ifaces.extend(
[name.rsplit('.', 1)[0] for name in self.include if '.' in name])
for behavior_schema, behavior in await get_all_behaviors(self.context,
load=False):
dotted_name = behavior_schema.__identifier__
if (dotted_name in self.omit
or (len(included_ifaces) > 0
and dotted_name not in included_ifaces)):
# make sure the schema isn't filtered
continue
if (not getattr(behavior, 'auto_serialize', True)
and dotted_name not in included_ifaces):
continue
if IAsyncBehavior.implementedBy(behavior.__class__):
# providedBy not working here?
await behavior.load(create=False)
await self.get_schema(behavior_schema, behavior, result, True)
return result
async def _process_object(self, obj, data):
if data["count"] % 200 == 0:
await data["tm"].commit(txn=data["transaction"])
data["transaction"] = await data["tm"].begin()
obj.__txn__ = data["transaction"]
uuid = obj.__uuid__
writer = IWriter(obj)
await self._index(uuid, writer, data["transaction"], data["table_name"])
data["count"] += 1
if IFolder.providedBy(obj):
await self._process_folder(obj, data)
del obj
async def reindex_all_content(self, container, security=False):
"""
recursively go through all content to reindex jsonb...
"""
data = {"count": 0, "transaction": None, "transactions": 0, "tm": get_current_transaction()._manager}
try:
data["table_name"] = data["tm"]._storage._objects_table_name
except AttributeError:
# Not supported DB
return
data["transaction"] = await data["tm"].begin()
container.__txn__ = data["transaction"]
await self._process_object(container, data)
if IFolder.providedBy(container):
await self._process_folder(container, data)
await data["tm"].commit(txn=data["transaction"])
async def remove(self, container, objects, index_name=None, request=None):
"""List of UIDs to remove from index.
It will remove all the children on the index"""
if not self.enabled:
return
if len(objects) > 0:
if index_name is None:
indexes = await self.get_current_indexes(container)
else:
indexes = [index_name]
bulk_data = []
for obj in objects:
item_indexes = indexes
im = find_index_manager(obj)
if im:
item_indexes = await im.get_indexes()
for index in item_indexes:
bulk_data.append(
{"delete": {
"_index": index,
"_id": obj.__uuid__
}})
if IFolder.providedBy(obj):
# only folders need to have children cleaned
if IIndexActive.providedBy(obj):
# delete this index...
im = get_adapter(obj, IIndexManager)
await self._delete_index(im)
else:
await self.unindex_all_children(
container, obj, index_name=",".join(item_indexes))
conn = self.get_connection()
await conn.bulk(
index=indexes[0],
body=bulk_data,
doc_type=DOC_TYPE,
refresh=self._refresh(),
)
async def __call__(self, include=None, omit=None):
self.include = include or []
self.omit = omit or []
parent = self.context.__parent__
if parent is not None:
# We render the summary of the parent
try:
parent_summary = await get_multi_adapter(
(parent, self.request), IResourceSerializeToJsonSummary)()
except ComponentLookupError:
parent_summary = {}
else:
parent_summary = {}
factory = get_cached_factory(self.context.type_name)
behaviors = []
for behavior_schema in factory.behaviors or ():
behaviors.append(behavior_schema.__identifier__)
result = {
"@id": get_object_url(self.context, self.request),
"@type": self.context.type_name,
"@name": self.context.__name__,
"@uid": self.context.uuid,
"@static_behaviors": behaviors,
"parent": parent_summary, # should be @parent
"is_folderish":
IFolder.providedBy(self.context), # eek, should be @folderish?
"creation_date": json_compatible(self.context.creation_date),
"modification_date":
json_compatible(self.context.modification_date),
}
main_schema = factory.schema
await self.get_schema(main_schema, self.context, result, False)
# include can be one of:
# - <field name> on content schema
# - namespace.IBehavior
# - namespace.IBehavior.field_name
included_ifaces = [name for name in self.include if "." in name]
included_ifaces.extend(
[name.rsplit(".", 1)[0] for name in self.include if "." in name])
for behavior_schema, behavior in await get_all_behaviors(self.context,
load=False):
if "*" not in self.include:
dotted_name = behavior_schema.__identifier__
if dotted_name in self.omit or (len(included_ifaces) > 0
and dotted_name
not in included_ifaces):
# make sure the schema isn't filtered
continue
if not getattr(behavior, "auto_serialize",
True) and dotted_name not in included_ifaces:
continue
if IAsyncBehavior.implementedBy(behavior.__class__):
# providedBy not working here?
await behavior.load(create=False)
await self.get_schema(behavior_schema, behavior, result, True)
for post_serialize_processors in app_settings["post_serialize"]:
await apply_coroutine(post_serialize_processors, self.context,
result)
return result
def is_folderish(self):
return IFolder.providedBy(self.context)