def get_popularity(site):
setSite(site)
catalog = api.portal.get_tool('portal_catalog')
es = ElasticSearchCatalog(catalog)
if not es.enabled:
return
service = analytics.get_ga_service()
if not service:
return
profile = analytics.get_ga_profile(service)
if not profile:
return
bulk_data = []
bulk_size = es.get_setting('bulk_size', 50)
conn = es.connection
site._p_jar.sync()
for path, page_views in get_results(service, profile)['rows']:
path = path.split('?')[0].lstrip('/').replace('/view',
'').split('@@')[0]
ob = site.restrictedTraverse(str(path), None)
if ob is None:
continue
annotations = IAnnotations(ob)
data = {'page_views': int(page_views)}
counts = annotations.get(COUNT_ANNOTATION_KEY, OOBTree())
counts['page_views'] = int(page_views)
annotations[COUNT_ANNOTATION_KEY] = counts
for key, value in counts.items():
if key in ('page_views', ):
continue
data[key + '_shares'] = value
if IPloneSiteRoot.providedBy(ob):
ob = ob[get_default_page(ob)]
bulk_data.extend([{
'update': {
'_index': es.index_name,
'_id': IUUID(ob)
}
}, {
'doc': data
}])
if len(bulk_data) % bulk_size == 0:
conn.bulk(index=es.index_name, body=bulk_data)
bulk_data = []
transaction.commit()
site._p_jar.sync()
if len(bulk_data) > 0:
conn.bulk(index=es.index_name, body=bulk_data)
transaction.commit()
class ElasticControlPanelFormWrapper(ControlPanelFormWrapper):
index = ViewPageTemplateFile('controlpanel_layout.pt')
def __init__(self, *args, **kwargs):
super(ElasticControlPanelFormWrapper, self).__init__(*args, **kwargs)
self.portal_catalog = getToolByName(self.context, 'portal_catalog')
self.es = ElasticSearchCatalog(self.portal_catalog)
@property
def connection_status(self):
try:
return self.es.connection.status()['ok']
except AttributeError:
try:
health_status = self.es.connection.cluster.health()['status']
return health_status in ('green', 'yellow')
except Exception:
return False
except Exception:
return False
@property
def es_info(self):
try:
info = self.es.connection.info()
try:
stats = self.es.connection.indices.stats(
index=self.es.real_index_name
)['indices'][self.es.real_index_name]['primaries']
size_in_mb = stats['store']['size_in_bytes'] / 1024.0 / 1024.0
return [
('Cluster Name', info.get('name')),
('Elastic Search Version', info['version']['number']),
('Number of docs', stats['docs']['count']),
('Deleted docs', stats['docs']['deleted']),
('Size', str(int(math.ceil(size_in_mb))) + 'MB'),
('Query Count', stats['search']['query_total'])
]
except KeyError:
return [
('Cluster Name', info.get('name')),
('Elastic Search Version', info['version']['number'])
]
except Exception:
logger.warning('Error getting stats', exc_info=True)
return []
@property
def active(self):
return self.es.get_setting('enabled')
class ElasticControlPanelFormWrapper(ControlPanelFormWrapper):
index = ViewPageTemplateFile("controlpanel_layout.pt")
def __init__(self, *args, **kwargs):
super(ElasticControlPanelFormWrapper, self).__init__(*args, **kwargs)
self.portal_catalog = getToolByName(self.context, "portal_catalog")
self.es = ElasticSearchCatalog(self.portal_catalog)
@property
def connection_status(self):
try:
return self.es.connection.status()["ok"]
except AttributeError:
try:
return self.es.connection.cluster.health()["status"] in ("green", "yellow")
except:
return False
except:
return False
@property
def es_info(self):
try:
info = self.es.connection.info()
stats = self.es.connection.indices.stats(index=self.es.real_index_name)["indices"][self.es.real_index_name][
"total"
]
return [
("Cluster Name", info.get("name")),
("Elastic Search Version", info["version"]["number"]),
("Number of docs", stats["docs"]["count"]),
("Deleted docs", stats["docs"]["deleted"]),
("Size", str(int(math.ceil(stats["store"]["size_in_bytes"] / 1024.0 / 1024.0))) + "MB"),
]
except Exception:
return []
@property
def active(self):
return self.es.get_setting("enabled")
def index_batch(remove, index, positions, es=None):
if es is None:
from collective.elasticsearch.es import ElasticSearchCatalog
es = ElasticSearchCatalog(api.portal.get_tool('portal_catalog'))
setSite(api.portal.get())
conn = es.connection
bulk_size = es.get_setting('bulk_size', 50)
if len(remove) > 0:
bulk_data = []
for uid in remove:
bulk_data.append({
'delete': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': uid
}
})
es.connection.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
if len(index) > 0:
if type(index) in (list, tuple, set):
# does not contain objects, must be async, convert to dict
index = dict([(k, None) for k in index])
bulk_data = []
for uid, obj in index.items():
if obj is None:
obj = uuidToObject(uid)
if obj is None:
continue
bulk_data.extend([{
'index': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': uid
}
},
get_index_data(obj, es)])
if len(bulk_data) % bulk_size == 0:
conn.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
bulk_data = []
if len(bulk_data) > 0:
conn.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
if len(positions) > 0:
bulk_data = []
index = getIndex(es.catalogtool._catalog, 'getObjPositionInParent')
for uid, ids in positions.items():
if uid == '/':
parent = getSite()
else:
parent = uuidToObject(uid)
if parent is None:
logger.warn('could not find object to index positions')
continue
for _id in ids:
ob = parent[_id]
wrapped_object = get_wrapped_object(ob, es)
try:
value = index.get_value(wrapped_object)
except Exception:
continue
bulk_data.extend([{
'update': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': IUUID(ob)
}
}, {
'doc': {
'getObjPositionInParent': value
}
}])
if len(bulk_data) % bulk_size == 0:
conn.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
bulk_data = []
if len(bulk_data) > 0:
conn.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
def index_batch(remove, index, positions, es=None):
if es is None:
from collective.elasticsearch.es import ElasticSearchCatalog
es = ElasticSearchCatalog(api.portal.get_tool('portal_catalog'))
conn = es.connection
bulk_size = es.get_setting('bulk_size', 50)
if len(remove) > 0:
bulk_data = []
for uid in remove:
bulk_data.append({
'delete': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': uid
}
})
es.connection.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data)
if len(index) > 0:
if type(index) in (list, tuple, set):
# does not contain objects, must be async, convert to dict
index = dict([(k, None) for k in index])
bulk_data = []
for uid, obj in index.items():
if obj is None:
obj = uuidToObject(uid)
if obj is None:
continue
bulk_data.extend([{
'index': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': uid
}
}, get_index_data(uid, obj, es)])
if len(bulk_data) % bulk_size == 0:
conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data)
bulk_data = []
if len(bulk_data) > 0:
conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data)
if len(positions) > 0:
bulk_data = []
index = getIndex(es.catalogtool._catalog, 'getObjPositionInParent')
for uid, ids in positions.items():
if uid == '/':
parent = getSite()
else:
parent = uuidToObject(uid)
if parent is None:
logger.warn('could not find object to index positions')
continue
for _id in ids:
ob = parent[_id]
wrapped_object = get_wrapped_object(ob, es)
try:
value = index.get_value(wrapped_object)
except:
continue
bulk_data.extend([{
'update': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': IUUID(ob)
}
}, {
'doc': {
'getObjPositionInParent': value
}
}])
if len(bulk_data) % bulk_size == 0:
conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data)
bulk_data = []
if len(bulk_data) > 0:
conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data)
def index_batch(remove, index, positions, es=None): # noqa: C901
if es is None:
from collective.elasticsearch.es import ElasticSearchCatalog
es = ElasticSearchCatalog(api.portal.get_tool('portal_catalog'))
setSite(api.portal.get())
conn = es.connection
bulk_size = es.get_setting('bulk_size', 50)
if len(remove) > 0:
bulk_data = []
for uid in remove:
bulk_data.append({
'delete': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': uid
}
})
result = es.connection.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
if "errors" in result and result["errors"] is True:
logger.error("Error in bulk indexing removal: %s" % result)
if len(index) > 0:
if type(index) in (list, tuple, set):
# does not contain objects, must be async, convert to dict
index = dict([(k, None) for k in index])
bulk_data = []
for uid, obj in index.items():
# If content has been moved (ie by a contentrule) then the object
# passed here is the original object, not the moved one.
# So if there is a uuid, we use this to get the correct object.
# See https://github.com/collective/collective.elasticsearch/issues/65 # noqa
if uid is not None:
obj = uuidToObject(uid)
if obj is None:
obj = uuidToObject(uid)
if obj is None:
continue
bulk_data.extend([{
'index': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': uid
}
},
get_index_data(obj, es)])
if len(bulk_data) % bulk_size == 0:
result = conn.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
if "errors" in result and result["errors"] is True:
logger.error("Error in bulk indexing: %s" % result)
bulk_data = []
if len(bulk_data) > 0:
result = conn.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
if "errors" in result and result["errors"] is True:
logger.error("Error in bulk indexing: %s" % result)
if len(positions) > 0:
bulk_data = []
index = getIndex(es.catalogtool._catalog, 'getObjPositionInParent')
for uid, ids in positions.items():
if uid == '/':
parent = getSite()
else:
parent = uuidToObject(uid)
if parent is None:
logger.warn('could not find object to index positions')
continue
for _id in ids:
ob = parent[_id]
wrapped_object = get_wrapped_object(ob, es)
try:
value = index.get_value(wrapped_object)
except Exception:
continue
bulk_data.extend([{
'update': {
'_index': es.index_name,
'_type': es.doc_type,
'_id': IUUID(ob)
}
}, {
'doc': {
'getObjPositionInParent': value
}
}])
if len(bulk_data) % bulk_size == 0:
conn.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
bulk_data = []
if len(bulk_data) > 0:
conn.bulk(index=es.index_name,
doc_type=es.doc_type,
body=bulk_data)
