def get_popularity(site): setSite(site) catalog = api.portal.get_tool('portal_catalog') es = ElasticSearchCatalog(catalog) if not es.enabled: return service = analytics.get_ga_service() if not service: return profile = analytics.get_ga_profile(service) if not profile: return bulk_data = [] bulk_size = es.get_setting('bulk_size', 50) conn = es.connection site._p_jar.sync() for path, page_views in get_results(service, profile)['rows']: path = path.split('?')[0].lstrip('/').replace('/view', '').split('@@')[0] ob = site.restrictedTraverse(str(path), None) if ob is None: continue annotations = IAnnotations(ob) data = {'page_views': int(page_views)} counts = annotations.get(COUNT_ANNOTATION_KEY, OOBTree()) counts['page_views'] = int(page_views) annotations[COUNT_ANNOTATION_KEY] = counts for key, value in counts.items(): if key in ('page_views', ): continue data[key + '_shares'] = value if IPloneSiteRoot.providedBy(ob): ob = ob[get_default_page(ob)] bulk_data.extend([{ 'update': { '_index': es.index_name, '_id': IUUID(ob) } }, { 'doc': data }]) if len(bulk_data) % bulk_size == 0: conn.bulk(index=es.index_name, body=bulk_data) bulk_data = [] transaction.commit() site._p_jar.sync() if len(bulk_data) > 0: conn.bulk(index=es.index_name, body=bulk_data) transaction.commit()
class ElasticControlPanelFormWrapper(ControlPanelFormWrapper): index = ViewPageTemplateFile('controlpanel_layout.pt') def __init__(self, *args, **kwargs): super(ElasticControlPanelFormWrapper, self).__init__(*args, **kwargs) self.portal_catalog = getToolByName(self.context, 'portal_catalog') self.es = ElasticSearchCatalog(self.portal_catalog) @property def connection_status(self): try: return self.es.connection.status()['ok'] except AttributeError: try: health_status = self.es.connection.cluster.health()['status'] return health_status in ('green', 'yellow') except Exception: return False except Exception: return False @property def es_info(self): try: info = self.es.connection.info() try: stats = self.es.connection.indices.stats( index=self.es.real_index_name )['indices'][self.es.real_index_name]['primaries'] size_in_mb = stats['store']['size_in_bytes'] / 1024.0 / 1024.0 return [ ('Cluster Name', info.get('name')), ('Elastic Search Version', info['version']['number']), ('Number of docs', stats['docs']['count']), ('Deleted docs', stats['docs']['deleted']), ('Size', str(int(math.ceil(size_in_mb))) + 'MB'), ('Query Count', stats['search']['query_total']) ] except KeyError: return [ ('Cluster Name', info.get('name')), ('Elastic Search Version', info['version']['number']) ] except Exception: logger.warning('Error getting stats', exc_info=True) return [] @property def active(self): return self.es.get_setting('enabled')
class ElasticControlPanelFormWrapper(ControlPanelFormWrapper): index = ViewPageTemplateFile("controlpanel_layout.pt") def __init__(self, *args, **kwargs): super(ElasticControlPanelFormWrapper, self).__init__(*args, **kwargs) self.portal_catalog = getToolByName(self.context, "portal_catalog") self.es = ElasticSearchCatalog(self.portal_catalog) @property def connection_status(self): try: return self.es.connection.status()["ok"] except AttributeError: try: return self.es.connection.cluster.health()["status"] in ("green", "yellow") except: return False except: return False @property def es_info(self): try: info = self.es.connection.info() stats = self.es.connection.indices.stats(index=self.es.real_index_name)["indices"][self.es.real_index_name][ "total" ] return [ ("Cluster Name", info.get("name")), ("Elastic Search Version", info["version"]["number"]), ("Number of docs", stats["docs"]["count"]), ("Deleted docs", stats["docs"]["deleted"]), ("Size", str(int(math.ceil(stats["store"]["size_in_bytes"] / 1024.0 / 1024.0))) + "MB"), ] except Exception: return [] @property def active(self): return self.es.get_setting("enabled")
def index_batch(remove, index, positions, es=None): if es is None: from collective.elasticsearch.es import ElasticSearchCatalog es = ElasticSearchCatalog(api.portal.get_tool('portal_catalog')) setSite(api.portal.get()) conn = es.connection bulk_size = es.get_setting('bulk_size', 50) if len(remove) > 0: bulk_data = [] for uid in remove: bulk_data.append({ 'delete': { '_index': es.index_name, '_type': es.doc_type, '_id': uid } }) es.connection.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) if len(index) > 0: if type(index) in (list, tuple, set): # does not contain objects, must be async, convert to dict index = dict([(k, None) for k in index]) bulk_data = [] for uid, obj in index.items(): if obj is None: obj = uuidToObject(uid) if obj is None: continue bulk_data.extend([{ 'index': { '_index': es.index_name, '_type': es.doc_type, '_id': uid } }, get_index_data(obj, es)]) if len(bulk_data) % bulk_size == 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) bulk_data = [] if len(bulk_data) > 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) if len(positions) > 0: bulk_data = [] index = getIndex(es.catalogtool._catalog, 'getObjPositionInParent') for uid, ids in positions.items(): if uid == '/': parent = getSite() else: parent = uuidToObject(uid) if parent is None: logger.warn('could not find object to index positions') continue for _id in ids: ob = parent[_id] wrapped_object = get_wrapped_object(ob, es) try: value = index.get_value(wrapped_object) except Exception: continue bulk_data.extend([{ 'update': { '_index': es.index_name, '_type': es.doc_type, '_id': IUUID(ob) } }, { 'doc': { 'getObjPositionInParent': value } }]) if len(bulk_data) % bulk_size == 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) bulk_data = [] if len(bulk_data) > 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data)
def index_batch(remove, index, positions, es=None): if es is None: from collective.elasticsearch.es import ElasticSearchCatalog es = ElasticSearchCatalog(api.portal.get_tool('portal_catalog')) conn = es.connection bulk_size = es.get_setting('bulk_size', 50) if len(remove) > 0: bulk_data = [] for uid in remove: bulk_data.append({ 'delete': { '_index': es.index_name, '_type': es.doc_type, '_id': uid } }) es.connection.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) if len(index) > 0: if type(index) in (list, tuple, set): # does not contain objects, must be async, convert to dict index = dict([(k, None) for k in index]) bulk_data = [] for uid, obj in index.items(): if obj is None: obj = uuidToObject(uid) if obj is None: continue bulk_data.extend([{ 'index': { '_index': es.index_name, '_type': es.doc_type, '_id': uid } }, get_index_data(uid, obj, es)]) if len(bulk_data) % bulk_size == 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) bulk_data = [] if len(bulk_data) > 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) if len(positions) > 0: bulk_data = [] index = getIndex(es.catalogtool._catalog, 'getObjPositionInParent') for uid, ids in positions.items(): if uid == '/': parent = getSite() else: parent = uuidToObject(uid) if parent is None: logger.warn('could not find object to index positions') continue for _id in ids: ob = parent[_id] wrapped_object = get_wrapped_object(ob, es) try: value = index.get_value(wrapped_object) except: continue bulk_data.extend([{ 'update': { '_index': es.index_name, '_type': es.doc_type, '_id': IUUID(ob) } }, { 'doc': { 'getObjPositionInParent': value } }]) if len(bulk_data) % bulk_size == 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) bulk_data = [] if len(bulk_data) > 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data)
def index_batch(remove, index, positions, es=None): # noqa: C901 if es is None: from collective.elasticsearch.es import ElasticSearchCatalog es = ElasticSearchCatalog(api.portal.get_tool('portal_catalog')) setSite(api.portal.get()) conn = es.connection bulk_size = es.get_setting('bulk_size', 50) if len(remove) > 0: bulk_data = [] for uid in remove: bulk_data.append({ 'delete': { '_index': es.index_name, '_type': es.doc_type, '_id': uid } }) result = es.connection.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) if "errors" in result and result["errors"] is True: logger.error("Error in bulk indexing removal: %s" % result) if len(index) > 0: if type(index) in (list, tuple, set): # does not contain objects, must be async, convert to dict index = dict([(k, None) for k in index]) bulk_data = [] for uid, obj in index.items(): # If content has been moved (ie by a contentrule) then the object # passed here is the original object, not the moved one. # So if there is a uuid, we use this to get the correct object. # See https://github.com/collective/collective.elasticsearch/issues/65 # noqa if uid is not None: obj = uuidToObject(uid) if obj is None: obj = uuidToObject(uid) if obj is None: continue bulk_data.extend([{ 'index': { '_index': es.index_name, '_type': es.doc_type, '_id': uid } }, get_index_data(obj, es)]) if len(bulk_data) % bulk_size == 0: result = conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) if "errors" in result and result["errors"] is True: logger.error("Error in bulk indexing: %s" % result) bulk_data = [] if len(bulk_data) > 0: result = conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) if "errors" in result and result["errors"] is True: logger.error("Error in bulk indexing: %s" % result) if len(positions) > 0: bulk_data = [] index = getIndex(es.catalogtool._catalog, 'getObjPositionInParent') for uid, ids in positions.items(): if uid == '/': parent = getSite() else: parent = uuidToObject(uid) if parent is None: logger.warn('could not find object to index positions') continue for _id in ids: ob = parent[_id] wrapped_object = get_wrapped_object(ob, es) try: value = index.get_value(wrapped_object) except Exception: continue bulk_data.extend([{ 'update': { '_index': es.index_name, '_type': es.doc_type, '_id': IUUID(ob) } }, { 'doc': { 'getObjPositionInParent': value } }]) if len(bulk_data) % bulk_size == 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data) bulk_data = [] if len(bulk_data) > 0: conn.bulk(index=es.index_name, doc_type=es.doc_type, body=bulk_data)