def get_total_datastreams(self): c = Cache(db=0) total_datastreams = c.get('my_total_datastreams_' + str(self.id)) if not total_datastreams: total_datastreams = DataStream.objects.filter(user=self.id).count() if total_datastreams > 0: c.set('my_total_datastreams_' + str(self.id), total_datastreams, settings.REDIS_STATS_TTL) return total_datastreams
def get_total_visualizations(self): c = Cache(db=0) total_visualizations = c.get('my_total_visualizations_' + str(self.id)) if not total_visualizations: total_visualizations = Visualization.objects.filter(user=self.id).count() if total_visualizations > 0: c.set('my_total_visualizations_' + str(self.id), total_visualizations, settings.REDIS_STATS_TTL) return total_visualizations
def get_total_datasets(self): c = Cache(db=0) users = User.objects.filter(account=self) total_datasets = c.get('account_total_datasets_' + str(self.id)) if not total_datasets: total_datasets = Dataset.objects.filter(user__in=users).count() if total_datasets > 0: c.set('account_total_datasets_' + str(self.id), total_datasets, settings.REDIS_STATS_TTL) return total_datasets
def get_total_datasets(self): c = Cache(db=0) users = User.objects.filter(account=self) total_datasets = c.get('account_total_datasets_' + str(self.id)) if not total_datasets: total_datasets = Dataset.objects.filter(user__in=users).count() #if settings.DEBUG: logger.info('get_total_datasets from database %d' % total_datasets) if total_datasets > 0: c.set('account_total_datasets_' + str(self.id), total_datasets, settings.REDIS_STATS_TTL) #else: # if settings.DEBUG: logger.info('get_total_datasets from cache %s' % total_datasets) return total_datasets
def get_total_datasets(self): c = Cache(db=0) users = User.objects.filter(account=self) total_datasets = c.get('account_total_datasets_' + str(self.id)) if not total_datasets: total_datasets = Dataset.objects.filter(user__in=users).count() if settings.DEBUG: logger.info('get_total_datasets from database %d' % total_datasets) if total_datasets > 0: c.set('account_total_datasets_' + str(self.id), total_datasets, settings.REDIS_STATS_TTL) else: if settings.DEBUG: logger.info('get_total_datasets from cache %s' % total_datasets) return total_datasets
class VisualizationHitsDAO: """class for manage access to Hits in DB and index""" doc_type = "vz" from_cache = False # cache ttl, 1 hora TTL = 3600 def __init__(self, visualization): self.visualization = visualization self.search_index = ElasticsearchIndex() self.logger = logging.getLogger(__name__) self.cache = Cache() def add(self, channel_type): """agrega un hit al datastream. """ try: hit = VisualizationHits.objects.create( visualization_id=self.visualization.visualization_id, channel_type=channel_type ) except IntegrityError: # esta correcto esta excepcion? raise VisualizationNotFoundException() self.logger.info("VisualizationHitsDAO hit! (guid: %s)" % (self.datastream.guid)) # armo el documento para actualizar el index. doc = { "docid": "%s::%s" % (self.doc_type.upper(), self.visualization.guid), "type": self.doc_type, "script": "ctx._source.fields.hits+=1", } return self.search_index.update(doc) def count(self): return VisualizationHits.objects.filter(visualization_id=self.visualization.visualization_id).count() def _get_cache(self, cache_key): cache = self.cache.get(cache_key) return cache def _set_cache(self, cache_key, value): return self.cache.set(cache_key, value, self.TTL) def count_by_day(self, day): """retorna los hits de un día determinado""" # si es datetime, usar solo date if type(day) == type(datetime.today()): day = day.date() cache_key = "%s_hits_%s_by_date_%s" % (self.doc_type, self.visualization.guid, str(day)) hits = self._get_cache(cache_key) # si el día particular no esta en el caché, lo guarda # salvo que sea el parametro pasado sea de hoy, lo guarda en el cache pero usa siempre el de la DB if not hits or day == date.today(): hits = VisualizationHits.objects.filter( visualization=self.visualization, created_at__startswith=day ).count() self._set_cache(cache_key, hits) return (date, hits) def count_by_days(self, day=30, channel_type=None): """trae un dict con los hits totales de los ultimos day y los hits particulares de los días desde day hasta today""" # no sé si es necesario esto if day < 1: return {} cache_key = "%s_hits_%s_%s" % (self.doc_type, self.visualization.guid, day) if channel_type: cache_key += "_channel_type_%s" % channel_type hits = self._get_cache(cache_key) # me cachendié! no esta en la cache if not hits: # tenemos la fecha de inicio start_date = datetime.today() - timedelta(days=day) # tomamos solo la parte date truncate_date = connection.ops.date_trunc_sql("day", "created_at") qs = VisualizationHits.objects.filter(visualization=self.visualization, created_at__gte=start_date) if channel_type: qs = qs.filter(channel_type=channel_type) hits = ( qs.extra(select={"_date": truncate_date, "fecha": "DATE(created_at)"}) .values("fecha") .order_by("created_at") .annotate(hits=Count("created_at")) ) control = [date.today() - timedelta(days=x) for x in range(day - 1, 0, -1)] control.append(date.today()) for i in hits: try: control.remove(i["fecha"]) except ValueError: pass hits = list(hits) for i in control: hits.append({"fecha": i, "hits": 0}) hits = sorted(hits, key=lambda k: k["fecha"]) # transformamos las fechas en isoformat hits = map(self._date_isoformat, hits) # lo dejamos, amablemente, en la cache! self._set_cache(cache_key, json.dumps(hits, cls=DjangoJSONEncoder)) self.from_cache = False else: hits = json.loads(hits) self.from_cache = True return hits def _date_isoformat(self, row): row["fecha"] = row["fecha"].isoformat() return row
class DatastreamHitsDAO(): """class for manage access to Hits in DB and index""" doc_type = "ds" from_cache = False # cache ttl, 1 hora TTL=3600 def __init__(self, datastream): self.datastream = datastream #self.datastream_revision = datastream.last_published_revision self.search_index = ElasticsearchIndex() self.logger=logging.getLogger(__name__) self.cache=Cache() def add(self, channel_type): """agrega un hit al datastream. """ # TODO: Fix temporal por el paso de DT a DAO. # Es problema es que por momentos el datastream viene de un queryset y otras veces de un DAO y son objetos # distintos try: datastream_id = self.datastream.datastream_id except: datastream_id = self.datastream['datastream_id'] try: guid = self.datastream.guid except: guid = self.datastream['guid'] try: hit=DataStreamHits.objects.create(datastream_id=datastream_id, channel_type=channel_type) except IntegrityError: # esta correcto esta excepcion? raise DataStreamNotFoundException() self.logger.info("DatastreamHitsDAO hit! (guid: %s)" % ( guid)) # armo el documento para actualizar el index. doc={'docid':"DS::%s" % guid, "type": "ds", "script": "ctx._source.fields.hits+=1"} return self.search_index.update(doc) def count(self): return DataStreamHits.objects.filter(datastream_id=self.datastream['datastream_id']).count() def _get_cache(self, cache_key): cache=self.cache.get(cache_key) return cache def _set_cache(self, cache_key, value): return self.cache.set(cache_key, value, self.TTL) def count_by_days(self, day=30, channel_type=None): """trae un dict con los hits totales de los ultimos day y los hits particulares de los días desde day hasta today""" # no sé si es necesario esto if day < 1: return {} cache_key="%s_hits_%s_%s" % ( self.doc_type, self.datastream.guid, day) if channel_type: cache_key+="_channel_type_%s" % channel_type hits = self._get_cache(cache_key) # me cachendié! no esta en la cache if not hits : # tenemos la fecha de inicio start_date=datetime.today()-timedelta(days=day) # tomamos solo la parte date truncate_date = connection.ops.date_trunc_sql('day', 'created_at') qs=DataStreamHits.objects.filter(datastream=self.datastream,created_at__gte=start_date) if channel_type: qs=qs.filter(channel_type=channel_type) hits=qs.extra(select={'_date': truncate_date, "fecha": 'DATE(created_at)'}).values("fecha").order_by("created_at").annotate(hits=Count("created_at")) control=[ date.today()-timedelta(days=x) for x in range(day-1,0,-1)] control.append(date.today()) for i in hits: try: control.remove(i['fecha']) except ValueError: pass hits=list(hits) for i in control: hits.append({"fecha": i, "hits": 0}) hits = sorted(hits, key=lambda k: k['fecha']) # transformamos las fechas en isoformat hits=map(self._date_isoformat, hits) # lo dejamos, amablemente, en la cache! self._set_cache(cache_key, json.dumps(hits, cls=DjangoJSONEncoder)) self.from_cache=False else: hits=json.loads(hits) self.from_cache = True return hits def _date_isoformat(self, row): row['fecha']=row['fecha'].isoformat() return row
def workspace_open_data_metrics(auth_manager): from core.cache import Cache from datetime import date, timedelta from django.db import connection cursor = connection.cursor() user_id = auth_manager.id account_id = auth_manager.account_id language = auth_manager.language last_7_days = date.today() - timedelta(days=7) c = Cache(db=0) published_datasets = c.get('published_datasets_' + str(account_id)) if not published_datasets: cursor.execute("SELECT COUNT(1) as val FROM ao_datasets d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_dataset_revisions b WHERE b.dataset_id = d.id AND NOT EXISTS(SELECT * FROM ao_dataset_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.dataset_id = c.dataset_id) AND b.status = 3)", [str(account_id)]) row = cursor.fetchone() published_datasets = row[0] c.set('published_datasets_' + str(account_id), published_datasets, settings.REDIS_STATS_TTL) total_datasets = c.get('total_datasets_' + str(account_id)) if not total_datasets: total_datasets = Dataset.objects.filter(user__account=account_id).count() c.set('total_datasets_' + str(account_id), total_datasets, settings.REDIS_STATS_TTL) published_datastreams = c.get('published_datastreams_' + str(account_id)) if not published_datastreams: cursor.execute("SELECT COUNT(1) as val FROM ao_datastreams d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_datastream_revisions b WHERE b.datastream_id = d.id AND NOT EXISTS(SELECT * FROM ao_datastream_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.datastream_id = c.datastream_id) AND b.status = 3)", [str(account_id)]) row = cursor.fetchone() published_datastreams = row[0] c.set('published_datastreams_' + str(account_id), published_datastreams, settings.REDIS_STATS_TTL) total_datastreams = c.get('total_datastreams_' + str(account_id)) if not total_datastreams: total_datastreams = DataStream.objects.filter(user__account=account_id).count() c.set('total_datastreams_' + str(account_id), total_datastreams, settings.REDIS_STATS_TTL) published_dashboards = c.get('published_dashboards_' + str(account_id)) if not published_dashboards: cursor.execute("SELECT COUNT(1) as val FROM ao_dashboards d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_dashboard_revisions b WHERE b.dashboard_id = d.id AND NOT EXISTS(SELECT * FROM ao_dashboard_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.dashboard_id = c.dashboard_id) AND b.status = 3)", [str(account_id)]) row = cursor.fetchone() published_dashboards = row[0] c.set('published_dashboards_' + str(account_id), published_dashboards, settings.REDIS_STATS_TTL) published_visualizations = c.get('published_visualizations_' + str(account_id)) if not published_visualizations: cursor.execute("SELECT COUNT(1) as val FROM ao_visualizations d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_visualizations_revisions b WHERE b.visualization_id = d.id AND NOT EXISTS(SELECT * FROM ao_visualizations_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.visualization_id = c.visualization_id) AND b.status = 3)", [str(account_id)]) row = cursor.fetchone() published_visualizations = row[0] c.set('published_visualizations_' + str(account_id), published_visualizations, settings.REDIS_STATS_TTL) total_visualizations = c.get('total_visualizations_' + str(account_id)) if not total_visualizations: total_visualizations = Visualization.objects.filter(user__account=account_id).count() c.set('total_visualizations_' + str(account_id), total_visualizations, settings.REDIS_STATS_TTL) return locals()
def workspace_open_data_metrics(auth_manager): from core.cache import Cache from datetime import date, timedelta from django.db import connection cursor = connection.cursor() user_id = auth_manager.id account_id = auth_manager.account_id language = auth_manager.language last_7_days = date.today() - timedelta(days=7) c = Cache(db=0) published_datasets = c.get('published_datasets_' + str(account_id)) if not published_datasets: cursor.execute( "SELECT COUNT(1) as val FROM ao_datasets d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_dataset_revisions b WHERE b.dataset_id = d.id AND NOT EXISTS(SELECT * FROM ao_dataset_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.dataset_id = c.dataset_id) AND b.status = 3)", [str(account_id)]) row = cursor.fetchone() published_datasets = row[0] c.set('published_datasets_' + str(account_id), published_datasets, settings.REDIS_STATS_TTL) total_datasets = c.get('total_datasets_' + str(account_id)) if not total_datasets: total_datasets = Dataset.objects.filter( user__account=account_id).count() c.set('total_datasets_' + str(account_id), total_datasets, settings.REDIS_STATS_TTL) published_datastreams = c.get('published_datastreams_' + str(account_id)) if not published_datastreams: cursor.execute( "SELECT COUNT(1) as val FROM ao_datastreams d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_datastream_revisions b WHERE b.datastream_id = d.id AND NOT EXISTS(SELECT * FROM ao_datastream_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.datastream_id = c.datastream_id) AND b.status = 3)", [str(account_id)]) row = cursor.fetchone() published_datastreams = row[0] c.set('published_datastreams_' + str(account_id), published_datastreams, settings.REDIS_STATS_TTL) total_datastreams = c.get('total_datastreams_' + str(account_id)) if not total_datastreams: total_datastreams = DataStream.objects.filter( user__account=account_id).count() c.set('total_datastreams_' + str(account_id), total_datastreams, settings.REDIS_STATS_TTL) published_visualizations = c.get('published_visualizations_' + str(account_id)) if not published_visualizations: cursor.execute( "SELECT COUNT(1) as val FROM ao_visualizations d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_visualizations_revisions b WHERE b.visualization_id = d.id AND NOT EXISTS(SELECT * FROM ao_visualizations_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.visualization_id = c.visualization_id) AND b.status = 3)", [str(account_id)]) row = cursor.fetchone() published_visualizations = row[0] c.set('published_visualizations_' + str(account_id), published_visualizations, settings.REDIS_STATS_TTL) total_visualizations = c.get('total_visualizations_' + str(account_id)) if not total_visualizations: total_visualizations = Visualization.objects.filter( user__account=account_id).count() c.set('total_visualizations_' + str(account_id), total_visualizations, settings.REDIS_STATS_TTL) return locals()