def get_total_datastreams(self):
c = Cache(db=0)
total_datastreams = c.get('my_total_datastreams_' + str(self.id))
if not total_datastreams:
total_datastreams = DataStream.objects.filter(user=self.id).count()
if total_datastreams > 0:
c.set('my_total_datastreams_' + str(self.id), total_datastreams, settings.REDIS_STATS_TTL)
return total_datastreams
def get_total_visualizations(self):
c = Cache(db=0)
total_visualizations = c.get('my_total_visualizations_' + str(self.id))
if not total_visualizations:
total_visualizations = Visualization.objects.filter(user=self.id).count()
if total_visualizations > 0:
c.set('my_total_visualizations_' + str(self.id), total_visualizations, settings.REDIS_STATS_TTL)
return total_visualizations
def get_total_visualizations(self):
c = Cache(db=0)
total_visualizations = c.get('my_total_visualizations_' + str(self.id))
if not total_visualizations:
total_visualizations = Visualization.objects.filter(user=self.id).count()
if total_visualizations > 0:
c.set('my_total_visualizations_' + str(self.id), total_visualizations, settings.REDIS_STATS_TTL)
return total_visualizations
def get_total_datastreams(self):
c = Cache(db=0)
total_datastreams = c.get('my_total_datastreams_' + str(self.id))
if not total_datastreams:
total_datastreams = DataStream.objects.filter(user=self.id).count()
if total_datastreams > 0:
c.set('my_total_datastreams_' + str(self.id), total_datastreams, settings.REDIS_STATS_TTL)
return total_datastreams
def get_total_datasets(self):
c = Cache(db=0)
users = User.objects.filter(account=self)
total_datasets = c.get('account_total_datasets_' + str(self.id))
if not total_datasets:
total_datasets = Dataset.objects.filter(user__in=users).count()
if total_datasets > 0:
c.set('account_total_datasets_' + str(self.id), total_datasets, settings.REDIS_STATS_TTL)
return total_datasets
def get_total_datasets(self):
c = Cache(db=0)
users = User.objects.filter(account=self)
total_datasets = c.get('account_total_datasets_' + str(self.id))
if not total_datasets:
total_datasets = Dataset.objects.filter(user__in=users).count()
#if settings.DEBUG: logger.info('get_total_datasets from database %d' % total_datasets)
if total_datasets > 0:
c.set('account_total_datasets_' + str(self.id), total_datasets, settings.REDIS_STATS_TTL)
#else:
# if settings.DEBUG: logger.info('get_total_datasets from cache %s' % total_datasets)
return total_datasets
def get_total_datasets(self):
c = Cache(db=0)
users = User.objects.filter(account=self)
total_datasets = c.get('account_total_datasets_' + str(self.id))
if not total_datasets:
total_datasets = Dataset.objects.filter(user__in=users).count()
if settings.DEBUG: logger.info('get_total_datasets from database %d' % total_datasets)
if total_datasets > 0:
c.set('account_total_datasets_' + str(self.id), total_datasets, settings.REDIS_STATS_TTL)
else:
if settings.DEBUG: logger.info('get_total_datasets from cache %s' % total_datasets)
return total_datasets
class VisualizationHitsDAO:
"""class for manage access to Hits in DB and index"""
doc_type = "vz"
from_cache = False
# cache ttl, 1 hora
TTL = 3600
def __init__(self, visualization):
self.visualization = visualization
self.search_index = ElasticsearchIndex()
self.logger = logging.getLogger(__name__)
self.cache = Cache()
def add(self, channel_type):
"""agrega un hit al datastream. """
try:
hit = VisualizationHits.objects.create(
visualization_id=self.visualization.visualization_id, channel_type=channel_type
)
except IntegrityError:
# esta correcto esta excepcion?
raise VisualizationNotFoundException()
self.logger.info("VisualizationHitsDAO hit! (guid: %s)" % (self.datastream.guid))
# armo el documento para actualizar el index.
doc = {
"docid": "%s::%s" % (self.doc_type.upper(), self.visualization.guid),
"type": self.doc_type,
"script": "ctx._source.fields.hits+=1",
}
return self.search_index.update(doc)
def count(self):
return VisualizationHits.objects.filter(visualization_id=self.visualization.visualization_id).count()
def _get_cache(self, cache_key):
cache = self.cache.get(cache_key)
return cache
def _set_cache(self, cache_key, value):
return self.cache.set(cache_key, value, self.TTL)
def count_by_day(self, day):
"""retorna los hits de un día determinado"""
# si es datetime, usar solo date
if type(day) == type(datetime.today()):
day = day.date()
cache_key = "%s_hits_%s_by_date_%s" % (self.doc_type, self.visualization.guid, str(day))
hits = self._get_cache(cache_key)
# si el día particular no esta en el caché, lo guarda
# salvo que sea el parametro pasado sea de hoy, lo guarda en el cache pero usa siempre el de la DB
if not hits or day == date.today():
hits = VisualizationHits.objects.filter(
visualization=self.visualization, created_at__startswith=day
).count()
self._set_cache(cache_key, hits)
return (date, hits)
def count_by_days(self, day=30, channel_type=None):
"""trae un dict con los hits totales de los ultimos day y los hits particulares de los días desde day hasta today"""
# no sé si es necesario esto
if day < 1:
return {}
cache_key = "%s_hits_%s_%s" % (self.doc_type, self.visualization.guid, day)
if channel_type:
cache_key += "_channel_type_%s" % channel_type
hits = self._get_cache(cache_key)
# me cachendié! no esta en la cache
if not hits:
# tenemos la fecha de inicio
start_date = datetime.today() - timedelta(days=day)
# tomamos solo la parte date
truncate_date = connection.ops.date_trunc_sql("day", "created_at")
qs = VisualizationHits.objects.filter(visualization=self.visualization, created_at__gte=start_date)
if channel_type:
qs = qs.filter(channel_type=channel_type)
hits = (
qs.extra(select={"_date": truncate_date, "fecha": "DATE(created_at)"})
.values("fecha")
.order_by("created_at")
.annotate(hits=Count("created_at"))
)
control = [date.today() - timedelta(days=x) for x in range(day - 1, 0, -1)]
control.append(date.today())
for i in hits:
try:
control.remove(i["fecha"])
except ValueError:
pass
hits = list(hits)
for i in control:
hits.append({"fecha": i, "hits": 0})
hits = sorted(hits, key=lambda k: k["fecha"])
# transformamos las fechas en isoformat
hits = map(self._date_isoformat, hits)
# lo dejamos, amablemente, en la cache!
self._set_cache(cache_key, json.dumps(hits, cls=DjangoJSONEncoder))
self.from_cache = False
else:
hits = json.loads(hits)
self.from_cache = True
return hits
def _date_isoformat(self, row):
row["fecha"] = row["fecha"].isoformat()
return row
class DatastreamHitsDAO():
"""class for manage access to Hits in DB and index"""
doc_type = "ds"
from_cache = False
# cache ttl, 1 hora
TTL=3600
def __init__(self, datastream):
self.datastream = datastream
#self.datastream_revision = datastream.last_published_revision
self.search_index = ElasticsearchIndex()
self.logger=logging.getLogger(__name__)
self.cache=Cache()
def add(self, channel_type):
"""agrega un hit al datastream. """
# TODO: Fix temporal por el paso de DT a DAO.
# Es problema es que por momentos el datastream viene de un queryset y otras veces de un DAO y son objetos
# distintos
try:
datastream_id = self.datastream.datastream_id
except:
datastream_id = self.datastream['datastream_id']
try:
guid = self.datastream.guid
except:
guid = self.datastream['guid']
try:
hit=DataStreamHits.objects.create(datastream_id=datastream_id, channel_type=channel_type)
except IntegrityError:
# esta correcto esta excepcion?
raise DataStreamNotFoundException()
self.logger.info("DatastreamHitsDAO hit! (guid: %s)" % ( guid))
# armo el documento para actualizar el index.
doc={'docid':"DS::%s" % guid,
"type": "ds",
"script": "ctx._source.fields.hits+=1"}
return self.search_index.update(doc)
def count(self):
return DataStreamHits.objects.filter(datastream_id=self.datastream['datastream_id']).count()
def _get_cache(self, cache_key):
cache=self.cache.get(cache_key)
return cache
def _set_cache(self, cache_key, value):
return self.cache.set(cache_key, value, self.TTL)
def count_by_days(self, day=30, channel_type=None):
"""trae un dict con los hits totales de los ultimos day y los hits particulares de los días desde day hasta today"""
# no sé si es necesario esto
if day < 1:
return {}
cache_key="%s_hits_%s_%s" % ( self.doc_type, self.datastream.guid, day)
if channel_type:
cache_key+="_channel_type_%s" % channel_type
hits = self._get_cache(cache_key)
# me cachendié! no esta en la cache
if not hits :
# tenemos la fecha de inicio
start_date=datetime.today()-timedelta(days=day)
# tomamos solo la parte date
truncate_date = connection.ops.date_trunc_sql('day', 'created_at')
qs=DataStreamHits.objects.filter(datastream=self.datastream,created_at__gte=start_date)
if channel_type:
qs=qs.filter(channel_type=channel_type)
hits=qs.extra(select={'_date': truncate_date, "fecha": 'DATE(created_at)'}).values("fecha").order_by("created_at").annotate(hits=Count("created_at"))
control=[ date.today()-timedelta(days=x) for x in range(day-1,0,-1)]
control.append(date.today())
for i in hits:
try:
control.remove(i['fecha'])
except ValueError:
pass
hits=list(hits)
for i in control:
hits.append({"fecha": i, "hits": 0})
hits = sorted(hits, key=lambda k: k['fecha'])
# transformamos las fechas en isoformat
hits=map(self._date_isoformat, hits)
# lo dejamos, amablemente, en la cache!
self._set_cache(cache_key, json.dumps(hits, cls=DjangoJSONEncoder))
self.from_cache=False
else:
hits=json.loads(hits)
self.from_cache = True
return hits
def _date_isoformat(self, row):
row['fecha']=row['fecha'].isoformat()
return row
def workspace_open_data_metrics(auth_manager):
from core.cache import Cache
from datetime import date, timedelta
from django.db import connection
cursor = connection.cursor()
user_id = auth_manager.id
account_id = auth_manager.account_id
language = auth_manager.language
last_7_days = date.today() - timedelta(days=7)
c = Cache(db=0)
published_datasets = c.get('published_datasets_' + str(account_id))
if not published_datasets:
cursor.execute("SELECT COUNT(1) as val FROM ao_datasets d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_dataset_revisions b WHERE b.dataset_id = d.id AND NOT EXISTS(SELECT * FROM ao_dataset_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.dataset_id = c.dataset_id) AND b.status = 3)", [str(account_id)])
row = cursor.fetchone()
published_datasets = row[0]
c.set('published_datasets_' + str(account_id), published_datasets, settings.REDIS_STATS_TTL)
total_datasets = c.get('total_datasets_' + str(account_id))
if not total_datasets:
total_datasets = Dataset.objects.filter(user__account=account_id).count()
c.set('total_datasets_' + str(account_id), total_datasets, settings.REDIS_STATS_TTL)
published_datastreams = c.get('published_datastreams_' + str(account_id))
if not published_datastreams:
cursor.execute("SELECT COUNT(1) as val FROM ao_datastreams d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_datastream_revisions b WHERE b.datastream_id = d.id AND NOT EXISTS(SELECT * FROM ao_datastream_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.datastream_id = c.datastream_id) AND b.status = 3)", [str(account_id)])
row = cursor.fetchone()
published_datastreams = row[0]
c.set('published_datastreams_' + str(account_id), published_datastreams, settings.REDIS_STATS_TTL)
total_datastreams = c.get('total_datastreams_' + str(account_id))
if not total_datastreams:
total_datastreams = DataStream.objects.filter(user__account=account_id).count()
c.set('total_datastreams_' + str(account_id), total_datastreams, settings.REDIS_STATS_TTL)
published_dashboards = c.get('published_dashboards_' + str(account_id))
if not published_dashboards:
cursor.execute("SELECT COUNT(1) as val FROM ao_dashboards d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_dashboard_revisions b WHERE b.dashboard_id = d.id AND NOT EXISTS(SELECT * FROM ao_dashboard_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.dashboard_id = c.dashboard_id) AND b.status = 3)", [str(account_id)])
row = cursor.fetchone()
published_dashboards = row[0]
c.set('published_dashboards_' + str(account_id), published_dashboards, settings.REDIS_STATS_TTL)
published_visualizations = c.get('published_visualizations_' + str(account_id))
if not published_visualizations:
cursor.execute("SELECT COUNT(1) as val FROM ao_visualizations d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_visualizations_revisions b WHERE b.visualization_id = d.id AND NOT EXISTS(SELECT * FROM ao_visualizations_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.visualization_id = c.visualization_id) AND b.status = 3)", [str(account_id)])
row = cursor.fetchone()
published_visualizations = row[0]
c.set('published_visualizations_' + str(account_id), published_visualizations, settings.REDIS_STATS_TTL)
total_visualizations = c.get('total_visualizations_' + str(account_id))
if not total_visualizations:
total_visualizations = Visualization.objects.filter(user__account=account_id).count()
c.set('total_visualizations_' + str(account_id), total_visualizations, settings.REDIS_STATS_TTL)
return locals()
def workspace_open_data_metrics(auth_manager):
from core.cache import Cache
from datetime import date, timedelta
from django.db import connection
cursor = connection.cursor()
user_id = auth_manager.id
account_id = auth_manager.account_id
language = auth_manager.language
last_7_days = date.today() - timedelta(days=7)
c = Cache(db=0)
published_datasets = c.get('published_datasets_' + str(account_id))
if not published_datasets:
cursor.execute(
"SELECT COUNT(1) as val FROM ao_datasets d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_dataset_revisions b WHERE b.dataset_id = d.id AND NOT EXISTS(SELECT * FROM ao_dataset_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.dataset_id = c.dataset_id) AND b.status = 3)",
[str(account_id)])
row = cursor.fetchone()
published_datasets = row[0]
c.set('published_datasets_' + str(account_id), published_datasets,
settings.REDIS_STATS_TTL)
total_datasets = c.get('total_datasets_' + str(account_id))
if not total_datasets:
total_datasets = Dataset.objects.filter(
user__account=account_id).count()
c.set('total_datasets_' + str(account_id), total_datasets,
settings.REDIS_STATS_TTL)
published_datastreams = c.get('published_datastreams_' + str(account_id))
if not published_datastreams:
cursor.execute(
"SELECT COUNT(1) as val FROM ao_datastreams d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_datastream_revisions b WHERE b.datastream_id = d.id AND NOT EXISTS(SELECT * FROM ao_datastream_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.datastream_id = c.datastream_id) AND b.status = 3)",
[str(account_id)])
row = cursor.fetchone()
published_datastreams = row[0]
c.set('published_datastreams_' + str(account_id),
published_datastreams, settings.REDIS_STATS_TTL)
total_datastreams = c.get('total_datastreams_' + str(account_id))
if not total_datastreams:
total_datastreams = DataStream.objects.filter(
user__account=account_id).count()
c.set('total_datastreams_' + str(account_id), total_datastreams,
settings.REDIS_STATS_TTL)
published_visualizations = c.get('published_visualizations_' +
str(account_id))
if not published_visualizations:
cursor.execute(
"SELECT COUNT(1) as val FROM ao_visualizations d JOIN ao_users u ON u.id=d.user_id JOIN ao_accounts ac ON u.account_id=ac.id WHERE ac.id = %s and EXISTS(SELECT * FROM ao_visualizations_revisions b WHERE b.visualization_id = d.id AND NOT EXISTS(SELECT * FROM ao_visualizations_revisions c WHERE c.created_at > b.created_at AND c.status = 4 AND b.visualization_id = c.visualization_id) AND b.status = 3)",
[str(account_id)])
row = cursor.fetchone()
published_visualizations = row[0]
c.set('published_visualizations_' + str(account_id),
published_visualizations, settings.REDIS_STATS_TTL)
total_visualizations = c.get('total_visualizations_' + str(account_id))
if not total_visualizations:
total_visualizations = Visualization.objects.filter(
user__account=account_id).count()
c.set('total_visualizations_' + str(account_id), total_visualizations,
settings.REDIS_STATS_TTL)
return locals()
