async def get_data(self):
coll_name = self._collection_name
resp_data = await self.make_request('get')
if self._cached:
self.end_datetime = utils.now()
else:
prev = await self.find_previous()
if prev is not None:
self.start_datetime = utils.now()
self.end_datetime = self.start_datetime + datetime.timedelta(seconds=1)
prev['end_datetime'] = self.start_datetime
prev['is_complete'] = True
await self.db_store.update_doc(coll_name, {'_id':prev['_id']}, prev)
self.referrers = {}
for d in resp_data:
key = d['referrer']
r = self.referrers.get(key)
if r is not None:
exists = True
r.count = d['count']
r.uniques = d['uniques']
else:
exists = False
rkwargs = {'traffic_referrals':self}
rkwargs.update(d)
r = TrafficReferrer(**rkwargs)
self.referrers[key] = r
async def get_traffic_paths(self, now=None):
if now is None:
now = utils.now()
now_ts = utils.dt_to_timestamp(now)
tp = RepoTrafficPaths(repo=self, datetime=now, db_store=self.db_store)
self.traffic_paths = tp
await tp.get_data()
return tp
async def get_data(self, now=None):
if now is None:
now = utils.now()
tasks = [
asyncio.ensure_future(self.get_traffic_views(now=now)),
asyncio.ensure_future(self.get_traffic_paths(now=now)),
asyncio.ensure_future(self.get_traffic_referrals(now=now)),
]
await asyncio.wait(tasks)
async def get_repo_data(self, now=None):
if now is None:
now = utils.now()
tasks = []
for repo in self.repos.values():
tasks.append(asyncio.ensure_future(repo.get_data(now=now)))
await asyncio.wait(tasks)
if self.db_store is not None:
await self.store_to_db()
async def get_traffic_referrals(self, now=None):
if now is None:
now = utils.now()
tkwargs = {'repo':self, 'db_store':self.db_store}
tr = await RepoTrafficReferrals.find_last_item(**tkwargs)
if tr is None:
tr = RepoTrafficReferrals(**tkwargs)
self.traffic_referrals = tr
await tr.get_data()
return tr
async def get_traffic_views(self, per='day', now=None):
if now is None:
now = utils.now()
now_ts = utils.dt_to_timestamp(now)
tv = RepoTrafficViews(
repo=self, per=per, datetime=now, db_store=self.db_store,
)
self.traffic_views = tv
await tv.get_data()
return tv
def build_datetime_filter(filter_key, **kwargs):
start_dt = kwargs.get('start_datetime')
end_dt = kwargs.get('end_datetime', utils.now())
if start_dt is None:
filt = {filter_key:{'$lte':end_dt}}
else:
filt = {'$and':[
{filter_key:{'$gte':start_dt}},
{filter_key:{'$lte':end_dt}},
]}
return filt
def __init__(self, **kwargs):
self._repo_slug = kwargs.get('repo_slug')
self.repo = kwargs.get('repo')
kwargs.setdefault('db_store', self.repo.db_store)
kwargs['request_handler'] = self.repo.request_handler
super().__init__(**kwargs)
start_dt = utils.now()
end_dt = start_dt + datetime.timedelta(seconds=1)
self.start_datetime = kwargs.get('start_datetime', start_dt)
self.end_datetime = kwargs.get('end_datetime', end_dt)
self.is_complete = kwargs.get('is_complete', False)
self.referrers = kwargs.get('referrers', {})
async def store_to_db(self, log_timestamp=None):
if log_timestamp is None:
log_timestamp = utils.now()
db_store = self.db_store
await self.log_db_update(log_timestamp, 'repos', 0)
tasks = []
for repo in self.repos.values():
tasks.append(asyncio.ensure_future(repo.store_to_db(log_timestamp)))
await asyncio.wait(tasks)
log_doc = await self.get_db_update_log(log_timestamp)
for coll_name, update_count in log_doc['collection_updates'].items():
logger.info('{} Updates: {}'.format(coll_name, update_count))
logger.info('Total Updates: {}'.format(log_doc['total_updates']))
def update_context_dt_range(request, context=None):
if context is None:
context = {}
for key in ['start_datetime', 'end_datetime']:
dt = request.query.get(key)
if isinstance(dt, str) and not len(dt):
dt = None
if dt is None:
if key in context:
del context[key]
else:
if isinstance(dt, str) and not dt.endswith('Z'):
dt = '{}Z'.format(dt)
dt = parse_query_dt(dt)
context[key] = dt
context['{}_str'.format(key)] = utils.dt_to_str(dt)
if not context.get('end_datetime'):
now = utils.now()
context['end_datetime'] = now
context['end_datetime_str'] = utils.dt_to_str(now)
return context