def populate(self): results = (self.get_query() .order_by(desc(self.count_col)) .limit(10) .all()) visits = [{ 'label': ( parse_referrer(visit.key, host_only=True, second_pass=True) if self.criteria_name == 'pretty_referrer' else visit.key), 'data': visit.count } for visit in results] all_visits = (self.filter(self.db .query(self.count_col.label("all"))) .first()).all or 0 other = all_visits - sum(visit['data'] for visit in visits) if other: visits = visits + [{'label': 'Other', 'data': other}] for visit in visits: self.chart.add(str(visit['label']), float(visit['data']))
def process(self, db): self.log.debug('Processing message %r' % self) visits = Visit.__table__ if len(self.qs_args) == 0: self.log.warn('No params in request. Must be a bot.') return None, False def get(key, default=None, from_encoding=None): value = self.qs_args.get(key, [default])[0] if value and value == b'undefined': value = None if value is not None: if from_encoding: value = value.decode(from_encoding) else: value = try_decode(value) return value visit = None kind = get('d') if kind == 'e': raise RuntimeError('Problem in tracker: "%s"' % get('r')) uuid = get('_') if '; ' in uuid: uuid = uuid.split('; ')[0] platform, browser, version = parse_ua(self.ua) if kind == 'c': try: id = db.execute( select([visits.c.id]) .where(visits.c.uuid == uuid) .order_by(desc(visits.c.date)) ).fetchone()['id'] except: self.log.warn('Could not find uuid %s' % uuid, exc_info=True) else: db.execute( visits .update() .where(visits.c.id == id) .values(time=timedelta(seconds=int(get('t', 0)) / 1000))) self.log.debug('%r inserted' % self) return uuid, False elif kind == 'o': last_visit = get('l') if last_visit and 'undefined' not in last_visit: last_visit = datetime.fromtimestamp(int(last_visit) / 1000) else: last_visit = None visit = {'uuid': uuid, 'host': get('k'), 'site': get('u'), 'client_tz_offset': int(float(get('z', 0))), 'date': self.stamp, 'last_visit': last_visit, 'ip': self.ip, 'referrer': get('r'), 'pretty_referrer': parse_referrer( get('r', from_encoding='latin-1')), 'referrer_domain': parse_domain( get('r', from_encoding='latin-1')), 'size': get('s'), 'page': get('p', from_encoding='latin-1'), 'hash': get('h', from_encoding='latin-1'), 'query': get('q'), 'language': get('i'), 'browser_name': browser, 'browser_version': version, 'platform': platform} lat = None lng = None city_name = None country_code = None country_name = None asn_name = None if self.ip.startswith('::ffff'): self.ip = self.ip.replace('::ffff:', '') if (self.ip == '127.0.0.1' or self.ip.startswith('192.168.') or self.ip.startswith('10.')): city_name = 'Local' country_name = 'Local' asn_name = 'Local' elif self.ip != 'unknown': countries = list(db.execute( country.select().where( country.c.ipr.op('>>=')(visit['ip'])))) if len(countries) > 0: country_name = countries[0].country_name country_code = countries[0].country_code cities = list(db.execute( city.select().where( city.c.ipr.op('>>=')(visit['ip'])))) if len(cities) > 0: city_name = cities[0].city lat = cities[0].latitude lng = cities[0].longitude asns = list(db.execute( asn.select().where( asn.c.ipr.op('>>=')(visit['ip'])))) if len(asns) > 0: asn_name = asns[0].asn visit['country'] = country_name visit['country_code'] = country_code visit['city'] = city_name visit['lat'] = lat visit['lng'] = lng visit['asn'] = asn_name domain_parts = visit['host'].split('.') if len(domain_parts) > 2: visit['subdomain'] = '.'.join(domain_parts[:-2]) else: visit['subdomain'] = None visit['domain'] = '.'.join(domain_parts[-2:]) visit['day'] = visit['date'].date() visit['hour'] = visit['date'].hour browser_minor_version = '' visit['browser_version'] = visit['browser_version'] or '' if (visit['browser_version'] and visit['browser_name'] not in ( 'opera', 'safari', 'chrome') and len( visit['browser_version'].split('.')) > 1): browser_minor_version = '.%s' % visit[ 'browser_version'].split('.')[1] visit['browser_name_version'] = '%s %s%s' % ( visit['browser_name'], visit['browser_version'].split('.')[0], browser_minor_version) visit['id'] = db.execute( select([VisitIdSeq.next_value()])).scalar() db.execute(visits.insert().returning(visits.c.id), **visit) self.log.debug('%r inserted' % self) return visit, True raise NotImplementedError('Unknown kind %s' % kind)