Example #1
0
def get_client_details(request) -> dict:
    '''
    Given valid request returns user's ip, location and user-agent.
    '''
    if not request:
        return dict()

    client_ip, is_routable = get_client_ip(request)
    client_ip = client_ip if client_ip and is_routable else None
    location = ''

    if client_ip:
        result = GeoIP2().city(client_ip)
        location = '{city}, {country}'.format(
            country=result.get('country_name', ''),
            city=result.get('city', ''),
        )

    user_agent = request.META.get('HTTP_USER_AGENT', '')

    if user_agent:
        user_agent = str(parse_ua(user_agent))

    return {
        'ip': client_ip,
        'location': location,
        'user_agent': user_agent,
    }
Example #2
0
def hlar_top(request):
    current_site = get_current_site(request)
    print(current_site.domain)

    if request.user.is_authenticated() == False:
        try:
            # oauth で返ってきた時はsessionにid が入っているのでそれを取得する。
            user = User.objects.filter(
                id=request.session._session_cache['_auth_user_id'])[0]

            user_auth = authenticate(username=user.email,
                                     password=DEFAULT_PASS)
            login(request, user_auth)

        except Exception as e:
            print('error')

    # 人気ターゲット一覧を取得
    targets = get_targets_popular()

    ua = parse_ua(request.META['HTTP_USER_AGENT'])

    return render(
        request,
        'hlar/hlar_top.html',  # 使用するテンプレート
        {
            'user': request.user,
            'msg': _("使い方"),
            'targets': targets,
            's3_FQDN': s3_FQDN,
            'is_mobile': ua.is_mobile,
        }  # テンプレートに渡すデータ
    )
Example #3
0
 def setUp(self):
     super(ReportRequestRenderedViewTests, self).setUp()
     cache_key = 'request:%s' % self.request_id
     data = {
         'started': 5000,
         'server_time': 150,
         'agent': parse_ua(CHROME_UA),
         'view_name': 'bar',
     }
     cache.set(cache_key, data)
Example #4
0
 def process(self, device):
     """Process a device to compute it's privacy features and PI."""
     features = {}
     if device.user_agent:
         ua = parse_ua(device.user_agent)
         browser_version = ua.browser.version_string.lower()
         features.update({
             'browser_family': ua.browser.family.lower(),
             'browser_version': browser_version,
             'os_family': ua.os.family.lower(),
             'os_version': ua.os.version_string.lower(),
             'device_family': ua.device.family.lower(),
             'device_type': self._get_device_type(ua)
         })
     # XXX processing IP address to detect some informations
     features.update(self._process_ip_address(device.ip_creation))
     return None, features
Example #5
0
 def process(self, device):
     """Process a device to compute it's privacy features and PI."""
     features = {}
     if device.user_agent:
         ua = parse_ua(device.user_agent)
         browser_version = ua.browser.version_string.lower()
         features.update({'browser_family': ua.browser.family.lower(),
                          'browser_version': browser_version,
                          'os_family': ua.os.family.lower(),
                          'os_version': ua.os.version_string.lower(),
                          'device_family': ua.device.family.lower(),
                          'device_type': self._get_device_type(ua)})
     # XXX processing IP address to detect some informations
     if device.ip_creation:
         features.update(self._process_ip_address(device.ip_creation))
     else:
         log.warn('No ip address found for device')
     return None, features
Example #6
0
def target_list(request):

    if request.user.is_authenticated() == False:
        return HttpResponseRedirect('/accounts/login/?next=%s' % request.path)

    ua = parse_ua(request.META['HTTP_USER_AGENT'])

    # ターゲット一覧を取得
    targets = Target.objects.filter(user_id=str(request.user.id),
                                    del_flg=False)

    addTarget = True
    if len(targets) >= settings.TARGET_LIMIT_COUNT:
        addTarget = False

    for target in targets:
        # シェアのリンクを作成
        arrContentName = target.img_name.split(".")
        targetImgURL = settings.URL_ROOT + "hlar/target/preview_img/" + arrContentName[
            0]

        # Twitter
        twitterText = _("ARアプリ【UNIVERSE.AR】でこの画像を読み取ってみましょう! #universear")
        twitterParam = {'text': twitterText, 'url': targetImgURL}
        target.twitter_url = 'https://twitter.com/share?' + urllib.parse.urlencode(
            twitterParam)

        # facebook
        facebookParam = {'u': targetImgURL}
        target.fb_url = 'https://www.facebook.com/share.php?' + urllib.parse.urlencode(
            facebookParam)

    return render(
        request,
        'hlar/target_list.html',  # 使用するテンプレート
        {
            'targets': targets,
            's3_FQDN': s3_FQDN,
            'is_mobile': ua.is_mobile,
            'addTarget': addTarget,
            'TARGET_LIMIT_COUNT': settings.TARGET_LIMIT_COUNT,
        })  # テンプレートに渡すデータ
Example #7
0
    def prepare_email_data(self, markers):
        data = defaultdict(lambda: defaultdict(list))
        """email -> {link -> [{ip, date}]}"""
        for marker_path, content_string in markers.iteritems():
            content = json.loads(content_string)
            link = self.obtain_url(content.get('token'), content.get('path'))
            marker_name = os.path.basename(marker_path)
            address, timestamp, padding = marker_name.rsplit('.', 2)
            try:
                timestamp = int(timestamp)
            except ValueError:
                timestamp = None
            if timestamp:
                date = datetime.fromtimestamp(timestamp).isoformat(sep=b' ')
                date += ' (UTC)'
            else:
                date = None
            item = {
                'date': date,
                'ip': content.get('ip'),
            }

            # Process user agent:
            ua = parse_ua(content.get('user_agent', ''))
            unknown = 'Other'
            if ua.browser.family != unknown:
                item['browser'] = (ua.browser.family + ' ' +
                                   ua.browser.version_string).strip()
            if ua.os.family != unknown:
                item['os'] = (ua.os.family + ' ' +
                              ua.os.version_string).strip()
            if ua.device.family != unknown:
                item['device'] = ua.device.family

            data[address][link].append(item)

        for items in data.itervalues():
            for l in items.itervalues():
                l.sort(key=lambda d: (d['date'], d['ip']))

        return data
Example #8
0
def serialize_event(event):
    d = {
        'org_id': event.org_id,
        'user_id': event.user_id,
        'action': event.action,
        'object_type': event.object_type,
        'object_id': event.object_id,
        'created_at': event.created_at
    }

    if event.user_id:
        d['user_name'] = event.additional_properties.get('user_name', 'User {}'.format(event.user_id))

    if not event.user_id:
        d['user_name'] = event.additional_properties.get('api_key', 'Unknown')

    d['browser'] = str(parse_ua(event.additional_properties.get('user_agent', '')))
    d['location'] = get_location(event.additional_properties.get('ip'))
    d['details'] = event_details(event)

    return d
    def gather_view_data(self, request, view_func):
        """Discover the view name.
        """
        # View name is defined as module.view
        # (e.g. django.contrib.auth.views.login)
        name = view_func.__module__

        # CBV specific
        if hasattr(view_func, '__name__'):
            name = '%s.%s' % (name, view_func.__name__)
        elif hasattr(view_func, '__class__'):
            name = '%s.%s' % (name, view_func.__class__.__name__)
        method = request.method.lower()
        if request.is_ajax():
            method += '_ajax'
        name = '%s.%s' % (name, method)

        self.scope.id = id_request(request)

        self.scope.agent = parse_ua(request.META.get('HTTP_USER_AGENT', ''))
        self.scope.view_name = "view." + name
Example #10
0
def serialize_event(event):
    d = {
        'org_id': event.org_id,
        'user_id': event.user_id,
        'action': event.action,
        'object_type': event.object_type,
        'object_id': event.object_id,
        'created_at': event.created_at
    }

    if event.user_id:
        d['user_name'] = event.additional_properties.get('user_name', 'User {}'.format(event.user_id))

    if not event.user_id:
        d['user_name'] = event.additional_properties.get('api_key', 'Unknown')

    d['browser'] = str(parse_ua(event.additional_properties.get('user_agent', '')))
    d['location'] = get_location(event.additional_properties.get('ip'))
    d['details'] = event_details(event)

    return d
Example #11
0
def serialize_event(event):
    d = {
        "org_id": event.org_id,
        "user_id": event.user_id,
        "action": event.action,
        "object_type": event.object_type,
        "object_id": event.object_id,
        "created_at": event.created_at,
    }

    if event.user_id:
        d["user_name"] = event.additional_properties.get(
            "user_name", "User {}".format(event.user_id)
        )

    if not event.user_id:
        d["user_name"] = event.additional_properties.get("api_key", "Unknown")

    d["browser"] = str(parse_ua(event.additional_properties.get("user_agent", "")))
    d["location"] = get_location(event.additional_properties.get("ip"))
    d["details"] = event_details(event)

    return d
Example #12
0
def lookup(urlcode):
    """
    Endpoint that recieves the short url and redirects to the long url target
    Case insensitive for better usability

    :param urlcode: The code of the short url
    :returns: redirects to target
    """
    urlcode = urlcode.lower()
    result = sht.short_to_long(urlcode)
    if result:
        url = result['default_url']
        target = 'default'
        ua = parse_ua(request.headers.get('User-Agent'))
        if result['mobile_url'] and ua.is_mobile:
            url = result['mobile_url']
            target = 'mobile'
        elif result['tablet_url'] and ua.is_tablet:
            url = result['tablet_url']
            target = 'tablet'
        sht.hit(urlcode, target)
        return redirect(url)
        return Response(str(result['mobile_url ']))
    return Response('{"error":"URL NOT FOUND"}', status=404, mimetype="application/json")
def get_event_counts(entry):
    result = []
    for e in events:        
        ename = e.replace("Event: ", "")
        count = len(filter(lambda x: x['details'] != None and x['details'].startswith(ename),
                           entry.logs))
        result.append(count)
    return result
        

for key in logs:
    entry = logs[key]
    if not entry.user_agent or key[1] == 0 or len(entry.user) > 1: 
        continue
    ua = parse_ua(entry.user_agent)

    row = [entry.ip,
           entry.user_agent,
           ua.os.family,
           ua.device.family,
           ua.browser.family,
           int(ua.is_mobile or ua.is_tablet or ua.is_touch_capable),
           int(ua.is_pc),
           len(entry.user),
    ]

    if len(entry.user) == 1:
        u = entry.user[0]
    else:
        u = none_dict
Example #14
0
    def write(cursor, lock, msg=None):
        if uwsgi_mode:
            msg = pickle.loads(msg)
#        msg = self.msg

        if msg.cached:
            cached = '*'
        else:
            cached = ''
        if msg.compress:
            compress = '*'
        else:
            compress = ''
        if hasattr(settings,'ROOT_URL'):
            if msg.url.startswith(settings.ROOT_URL):
                url = msg.url[len(settings.ROOT_URL):]
            else:
                url = msg.url
            if msg.referer:
                if msg.referer.startswith(settings.ROOT_URL):
                    referer = msg.referer[len(settings.ROOT_URL):]
                else:
                    referer = msg.referer
            else:
                referer = '*'
        else:
            url = msg.url
            if msg.referer:
                referer = msg.referer
            else:
                referer = '*'
        if msg.user_id:
            user_id = msg.user_id
        else:
            user_id = '-'

        if msg.bot:
            bot = '*'
        else:
            bot = ''

        if METHOD_CHOICES_DICT[msg.request_method] == 'GET':
            method = '-'
        elif METHOD_CHOICES_DICT[msg.request_method] == 'POST':
            method = '+'
        else:
            method = '?'
        if msg.ajax:
            direction = f'<{method}'
        else:
            direction = f'{method}>'

        ua_string = msg.user_agent[:252] + (msg.user_agent[252:] and '..') if msg.user_agent else None
        if ua_string:
            user_agent = parse_ua(ua_string)
        else:
            user_agent = parse_ua('')
            msg.bot = True
        if len(user_agent.browser.version) > 0:
            browser_major_version = user_agent.browser.version[0]
        else:
            browser_major_version = None
        if len(user_agent.browser.version) > 1:
            browser_minor_version = user_agent.browser.version[1]
        else:
            browser_minor_version = None
        if len(user_agent.browser.version) > 2:
            browser_patch = user_agent.browser.version[2]
        else:
            browser_patch = None
        if len(user_agent.os.version) > 0:
            os_major_version = user_agent.os.version[0]
        else:
            os_major_version = None
        if len(user_agent.os.version) > 1:
            os_minor_version = user_agent.os.version[1]
        else:
            os_minor_version = None
        if len(user_agent.os.version) > 2:
            os_patch = user_agent.os.version[2]
        else:
            os_patch = None
        if len(user_agent.os.version) > 3:
            os_minor_patch = user_agent.os.version[3]
        else:
            os_minor_patch = None
        fields = [
                    msg.timestamp,
                    msg.ip,
                    msg.response_time,
                    msg.status_code,
                    msg.url[:508] + (msg.url[508:] and '..') if msg.url else None,
                    msg.request_content_type[:48] + (msg.request_content_type[48:] and '..') if msg.request_content_type else None,
                    msg.request_method,
                    msg.ajax,
                    msg.referer[:508] + (msg.referer[508:] and '..') if msg.referer else None,
                    ua_string,
                    msg.request_content_length,
                    msg.accept[:252] + (msg.accept[252:] and '..') if msg.accept else None,
                    msg.accept_language[:48] + (msg.accept_language[48:] and '..') if msg.accept_language else None,
                    msg.accept_encoding[:48] + (msg.accept_encoding[48:] and '..') if msg.accept_encoding else None,
                    msg.response_content_type[:48] + (msg.response_content_type[48:] and '..') if msg.response_content_type else None,
                    msg.response_content_length,
                    msg.compress,
                    msg.session_key,
                    msg.user_id,
                    msg.latitude,
                    msg.longitude,
                    msg.protocol,
                    msg.cached,
                    msg.session_start_time,
                    msg.preview,
                    msg.prefetch,
                    msg.bot,
                ]
        with lock:
        
            try:
                cursor.execute("execute " + LogWriter.log_sql + "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);", fields)
                result = cursor.fetchone()
#            print(f'{dir(result)=}')
#            dir(result)=['__add__', '__class__', '__contains__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__getnewargs__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', '__module__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__rmul__', '__setattr__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', '_asdict', '_field_defaults', '_fields', '_fields_defaults', '_make', '_replace', 'accept_encoding_id', 'accept_language_id', 'accept_type_id', 'ajax', 'cached', 'compress', 'connect_time', 'count', 'host_name_id', 'id', 'index', 'ip_id', 'latitude', 'log_timestamp', 'longitude', 'lookup_time', 'method', 'prefetch', 'preview', 'protocol', 'referer_url_id', 'request_content_length', 'request_content_type_id', 'request_url_id', 'response_content_length', 'response_content_type_id', 'response_time', 'session_id', 'session_log_id', 'ssl_time', 'status', 'timestamp', 'user_agent_id', 'user_id']

            except psycopg2.errors.StringDataRightTruncation:
                logger.error("Log entry contains too many characters in fields. Please review fields:")
                count = 0
                for i in fields:
                    logger.error(f'{c}: length {len(i)}: {i}')
            host = None
            cursor.execute('execute get_host(%s);' , [result.ip_id])
            host_result = cursor.fetchone()
            if host_result.hostname:
                host = host_result.hostname
                if host_result.domain_id == None:
                    names = host.split(".")[1:]
                    if names:
                        domain = names[-1]
                        if len(names) > 1:
                            domain = names[-2] + '.' + domain
                        if len(names) > 2:
                            domain = names[-3] + "." + domain
                        try:
                            cursor.execute('execute create_domain(%s, %s);' ,
                            [domain, msg.bot])
                        except psycopg2.errors.StringDataRightTruncation:
                            logger.error("Log create_domain entry contains too many characters for domain name.")
                            logger.error(f'length {len(domain)}: {domain}')

                        domain_result = cursor.fetchone()
                        if domain_result:
                            cursor.execute('execute update_host_domain(%s, %s);' ,
                                [host_result.host_id, domain_result.id])
                    else:
                        domain = '-'
                else:
                    domain = host_result.domainname
            else:
                host = None
                try:
                    host = socket.gethostbyaddr(msg.ip)[0]
                except:
                    domain = '-'
                if host:
                    names = host.split(".")[1:]
                    if len(names) <= 1:
                        logger = logging.warning(f"No hostname found for host {host}")
                        names = host.split(".")
                    domain = names[-1]
                    if len(names) > 1:
                        domain = names[-2] + '.' + domain
                    if len(names) > 2:
                        domain = names[-3] + "." + domain
                    cursor.execute('execute create_domain(%s, %s);' ,
                        [domain, msg.bot])
                    domain_result = cursor.fetchone()
                    if domain_result:
                        if host_result.host_id != None:
                            cursor.execute('execute update_host_domain(%s, %s);' ,
                                [host_result.host_id, domain_result.id])
                        else:
                            try:
                                cursor.execute('execute update_host(%s, %s, %s, %s);' ,
                                    [result.ip_id, domain_result.id, host, msg.bot])
                            except psycopg2.errors.StringDataRightTruncation:
                                logger.error("Log update_host entry contains too many characters for host name!")
                                logger.error(f'length {len(host)}: {host}')

            cursor.execute('execute get_user_agent(%s);' , [result.user_agent_id])
            useragent_result = cursor.fetchone()
            if useragent_result:
                if useragent_result.browser_id is None:
                    cursor.execute(
                        'execute update_browser(%s, %s, %s, %s, %s);' ,
                        [
                            result.user_agent_id,
                            user_agent.browser.family,
                            browser_major_version,
                            browser_minor_version,
                            browser_patch,
                        ])

                if useragent_result.os_id is None:
                    cursor.execute(
                        'execute update_os(%s, %s, %s, %s, %s, %s);' ,
                        [
                            result.user_agent_id,
                            user_agent.os.family,
                            os_major_version,
                            os_minor_version,
                            os_patch,
                            os_minor_patch,
                        ])

                if useragent_result.device_id is None:
                    cursor.execute(
                        'execute update_device(%s, %s, %s, %s, %s, %s, %s, %s, %s);' ,
                        [
                            result.user_agent_id,
                            user_agent.device.family,
                            user_agent.device.brand,
                            user_agent.device.model,
                            user_agent.is_mobile,
                            user_agent.is_pc,
                            user_agent.is_tablet,
                            user_agent.is_touch_capable,
                            user_agent.is_bot
                        ])

            cursor.execute('execute record_timestamp(%s);', [ result.id ])
            log_timestamp_result = cursor.fetchone()

        if user_agent.browser.family != 'Other':
            browser = f'{user_agent.browser.family}'
            if browser_major_version != None:
                browser = f'{browser} {browser_major_version}'
            if browser_minor_version != None:
                browser = f'{browser}.{browser_minor_version}'
            if user_agent.os.family == 'Other':
                os = ''
            else:
                os = f', {user_agent.os.family}'
                if os_major_version != None:
                    os = f'{os} {os_major_version}'
                if os_minor_version != None:
                    os = f'{os}.{os_minor_version}'
            bot = ''
            device = ''
            if user_agent.device.family in [
                'Spider', 'Generic Smartphone', 'Generic Desktop']:
                bot = '*'
            elif user_agent.device.family != 'Other':
                if user_agent.device.brand:
                    device = f', {user_agent.device.brand}'
                    if user_agent.device.model:
                        device = f'{device} {user_agent.device.model}'
                else:
                    device = f', {user_agent.device.family}'
            ua = f'({browser}{os}{device}){bot}'
        else:
            if ua_string == '':
                ua = '(Empty User Agent)'
            else:
                ua = f'({ua_string})*'

        timestamp = timezone.localtime(result.timestamp).strftime("%Y-%m-%d %H:%M:%S")
        log_response_time = (log_timestamp_result.log_timestamp-result.timestamp).microseconds/1000
#        log_response_time = (time.perf_counter()-msg.perf_counter)*1000
        log_delay = log_response_time - result.response_time
        analytics_logger.info(
            f'{msg.ip} '
            f'{domain} '
            f'{user_id} '
            f'{result.response_time:.3f}ms{cached} '
            f'{log_delay:.3f}ms '
            f'{msg.response_content_length}b{compress} '
            f'{msg.status_code} '
            f'{referer} {direction} {url} '
            f'{ua} '
        )
Example #15
0
    def __str__(self):
        if self.bot == False:
            color = RED
        else:
            color = ''

        user_agent = parse_ua(self.user_agent_string)
        if len(user_agent.browser.version) > 0:
            browser_major_version = user_agent.browser.version[0]
        else:
            browser_major_version = None
        if len(user_agent.browser.version) > 1:
            browser_minor_version = user_agent.browser.version[1]
        else:
            browser_minor_version = None
        if len(user_agent.os.version) > 0:
            os_major_version = user_agent.os.version[0]
        else:
            os_major_version = None
        if len(user_agent.os.version) > 1:
            os_minor_version = user_agent.os.version[1]
        else:
            os_minor_version = None

        if user_agent.browser.family != 'Other':

            browser = f'{user_agent.browser.family}'
            if browser_major_version != None:
                browser = f'{browser} {browser_major_version}'
            if browser_minor_version != None:
                browser = f'{browser}.{browser_minor_version}'
            if user_agent.os.family == 'Other':
                os = ''
            else:
                os = f', {user_agent.os.family}'
                if os_major_version != None:
                    os = f'{os} {os_major_version}'
                if os_minor_version != None:
                    os = f'{os}.{os_minor_version}'
            bot = ''
            device = ''
            if user_agent.device.family in [
                    'Spider', 'Generic Smartphone', 'Generic Desktop'
            ]:
                bot = '*'
            elif user_agent.device.family != 'Other':
                if user_agent.device.brand:
                    device = f', {user_agent.device.brand}'
                    if user_agent.device.model:
                        device = f'{device} {user_agent.device.model}'
                else:
                    device = f', {user_agent.device.family}'
            ua = f'{browser}{os}{device}'
        else:
            ua = f'({self.user_agent_string})*'
            bot = ''

        return format_html(
            '{}<span style="color: {};">{}</span>',
            ua,
            color,
            bot,
        )