Example #1
0
 def device_func(self, obj):
     """
     Reformat the device format. In this, values have been sent with the hits, but if User Agent has been saved
     this could be parsed in a similar way. This equates to the 'deviceInformation' record.
     @param obj: Session object
     @return: The device information
     """
     user_agent = self.retrieve_value(obj, 'cd' + USER_AGENT_CD, 0)
     if user_agent:
         ua = ua_parse(user_agent)
         if ua.is_pc:
             device = 'desktop'
         elif ua.is_tablet:
             device = 'tablet'
         else:
             device = 'mobile'
         dev = {'screenResolution': self.retrieve_value(obj, 'sr', 0),
                'browserSize': self.retrieve_value(obj, 'vp', 0),
                'language': self.retrieve_value(obj, 'ul', 0),
                'browser': ua.browser.family,
                'browserVersion': ua.browser.version_string,
                'deviceCategory': device,
                'operatingSystem': ua.os.family + " " + ua.os.version_string,
                'isMobile': False if ua.is_pc else True,
                'mobileDeviceModel': None if ua.is_pc else ua.device.model,
                'mobileDeviceBranding': None if ua.is_pc else ua.device.brand,
                'mobileDeviceInfo': None if ua.is_pc else ua.device.brand + " " + ua.device.family + " " \
                                                          + ua.device.model
                }
     else:
         dev = empty_device
     return {k: dev.get(k) for k in device_order}
Example #2
0
    def get_template_namespace(self):
        """Override `tornado.web.RequestHandler.get_template_namespace` static method
        add those new features:

        """

        namespace = super(BaseHandler, self).get_template_namespace()
        namespace['site'] = self.config.read_config()

        try:
            user_agent = self.request.headers['User-Agent']
        except KeyError:
            user_agent = '-'

        namespace['req'] = {

            # 当前请求路径
            'path': urlparse.urlparse(self.request.uri).path,

            'hostname': self.request.protocol + '://' + self.request.host,

            # userAgent 实例
            'user_agent': ua_parse(user_agent)
        }

        namespace['archives'] = self.posts.get_archives()
        namespace['tags'] = self.posts.get_tags()
        namespace['categories'] = []

        return namespace
Example #3
0
 def check_user_agent(self, value: str):
     """ 根据UA,获取设备信息 """
     if not value or self.user_agent == value:
         return
     self.user_agent = value
     self.device_key = idshift.hash_md5(f'{self.usrid}:{value}')
     ua_info = ua_parse(self.user_agent)
     self.ua_info = dict(
         device=dict(
             family=ua_info.device.family,
             brand=ua_info.device.brand,
             model=ua_info.device.model,
         ),
         os=dict(
             family=ua_info.os.family,
             version=ua_info.os.version_string,
         ),
         browser=dict(
             family=ua_info.browser.family,
             version=ua_info.browser.version_string,
         )
     )
     self.app_type = self.get_app_type()
     self.device_name = ' '.join([i.strip() for i in str(ua_info).split('/')][:2])
     up_fields = ['app_type', 'user_agent', 'device_key', 'device_name', 'ua_info', 'updated_at']
     self.save(update_fields=up_fields)
Example #4
0
 def process_data(self, dfs, sids):
     """
     Create an array of sessions, with each session taking a single row
     @param dfs: The dataframe of sessions
     @param sids: The session ids
     @return:
     """
     array = []
     # Fetch the Array
     for x in range(0, len(sids)):
         # Clear Session Level Values, Dicts and Lists
         key = sids.iloc[x]
         obj = dfs[key]
         user_agent = self.retrieve_value(obj, 'cd30', 0)
         if user_agent:
             ua = ua_parse(user_agent)
             if ua.is_bot:
                 continue
         try:
             session = self.session_func(obj)
         except Exception as ex:
             self.logger.critical('There was an Execption: {}'.format(ex))
             raise Exception(ex)
         # Add session to list
         array.append(session.copy())
     return array
Example #5
0
def parser_ua(ua_string):
    '''
    解析user-agent
    '''
    info = {}
    info['spider'] = False
    try:
        msg = ua_parse(ua_string)

        if msg.is_pc:
            info['dev'] = 'PC'
        elif msg.is_tablet:
            info['dev'] = 'Pad'
        elif msg.is_mobile:
            info['dev'] = 'MObile'
        else:
            info['dev'] = 'Unknow'

        if msg.is_bot:
            info['spider'] = True
        info["type"] = msg.os.family+' '+str(msg.os.version_string)

        info["ua"] = msg.browser.family+' '+str(msg.browser.version_string)
        return info
    except Exception as e:
        return info
Example #6
0
def get_user_agent():
    """ Returns a user-agents object if we can get a user agent. Otherwise,
    returns None. """

    if "HTTP_USER_AGENT" not in os.environ:
        return None
    else:
        return ua_parse(os.environ["HTTP_USER_AGENT"])
Example #7
0
def calendars(request):
    try:
        is_mobile = ua_parse(str(request.META['HTTP_USER_AGENT'])).is_mobile
    except BaseException:
        is_mobile = False

    return render(
        request,
        'calendars.html',
        context={'is_mobile': is_mobile}
    )
	def get_browser_from_user_agent(self, request):
		try:
			user_agent = ua_parse(request.headers['User-Agent']).browser
			return ("%s_%s" % (user_agent.family, user_agent.version_string)).lower().replace(" ", "_").replace(".", "_")

		except Exception as e:
			if DEBUG:
				print e, type(e)
				print "could not get User-Agent"

		return ""
Example #9
0
    def do_GET(self):
        parsed_path = urlparse(self.path)

        msg = """CLIENT VALUES:
              client_address=%s (%s)
              command=%s
              path=%s
              real path=%s
              query=%s
              request_version=%s
              """ % (self.client_address, self.address_string(), self.command, self.path,
               parsed_path.path,parsed_path.query, self.request_version,)

        msg += """SERVER VALUES:
              server_version=%s
              sys_version=%s
              protocol_version=%s
              """ % (self.server_version, self.sys_version,self.protocol_version,)


        msg += '\nHEADERS:\n'
        for name, value in sorted(self.headers.items()):
            msg += ('\t\t%s=%s\n' % (name, value.rstrip()))

        #OS guesing feature
        if self.headers.has_key('user-agent'):
            user_agent = ua_parse(self.headers['user-agent'])
            msg += """\nUser-Agent guessing:
                Browser: %s %s
                OS: %s %s
                Device: %s

                PC: %s
                Mobile: %s
                Tablet: %s

                Touch: %s
                Bot: %s\n""" % \
            (user_agent.browser.family, user_agent.browser.version_string,
             user_agent.os.family,user_agent.os.version_string,
             user_agent.device.family,

             user_agent.is_pc, user_agent.is_mobile,user_agent.is_tablet,

             user_agent.is_touch_capable,user_agent.is_bot,
            )

        self.send_response(200)
        self.end_headers()

        self.wfile.write(msg)
        #self.wfile.write('OK')
        #logging.info(message)
        return
Example #10
0
def get_default_view(request):

    if 'default_mode' not in request.session:
        user_agent = ua_parse(request.META['HTTP_USER_AGENT'])
        if user_agent.is_mobile:
            mode = 'smartphone'
        else:
            mode = 'classic'

        request.session['default_mode'] = mode

    return request.session['default_mode']
Example #11
0
def get_default_view(request):

    if 'default_mode' not in request.session:
        user_agent = ua_parse(request.META.get('HTTP_USER_AGENT', ''))
        if user_agent.is_mobile:
            mode = 'smartphone'
        else:
            mode = 'classic'

        request.session['default_mode'] = mode

    return request.session['default_mode']
Example #12
0
    def get_browser_from_user_agent(self, request):
        try:
            user_agent = ua_parse(request.headers['User-Agent']).browser
            return ("%s_%s" % (user_agent.family, user_agent.version_string)
                    ).lower().replace(" ", "_").replace(".", "_")

        except Exception as e:
            if DEBUG:
                print e, type(e)
                print "could not get User-Agent"

        return ""
Example #13
0
    async def on_connect(self, sid, environ):
        logger.info('Client connected, sid=%s' % (sid))

        print("=" * 100)
        print(environ)
        print(dir(environ['aiohttp.request']))
        print(environ['aiohttp.request'].scheme)
        print("=" * 100)

        client = {
            'user_agent': environ.get('HTTP_USER_AGENT'),
            'remote_ip': environ['aiohttp.request'].remote,
            'device': {},
        }

        payload = {
            'sid': sid,
            'client': client,
        }

        try:
            user_agent = ua_parse(environ['HTTP_USER_AGENT'])
            client['device'].update({
                'is_mobile': user_agent.is_mobile,
                'is_tablet': user_agent.is_tablet,
                'is_touch_capable': user_agent.is_touch_capable,
                'is_pc': user_agent.is_pc,
                'is_bot': user_agent.is_bot,
                'os': {
                    'family': user_agent.os.family,
                    'version': user_agent.os.version_string,
                },
                'device': {
                    'family': user_agent.device.family,
                    'brand': user_agent.device.brand,
                    'model': user_agent.device.model,
                },
                'browser': {
                    'family': user_agent.browser.family,
                    'version': user_agent.browser.version_string,
                }
            })
        except:
            logger.warning('Could not parse client user agent (ua=%s)' %
                           (environ.get('HTTP_USER_AGENT')))

        self.context.socket_clients[sid] = client
        self.context.queue_event('socket', 'client.connected', payload=payload)

        await self.emit('status', self.context.status, room=sid)
Example #14
0
 def parse_useragent(self, ua):
     sys.path.append(self.prj_path + "/script/useragent_parser")
     from user_agents import parse as ua_parse
     ua_dict = {}
     user_agent = ua_parse(ua)
     if user_agent.browser.family != None:
         ua_dict['browser'] = user_agent.browser.family
     if user_agent.browser.version_string != None:
         ua_dict['browser_version'] = user_agent.browser.version_string
     if user_agent.os.family != None:
         ua_dict['os'] = user_agent.os.family
     if user_agent.device.family != None:
         ua_dict['device'] = user_agent.device.family
     if user_agent.device.brand != None:
         ua_dict['device_type'] = user_agent.device.brand
     if user_agent.device.model != None:
         ua_dict['device_version'] = user_agent.device.model
     return ua_dict
Example #15
0
    def __process_pageviews(self, f):
        print("BrowserParser - processing pageviews from: " + f)

        ua_cache = {}
        with open(f) as csvfile:
            r = csv.DictReader(csvfile, delimiter=',')

            for row in r:
                # save UA to cache
                user_agent = row['user_agent']
                if user_agent not in ua_cache:
                    ua_cache[user_agent] = ua_parse(user_agent)

                # save all user devices for (user <-> device mapping)
                browser_id = row['browser_id']
                if browser_id not in self.browsers_with_users:
                    self.browsers_with_users[browser_id] = {'user_ids': set()}
                if row['user_id']:
                    self.browsers_with_users[browser_id]['user_ids'].add(
                        row['user_id'])
                self.browsers_with_users[browser_id]['ua'] = ua_cache[
                    user_agent]

                # continue with pageviews only for subscribers
                if row['subscriber'] == 'True':
                    continue

                if row['derived_referer_medium'] == '':
                    continue

                if browser_id not in self.data:
                    self.data[browser_id] = empty_data_entry(
                        {'user_ids': set()})

                record = self.data[browser_id]
                update_record_from_pageviews_row(record, row)

                if row['user_id']:
                    record['user_ids'].add(row['user_id'])
                record['ua'] = ua_cache[user_agent]

                self.data[browser_id] = record
Example #16
0
    def parser_ua(self,ua_string):
        info = {}
        info['spider'] = False
        msg = ua_parse(ua_string)

        if msg.is_pc:
            info['dev'] = 'PC'
        elif msg.is_tablet:
            info['dev'] = 'Pad'
        elif msg.is_mobile:
            info['dev'] = 'MObile'
        else:
            info['dev'] = 'Unknow'

        if msg.is_bot:
            info['spider'] = True
        info["type"] = msg.os.family+' '+str(msg.os.version_string)

        info["ua"] = msg.browser.family+' '+str(msg.browser.version_string)
        return info
Example #17
0
    def log_event(self,
                  msg=None,
                  event_type=None,
                  action=None,
                  key=None,
                  value=None,
                  agent=None):
        """ This is the primary user-facing logging interface, so there' s a bit
        of a high bar for using it.

        The basic idea of creating a log entry is that we're doing a bit of the
        semantic logging (i.e. strongly typed) thing, so, depending on which
        kwargs you use when calling this method, your final outcome/message is
        going to vary somewhat.

        That said, none of the kwargs here are mandatory, because context.
        """

        #
        #   baseline attributes
        #

        # for those who still raw-dog it; force to ASCII:
        if msg is not None:
            msg = msg.encode("ascii", 'ignore')

        # 0.) method: determine caller method
        curframe = inspect.currentframe()
        calframe = inspect.getouterframes(curframe, 2)
        method = calframe[2][3]

        # 1.) event: determine event type if it's None
        if event_type is None:
            event_type = method

        # 2.) action: figure out the action; set the special action vars
        if action is None:
            action = method.split("_")[0]
        action_word, action_preposition = utils.action_keyword(action)

        # 3.) key: default the key if we don't get one
        if key is None:
            key = " ".join(method.split("_")[1:])
        key = key.encode('ascii', 'ignore')
        if key == "settlement":
            key = ""

        # 4.) value; default the value if we don't get one
        if value is None:
            value = "UNKNOWN"
        if type(value) != int:
            value = value.encode('ascii', 'ignore')

        # set 'created_by'
        created_by = None
        created_by_email = None
        if request:
            if hasattr(request, 'User'):
                created_by = request.User.user['_id']
                created_by_email = request.User.user['login']
                if agent is None:
                    agent = "user"

        # set 'attribute_modified'

        attribute_modified = {
            'key': key,
            'value': value,
        }

        if attribute_modified['key'] is not None:
            attribute_modified['key_pretty'] = key.replace("_", " ").replace(
                "and", "&").title()
        if attribute_modified['value'] is not None:
            attribute_modified['value_pretty'] = str(value).replace("_", " ")

        d = {
            'version': 1.3,
            'agent': agent,
            "created_on": datetime.now(),
            'created_by': created_by,
            'created_by_email': created_by_email,
            "settlement_id": self.settlement_id,
            "ly": self.get_current_ly(),
            'event_type': event_type,
            'event': msg,
            'modified': {
                'attribute': attribute_modified
            },
        }

        # survivor, if it's a survivor
        if self.collection == 'survivors':
            d['survivor_id'] = self.survivor['_id']

        # target is the settlement, unless a survivor object calls this method
        action_target = "settlement"
        if 'survivor_id' in d.keys():
            d['modified']['asset'] = {
                'type': 'survivor',
                '_id': d['survivor_id'],
                'name': self.survivor['name'],
                'sex': self.get_sex(),
            }
            action_target = "survivor"
        else:
            d['modified']['asset'] = {
                "type": "settlement",
                "name": self.settlement['name'],
                '_id': self.settlement_id
            }

        # create the 'action'
        d['action'] = {
            'word': action_word,
            'preposition': str(action_preposition)
        }

        # now write the repr, which is like...the simplified sentence of the event
        if key is None and value is None:
            d['action']['repr'] = " ".join(['modified', action_target])
        elif key is not None and value is None:
            d['action']['repr'] = " ".join(['modified', action_target, key])
        else:
            if action_preposition is None:
                d['action']['repr'] = action_word
            elif action_word in ['set']:
                d['action']['repr'] = " ".join([
                    action_word, action_target, key, action_preposition,
                    str(value)
                ])
            elif action_word in ['unset']:
                d['action']['repr'] = " ".join(
                    [action_word, action_target, key])
            elif action_target == "survivor" and action_preposition is not None:
                d['action']['repr'] = " ".join([
                    action_word,
                    "'%s'" % value, action_preposition,
                    str(key)
                ])
            else:
                d['action']['repr'] = " ".join([
                    action_word,
                    "'%s'" % value, action_preposition, action_target,
                    str(key)
                ])

        # default a message, if incoming message is none
        if msg is None:

            # create messages for survivor updates
            if d['modified']['asset']['type'] == 'survivor':
                if d['agent'] == 'user' and action_preposition is None:
                    d['event'] = " ".join([
                        d['created_by_email'],
                        d['action']['word'],
                        "%s [%s]" % (self.survivor['name'], self.get_sex()),
                    ])
                elif d['agent'] == 'user' and action_preposition is not None:
                    d['event'] = " ".join([
                        d['created_by_email'],
                        d['action']['word'],
                        d['modified']['attribute']['value_pretty'],
                        d['action']['preposition'],
                        "%s [%s]" % (self.survivor['name'], self.get_sex()),
                        d['modified']['attribute']['key_pretty'],
                    ])
                else:
                    d['event'] = " ".join([
                        "%s [%s]" % (self.survivor['name'], self.get_sex()),
                        d['action']['word'],
                        d['modified']['attribute']['value_pretty'],
                        d['action']['preposition'],
                        d['modified']['attribute']['key_pretty'],
                    ])
            # create messages for settlements
            elif d['modified']['asset']['type'] == 'settlement':
                if d['agent'] == 'user' and action_preposition is None:
                    d['event'] = " ".join([
                        d['created_by_email'],
                        d['action']['word'],
                        self.settlement['name'],
                    ])
                else:
                    d['event'] = " ".join([
                        str(d['created_by_email']),
                        str(d['action']['repr']),
                    ])
            else:
                d['event'] = 'Updated %s' % self
#            elif agent == "automation":
#                d['event'] = d['action']['repr']

# enforce terminal punctuation on the event "sentence"
        if d['event'][-1] not in ['.', '!']:
            d['event'] = d['event'].strip()
            d['event'] += "."

#        d['event'] = d['event'].decode('ascii','replace').encode('utf-8','replace')

# finally, if we had a requester, now that we've settled on a message
# text, update the requester's latest action with it
        if 'created_by' is not None:
            if request:
                ua_string = str(ua_parse(request.user_agent.string))
                request.User.set_latest_action(d['event'], ua_string)

        # finally, insert the event (i.e. save)
        utils.mdb.settlement_events.insert(d)
        self.logger.info("%s event: %s" % (self, d['event']))
Example #18
0
 def ua(self):
     if not hasattr(self, '_ua'):
         self._ua = ua_parse(self.user_agent)
     return self._ua
Example #19
0
def parser_ua(ua_string):
    '''
    解析user-agent
    '''
    info = {}
    info['spider'] = False
    try:
        msg = ua_parse(ua_string)

        if msg.is_pc:
            info['dev'] = 'PC'
        elif msg.is_tablet:
            info['dev'] = 'Pad'
        elif msg.is_mobile:
            info['dev'] = 'MObile'
        else:
            info['dev'] = 'Unknow'

        if msg.is_bot:
            info['spider'] = True
        info["type"] = msg.os.family + ' ' + str(msg.os.version_string)

        info["ua"] = msg.browser.family + ' ' + str(msg.browser.version_string)
        return info
    except Exception as e:
        return info

    info = {}
    addr = ""
    g1 = GeoIPUtil()
    g2 = GeoIP2Util()
    if is_intranet(ipaddr):
        info["address"] = "局域网内地址"
        info['weidu'] = info['jingdu'] = info['country'] = info[
            'subdivision'] = info['city'] = ""
    else:
        try:
            longitude, latitude = g1.get_lat_alt(ipaddr)
            info['weidu'] = latitude
            info['jingdu'] = longitude
            country, subdivision, city = g2.get_ip_location(ipaddr)
            if country == u"中国":
                if not subdivision and not city:
                    addr = country
                elif subdivision.find(u'市') == -1 and subdivision not in [
                        u'上海'
                ] and city:
                    addr = subdivision.strip(u'省') + u"省\t" + city
                elif subdivision and not city:
                    addr = country + ' ' + subdivision
                else:
                    addr = subdivision + "\t" + city
            else:
                if subdivision in [
                        u'台北市', u'新北市', u'基隆市', u'新竹市', u'嘉义市', u'台中市', u'台南市',
                        u'高雄市', u'屏东市'
                ]:
                    subdivision = "台湾省" + ' ' + subdivision
                if country in [u'香港', u'澳门'] and subdivision:
                    subdivision = country + ' ' + subdivision
                    country = '中国'
                elif country in [u'香港', u'澳门'] and not subdivision:
                    subdivision = country
                    country = '中国'

                if not subdivision and not city:
                    addr = country
                elif subdivision and not city:
                    addr = country + ' ' + subdivision
                else:
                    addr = country.replace(
                        u'台湾', u'中国') + "\t" + subdivision + "\t" + city
            if addr:
                info["address"] = addr

            info["country"] = country.replace(u'台湾', u'中国')
            info["subdivision"] = subdivision
            info["city"] = city
        except Exception as e:
            print str(e)
            return info
Example #20
0
 def ua(self):
     if not hasattr(self, '_ua'):
         self._ua = ua_parse(self.requests.first().user_agent)
     return self._ua
Example #21
0
    def do_GET(self):
        parsed_path = urlparse(self.path)

        msg = """CLIENT VALUES:
              client_address=%s (%s)
              command=%s
              path=%s
              real path=%s
              query=%s
              request_version=%s
              """ % (
            self.client_address,
            self.address_string(),
            self.command,
            self.path,
            parsed_path.path,
            parsed_path.query,
            self.request_version,
        )

        msg += """SERVER VALUES:
              server_version=%s
              sys_version=%s
              protocol_version=%s
              """ % (
            self.server_version,
            self.sys_version,
            self.protocol_version,
        )

        msg += '\nHEADERS:\n'
        for name, value in sorted(self.headers.items()):
            msg += ('\t\t%s=%s\n' % (name, value.rstrip()))

        #OS guesing feature
        if self.headers.has_key('user-agent'):
            user_agent = ua_parse(self.headers['user-agent'])
            msg += """\nUser-Agent guessing:
                Browser: %s %s
                OS: %s %s
                Device: %s

                PC: %s
                Mobile: %s
                Tablet: %s

                Touch: %s
                Bot: %s\n""" % \
            (user_agent.browser.family, user_agent.browser.version_string,
             user_agent.os.family,user_agent.os.version_string,
             user_agent.device.family,

             user_agent.is_pc, user_agent.is_mobile,user_agent.is_tablet,

             user_agent.is_touch_capable,user_agent.is_bot,
            )

        self.send_response(200)
        self.end_headers()

        self.wfile.write(msg)
        #self.wfile.write('OK')
        #logging.info(message)
        return