def get_client_details(request) -> dict:
'''
Given valid request returns user's ip, location and user-agent.
'''
if not request:
return dict()
client_ip, is_routable = get_client_ip(request)
client_ip = client_ip if client_ip and is_routable else None
location = ''
if client_ip:
result = GeoIP2().city(client_ip)
location = '{city}, {country}'.format(
country=result.get('country_name', ''),
city=result.get('city', ''),
)
user_agent = request.META.get('HTTP_USER_AGENT', '')
if user_agent:
user_agent = str(parse_ua(user_agent))
return {
'ip': client_ip,
'location': location,
'user_agent': user_agent,
}
def hlar_top(request):
current_site = get_current_site(request)
print(current_site.domain)
if request.user.is_authenticated() == False:
try:
# oauth で返ってきた時はsessionにid が入っているのでそれを取得する。
user = User.objects.filter(
id=request.session._session_cache['_auth_user_id'])[0]
user_auth = authenticate(username=user.email,
password=DEFAULT_PASS)
login(request, user_auth)
except Exception as e:
print('error')
# 人気ターゲット一覧を取得
targets = get_targets_popular()
ua = parse_ua(request.META['HTTP_USER_AGENT'])
return render(
request,
'hlar/hlar_top.html', # 使用するテンプレート
{
'user': request.user,
'msg': _("使い方"),
'targets': targets,
's3_FQDN': s3_FQDN,
'is_mobile': ua.is_mobile,
} # テンプレートに渡すデータ
)
def setUp(self):
super(ReportRequestRenderedViewTests, self).setUp()
cache_key = 'request:%s' % self.request_id
data = {
'started': 5000,
'server_time': 150,
'agent': parse_ua(CHROME_UA),
'view_name': 'bar',
}
cache.set(cache_key, data)
def process(self, device):
"""Process a device to compute it's privacy features and PI."""
features = {}
if device.user_agent:
ua = parse_ua(device.user_agent)
browser_version = ua.browser.version_string.lower()
features.update({
'browser_family': ua.browser.family.lower(),
'browser_version': browser_version,
'os_family': ua.os.family.lower(),
'os_version': ua.os.version_string.lower(),
'device_family': ua.device.family.lower(),
'device_type': self._get_device_type(ua)
})
# XXX processing IP address to detect some informations
features.update(self._process_ip_address(device.ip_creation))
return None, features
def process(self, device):
"""Process a device to compute it's privacy features and PI."""
features = {}
if device.user_agent:
ua = parse_ua(device.user_agent)
browser_version = ua.browser.version_string.lower()
features.update({'browser_family': ua.browser.family.lower(),
'browser_version': browser_version,
'os_family': ua.os.family.lower(),
'os_version': ua.os.version_string.lower(),
'device_family': ua.device.family.lower(),
'device_type': self._get_device_type(ua)})
# XXX processing IP address to detect some informations
if device.ip_creation:
features.update(self._process_ip_address(device.ip_creation))
else:
log.warn('No ip address found for device')
return None, features
def target_list(request):
if request.user.is_authenticated() == False:
return HttpResponseRedirect('/accounts/login/?next=%s' % request.path)
ua = parse_ua(request.META['HTTP_USER_AGENT'])
# ターゲット一覧を取得
targets = Target.objects.filter(user_id=str(request.user.id),
del_flg=False)
addTarget = True
if len(targets) >= settings.TARGET_LIMIT_COUNT:
addTarget = False
for target in targets:
# シェアのリンクを作成
arrContentName = target.img_name.split(".")
targetImgURL = settings.URL_ROOT + "hlar/target/preview_img/" + arrContentName[
0]
# Twitter
twitterText = _("ARアプリ【UNIVERSE.AR】でこの画像を読み取ってみましょう！ #universear")
twitterParam = {'text': twitterText, 'url': targetImgURL}
target.twitter_url = 'https://twitter.com/share?' + urllib.parse.urlencode(
twitterParam)
# facebook
facebookParam = {'u': targetImgURL}
target.fb_url = 'https://www.facebook.com/share.php?' + urllib.parse.urlencode(
facebookParam)
return render(
request,
'hlar/target_list.html', # 使用するテンプレート
{
'targets': targets,
's3_FQDN': s3_FQDN,
'is_mobile': ua.is_mobile,
'addTarget': addTarget,
'TARGET_LIMIT_COUNT': settings.TARGET_LIMIT_COUNT,
}) # テンプレートに渡すデータ
def prepare_email_data(self, markers):
data = defaultdict(lambda: defaultdict(list))
"""email -> {link -> [{ip, date}]}"""
for marker_path, content_string in markers.iteritems():
content = json.loads(content_string)
link = self.obtain_url(content.get('token'), content.get('path'))
marker_name = os.path.basename(marker_path)
address, timestamp, padding = marker_name.rsplit('.', 2)
try:
timestamp = int(timestamp)
except ValueError:
timestamp = None
if timestamp:
date = datetime.fromtimestamp(timestamp).isoformat(sep=b' ')
date += ' (UTC)'
else:
date = None
item = {
'date': date,
'ip': content.get('ip'),
}
# Process user agent:
ua = parse_ua(content.get('user_agent', ''))
unknown = 'Other'
if ua.browser.family != unknown:
item['browser'] = (ua.browser.family + ' ' +
ua.browser.version_string).strip()
if ua.os.family != unknown:
item['os'] = (ua.os.family + ' ' +
ua.os.version_string).strip()
if ua.device.family != unknown:
item['device'] = ua.device.family
data[address][link].append(item)
for items in data.itervalues():
for l in items.itervalues():
l.sort(key=lambda d: (d['date'], d['ip']))
return data
def serialize_event(event):
d = {
'org_id': event.org_id,
'user_id': event.user_id,
'action': event.action,
'object_type': event.object_type,
'object_id': event.object_id,
'created_at': event.created_at
}
if event.user_id:
d['user_name'] = event.additional_properties.get('user_name', 'User {}'.format(event.user_id))
if not event.user_id:
d['user_name'] = event.additional_properties.get('api_key', 'Unknown')
d['browser'] = str(parse_ua(event.additional_properties.get('user_agent', '')))
d['location'] = get_location(event.additional_properties.get('ip'))
d['details'] = event_details(event)
return d
def gather_view_data(self, request, view_func):
"""Discover the view name.
"""
# View name is defined as module.view
# (e.g. django.contrib.auth.views.login)
name = view_func.__module__
# CBV specific
if hasattr(view_func, '__name__'):
name = '%s.%s' % (name, view_func.__name__)
elif hasattr(view_func, '__class__'):
name = '%s.%s' % (name, view_func.__class__.__name__)
method = request.method.lower()
if request.is_ajax():
method += '_ajax'
name = '%s.%s' % (name, method)
self.scope.id = id_request(request)
self.scope.agent = parse_ua(request.META.get('HTTP_USER_AGENT', ''))
self.scope.view_name = "view." + name
def serialize_event(event):
d = {
'org_id': event.org_id,
'user_id': event.user_id,
'action': event.action,
'object_type': event.object_type,
'object_id': event.object_id,
'created_at': event.created_at
}
if event.user_id:
d['user_name'] = event.additional_properties.get('user_name', 'User {}'.format(event.user_id))
if not event.user_id:
d['user_name'] = event.additional_properties.get('api_key', 'Unknown')
d['browser'] = str(parse_ua(event.additional_properties.get('user_agent', '')))
d['location'] = get_location(event.additional_properties.get('ip'))
d['details'] = event_details(event)
return d
def serialize_event(event):
d = {
"org_id": event.org_id,
"user_id": event.user_id,
"action": event.action,
"object_type": event.object_type,
"object_id": event.object_id,
"created_at": event.created_at,
}
if event.user_id:
d["user_name"] = event.additional_properties.get(
"user_name", "User {}".format(event.user_id)
)
if not event.user_id:
d["user_name"] = event.additional_properties.get("api_key", "Unknown")
d["browser"] = str(parse_ua(event.additional_properties.get("user_agent", "")))
d["location"] = get_location(event.additional_properties.get("ip"))
d["details"] = event_details(event)
return d
def lookup(urlcode):
"""
Endpoint that recieves the short url and redirects to the long url target
Case insensitive for better usability
:param urlcode: The code of the short url
:returns: redirects to target
"""
urlcode = urlcode.lower()
result = sht.short_to_long(urlcode)
if result:
url = result['default_url']
target = 'default'
ua = parse_ua(request.headers.get('User-Agent'))
if result['mobile_url'] and ua.is_mobile:
url = result['mobile_url']
target = 'mobile'
elif result['tablet_url'] and ua.is_tablet:
url = result['tablet_url']
target = 'tablet'
sht.hit(urlcode, target)
return redirect(url)
return Response(str(result['mobile_url ']))
return Response('{"error":"URL NOT FOUND"}', status=404, mimetype="application/json")
def get_event_counts(entry):
result = []
for e in events:
ename = e.replace("Event: ", "")
count = len(filter(lambda x: x['details'] != None and x['details'].startswith(ename),
entry.logs))
result.append(count)
return result
for key in logs:
entry = logs[key]
if not entry.user_agent or key[1] == 0 or len(entry.user) > 1:
continue
ua = parse_ua(entry.user_agent)
row = [entry.ip,
entry.user_agent,
ua.os.family,
ua.device.family,
ua.browser.family,
int(ua.is_mobile or ua.is_tablet or ua.is_touch_capable),
int(ua.is_pc),
len(entry.user),
]
if len(entry.user) == 1:
u = entry.user[0]
else:
u = none_dict
def write(cursor, lock, msg=None):
if uwsgi_mode:
msg = pickle.loads(msg)
# msg = self.msg
if msg.cached:
cached = '*'
else:
cached = ''
if msg.compress:
compress = '*'
else:
compress = ''
if hasattr(settings,'ROOT_URL'):
if msg.url.startswith(settings.ROOT_URL):
url = msg.url[len(settings.ROOT_URL):]
else:
url = msg.url
if msg.referer:
if msg.referer.startswith(settings.ROOT_URL):
referer = msg.referer[len(settings.ROOT_URL):]
else:
referer = msg.referer
else:
referer = '*'
else:
url = msg.url
if msg.referer:
referer = msg.referer
else:
referer = '*'
if msg.user_id:
user_id = msg.user_id
else:
user_id = '-'
if msg.bot:
bot = '*'
else:
bot = ''
if METHOD_CHOICES_DICT[msg.request_method] == 'GET':
method = '-'
elif METHOD_CHOICES_DICT[msg.request_method] == 'POST':
method = '+'
else:
method = '?'
if msg.ajax:
direction = f'<{method}'
else:
direction = f'{method}>'
ua_string = msg.user_agent[:252] + (msg.user_agent[252:] and '..') if msg.user_agent else None
if ua_string:
user_agent = parse_ua(ua_string)
else:
user_agent = parse_ua('')
msg.bot = True
if len(user_agent.browser.version) > 0:
browser_major_version = user_agent.browser.version[0]
else:
browser_major_version = None
if len(user_agent.browser.version) > 1:
browser_minor_version = user_agent.browser.version[1]
else:
browser_minor_version = None
if len(user_agent.browser.version) > 2:
browser_patch = user_agent.browser.version[2]
else:
browser_patch = None
if len(user_agent.os.version) > 0:
os_major_version = user_agent.os.version[0]
else:
os_major_version = None
if len(user_agent.os.version) > 1:
os_minor_version = user_agent.os.version[1]
else:
os_minor_version = None
if len(user_agent.os.version) > 2:
os_patch = user_agent.os.version[2]
else:
os_patch = None
if len(user_agent.os.version) > 3:
os_minor_patch = user_agent.os.version[3]
else:
os_minor_patch = None
fields = [
msg.timestamp,
msg.ip,
msg.response_time,
msg.status_code,
msg.url[:508] + (msg.url[508:] and '..') if msg.url else None,
msg.request_content_type[:48] + (msg.request_content_type[48:] and '..') if msg.request_content_type else None,
msg.request_method,
msg.ajax,
msg.referer[:508] + (msg.referer[508:] and '..') if msg.referer else None,
ua_string,
msg.request_content_length,
msg.accept[:252] + (msg.accept[252:] and '..') if msg.accept else None,
msg.accept_language[:48] + (msg.accept_language[48:] and '..') if msg.accept_language else None,
msg.accept_encoding[:48] + (msg.accept_encoding[48:] and '..') if msg.accept_encoding else None,
msg.response_content_type[:48] + (msg.response_content_type[48:] and '..') if msg.response_content_type else None,
msg.response_content_length,
msg.compress,
msg.session_key,
msg.user_id,
msg.latitude,
msg.longitude,
msg.protocol,
msg.cached,
msg.session_start_time,
msg.preview,
msg.prefetch,
msg.bot,
]
with lock:
try:
cursor.execute("execute " + LogWriter.log_sql + "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);", fields)
result = cursor.fetchone()
# print(f'{dir(result)=}')
# dir(result)=['__add__', '__class__', '__contains__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__getnewargs__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', '__module__', '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__rmul__', '__setattr__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', '_asdict', '_field_defaults', '_fields', '_fields_defaults', '_make', '_replace', 'accept_encoding_id', 'accept_language_id', 'accept_type_id', 'ajax', 'cached', 'compress', 'connect_time', 'count', 'host_name_id', 'id', 'index', 'ip_id', 'latitude', 'log_timestamp', 'longitude', 'lookup_time', 'method', 'prefetch', 'preview', 'protocol', 'referer_url_id', 'request_content_length', 'request_content_type_id', 'request_url_id', 'response_content_length', 'response_content_type_id', 'response_time', 'session_id', 'session_log_id', 'ssl_time', 'status', 'timestamp', 'user_agent_id', 'user_id']
except psycopg2.errors.StringDataRightTruncation:
logger.error("Log entry contains too many characters in fields. Please review fields:")
count = 0
for i in fields:
logger.error(f'{c}: length {len(i)}: {i}')
host = None
cursor.execute('execute get_host(%s);' , [result.ip_id])
host_result = cursor.fetchone()
if host_result.hostname:
host = host_result.hostname
if host_result.domain_id == None:
names = host.split(".")[1:]
if names:
domain = names[-1]
if len(names) > 1:
domain = names[-2] + '.' + domain
if len(names) > 2:
domain = names[-3] + "." + domain
try:
cursor.execute('execute create_domain(%s, %s);' ,
[domain, msg.bot])
except psycopg2.errors.StringDataRightTruncation:
logger.error("Log create_domain entry contains too many characters for domain name.")
logger.error(f'length {len(domain)}: {domain}')
domain_result = cursor.fetchone()
if domain_result:
cursor.execute('execute update_host_domain(%s, %s);' ,
[host_result.host_id, domain_result.id])
else:
domain = '-'
else:
domain = host_result.domainname
else:
host = None
try:
host = socket.gethostbyaddr(msg.ip)[0]
except:
domain = '-'
if host:
names = host.split(".")[1:]
if len(names) <= 1:
logger = logging.warning(f"No hostname found for host {host}")
names = host.split(".")
domain = names[-1]
if len(names) > 1:
domain = names[-2] + '.' + domain
if len(names) > 2:
domain = names[-3] + "." + domain
cursor.execute('execute create_domain(%s, %s);' ,
[domain, msg.bot])
domain_result = cursor.fetchone()
if domain_result:
if host_result.host_id != None:
cursor.execute('execute update_host_domain(%s, %s);' ,
[host_result.host_id, domain_result.id])
else:
try:
cursor.execute('execute update_host(%s, %s, %s, %s);' ,
[result.ip_id, domain_result.id, host, msg.bot])
except psycopg2.errors.StringDataRightTruncation:
logger.error("Log update_host entry contains too many characters for host name!")
logger.error(f'length {len(host)}: {host}')
cursor.execute('execute get_user_agent(%s);' , [result.user_agent_id])
useragent_result = cursor.fetchone()
if useragent_result:
if useragent_result.browser_id is None:
cursor.execute(
'execute update_browser(%s, %s, %s, %s, %s);' ,
[
result.user_agent_id,
user_agent.browser.family,
browser_major_version,
browser_minor_version,
browser_patch,
])
if useragent_result.os_id is None:
cursor.execute(
'execute update_os(%s, %s, %s, %s, %s, %s);' ,
[
result.user_agent_id,
user_agent.os.family,
os_major_version,
os_minor_version,
os_patch,
os_minor_patch,
])
if useragent_result.device_id is None:
cursor.execute(
'execute update_device(%s, %s, %s, %s, %s, %s, %s, %s, %s);' ,
[
result.user_agent_id,
user_agent.device.family,
user_agent.device.brand,
user_agent.device.model,
user_agent.is_mobile,
user_agent.is_pc,
user_agent.is_tablet,
user_agent.is_touch_capable,
user_agent.is_bot
])
cursor.execute('execute record_timestamp(%s);', [ result.id ])
log_timestamp_result = cursor.fetchone()
if user_agent.browser.family != 'Other':
browser = f'{user_agent.browser.family}'
if browser_major_version != None:
browser = f'{browser} {browser_major_version}'
if browser_minor_version != None:
browser = f'{browser}.{browser_minor_version}'
if user_agent.os.family == 'Other':
os = ''
else:
os = f', {user_agent.os.family}'
if os_major_version != None:
os = f'{os} {os_major_version}'
if os_minor_version != None:
os = f'{os}.{os_minor_version}'
bot = ''
device = ''
if user_agent.device.family in [
'Spider', 'Generic Smartphone', 'Generic Desktop']:
bot = '*'
elif user_agent.device.family != 'Other':
if user_agent.device.brand:
device = f', {user_agent.device.brand}'
if user_agent.device.model:
device = f'{device} {user_agent.device.model}'
else:
device = f', {user_agent.device.family}'
ua = f'({browser}{os}{device}){bot}'
else:
if ua_string == '':
ua = '(Empty User Agent)'
else:
ua = f'({ua_string})*'
timestamp = timezone.localtime(result.timestamp).strftime("%Y-%m-%d %H:%M:%S")
log_response_time = (log_timestamp_result.log_timestamp-result.timestamp).microseconds/1000
# log_response_time = (time.perf_counter()-msg.perf_counter)*1000
log_delay = log_response_time - result.response_time
analytics_logger.info(
f'{msg.ip} '
f'{domain} '
f'{user_id} '
f'{result.response_time:.3f}ms{cached} '
f'{log_delay:.3f}ms '
f'{msg.response_content_length}b{compress} '
f'{msg.status_code} '
f'{referer} {direction} {url} '
f'{ua} '
)
def __str__(self):
if self.bot == False:
color = RED
else:
color = ''
user_agent = parse_ua(self.user_agent_string)
if len(user_agent.browser.version) > 0:
browser_major_version = user_agent.browser.version[0]
else:
browser_major_version = None
if len(user_agent.browser.version) > 1:
browser_minor_version = user_agent.browser.version[1]
else:
browser_minor_version = None
if len(user_agent.os.version) > 0:
os_major_version = user_agent.os.version[0]
else:
os_major_version = None
if len(user_agent.os.version) > 1:
os_minor_version = user_agent.os.version[1]
else:
os_minor_version = None
if user_agent.browser.family != 'Other':
browser = f'{user_agent.browser.family}'
if browser_major_version != None:
browser = f'{browser} {browser_major_version}'
if browser_minor_version != None:
browser = f'{browser}.{browser_minor_version}'
if user_agent.os.family == 'Other':
os = ''
else:
os = f', {user_agent.os.family}'
if os_major_version != None:
os = f'{os} {os_major_version}'
if os_minor_version != None:
os = f'{os}.{os_minor_version}'
bot = ''
device = ''
if user_agent.device.family in [
'Spider', 'Generic Smartphone', 'Generic Desktop'
]:
bot = '*'
elif user_agent.device.family != 'Other':
if user_agent.device.brand:
device = f', {user_agent.device.brand}'
if user_agent.device.model:
device = f'{device} {user_agent.device.model}'
else:
device = f', {user_agent.device.family}'
ua = f'{browser}{os}{device}'
else:
ua = f'({self.user_agent_string})*'
bot = ''
return format_html(
'{}<span style="color: {};">{}</span>',
ua,
color,
bot,
)
