def w4_output_controller_download(request):
"""
User can download content in W4 by clicking on link
:param request: user request
:return: content data onto a file
"""
response = None
fng = w4_output_controller_download_filename # file name generator :-D
if request.method == 'GET':
user_agent = user_agent_parser.ParseOS(
request.META.get("HTTP_USER_AGENT"))
controller_job_id = request.GET.get('job_id')
controller_job_number = request.GET.get('job_number', "")
content = get_brain_output_content(controller_job_id, max_size=None)
if "windows" in user_agent.get("family").lower() and isinstance(
content, str):
content = content.replace("\n", "\r\n")
response = HttpResponse(content,
content_type='application/octet-stream')
content_dispo = 'attachment; \
filename="{}"'.format(
fng(controller_job_id, controller_job_number))
response['Content-Disposition'] = content_dispo
response.status_code = 200
return response
def show_apple_maps(request):
"""
Test if we can show an Apple Maps link to a device. There's no Web version
of Apple Maps; it redirects on Google Maps on non-iOS/OSX platforms.
"""
ua = user_agent_parser.ParseOS(request.user_agent.string)
return ua.get("family") in ("iOS", "Mac OS X")
def check_os(user_agent_str: str) -> list:
result = config.types_by_ua[-1:]
family = user_agent_parser.ParseOS(user_agent_str)["family"].lower()
os_names = dict(
zip(["mac os x", "window", "solaris", "linux"],
config.types_by_ua[:-1]))
if family in os_names:
result.append(os_names[family])
return result
def process_tcp_packet(packet):
if not packet.haslayer(http.HTTPRequest):
# This packet doesn't contain an HTTP request so we skip it
return
http_layer = packet.getlayer(http.HTTPRequest)
ip_layer = packet.getlayer(IP)
try:
ip = '{0[src]}'.format(ip_layer.fields, http_layer.fields)
ua_string = '{1[User-Agent]}'.format(ip_layer.fields, http_layer.fields)
agent = user_agent_parser.ParseUserAgent(ua_string)
print ua_string
device = user_agent_parser.ParseDevice(ua_string)
os = user_agent_parser.ParseOS(ua_string)
browser_family = agent['family']
browser_major = agent['major']
browser_minor = agent['minor']
browser_patch = agent['patch']
device_brand = device['brand']
device_family = device['family']
device_model = device['model']
os_family = os['family']
os_major = os['major']
os_minor = os['minor']
os_patch = os['patch']
os_patch_minor = os['patch_minor']
key = find_key(ip)
device_hash = wdb.Hash(key)
hostname = device_hash['hostname']
#print key,hostname,ip,browser_family,browser_major,browser_minor,browser_patch,device_brand,device_family,device_model,os_family,os_major,os_minor,os_patch,os_patch_minor
os_version = str(os_major) +'.'+ str(os_minor) +'.'+ str(os_patch)
browser_combined = browser_family +'('+ str(browser_major) +'.'+ str(browser_minor) +'.'+ str(browser_patch) +')'
k = wdb.Hash(key)
if "Other" not in os_family:
k.update(os=os_family)
print ip,key,os_family
if os_family == "Mac OS X":
k.update(category="Desktop/Laptop")
if os_family == "iOS":
k.update(category="Smartphones/PDAs/Tablets")
if ("None" or "Other") not in os_version:
k.update(os_version=os_version)
print ip,key,os_version
if ("None" or "Other") not in browser_combined:
k.update(browser_family=browser_combined)
print ip,key,browser_combined
except KeyError:
pass
def parse_brand():
# pp = pprint.PrettyPrinter(indent=4)
with open("/Users/zaniu/Documents/logs/analysis/user_agent_file.txt", "r") as user_agent_file:
user_agent_file_result = open("/Users/zaniu/Documents/logs/analysis/user_agent_brand_result_os.txt", "w")
for user_agent in user_agent_file:
if user_agent:
parsed_map = user_agent_parser.ParseOS(user_agent)
if parsed_map:
print(parsed_map["family"])
user_agent_file_result.write("{}".format(parsed_map["family"]) + "\n")
user_agent_file_result.close
def extract_os_from_user_agent(user_agent, default_os='Other'):
parsed_os = user_agent_parser.ParseOS(user_agent)
os_family = parsed_os['family']
if 'Symbian' in os_family:
os_family = 'Symbian OS'
elif 'BlackBerry' in os_family:
os_family = 'BlackBerry OS'
if os_family is None or os_family not in get_os_list():
os_family = default_os
return os_family
def parse_user_agent(user_agent):
if pd.isnull(user_agent):
return "unknown os"
try:
parsed_string = user_agent_parser.ParseOS(user_agent)
result_string = parsed_string['family']
if parsed_string['major']:
result_string += ' ' + parsed_string['major']
if parsed_string['minor']:
result_string += '.' + parsed_string['minor']
if parsed_string['patch']:
result_string += '.' + parsed_string['patch']
return result_string
except:
return user_agent
def checkTags(self, fp):
parsedUA = user_agent_parser.ParseOS(fp["User-Agent"])
family = parsedUA["family"]
if "Windows" in family:
return [windows]
elif "Mac" in family:
return [mac]
elif family in linuxOS:
return [linux]
elif "Android" in family:
return [android]
elif "iOS" in family:
return [iOS]
else:
return [others]
def get_user_agent_data(request):
userAgent = request.META['HTTP_USER_AGENT']
from ua_parser import user_agent_parser
browser = user_agent_parser.ParseUserAgent(userAgent)
os = user_agent_parser.ParseOS(userAgent)
device = user_agent_parser.ParseDevice(userAgent)
return {
"os":
os['family'] or '' + ' ' + os['major'] or '' + '.' + os['minor']
or '',
"browser":
browser['family'] or '' + ' ' + browser['major'] or '',
"device":
device['brand'] or '' + ' ' + device['family']
or '' + ' ' + device['model'] or '',
}
def get_request_parsed_ua_string(request, parse=None):
ua_string = get_request_ua_string(request)
if not isinstance(ua_string, str):
ua_string = ua_string.decode('utf-8', 'ignore')
if parse == "agent":
return user_agent_parser.ParseUserAgent(ua_string)
elif parse == "os":
return user_agent_parser.ParseOS(ua_string)
elif parse == "device":
return user_agent_parser.ParseDevice(ua_string)
return user_agent_parser.Parse(ua_string)
def w4_output_controller(request):
"""
Outputs data in W4
:param request: user request
:return: job content
"""
response = None
user_agent = user_agent_parser.ParseOS(request.META.get("HTTP_USER_AGENT"))
convert = False
if "windows" in user_agent.get("family").lower():
convert = True
if request.method == 'GET':
controller_job_id = request.GET.get('job_id')
truncate_to = int(request.GET.get("truncate", 0))
result = _w4_get_content(controller_job_id, truncate_to, convert)
response = HttpResponse(json.dumps(result),
content_type='application/json')
response.status_code = int(result['status'])
return response
def parse_ua_text(ua_text):
#parsed_string = user_agent_parser.Parse(ua_text)
#print(parsed_string)
browser_data = user_agent_parser.ParseUserAgent(ua_text)
os_data = user_agent_parser.ParseOS(ua_text)
device_data = user_agent_parser.ParseDevice(ua_text)
browser_name = str(browser_data['family'])
browser_version = str(browser_data['major'])
os_name = str(os_data['family'])
os_version = str(os_data['major'])
device_name = str(device_data['family'])
device_brand = str(device_data['brand'])
device_model = str(device_data['model'])
csv_data = [ua_text,browser_name,browser_version,os_name,os_version,device_name,device_brand,device_model]
return csv_data
def runOSTestsFromYAML(self, file_name):
yamlFile = open(os.path.join(TEST_RESOURCES_DIR, file_name))
yamlContents = yaml.load(yamlFile, Loader=SafeLoader)
yamlFile.close()
for test_case in yamlContents["test_cases"]:
# Inputs to Parse()
user_agent_string = test_case["user_agent_string"]
kwds = {}
if "js_ua" in test_case:
kwds = eval(test_case["js_ua"])
# The expected results
expected = {
"family": test_case["family"],
"major": test_case["major"],
"minor": test_case["minor"],
"patch": test_case["patch"],
"patch_minor": test_case["patch_minor"],
}
result = user_agent_parser.ParseOS(user_agent_string, **kwds)
self.assertEqual(
result,
expected,
"UA: {0}\n expected<{1} {2} {3} {4} {5}> != actual<{6} {7} {8} {9} {10}>"
.format(
user_agent_string,
expected["family"],
expected["major"],
expected["minor"],
expected["patch"],
expected["patch_minor"],
result["family"],
result["major"],
result["minor"],
result["patch"],
result["patch_minor"],
),
)
def get_os(self, request):
if 'os' in request.GET:
os = request.GET['os']
# Only allow known OS, otherwise this might open us to XSS attacks
if os in _KNOWN_OS:
return os
ua_parsed = user_agent_parser.ParseOS(
request.META.get('HTTP_USER_AGENT', ''))
os = ua_parsed['family'].lower().strip()
if os == 'mac os x':
return 'osx'
elif os == 'ios':
return 'ios'
elif os == 'android':
return 'android'
elif os == 'linux':
return 'linux'
elif os.startswith('windows'):
return 'win'
return 'any'
def sessions(self):
from .models import LoggedInUser
sessions = []
logged_in = LoggedInUser.objects.filter(user=self)
for li in logged_in:
try:
s = Session.objects.get(session_key=li.session_key)
puaos = user_agent_parser.ParseOS(li.user_agent)
puaoss = '.'.join(
[puaos[k] for k in puaos.keys() if puaos[k] is not None])
puabr = user_agent_parser.ParseUserAgent(li.user_agent)
puabrs = '.'.join(
[puabr[k] for k in puabr.keys() if puabr[k] is not None])
puad = user_agent_parser.ParseDevice(li.user_agent)
puads = ' '.join(
[puad[k] for k in puad.keys() if puad[k] is not None])
sessions.append((s, ' '.join([puads, puaoss, puabrs])))
except Session.DoesNotExist:
li.delete()
return sessions
def portal_page():
""".. http:get:: /portal
renders the portal page.
.. http:post:: /portal
receives IMSI and redirects to the correct end point based on the device model.
:form uid: IMSI of the device
"""
_user_agent = request.headers['User-Agent']
_device = user_agent_parser.ParseDevice(_user_agent)
_os = user_agent_parser.ParseOS(_user_agent)
_os_ver = '{0}.{1}'.format(str(_os['major'] or ''), str(_os['minor']
or ''))
_form = _UserIdForm(request.form)
if request.method == 'POST' and _form.validate_on_submit():
return redirect(
'{0}?imsi={1}&vendor={2}&model={3}&,version={4}&origin=CL'.format(
_find_passpoint_url(_device), request.form["uid"],
_device['brand'], _device['model'], _os_ver))
return render_template('portal.html',
device=_device['model'],
os_ver=_os_ver,
os_family=_os['family'],
form=_form,
supported=check_if_device_supported(
_device['model'], _os_ver),
captions=_CAPTIONS)
def runOSTestsFromYAML(self, file_name):
yamlFile = open(os.path.join(TEST_RESOURCES_DIR, file_name))
yamlContents = yaml.load(yamlFile)
yamlFile.close()
for test_case in yamlContents['test_cases']:
# Inputs to Parse()
user_agent_string = test_case['user_agent_string']
kwds = {}
if 'js_ua' in test_case:
kwds = eval(test_case['js_ua'])
# The expected results
expected = {
'family': test_case['family'],
'major': test_case['major'],
'minor': test_case['minor'],
'patch': test_case['patch'],
'patch_minor': test_case['patch_minor']
}
result = user_agent_parser.ParseOS(user_agent_string, **kwds)
self.assertEqual(
result, expected,
"UA: {0}\n expected<{1} {2} {3} {4} {5}> != actual<{6} {7} {8} {9} {10}>".format(
user_agent_string,
expected['family'],
expected['major'],
expected['minor'],
expected['patch'],
expected['patch_minor'],
result['family'],
result['major'],
result['minor'],
result['patch'],
result['patch_minor']))
from ua_parser import user_agent_parser
sys.path.append('../')
import processing
parser = argparse.ArgumentParser(
description='Parse the user agent of the participants data')
parser.add_argument('--path',
required=True,
help='Path to the participant data to be parsed')
args = parser.parse_args()
if __name__ == "__main__":
print('Reading participants.csv...')
participants = pd.read_csv(args.path, sep='\t')
print('Parsing user agent (this may take a few minutes)...')
participants['browser'] = participants.USER_AGENT.apply(
lambda x: user_agent_parser.ParseUserAgent(x)['family'])
participants['os'] = participants.USER_AGENT.apply(
lambda x: user_agent_parser.ParseOS(x)['family'])
participants['device_family'] = participants.USER_AGENT.apply(
lambda x: user_agent_parser.ParseDevice(x)['family'])
participants['device_brand'] = participants.USER_AGENT.apply(
lambda x: user_agent_parser.ParseDevice(x)['brand'])
participants['device_model'] = participants.USER_AGENT.apply(
lambda x: user_agent_parser.ParseDevice(x)['model'])
print('Saving to participants.csv...')
participants.to_csv(args.path, sep='\t', index=False)
print('DONE')
df['created_at'] = pd.to_datetime(df['created_at'])
df = pd.merge(df, refs)
idx1 = df.loc[(df['user_id'] == 698) & (df['is_referred'] == False), ].index
idx2 = df.loc[(df['user_id'] == 142711) & (df['is_referred'] == False), ].index
df.drop(idx1, inplace=True)
df.drop(idx2, inplace=True)
fc = pd.read_sql_query(first_clicks, localdb)
df = pd.merge(df, fc, how='left')
df = pd.merge(df, uas, how='left')
df['device'] = \
df['context_user_agent'].apply(lambda x: up.ParseDevice(x)['family']
if pd.notnull(x) else None)
df['OS'] = df['context_user_agent'].apply(
lambda x: up.ParseOS(x)['family'] if pd.notnull(x) else None)
df.loc[df['OS'] == 'Mac OS X', 'device'] = 'Mac'
df.drop('context_user_agent', axis=1, inplace=True)
df['zipcode'] = df['zipcode'].str[:5]
mems_per_zip = \
df.groupby('zipcode')['user_id'].\
count().\
reset_index().\
rename(columns={'user_id': 'members_in_zip'})
df = pd.merge(df, gc, how='left')
df = pd.merge(df, census, how='left', left_on='zipcode', right_on='zip')
df.drop('zip', axis=1, inplace=True)
def log2database(nameFile):
# leer el archivo
try:
file_object = open(nameFile, "r")
except Exception as e:
print(e)
return False
lines = file_object.readlines()
contador = 1
# abrir conexion a la base de datos
db = pymysql.connect(host,username,password,database)
for linea in lines:
objectLog = {}
print("\n[procesando linea del log] = " + str(contador) + "\n")
linea_1 = linea.replace("\n","")
try:
first_split = linea_1.split('\x22')
# agregando user_agent
objectLog['user_agent'] = first_split[5]
# extraer informacion de user agent
parsed_ua = user_agent_parser.ParseUserAgent(objectLog['user_agent'])
print("USER AGENT")
if parsed_ua['family'] is None:
parsed_ua['family'] = "None"
if parsed_ua['major'] is None:
parsed_ua['major'] = "None"
if parsed_ua['minor'] is None:
parsed_ua['minor'] = "None"
if parsed_ua['patch'] is None:
parsed_ua['patch'] = "None"
print(parsed_ua)
# extraer informacion del SO
parsed_os = user_agent_parser.ParseOS(objectLog['user_agent'])
print("Sistema operativo")
print(parsed_os)
if parsed_os['family'] is None:
parsed_os['family'] = "None"
if parsed_os['major'] is None:
parsed_os['major'] = "None"
if parsed_os['minor'] is None:
parsed_os['minor'] = "None"
if parsed_os['patch'] is None:
parsed_os['patch'] = "None"
if parsed_os['patch_minor'] is None:
parsed_os['patch_minor'] = "None"
# extraer informacion del dispositivo
parsed_device = user_agent_parser.ParseDevice(objectLog['user_agent'])
print("Dispositivo")
if parsed_device['family'] is None:
parsed_device['family'] = "None"
if parsed_device['brand'] is None:
parsed_device['brand'] = "None"
if parsed_device['model'] is None:
parsed_device['model'] = "None"
print(parsed_device)
# agregando tcp_log
objectLog['tcp_log'] = first_split[6].replace(" ", "")
# filtrando ip y timestamp
first_split[0] = first_split[0].replace("[","")
first_split[0] = first_split[0].replace("]","")
first_split[0] = first_split[0].replace(" +0000 ","")
ip_date = first_split[0].split(" - - ")
objectLog['ip'] = ip_date[0]
objectLog['fecha'] = ip_date[1]
objectLog['fecha'] = objectLog['fecha'].replace("/", "-")
objectLog['fecha'] = objectLog['fecha'].replace(':', ' ', 1)
objectLog['fecha'] = datetime.strptime(objectLog['fecha'], '%d-%b-%Y %H:%M:%S')
# agregando metodo, url, version de http
met_uri_http = first_split[1].split(" ")
objectLog['metodo'] = met_uri_http[0]
objectLog['url'] = met_uri_http[1]
objectLog['version_http'] = met_uri_http[2]
# agregar codigo de respuesta
cod_res = first_split[2].split(" ")
objectLog['res_codigo'] = cod_res[1]
objectLog['res_codigo_2'] = cod_res[2]
# agregar url de redirccion
objectLog['redireccion'] = "" if ( first_split[3] == "-" ) else first_split[3]
print()
print(objectLog)
# ENVIAR REGISTRO A BASE DE DATOS
try:
###################################################################################
###################################################################################
cursor = db.cursor()
sql = "INSERT INTO db_proxy.logs_squid \
(user_agent,tcp_log,ip,fecha,metodo,url,res_codigo,size, \
redireccion,version_http) \
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);"
cursor.execute(sql, (objectLog['user_agent'], objectLog['tcp_log'], objectLog['ip'], objectLog['fecha'], objectLog['metodo'],objectLog['url'], objectLog['res_codigo'], objectLog['res_codigo_2'], objectLog['redireccion'],objectLog['version_http']))
db.commit()
last_indice = cursor.lastrowid
print("INDEX ID INSERT :" + str(last_indice))
print("[info] : datos insertados en la tabla log_squid")
###################################################################################
###################################################################################
cursor = db.cursor()
sql = "INSERT INTO db_proxy.dispositivo \
(logs_squid_idlogs_squid,family,brand,model) \
VALUES (%s,%s,%s,%s);"
cursor.execute(sql, (last_indice , parsed_device['family'], parsed_device['brand'], parsed_device['model'] ))
db.commit()
print("[info] : datos insertados en la tabla dispositivo")
###################################################################################
###################################################################################
cursor = db.cursor()
sql = "INSERT INTO db_proxy.navegador \
(logs_squid_idlogs_squid,family,major,minor,patch) \
VALUES \
(%s,%s,%s,%s,%s);"
cursor.execute(sql, (last_indice, parsed_ua['family'], parsed_ua['major'], parsed_ua['minor'], parsed_ua['patch'] ))
db.commit()
print("[info] : datos insertados en la tabla navegador")
###################################################################################
###################################################################################
cursor = db.cursor()
sql = "INSERT INTO db_proxy.sistema_operativo \
(logs_squid_idlogs_squid,family,major,minor,patch,patch_minor) \
VALUES \
(%s,%s,%s,%s,%s,%s);"
cursor.execute(sql, (last_indice, parsed_os['family'], parsed_os['major'], parsed_os['minor'], parsed_os['patch'], parsed_os['patch_minor'] ))
db.commit()
print("[info] : datos insertados en la tabla sistema operativo")
except Exception as e:
print(e)
except Exception as e:
print(e)
contador += 1
db.close()
def oper_system_extraction(x):
user_agent = user_agent_parser.ParseOS(str(x))
return user_agent['family']
def parse_single_brand():
pp = pprint.PrettyPrinter(indent=4)
parsed_map = user_agent_parser.ParseOS("Mozilla%2F5.0+%28Windows+NT+10.0%3B+Win64%3B+x64%29+AppleWebKit%2F537.36+%28KHTML%2C+like+Gecko%29+Chrome%2F74.0.3729.169+Safari%2F537.36")
if parsed_map:
print(parsed_map["family"])
