def w4_output_controller_download(request): """ User can download content in W4 by clicking on link :param request: user request :return: content data onto a file """ response = None fng = w4_output_controller_download_filename # file name generator :-D if request.method == 'GET': user_agent = user_agent_parser.ParseOS( request.META.get("HTTP_USER_AGENT")) controller_job_id = request.GET.get('job_id') controller_job_number = request.GET.get('job_number', "") content = get_brain_output_content(controller_job_id, max_size=None) if "windows" in user_agent.get("family").lower() and isinstance( content, str): content = content.replace("\n", "\r\n") response = HttpResponse(content, content_type='application/octet-stream') content_dispo = 'attachment; \ filename="{}"'.format( fng(controller_job_id, controller_job_number)) response['Content-Disposition'] = content_dispo response.status_code = 200 return response
def show_apple_maps(request): """ Test if we can show an Apple Maps link to a device. There's no Web version of Apple Maps; it redirects on Google Maps on non-iOS/OSX platforms. """ ua = user_agent_parser.ParseOS(request.user_agent.string) return ua.get("family") in ("iOS", "Mac OS X")
def check_os(user_agent_str: str) -> list: result = config.types_by_ua[-1:] family = user_agent_parser.ParseOS(user_agent_str)["family"].lower() os_names = dict( zip(["mac os x", "window", "solaris", "linux"], config.types_by_ua[:-1])) if family in os_names: result.append(os_names[family]) return result
def process_tcp_packet(packet): if not packet.haslayer(http.HTTPRequest): # This packet doesn't contain an HTTP request so we skip it return http_layer = packet.getlayer(http.HTTPRequest) ip_layer = packet.getlayer(IP) try: ip = '{0[src]}'.format(ip_layer.fields, http_layer.fields) ua_string = '{1[User-Agent]}'.format(ip_layer.fields, http_layer.fields) agent = user_agent_parser.ParseUserAgent(ua_string) print ua_string device = user_agent_parser.ParseDevice(ua_string) os = user_agent_parser.ParseOS(ua_string) browser_family = agent['family'] browser_major = agent['major'] browser_minor = agent['minor'] browser_patch = agent['patch'] device_brand = device['brand'] device_family = device['family'] device_model = device['model'] os_family = os['family'] os_major = os['major'] os_minor = os['minor'] os_patch = os['patch'] os_patch_minor = os['patch_minor'] key = find_key(ip) device_hash = wdb.Hash(key) hostname = device_hash['hostname'] #print key,hostname,ip,browser_family,browser_major,browser_minor,browser_patch,device_brand,device_family,device_model,os_family,os_major,os_minor,os_patch,os_patch_minor os_version = str(os_major) +'.'+ str(os_minor) +'.'+ str(os_patch) browser_combined = browser_family +'('+ str(browser_major) +'.'+ str(browser_minor) +'.'+ str(browser_patch) +')' k = wdb.Hash(key) if "Other" not in os_family: k.update(os=os_family) print ip,key,os_family if os_family == "Mac OS X": k.update(category="Desktop/Laptop") if os_family == "iOS": k.update(category="Smartphones/PDAs/Tablets") if ("None" or "Other") not in os_version: k.update(os_version=os_version) print ip,key,os_version if ("None" or "Other") not in browser_combined: k.update(browser_family=browser_combined) print ip,key,browser_combined except KeyError: pass
def parse_brand(): # pp = pprint.PrettyPrinter(indent=4) with open("/Users/zaniu/Documents/logs/analysis/user_agent_file.txt", "r") as user_agent_file: user_agent_file_result = open("/Users/zaniu/Documents/logs/analysis/user_agent_brand_result_os.txt", "w") for user_agent in user_agent_file: if user_agent: parsed_map = user_agent_parser.ParseOS(user_agent) if parsed_map: print(parsed_map["family"]) user_agent_file_result.write("{}".format(parsed_map["family"]) + "\n") user_agent_file_result.close
def extract_os_from_user_agent(user_agent, default_os='Other'): parsed_os = user_agent_parser.ParseOS(user_agent) os_family = parsed_os['family'] if 'Symbian' in os_family: os_family = 'Symbian OS' elif 'BlackBerry' in os_family: os_family = 'BlackBerry OS' if os_family is None or os_family not in get_os_list(): os_family = default_os return os_family
def parse_user_agent(user_agent): if pd.isnull(user_agent): return "unknown os" try: parsed_string = user_agent_parser.ParseOS(user_agent) result_string = parsed_string['family'] if parsed_string['major']: result_string += ' ' + parsed_string['major'] if parsed_string['minor']: result_string += '.' + parsed_string['minor'] if parsed_string['patch']: result_string += '.' + parsed_string['patch'] return result_string except: return user_agent
def checkTags(self, fp): parsedUA = user_agent_parser.ParseOS(fp["User-Agent"]) family = parsedUA["family"] if "Windows" in family: return [windows] elif "Mac" in family: return [mac] elif family in linuxOS: return [linux] elif "Android" in family: return [android] elif "iOS" in family: return [iOS] else: return [others]
def get_user_agent_data(request): userAgent = request.META['HTTP_USER_AGENT'] from ua_parser import user_agent_parser browser = user_agent_parser.ParseUserAgent(userAgent) os = user_agent_parser.ParseOS(userAgent) device = user_agent_parser.ParseDevice(userAgent) return { "os": os['family'] or '' + ' ' + os['major'] or '' + '.' + os['minor'] or '', "browser": browser['family'] or '' + ' ' + browser['major'] or '', "device": device['brand'] or '' + ' ' + device['family'] or '' + ' ' + device['model'] or '', }
def get_request_parsed_ua_string(request, parse=None): ua_string = get_request_ua_string(request) if not isinstance(ua_string, str): ua_string = ua_string.decode('utf-8', 'ignore') if parse == "agent": return user_agent_parser.ParseUserAgent(ua_string) elif parse == "os": return user_agent_parser.ParseOS(ua_string) elif parse == "device": return user_agent_parser.ParseDevice(ua_string) return user_agent_parser.Parse(ua_string)
def w4_output_controller(request): """ Outputs data in W4 :param request: user request :return: job content """ response = None user_agent = user_agent_parser.ParseOS(request.META.get("HTTP_USER_AGENT")) convert = False if "windows" in user_agent.get("family").lower(): convert = True if request.method == 'GET': controller_job_id = request.GET.get('job_id') truncate_to = int(request.GET.get("truncate", 0)) result = _w4_get_content(controller_job_id, truncate_to, convert) response = HttpResponse(json.dumps(result), content_type='application/json') response.status_code = int(result['status']) return response
def parse_ua_text(ua_text): #parsed_string = user_agent_parser.Parse(ua_text) #print(parsed_string) browser_data = user_agent_parser.ParseUserAgent(ua_text) os_data = user_agent_parser.ParseOS(ua_text) device_data = user_agent_parser.ParseDevice(ua_text) browser_name = str(browser_data['family']) browser_version = str(browser_data['major']) os_name = str(os_data['family']) os_version = str(os_data['major']) device_name = str(device_data['family']) device_brand = str(device_data['brand']) device_model = str(device_data['model']) csv_data = [ua_text,browser_name,browser_version,os_name,os_version,device_name,device_brand,device_model] return csv_data
def runOSTestsFromYAML(self, file_name): yamlFile = open(os.path.join(TEST_RESOURCES_DIR, file_name)) yamlContents = yaml.load(yamlFile, Loader=SafeLoader) yamlFile.close() for test_case in yamlContents["test_cases"]: # Inputs to Parse() user_agent_string = test_case["user_agent_string"] kwds = {} if "js_ua" in test_case: kwds = eval(test_case["js_ua"]) # The expected results expected = { "family": test_case["family"], "major": test_case["major"], "minor": test_case["minor"], "patch": test_case["patch"], "patch_minor": test_case["patch_minor"], } result = user_agent_parser.ParseOS(user_agent_string, **kwds) self.assertEqual( result, expected, "UA: {0}\n expected<{1} {2} {3} {4} {5}> != actual<{6} {7} {8} {9} {10}>" .format( user_agent_string, expected["family"], expected["major"], expected["minor"], expected["patch"], expected["patch_minor"], result["family"], result["major"], result["minor"], result["patch"], result["patch_minor"], ), )
def get_os(self, request): if 'os' in request.GET: os = request.GET['os'] # Only allow known OS, otherwise this might open us to XSS attacks if os in _KNOWN_OS: return os ua_parsed = user_agent_parser.ParseOS( request.META.get('HTTP_USER_AGENT', '')) os = ua_parsed['family'].lower().strip() if os == 'mac os x': return 'osx' elif os == 'ios': return 'ios' elif os == 'android': return 'android' elif os == 'linux': return 'linux' elif os.startswith('windows'): return 'win' return 'any'
def sessions(self): from .models import LoggedInUser sessions = [] logged_in = LoggedInUser.objects.filter(user=self) for li in logged_in: try: s = Session.objects.get(session_key=li.session_key) puaos = user_agent_parser.ParseOS(li.user_agent) puaoss = '.'.join( [puaos[k] for k in puaos.keys() if puaos[k] is not None]) puabr = user_agent_parser.ParseUserAgent(li.user_agent) puabrs = '.'.join( [puabr[k] for k in puabr.keys() if puabr[k] is not None]) puad = user_agent_parser.ParseDevice(li.user_agent) puads = ' '.join( [puad[k] for k in puad.keys() if puad[k] is not None]) sessions.append((s, ' '.join([puads, puaoss, puabrs]))) except Session.DoesNotExist: li.delete() return sessions
def portal_page(): """.. http:get:: /portal renders the portal page. .. http:post:: /portal receives IMSI and redirects to the correct end point based on the device model. :form uid: IMSI of the device """ _user_agent = request.headers['User-Agent'] _device = user_agent_parser.ParseDevice(_user_agent) _os = user_agent_parser.ParseOS(_user_agent) _os_ver = '{0}.{1}'.format(str(_os['major'] or ''), str(_os['minor'] or '')) _form = _UserIdForm(request.form) if request.method == 'POST' and _form.validate_on_submit(): return redirect( '{0}?imsi={1}&vendor={2}&model={3}&,version={4}&origin=CL'.format( _find_passpoint_url(_device), request.form["uid"], _device['brand'], _device['model'], _os_ver)) return render_template('portal.html', device=_device['model'], os_ver=_os_ver, os_family=_os['family'], form=_form, supported=check_if_device_supported( _device['model'], _os_ver), captions=_CAPTIONS)
def runOSTestsFromYAML(self, file_name): yamlFile = open(os.path.join(TEST_RESOURCES_DIR, file_name)) yamlContents = yaml.load(yamlFile) yamlFile.close() for test_case in yamlContents['test_cases']: # Inputs to Parse() user_agent_string = test_case['user_agent_string'] kwds = {} if 'js_ua' in test_case: kwds = eval(test_case['js_ua']) # The expected results expected = { 'family': test_case['family'], 'major': test_case['major'], 'minor': test_case['minor'], 'patch': test_case['patch'], 'patch_minor': test_case['patch_minor'] } result = user_agent_parser.ParseOS(user_agent_string, **kwds) self.assertEqual( result, expected, "UA: {0}\n expected<{1} {2} {3} {4} {5}> != actual<{6} {7} {8} {9} {10}>".format( user_agent_string, expected['family'], expected['major'], expected['minor'], expected['patch'], expected['patch_minor'], result['family'], result['major'], result['minor'], result['patch'], result['patch_minor']))
from ua_parser import user_agent_parser sys.path.append('../') import processing parser = argparse.ArgumentParser( description='Parse the user agent of the participants data') parser.add_argument('--path', required=True, help='Path to the participant data to be parsed') args = parser.parse_args() if __name__ == "__main__": print('Reading participants.csv...') participants = pd.read_csv(args.path, sep='\t') print('Parsing user agent (this may take a few minutes)...') participants['browser'] = participants.USER_AGENT.apply( lambda x: user_agent_parser.ParseUserAgent(x)['family']) participants['os'] = participants.USER_AGENT.apply( lambda x: user_agent_parser.ParseOS(x)['family']) participants['device_family'] = participants.USER_AGENT.apply( lambda x: user_agent_parser.ParseDevice(x)['family']) participants['device_brand'] = participants.USER_AGENT.apply( lambda x: user_agent_parser.ParseDevice(x)['brand']) participants['device_model'] = participants.USER_AGENT.apply( lambda x: user_agent_parser.ParseDevice(x)['model']) print('Saving to participants.csv...') participants.to_csv(args.path, sep='\t', index=False) print('DONE')
df['created_at'] = pd.to_datetime(df['created_at']) df = pd.merge(df, refs) idx1 = df.loc[(df['user_id'] == 698) & (df['is_referred'] == False), ].index idx2 = df.loc[(df['user_id'] == 142711) & (df['is_referred'] == False), ].index df.drop(idx1, inplace=True) df.drop(idx2, inplace=True) fc = pd.read_sql_query(first_clicks, localdb) df = pd.merge(df, fc, how='left') df = pd.merge(df, uas, how='left') df['device'] = \ df['context_user_agent'].apply(lambda x: up.ParseDevice(x)['family'] if pd.notnull(x) else None) df['OS'] = df['context_user_agent'].apply( lambda x: up.ParseOS(x)['family'] if pd.notnull(x) else None) df.loc[df['OS'] == 'Mac OS X', 'device'] = 'Mac' df.drop('context_user_agent', axis=1, inplace=True) df['zipcode'] = df['zipcode'].str[:5] mems_per_zip = \ df.groupby('zipcode')['user_id'].\ count().\ reset_index().\ rename(columns={'user_id': 'members_in_zip'}) df = pd.merge(df, gc, how='left') df = pd.merge(df, census, how='left', left_on='zipcode', right_on='zip') df.drop('zip', axis=1, inplace=True)
def log2database(nameFile): # leer el archivo try: file_object = open(nameFile, "r") except Exception as e: print(e) return False lines = file_object.readlines() contador = 1 # abrir conexion a la base de datos db = pymysql.connect(host,username,password,database) for linea in lines: objectLog = {} print("\n[procesando linea del log] = " + str(contador) + "\n") linea_1 = linea.replace("\n","") try: first_split = linea_1.split('\x22') # agregando user_agent objectLog['user_agent'] = first_split[5] # extraer informacion de user agent parsed_ua = user_agent_parser.ParseUserAgent(objectLog['user_agent']) print("USER AGENT") if parsed_ua['family'] is None: parsed_ua['family'] = "None" if parsed_ua['major'] is None: parsed_ua['major'] = "None" if parsed_ua['minor'] is None: parsed_ua['minor'] = "None" if parsed_ua['patch'] is None: parsed_ua['patch'] = "None" print(parsed_ua) # extraer informacion del SO parsed_os = user_agent_parser.ParseOS(objectLog['user_agent']) print("Sistema operativo") print(parsed_os) if parsed_os['family'] is None: parsed_os['family'] = "None" if parsed_os['major'] is None: parsed_os['major'] = "None" if parsed_os['minor'] is None: parsed_os['minor'] = "None" if parsed_os['patch'] is None: parsed_os['patch'] = "None" if parsed_os['patch_minor'] is None: parsed_os['patch_minor'] = "None" # extraer informacion del dispositivo parsed_device = user_agent_parser.ParseDevice(objectLog['user_agent']) print("Dispositivo") if parsed_device['family'] is None: parsed_device['family'] = "None" if parsed_device['brand'] is None: parsed_device['brand'] = "None" if parsed_device['model'] is None: parsed_device['model'] = "None" print(parsed_device) # agregando tcp_log objectLog['tcp_log'] = first_split[6].replace(" ", "") # filtrando ip y timestamp first_split[0] = first_split[0].replace("[","") first_split[0] = first_split[0].replace("]","") first_split[0] = first_split[0].replace(" +0000 ","") ip_date = first_split[0].split(" - - ") objectLog['ip'] = ip_date[0] objectLog['fecha'] = ip_date[1] objectLog['fecha'] = objectLog['fecha'].replace("/", "-") objectLog['fecha'] = objectLog['fecha'].replace(':', ' ', 1) objectLog['fecha'] = datetime.strptime(objectLog['fecha'], '%d-%b-%Y %H:%M:%S') # agregando metodo, url, version de http met_uri_http = first_split[1].split(" ") objectLog['metodo'] = met_uri_http[0] objectLog['url'] = met_uri_http[1] objectLog['version_http'] = met_uri_http[2] # agregar codigo de respuesta cod_res = first_split[2].split(" ") objectLog['res_codigo'] = cod_res[1] objectLog['res_codigo_2'] = cod_res[2] # agregar url de redirccion objectLog['redireccion'] = "" if ( first_split[3] == "-" ) else first_split[3] print() print(objectLog) # ENVIAR REGISTRO A BASE DE DATOS try: ################################################################################### ################################################################################### cursor = db.cursor() sql = "INSERT INTO db_proxy.logs_squid \ (user_agent,tcp_log,ip,fecha,metodo,url,res_codigo,size, \ redireccion,version_http) \ VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);" cursor.execute(sql, (objectLog['user_agent'], objectLog['tcp_log'], objectLog['ip'], objectLog['fecha'], objectLog['metodo'],objectLog['url'], objectLog['res_codigo'], objectLog['res_codigo_2'], objectLog['redireccion'],objectLog['version_http'])) db.commit() last_indice = cursor.lastrowid print("INDEX ID INSERT :" + str(last_indice)) print("[info] : datos insertados en la tabla log_squid") ################################################################################### ################################################################################### cursor = db.cursor() sql = "INSERT INTO db_proxy.dispositivo \ (logs_squid_idlogs_squid,family,brand,model) \ VALUES (%s,%s,%s,%s);" cursor.execute(sql, (last_indice , parsed_device['family'], parsed_device['brand'], parsed_device['model'] )) db.commit() print("[info] : datos insertados en la tabla dispositivo") ################################################################################### ################################################################################### cursor = db.cursor() sql = "INSERT INTO db_proxy.navegador \ (logs_squid_idlogs_squid,family,major,minor,patch) \ VALUES \ (%s,%s,%s,%s,%s);" cursor.execute(sql, (last_indice, parsed_ua['family'], parsed_ua['major'], parsed_ua['minor'], parsed_ua['patch'] )) db.commit() print("[info] : datos insertados en la tabla navegador") ################################################################################### ################################################################################### cursor = db.cursor() sql = "INSERT INTO db_proxy.sistema_operativo \ (logs_squid_idlogs_squid,family,major,minor,patch,patch_minor) \ VALUES \ (%s,%s,%s,%s,%s,%s);" cursor.execute(sql, (last_indice, parsed_os['family'], parsed_os['major'], parsed_os['minor'], parsed_os['patch'], parsed_os['patch_minor'] )) db.commit() print("[info] : datos insertados en la tabla sistema operativo") except Exception as e: print(e) except Exception as e: print(e) contador += 1 db.close()
def oper_system_extraction(x): user_agent = user_agent_parser.ParseOS(str(x)) return user_agent['family']
def parse_single_brand(): pp = pprint.PrettyPrinter(indent=4) parsed_map = user_agent_parser.ParseOS("Mozilla%2F5.0+%28Windows+NT+10.0%3B+Win64%3B+x64%29+AppleWebKit%2F537.36+%28KHTML%2C+like+Gecko%29+Chrome%2F74.0.3729.169+Safari%2F537.36") if parsed_map: print(parsed_map["family"])