def flag_machines(doc): """Flag events which are created by machines. The list of machines is defined by the `COUNTER-robots Python package <https://github.com/inveniosoftware/counter-robots>`_ , which follows the `list defined by Project COUNTER <https://www.projectcounter.org/appendices/850-2/>`_ that was later split into robots and machines by `the Make Data Count project <https://github.com/CDLUC3/Make-Data-Count/tree/master/user-agents>`_. """ doc['is_machine'] = 'user_agent' in doc and is_machine(doc['user_agent']) return doc
def lambda_handler(event, context): output = [] for record in event['records']: payload = base64.b64decode(record['data']) log = json.loads(payload) # Flattens the header fields flatheaders = {} for item in log['httpRequest']['headers']: flatheaders[item['name'].lower()] = item['value'] del (log['httpRequest']['headers']) log['httpRequest']['headers'] = flatheaders # Adds ASN number log['httpRequest']['asn'] = asndb.lookup( log['httpRequest']['clientIp'])[0] user_agent_string = log['httpRequest']['headers']['user-agent'] # Adds user-agent information ua_info = getUAInfo(user_agent_string) log['httpRequest']['browser'] = ua_info['browser'] log['httpRequest']['os'] = ua_info['os'] log['httpRequest']['device'] = ua_info['device'] # Adds robot information device_type = '' if is_machine(user_agent_string): device_type = 'machine' elif is_robot(user_agent_string): device_type = 'robot' else: device_type = 'browser' log['httpRequest']['deviceType'] = device_type payload = json.dumps(log) output_record = { 'recordId': record['recordId'], 'result': 'Ok', 'data': base64.b64encode(payload) } output.append(output_record) print('Successfully processed {} records.'.format(len(event['records']))) return {'records': output}
def test_is_machine(): machine_ua = 'Wget/1.14 (linux-gnu)' robot_ua = 'AdsBot-Google (+http://www.google.com/adsbot.html)' assert is_machine(machine_ua) is True assert is_machine(robot_ua) is not True