def _hawk_api_request( url: str, credentials: dict, results_key: Optional[str], next_key: Optional[str], validate_response: Optional[bool] = True, force_http: Optional[bool] = False, ): sender = Sender( credentials, # Currently data workspace denies hawk requests signed with https urls. # Once fixed the protocol replacement can be removed. url.replace('https', 'http') if force_http else url, "get", content="", content_type="", always_hash_content=True, ) logger.info(f"Fetching page {url}") response = requests.get( url, headers={ "Authorization": sender.request_header, "Content-Type": "" }, timeout=300, ) try: response.raise_for_status() except requests.exceptions.HTTPError: logger.warning(f"Request failed: {response.text}") raise if validate_response: try: sender.accept_response( response.headers["Server-Authorization"], content=response.content, content_type=response.headers["Content-Type"], ) except HawkFail as e: logger.error(f"HAWK Authentication failed {str(e)}") raise response_json = response.json() if (next_key and next_key not in response_json) or ( results_key and results_key not in response_json): raise ValueError("Unexpected response structure") return response_json
def handle(self, *args, **options): hawk_log = logging.getLogger('mohawk') hawk_log.setLevel(logging.DEBUG) hawk_log.addHandler(logging.StreamHandler()) try: import requests except ImportError: raise CommandError('To use this command you first need to ' 'install the requests module') url = options['url'] if not url: raise CommandError('Specify a URL to load with --url') qs = options['d'] or '' request_content_type = ('application/x-www-form-urlencoded' if qs else 'text/plain') method = options['X'] credentials = lookup_credentials(options['creds']) sender = Sender(credentials, url, method.upper(), content=qs, content_type=request_content_type) headers = { 'Authorization': sender.request_header, 'Content-Type': request_content_type } do_request = getattr(requests, method.lower()) res = do_request(url, data=qs, headers=headers) print '{method} -d {qs} {url}'.format(method=method.upper(), qs=qs or 'None', url=url) print res.text # Verify we're talking to our trusted server. print res.headers auth_hdr = res.headers.get('Server-Authorization', None) if auth_hdr: sender.accept_response(auth_hdr, content=res.text, content_type=res.headers['Content-Type']) print '<response was Hawk verified>' else: print '** NO Server-Authorization header **' print '<response was NOT Hawk verified>'
def handle(self, *args, **options): # Configure the mohawk lib for debug logging so we can see inputs to # the signature functions and other useful stuff. hawk_log = logging.getLogger('mohawk') hawk_log.setLevel(logging.DEBUG) hawk_log.addHandler(logging.StreamHandler()) url = options['url'] if not url: raise CommandError('Specify a URL to load with --url') creds_key = options['creds'] if not creds_key: raise CommandError('Specify ID for Hawk credentials with --creds') method = options['X'] qs = options['d'] or '' request_content_type = ('application/x-www-form-urlencoded' if qs else 'text/plain') credentials = lookup_credentials(creds_key) sender = Sender(credentials, url, method.upper(), content=qs, content_type=request_content_type) headers = { 'Authorization': sender.request_header, 'Content-Type': request_content_type } res = request(url, method.lower(), data=qs, headers=headers) self.stdout.write('{method} -d {qs} {url}'.format( method=method.upper(), qs=qs or 'None', url=url)) self.stdout.write(res.text) # Verify we're talking to our trusted server. self.stdout.write(str(res.headers)) auth_hdr = res.headers.get('Server-Authorization', None) if auth_hdr: sender.accept_response(auth_hdr, content=res.text, content_type=res.headers['Content-Type']) self.stdout.write('<response was Hawk verified>') else: self.stdout.write('** NO Server-Authorization header **') self.stdout.write('<response was NOT Hawk verified>')
def handle(self, *args, **options): hawk_log = logging.getLogger('mohawk') hawk_log.setLevel(logging.DEBUG) hawk_log.addHandler(logging.StreamHandler()) try: import requests except ImportError: raise CommandError('To use this command you first need to ' 'install the requests module') url = options['url'] if not url: raise CommandError('Specify a URL to load with --url') qs = options['d'] or '' request_content_type = ('application/x-www-form-urlencoded' if qs else 'text/plain') method = options['X'] credentials = lookup_credentials(options['creds']) sender = Sender(credentials, url, method.upper(), content=qs, content_type=request_content_type) headers = {'Authorization': sender.request_header, 'Content-Type': request_content_type} do_request = getattr(requests, method.lower()) res = do_request(url, data=qs, headers=headers) print '{method} -d {qs} {url}'.format(method=method.upper(), qs=qs or 'None', url=url) print res.text # Verify we're talking to our trusted server. print res.headers auth_hdr = res.headers.get('Server-Authorization', None) if auth_hdr: sender.accept_response(auth_hdr, content=res.text, content_type=res.headers['Content-Type']) print '<response was Hawk verified>' else: print '** NO Server-Authorization header **' print '<response was NOT Hawk verified>'
def handle(self, *args, **options): # Configure the mohawk lib for debug logging so we can see inputs to # the signature functions and other useful stuff. hawk_log = logging.getLogger('mohawk') hawk_log.setLevel(logging.DEBUG) hawk_log.addHandler(logging.StreamHandler()) url = options['url'] if not url: raise CommandError('Specify a URL to load with --url') creds_key = options['creds'] if not creds_key: raise CommandError('Specify ID for Hawk credentials with --creds') method = options['X'] qs = options['d'] or '' request_content_type = ('application/x-www-form-urlencoded' if qs else 'text/plain') credentials = lookup_credentials(creds_key) sender = Sender(credentials, url, method.upper(), content=qs, content_type=request_content_type) headers = {'Authorization': sender.request_header, 'Content-Type': request_content_type} res = request(url, method.lower(), data=qs, headers=headers) self.stdout.write('{method} -d {qs} {url}'.format(method=method.upper(), qs=qs or 'None', url=url)) self.stdout.write(res.text) # Verify we're talking to our trusted server. self.stdout.write(str(res.headers)) auth_hdr = res.headers.get('Server-Authorization', None) if auth_hdr: sender.accept_response(auth_hdr, content=res.text, content_type=res.headers['Content-Type']) self.stdout.write('<response was Hawk verified>') else: self.stdout.write('** NO Server-Authorization header **') self.stdout.write('<response was NOT Hawk verified>')
def search_with_activitystream(query): """ Searches ActivityStream services with given Elasticsearch query. Note that this must be at root level in SearchView class to enable it to be mocked in tests. """ request = requests.Request( method="GET", url=settings.ACTIVITY_STREAM_API_URL, data=query).prepare() auth = Sender( { 'id': settings.ACTIVITY_STREAM_API_ACCESS_KEY, 'key': settings.ACTIVITY_STREAM_API_SECRET_KEY, 'algorithm': 'sha256' }, settings.ACTIVITY_STREAM_API_URL, "GET", content=query, content_type='application/json', ).request_header request.headers.update({ 'X-Forwarded-Proto': 'https', 'Authorization': auth, 'Content-Type': 'application/json' }) return requests.Session().send(request)
def get_dataflow_dag_status(execution_date): config = settings.DATAFLOW_API_CONFIG url = ( f'{config["DATAFLOW_BASE_URL"]}/api/experimental/' f'dags/{config["DATAFLOW_S3_IMPORT_DAG"]}/dag_runs/{execution_date.split("+")[0]}' ) hawk_creds = { 'id': config['DATAFLOW_HAWK_ID'], 'key': config['DATAFLOW_HAWK_KEY'], 'algorithm': 'sha256', } header = Sender( hawk_creds, url, 'get', content='', content_type='', ).request_header response = requests.get( url, headers={ 'Authorization': header, 'Content-Type': '' }, ) response.raise_for_status() return response.json()
def _sign_content(sign_payload): # if the list is empty, return an empty signature list if len(sign_payload) == 0: return [] content_type = 'application/json' sender = Sender( { 'id': env.config.SIGNING_HAWK_ID, 'key': env.config.SIGNING_HAWK_KEY, 'algorithm': 'sha256' }, # credentials env.config.SIGNING_SERVICE_URL, # url 'POST', # method content=json.dumps(sign_payload), content_type=content_type) try: r = requests.post(env.config.SIGNING_SERVICE_URL, json=sign_payload, headers={ 'Content-Type': content_type, 'Authorization': sender.request_header }) if r.status_code != 201: msg = "signing server returned an error code: %s" % (r.status_code) raise Exception(msg) except Exception as e: msg = "Error when signing: %s" % e env.log(msg) raise else: return r.json()
def _hawk_api_request( url: str, method: str, query: dict, credentials: dict, expected_response_structure: str = None, ): body = json.dumps(query) header = Sender( credentials, url, method.lower(), content_type="application/json", content=body ).request_header response = requests.request( method, url, data=body, headers={"Authorization": header, "Content-Type": "application/json"}, ) try: response.raise_for_status() except requests.exceptions.HTTPError: logger.warning(f"Request failed: {response.text}") raise response_json = response.json() if expected_response_structure and expected_response_structure not in response_json: raise ValueError("Unexpected response structure") return response_json
def hawk_api_request( url, method, credentials, body=None, ): if body: body = json.dumps(body) auth_header = Sender( credentials, url, method.lower(), content_type="application/json" if body else None, content=body, ).request_header headers = {"Authorization": auth_header} if body: headers["Content-Type"] = "application/json" response = requests.request( method, url, data=body, headers=headers, ) response.raise_for_status() response_json = response.json() return response_json
def start(): entry = { "userId": user_id, "start": now_string(), "end": None, "timezoneName": "CET", "timezone": "+0100", "type": "work" } url = 'https://app.absence.io/api/v2/timespans/create' method = 'POST' content = json.dumps(entry) content_type = 'application/json' sender = Sender({ 'id': user_id, 'key': key, 'algorithm': 'sha256' }, url, method, content=content, content_type=content_type) response = requests.post(url, data=content, headers={ 'Authorization': sender.request_header, 'Content-Type': content_type }) return response
def request(self, method, path, **kwargs): if not settings.DATAHUB_URL: raise DataHubException("DATAHUB_URL is not set") url = f"{settings.DATAHUB_URL}{path}" credentials = { "id": settings.DATAHUB_HAWK_ID, "key": settings.DATAHUB_HAWK_KEY, "algorithm": "sha256", } sender = Sender( credentials, url, method, content=json.dumps(kwargs), content_type="application/json", always_hash_content=False, ) headers = {"Authorization": sender.request_header} response = getattr(requests, method)(url, verify=not settings.DEBUG, headers=headers, json=kwargs) try: response.raise_for_status() except requests.exceptions.HTTPError as e: raise APIHttpException(e) return response.json()
def hawk_request(method, url, body): hawk_id = settings.ACTIVITY_STREAM_HAWK_CREDENTIALS_ID hawk_key = settings.ACTIVITY_STREAM_HAWK_CREDENTIALS_KEY if not hawk_id or not hawk_key: raise HawkException("Hawk id or key not configured") content_type = 'application/json' header = Sender( { 'id': hawk_id, 'key': hawk_key, 'algorithm': 'sha256' }, url, method, content=body, content_type=content_type, ).request_header response = requests.request( method, url, data=body, headers={ 'Authorization': header, 'Content-Type': content_type }, ) return response.status_code, response.content
def trigger_dataflow_dag(conf, dag, dag_run_id): config = settings.DATAFLOW_API_CONFIG trigger_url = f'{config["DATAFLOW_BASE_URL"]}/api/experimental/' f"dags/{dag}/dag_runs" hawk_creds = { "id": config["DATAFLOW_HAWK_ID"], "key": config["DATAFLOW_HAWK_KEY"], "algorithm": "sha256", } method = "POST" content_type = "application/json" body = json.dumps( { "run_id": dag_run_id, "replace_microseconds": "false", "conf": conf, } ) header = Sender( hawk_creds, trigger_url, method.lower(), content=body, content_type=content_type, ).request_header response = requests.request( method, trigger_url, data=body, headers={"Authorization": header, "Content-Type": content_type}, ) response.raise_for_status() return response.json()
def _get_hawk_response(client_id, secret, method='GET', content='', content_type='application/json'): auth = {'id': client_id, 'key': secret, 'algorithm': 'sha256'} url = 'http://testserver/' sender = Sender(auth, url, method, content=content, content_type='application/json') do_request = getattr(factory, method.lower()) request = do_request( url, data=content, content_type='application/json', # factory.get doesn't set the CONTENT_TYPE header # I'm setting it manually here for simplicity CONTENT_TYPE='application/json', HTTP_AUTHORIZATION=sender.request_header) view = AuthenticatedView.as_view() return view(request)
def search_with_activitystream(query): """ Searches ActivityStream services with given Elasticsearch query. Note that this must be at root level in SearchView class to enable it to be mocked in tests. """ request = requests.Request( method="GET", url=settings.ACTIVITY_STREAM_API_URL, data=query).prepare() auth = Sender( { 'id': settings.ACTIVITY_STREAM_API_ACCESS_KEY, 'key': settings.ACTIVITY_STREAM_API_SECRET_KEY, 'algorithm': 'sha256' }, settings.ACTIVITY_STREAM_API_URL, "GET", content=query, content_type='application/json', ).request_header # Note that the X-Forwarded-* items are overridden by Gov PaaS values # in production, and thus the value of ACTIVITY_STREAM_API_IP_WHITELIST # in production is irrelivant. It is included here to allow the app to # run locally or outside of Gov PaaS. request.headers.update({ 'X-Forwarded-Proto': 'https', 'X-Forwarded-For': settings.ACTIVITY_STREAM_API_IP_WHITELIST, 'Authorization': auth, 'Content-Type': 'application/json' }) return requests.Session().send(request)
def api_check(): data = {"status": HealthStatus.FAIL, "duration": None} # trailing / is important here url = f"{settings.MARKET_ACCESS_API_URI}check/" sender = Sender( settings.MARKET_ACCESS_API_HAWK_CREDS, url, "GET", content="", content_type="text/plain", always_hash_content=False, ) try: response = requests.get( url, verify=not settings.DEBUG, headers={ "Authorization": sender.request_header, "Content-Type": "text/plain", }, ) response.raise_for_status() response_data = response.json() except Exception: pass else: data["status"] = response_data["status"] data["duration"] = response_data.get("duration") return data
def save_pipeline_to_dataflow(pipeline, method): url = f"{API_URL}/dag/{pipeline.dag_id}" content_type = "application/json" schema_name, table_name = pipeline.table_name.split(".") body = json.dumps({ "schedule": "@daily", "schema_name": schema_name, "table_name": table_name, "type": pipeline.type, "enabled": True, "config": pipeline.config, }) header = Sender( HAWK_CREDS, url, method.lower(), content=body, content_type=content_type, ).request_header response = requests.request( method, url, data=body, headers={ "Authorization": header, "Content-Type": content_type }, ) response.raise_for_status() return response.json()
def import_api_results(endpoint): # Avoid calling DH fake_it = settings.FAKE_METADATA if fake_it: # TODO: fake all metadata, not just a part of it # currently only a few countries are made available file_path = os.path.join(settings.BASE_DIR, f"static/{endpoint}.json") return json.loads(open(file_path).read()) base_url = URLObject(settings.DH_METADATA_URL) # v4 endpoints do not have trailing forward slash meta_url = base_url.relative(f"./{endpoint}") credentials = settings.HAWK_CREDENTIALS[settings.DATAHUB_HAWK_ID] sender = Sender( credentials, meta_url, 'GET', content=None, content_type=None, always_hash_content=False, ) response = requests.get(meta_url, verify=not settings.DEBUG, headers={'Authorization': sender.request_header}) if response.ok: return response.json() return None
def _get_hawk_sender(url, method, content_type, content): return Sender( {"id": settings.LITE_HAWK_ID, "key": settings.LITE_HAWK_KEY, "algorithm": "sha256"}, url, method, content_type=content_type, content=content, seen_nonce=_seen_nonce, )
def _get_hawk_sender(url, method, content_type, content): return Sender( {"id": "internal-frontend", "key": env("LITE_INTERNAL_HAWK_KEY"), "algorithm": "sha256"}, url, method, content_type=content_type, content=content, seen_nonce=_seen_nonce, )
def get_hawk_sender(method, url, data, credentials): content = serialize(data) if data else data credentials = settings.HAWK_CREDENTIALS.get(credentials) return Sender(credentials, url, method, content=content, content_type="application/json", seen_nonce=_seen_nonce)
def _sender(self, method='GET', content_type='', url=None, credentials=None, content='', **kw): if not url: url = self.url if not credentials: credentials = self.credentials return Sender(credentials, url, method, content=content, content_type=content_type, **kw)
def request_headers(self, method, url): credentials = { 'id': self.user_id, 'key': self.api_key, 'algorithm': 'sha256' } sender = Sender(credentials, url, method, always_hash_content=False, ext=self.org_id) return {'Authorization': sender.request_header}
def main(): apikey = raw_input("Enter API Key: ") userid = raw_input("Enter userid: ") nDays = raw_input("Enter number of Days to review: ") #Sanitize inputs if nDays == None: nDays = 1 else: nDays = int(nDays) #These will be consistent for all requests for hawk credentials = {'id': userid, 'key': apikey, 'algorithm': 'sha256'} date_N_days_ago = datetime.now() - timedelta(days=nDays) #these are for the from/until parameters in the api call fromDate = (str(date_N_days_ago).split(' ')[0]) ORGANIZATION_ID = raw_input("Enter Org ID: ") token = None status, severity = 'active', 1 allAlerts = [] while True: if token == None: URL = 'https://api.threatstack.com/v2/alerts?status=%s&from=%s&severity%s' % ( status, fromDate, severity) else: URL = 'https://api.threatstack.com/v2/alerts?status=%s&from=%s&severity%s&token=%s' % ( status, fromDate, severity, token) sender = Sender(credentials, URL, "GET", always_hash_content=False, ext=ORGANIZATION_ID) response = requests.get( URL, headers={'Authorization': sender.request_header}) vals = response.json() token = vals.get('token') alerts = vals.get('alerts') #have our alerts, now do stuff for alert in alerts: if alert != None: allAlerts.append(alert) if vals['token'] == None: break #You have all the alerts, do whatever next, example below is print out the first 10 print(allAlerts[:10])
def get_sender(): return Sender( credentials={ 'id': flask_app.config['access_control']['hawk_client_id'], 'key': flask_app.config['access_control']['hawk_client_key'], 'algorithm': 'sha256', }, url=url, method='POST', content=json_data, content_type='application/json', )
def get_mohawk_sender(url): sender = Sender( credentials={ 'id': 'iss1', 'key': 'secret1', 'algorithm': 'sha256' }, url=url, method='GET', content='', content_type='', ) return sender
def test_hmac_auth(self): credentials = { 'id': self.access_key, 'key': self.secret_key, 'algorithm': 'sha256' } sender = Sender( url='http://localhost/alerts', method='GET', content='', content_type='application/json', credentials=credentials ) headers = { 'Authorization': sender.request_header, 'Content-Type': 'application/json' } response = self.client.get('/alerts', headers=headers) self.assertEqual(response.status_code, 200) data = json.loads(response.data.decode('utf-8')) sender = Sender( url='http://localhost/alert', method='POST', content=json.dumps(self.alert), content_type='application/json', credentials=credentials ) headers = { 'Authorization': sender.request_header, 'Content-Type': 'application/json' } response = self.client.post('/alert', data=json.dumps(self.alert), content_type='application/json', headers=headers) self.assertEqual(response.status_code, 201) data = json.loads(response.data.decode('utf-8')) self.assertEqual(data['alert']['event'], 'Foo')
def _get_hawk_sender(url, method, content_type, content): return Sender( { "id": "exporter-frontend", "key": env("LITE_EXPORTER_HAWK_KEY"), "algorithm": "sha256" }, url, method, content_type=content_type, content=content, seen_nonce=_seen_nonce, )
def send(self, value): """ :param json_data: A dict OF DATA TO SEND :return: URL OF WHERE DATA WAS STORED, AND etl.id OF RECORD """ content = json.dumps(value) # Hawk Sender WILL DO THE WORK OF SIGNINGs sender = Sender( self.hawk, self.url, b'POST', content=content, content_type=CONTENT_TYPE ) # STANDARD POST response = requests.post( url=self.url, data=content, headers={ 'Authorization': sender.request_header, 'Content-Type': CONTENT_TYPE } ) if response.status_code != 200: raise Exception(response.content) # SERVER SIGNED THE RESPONSE. VERIFY IT sender.accept_response( response.headers['Server-Authorization'], content=response.content, content_type=response.headers['Content-Type'] ) about = json.loads(response.content) return about['link'], about['etl']['id']
def get_signature_headers(self, url, body, method, content_type): sender = Sender( { 'id': self.sender_id, 'key': self.secret, 'algorithm': self.algorithm }, get_path(url), method, content=get_content(body), content_type=get_content_type(content_type), ) return {self.header_name: sender.request_header}
def test_successful_authentication(self): sender = Sender( credentials={ 'id': self.client_id, 'key': self.client_key, 'algorithm': 'sha256' }, url='http://localhost:80/test/', method='GET', content='', content_type='', ) with self.app.test_client() as c: response = c.get('/test/', headers={'Authorization': sender.request_header}) assert response.status_code == 200 assert response.get_data() == b'OK' # check if accepted response doesn't throw exception sender.accept_response( response.headers.get('Server-Authorization'), content=response.get_data(), content_type=response.mimetype, )
def get(self): sender = Sender( { 'id': self.secret_id, 'key': self.secret, 'algorithm': self.algorithm, }, self.target_url, 'GET', '', '') # Mohawk provides this authorization header, should start with Hawk auth = {'authorization': sender.request_header} response = self.session.get(self.target_url, timeout=5, headers=auth) response.raise_for_status() response_json = response.json() if not self.fields: return response_json return {key: response_json[key] for key in self.fields}