def _hawk_api_request(
url: str,
credentials: dict,
results_key: Optional[str],
next_key: Optional[str],
validate_response: Optional[bool] = True,
force_http: Optional[bool] = False,
):
sender = Sender(
credentials,
# Currently data workspace denies hawk requests signed with https urls.
# Once fixed the protocol replacement can be removed.
url.replace('https', 'http') if force_http else url,
"get",
content="",
content_type="",
always_hash_content=True,
)
logger.info(f"Fetching page {url}")
response = requests.get(
url,
headers={
"Authorization": sender.request_header,
"Content-Type": ""
},
timeout=300,
)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
logger.warning(f"Request failed: {response.text}")
raise
if validate_response:
try:
sender.accept_response(
response.headers["Server-Authorization"],
content=response.content,
content_type=response.headers["Content-Type"],
)
except HawkFail as e:
logger.error(f"HAWK Authentication failed {str(e)}")
raise
response_json = response.json()
if (next_key and next_key not in response_json) or (
results_key and results_key not in response_json):
raise ValueError("Unexpected response structure")
return response_json
def handle(self, *args, **options):
hawk_log = logging.getLogger('mohawk')
hawk_log.setLevel(logging.DEBUG)
hawk_log.addHandler(logging.StreamHandler())
try:
import requests
except ImportError:
raise CommandError('To use this command you first need to '
'install the requests module')
url = options['url']
if not url:
raise CommandError('Specify a URL to load with --url')
qs = options['d'] or ''
request_content_type = ('application/x-www-form-urlencoded'
if qs else 'text/plain')
method = options['X']
credentials = lookup_credentials(options['creds'])
sender = Sender(credentials,
url,
method.upper(),
content=qs,
content_type=request_content_type)
headers = {
'Authorization': sender.request_header,
'Content-Type': request_content_type
}
do_request = getattr(requests, method.lower())
res = do_request(url, data=qs, headers=headers)
print '{method} -d {qs} {url}'.format(method=method.upper(),
qs=qs or 'None',
url=url)
print res.text
# Verify we're talking to our trusted server.
print res.headers
auth_hdr = res.headers.get('Server-Authorization', None)
if auth_hdr:
sender.accept_response(auth_hdr,
content=res.text,
content_type=res.headers['Content-Type'])
print '<response was Hawk verified>'
else:
print '** NO Server-Authorization header **'
print '<response was NOT Hawk verified>'
def handle(self, *args, **options):
# Configure the mohawk lib for debug logging so we can see inputs to
# the signature functions and other useful stuff.
hawk_log = logging.getLogger('mohawk')
hawk_log.setLevel(logging.DEBUG)
hawk_log.addHandler(logging.StreamHandler())
url = options['url']
if not url:
raise CommandError('Specify a URL to load with --url')
creds_key = options['creds']
if not creds_key:
raise CommandError('Specify ID for Hawk credentials with --creds')
method = options['X']
qs = options['d'] or ''
request_content_type = ('application/x-www-form-urlencoded'
if qs else 'text/plain')
credentials = lookup_credentials(creds_key)
sender = Sender(credentials,
url,
method.upper(),
content=qs,
content_type=request_content_type)
headers = {
'Authorization': sender.request_header,
'Content-Type': request_content_type
}
res = request(url, method.lower(), data=qs, headers=headers)
self.stdout.write('{method} -d {qs} {url}'.format(
method=method.upper(), qs=qs or 'None', url=url))
self.stdout.write(res.text)
# Verify we're talking to our trusted server.
self.stdout.write(str(res.headers))
auth_hdr = res.headers.get('Server-Authorization', None)
if auth_hdr:
sender.accept_response(auth_hdr,
content=res.text,
content_type=res.headers['Content-Type'])
self.stdout.write('<response was Hawk verified>')
else:
self.stdout.write('** NO Server-Authorization header **')
self.stdout.write('<response was NOT Hawk verified>')
def handle(self, *args, **options):
hawk_log = logging.getLogger('mohawk')
hawk_log.setLevel(logging.DEBUG)
hawk_log.addHandler(logging.StreamHandler())
try:
import requests
except ImportError:
raise CommandError('To use this command you first need to '
'install the requests module')
url = options['url']
if not url:
raise CommandError('Specify a URL to load with --url')
qs = options['d'] or ''
request_content_type = ('application/x-www-form-urlencoded'
if qs else 'text/plain')
method = options['X']
credentials = lookup_credentials(options['creds'])
sender = Sender(credentials,
url, method.upper(),
content=qs,
content_type=request_content_type)
headers = {'Authorization': sender.request_header,
'Content-Type': request_content_type}
do_request = getattr(requests, method.lower())
res = do_request(url, data=qs, headers=headers)
print '{method} -d {qs} {url}'.format(method=method.upper(),
qs=qs or 'None',
url=url)
print res.text
# Verify we're talking to our trusted server.
print res.headers
auth_hdr = res.headers.get('Server-Authorization', None)
if auth_hdr:
sender.accept_response(auth_hdr,
content=res.text,
content_type=res.headers['Content-Type'])
print '<response was Hawk verified>'
else:
print '** NO Server-Authorization header **'
print '<response was NOT Hawk verified>'
def handle(self, *args, **options):
# Configure the mohawk lib for debug logging so we can see inputs to
# the signature functions and other useful stuff.
hawk_log = logging.getLogger('mohawk')
hawk_log.setLevel(logging.DEBUG)
hawk_log.addHandler(logging.StreamHandler())
url = options['url']
if not url:
raise CommandError('Specify a URL to load with --url')
creds_key = options['creds']
if not creds_key:
raise CommandError('Specify ID for Hawk credentials with --creds')
method = options['X']
qs = options['d'] or ''
request_content_type = ('application/x-www-form-urlencoded'
if qs else 'text/plain')
credentials = lookup_credentials(creds_key)
sender = Sender(credentials,
url, method.upper(),
content=qs,
content_type=request_content_type)
headers = {'Authorization': sender.request_header,
'Content-Type': request_content_type}
res = request(url, method.lower(), data=qs, headers=headers)
self.stdout.write('{method} -d {qs} {url}'.format(method=method.upper(),
qs=qs or 'None',
url=url))
self.stdout.write(res.text)
# Verify we're talking to our trusted server.
self.stdout.write(str(res.headers))
auth_hdr = res.headers.get('Server-Authorization', None)
if auth_hdr:
sender.accept_response(auth_hdr,
content=res.text,
content_type=res.headers['Content-Type'])
self.stdout.write('<response was Hawk verified>')
else:
self.stdout.write('** NO Server-Authorization header **')
self.stdout.write('<response was NOT Hawk verified>')
def search_with_activitystream(query):
""" Searches ActivityStream services with given Elasticsearch query.
Note that this must be at root level in SearchView class to
enable it to be mocked in tests.
"""
request = requests.Request(
method="GET",
url=settings.ACTIVITY_STREAM_API_URL,
data=query).prepare()
auth = Sender(
{
'id': settings.ACTIVITY_STREAM_API_ACCESS_KEY,
'key': settings.ACTIVITY_STREAM_API_SECRET_KEY,
'algorithm': 'sha256'
},
settings.ACTIVITY_STREAM_API_URL,
"GET",
content=query,
content_type='application/json',
).request_header
request.headers.update({
'X-Forwarded-Proto': 'https',
'Authorization': auth,
'Content-Type': 'application/json'
})
return requests.Session().send(request)
def get_dataflow_dag_status(execution_date):
config = settings.DATAFLOW_API_CONFIG
url = (
f'{config["DATAFLOW_BASE_URL"]}/api/experimental/'
f'dags/{config["DATAFLOW_S3_IMPORT_DAG"]}/dag_runs/{execution_date.split("+")[0]}'
)
hawk_creds = {
'id': config['DATAFLOW_HAWK_ID'],
'key': config['DATAFLOW_HAWK_KEY'],
'algorithm': 'sha256',
}
header = Sender(
hawk_creds,
url,
'get',
content='',
content_type='',
).request_header
response = requests.get(
url,
headers={
'Authorization': header,
'Content-Type': ''
},
)
response.raise_for_status()
return response.json()
def _sign_content(sign_payload):
# if the list is empty, return an empty signature list
if len(sign_payload) == 0:
return []
content_type = 'application/json'
sender = Sender(
{
'id': env.config.SIGNING_HAWK_ID,
'key': env.config.SIGNING_HAWK_KEY,
'algorithm': 'sha256'
}, # credentials
env.config.SIGNING_SERVICE_URL, # url
'POST', # method
content=json.dumps(sign_payload),
content_type=content_type)
try:
r = requests.post(env.config.SIGNING_SERVICE_URL,
json=sign_payload,
headers={
'Content-Type': content_type,
'Authorization': sender.request_header
})
if r.status_code != 201:
msg = "signing server returned an error code: %s" % (r.status_code)
raise Exception(msg)
except Exception as e:
msg = "Error when signing: %s" % e
env.log(msg)
raise
else:
return r.json()
def _hawk_api_request(
url: str,
method: str,
query: dict,
credentials: dict,
expected_response_structure: str = None,
):
body = json.dumps(query)
header = Sender(
credentials, url, method.lower(), content_type="application/json", content=body
).request_header
response = requests.request(
method,
url,
data=body,
headers={"Authorization": header, "Content-Type": "application/json"},
)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
logger.warning(f"Request failed: {response.text}")
raise
response_json = response.json()
if expected_response_structure and expected_response_structure not in response_json:
raise ValueError("Unexpected response structure")
return response_json
def hawk_api_request(
url,
method,
credentials,
body=None,
):
if body:
body = json.dumps(body)
auth_header = Sender(
credentials,
url,
method.lower(),
content_type="application/json" if body else None,
content=body,
).request_header
headers = {"Authorization": auth_header}
if body:
headers["Content-Type"] = "application/json"
response = requests.request(
method,
url,
data=body,
headers=headers,
)
response.raise_for_status()
response_json = response.json()
return response_json
def start():
entry = {
"userId": user_id,
"start": now_string(),
"end": None,
"timezoneName": "CET",
"timezone": "+0100",
"type": "work"
}
url = 'https://app.absence.io/api/v2/timespans/create'
method = 'POST'
content = json.dumps(entry)
content_type = 'application/json'
sender = Sender({
'id': user_id,
'key': key,
'algorithm': 'sha256'
},
url,
method,
content=content,
content_type=content_type)
response = requests.post(url,
data=content,
headers={
'Authorization': sender.request_header,
'Content-Type': content_type
})
return response
def request(self, method, path, **kwargs):
if not settings.DATAHUB_URL:
raise DataHubException("DATAHUB_URL is not set")
url = f"{settings.DATAHUB_URL}{path}"
credentials = {
"id": settings.DATAHUB_HAWK_ID,
"key": settings.DATAHUB_HAWK_KEY,
"algorithm": "sha256",
}
sender = Sender(
credentials,
url,
method,
content=json.dumps(kwargs),
content_type="application/json",
always_hash_content=False,
)
headers = {"Authorization": sender.request_header}
response = getattr(requests, method)(url,
verify=not settings.DEBUG,
headers=headers,
json=kwargs)
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
raise APIHttpException(e)
return response.json()
def hawk_request(method, url, body):
hawk_id = settings.ACTIVITY_STREAM_HAWK_CREDENTIALS_ID
hawk_key = settings.ACTIVITY_STREAM_HAWK_CREDENTIALS_KEY
if not hawk_id or not hawk_key:
raise HawkException("Hawk id or key not configured")
content_type = 'application/json'
header = Sender(
{
'id': hawk_id,
'key': hawk_key,
'algorithm': 'sha256'
},
url,
method,
content=body,
content_type=content_type,
).request_header
response = requests.request(
method,
url,
data=body,
headers={
'Authorization': header,
'Content-Type': content_type
},
)
return response.status_code, response.content
def trigger_dataflow_dag(conf, dag, dag_run_id):
config = settings.DATAFLOW_API_CONFIG
trigger_url = f'{config["DATAFLOW_BASE_URL"]}/api/experimental/' f"dags/{dag}/dag_runs"
hawk_creds = {
"id": config["DATAFLOW_HAWK_ID"],
"key": config["DATAFLOW_HAWK_KEY"],
"algorithm": "sha256",
}
method = "POST"
content_type = "application/json"
body = json.dumps(
{
"run_id": dag_run_id,
"replace_microseconds": "false",
"conf": conf,
}
)
header = Sender(
hawk_creds,
trigger_url,
method.lower(),
content=body,
content_type=content_type,
).request_header
response = requests.request(
method,
trigger_url,
data=body,
headers={"Authorization": header, "Content-Type": content_type},
)
response.raise_for_status()
return response.json()
def _get_hawk_response(client_id,
secret,
method='GET',
content='',
content_type='application/json'):
auth = {'id': client_id, 'key': secret, 'algorithm': 'sha256'}
url = 'http://testserver/'
sender = Sender(auth,
url,
method,
content=content,
content_type='application/json')
do_request = getattr(factory, method.lower())
request = do_request(
url,
data=content,
content_type='application/json',
# factory.get doesn't set the CONTENT_TYPE header
# I'm setting it manually here for simplicity
CONTENT_TYPE='application/json',
HTTP_AUTHORIZATION=sender.request_header)
view = AuthenticatedView.as_view()
return view(request)
def search_with_activitystream(query):
""" Searches ActivityStream services with given Elasticsearch query.
Note that this must be at root level in SearchView class to
enable it to be mocked in tests.
"""
request = requests.Request(
method="GET",
url=settings.ACTIVITY_STREAM_API_URL,
data=query).prepare()
auth = Sender(
{
'id': settings.ACTIVITY_STREAM_API_ACCESS_KEY,
'key': settings.ACTIVITY_STREAM_API_SECRET_KEY,
'algorithm': 'sha256'
},
settings.ACTIVITY_STREAM_API_URL,
"GET",
content=query,
content_type='application/json',
).request_header
# Note that the X-Forwarded-* items are overridden by Gov PaaS values
# in production, and thus the value of ACTIVITY_STREAM_API_IP_WHITELIST
# in production is irrelivant. It is included here to allow the app to
# run locally or outside of Gov PaaS.
request.headers.update({
'X-Forwarded-Proto': 'https',
'X-Forwarded-For': settings.ACTIVITY_STREAM_API_IP_WHITELIST,
'Authorization': auth,
'Content-Type': 'application/json'
})
return requests.Session().send(request)
def api_check():
data = {"status": HealthStatus.FAIL, "duration": None}
# trailing / is important here
url = f"{settings.MARKET_ACCESS_API_URI}check/"
sender = Sender(
settings.MARKET_ACCESS_API_HAWK_CREDS,
url,
"GET",
content="",
content_type="text/plain",
always_hash_content=False,
)
try:
response = requests.get(
url,
verify=not settings.DEBUG,
headers={
"Authorization": sender.request_header,
"Content-Type": "text/plain",
},
)
response.raise_for_status()
response_data = response.json()
except Exception:
pass
else:
data["status"] = response_data["status"]
data["duration"] = response_data.get("duration")
return data
def save_pipeline_to_dataflow(pipeline, method):
url = f"{API_URL}/dag/{pipeline.dag_id}"
content_type = "application/json"
schema_name, table_name = pipeline.table_name.split(".")
body = json.dumps({
"schedule": "@daily",
"schema_name": schema_name,
"table_name": table_name,
"type": pipeline.type,
"enabled": True,
"config": pipeline.config,
})
header = Sender(
HAWK_CREDS,
url,
method.lower(),
content=body,
content_type=content_type,
).request_header
response = requests.request(
method,
url,
data=body,
headers={
"Authorization": header,
"Content-Type": content_type
},
)
response.raise_for_status()
return response.json()
def import_api_results(endpoint):
# Avoid calling DH
fake_it = settings.FAKE_METADATA
if fake_it:
# TODO: fake all metadata, not just a part of it
# currently only a few countries are made available
file_path = os.path.join(settings.BASE_DIR, f"static/{endpoint}.json")
return json.loads(open(file_path).read())
base_url = URLObject(settings.DH_METADATA_URL)
# v4 endpoints do not have trailing forward slash
meta_url = base_url.relative(f"./{endpoint}")
credentials = settings.HAWK_CREDENTIALS[settings.DATAHUB_HAWK_ID]
sender = Sender(
credentials,
meta_url,
'GET',
content=None,
content_type=None,
always_hash_content=False,
)
response = requests.get(meta_url,
verify=not settings.DEBUG,
headers={'Authorization': sender.request_header})
if response.ok:
return response.json()
return None
def _get_hawk_sender(url, method, content_type, content):
return Sender(
{"id": settings.LITE_HAWK_ID, "key": settings.LITE_HAWK_KEY, "algorithm": "sha256"},
url,
method,
content_type=content_type,
content=content,
seen_nonce=_seen_nonce,
)
def _get_hawk_sender(url, method, content_type, content):
return Sender(
{"id": "internal-frontend", "key": env("LITE_INTERNAL_HAWK_KEY"), "algorithm": "sha256"},
url,
method,
content_type=content_type,
content=content,
seen_nonce=_seen_nonce,
)
def get_hawk_sender(method, url, data, credentials):
content = serialize(data) if data else data
credentials = settings.HAWK_CREDENTIALS.get(credentials)
return Sender(credentials,
url,
method,
content=content,
content_type="application/json",
seen_nonce=_seen_nonce)
def _sender(self, method='GET', content_type='', url=None,
credentials=None, content='', **kw):
if not url:
url = self.url
if not credentials:
credentials = self.credentials
return Sender(credentials,
url, method,
content=content,
content_type=content_type,
**kw)
def request_headers(self, method, url):
credentials = {
'id': self.user_id,
'key': self.api_key,
'algorithm': 'sha256'
}
sender = Sender(credentials,
url,
method,
always_hash_content=False,
ext=self.org_id)
return {'Authorization': sender.request_header}
def main():
apikey = raw_input("Enter API Key: ")
userid = raw_input("Enter userid: ")
nDays = raw_input("Enter number of Days to review: ")
#Sanitize inputs
if nDays == None:
nDays = 1
else:
nDays = int(nDays)
#These will be consistent for all requests for hawk
credentials = {'id': userid, 'key': apikey, 'algorithm': 'sha256'}
date_N_days_ago = datetime.now() - timedelta(days=nDays)
#these are for the from/until parameters in the api call
fromDate = (str(date_N_days_ago).split(' ')[0])
ORGANIZATION_ID = raw_input("Enter Org ID: ")
token = None
status, severity = 'active', 1
allAlerts = []
while True:
if token == None:
URL = 'https://api.threatstack.com/v2/alerts?status=%s&from=%s&severity%s' % (
status, fromDate, severity)
else:
URL = 'https://api.threatstack.com/v2/alerts?status=%s&from=%s&severity%s&token=%s' % (
status, fromDate, severity, token)
sender = Sender(credentials,
URL,
"GET",
always_hash_content=False,
ext=ORGANIZATION_ID)
response = requests.get(
URL, headers={'Authorization': sender.request_header})
vals = response.json()
token = vals.get('token')
alerts = vals.get('alerts')
#have our alerts, now do stuff
for alert in alerts:
if alert != None:
allAlerts.append(alert)
if vals['token'] == None:
break
#You have all the alerts, do whatever next, example below is print out the first 10
print(allAlerts[:10])
def get_sender():
return Sender(
credentials={
'id': flask_app.config['access_control']['hawk_client_id'],
'key':
flask_app.config['access_control']['hawk_client_key'],
'algorithm': 'sha256',
},
url=url,
method='POST',
content=json_data,
content_type='application/json',
)
def get_mohawk_sender(url):
sender = Sender(
credentials={
'id': 'iss1',
'key': 'secret1',
'algorithm': 'sha256'
},
url=url,
method='GET',
content='',
content_type='',
)
return sender
def test_hmac_auth(self):
credentials = {
'id': self.access_key,
'key': self.secret_key,
'algorithm': 'sha256'
}
sender = Sender(
url='http://localhost/alerts',
method='GET',
content='',
content_type='application/json',
credentials=credentials
)
headers = {
'Authorization': sender.request_header,
'Content-Type': 'application/json'
}
response = self.client.get('/alerts', headers=headers)
self.assertEqual(response.status_code, 200)
data = json.loads(response.data.decode('utf-8'))
sender = Sender(
url='http://localhost/alert',
method='POST',
content=json.dumps(self.alert),
content_type='application/json',
credentials=credentials
)
headers = {
'Authorization': sender.request_header,
'Content-Type': 'application/json'
}
response = self.client.post('/alert', data=json.dumps(self.alert),
content_type='application/json', headers=headers)
self.assertEqual(response.status_code, 201)
data = json.loads(response.data.decode('utf-8'))
self.assertEqual(data['alert']['event'], 'Foo')
def _get_hawk_sender(url, method, content_type, content):
return Sender(
{
"id": "exporter-frontend",
"key": env("LITE_EXPORTER_HAWK_KEY"),
"algorithm": "sha256"
},
url,
method,
content_type=content_type,
content=content,
seen_nonce=_seen_nonce,
)
def send(self, value):
"""
:param json_data: A dict OF DATA TO SEND
:return: URL OF WHERE DATA WAS STORED, AND etl.id OF RECORD
"""
content = json.dumps(value)
# Hawk Sender WILL DO THE WORK OF SIGNINGs
sender = Sender(
self.hawk,
self.url,
b'POST',
content=content,
content_type=CONTENT_TYPE
)
# STANDARD POST
response = requests.post(
url=self.url,
data=content,
headers={
'Authorization': sender.request_header,
'Content-Type': CONTENT_TYPE
}
)
if response.status_code != 200:
raise Exception(response.content)
# SERVER SIGNED THE RESPONSE. VERIFY IT
sender.accept_response(
response.headers['Server-Authorization'],
content=response.content,
content_type=response.headers['Content-Type']
)
about = json.loads(response.content)
return about['link'], about['etl']['id']
def get_signature_headers(self, url, body, method, content_type):
sender = Sender(
{
'id': self.sender_id,
'key': self.secret,
'algorithm': self.algorithm
},
get_path(url),
method,
content=get_content(body),
content_type=get_content_type(content_type),
)
return {self.header_name: sender.request_header}
def test_successful_authentication(self):
sender = Sender(
credentials={
'id': self.client_id,
'key': self.client_key,
'algorithm': 'sha256'
},
url='http://localhost:80/test/',
method='GET',
content='',
content_type='',
)
with self.app.test_client() as c:
response = c.get('/test/',
headers={'Authorization': sender.request_header})
assert response.status_code == 200
assert response.get_data() == b'OK'
# check if accepted response doesn't throw exception
sender.accept_response(
response.headers.get('Server-Authorization'),
content=response.get_data(),
content_type=response.mimetype,
)
def get(self):
sender = Sender(
{
'id': self.secret_id,
'key': self.secret,
'algorithm': self.algorithm,
}, self.target_url, 'GET', '', '')
# Mohawk provides this authorization header, should start with Hawk
auth = {'authorization': sender.request_header}
response = self.session.get(self.target_url, timeout=5, headers=auth)
response.raise_for_status()
response_json = response.json()
if not self.fields:
return response_json
return {key: response_json[key] for key in self.fields}
