def get_hawk_headers(
url,
client_id,
client_key,
content='',
content_type='',
https=False,
method='GET',
):
if https is False:
# todo: ask data workspace to fix the https/http x-forwarded-for
url = url.replace('https', 'http')
credentials = {'id': client_id, 'key': client_key, 'algorithm': 'sha256'}
sender = mohawk.Sender(
credentials=credentials,
url=url,
method=method,
content=content,
content_type=content_type,
)
headers = {
'Authorization': sender.request_header,
'Content-Type': content_type
}
return headers
def request(self, method, path, params=None, json_=unset, content_type=''):
"""Make a request with a specified HTTP method."""
params = urlencode(params) if params else ''
url = join_truthy_strings(f'http://testserver{path}', params, sep='?')
if json_ is not self.unset:
content_type = 'application/json'
body = json.dumps(json_, cls=DjangoJSONEncoder).encode('utf-8')
else:
body = b''
sender = mohawk.Sender(
self.credentials,
url,
method,
content=body,
content_type=content_type,
)
return self.api_client.generic(
method,
url,
HTTP_AUTHORIZATION=sender.request_header,
HTTP_X_FORWARDED_FOR=self.http_x_forwarded_for,
data=body,
content_type=content_type,
)
def make_request(self, method, url, data=''):
"""
Make an HAWK authenticated request on remote server
"""
request = getattr(requests, method)
if not request:
raise Exception('Invalid method {}'.format(method))
# Build HAWK token
url = self.api_url + url
hawk = mohawk.Sender(self.credentials,
url,
method,
content=data,
content_type='application/json')
# Send request
headers = {
'Authorization': hawk.request_header,
'Content-Type': 'application/json',
}
response = request(url, data=data, headers=headers, verify=False)
if not response.ok:
raise Exception('Invalid response from {} {} : {}'.format(
method, url, response.content))
return response.json()
def sign(self, url, method, *args, **kw):
headers = kw.pop('headers', {})
if self.host is not None:
headers['Host'] = self.host
data = kw.get('data')
content = ''
if data:
if not isinstance(data, str):
# XXX order?
content = json.dumps(data)
data = content
headers['Content-Type'] = 'application/json'
else:
raise NotImplementedError()
sender = mohawk.Sender(self.credentials,
url,
method,
content=content,
content_type=headers.get('Content-Type', ''),
_timestamp=self._timestamp)
headers['Authorization'] = sender.request_header
return headers, data
def test_valid_credentials(mocker):
app = Flask('dataflow-test-app')
mocker.patch.object(
api_auth_backend.config,
'AIRFLOW_API_HAWK_CREDENTIALS',
{TEST_CLIENT_ID: TEST_CLIENT_KEY},
)
credentials = {
'id': TEST_CLIENT_ID,
'key': TEST_CLIENT_KEY,
'algorithm': 'sha256'
}
sender = mohawk.Sender(
credentials=credentials,
url='http://localhost/',
method='GET',
content='',
content_type='',
)
mock_view = mock.Mock()
mock_view.return_value = {}
headers = {'Authorization': sender.request_header, 'Content-Type': ''}
with app.test_request_context('/', headers=headers):
response = api_auth_backend.requires_authentication(mock_view)()
assert response.status_code == 200
mock_view.assert_called_once()
def make_request(self, method, url, data=''):
"""
Make an HAWK authenticated request on remote server
"""
request = getattr(requests, method)
if not request:
raise Exception('Invalid method {}'.format(method))
# Build HAWK token
url = self.api_url + url
hawk = mohawk.Sender(self.credentials,
url,
method,
content=data,
content_type='application/json')
# Support dev ssl ca cert
ssl_dev_ca = os.environ.get('SSL_DEV_CA')
if ssl_dev_ca is not None:
assert os.path.isdir(ssl_dev_ca), \
'SSL_DEV_CA must be a dir with hashed dev ca certs'
# Send request, using optional dev ca
headers = {
'Authorization': hawk.request_header,
'Content-Type': 'application/json',
}
response = request(url, data=data, headers=headers, verify=ssl_dev_ca)
if not response.ok:
raise Exception('Invalid response from {} {} : {}'.format(
method, url, response.content))
return response.json()
def schedule_nagbot_message(message: str,
short_message: str,
deadline: datetime,
policies: List[dict],
uid: str = None) -> str:
'''
Instantiates a new message to be sent repeatedly by NagBot
:param message: Long description of message (ie email body)
:param short_message: Short description of message (ie email subject, IRC message)
:param deadline: Message expiry date
:param policies: Notification policies described in dict format
:param uid: Optionally specify tracking uid. A random uid will be generated if not given
:return: Tracking uid for the notification
'''
for policy in policies:
verify_policy_structure(policy)
if uid is None:
uid = generate_random_uid()
request_url = current_app.config[
'RELENG_NOTIFICATION_POLICY_URL'] + '/message/' + uid
message_body = json.dumps({
'deadline': deadline.isoformat(),
'message': message,
'shortMessage': short_message,
'policies': policies,
})
hawk = mohawk.Sender(get_current_app_credentials(),
request_url,
'put',
content=message_body,
content_type='application/json')
headers = {
'Authorization': hawk.request_header,
'Content-Type': 'application/json',
}
# Support dev ssl ca cert
ssl_dev_ca = current_app.config.get('SSL_DEV_CA')
if ssl_dev_ca is not None:
assert os.path.isdir(
ssl_dev_ca), 'SSL_DEV_CA must be a dir with hashed dev ca certs'
response = put(request_url,
headers=headers,
data=message_body,
verify=ssl_dev_ca)
response.raise_for_status()
return uid
def __call__(self, r):
r.headers['Host'] = urlparse(self.server_url).netloc
sender = mohawk.Sender(self.credentials,
r.url,
r.method,
content=r.body or '',
content_type=r.headers.get('Content-Type', ''))
r.headers['Authorization'] = sender.request_header
return r
def hawk_auth_header(key_id, secret_key, url, method, content_type, content):
return mohawk.Sender(
{
'id': key_id,
'key': secret_key,
'algorithm': 'sha256',
},
url,
method,
content_type=content_type,
content=content).request_header
def get_hawk_token(user):
sender = mohawk.Sender(credentials={
'id': user.username,
'key': user.password,
'algorithm': 'sha256',
},
url='http://falconframework.org/auth',
method='GET',
nonce='ABC123',
always_hash_content=False)
return str(sender.request_header)
def hawk_auth_header(key_id, secret_key, url, method, content, content_type):
return mohawk.Sender(
{
"id": key_id,
"key": secret_key,
"algorithm": "sha256",
},
url,
method,
content=content,
content_type=content_type,
).request_header
def get_hawk_auth_header(method, url, client_id=None, access_token=None, content_type='application/json'):
return mohawk.Sender(
credentials={
'id': client_id or settings.TASKCLUSTER_CLIENT_ID,
'key': access_token or settings.TASKCLUSTER_ACCESS_TOKEN,
'algorithm': 'sha256',
},
ext={},
url=url,
content='',
content_type='application/json',
method=method,
).request_header
def signed_request(self, credentials, method='GET', path='/', data=None):
url = 'http://localhost' + path
content = json.dumps(data)
content_type = 'application/json'
sender = mohawk.Sender(credentials, url, method, content, content_type)
return self.client.open(
method=method,
path=path,
headers={'Authorization': sender.request_header},
data=content,
content_type=content_type)
def __call__(self, r):
if self.host is not None:
r.headers['Host'] = self.host
sender = mohawk.Sender(self.credentials,
r.url,
r.method,
content=r.body or '',
content_type=r.headers.get('Content-Type', ''),
_timestamp=self._timestamp)
r.headers['Authorization'] = sender.request_header
return r
def get_taskcluster_headers(request_url, method, content,
taskcluster_client_id, taskcluster_access_token):
hawk = mohawk.Sender(
{
"id": taskcluster_client_id,
"key": taskcluster_access_token,
"algorithm": "sha256"
},
request_url,
method,
content,
content_type="application/json")
return {
"Authorization": hawk.request_header,
"Content-Type": "application/json"
}
def make_request(self, method, url, data=None):
'''
Make an HAWK authenticated request on remote server
Low level API access
'''
request = getattr(requests, method)
if not request:
raise Exception('Invalid method {}'.format(method))
# Encode optional data as json
if data is not None:
data = json.dumps(data, cls=ShipitJSONEncoder)
else:
data = ''
# Build HAWK token
url = self.api_url + url
hawk = mohawk.Sender(self.credentials,
url,
method,
content=data,
content_type='application/json')
# Support dev ssl ca cert
ssl_dev_ca = os.environ.get('SSL_DEV_CA')
if ssl_dev_ca is not None:
assert os.path.isdir(ssl_dev_ca), \
'SSL_DEV_CA must be a dir with hashed dev ca certs'
# Send request, using optional dev ca
headers = {
'Authorization': hawk.request_header,
'Content-Type': 'application/json',
}
response = request(url, data=data, headers=headers, verify=ssl_dev_ca)
if not response.ok:
if response.status_code == 404:
logger.debug('Not found response on {}'.format(url))
raise NotFound
raise Exception('Invalid response from {} {} : {}'.format(
method, url, response.content))
if not response.content:
return # avoid empty json parsing
return response.json()
def _get_taskcluster_headers(request_url, method, content,
taskcluster_client_id,
taskcluster_access_token):
hawk = mohawk.Sender(
{
'id': taskcluster_client_id,
'key': taskcluster_access_token,
'algorithm': 'sha256',
},
request_url,
method,
content,
content_type='application/json',
)
return {
'Authorization': hawk.request_header,
'Content-Type': 'application/json',
}
def _auth_sender(key_id='some-id',
secret_key='some-secret',
url=_url,
method='GET',
content='',
content_type=''):
credentials = {
'id': key_id,
'key': secret_key,
'algorithm': 'sha256',
}
return mohawk.Sender(
credentials,
url(),
method,
content=content,
content_type=content_type,
)
def hawk_request(method, url, id_, secret_key, body):
"""
Makes a :mod:`hawk` request
:param method: A valid HTTP method
:type method: str
:param url: A valid URL
:type url: str
:param id_: The ``id`` for the ``credentials`` argument to :class:`mohawk.Sender`
:type id_: str
:param secret_key:
The ``secret_key`` for the ``credentials`` argument to :class:`mohawk.Sender`
:type secret_key: str
:param body: A JSON serializable data structure to be sent as the request body
:returns: :class:`requests.Response`
"""
header = mohawk.Sender(
{
"id": id_,
"key": secret_key,
"algorithm": "sha256"
},
url,
method,
content_type="application/json",
content=body,
).request_header
response = requests.request(
method,
url,
data=body,
headers={
"Authorization": header,
"Content-Type": "application/json"
},
)
return response
def test_hawk_auth(key, expected_status, mocker):
url = 'http://localhost:8080/api/experimental/test'
sender = mohawk.Sender(
credentials={
'id': 'dataflowapi',
'key': key,
'algorithm': 'sha256'
},
url=url,
method='GET',
content='',
content_type='',
)
response = requests.get(url,
headers={
'Authorization': sender.request_header,
'Content-Type': ''
})
assert response.status_code == expected_status
def auth_sender(key_id=settings.ACTIVITY_STREAM_ACCESS_KEY_ID,
secret_key=settings.ACTIVITY_STREAM_SECRET_ACCESS_KEY,
url=URL,
method='GET',
content='',
content_type=''):
credentials = {
'id': key_id,
'key': secret_key,
'algorithm': 'sha256',
}
return mohawk.Sender(
credentials,
url,
method,
content=content,
content_type=content_type,
)
def hawk_auth_sender(
key_id="some-id",
secret_key="some-secret",
url=test_url,
method="GET",
content="",
content_type="",
):
credentials = {
"id": key_id,
"key": secret_key,
"algorithm": "sha256",
}
return mohawk.Sender(
credentials,
url,
method,
content=content,
content_type=content_type,
)
def _auth_sender(
key_id="test-id-without-scope",
secret_key="test-key-without-scope",
url=None,
method="GET",
content="",
content_type="",
):
credentials = {
"id": key_id,
"key": secret_key,
"algorithm": "sha256",
}
return mohawk.Sender(
credentials,
url,
method,
content=content,
content_type=content_type,
)
def _auth_sender(
key_id='test-id-without-scope',
secret_key='test-key-without-scope',
url=_url,
method='GET',
content='',
content_type='',
):
credentials = {
'id': key_id,
'key': secret_key,
'algorithm': 'sha256',
}
return mohawk.Sender(
credentials,
url(),
method,
content=content,
content_type=content_type,
)
def sender(
url,
key_id='some-id',
secret_key='some-secret',
method='GET',
content='',
content_type='',
):
"""
Return hawk sender for the given URL.
"""
return mohawk.Sender(
{
'id': key_id,
'key': secret_key,
'algorithm': 'sha256',
},
url,
method,
content=content,
content_type=content_type,
)
async def tryRequest(retryFor):
# Construct header
if self._hasCredentials():
sender = mohawk.Sender(
credentials={
'id': self.options['credentials']['clientId'],
'key': self.options['credentials']['accessToken'],
'algorithm': 'sha256',
},
ext=hawkExt if hawkExt else {},
url=url,
content=payload if payload else '',
content_type='application/json' if payload else '',
method=method,
)
headers = {'Authorization': sender.request_header}
else:
log.debug('Not using hawk!')
headers = {}
if payload:
# Set header for JSON if payload is given, note that we serialize
# outside this loop.
headers['Content-Type'] = 'application/json'
try:
response = await asyncutils.makeSingleHttpRequest(
method, url, payload, headers, session=self.session)
except aiohttp.ClientError as rerr:
return retryFor(
exceptions.TaskclusterConnectionError(
"Failed to establish connection", superExc=rerr))
status = response.status
if status == 204:
return None
# Catch retryable errors and go to the beginning of the loop
# to do the retry
if 500 <= status and status < 600:
try:
response.raise_for_status()
except Exception as exc:
return retryFor(exc)
# Throw errors for non-retryable errors
if status < 200 or status >= 300:
# Parse messages from errors
data = {}
try:
data = await response.json()
except Exception:
pass # Ignore JSON errors in error messages
# Find error message
message = "Unknown Server Error"
if isinstance(data, dict) and 'message' in data:
message = data['message']
else:
if status == 401:
message = "Authentication Error"
elif status == 500:
message = "Internal Server Error"
else:
message = "Unknown Server Error %s\n%s" % (
str(status), str(data)[:1024])
# Raise TaskclusterAuthFailure if this is an auth issue
if status == 401:
raise exceptions.TaskclusterAuthFailure(message,
status_code=status,
body=data,
superExc=None)
# Raise TaskclusterRestFailure for all other issues
raise exceptions.TaskclusterRestFailure(message,
status_code=status,
body=data,
superExc=None)
# Try to load JSON
try:
await response.release()
return await response.json()
except (ValueError, aiohttp.client_exceptions.ContentTypeError):
return {"response": response}
async def _makeHttpRequest(self, method, route, payload):
""" Make an HTTP Request for the API endpoint. This method wraps
the logic about doing failure retry and passes off the actual work
of doing an HTTP request to another method."""
baseUrl = self.options['baseUrl']
# urljoin ignores the last param of the baseUrl if the base url doesn't end
# in /. I wonder if it's better to just do something basic like baseUrl +
# route instead
if not baseUrl.endswith('/'):
baseUrl += '/'
url = urllib.parse.urljoin(baseUrl, route.lstrip('/'))
log.debug('Full URL used is: %s', url)
hawkExt = self.makeHawkExt()
# Serialize payload if given
if payload is not None:
payload = utils.dumpJson(payload)
# Do a loop of retries
retry = -1 # we plus first in the loop, and attempt 1 is retry 0
retries = self.options['maxRetries']
while retry < retries:
retry += 1
# if this isn't the first retry then we sleep
if retry > 0:
snooze = float(retry * retry) / 10.0
log.info('Sleeping %0.2f seconds for exponential backoff', snooze)
await asyncio.sleep(utils.calculateSleepTime(retry))
# Construct header
if self._hasCredentials():
sender = mohawk.Sender(
credentials={
'id': self.options['credentials']['clientId'],
'key': self.options['credentials']['accessToken'],
'algorithm': 'sha256',
},
ext=hawkExt if hawkExt else {},
url=url,
content=payload if payload else '',
content_type='application/json' if payload else '',
method=method,
)
headers = {'Authorization': sender.request_header}
else:
log.debug('Not using hawk!')
headers = {}
if payload:
# Set header for JSON if payload is given, note that we serialize
# outside this loop.
headers['Content-Type'] = 'application/json'
log.debug('Making attempt %d', retry)
try:
response = await asyncutils.makeSingleHttpRequest(
method, url, payload, headers, session=self.session
)
except aiohttp.ClientError as rerr:
if retry < retries:
log.warn('Retrying because of: %s' % rerr)
continue
# raise a connection exception
raise exceptions.TaskclusterConnectionError(
"Failed to establish connection",
superExc=rerr
)
status = response.status
if status == 204:
return None
# Catch retryable errors and go to the beginning of the loop
# to do the retry
if 500 <= status and status < 600 and retry < retries:
log.warn('Retrying because of a %s status code' % status)
continue
# Throw errors for non-retryable errors
if status < 200 or status >= 300:
# Parse messages from errors
data = {}
try:
data = await response.json()
except:
pass # Ignore JSON errors in error messages
# Find error message
message = "Unknown Server Error"
if isinstance(data, dict):
message = data.get('message')
else:
if status == 401:
message = "Authentication Error"
elif status == 500:
message = "Internal Server Error"
else:
message = "Unknown Server Error %s\n%s" % (str(status), str(data)[:1024])
# Raise TaskclusterAuthFailure if this is an auth issue
if status == 401:
raise exceptions.TaskclusterAuthFailure(
message,
status_code=status,
body=data,
superExc=None
)
# Raise TaskclusterRestFailure for all other issues
raise exceptions.TaskclusterRestFailure(
message,
status_code=status,
body=data,
superExc=None
)
# Try to load JSON
try:
await response.release()
return await response.json()
except ValueError:
return {"response": response}
# This code-path should be unreachable
assert False, "Error from last retry should have been raised!"
def post_tick_tock() -> dict:
'''
Trigger pending notifications according to their notification policies
:return: Information about notification triggered by this call in JSON format.
'''
session = current_app.db.session
current_time = datetime.now()
pending_messages = session.query(Message).all()
if not pending_messages:
raise NotFound('No pending policies to trigger.')
notifications = []
for message, is_past_deadline in determine_message_action(
pending_messages):
if is_past_deadline:
session.delete(message)
continue
policies = session.query(Policy).filter(
Policy.uid == message.uid).all()
for policy, identity_preference_url in get_identity_url_for_actionable_policies(
policies):
try:
service_credentials = {
'id': current_app.config['TASKCLUSTER_CLIENT_ID'],
'key': current_app.config['TASKCLUSTER_ACCESS_TOKEN'],
'algorithm': 'sha256',
}
hawk = mohawk.Sender(service_credentials,
identity_preference_url,
'get',
content='',
content_type='application/json')
# Support dev ssl ca cert
ssl_dev_ca = current_app.config.get('SSL_DEV_CA')
if ssl_dev_ca is not None:
assert os.path.isdir(ssl_dev_ca), \
'SSL_DEV_CA must be a dir with hashed dev ca certs'
headers = {
'Authorization': hawk.request_header,
'Content-Type': 'application/json',
}
identity_preference = get(
identity_preference_url,
headers=headers,
verify=ssl_dev_ca).json()['preferences'].pop()
notification_info = send_notifications(message,
identity_preference)
notifications.append(notification_info)
policy.last_notified = current_time
except JSONDecodeError:
logger.warn('')
session.add_all(policies)
session.commit()
return {
'notifications': notifications,
}
def _makeHttpRequest(self, method, route, payload):
""" Make an HTTP Request for the API endpoint. This method wraps
the logic about doing failure retry and passes off the actual work
of doing an HTTP request to another method."""
url = self._constructUrl(route)
log.debug('Full URL used is: %s', url)
hawkExt = self.makeHawkExt()
# Serialize payload if given
if payload is not None:
payload = utils.dumpJson(payload)
# Do a loop of retries
retry = -1 # we plus first in the loop, and attempt 1 is retry 0
retries = self.options['maxRetries']
while retry < retries:
retry += 1
# if this isn't the first retry then we sleep
if retry > 0:
time.sleep(utils.calculateSleepTime(retry))
# Construct header
if self._hasCredentials():
sender = mohawk.Sender(
credentials={
'id': self.options['credentials']['clientId'],
'key': self.options['credentials']['accessToken'],
'algorithm': 'sha256',
},
ext=hawkExt if hawkExt else {},
url=url,
content=payload if payload else '',
content_type='application/json' if payload else '',
method=method,
)
headers = {'Authorization': sender.request_header}
else:
log.debug('Not using hawk!')
headers = {}
if payload:
# Set header for JSON if payload is given, note that we serialize
# outside this loop.
headers['Content-Type'] = 'application/json'
log.debug('Making attempt %d', retry)
try:
response = utils.makeSingleHttpRequest(method, url, payload,
headers)
except requests.exceptions.RequestException as rerr:
if retry < retries:
log.warn('Retrying because of: %s' % rerr)
continue
# raise a connection exception
raise exceptions.TaskclusterConnectionError(
"Failed to establish connection", superExc=rerr)
# Handle non 2xx status code and retry if possible
status = response.status_code
if status == 204:
return None
# Catch retryable errors and go to the beginning of the loop
# to do the retry
if 500 <= status and status < 600 and retry < retries:
log.warn('Retrying because of a %s status code' % status)
continue
# Throw errors for non-retryable errors
if status < 200 or status >= 300:
data = {}
try:
data = response.json()
except:
pass # Ignore JSON errors in error messages
# Find error message
message = "Unknown Server Error"
if isinstance(data, dict):
message = data.get('message')
else:
if status == 401:
message = "Authentication Error"
elif status == 500:
message = "Internal Server Error"
# Raise TaskclusterAuthFailure if this is an auth issue
if status == 401:
raise exceptions.TaskclusterAuthFailure(message,
status_code=status,
body=data,
superExc=None)
# Raise TaskclusterRestFailure for all other issues
raise exceptions.TaskclusterRestFailure(message,
status_code=status,
body=data,
superExc=None)
# Try to load JSON
try:
return response.json()
except ValueError:
return {"response": response}
# This code-path should be unreachable
assert False, "Error from last retry should have been raised!"
