def sig2(method, endpoint, params, provider, aws_api_version):
'''
Sign a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
'''
timenow = datetime.datetime.utcnow()
timestamp = timenow.strftime('%Y-%m-%dT%H:%M:%SZ')
params_with_headers = params.copy()
params_with_headers['AWSAccessKeyId'] = provider.get('id', '')
params_with_headers['SignatureVersion'] = '2'
params_with_headers['SignatureMethod'] = 'HmacSHA256'
params_with_headers['Timestamp'] = '{0}'.format(timestamp)
params_with_headers['Version'] = aws_api_version
keys = sorted(params_with_headers.keys())
values = list(list(map(params_with_headers.get, keys)))
querystring = urlencode(list(zip(keys, values)))
canonical = '{0}\n{1}\n/\n{2}'.format(
method.encode('utf-8'),
endpoint.encode('utf-8'),
querystring.encode('utf-8'),
)
hashed = hmac.new(provider['key'], canonical, hashlib.sha256)
sig = binascii.b2a_base64(hashed.digest())
params_with_headers['Signature'] = sig.strip()
return params_with_headers
def test_urlencoded_ctype(self):
data = {'valid': 'stuff'}
request, response = self.request('/', method='POST',
body=urlencode(data), headers=(
('Content-type', 'application/x-www-form-urlencoded'),
))
self.assertEqual(response.status, '200 OK')
self.assertDictEqual(request.unserialized_data, data)
def test_bad_login(self):
'''
Test logging in
'''
body = urlencode({'totally': 'invalid_creds'})
request, response = self.request('/login', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '401 Unauthorized')
def test_webhook_noauth(self):
'''
Auth can be disabled for requests to the webhook URL
'''
body = urlencode({'foo': 'Foo!'})
request, response = self.request('/hook', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '200 OK')
def test_good_login(self):
'''
Test logging in
'''
body = urlencode(self.auth_creds)
request, response = self.request('/login', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '200 OK')
return response
def test_logout(self):
ret = self.test_good_login()
token = ret.headers['X-Auth-Token']
body = urlencode({})
request, response = self.request('/logout', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded',
'X-Auth-Token': token,
})
self.assertEqual(response.status, '200 OK')
def test_run_bad_login(self):
'''
Test the run URL with bad auth credentials
'''
cmd = dict(self.low, **{'totally': 'invalid_creds'})
body = urlencode(cmd)
request, response = self.request('/run', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '401 Unauthorized')
def test_run_good_login(self):
'''
Test the run URL with good auth credentials
'''
cmd = dict(self.low, **dict(self.auth_creds))
body = urlencode(cmd)
request, response = self.request('/run', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '200 OK')
def test_bad_login(self):
'''
Test logging in
'''
# Mock mk_token for a negative return
self.Resolver.return_value.mk_token.return_value = {}
body = urlencode({'totally': 'invalid_creds'})
request, response = self.request('/login', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '401 Unauthorized')
def test_run_good_login(self):
'''
Test the run URL with good auth credentials
'''
cmd = dict(self.low, **dict(self.auth_creds))
body = urlencode(cmd)
request, response = self.request(
'/run',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
self.assertEqual(response.status, '200 OK')
def test_webhook_noauth(self):
'''
Auth can be disabled for requests to the webhook URL
'''
# Mock fire_event() since we're only testing auth here.
self.get_event.return_value.fire_event.return_value = True
body = urlencode({'foo': 'Foo!'})
request, response = self.request('/hook', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '200 OK')
def test_run_wrong_token(self):
'''
Test the run URL with incorrect token
'''
cmd = dict(self.low, **{'token': 'bad'})
body = urlencode(cmd)
request, response = self.request(
'/run',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
assert response.status == '401 Unauthorized'
def test_websocket_handler_upgrade_to_websocket(self):
response = yield self.http_client.fetch(self.get_url('/login'),
method='POST',
body=urlencode(self.auth_creds),
headers={'Content-Type': self.content_type_map['form']})
token = json.loads(response.body)['return'][0]['token']
url = 'ws://127.0.0.1:{0}/all_events/{1}'.format(self.get_http_port(), token)
request = HTTPRequest(url, headers={'Origin': 'http://example.com',
'Host': 'example.com'})
ws = yield websocket_connect(request)
ws.write_message('websocket client ready')
ws.close()
def test_run_pathname_not_exists_token(self):
'''
Test the run URL with path that does not exist in token
'''
cmd = dict(self.low, **{'token': os.path.join('tmp', 'doesnotexist')})
body = urlencode(cmd)
request, response = self.request(
'/run',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
assert response.status == '401 Unauthorized'
def test_good_login(self):
"""
Test logging in
"""
body = urlencode(self.auth_creds)
request, response = self.request(
"/login",
method="POST",
body=body,
headers={"content-type": "application/x-www-form-urlencoded"},
)
self.assertEqual(response.status, "200 OK")
return response
def test_run_bad_login(self):
'''
Test the run URL with bad auth credentials
'''
cmd = dict(self.low, **{'totally': 'invalid_creds'})
body = urlencode(cmd)
request, response = self.request(
'/run',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
self.assertEqual(response.status, '401 Unauthorized')
def _add_job(self):
'''
Helper function to add a job to the job cache
'''
cmd = dict(self.low, **dict(self.auth_creds))
body = urlencode(cmd)
request, response = self.request(
'/run',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
self.assertEqual(response.status, '200 OK')
def test_run_empty_token_upercase(self):
'''
Test the run URL with empty token with upercase characters
'''
cmd = dict(self.low, **{'ToKen': ''})
body = urlencode(cmd)
request, response = self.request(
'/run',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
assert response.status == '401 Unauthorized'
def test_websocket_handler_upgrade_to_websocket(self):
response = yield self.http_client.fetch(self.get_url('/login'),
method='POST',
body=urlencode(self.auth_creds),
headers={'Content-Type': self.content_type_map['form']})
token = json.loads(response.body)['return'][0]['token']
url = 'ws://127.0.0.1:{0}/hook/{1}'.format(self.get_http_port(), token)
request = HTTPRequest(url, headers={'Origin': 'http://example.com',
'Host': 'example.com'})
ws = yield websocket_connect(request)
ws.write_message('websocket client ready')
ws.close()
def test_login(self):
'''
Test valid logins
'''
# Test in form encoded
response = self.fetch('/login',
method='POST',
body=urlencode(self.auth_creds),
headers={'Content-Type': self.content_type_map['form']})
cookies = response.headers['Set-Cookie']
self.assertEqual(response.code, 200)
response_obj = salt.utils.json.loads(response.body)['return'][0]
token = response_obj['token']
self.assertIn('session_id={0}'.format(token), cookies)
self.assertEqual(sorted(response_obj['perms']), sorted(self.opts['external_auth']['auto'][self.auth_creds_dict['username']]))
self.assertIn('token', response_obj) # TODO: verify that its valid?
self.assertEqual(response_obj['user'], self.auth_creds_dict['username'])
self.assertEqual(response_obj['eauth'], self.auth_creds_dict['eauth'])
# Test in JSON
response = self.fetch('/login',
method='POST',
body=salt.utils.json.dumps(self.auth_creds_dict),
headers={'Content-Type': self.content_type_map['json']})
cookies = response.headers['Set-Cookie']
self.assertEqual(response.code, 200)
response_obj = salt.utils.json.loads(response.body)['return'][0]
token = response_obj['token']
self.assertIn('session_id={0}'.format(token), cookies)
self.assertEqual(sorted(response_obj['perms']), sorted(self.opts['external_auth']['auto'][self.auth_creds_dict['username']]))
self.assertIn('token', response_obj) # TODO: verify that its valid?
self.assertEqual(response_obj['user'], self.auth_creds_dict['username'])
self.assertEqual(response_obj['eauth'], self.auth_creds_dict['eauth'])
# Test in YAML
response = self.fetch('/login',
method='POST',
body=salt.utils.yaml.safe_dump(self.auth_creds_dict),
headers={'Content-Type': self.content_type_map['yaml']})
cookies = response.headers['Set-Cookie']
self.assertEqual(response.code, 200)
response_obj = salt.utils.json.loads(response.body)['return'][0]
token = response_obj['token']
self.assertIn('session_id={0}'.format(token), cookies)
self.assertEqual(sorted(response_obj['perms']), sorted(self.opts['external_auth']['auto'][self.auth_creds_dict['username']]))
self.assertIn('token', response_obj) # TODO: verify that its valid?
self.assertEqual(response_obj['user'], self.auth_creds_dict['username'])
self.assertEqual(response_obj['eauth'], self.auth_creds_dict['eauth'])
def test_run_bad_login(self):
"""
Test the run URL with bad auth credentials
"""
cmd = dict(self.low, **{"totally": "invalid_creds"})
body = urlencode(cmd)
request, response = self.request(
"/run",
method="POST",
body=body,
headers={"content-type": "application/x-www-form-urlencoded"},
)
self.assertEqual(response.status, "401 Unauthorized")
def test_run_good_login(self):
"""
Test the run URL with good auth credentials
"""
cmd = dict(self.low, **dict(self.auth_creds))
body = urlencode(cmd)
request, response = self.request(
"/run",
method="POST",
body=body,
headers={"content-type": "application/x-www-form-urlencoded"},
)
self.assertEqual(response.status, "200 OK")
def test_bad_login(self):
'''
Test logging in
'''
# Mock mk_token for a negative return
self.Resolver.return_value.mk_token.return_value = {}
body = urlencode({'totally': 'invalid_creds'})
request, response = self.request(
'/login',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
self.assertEqual(response.status, '401 Unauthorized')
def test_webhook_noauth(self):
'''
Auth can be disabled for requests to the webhook URL
'''
# Mock fire_event() since we're only testing auth here.
self.get_event.return_value.fire_event.return_value = True
body = urlencode({'foo': 'Foo!'})
request, response = self.request(
'/hook',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
self.assertEqual(response.status, '200 OK')
def test_login(self):
'''
Test valid logins
'''
response = self.fetch('/login',
method='POST',
body=urlencode(self.auth_creds),
headers={'Content-Type': self.content_type_map['form']})
response_obj = json.loads(response.body)['return'][0]
self.assertEqual(response_obj['perms'], self.opts['external_auth']['auto'][self.auth_creds_dict['username']])
self.assertIn('token', response_obj) # TODO: verify that its valid?
self.assertEqual(response_obj['user'], self.auth_creds_dict['username'])
self.assertEqual(response_obj['eauth'], self.auth_creds_dict['eauth'])
def _token(self):
'''
Return the token
'''
body = urlencode(self.auth_creds)
request, response = self.request(
'/login',
method='POST',
body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
}
)
return response.headers['X-Auth-Token']
def test_run_pathname_not_exists_token(self):
"""
Test the run URL with path that does not exist in token
"""
cmd = dict(self.low, **{"token": os.path.join("tmp", "doesnotexist")})
body = urlencode(cmd)
request, response = self.request(
"/run",
method="POST",
body=body,
headers={"content-type": "application/x-www-form-urlencoded"},
)
assert response.status == "401 Unauthorized"
def test_run_wrong_token(self):
"""
Test the run URL with incorrect token
"""
cmd = dict(self.low, **{"token": "bad"})
body = urlencode(cmd)
request, response = self.request(
"/run",
method="POST",
body=body,
headers={"content-type": "application/x-www-form-urlencoded"},
)
assert response.status == "401 Unauthorized"
def test_good_pwd_pam_chsh_service(self):
'''
Test login while specifying chsh service with good passwd
This test ensures this PR is working correctly:
https://github.com/saltstack/salt/pull/31826
'''
copyauth_creds = AUTH_CREDS.copy()
copyauth_creds['service'] = 'chsh'
body = urlencode(copyauth_creds)
request, response = self.request('/login', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '200 OK')
def _add_job(self):
"""
Helper function to add a job to the job cache
"""
cmd = dict(self.low, **dict(self.auth_creds))
body = urlencode(cmd)
request, response = self.request(
"/run",
method="POST",
body=body,
headers={"content-type": "application/x-www-form-urlencoded"},
)
self.assertEqual(response.status, "200 OK")
def test_login(self):
'''
Test valid logins
'''
response = self.fetch('/login',
method='POST',
body=urlencode(self.auth_creds),
headers={'Content-Type': self.content_type_map['form']})
response_obj = json.loads(response.body)['return'][0]
self.assertEqual(response_obj['perms'], self.opts['external_auth']['auto'][self.auth_creds_dict['username']])
self.assertIn('token', response_obj) # TODO: verify that its valid?
self.assertEqual(response_obj['user'], self.auth_creds_dict['username'])
self.assertEqual(response_obj['eauth'], self.auth_creds_dict['eauth'])
def test_run_empty_token_upercase(self):
"""
Test the run URL with empty token with upercase characters
"""
cmd = dict(self.low, **{"ToKen": ""})
body = urlencode(cmd)
request, response = self.request(
"/run",
method="POST",
body=body,
headers={"content-type": "application/x-www-form-urlencoded"},
)
assert response.status == "401 Unauthorized"
def _token(self):
'''
Return the token
'''
body = urlencode(self.auth_creds)
request, response = self.request(
'/login',
method='POST',
body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
}
)
return response.headers['X-Auth-Token']
def test_logout(self):
ret = self.test_good_login()
token = ret.headers['X-Auth-Token']
body = urlencode({})
request, response = self.request(
'/logout',
method='POST',
body=body,
headers={
'content-type': 'application/x-www-form-urlencoded',
'X-Auth-Token': token,
})
self.assertEqual(response.status, '200 OK')
def test_websocket_handler_cors_origin_wildcard(self):
self._app.mod_opts['cors_origin'] = '*'
response = yield self.http_client.fetch(self.get_url('/login'),
method='POST',
body=urlencode(self.auth_creds),
headers={'Content-Type': self.content_type_map['form']})
token = salt.utils.json.loads(self.decode_body(response).body)['return'][0]['token']
url = 'ws://127.0.0.1:{0}/all_events/{1}'.format(self.get_http_port(), token)
request = HTTPRequest(url, headers={'Origin': 'http://foo.bar',
'Host': 'example.com'})
ws = yield websocket_connect(request)
ws.write_message('websocket client ready')
ws.close()
def test_logout(self):
ret = self.test_good_login()
token = ret.headers["X-Auth-Token"]
body = urlencode({})
request, response = self.request(
"/logout",
method="POST",
body=body,
headers={
"content-type": "application/x-www-form-urlencoded",
"X-Auth-Token": token,
},
)
self.assertEqual(response.status, "200 OK")
def test_login_bad_creds(self):
'''
Test logins with bad/missing passwords
'''
bad_creds = []
for key, val in self.auth_creds_dict.iteritems():
if key == 'username':
val = val + 'foo'
bad_creds.append((key, val))
response = self.fetch('/login',
method='POST',
body=urlencode(bad_creds),
headers={'Content-Type': self.content_type_map['form']})
self.assertEqual(response.code, 401)
def test_login_missing_password(self):
'''
Test logins with bad/missing passwords
'''
bad_creds = []
for key, val in six.iteritems(self.auth_creds_dict):
if key == 'password':
continue
bad_creds.append((key, val))
response = self.fetch('/login',
method='POST',
body=urlencode(bad_creds),
headers={'Content-Type': self.content_type_map['form']})
self.assertEqual(response.code, 400)
def test_good_pwd_pam_chsh_service(self):
"""
Test login while specifying chsh service with good passwd
This test ensures this PR is working correctly:
https://github.com/saltstack/salt/pull/31826
"""
copyauth_creds = AUTH_CREDS.copy()
copyauth_creds["service"] = "chsh"
body = urlencode(copyauth_creds)
request, response = self.request(
"/login",
method="POST",
body=body,
headers={"content-type": "application/x-www-form-urlencoded"},
)
self.assertEqual(response.status, "200 OK")
def test_run_bad_login(self):
'''
Test the run URL with bad auth credentials
'''
cmd = dict(self.low, **{'totally': 'invalid_creds'})
body = urlencode(cmd)
# Mock the interaction with Salt so we can focus on the API.
with mock.patch.object(self.app.salt.netapi.NetapiClient, 'run',
side_effect=EauthAuthenticationError('Oh noes!')):
request, response = self.request('/run', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '401 Unauthorized')
def test_run_good_login(self):
'''
Test the run URL with good auth credentials
'''
cmd = dict(self.low, **dict(self.auth_creds))
body = urlencode(cmd)
# Mock the interaction with Salt so we can focus on the API.
with mock.patch.object(self.app.salt.netapi.NetapiClient, 'run',
return_value=True):
request, response = self.request('/run', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '200 OK')
def _get_response(self, action, params, method='POST'):
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json; charset=UTF-8'
}
if not params:
params = {}
query_params = {}
if ('server_basic_username' in globals()
and len(self.server_basic_username) > 0):
userAndPass = b64encode(
str.encode(self.server_basic_username + ':' +
self.server_basic_password)).decode('ascii')
headers['Authorization'] = 'Basic %s' % userAndPass
if (len(self._session) > 0):
params['ses'] = self._session
query_params['a'] = action
if not method:
method = 'POST'
if method == 'POST':
data = urlencode(params)
else:
data = None
query_params.update(params)
http_timeout = 10 * 60
result = salt.utils.http.query(
self._url,
method=method,
params=query_params,
data=data,
headers_dict=headers,
decode=True,
status=True,
opts=__opts__,
)
return result
def test_login_missing_password(self):
"""
Test logins with bad/missing passwords
"""
bad_creds = []
for key, val in six.iteritems(self.auth_creds_dict):
if key == "password":
continue
bad_creds.append((key, val))
response = self.fetch(
"/login",
method="POST",
body=urlencode(bad_creds),
headers={"Content-Type": self.content_type_map["form"]},
)
self.assertEqual(response.code, 400)
def test_run_good_login(self):
'''
Test the run URL with good auth credentials
'''
cmd = dict(self.low, **dict(self.auth_creds))
body = urlencode(cmd)
# Mock the interaction with Salt so we can focus on the API.
with mock.patch.object(self.app.salt.netapi.NetapiClient,
'run',
return_value=True):
request, response = self.request(
'/run',
method='POST',
body=body,
headers={'content-type': 'application/x-www-form-urlencoded'})
self.assertEqual(response.status, '200 OK')
def test_websocket_handler_upgrade_to_websocket(self):
response = yield self.http_client.fetch(
self.get_url("/login"),
method="POST",
body=urlencode(self.auth_creds),
headers={"Content-Type": self.content_type_map["form"]},
)
token = salt.utils.json.loads(self.decode_body(response).body)["return"][0][
"token"
]
url = "ws://127.0.0.1:{0}/all_events/{1}".format(self.get_http_port(), token)
request = HTTPRequest(
url, headers={"Origin": "http://example.com", "Host": "example.com"}
)
ws = yield websocket_connect(request)
ws.write_message("websocket client ready")
ws.close()
def test_login_bad_creds(self):
'''
Test logins with bad/missing passwords
'''
bad_creds = []
for key, val in six.iteritems(self.auth_creds_dict):
if key == 'username':
val = val + 'foo'
if key == 'eauth':
val = 'sharedsecret'
bad_creds.append((key, val))
response = self.fetch('/login',
method='POST',
body=urlencode(bad_creds),
headers={'Content-Type': self.content_type_map['form']})
self.assertEqual(response.code, 401)
def test_good_login(self):
'''
Test logging in
'''
# Mock mk_token for a positive return
self.Resolver.return_value.mk_token.return_value = {
'token': '6d1b722e',
'start': 1363805943.776223,
'expire': 1363849143.776224,
'name': 'saltdev',
'eauth': 'auto',
}
body = urlencode(self.auth_creds)
request, response = self.request('/login', method='POST', body=body,
headers={
'content-type': 'application/x-www-form-urlencoded'
})
self.assertEqual(response.status, '200 OK')
return response
def sig2(method, endpoint, params, provider, aws_api_version):
'''
Sign a query against AWS services using Signature Version 2 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
'''
timenow = datetime.datetime.utcnow()
timestamp = timenow.strftime('%Y-%m-%dT%H:%M:%SZ')
# Retrieve access credentials from meta-data, or use provided
access_key_id, secret_access_key, token = creds(provider)
params_with_headers = params.copy()
params_with_headers['AWSAccessKeyId'] = access_key_id
params_with_headers['SignatureVersion'] = '2'
params_with_headers['SignatureMethod'] = 'HmacSHA256'
params_with_headers['Timestamp'] = '{0}'.format(timestamp)
params_with_headers['Version'] = aws_api_version
keys = sorted(params_with_headers.keys())
values = list(list(map(params_with_headers.get, keys)))
querystring = urlencode(list(zip(keys, values)))
canonical = '{0}\n{1}\n/\n{2}'.format(
method.encode('utf-8'),
endpoint.encode('utf-8'),
querystring.encode('utf-8'),
)
hashed = hmac.new(secret_access_key, canonical, hashlib.sha256)
sig = binascii.b2a_base64(hashed.digest())
params_with_headers['Signature'] = sig.strip()
# Add in security token if we have one
if token != '':
params_with_headers['SecurityToken'] = token
return params_with_headers
def test_cors_origin_multiple(self):
self._app.mod_opts['cors_origin'] = ['http://example.com', 'http://foo.bar']
response = yield self.http_client.fetch(self.get_url('/login'),
method='POST',
body=urlencode(self.auth_creds),
headers={'Content-Type': self.content_type_map['form']})
token = json.loads(response.body)['return'][0]['token']
url = 'ws://127.0.0.1:{0}/hook/{1}'.format(self.get_http_port(), token)
# Example.com should works
request = HTTPRequest(url, headers={'Origin': 'http://example.com',
'Host': 'example.com'})
ws = yield websocket_connect(request)
ws.write_message('websocket client ready')
ws.close()
# Foo.bar too
request = HTTPRequest(url, headers={'Origin': 'http://foo.bar',
'Host': 'example.com'})
ws = yield websocket_connect(request)
ws.write_message('websocket client ready')
ws.close()
def sig4(method, endpoint, params, prov_dict, aws_api_version, location,
product='ec2', uri='/', requesturl=None):
'''
Sign a query against AWS services using Signature Version 4 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
'''
timenow = datetime.datetime.utcnow()
timestamp = timenow.strftime('%Y-%m-%dT%H:%M:%SZ')
params_with_headers = params.copy()
params_with_headers['Version'] = aws_api_version
keys = sorted(params_with_headers.keys())
values = list(map(params_with_headers.get, keys))
querystring = urlencode(list(zip(keys, values)))
amzdate = timenow.strftime('%Y%m%dT%H%M%SZ')
datestamp = timenow.strftime('%Y%m%d')
payload_hash = hashlib.sha256('').hexdigest()
canonical_headers = 'host:{0}\nx-amz-date:{1}\n'.format(
endpoint,
amzdate,
)
signed_headers = 'host;x-amz-date'
request = '\n'.join((
method, endpoint, querystring, canonical_headers,
signed_headers, payload_hash
))
algorithm = 'AWS4-HMAC-SHA256'
# Create payload hash (hash of the request body content). For GET
# requests, the payload is an empty string ('').
payload_hash = hashlib.sha256('').hexdigest()
# Combine elements to create create canonical request
canonical_request = '\n'.join((
method,
uri,
querystring,
canonical_headers,
signed_headers,
payload_hash
))
# Create the string to sign
credential_scope = '/'.join((
datestamp, location, product, 'aws4_request'
))
string_to_sign = '\n'.join((
algorithm,
amzdate,
credential_scope,
hashlib.sha256(canonical_request).hexdigest()
))
# Create the signing key using the function defined above.
signing_key = _sig_key(
prov_dict.get('key', ''),
datestamp,
location,
product
)
# Sign the string_to_sign using the signing_key
signature = hmac.new(
signing_key,
string_to_sign.encode('utf-8'),
hashlib.sha256).hexdigest()
# Add signing information to the request
authorization_header = (
'{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}'
).format(
algorithm,
prov_dict.get('id', ''),
credential_scope,
signed_headers,
signature,
)
headers = {
'x-amz-date': amzdate,
'Authorization': authorization_header
}
requesturl = '{0}?{1}'.format(requesturl, querystring)
return headers, requesturl
def sig4(method, endpoint, params, prov_dict,
aws_api_version=DEFAULT_AWS_API_VERSION, location=DEFAULT_LOCATION,
product='ec2', uri='/', requesturl=None, data='', headers=None):
'''
Sign a query against AWS services using Signature Version 4 Signing
Process. This is documented at:
http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
'''
timenow = datetime.datetime.utcnow()
# Retrieve access credentials from meta-data, or use provided
access_key_id, secret_access_key, token = creds(prov_dict)
params_with_headers = params.copy()
if product != 's3':
params_with_headers['Version'] = aws_api_version
keys = sorted(params_with_headers.keys())
values = list(map(params_with_headers.get, keys))
querystring = urlencode(list(zip(keys, values))).replace('+', '%20')
amzdate = timenow.strftime('%Y%m%dT%H%M%SZ')
datestamp = timenow.strftime('%Y%m%d')
canonical_headers = 'host:{0}\nx-amz-date:{1}'.format(
endpoint,
amzdate,
)
signed_headers = 'host;x-amz-date'
if isinstance(headers, dict):
for header in sorted(headers.keys()):
canonical_headers += '\n{0}:{1}'.format(header, headers[header])
signed_headers += ';{0}'.format(header)
canonical_headers += '\n'
algorithm = 'AWS4-HMAC-SHA256'
# Create payload hash (hash of the request body content). For GET
# requests, the payload is an empty string ('').
payload_hash = hashlib.sha256(data).hexdigest()
# Combine elements to create create canonical request
canonical_request = '\n'.join((
method,
uri,
querystring,
canonical_headers,
signed_headers,
payload_hash
))
# Create the string to sign
credential_scope = '/'.join((
datestamp, location, product, 'aws4_request'
))
string_to_sign = '\n'.join((
algorithm,
amzdate,
credential_scope,
hashlib.sha256(canonical_request).hexdigest()
))
# Create the signing key using the function defined above.
signing_key = _sig_key(
secret_access_key,
datestamp,
location,
product
)
# Sign the string_to_sign using the signing_key
signature = hmac.new(
signing_key,
string_to_sign.encode('utf-8'),
hashlib.sha256).hexdigest()
# Add signing information to the request
authorization_header = (
'{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}'
).format(
algorithm,
access_key_id,
credential_scope,
signed_headers,
signature,
)
new_headers = {
'x-amz-date': amzdate,
'x-amz-content-sha256': payload_hash,
'Authorization': authorization_header,
}
if isinstance(headers, dict):
for header in sorted(headers.keys()):
new_headers[header] = headers[header]
# Add in security token if we have one
if token != '':
new_headers['X-Amz-Security-Token'] = token
requesturl = '{0}?{1}'.format(requesturl, querystring)
return new_headers, requesturl
def query(key, keyid, method='GET', params=None, headers=None,
requesturl=None, return_url=False, bucket=None, service_url=None,
path=None, return_bin=False, action=None, local_file=None,
verify_ssl=True, full_headers=False):
'''
Perform a query against an S3-like API. This function requires that a
secret key and the id for that key are passed in. For instance:
s3.keyid: GKTADJGHEIQSXMKKRBJ08H
s3.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
A service_url may also be specified in the configuration::
s3.service_url: s3.amazonaws.com
If a service_url is not specified, the default is s3.amazonaws.com. This
may appear in various documentation as an "endpoint". A comprehensive list
for Amazon S3 may be found at::
http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
The service_url will form the basis for the final endpoint that is used to
query the service.
SSL verification may also be turned off in the configuration:
s3.verify_ssl: False
This is required if using S3 bucket names that contain a period, as
these will not match Amazon's S3 wildcard certificates. Certificate
verification is enabled by default.
'''
if not HAS_REQUESTS:
log.error('There was an error: requests is required for s3 access')
if not headers:
headers = {}
if not params:
params = {}
if path is None:
path = ''
if not service_url:
service_url = 's3.amazonaws.com'
if bucket:
endpoint = '{0}.{1}'.format(bucket, service_url)
else:
endpoint = service_url
# Try grabbing the credentials from the EC2 instance IAM metadata if available
token = None
if not key or not keyid:
iam_creds = iam.get_iam_metadata()
key = iam_creds['secret_key']
keyid = iam_creds['access_key']
token = iam_creds['security_token']
if not requesturl:
x_amz_date = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
content_type = 'text/plain'
if method == 'GET':
if bucket:
can_resource = '/{0}/{1}'.format(bucket, path)
else:
can_resource = '/'
elif method == 'PUT' or method == 'HEAD' or method == 'DELETE':
if path:
can_resource = '/{0}/{1}'.format(bucket, path)
else:
can_resource = '/{0}/'.format(bucket)
if action:
can_resource += '?{0}'.format(action)
log.debug('CanonicalizedResource: {0}'.format(can_resource))
headers['Host'] = endpoint
headers['Content-type'] = content_type
headers['Date'] = x_amz_date
if token:
headers['x-amz-security-token'] = token
string_to_sign = '{0}\n'.format(method)
new_headers = []
for header in sorted(headers):
if header.lower().startswith('x-amz'):
log.debug(header.lower())
new_headers.append('{0}:{1}'.format(header.lower(),
headers[header]))
can_headers = '\n'.join(new_headers)
log.debug('CanonicalizedAmzHeaders: {0}'.format(can_headers))
string_to_sign += '\n{0}'.format(content_type)
string_to_sign += '\n{0}'.format(x_amz_date)
if can_headers:
string_to_sign += '\n{0}'.format(can_headers)
string_to_sign += '\n{0}'.format(can_resource)
log.debug('String To Sign:: \n{0}'.format(string_to_sign))
hashed = hmac.new(key, string_to_sign, hashlib.sha1)
sig = binascii.b2a_base64(hashed.digest())
headers['Authorization'] = 'AWS {0}:{1}'.format(keyid, sig.strip())
querystring = urlencode(params)
if action:
if querystring:
querystring = '{0}&{1}'.format(action, querystring)
else:
querystring = action
requesturl = 'https://{0}/'.format(endpoint)
if path:
requesturl += path
if querystring:
requesturl += '?{0}'.format(querystring)
data = None
if method == 'PUT':
if local_file:
with salt.utils.fopen(local_file, 'r') as ifile:
data = ifile.read()
log.debug('S3 Request: {0}'.format(requesturl))
log.debug('S3 Headers::')
log.debug(' Authorization: {0}'.format(headers['Authorization']))
try:
result = requests.request(method, requesturl, headers=headers,
data=data,
verify=verify_ssl)
response = result.content
except requests.exceptions.HTTPError as exc:
log.error('There was an error::')
if hasattr(exc, 'code') and hasattr(exc, 'msg'):
log.error(' Code: {0}: {1}'.format(exc.code, exc.msg))
log.error(' Content: \n{0}'.format(exc.read()))
return False
log.debug('S3 Response Status Code: {0}'.format(result.status_code))
if method == 'PUT':
if result.status_code == 200:
if local_file:
log.debug('Uploaded from {0} to {1}'.format(local_file, path))
else:
log.debug('Created bucket {0}'.format(bucket))
else:
if local_file:
log.debug('Failed to upload from {0} to {1}: {2}'.format(
local_file,
path,
result.status_code,
))
else:
log.debug('Failed to create bucket {0}'.format(bucket))
return
if method == 'DELETE':
if str(result.status_code).startswith('2'):
if path:
log.debug('Deleted {0} from bucket {1}'.format(path, bucket))
else:
log.debug('Deleted bucket {0}'.format(bucket))
else:
if path:
log.debug('Failed to delete {0} from bucket {1}: {2}'.format(
path,
bucket,
result.status_code,
))
else:
log.debug('Failed to delete bucket {0}'.format(bucket))
return
# This can be used to save a binary object to disk
if local_file and method == 'GET':
log.debug('Saving to local file: {0}'.format(local_file))
with salt.utils.fopen(local_file, 'w') as out:
out.write(response)
return 'Saved to local file: {0}'.format(local_file)
# This can be used to return a binary object wholesale
if return_bin:
return response
if response:
items = ET.fromstring(response)
ret = []
for item in items:
ret.append(xml.to_dict(item))
if return_url is True:
return ret, requesturl
else:
if result.status_code != requests.codes.ok:
return
ret = {'headers': []}
if full_headers:
ret['headers'] = dict(result.headers)
else:
for header in result.headers:
ret['headers'].append(header.strip())
return ret
def test_get_lowstate(self):
'''
Test transformations low data of the function _get_lowstate
'''
valid_lowstate = [{
"client": "local",
"tgt": "*",
"fun": "test.fib",
"arg": ["10"]
}]
# Case 1. dictionary type of lowstate
request_lowstate = {
"client": "local",
"tgt": "*",
"fun": "test.fib",
"arg": ["10"]
}
response = self.fetch('/',
method='POST',
body=json.dumps(request_lowstate),
headers={'Content-Type': self.content_type_map['json']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# Case 2. string type of arg
request_lowstate = [{
"client": "local",
"tgt": "*",
"fun": "test.fib",
"arg": "10"
}]
response = self.fetch('/',
method='POST',
body=json.dumps(request_lowstate),
headers={'Content-Type': self.content_type_map['json']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# Case 3. Combine Case 1 and Case 2.
request_lowstate = {
"client": "local",
"tgt": "*",
"fun": "test.fib",
"arg": "10"
}
# send as json
response = self.fetch('/',
method='POST',
body=json.dumps(request_lowstate),
headers={'Content-Type': self.content_type_map['json']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# send as yaml
response = self.fetch('/',
method='POST',
body=yaml.dump(request_lowstate),
headers={'Content-Type': self.content_type_map['yaml']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# send as plain text
response = self.fetch('/',
method='POST',
body=json.dumps(request_lowstate),
headers={'Content-Type': self.content_type_map['text']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# send as form-urlencoded
request_form_lowstate = (
('client', 'local'),
('tgt', '*'),
('fun', 'test.fib'),
('arg', '10'),
)
response = self.fetch('/',
method='POST',
body=urlencode(request_form_lowstate),
headers={'Content-Type': self.content_type_map['form']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
def test_deserialize(self):
'''
Send various encoded forms of lowstates (and bad ones) to make sure we
handle deserialization correctly
'''
valid_lowstate = [{
"client": "local",
"tgt": "*",
"fun": "test.fib",
"arg": ["10"]
},
{
"client": "runner",
"fun": "jobs.lookup_jid",
"jid": "20130603122505459265"
}]
# send as JSON
response = self.fetch('/',
method='POST',
body=json.dumps(valid_lowstate),
headers={'Content-Type': self.content_type_map['json']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# send yaml as json (should break)
response = self.fetch('/',
method='POST',
body=yaml.dump(valid_lowstate),
headers={'Content-Type': self.content_type_map['json']})
self.assertEqual(response.code, 400)
# send as yaml
response = self.fetch('/',
method='POST',
body=yaml.dump(valid_lowstate),
headers={'Content-Type': self.content_type_map['yaml']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# send json as yaml (works since yaml is a superset of json)
response = self.fetch('/',
method='POST',
body=json.dumps(valid_lowstate),
headers={'Content-Type': self.content_type_map['yaml']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# send json as text/plain
response = self.fetch('/',
method='POST',
body=json.dumps(valid_lowstate),
headers={'Content-Type': self.content_type_map['text']})
self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate'])
# send form-urlencoded
form_lowstate = (
('client', 'local'),
('tgt', '*'),
('fun', 'test.fib'),
('arg', '10'),
('arg', 'foo'),
)
response = self.fetch('/',
method='POST',
body=urlencode(form_lowstate),
headers={'Content-Type': self.content_type_map['form']})
returned_lowstate = json.loads(response.body)['lowstate']
self.assertEqual(len(returned_lowstate), 1)
returned_lowstate = returned_lowstate[0]
self.assertEqual(returned_lowstate['client'], 'local')
self.assertEqual(returned_lowstate['tgt'], '*')
self.assertEqual(returned_lowstate['fun'], 'test.fib')
self.assertEqual(returned_lowstate['arg'], ['10', 'foo'])
