def __init__(self, app, instance_keys, storage):
self.app = app
self._legacy_secscan_api = None
validator = V2SecurityConfigValidator(
app.config.get("FEATURE_SECURITY_SCANNER", False),
app.config.get("SECURITY_SCANNER_ENDPOINT", None),
)
if not validator.valid():
msg = "Failed to validate security scanner V2 configuration"
logger.warning(msg)
raise InvalidConfigurationException(msg)
url_scheme_and_hostname = URLSchemeAndHostname(
app.config["PREFERRED_URL_SCHEME"], app.config["SERVER_HOSTNAME"])
self._legacy_secscan_api = SecurityScannerAPI(
app.config,
storage,
app.config["SERVER_HOSTNAME"],
app.config["HTTPCLIENT"],
uri_creator=get_blob_download_uri_getter(
app.test_request_context("/"), url_scheme_and_hostname),
instance_keys=instance_keys,
)
def test_validate_bitbucket_trigger(app):
url_hit = [False]
@urlmatch(netloc=r"bitbucket.org")
def handler(url, request):
url_hit[0] = True
return {
"status_code": 200,
"content": "oauth_token=foo&oauth_token_secret=bar",
}
with HTTMock(handler):
validator = BitbucketTriggerValidator()
url_scheme_and_hostname = URLSchemeAndHostname("http", "localhost:5000")
unvalidated_config = ValidatorContext(
{
"BITBUCKET_TRIGGER_CONFIG": {
"CONSUMER_KEY": "foo",
"CONSUMER_SECRET": "bar",
},
},
url_scheme_and_hostname=url_scheme_and_hostname,
)
validator.validate(unvalidated_config)
assert url_hit[0]
def __init__(self, app, instance_keys, storage):
self.app = app
self._legacy_secscan_api = None
validator = V2SecurityConfigValidator(
app.config.get("FEATURE_SECURITY_SCANNER", False),
app.config.get("SECURITY_SCANNER_ENDPOINT"),
)
if not validator.valid():
msg = "Failed to validate security scanner V2 configuration"
logger.warning(msg)
raise InvalidConfigurationException(msg)
url_scheme_and_hostname = URLSchemeAndHostname(
app.config["PREFERRED_URL_SCHEME"], app.config["SERVER_HOSTNAME"])
self._legacy_secscan_api = SecurityScannerAPI(
app.config,
storage,
app.config["SERVER_HOSTNAME"],
app.config["HTTPCLIENT"],
uri_creator=get_blob_download_uri_getter(
app.test_request_context("/"), url_scheme_and_hostname),
instance_keys=instance_keys,
)
# NOTE: This import is in here because otherwise this class would depend upon app.
# Its not great, but as this is intended to be legacy until its removed, its okay.
from util.secscan.analyzer import LayerAnalyzer
self._target_version = app.config.get(
"SECURITY_SCANNER_ENGINE_VERSION_TARGET", 3)
self._analyzer = LayerAnalyzer(app.config, self._legacy_secscan_api)
def test_validate_gitlab_enterprise_trigger(app):
url_hit = [False]
@urlmatch(netloc=r"somegitlab", path="/oauth/token")
def handler(_, __):
url_hit[0] = True
return {
"status_code": 400,
"content": json.dumps({"error": "invalid code"})
}
with HTTMock(handler):
validator = GitLabTriggerValidator()
url_scheme_and_hostname = URLSchemeAndHostname("http",
"localhost:5000")
unvalidated_config = ValidatorContext(
{
"GITLAB_TRIGGER_CONFIG": {
"GITLAB_ENDPOINT": "http://somegitlab",
"CLIENT_ID": "foo",
"CLIENT_SECRET": "bar",
},
},
http_client=build_requests_session(),
url_scheme_and_hostname=url_scheme_and_hostname,
)
validator.validate(unvalidated_config)
assert url_hit[0]
def test_validate_bitbucket_trigger(app):
url_hit = [False]
@urlmatch(netloc=r'bitbucket.org')
def handler(url, request):
url_hit[0] = True
return {
'status_code': 200,
'content': 'oauth_token=foo&oauth_token_secret=bar',
}
with HTTMock(handler):
validator = BitbucketTriggerValidator()
url_scheme_and_hostname = URLSchemeAndHostname('http',
'localhost:5000')
unvalidated_config = ValidatorContext(
{
'BITBUCKET_TRIGGER_CONFIG': {
'CONSUMER_KEY': 'foo',
'CONSUMER_SECRET': 'bar',
},
},
url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]
def test_validate_gitlab_enterprise_trigger(app):
url_hit = [False]
@urlmatch(netloc=r'somegitlab', path='/oauth/token')
def handler(_, __):
url_hit[0] = True
return {
'status_code': 400,
'content': json.dumps({'error': 'invalid code'})
}
with HTTMock(handler):
validator = GitLabTriggerValidator()
url_scheme_and_hostname = URLSchemeAndHostname('http',
'localhost:5000')
unvalidated_config = ValidatorContext(
{
'GITLAB_TRIGGER_CONFIG': {
'GITLAB_ENDPOINT': 'http://somegitlab',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
},
http_client=build_requests_session(),
url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]
def exchange_code(self,
app_config,
http_client,
code,
form_encode=False,
redirect_suffix='',
client_auth=False):
""" Exchanges an OAuth access code for associated OAuth token and other data. """
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(
app_config)
payload = {
'code':
code,
'grant_type':
'authorization_code',
'redirect_uri':
self.get_redirect_uri(url_scheme_and_hostname, redirect_suffix)
}
headers = {'Accept': 'application/json'}
auth = None
if client_auth:
auth = (self.client_id(), self.client_secret())
else:
payload['client_id'] = self.client_id()
payload['client_secret'] = self.client_secret()
token_url = self.token_endpoint().to_url()
if form_encode:
get_access_token = http_client.post(token_url,
data=payload,
headers=headers,
auth=auth)
else:
get_access_token = http_client.post(token_url,
params=payload,
headers=headers,
auth=auth)
if get_access_token.status_code // 100 != 2:
logger.debug('Got get_access_token response %s',
get_access_token.text)
raise OAuthExchangeCodeException(
'Got non-2XX response for code exchange: %s' %
get_access_token.status_code)
json_data = get_access_token.json()
if not json_data:
raise OAuthExchangeCodeException(
'Got non-JSON response for code exchange')
if 'error' in json_data:
raise OAuthExchangeCodeException(
json_data.get('error_description', json_data['error']))
return json_data
def test_validate_noop(unvalidated_config, app):
unvalidated_config = ValidatorContext(
unvalidated_config,
feature_sec_scanner=False,
is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
SecurityScannerValidator.validate(unvalidated_config)
def test_auth_url(oidc_service, discovery_handler, http_client,
authorize_handler):
config = {"PREFERRED_URL_SCHEME": "https", "SERVER_HOSTNAME": "someserver"}
with HTTMock(discovery_handler, authorize_handler):
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(config)
auth_url = oidc_service.get_auth_url(url_scheme_and_hostname, "",
"some csrf token", ["one", "two"])
# Hit the URL and ensure it works.
result = http_client.get(auth_url).json()
assert result["state"] == "some csrf token"
assert result["scope"] == "one two"
def test_auth_url(oidc_service, discovery_handler, http_client,
authorize_handler):
config = {'PREFERRED_URL_SCHEME': 'https', 'SERVER_HOSTNAME': 'someserver'}
with HTTMock(discovery_handler, authorize_handler):
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(config)
auth_url = oidc_service.get_auth_url(url_scheme_and_hostname, '',
'some csrf token', ['one', 'two'])
# Hit the URL and ensure it works.
result = http_client.get(auth_url).json()
assert result['state'] == 'some csrf token'
assert result['scope'] == 'one two'
def test_validate(unvalidated_config, expected_error, app):
unvalidated_config = ValidatorContext(
unvalidated_config,
feature_sec_scanner=True,
is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
with fake_security_scanner(hostname='fakesecurityscanner'):
if expected_error is not None:
with pytest.raises(expected_error):
SecurityScannerValidator.validate(unvalidated_config)
else:
SecurityScannerValidator.validate(unvalidated_config)
def exchange_code(
self,
app_config,
http_client,
code,
form_encode=False,
redirect_suffix="",
client_auth=False,
):
""" Exchanges an OAuth access code for associated OAuth token and other data. """
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app_config)
payload = {
"code": code,
"grant_type": "authorization_code",
"redirect_uri": self.get_redirect_uri(url_scheme_and_hostname, redirect_suffix),
}
headers = {"Accept": "application/json"}
auth = None
if client_auth:
auth = (self.client_id(), self.client_secret())
else:
payload["client_id"] = self.client_id()
payload["client_secret"] = self.client_secret()
token_url = self.token_endpoint().to_url()
if form_encode:
get_access_token = http_client.post(token_url, data=payload, headers=headers, auth=auth)
else:
get_access_token = http_client.post(
token_url, params=payload, headers=headers, auth=auth
)
if get_access_token.status_code // 100 != 2:
logger.debug("Got get_access_token response %s", get_access_token.text)
raise OAuthExchangeCodeException(
"Got non-2XX response for code exchange: %s" % get_access_token.status_code
)
json_data = get_access_token.json()
if not json_data:
raise OAuthExchangeCodeException("Got non-JSON response for code exchange")
if "error" in json_data:
raise OAuthExchangeCodeException(json_data.get("error_description", json_data["error"]))
return json_data
def from_app(
cls,
app,
config,
user_password,
ip_resolver,
instance_keys,
client=None,
config_provider=None,
init_scripts_location=None,
):
"""
Creates a ValidatorContext from an app config, with a given config to validate.
:param app: the Flask app to pull configuration information from
:param config: the config to validate
:param user_password: request password
:param instance_keys: The instance keys handler
:param ip_resolver: an App
:param client: http client used to connect to services
:param config_provider: config provider used to access config volume(s)
:param init_scripts_location: location where initial load scripts are stored
:return: ValidatorContext
"""
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(
app.config)
return cls(
config,
user_password=user_password,
http_client=client or app.config["HTTPCLIENT"],
context=app.app_context,
url_scheme_and_hostname=url_scheme_and_hostname,
jwt_auth_max=app.config.get("JWT_AUTH_MAX_FRESH_S", 300),
registry_title=app.config["REGISTRY_TITLE"],
ip_resolver=ip_resolver,
feature_sec_scanner=app.config.get("FEATURE_SECURITY_SCANNER",
False),
is_testing=app.config.get("TESTING", False),
uri_creator=get_blob_download_uri_getter(
app.test_request_context("/"), url_scheme_and_hostname),
config_provider=config_provider,
instance_keys=instance_keys,
init_scripts_location=init_scripts_location,
)
import pytest
from app import app
from util.config import URLSchemeAndHostname
from util.secscan.secscan_util import get_blob_download_uri_getter
from test.fixtures import *
@pytest.mark.parametrize(
'url_scheme_and_hostname, repo_namespace, checksum, expected_value,', [
(URLSchemeAndHostname(
'http', 'localhost:5000'), 'devtable/simple', 'tarsum+sha256:123',
'http://localhost:5000/v2/devtable/simple/blobs/tarsum%2Bsha256:123'),
])
def test_blob_download_uri_getter(app, url_scheme_and_hostname, repo_namespace,
checksum, expected_value):
blob_uri_getter = get_blob_download_uri_getter(
app.test_request_context('/'), url_scheme_and_hostname)
assert blob_uri_getter(repo_namespace, checksum) == expected_value
# Note: We set `has_namespace` to `False` here, as we explicitly want this queue to not be emptied
# when a namespace is marked for deletion.
namespace_gc_queue = WorkQueue(app.config["NAMESPACE_GC_QUEUE_NAME"], tf, has_namespace=False)
all_queues = [
image_replication_queue,
dockerfile_build_queue,
notification_queue,
secscan_notification_queue,
chunk_cleanup_queue,
repository_gc_queue,
namespace_gc_queue,
]
url_scheme_and_hostname = URLSchemeAndHostname(
app.config["PREFERRED_URL_SCHEME"], app.config["SERVER_HOSTNAME"]
)
repo_mirror_api = RepoMirrorAPI(
app.config,
app.config["SERVER_HOSTNAME"],
app.config["HTTPCLIENT"],
instance_keys=instance_keys,
)
tuf_metadata_api = TUFMetadataAPI(app, app.config)
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
_v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)
if os.path.exists(_v2_key_path):
docker_v2_signing_key = RSAKey().load(_v2_key_path)
def exchange_code(
self,
app_config,
http_client,
code,
form_encode=False,
redirect_suffix="",
client_auth=False,
):
"""
Exchanges an OAuth access code for associated OAuth token and other data.
"""
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app_config)
payload = {
"code": code,
"grant_type": "authorization_code",
"redirect_uri": self.get_redirect_uri(url_scheme_and_hostname, redirect_suffix),
}
headers = {"Accept": "application/json"}
auth = None
if client_auth:
auth = (self.client_id(), self.client_secret())
else:
payload["client_id"] = self.client_id()
payload["client_secret"] = self.client_secret()
token_url = self.token_endpoint().to_url()
def perform_request():
attempts = 0
max_attempts = 3
timeout = 5 / 1000
while attempts < max_attempts:
if self._is_testing:
headers["X-Quay-Retry-Attempts"] = str(attempts)
try:
if form_encode:
return http_client.post(
token_url, data=payload, headers=headers, auth=auth, timeout=5
)
else:
return http_client.post(
token_url, params=payload, headers=headers, auth=auth, timeout=5
)
except requests.ConnectionError:
logger.debug("Got ConnectionError during OAuth token exchange, retrying.")
attempts += 1
time.sleep(timeout)
get_access_token = perform_request()
if get_access_token is None:
logger.debug("Received too many ConnectionErrors during code exchange")
raise OAuthExchangeCodeException(
"Received too many ConnectionErrors during code exchange"
)
if get_access_token.status_code // 100 != 2:
logger.debug("Got get_access_token response %s", get_access_token.text)
raise OAuthExchangeCodeException(
"Got non-2XX response for code exchange: %s" % get_access_token.status_code
)
json_data = get_access_token.json()
if not json_data:
raise OAuthExchangeCodeException("Got non-JSON response for code exchange")
if "error" in json_data:
raise OAuthExchangeCodeException(json_data.get("error_description", json_data["error"]))
return json_data
