def test_is_docker_image_latest_tag_without_tag():
"""
Given
- A latest docker image has an empty tag
When
- The most updated docker image in docker-hub is '1.0.3'
Then
- If the docker image is numeric and the most update one, it is Valid
- If the docker image is not numeric and labeled "latest", it is Invalid
"""
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.docker_image_latest_tag = ''
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.is_latest_tag = True
docker_image_validator.docker_image_tag = '1.0.2'
docker_image_validator.is_valid = True
assert docker_image_validator.is_docker_image_latest_tag() is False
assert docker_image_validator.is_latest_tag is False
assert docker_image_validator.is_docker_image_valid() is False
def test_get_latest_commit_fails(self, mocker, integration,
requests_mock, mock_results,
mocked_status, expected):
"""
Given:
- A project with no successful commit in master in Iron Bank.
- A project that does not exists in Iron Bank.
When:
Validating docker image of Iron Bank pack.
Then:
Validates we show the correct error.
"""
api_url = 'https://repo1.dso.mil/api/v4/projects/dsop%2Fopensource%2Fpalo-alto-networks%2Ftest%2Ftest_project/pipelines'
requests_mock.get(api_url,
status_code=mocked_status,
json=mock_results)
DockerImageValidator.is_iron_bank = True
docker_image_name = 'test/test_project:1.0.2'
DockerImageValidator.yml_docker_image = docker_image_name
DockerImageValidator.file_path = integration.yml.path
with pytest.raises(Exception) as e:
DockerImageValidator._get_latest_commit(
api_url, docker_image_name)
assert str(e.value) == expected
def test_is_docker_image_latest_tag_with_numeric_but_not_most_updated():
"""
Given
- A docker image with '1.0.2' as tag
When
- The most updated docker image in docker-hub is '1.0.3'
Then
- If the docker image is numeric and the most update one, it is Valid
- If the docker image is not numeric and labeled "latest", it is Invalid
- If the docker image is not the most updated one it is still valid
(however, a warning will be printed)
"""
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.docker_image_latest_tag = '1.0.3'
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.is_latest_tag = True
docker_image_validator.docker_image_tag = '1.0.2'
docker_image_validator.is_valid = True
assert docker_image_validator.is_docker_image_latest_tag() is True
assert docker_image_validator.is_latest_tag is True
assert docker_image_validator.is_docker_image_valid() is True
def test_get_manifest_from_commit(self, integration, requests_mock):
"""
Given:
An example existing commit with successful commits in master with Manifest file in Iron Bank.
When:
Validating docker image of Iron Bank pack.
Then:
Validates we send the correct request to Iron Bank.
"""
manifest_url = 'https://repo1.dso.mil/api/v4/projects/dsop%2Fopensource%2Fpalo-alto-networks%2Ftest%2F'\
'test_project/repository/files/hardening_manifest.yaml/raw'
request_mock = requests_mock.get(
manifest_url,
text=
"""apiVersion: v1\nname: opensource/palo-alto-networks/test/test_project\ntags:\n- 1.0.1.23955\n"""
)
DockerImageValidator.file_path = integration.yml.path
DockerImageValidator.is_iron_bank = True
docker_image_name = 'test/test_project:1.0.2'
DockerImageValidator.yml_docker_image = docker_image_name
DockerImageValidator._get_manifest_from_commit(
manifest_url, 'sha1')
assert request_mock.last_request.query == 'ref=sha1'
assert request_mock.last_request.path == '/api/v4/projects/dsop%2fopensource%2fpalo-alto-networks%2f' \
'test%2ftest_project/repository/files/hardening_manifest.yaml/raw'
def test_get_manifest_from_commit_fails(self, mocker, integration,
requests_mock, mock_results,
mocked_status, expected):
"""
Given:
- A project without manifest file in master in Iron Bank.
When:
Validating docker image of Iron Bank pack.
Then:
Validates we show the correct error.
"""
manifest_url = 'https://repo1.dso.mil/api/v4/projects/dsop%2Fopensource%2Fpalo-alto-networks%2Ftest%2F' \
'test_project/repository/files/hardening_manifest.yaml/raw'
requests_mock.get(manifest_url,
status_code=mocked_status,
text=mock_results)
DockerImageValidator.is_iron_bank = True
docker_image_name = 'test/test_project:1.0.2'
DockerImageValidator.yml_docker_image = docker_image_name
DockerImageValidator.file_path = integration.yml.path
with pytest.raises(Exception) as e:
DockerImageValidator._get_manifest_from_commit(
manifest_url, 'sha1')
assert str(e.value) == expected
def is_docker_image_valid(self):
# type: () -> bool
docker_image_validator = DockerImageValidator(self.file_path,
is_modified_file=True,
is_integration=False)
if docker_image_validator.is_docker_image_valid():
return True
return False
def test_parse_docker_image():
assert 'demisto/python', '1.3-alpine' == DockerImageValidator.parse_docker_image(
'demisto/python:1.3-alpine')
assert 'demisto/slack', '1.2.3.4' == DockerImageValidator.parse_docker_image(
'demisto/slack:1.2.3.4')
assert 'demisto/python', '' == DockerImageValidator.parse_docker_image(
'demisto/python/1.2.3.4')
assert ('',
'') == DockerImageValidator.parse_docker_image('blah/blah:1.2.3.4')
def is_docker_image_valid(self):
# type: () -> bool
# dockers should not be checked when running on all files
if self.skip_docker_check:
return True
docker_image_validator = DockerImageValidator(self.file_path, is_modified_file=True, is_integration=False,
ignored_errors=self.ignored_errors,
print_as_warnings=self.print_as_warnings)
if docker_image_validator.is_docker_image_valid():
return True
return False
def is_docker_image_valid(self):
# type: () -> bool
# dockers should not be checked when running on all files
# dockers should not be checked when running on ApiModules scripts
if self.skip_docker_check or API_MODULES_PACK in self.file_path:
return True
docker_image_validator = DockerImageValidator(self.file_path, is_modified_file=True, is_integration=False,
ignored_errors=self.ignored_errors,
print_as_warnings=self.print_as_warnings,
suppress_print=self.suppress_print)
if docker_image_validator.is_docker_image_valid():
return True
return False
def test_none_demisto_docker(docker, docker_tag, expected_output):
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.file_path = "PATH"
docker_image_validator.ignored_errors = {}
docker_image_validator.checked_files = set()
docker_image_validator.docker_image_latest_tag = '1.0.3'
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.suppress_print = False
assert docker_image_validator.get_docker_image_latest_tag(
docker_image_name=docker,
yml_docker_image='{}:{}'.format(docker,
docker_tag)) == expected_output
def is_docker_image_valid(self):
# type: () -> bool
# dockers should not be checked on master branch
if self.branch_name == 'master':
return True
docker_image_validator = DockerImageValidator(
self.file_path,
is_modified_file=True,
is_integration=False,
ignored_errors=self.ignored_errors,
print_as_warnings=self.print_as_warnings)
if docker_image_validator.is_docker_image_valid():
return True
return False
def update_docker_image_in_script(script_obj: dict, from_version: Optional[str] = None):
"""Update the docker image for the passed script object. Will ignore if this is a javascript
object or using default image (not set).
Args:
script_obj (dict): script object
"""
if script_obj.get('type') == TYPE_JS:
print_color('Skipping docker image update as this is a Javascript automation.', LOG_COLORS.YELLOW)
return
dockerimage = script_obj.get('dockerimage')
if not dockerimage: # default image -> nothing to do
print_color('Skipping docker image update as default docker image is being used.', LOG_COLORS.YELLOW)
return
image_name = dockerimage.split(':')[0]
latest_tag = DockerImageValidator.get_docker_image_latest_tag_request(image_name)
full_name = f'{image_name}:{latest_tag}'
if full_name != dockerimage:
print(f'Updating docker image to: {full_name}')
script_obj['dockerimage'] = full_name
if (not from_version) or server_version_compare('5.0.0', from_version):
# if this is a script that supports 4.5 and earlier. Make sure dockerimage45 is set
if not script_obj.get('dockerimage45'):
print(f'Setting dockerimage45 to previous image value: {dockerimage} for 4.5 and earlier support')
script_obj['dockerimage45'] = dockerimage
else:
print(f'Already using latest docker image: {dockerimage}. Nothing to update.')
def test_get_docker_image_latest_tag(image, mocker):
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.file_path = "PATH"
docker_image_validator.ignored_errors = []
docker_image_validator.docker_image_latest_tag = '1.0.3'
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.suppress_print = False
tag = docker_image_validator.get_docker_image_latest_tag(
docker_image_name='demisto/' + image, yml_docker_image='')
# current latest tag is 2.7.16.2728 or 3.7.2.2728 disable-secrets-detection
assert int(tag.split('.')[3]) >= 2728
def mock_docker_image_validator():
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.file_path = "PATH"
docker_image_validator.ignored_errors = {}
docker_image_validator.checked_files = set()
docker_image_validator.suppress_print = False
docker_image_validator.json_file_path = ''
return docker_image_validator
def test_get_docker_image_from_yml():
with patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
# Test integration case
docker_validator = DockerImageValidator(None, None, None)
docker_validator.yml_file = get_yaml(TEST_INTEGRATION_FILE)
docker_validator.is_integration = True
docker_validator.suppress_print = False
docker_image = docker_validator.get_docker_image_from_yml()
assert docker_image == "demisto/pyjwt:1.0"
# Test script case
docker_validator.yml_file = get_yaml(TEST_SCRIPT_FILE)
docker_validator.is_integration = False
docker_image = docker_validator.get_docker_image_from_yml()
assert docker_image == "demisto/stix2:1.0.0.204"
def test_get_latest_commit(self, integration, requests_mock):
"""
Given:
An example existing project with successful commits in master in Iron Bank.
When:
Validating docker image of Iron Bank pack.
Then:
Validates we extract correctly the commit id.
"""
api_url = 'https://repo1.dso.mil/api/v4/projects/dsop%2Fopensource%2Fpalo-alto-networks%2Ftest%2Ftest_project/pipelines'
requests_mock.get(
api_url,
json=[{
'id':
433333,
'project_id':
7070,
'sha':
'sha_1',
'ref':
'master',
'status':
'success',
'created_at':
'2021-08-19T09:18:35.547Z',
'updated_at':
'2021-08-19T09:38:21.743Z',
'web_url':
'https://repo1.dso.mil/dsop/opensource/palo-alto-networks/test/test_project/-/pipelines/433333'
}, {
'id':
432507,
'project_id':
7070,
'sha':
'sha_2',
'ref':
'master',
'status':
'success',
'created_at':
'2021-08-18T22:19:19.843Z',
'updated_at':
'2021-08-18T22:40:29.950Z',
'web_url':
'https://repo1.dso.mil/dsop/opensource/palo-alto-networks/test/test_project/-/pipelines/432507'
}])
DockerImageValidator.file_path = integration.yml.path
DockerImageValidator.is_iron_bank = True
docker_image_name = 'test/test_project:1.0.2'
DockerImageValidator.yml_docker_image = docker_image_name
res = DockerImageValidator._get_latest_commit(
api_url, docker_image_name)
assert 'sha_1' == res
def get_docker_image():
try:
latest_tag = DockerImageValidator.get_docker_image_latest_tag_request(
'demisto/python3')
docker_image = f'demisto/python3:{latest_tag}'
logger.debug(f'docker image set to: {docker_image}')
except Exception as e:
# set default docker image
docker_image = 'demisto/python3:3.9.1.14969'
logger.warning(
f'Failed getting latest docker image for demisto/python3: {e}')
return docker_image
def test_non_existing_docker(self, integration, capsys, requests_mock,
mocker):
docker_image = 'demisto/nonexistingdocker:1.4.0'
integration.yml.write_dict({
'script': {
'subtype': 'python3',
'type': 'python',
'dockerimage': docker_image
}
})
error, code = Errors.non_existing_docker(docker_image)
mocker.patch.object(DockerImageValidator,
'docker_auth',
return_value='auth')
requests_mock.get(
"https://hub.docker.com/v2/repositories/demisto/nonexistingdocker/tags",
json={'results': []})
validator = DockerImageValidator(integration.yml.path, True, True)
assert validator.is_docker_image_valid() is False
captured = capsys.readouterr()
assert validator.is_valid is False
assert error in captured.out
assert code in captured.out
def test_none_demisto_docker(docker, docker_tag, expected_output):
assert DockerImageValidator.get_docker_image_latest_tag(
docker, '{}:{}'.format(docker, docker_tag)) == expected_output
def mock_docker_image_validator():
with mock.patch.object(DockerImageValidator, '__init__', lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.file_path = "PATH"
docker_image_validator.ignored_errors = {}
docker_image_validator.checked_files = set()
docker_image_validator.suppress_print = False
docker_image_validator.json_file_path = ''
docker_image_validator.specific_validations = None
docker_image_validator.predefined_deprecated_ignored_errors = {}
docker_image_validator.predefined_by_support_ignored_errors = {}
return docker_image_validator
def test_is_docker_image_latest_tag_with_default_image():
"""
Given
- The default docker image - 'demisto/python:1.3-alpine'
When
- The most updated docker image in docker-hub is '1.0.3'
Then
- If the docker image is numeric and the most update one, it is Valid
- If the docker image is not numeric and labeled "latest", it is Invalid
"""
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.file_path = "PATH"
docker_image_validator.ignored_errors = {}
docker_image_validator.code_type = 'python'
docker_image_validator.checked_files = set()
docker_image_validator.docker_image_latest_tag = '1.0.3'
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.is_latest_tag = True
docker_image_validator.is_modified_file = False
docker_image_validator.docker_image_tag = '1.3-alpine'
docker_image_validator.is_valid = True
assert docker_image_validator.is_docker_image_latest_tag() is False
assert docker_image_validator.is_latest_tag is False
assert docker_image_validator.is_docker_image_valid() is False
def test_clear_non_numbered_tags(self, input_tags, output_tags):
assert sorted(output_tags) == sorted(
DockerImageValidator.clear_non_numbered_tags(input_tags))
def test_parse_www_auth(self, www_auth, expected):
assert expected == DockerImageValidator.parse_www_auth(www_auth)
def test_find_latest_tag_by_date(self):
tag = DockerImageValidator.find_latest_tag_by_date(MOCK_TAG_LIST)
assert tag == "1.0.0.2876"
def test_lexical_find_latest_tag(self):
tag_list = ["2.0.2000", "2.1.2700", "2.1.373", "latest"]
tag = DockerImageValidator.lexical_find_latest_tag(tag_list)
assert tag == "2.1.2700"
def test_is_docker_image_latest_tag_with_tag_labeled_latest():
"""
Given
- A docker image with "latest" as tag
When
- The most updated docker image in docker-hub is '1.0.3'
Then
- If the docker image is numeric and the most update one, it is Valid
- If the docker image is not numeric and labeled "latest", it is Invalid
"""
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.ignored_errors = {}
docker_image_validator.docker_image_latest_tag = 'latest'
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.file_path = "PATH"
docker_image_validator.is_latest_tag = True
docker_image_validator.is_valid = True
docker_image_validator.docker_image_tag = 'latest'
assert docker_image_validator.is_docker_image_latest_tag() is False
assert docker_image_validator.is_latest_tag is False
assert docker_image_validator.is_docker_image_valid() is False
def test_is_docker_image_latest_tag_with_numeric_but_not_most_updated():
"""
Given
- A docker image with '1.0.2' as tag
When
- The most updated docker image in docker-hub is '1.0.3'
Then
- If the docker image is numeric and the most update one, it is Valid
- If the docker image is not numeric and labeled "latest", it is Invalid
- If the docker image is not the most updated one it is invalid
"""
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.ignored_errors = {}
docker_image_validator.file_path = "path"
docker_image_validator.docker_image_latest_tag = '1.0.3'
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.code_type = 'python'
docker_image_validator.checked_files = set()
docker_image_validator.suppress_print = False
docker_image_validator.is_latest_tag = True
docker_image_validator.docker_image_tag = '1.0.2'
docker_image_validator.is_valid = True
assert docker_image_validator.is_docker_image_latest_tag() is False
assert docker_image_validator.is_latest_tag is False
assert docker_image_validator.is_docker_image_valid() is False
def test_parse_docker_image():
with mock.patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.file_path = "PATH"
docker_image_validator.ignored_errors = {}
docker_image_validator.checked_files = set()
docker_image_validator.docker_image_latest_tag = '1.0.3'
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.suppress_print = False
assert 'demisto/python', '1.3-alpine' == docker_image_validator.parse_docker_image(
docker_image='demisto/python:1.3-alpine')
assert 'demisto/slack', '1.2.3.4' == docker_image_validator.parse_docker_image(
docker_image='demisto/slack:1.2.3.4')
assert 'demisto/python', '' == docker_image_validator.parse_docker_image(
docker_image='demisto/python/1.2.3.4')
assert ('', '') == docker_image_validator.parse_docker_image(
docker_image='blah/blah:1.2.3.4')
def test_is_docker_image_latest_tag():
with patch.object(DockerImageValidator, '__init__',
lambda x, y, z, w: None):
docker_image_validator = DockerImageValidator(None, None, None)
docker_image_validator.yml_file = {}
docker_image_validator.docker_image_latest_tag = 'latest_tag'
docker_image_validator.docker_image_name = 'demisto/python'
docker_image_validator.from_version = '5.0.0'
# ===== Added File Tests =====
# default docker image
docker_image_validator.is_latest_tag = True
docker_image_validator.is_modified_file = False
docker_image_validator.docker_image_tag = '1.3-alpine'
assert docker_image_validator.is_docker_image_latest_tag() is False
# regular docker image, not latest tag
docker_image_validator.is_latest_tag = True
docker_image_validator.docker_image_tag = 'not_latest_tag'
assert docker_image_validator.is_docker_image_latest_tag() is False
# regular docker image, latest tag
docker_image_validator.is_latest_tag = True
docker_image_validator.docker_image_tag = 'latest_tag'
assert docker_image_validator.is_docker_image_latest_tag() is True
# ===== Modified File Tests =====
# from version 4.1.0
docker_image_validator.is_latest_tag = True
docker_image_validator.is_modified_file = True
docker_image_validator.from_version = '4.1.0'
assert docker_image_validator.is_docker_image_latest_tag() is True
# from version 5.0.0 - regular docker image, latest tag
docker_image_validator.is_latest_tag = True
docker_image_validator.from_version = '5.0.0'
assert docker_image_validator.is_docker_image_latest_tag() is True
# from version 5.0.0 - regular docker image, not latest tag
docker_image_validator.is_latest_tag = True
docker_image_validator.from_version = '5.0.0'
docker_image_validator.docker_image_tag = 'not_latest_tag'
assert docker_image_validator.is_docker_image_latest_tag() is False
# from version 5.0.0 - default docker image
docker_image_validator.is_latest_tag = True
docker_image_validator.docker_image_tag = '1.3-alpine'
assert docker_image_validator.is_docker_image_latest_tag() is True
def test_get_docker_image_latest_tag(image):
tag = DockerImageValidator.get_docker_image_latest_tag(
'demisto/' + image, '')
# current latest tag is 2.7.16.2728 or 3.7.2.2728 disable-secrets-detection
assert int(tag.split('.')[3]) >= 2728
