def run_npm(npm_output_file_path, npm_args):
"""
Run an npm command
Parameters
----------
npm_output_file_path:
String
npm_args:
Commandline arguments to npm
"""
try:
with open(npm_output_file_path, 'w',
encoding='utf-8') as npm_output_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, npm_output_file])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stderr, npm_output_file])
sh.npm( # pylint: disable=no-member
npm_args,
_out=out_callback,
_err=err_callback)
except sh.ErrorReturnCode as error:
raise StepRunnerException(f"Error running npm. {error}") from error
def run_tox(tox_output_file_path, tox_args):
"""
Run a tox command
Paramters
---------
tox_output_file_path:
String
tox_args:
Commandline arguments to tox
"""
try:
with open(tox_output_file_path, 'w', encoding='utf-8') as tox_output_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
tox_output_file
])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stderr,
tox_output_file
])
sh.tox( # pylint: disable=no-member
tox_args,
_out=out_callback,
_err=err_callback
)
except sh.ErrorReturnCode as error:
raise StepRunnerException(
f"Error running tox. {error}"
) from error
def _argocd_app_wait_for_health(argocd_app_name, argocd_timeout_seconds):
"""Waits for ArgoCD Application to reach Healthy state.
Parameters
----------
argocd_app_name : str
Name of ArgoCD Application to wait for Healthy state of.
argocd_timeout_seconds : int
Number of sections to wait before timing out waiting for Healthy state.
Raises
------
StepRunnerException
If error (including timeout) waiting for existing ArgoCD Application Healthy state.
If ArgoCD Application transitions from Healthy to Degraded while waiting for Healthy
state.
"""
for wait_for_health_retry in range(
ArgoCDGeneric.MAX_ATTEMPT_TO_WAIT_FOR_ARGOCD_OP_RETRIES):
argocd_output_buff = StringIO()
try:
print(
f"Wait for Healthy ArgoCD Application ({argocd_app_name}")
out_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, argocd_output_buff])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stderr, argocd_output_buff])
sh.argocd.app.wait( # pylint: disable=no-member
argocd_app_name,
'--health',
'--timeout',
argocd_timeout_seconds,
_out=out_callback,
_err=err_callback)
break
except sh.ErrorReturnCode as error:
# if error waiting for Healthy state because entered Degraded state
# while waiting for Healthy state
# try again to wait for Healthy state assuming that on next attempt the
# new degradation of Health will resolve itself.
#
# NOTE: this can happen based on bad timing if for instance an
# HorizontalPodAutoscaller doesn't enter Degraded state until after we are
# already waiting for the ArgoCD Application to enter Healthy state,
# but then the HorizontalPodAutoscaller will, after a time, become Healthy.
if re.match(
ArgoCDGeneric.
ARGOCD_HEALTH_STATE_TRANSITIONED_FROM_HEALTHY_TO_DEGRADED,
argocd_output_buff.getvalue()):
print(
f"ArgoCD Application ({argocd_app_name}) entered Degraded state"
" while waiting for it to enter Healthy state."
f" Try ({wait_for_health_retry} out of"
f" {ArgoCDGeneric.MAX_ATTEMPT_TO_WAIT_FOR_ARGOCD_OP_RETRIES}) again to"
" wait for Healthy state.")
else:
raise StepRunnerException(
f"Error waiting for Healthy ArgoCD Application ({argocd_app_name}): {error}"
) from error
def upload_to_rekor(self, rekor_server, extra_data_file,
signer_pgp_public_key_path,
signer_pgp_private_key_user):
"""TODO: function will be refactored in next release. will doc then.
"""
rekor_entry = Rekor.create_rekor_entry(
signer_pgp_public_key_path=signer_pgp_public_key_path,
signer_pgp_private_key_user=signer_pgp_private_key_user,
extra_data_file=extra_data_file)
rekor_entry_path = self.write_working_file(filename='entry.json',
contents=bytes(
json.dumps(rekor_entry),
'utf-8'))
rekor_upload_stdout_result = StringIO()
rekor_upload_stdout_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, rekor_upload_stdout_result])
rekor = sh.rekor( # pylint: disable=no-member
'upload',
'--rekor_server',
rekor_server,
'--entry',
rekor_entry_path,
_out=rekor_upload_stdout_callback,
_err_to_out=True,
_tee='out')
rekor_uuid = str(rekor).split('/')[-1].strip(' \n')
return rekor_entry, rekor_uuid
def test_one_stream(self):
stream_one = StringIO()
sh_redirect_to_multiple_streams_fn_callback = \
create_sh_redirect_to_multiple_streams_fn_callback([
stream_one
])
sh_redirect_to_multiple_streams_fn_callback('data1')
self.assertEqual('data1', stream_one.getvalue())
def __curl_file(
container_image_signature_file_path,
container_image_signature_name,
signature_server_url,
signature_server_username,
signature_server_password
):
"""Sends the signature file
Raises
------
StepRunnerException
If error pushing image signature.
"""
# remove any trailing / from url
signature_server_url = re.sub(r'/$', '', signature_server_url)
container_image_signature_url = f"{signature_server_url}/{container_image_signature_name}"
# calculate hashes
with open(container_image_signature_file_path, 'rb') as container_image_signature_file:
container_image_signature_file_contents = container_image_signature_file.read()
signature_file_md5 = hashlib.md5(container_image_signature_file_contents).hexdigest()
signature_file_sha1 = hashlib.sha1(container_image_signature_file_contents).hexdigest()
try:
stdout_result = StringIO()
stdout_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
stdout_result
])
# -s: Silent
# -S: Show error
# -f: Don't print out failure document
# -v: Verbose
sh.curl( # pylint: disable=no-member
'-sSfv',
'-X', 'PUT',
'--header', f'X-Checksum-Sha1:{signature_file_sha1}',
'--header', f'X-Checksum-MD5:{signature_file_md5}',
'--user', f"{signature_server_username}:{signature_server_password}",
'--upload-file', container_image_signature_file_path,
container_image_signature_url,
_out=stdout_callback,
_err_to_out=True,
_tee='out'
)
except sh.ErrorReturnCode as error:
raise StepRunnerException(
f"Error pushing signature file to signature server using curl: {error}"
) from error
return container_image_signature_url, signature_file_md5, signature_file_sha1
def create_container_from_image(
image_address,
repository_type='container-storage:'
):
"""Import a container image using buildah form a TAR file.
Parameters
----------
image_address : str
Image tag to create a container from.
ex:
* localhost/my-app:latest
* quay.io/my-org/my-app:latest
* docker-archive:/local/path/to/my-app-container-image.tar
container_name : str
name for the working container.
repository_type : str
The type of repository to mount the given image tag from.
See https://github.com/containers/skopeo for details on different repository types.
Returns
-------
str
Name of the imported container.
Raises
------
RuntimeError
If error importing image.
"""
container_name = None
try:
buildah_from_out_buff = StringIO()
buildah_from_out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
buildah_from_out_buff
])
sh.buildah( # pylint: disable=no-member
'from',
f"{repository_type}{image_address}",
_out=buildah_from_out_callback,
_err=sys.stderr,
_tee='err'
)
container_name = buildah_from_out_buff.getvalue().rstrip()
except sh.ErrorReturnCode as error:
raise RuntimeError(
f'Error creating container from image ({image_address}): {error}'
) from error
return container_name
def run_npm(npm_output_file_path, npm_args, npm_envs=None):
"""
Run an npm command
Parameters
----------
npm_output_file_path:
String
npm_args:
Commandline arguments to npm
npm_envs:
Dictionary representing additional environment variables
"""
try:
with open(npm_output_file_path, 'w',
encoding='utf-8') as npm_output_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, npm_output_file])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stderr, npm_output_file])
if npm_envs:
new_env = os.environ.copy()
new_env.update(npm_envs)
sh.npm( # pylint: disable=no-member
npm_args,
_env=new_env,
_out=out_callback,
_err=err_callback)
else:
sh.npm( # pylint: disable=no-member
npm_args,
_out=out_callback,
_err=err_callback)
except sh.ErrorReturnCode as error:
raise StepRunnerException(f"Error running npm. {error}") from error
def __audit_attestation(
self, # pylint: disable=no-self-use
workflow_attestation_file_path,
workflow_policy_file_path,
workflow_policy_query):
"""Method to run the opa shell command to evaluate a data file with a provided
query file and given query.
Parameters
----------
workflow_attestation_file_path
File path to the workflow attestation file
workflow_policy_file_path
File path to the workflow policy file
workflow_policy_query
Query that is run against the policy file
Returns
-------
string
Output of the opa shell command. This is more important
to output if the query fails.
int
Return code for opa shell command. A value of 0
means the query succeeeded whereas a value of
1 means the query failed.
"""
opa_attestation_stdout_result = StringIO()
opa_attestation_stdout_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, opa_attestation_stdout_result])
try:
sh.opa( # pylint: disable=no-member
'eval',
'--fail-defined',
'-d',
workflow_attestation_file_path,
'-i',
workflow_policy_file_path,
workflow_policy_query,
_out=opa_attestation_stdout_callback,
_err_to_out=True,
_tee='out')
except sh.ErrorReturnCode as error:
return f"Error evaluating query against data: {error}", 1
return 'Audit was successful', 0
def __verify_sig(
self,
signature_file_path,
artifact_file_path,
private_key_fingerprint,
):
# GPG_OUTPUT_REGEX = re.compile(r"using RSA key ([A-Za-z0-9]+).*(Good signature)", re.DOTALL)
GPG_OUTPUT_REGEX = re.compile(f"using RSA key {private_key_fingerprint}.*Good signature", re.DOTALL)
with TempDirectory() as temp_dir:
signature_file_path=Path(signature_file_path)
try:
stdout_result = StringIO()
stdout_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
stdout_result
])
sh.gpg(
'--verify', signature_file_path,
artifact_file_path,
_out=stdout_callback,
_err_to_out=True,
_tee='out'
)
verify_matches = re.findall(
GPG_OUTPUT_REGEX,
stdout_result.getvalue()
)
if len(verify_matches) < 1:
return False
except sh.ErrorReturnCode as error:
print(
f"Error verifying sig with gpg: {error}"
)
return False
# if here, then verification successful
return True
def mount_container(buildah_unshare_command, container_id):
"""Use buildah to mount a container.
Parameters
----------
buildah_unshare_command : sh.buildah.unshare.bake()
A baked sh.buildah.unshare command to use to run this command in the context off
so that this can be done "rootless".
container_id : str
ID of the container to mount.
Returns
-------
str
Absolute path to the mounted container.
Raises
------
RuntimeError
If error mounting the container.
"""
mount_path = None
try:
buildah_mount_out_buff = StringIO()
buildah_mount_out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
buildah_mount_out_buff
])
buildah_mount_command = buildah_unshare_command.bake("buildah", "mount")
buildah_mount_command(
container_id,
_out=buildah_mount_out_callback,
_err=sys.stderr,
_tee='err'
)
mount_path = buildah_mount_out_buff.getvalue().rstrip()
except sh.ErrorReturnCode as error:
raise RuntimeError(
f'Error mounting container ({container_id}): {error}'
) from error
return mount_path
def __import_pgp_key(pgp_private_key):
print("Import PGP private key to sign container image(s) with")
try:
# import the key
# NOTE: GPG is weird in that it sends "none error" output to stderr even on success...
# so merge the stderr into stdout
gpg_import_stdout_result = StringIO()
gpg_import_stdout_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, gpg_import_stdout_result])
sh.gpg( # pylint: disable=no-member
'--import',
'--fingerprint',
'--with-colons',
'--import-options=import-show',
_in=pgp_private_key,
_out=gpg_import_stdout_callback,
_err_to_out=True,
_tee='out')
# get the fingerprint of the imported key
#
# NOTE: if more then one match just using first one...
gpg_imported_pgp_private_key_fingerprints = re.findall(
PodmanSign.GPG_IMPORT_FINGER_PRINT_REGEX,
gpg_import_stdout_result.getvalue())
if len(gpg_imported_pgp_private_key_fingerprints) < 1:
raise StepRunnerException(
"Error getting PGP fingerprint for PGP key"
" to sign container image(s) with. See stdout and stderr for more info."
)
pgp_private_key_fingerprint = gpg_imported_pgp_private_key_fingerprints[
0]
print("Imported PGP private key to sign container image(s) with: "
f"fingerprint='{pgp_private_key_fingerprint}'")
except sh.ErrorReturnCode as error:
raise StepRunnerException(
f"Error importing pgp private key: {error}") from error
return pgp_private_key_fingerprint
def _upload_to_rekor(self, rekor_server, rekor_entry):
"""Method to upload a rekor entry to provided rekor server
Parameters
----------
rekor_server: str
URL to rekor server
signer_pgp_private_key_fingerprint: str
PGP fingerprint obtained from importing the private key
path_to_file: str
Path to file to be signed
artifact_to_sign_uri: str
URI where artifact was pulled from
Returns
-------
str:
Returns rekor uuid returned from upload command
"""
rekor_entry_path = self.write_working_file(filename='entry.json',
contents=bytes(
json.dumps(rekor_entry),
'utf-8'))
rekor_upload_stdout_result = StringIO()
rekor_upload_stdout_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, rekor_upload_stdout_result])
rekor = sh.rekor( # pylint: disable=no-member
'upload',
'--rekor_server',
rekor_server,
'--entry',
rekor_entry_path,
_out=rekor_upload_stdout_callback,
_err_to_out=True,
_tee='out')
rekor_uuid = str(rekor).rsplit('/', maxsplit=1)[-1].strip(' \n')
return rekor_uuid
def import_pgp_key(pgp_private_key):
"""Imports a PGP key.
Parameters
----------
pgp_private_key : str
PGP key to import.
Returns
-------
str
Fingerprint of the imported PGP key.
Raises
------
RuntimeError
If error getting PGP fingerprint for imported PGP key
If error importing PGP key.
"""
# Example input to match on:
# sec:-:3072:1:CF4AC14A3D109637:1601483310:1664555310::-:::scESC::::::23::0:
# fpr:::::::::DD7208BA0A6359F65B906B29CF4AC14A3D109637:
# grp:::::::::A483EE079EC1D58A954E3AAF3BCC61EDD7596BF0:
gpg_regex = re.compile(r"^fpr:+([^:]+):$", re.MULTILINE)
print("Import PGP private key to sign artifacts with")
try:
# import the key
# NOTE: GPG is weird in that it sends "none error" output to stderr even on success...
# so merge the stderr into stdout
gpg_import_stdout_result = StringIO()
gpg_import_stdout_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
gpg_import_stdout_result
])
sh.gpg( # pylint: disable=no-member
'--import',
'--fingerprint',
'--with-colons',
'--import-options=import-show',
_in=pgp_private_key,
_out=gpg_import_stdout_callback,
_err_to_out=True,
_tee='out'
)
# get the fingerprint of the imported key
#
# NOTE: if more then one match just using first one...
gpg_imported_pgp_private_key_fingerprints = re.findall(
gpg_regex,
gpg_import_stdout_result.getvalue()
)
if len(gpg_imported_pgp_private_key_fingerprints) < 1:
raise RuntimeError(
"Error getting PGP fingerprint for PGP key"
" to sign container image(s) with. See stdout and stderr for more info."
)
pgp_private_key_fingerprint = gpg_imported_pgp_private_key_fingerprints[0]
print(
"Imported PGP private key to sign artifacts with: "
f"fingerprint='{pgp_private_key_fingerprint}'"
)
except sh.ErrorReturnCode as error:
raise RuntimeError(
f"Error importing pgp private key: {error}"
) from error
return pgp_private_key_fingerprint
def _run_step(self): # pylint: disable=too-many-locals
"""Runs the step implemented by this StepImplementer.
Returns
-------
StepResult
Object containing the dictionary results of this step.
"""
step_result = StepResult.from_step_implementer(self)
pom_file = self.get_value('pom-file')
artifact_extensions = self.get_value('artifact-extensions')
artifact_parent_dir = self.get_value('artifact-parent-dir')
if not os.path.exists(pom_file):
step_result.success = False
step_result.message = f'Given pom file does not exist: {pom_file}'
return step_result
settings_file = self._generate_maven_settings()
mvn_output_file_path = self.write_working_file('mvn_test_output.txt')
try:
with open(mvn_output_file_path, 'w') as mvn_output_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, mvn_output_file])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stderr, mvn_output_file])
sh.mvn( # pylint: disable=no-member
'clean',
'install',
'-f',
pom_file,
'-s',
settings_file,
_out=out_callback,
_err=err_callback)
except sh.ErrorReturnCode as error:
step_result.success = False
step_result.message = "Package failures. See 'maven-output' report artifacts " \
f"for details: {error}"
return step_result
finally:
step_result.add_artifact(
description=
"Standard out and standard error from 'mvn install'.",
name='maven-output',
value=mvn_output_file_path)
# find the artifacts
artifact_file_names = []
artifact_parent_dir_full_path = \
os.listdir(os.path.join(
os.path.dirname(os.path.abspath(pom_file)),
artifact_parent_dir))
for filename in artifact_parent_dir_full_path:
if any(filename.endswith(ext) for ext in artifact_extensions):
artifact_file_names.append(filename)
# error if we find more then one artifact
# see https://projects.engineering.redhat.com/browse/NAPSSPO-546
if len(artifact_file_names) > 1:
step_result.success = False
step_result.message = 'pom resulted in multiple artifacts with expected artifact ' \
f'extensions ({artifact_extensions}), this is unsupported'
return step_result
if len(artifact_file_names) < 1:
step_result.success = False
step_result.message = 'pom resulted in 0 with expected artifact extensions ' \
f'({artifact_extensions}), this is unsupported'
return step_result
artifact_id = get_xml_element(pom_file, 'artifactId').text
group_id = get_xml_element(pom_file, 'groupId').text
try:
package_type = get_xml_element(pom_file, 'package').text
except ValueError:
package_type = 'jar'
package_artifacts = {
'path':
os.path.join(os.path.dirname(os.path.abspath(pom_file)),
artifact_parent_dir, artifact_file_names[0]),
'artifact-id':
artifact_id,
'group-id':
group_id,
'package-type':
package_type,
'pom-path':
pom_file
}
# Currently, package returns ONE 'artifact', eg: one war file
# However, in the future, an ARRAY could be returned, eg: several jar files
step_result.add_artifact(name='package-artifacts',
value=[package_artifacts])
return step_result
def _run_step(self): # pylint: disable=too-many-locals,too-many-statements
"""Runs the step implemented by this StepImplementer.
Returns
-------
StepResult
Object containing the dictionary results of this step.
"""
step_result = StepResult.from_step_implementer(self)
settings_file = self._generate_maven_settings()
pom_file = self.get_value('pom-file')
fail_on_no_tests = self.get_value('fail-on-no-tests')
selenium_hub_url = self.get_value('selenium-hub-url')
deployed_host_urls = ConfigValue.convert_leaves_to_values(
self.get_value('deployed-host-urls')
)
uat_maven_profile = self.get_value('uat-maven-profile')
tls_verify = self.get_value('tls-verify')
# NOTE:
# at some point may need to do smarter logic if a deployable has more then one deployed
# host URL to do UAT against all of them, but for now, use first one as target of UAT
if isinstance(deployed_host_urls, list):
target_base_url = deployed_host_urls[0]
if len(deployed_host_urls) > 1:
step_result.message = \
f"Given more then one deployed host URL ({deployed_host_urls})," \
f" targeting first one ({target_base_url}) for user acceptance test (UAT)."
print(step_result.message)
elif deployed_host_urls:
target_base_url = deployed_host_urls
else:
target_base_url = self.get_value('target-host-url')
# ensure surefire plugin enabled
maven_surefire_plugin = self._get_effective_pom_element(
element_path=MavenGeneric.SUREFIRE_PLUGIN_XML_ELEMENT_PATH
)
if maven_surefire_plugin is None:
step_result.success = False
step_result.message = 'Unit test dependency "maven-surefire-plugin" ' \
f'missing from effective pom ({self._get_effective_pom()}).'
return step_result
# get surefire test results dir
reports_dir = self._get_effective_pom_element(
element_path=MavenGeneric.SUREFIRE_PLUGIN_REPORTS_DIR_XML_ELEMENT_PATH
)
if reports_dir is not None:
test_results_dir = reports_dir.text
else:
test_results_dir = os.path.join(
os.path.dirname(os.path.abspath(pom_file)),
MavenGeneric.DEFAULT_SUREFIRE_PLUGIN_REPORTS_DIR
)
mvn_additional_options = []
if not tls_verify:
mvn_additional_options += [
'-Dmaven.wagon.http.ssl.insecure=true',
'-Dmaven.wagon.http.ssl.allowall=true',
'-Dmaven.wagon.http.ssl.ignore.validity.dates=true',
]
cucumber_html_report_path = os.path.join(self.work_dir_path, 'cucumber.html')
cucumber_json_report_path = os.path.join(self.work_dir_path, 'cucumber.json')
mvn_output_file_path = self.write_working_file('mvn_test_output.txt')
try:
with open(mvn_output_file_path, 'w') as mvn_output_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
mvn_output_file
])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stderr,
mvn_output_file
])
sh.mvn( # pylint: disable=no-member
'clean',
'test',
f'-P{uat_maven_profile}',
f'-Dselenium.hub.url={selenium_hub_url}',
f'-Dtarget.base.url={target_base_url}',
f'-Dcucumber.plugin=' \
f'html:{cucumber_html_report_path},' \
f'json:{cucumber_json_report_path}',
'-f', pom_file,
'-s', settings_file,
*mvn_additional_options,
_out=out_callback,
_err=err_callback
)
if not os.path.isdir(test_results_dir) or len(os.listdir(test_results_dir)) == 0:
if fail_on_no_tests:
step_result.message = "No user acceptance tests defined" \
f" using maven profile ({uat_maven_profile})."
step_result.success = False
else:
step_result.message = "No user acceptance tests defined" \
f" using maven profile ({uat_maven_profile})," \
" but 'fail-on-no-tests' is False."
except sh.ErrorReturnCode:
step_result.message = "User acceptance test failures. See 'maven-output'" \
", 'surefire-reports', 'cucumber-report-html', and 'cucumber-report-json'" \
" report artifacts for details."
step_result.success = False
step_result.add_artifact(
description=f"Standard out and standard error by 'mvn -P{uat_maven_profile} test'.",
name='maven-output',
value=mvn_output_file_path
)
step_result.add_artifact(
description=f"Surefire reports generated by 'mvn -P{uat_maven_profile} test'.",
name='surefire-reports',
value=test_results_dir
)
step_result.add_artifact(
description=f"Cucumber (HTML) report generated by 'mvn -P{uat_maven_profile} test'.",
name='cucumber-report-html',
value=cucumber_html_report_path
)
step_result.add_artifact(
description=f"Cucumber (JSON) report generated by 'mvn -P{uat_maven_profile} test'.",
name='cucumber-report-json',
value=cucumber_json_report_path
)
return step_result
def _run_step(self): # pylint: disable=too-many-locals
"""Runs the step implemented by this StepImplementer.
Returns
-------
StepResult
Object containing the dictionary results of this step.
"""
step_result = StepResult.from_step_implementer(self)
# Get config items
maven_push_artifact_repo_id = self.get_value(
'maven-push-artifact-repo-id')
maven_push_artifact_repo_url = self.get_value(
'maven-push-artifact-repo-url')
version = self.get_value('version')
package_artifacts = self.get_value('package-artifacts')
tls_verify = self.get_value('tls-verify')
# disable tls verification
mvn_additional_options = []
if not tls_verify:
mvn_additional_options += [
'-Dmaven.wagon.http.ssl.insecure=true',
'-Dmaven.wagon.http.ssl.allowall=true',
'-Dmaven.wagon.http.ssl.ignore.validity.dates=true',
]
# Create settings.xml
settings_file = self._generate_maven_settings()
# push the artifacts
push_artifacts = []
mvn_output_file_path = self.write_working_file('mvn_test_output.txt')
try:
for package in package_artifacts:
artifact_path = package['path']
group_id = package['group-id']
artifact_id = package['artifact-id']
package_type = package['package-type']
# push the artifact
with open(mvn_output_file_path, 'a') as mvn_output_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, mvn_output_file])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stderr, mvn_output_file])
sh.mvn( # pylint: disable=no-member
'deploy:deploy-file',
'-Dversion=' + version,
'-Dfile=' + artifact_path,
'-DgroupId=' + group_id,
'-DartifactId=' + artifact_id,
'-Dpackaging=' + package_type,
'-Durl=' + maven_push_artifact_repo_url,
'-DrepositoryId=' + maven_push_artifact_repo_id,
'-s' + settings_file,
*mvn_additional_options,
_out=out_callback,
_err=err_callback)
# record the pushed artifact
push_artifacts.append({
'artifact-id': artifact_id,
'group-id': group_id,
'version': version,
'path': artifact_path,
'packaging': package_type,
})
except sh.ErrorReturnCode as error:
step_result.success = False
step_result.message = "Push artifacts failures. See 'maven-output' report artifacts " \
f"for details: {error}"
step_result.add_artifact(
description="Standard out and standard error from 'mvn install'.",
name='maven-output',
value=mvn_output_file_path)
step_result.add_artifact(name='push-artifacts', value=push_artifacts)
return step_result
def __run_oscap_scan( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
buildah_unshare_command,
oscap_eval_type,
oscap_input_file,
oscap_out_file_path,
oscap_xml_results_file_path,
oscap_html_report_path,
container_mount_path,
oscap_profile=None,
oscap_tailoring_file=None,
oscap_fetch_remote_resources=True):
"""Run an oscap scan in the context of a buildah unshare to run "rootless".
Parameters
----------
buildah_unshare_command : sh.buildah.unshare.bake()
A baked sh.buildah.unshare command to use to run this command in the context off
so that this can be done "rootless".
oscap_eval_type : str
The type of oscap eval to perform. Must be a valid oscap eval type.
EX: xccdf, oval
oscap_input_file : str
Path to rules file passed to the oscap command.
oscap_out_file_path : str
Path to write the stdout and stderr of running the oscap command to.
oscap_xml_results_file_path : str
Write the scan results into this file.
oscap_html_report_path : str
Write the human readable (HTML) report into this file.
container_mount_path : str
Path to the mounted container to scan.
oscap_tailoring_file : str
XCCF Tailoring file.
See:
- https://www.open-scap.org/security-policies/customization/
- https://www.open-scap.org/resources/documentation/customizing-scap-security-guide-for-your-use-case/ # pylint: disable=line-too-long
- https://static.open-scap.org/openscap-1.2/oscap_user_manual.html#_how_to_tailor_source_data_stream # pylint: disable=line-too-long
oscap_profile : str
OpenSCAP profile to evaluate. Must be a valid profile in the given oscap_input_file.
EX: if you perform an `oscap info oscap_input_file` the profile must be listed.
Returns
-------
oscap_eval_success : bool
True if oscap eval passed all rules
False if oscap eval failed any rules
oscap_eval_fails : str
If oscap_eval_success is True then indeterminate.
If oscap_eval_success is False then string of all of the failed rules.
Raises
------
StepRunnerException
If unexpected error running oscap scan.
"""
oscap_profile_flag = None
if oscap_profile is not None:
oscap_profile_flag = f"--profile={oscap_profile}"
oscap_fetch_remote_resources_flag = None
if isinstance(oscap_fetch_remote_resources, str):
oscap_fetch_remote_resources = strtobool(
oscap_fetch_remote_resources)
if oscap_fetch_remote_resources:
oscap_fetch_remote_resources_flag = "--fetch-remote-resources"
oscap_tailoring_file_flag = None
if oscap_tailoring_file is not None:
oscap_tailoring_file_flag = f"--tailoring-file={oscap_tailoring_file}"
oscap_eval_success = None
oscap_eval_out_buff = StringIO()
oscap_eval_out = ""
oscap_eval_fails = None
try:
oscap_chroot_command = buildah_unshare_command.bake("oscap-chroot")
with open(oscap_out_file_path, 'w') as oscap_out_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[oscap_eval_out_buff, oscap_out_file])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[oscap_eval_out_buff, oscap_out_file])
oscap_chroot_command(
container_mount_path,
oscap_eval_type,
'eval',
oscap_profile_flag,
oscap_fetch_remote_resources_flag,
oscap_tailoring_file_flag,
f'--results={oscap_xml_results_file_path}',
f'--report={oscap_html_report_path}',
oscap_input_file,
_out=out_callback,
_err=err_callback,
_tee='err')
oscap_eval_success = True
except sh.ErrorReturnCode_1 as error: # pylint: disable=no-member
oscap_eval_success = error
except sh.ErrorReturnCode_2 as error: # pylint: disable=no-member
# XCCDF: If there is at least one rule with either fail or unknown result,
# oscap-scan finishes with return code 2.
# OVAL: Never returned
#
# Source: https://www.systutorials.com/docs/linux/man/8-oscap/
if oscap_eval_type == 'xccdf':
oscap_eval_success = False
else:
oscap_eval_success = error
except sh.ErrorReturnCode as error:
oscap_eval_success = error
# get the oscap output
oscap_eval_out = oscap_eval_out_buff.getvalue()
# parse the oscap output
# NOTE: oscap is puts carrage returns (\r / ^M) in their output, remove them
oscap_eval_out = re.sub('\r', '', oscap_eval_out)
# print the oscap output no matter the results
print(oscap_eval_out)
# if unexpected error throw error
if isinstance(oscap_eval_success, Exception):
raise StepRunnerException(
f"Error running 'oscap {oscap_eval_type} eval': {oscap_eval_success} "
) from oscap_eval_success
# NOTE: oscap oval eval returns exit code 0 whether or not any rules failed
# need to search output to determine if there were any rule failures
if oscap_eval_type == 'oval' and oscap_eval_success:
oscap_eval_fails = ""
for match in OpenSCAPGeneric.OSCAP_OVAL_STDOUT_PATTERN.finditer(
oscap_eval_out):
# NOTE: need to do regex and not == because may contain xterm color chars
if OpenSCAPGeneric.OSCAP_OVAL_STDOUT_FAIL_PATTERN.search(
match.groupdict()['ruleresult']):
oscap_eval_fails += match.groupdict()['ruleblock']
oscap_eval_fails += "\n"
oscap_eval_success = False
# if failed xccdf eval then parse out the fails
if oscap_eval_type == 'xccdf' and not oscap_eval_success:
oscap_eval_fails = ""
for match in OpenSCAPGeneric.OSCAP_XCCDF_STDOUT_PATTERN.finditer(
oscap_eval_out):
# NOTE: need to do regex and not == because may contain xterm color chars
if re.search(r'fail', match.groupdict()['ruleresult']):
oscap_eval_fails += "\n"
oscap_eval_fails += match.groupdict()['ruleblock']
oscap_eval_fails += "\n"
return oscap_eval_success, oscap_eval_fails
def _run_step(self):
"""Runs the step implemented by this StepImplementer.
Returns
-------
StepResult
Object containing the dictionary results of this step.
"""
step_result = StepResult.from_step_implementer(self)
pom_file = self.get_value('pom-file')
fail_on_no_tests = self.get_value('fail-on-no-tests')
# ensure surefire plugin enabled
maven_surefire_plugin = self._get_effective_pom_element(
element_path=MavenGeneric.SUREFIRE_PLUGIN_XML_ELEMENT_PATH
)
if maven_surefire_plugin is None:
step_result.success = False
step_result.message = 'Unit test dependency "maven-surefire-plugin" ' \
f'missing from effective pom ({self._get_effective_pom()}).'
return step_result
# get surefire test results dir
reports_dir = self._get_effective_pom_element(
element_path=MavenGeneric.SUREFIRE_PLUGIN_REPORTS_DIR_XML_ELEMENT_PATH
)
if reports_dir is not None:
test_results_dir = reports_dir.text
else:
test_results_dir = os.path.join(
os.path.dirname(os.path.abspath(pom_file)),
MavenGeneric.DEFAULT_SUREFIRE_PLUGIN_REPORTS_DIR
)
settings_file = self._generate_maven_settings()
mvn_output_file_path = self.write_working_file('mvn_test_output.txt')
try:
with open(mvn_output_file_path, 'w') as mvn_output_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
mvn_output_file
])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stderr,
mvn_output_file
])
sh.mvn( # pylint: disable=no-member
'clean',
'test',
'-f', pom_file,
'-s', settings_file,
_out=out_callback,
_err=err_callback
)
if not os.path.isdir(test_results_dir) or len(os.listdir(test_results_dir)) == 0:
if fail_on_no_tests:
step_result.message = 'No unit tests defined.'
step_result.success = False
else:
step_result.message = "No unit tests defined, but 'fail-on-no-tests' is False."
except sh.ErrorReturnCode as error:
step_result.message = "Unit test failures. See 'maven-output'" \
f" and 'surefire-reports' report artifacts for details: {error}"
step_result.success = False
finally:
step_result.add_artifact(
description="Standard out and standard error from 'mvn test'.",
name='maven-output',
value=mvn_output_file_path
)
step_result.add_artifact(
description="Surefire reports generated from 'mvn test'.",
name='surefire-reports',
value=test_results_dir
)
return step_result
def _run_step(self):
"""Runs the step implemented by this StepImplementer.
Returns
-------
StepResult
Object containing the dictionary results of this step.
"""
step_result = StepResult.from_step_implementer(self)
# configlint-yml-path is required
configlint_yml_path = self.get_value('configlint-yml-path')
if not os.path.exists(configlint_yml_path):
step_result.success = False
step_result.message = 'File specified in ' \
f'configlint-yml-path not found: {configlint_yml_path}'
return step_result
# Required: rules and exists
rules_file = self.get_value('rules')
if not os.path.exists(rules_file):
step_result.success = False
step_result.message = f'File specified in rules not found: {rules_file}'
return step_result
configlint_results_file_path = self.write_working_file('configlint_results_file.txt')
try:
# run config-lint writing stdout and stderr to the standard streams
# as well as to a results file.
with open(configlint_results_file_path, 'w') as configlint_results_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
configlint_results_file
])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stderr,
configlint_results_file
])
sh.config_lint( # pylint: disable=no-member
"-verbose",
"-debug",
"-rules",
rules_file,
configlint_yml_path,
_encoding='UTF-8',
_out=out_callback,
_err=err_callback,
_tee='err'
)
except sh.ErrorReturnCode_255: # pylint: disable=no-member
# NOTE: expected failure condition,
# aka, the config lint run, but found an issue
# stderr/stdout is captured in configlint_results_file_path
step_result.success = False
step_result.message = 'Failed config-lint scan.'
except sh.ErrorReturnCode:
# NOTE: un-expected failure condition
# aka, the config lint failed to run for some reason
# stderr/stdout is captured in configlint_results_file_path
step_result.success = False
step_result.message = 'Unexpected Error invoking config-lint.'
step_result.add_artifact(
name='configlint-result-set',
value=configlint_results_file_path
)
step_result.add_artifact(
name='configlint-yml-path',
value=configlint_yml_path
)
return step_result
def _argocd_app_sync(
argocd_app_name,
argocd_sync_timeout_seconds,
argocd_sync_retry_limit,
argocd_sync_prune=True
): # pylint: disable=line-too-long
# add any additional flags
argocd_sync_additional_flags = []
if argocd_sync_prune:
argocd_sync_additional_flags.append('--prune')
for wait_for_op_retry in range(
ArgoCDGeneric.MAX_ATTEMPT_TO_WAIT_FOR_ARGOCD_OP_RETRIES):
# wait for any existing operations before requesting new synchronization
#
# NOTE: attempted work around for 'FailedPrecondition desc = another operation is
# already in progress' error
# SEE: https://github.com/argoproj/argo-cd/issues/4505
ArgoCDGeneric._argocd_app_wait_for_operation(
argocd_app_name=argocd_app_name,
argocd_timeout_seconds=argocd_sync_timeout_seconds)
# sync app
argocd_output_buff = StringIO()
try:
print(
f"Request synchronization of ArgoCD app ({argocd_app_name})."
)
out_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stdout, argocd_output_buff])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback(
[sys.stderr, argocd_output_buff])
sh.argocd.app.sync( # pylint: disable=no-member
*argocd_sync_additional_flags,
'--timeout',
argocd_sync_timeout_seconds,
'--retry-limit',
argocd_sync_retry_limit,
argocd_app_name,
_out=out_callback,
_err=err_callback)
break
except sh.ErrorReturnCode as error:
# if error syncing because of in progress op
# try again to wait for in progress op and do sync again
#
# NOTE: this can happen if we do the wait for op, and then an op starts and then
# we try to do a sync
#
# SEE: https://github.com/argoproj/argo-cd/issues/4505
if re.match(ArgoCDGeneric.ARGOCD_OP_IN_PROGRESS_REGEX,
argocd_output_buff.getvalue()):
print(
f"ArgoCD Application ({argocd_app_name}) has an existing operation"
" that started after we already waited for existing operations but"
" before we tried to do a sync."
f" Try ({wait_for_op_retry} out of"
f" {ArgoCDGeneric.MAX_ATTEMPT_TO_WAIT_FOR_ARGOCD_OP_RETRIES}) again to"
" wait for the operation")
continue
if not argocd_sync_prune:
prune_warning = ". Sync 'prune' option is disabled." \
" If sync error (see logs) was due to resource(s) that need to be pruned," \
" and the pruneable resources are intentionally there then see the ArgoCD" \
" documentation for instructions for argo to ignore the resource(s)." \
" See: https://argoproj.github.io/argo-cd/user-guide/sync-options/#no-prune-resources" \
" and https://argoproj.github.io/argo-cd/user-guide/compare-options/#ignoring-resources-that-are-extraneous"
else:
prune_warning = ""
raise StepRunnerException(
f"Error synchronization ArgoCD Application ({argocd_app_name})"
f"{prune_warning}: {error}") from error
# wait for sync to finish
ArgoCDGeneric._argocd_app_wait_for_health(
argocd_app_name=argocd_app_name,
argocd_timeout_seconds=argocd_sync_timeout_seconds)
def run_maven( #pylint: disable=too-many-arguments
mvn_output_file_path,
settings_file,
pom_file,
phases_and_goals,
tls_verify=True,
additional_arguments=None,
profiles=None,
no_transfer_progress=True
):
"""Runs maven using the given configuration.
Parameters
----------
mvn_output_file_path : str
Path to file containing the maven stdout and stderr output.
phases_and_goals : [str]
List of maven phases and/or goals to execute.
additional_arguments : [str]
List of additional arguments to use.
pom_file : str (path)
pom used when executing maven.
tls_verify : boolean
Disables TLS Verification if set to False
profiles : [str]
List of maven profiles to use.
no_transfer_progress : boolean
`True` to suppress the transfer progress of packages maven downloads.
`False` to have the transfer progress printed.\
See https://maven.apache.org/ref/current/maven-embedder/cli.html
settings_file : str (path)
Maven settings file to use.
Returns
-------
str
Standard Out and Standard Error from running Maven.
Raises
------
StepRunnerException
If maven returns a none 0 exit code.
"""
if not isinstance(phases_and_goals, list):
phases_and_goals = [phases_and_goals]
# create profile argument
profiles_arguments = ""
if profiles:
profiles_arguments = ['-P', f"{','.join(profiles)}"]
# create no transfer progress argument
no_transfer_progress_argument = None
if no_transfer_progress:
no_transfer_progress_argument = '--no-transfer-progress'
# create tls arguments
tls_arguments = []
if not tls_verify:
tls_arguments += [
'-Dmaven.wagon.http.ssl.insecure=true',
'-Dmaven.wagon.http.ssl.allowall=true',
'-Dmaven.wagon.http.ssl.ignore.validity.dates=true',
]
if not additional_arguments:
additional_arguments = []
# run maven
try:
with open(mvn_output_file_path, 'w', encoding='utf-8') as mvn_output_file:
out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stdout,
mvn_output_file
])
err_callback = create_sh_redirect_to_multiple_streams_fn_callback([
sys.stderr,
mvn_output_file
])
sh.mvn( # pylint: disable=no-member
*phases_and_goals,
'-f', pom_file,
'-s', settings_file,
*profiles_arguments,
no_transfer_progress_argument,
*tls_arguments,
*additional_arguments,
_out=out_callback,
_err=err_callback
)
except sh.ErrorReturnCode as error:
raise StepRunnerException(
f"Error running maven. {error}"
) from error
