def delete_vm(options):
'''
Stop a VM
'''
cmd = []
if options.peer:
cmd.extend(['salt-call', 'publish.runner'])
else:
cmd.append('salt-run')
if options.lxc:
cmd.append('lxc.purge')
else:
cmd.append('cloud.destroy')
cmd.append(options.vm_name)
print('Running CMD: {0!r}'.format(' '.join(cmd)))
sys.stdout.flush()
proc = NonBlockingPopen(
' '.join(cmd),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
def win_cmd(command, **kwargs):
'''
Wrapper for commands to be run against Windows boxes
'''
try:
proc = NonBlockingPopen(
command,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
stream_stds=kwargs.get('display_ssh_output', True),
)
log.debug(
'Executing command(PID {0}): {1!r}'.format(
proc.pid, command
)
)
proc.poll_and_read_until_finish()
proc.communicate()
return proc.returncode
except Exception as err:
log.error(
'Failed to execute command {0!r}: {1}\n'.format(
command, err
),
exc_info=True
)
# Signal an error
return 1
def sync_minion(options):
if 'salt_minion_synced' not in options:
cmd = []
if options.peer:
cmd.extend(['salt-call', 'publish.runner'])
else:
cmd.append('salt')
cmd.extend([
build_minion_target(options),
'saltutil.sync_all'
])
print('Running CMD: {0!r}'.format(' '.join(cmd)))
sys.stdout.flush()
proc = NonBlockingPopen(
' '.join(cmd),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
sys.exit(proc.returncode)
setattr(options, 'salt_minion_synced', 'yes')
def download_unittest_reports(options):
'''
Download the generated unit test reports from minion
'''
print('Downloading remote unittest reports...')
sys.stdout.flush()
xml_reports_path = os.path.join(options.workspace, 'xml-test-reports')
if os.path.isdir(xml_reports_path):
shutil.rmtree(xml_reports_path)
if options.scp:
cmds = (
' '.join(build_scp_command(options,
'-r',
'root@{0}:/tmp/xml-unitests-output/*'.format(
get_minion_external_address(options)
),
os.path.join(options.workspace, 'xml-test-reports'))),
)
else:
os.makedirs(xml_reports_path)
cmds = (
'{0} {1} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/',
'{0} {1} cp.push /tmp/xml-test-reports.tar.gz',
'mv -f /var/cache/salt/master/minions/{2}/files/tmp/xml-test-reports.tar.gz {3} && '
'tar zxvf {3}/xml-test-reports.tar.gz -C {3}/xml-test-reports && '
'rm -f {3}/xml-test-reports.tar.gz'
)
for cmd in cmds:
cmd = cmd.format(
'salt-call publish.publish' if options.lxc else 'salt',
build_minion_target(options),
options.vm_name,
options.workspace
)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def download_remote_logs(options):
print('Downloading remote logs...')
sys.stdout.flush()
workspace = options.workspace
vm_name = options.download_remote_logs
for fname in ('salt-runtests.log', 'minion.log'):
if os.path.isfile(os.path.join(workspace, fname)):
os.unlink(os.path.join(workspace, fname))
if not options.remote_log_path:
options.remote_log_path = [
'/tmp/salt-runtests.log',
'/var/log/salt/minion'
]
cmds = []
for remote_log in options.remote_log_path:
cmds.extend([
'salt {{0}} archive.gzip {0}'.format(remote_log),
'salt {{0}} cp.push {0}.gz'.format(remote_log),
'gunzip /var/cache/salt/master/minions/{{1}}/files{0}.gz'.format(remote_log),
'mv /var/cache/salt/master/minions/{{1}}/files{0} {{2}}/{1}'.format(
remote_log,
'{0}{1}'.format(
os.path.basename(remote_log),
'' if remote_log.endswith('.log') else '.log'
)
)
])
for cmd in cmds:
cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def test_master_startup(self):
proc = NonBlockingPopen(
[
sys.executable,
self.get_script_path("master"),
"-c",
RUNTIME_VARS.TMP_CONF_DIR,
"-l",
"info",
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out = b""
err = b""
# Testing this should never be longer than 1 minute
max_time = time.time() + 60
try:
while True:
if time.time() > max_time:
assert False, "Max timeout ocurred"
time.sleep(0.5)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if b"DeprecationWarning: object() takes no parameters" in out:
self.fail(
"'DeprecationWarning: object() takes no parameters' was seen in"
" output"
)
if b"TypeError: object() takes no parameters" in out:
self.fail(
"'TypeError: object() takes no parameters' was seen in output"
)
if b"Setting up the master communication server" in out:
# We got past the place we need, stop the process
break
if out is None and err is None:
break
if proc.poll() is not None:
break
finally:
terminate_process(proc.pid, kill_children=True)
def download_coverage_report(options):
'''
Download the generated coverage report from minion
'''
print('Downloading remote coverage report...')
sys.stdout.flush()
if os.path.isfile(os.path.join(options.workspace, 'coverage.xml')):
os.unlink(os.path.join(options.workspace, 'coverage.xml'))
if options.scp:
cmds = (
' '.join(build_scp_command(options,
'root@{0}:/tmp/coverage.xml'.format(
get_minion_external_address(options)
),
os.path.join(options.workspace, 'coverage.xml'))),
)
else:
cmds = (
'{0} {1} archive.gzip /tmp/coverage.xml',
'{0} {1} cp.push /tmp/coverage.xml.gz',
'gunzip /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml.gz',
'mv /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml {3}'
)
for cmd in cmds:
cmd = cmd.format(
'salt-call publish.publish' if options.lxc else 'salt',
build_minion_target(options),
options.vm_name,
options.workspace
)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def prepare_ssh_access(options):
'''
Generate a temporary SSH key, valid for one hour, and set it as an
authorized key in the minion's root user account on the remote system.
'''
print('Generating temporary SSH Key')
ssh_key_path = os.path.join(options.workspace, 'jenkins_ssh_key_test')
subprocess.call(
'ssh-keygen -t ecdsa -b 521 -C "$(whoami)@$(hostname)-$(date --rfc-3339=seconds)" '
'-f {0} -N \'\' -V -10m:+1h'.format(ssh_key_path),
shell=True,
)
cmd = []
if options.peer:
cmd.extend(['salt-call', 'publish.publish'])
else:
cmd.append('salt')
pub_key_contents = open('{0}.pub'.format(ssh_key_path)).read().strip()
enc, key, comment = pub_key_contents.split(' ', 2)
cmd.extend([
build_minion_target(options),
'ssh.set_auth_key',
'root',
'{0!r}'.format(key),
'enc={0}'.format(enc),
'comment={0!r}'.format(comment)
])
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
def test_master_startup(self):
proc = NonBlockingPopen(
[
self.get_script_path('master'),
'-c',
self.config_dir,
'-l',
'info'
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
out = six.b('')
err = six.b('')
# Testing this should never be longer than 1 minute
max_time = time.time() + 60
try:
while True:
if time.time() > max_time:
assert False, 'Max timeout ocurred'
time.sleep(0.5)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if six.b('DeprecationWarning: object() takes no parameters') in out:
self.fail('\'DeprecationWarning: object() takes no parameters\' was seen in output')
if six.b('TypeError: object() takes no parameters') in out:
self.fail('\'TypeError: object() takes no parameters\' was seen in output')
if six.b('Setting up the master communication server') in out:
# We got past the place we need, stop the process
break
if out is None and err is None:
break
if proc.poll() is not None:
break
finally:
terminate_process(proc.pid, kill_children=True)
def cleanup(clean, vm_name):
if not clean:
return
cmd = 'salt-cloud -d {0} -y'.format(vm_name)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
def delete_vm(options):
'''
Stop a VM
'''
cmd = 'salt-cloud -d {0} -y'.format(options.delete_vm)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
def download_packages(options):
print('Downloading packages...')
sys.stdout.flush()
workspace = options.workspace
vm_name = options.download_packages
for fglob in ('salt-*.rpm',
'salt-*.deb',
'salt-*.pkg.xz',
'salt-buildpackage.log'):
for fname in glob.glob(os.path.join(workspace, fglob)):
if os.path.isfile(fname):
os.unlink(fname)
cmds = [
('salt {{0}} archive.tar czf {0}.tar.gz sources=\'*.*\' cwd={0}'
.format(options.package_artifact_dir)),
'salt {{0}} cp.push {0}.tar.gz'.format(options.package_artifact_dir),
('tar -C {{2}} -xzf /var/cache/salt/master/minions/{{1}}/files{0}.tar.gz'
.format(options.package_artifact_dir)),
]
for cmd in cmds:
cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def download_remote_logs(options):
print('Downloading remote logs...')
sys.stdout.flush()
workspace = options.workspace
vm_name = options.download_remote_logs
for fname in ('salt-runtests.log', 'minion.log'):
if os.path.isfile(os.path.join(workspace, fname)):
os.unlink(os.path.join(workspace, fname))
cmds = (
'salt {0} archive.gzip /tmp/salt-runtests.log',
'salt {0} archive.gzip /var/log/salt/minion',
'salt {0} cp.push /tmp/salt-runtests.log.gz',
'salt {0} cp.push /var/log/salt/minion.gz',
'gunzip /var/cache/salt/master/minions/{0}/files/tmp/salt-runtests.log.gz',
'gunzip /var/cache/salt/master/minions/{0}/files/var/log/salt/minion.gz',
'mv /var/cache/salt/master/minions/{0}/files/tmp/salt-runtests.log {1}/salt-runtests.log',
'mv /var/cache/salt/master/minions/{0}/files/var/log/salt/minion {1}/minion.log'
)
for cmd in cmds:
cmd = cmd.format(vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def download_unittest_reports(options):
print('Downloading remote unittest reports...')
sys.stdout.flush()
workspace = options.workspace
xml_reports_path = os.path.join(workspace, 'xml-test-reports')
if os.path.isdir(xml_reports_path):
shutil.rmtree(xml_reports_path)
os.makedirs(xml_reports_path)
cmds = (
'salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unittests-output/',
'salt {0} cp.push /tmp/xml-test-reports.tar.gz',
'mv -f /var/cache/salt/master/minions/{1}/files/tmp/xml-test-reports.tar.gz {2} && '
'tar zxvf {2}/xml-test-reports.tar.gz -C {2}/xml-test-reports && '
'rm -f {2}/xml-test-reports.tar.gz'
)
vm_name = options.download_unittest_reports
for cmd in cmds:
cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def download_unittest_reports(options):
print('Downloading remote unittest reports...')
sys.stdout.flush()
if os.path.isdir('xml-test-reports'):
shutil.rmtree('xml-test-reports')
os.makedirs('xml-test-reports')
cmds = (
'salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/',
'salt {0} cp.push /tmp/xml-test-reports.tar.gz',
'mv -f /var/cache/salt/master/minions/{0}/files/tmp/xml-test-reports.tar.gz {1}',
'tar zxvf {1}/xml-test-reports.tar.gz -C {1}/xml-test-reports',
'rm -f {1}/xml-test-reports.tar.gz')
vm_name = options.download_unittest_reports
workspace = options.workspace
for cmd in cmds:
cmd = cmd.format(vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('\nFailed to execute command. Exit code: {0}'.format(
proc.returncode))
time.sleep(0.25)
def download_coverage_report(options):
print('Downloading remote coverage report...')
sys.stdout.flush()
workspace = options.workspace
vm_name = options.download_coverage_report
if os.path.isfile(os.path.join(workspace, 'coverage.xml')):
os.unlink(os.path.join(workspace, 'coverage.xml'))
cmds = (
'salt {0} archive.gzip /tmp/coverage.xml',
'salt {0} cp.push /tmp/coverage.xml.gz',
'gunzip /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml.gz',
'mv /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml {2}')
for cmd in cmds:
cmd = cmd.format(build_minion_target(options, vm_name), vm_name,
workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
if proc.returncode != 0:
print('\nFailed to execute command. Exit code: {0}'.format(
proc.returncode))
time.sleep(0.25)
def sync_minion(options):
if 'salt_minion_synced' not in options:
cmd = []
if options.peer:
cmd.extend(['salt-call', 'publish.runner'])
else:
cmd.append('salt')
cmd.extend([build_minion_target(options), 'saltutil.sync_all'])
print('Running CMD: {0!r}'.format(' '.join(cmd)))
sys.stdout.flush()
proc = NonBlockingPopen(' '.join(cmd),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('\nFailed to execute command. Exit code: {0}'.format(
proc.returncode))
sys.exit(proc.returncode)
setattr(options, 'salt_minion_synced', 'yes')
def delete_vm(options):
'''
Stop a VM
'''
cmd = []
if options.peer:
cmd.extend(['salt-call', 'publish.runner'])
else:
cmd.append('salt-run')
if options.lxc:
cmd.append('lxc.purge')
else:
cmd.append('cloud.destroy')
cmd.append(options.vm_name)
print('Running CMD: {0!r}'.format(' '.join(cmd)))
sys.stdout.flush()
proc = NonBlockingPopen(' '.join(cmd),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
def download_unittest_reports(options):
print('Downloading remote unittest reports...')
sys.stdout.flush()
if os.path.isdir('xml-test-reports'):
shutil.rmtree('xml-test-reports')
os.makedirs('xml-test-reports')
cmds = (
'salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/',
'salt {0} cp.push /tmp/xml-test-reports.tar.gz',
'mv -f /var/cache/salt/master/minions/{0}/files/tmp/xml-test-reports.tar.gz {1}',
'tar zxvf {1}/xml-test-reports.tar.gz -C {1}/xml-test-reports',
'rm -f {1}/xml-test-reports.tar.gz'
)
vm_name = options.download_unittest_reports
workspace = options.workspace
for cmd in cmds:
cmd = cmd.format(vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
def download_coverage_report(options):
print('Downloading remote coverage report...')
sys.stdout.flush()
workspace = options.workspace
vm_name = options.download_coverage_report
if os.path.isfile(os.path.join(workspace, 'coverage.xml')):
os.unlink(os.path.join(workspace, 'coverage.xml'))
cmds = (
'salt {0} archive.gzip /tmp/coverage.xml',
'salt {0} cp.push /tmp/coverage.xml.gz',
'gunzip /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml.gz',
'mv /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml {2}'
)
for cmd in cmds:
cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def scp_file(dest_path, contents, kwargs):
'''
Use scp to copy a file to a server
'''
tmpfh, tmppath = tempfile.mkstemp()
with salt.utils.fopen(tmppath, 'w') as tmpfile:
tmpfile.write(contents)
log.debug('Uploading {0} to {1} (scp)'.format(dest_path,
kwargs['hostname']))
ssh_args = [
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, ie, non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none'
]
if 'key_filename' in kwargs:
# There should never be both a password and an ssh key passed in, so
ssh_args.extend([
# tell SSH to skip password authentication
'-oPasswordAuthentication=no',
'-oChallengeResponseAuthentication=no',
# Make sure public key authentication is enabled
'-oPubkeyAuthentication=yes',
# No Keyboard interaction!
'-oKbdInteractiveAuthentication=no',
# Also, specify the location of the key file
'-i {0}'.format(kwargs['key_filename'])
])
cmd = 'scp {0} {1} {2[username]}@{2[hostname]}:{3}'.format(
' '.join(ssh_args), tmppath, kwargs, dest_path)
log.debug('SCP command: {0!r}'.format(cmd))
if 'password' in kwargs:
cmd = 'sshpass -p {0} {1}'.format(kwargs['password'], cmd)
try:
proc = NonBlockingPopen(
cmd,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
stream_stds=kwargs.get('display_ssh_output', True),
)
log.debug('Uploading file(PID {0}): {1!r}'.format(proc.pid, dest_path))
proc.poll_and_read_until_finish()
proc.communicate()
return proc.returncode
except Exception as err:
log.error('Failed to upload file {0!r}: {1}\n'.format(dest_path, err),
exc_info=True)
# Signal an error
return 1
def download_remote_logs(options):
print('Downloading remote logs...')
sys.stdout.flush()
workspace = options.workspace
vm_name = options.download_remote_logs
for fname in ('salt-runtests.log', 'minion.log'):
if os.path.isfile(os.path.join(workspace, fname)):
os.unlink(os.path.join(workspace, fname))
if not options.remote_log_path:
options.remote_log_path = [
'/tmp/salt-runtests.log',
'/var/log/salt/minion'
]
cmds = []
for remote_log in options.remote_log_path:
cmds.extend([
'salt {{0}} archive.gzip {0}'.format(remote_log),
'salt {{0}} cp.push {0}.gz'.format(remote_log),
'gunzip /var/cache/salt/master/minions/{{1}}/files{0}.gz'.format(remote_log),
'mv /var/cache/salt/master/minions/{{1}}/files{0} {{2}}/{1}'.format(
remote_log,
'{0}{1}'.format(
os.path.basename(remote_log),
'' if remote_log.endswith('.log') else '.log'
)
)
])
for cmd in cmds:
cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def delete_vm(options):
'''
Stop a VM
'''
cmd = 'salt-cloud -d {0} -y'.format(options.delete_vm)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
def download_unittest_reports(options):
'''
Download the generated unit test reports from minion
'''
print('Downloading remote unittest reports...')
sys.stdout.flush()
xml_reports_path = os.path.join(options.workspace, 'xml-test-reports')
if os.path.isdir(xml_reports_path):
shutil.rmtree(xml_reports_path)
if options.scp:
cmds = (' '.join(
build_scp_command(
options, '-r', 'root@{0}:/tmp/xml-unitests-output/*'.format(
get_minion_external_address(options)),
os.path.join(options.workspace, 'xml-test-reports'))), )
else:
os.makedirs(xml_reports_path)
cmds = (
'{0} {1} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/',
'{0} {1} cp.push /tmp/xml-test-reports.tar.gz',
'mv -f /var/cache/salt/master/minions/{2}/files/tmp/xml-test-reports.tar.gz {3} && '
'tar zxvf {3}/xml-test-reports.tar.gz -C {3}/xml-test-reports && '
'rm -f {3}/xml-test-reports.tar.gz')
for cmd in cmds:
cmd = cmd.format(
'salt-call publish.publish' if options.lxc else 'salt',
build_minion_target(options), options.vm_name, options.workspace)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('\nFailed to execute command. Exit code: {0}'.format(
proc.returncode))
time.sleep(0.25)
def download_packages(options):
print('Downloading packages...')
sys.stdout.flush()
workspace = options.workspace
vm_name = options.download_packages
for fglob in ('salt-*.rpm',
'salt-*.deb',
'salt-*.pkg.xz',
'salt-buildpackage.log'):
for fname in glob.glob(os.path.join(workspace, fglob)):
if os.path.isfile(fname):
os.unlink(fname)
cmds = [
('salt {{0}} archive.tar czf {0}.tar.gz sources=\'*.*\' cwd={0}'
.format(options.package_artifact_dir)),
'salt {{0}} cp.push {0}.tar.gz'.format(options.package_artifact_dir),
('tar -C {{2}} -xzf /var/cache/salt/master/minions/{{1}}/files{0}.tar.gz'
.format(options.package_artifact_dir)),
]
for cmd in cmds:
cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def download_remote_logs(options):
print('Downloading remote logs...')
sys.stdout.flush()
workspace = options.workspace
vm_name = options.download_remote_logs
for fname in ('salt-runtests.log', 'minion.log'):
if os.path.isfile(os.path.join(workspace, fname)):
os.unlink(os.path.join(workspace, fname))
cmds = (
'salt {0} archive.gzip /tmp/salt-runtests.log',
'salt {0} archive.gzip /var/log/salt/minion',
'salt {0} cp.push /tmp/salt-runtests.log.gz',
'salt {0} cp.push /var/log/salt/minion.gz',
'gunzip /var/cache/salt/master/minions/{0}/files/tmp/salt-runtests.log.gz',
'gunzip /var/cache/salt/master/minions/{0}/files/var/log/salt/minion.gz',
'mv /var/cache/salt/master/minions/{0}/files/tmp/salt-runtests.log {1}/salt-runtests.log',
'mv /var/cache/salt/master/minions/{0}/files/var/log/salt/minion {1}/minion.log'
)
for cmd in cmds:
cmd = cmd.format(vm_name, workspace)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def download_coverage_report(options):
'''
Download the generated coverage report from minion
'''
print('Downloading remote coverage report...')
sys.stdout.flush()
if os.path.isfile(os.path.join(options.workspace, 'coverage.xml')):
os.unlink(os.path.join(options.workspace, 'coverage.xml'))
if options.scp:
cmds = (' '.join(
build_scp_command(
options, 'root@{0}:/tmp/coverage.xml'.format(
get_minion_external_address(options)),
os.path.join(options.workspace, 'coverage.xml'))), )
else:
cmds = (
'{0} {1} archive.gzip /tmp/coverage.xml',
'{0} {1} cp.push /tmp/coverage.xml.gz',
'gunzip /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml.gz',
'mv /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml {3}')
for cmd in cmds:
cmd = cmd.format(
'salt-call publish.publish' if options.lxc else 'salt',
build_minion_target(options), options.vm_name, options.workspace)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('\nFailed to execute command. Exit code: {0}'.format(
proc.returncode))
time.sleep(0.25)
def download_unittest_reports(options):
print("Downloading remote unittest reports...")
sys.stdout.flush()
workspace = options.workspace
xml_reports_path = os.path.join(workspace, "xml-test-reports")
if os.path.isdir(xml_reports_path):
shutil.rmtree(xml_reports_path)
os.makedirs(xml_reports_path)
cmds = (
"salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz '*.xml' cwd=/tmp/xml-unittests-output/",
"salt {0} cp.push /tmp/xml-test-reports.tar.gz",
"mv -f /var/cache/salt/master/minions/{1}/files/tmp/xml-test-reports.tar.gz {2} && "
"tar zxvf {2}/xml-test-reports.tar.gz -C {2}/xml-test-reports && "
"rm -f {2}/xml-test-reports.tar.gz",
)
vm_name = options.download_unittest_reports
for cmd in cmds:
cmd = cmd.format(build_minion_target(options, vm_name), vm_name,
workspace)
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True,
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
if proc.returncode != 0:
print("\nFailed to execute command. Exit code: {0}".format(
proc.returncode))
time.sleep(0.25)
def prepare_ssh_access(options):
'''
Generate a temporary SSH key, valid for one hour, and set it as an
authorized key in the minion's root user account on the remote system.
'''
print('Generating temporary SSH Key')
ssh_key_path = os.path.join(options.workspace, 'jenkins_ssh_key_test')
subprocess.call(
'ssh-keygen -t ecdsa -b 521 -C "$(whoami)@$(hostname)-$(date --rfc-3339=seconds)" '
'-f {0} -N \'\' -V -10m:+1h'.format(ssh_key_path),
shell=True,
)
cmd = []
if options.peer:
cmd.extend(['salt-call', 'publish.publish'])
else:
cmd.append('salt')
pub_key_contents = open('{0}.pub'.format(ssh_key_path)).read().strip()
enc, key, comment = pub_key_contents.split(' ', 2)
cmd.extend([
build_minion_target(options), 'ssh.set_auth_key', 'root',
'{0!r}'.format(key), 'enc={0}'.format(enc),
'comment={0!r}'.format(comment)
])
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('\nFailed to execute command. Exit code: {0}'.format(
proc.returncode))
def win_cmd(command, **kwargs):
'''
Wrapper for commands to be run against Windows boxes
'''
try:
proc = NonBlockingPopen(
command,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
stream_stds=kwargs.get('display_ssh_output', True),
)
log.debug('Executing command(PID {0}): {1!r}'.format(
proc.pid, command))
proc.poll_and_read_until_finish()
proc.communicate()
return proc.returncode
except Exception as err:
log.error('Failed to execute command {0!r}: {1}\n'.format(
command, err),
exc_info=True)
# Signal an error
return 1
def main():
'''
Main script execution
'''
parser = argparse.ArgumentParser(
description='Jenkins execution helper'
)
parser.add_argument(
'-w', '--workspace',
default=os.path.abspath(os.environ.get('WORKSPACE', os.getcwd())),
help='Path to the execution workspace'
)
# Output Options
output_group = parser.add_argument_group('Output Options')
output_group.add_argument(
'--no-color',
'--no-colour',
action='store_true',
default=False,
help='Don\'t use colors'
)
output_group.add_argument(
'--echo-parseable-output',
action='store_true',
default=False,
help='Print Jenkins related environment variables and exit'
)
output_group.add_argument(
'--pull-request',
type=int,
action=GetPullRequestAction,
default=None,
help='Include the Pull Request information in parseable output'
)
# Deployment Selection
deployment_group = parser.add_argument_group('Deployment Selection')
deployment_group.add_argument(
'--pre-mortem',
action='store_true',
default=False,
help='Don\'t try to deploy an new VM. Consider a VM deployed and only '
'execute post test suite execution commands. Right before killing the VM'
)
deployment_group_mutually_exclusive = deployment_group.add_mutually_exclusive_group()
deployment_group_mutually_exclusive.add_argument(
'--cloud',
action='store_true',
default=True,
help='Salt Cloud Deployment'
)
deployment_group_mutually_exclusive.add_argument(
'--lxc',
action='store_true',
default=False,
help='Salt LXC Deployment'
)
deployment_group.add_argument(
'--lxc-host',
default=None,
help='The host where to deploy the LXC VM'
)
# Execution Selections
execution_group = parser.add_argument_group('Execution Selection')
execution_group.add_argument(
'--peer', action='store_true', default=False, help='Run salt commands through the peer system'
)
execution_group.add_argument(
'--scp', action='store_true', default=False,
help='Download logs and reports using SCP'
)
bootstrap_script_options = parser.add_argument_group(
'Bootstrap Script Options',
'In case there\'s a need to provide the bootstrap script from an alternate URL and/or from a specific commit.'
)
bootstrap_script_options.add_argument(
'--bootstrap-salt-url',
default=None,
help='The salt git repository url used to bootstrap a minion'
)
bootstrap_script_options.add_argument(
'--bootstrap-salt-commit',
default=None,
help='The salt git commit used to bootstrap a minion'
)
vm_options_group = parser.add_argument_group('VM Options')
vm_options_group.add_argument('vm_name', nargs='?', help='Virtual machine name')
vm_options_group.add_argument(
'--vm-prefix',
default=os.environ.get('JENKINS_VM_NAME_PREFIX', 'ZJENKINS'),
help='The bootstrapped machine name prefix. Default: %(default)r'
)
vm_options_group.add_argument(
'--vm-source',
default=os.environ.get('JENKINS_VM_SOURCE', None),
help='The VM source. In case of --cloud usage, the could profile name. In case of --lxc usage, the image name.'
)
vm_options_group.add_argument(
'--grain-target',
action='append',
default=[],
help='Match minions using compound matchers, the minion ID, plus the passed grain.'
)
vm_preparation_group = parser.add_argument_group(
'VM Preparation Options',
'Salt SLS selection to prepare the VM. The defaults are based on the salt-jenkins repository. See '
'https://github.com/saltstack/salt-jenkins'
)
vm_preparation_group.add_argument(
'--prep-sls',
default='git.salt',
help='The sls file to execute to prepare the system. Default: %(default)r'
)
vm_preparation_group.add_argument(
'--prep-sls-2',
default=None,
help='An optional 2nd system preparation SLS'
)
vm_preparation_group.add_argument(
'--sls',
default='testrun-no-deps',
help='The final sls file to execute.'
)
vm_preparation_group.add_argument(
'--pillar',
action='append',
nargs=2,
help='Pillar (key, value)s to pass to the sls file. Example: \'--pillar pillar_key pillar_value\''
)
vm_actions = parser.add_argument_group(
'VM Actions',
'Action to execute on a running VM'
)
vm_actions.add_argument(
'--delete-vm',
action='store_true',
default=False,
help='Delete a running VM'
)
vm_actions.add_argument(
'--download-remote-reports',
default=False,
action='store_true',
help='Download remote reports when running remote \'testrun\' state'
)
vm_actions.add_argument(
'--download-unittest-reports',
default=False,
action='store_true',
help='Download the XML unittest results'
)
vm_actions.add_argument(
'--download-coverage-report',
default=False,
action='store_true',
help='Download the XML coverage reports'
)
vm_actions.add_argument(
'--download-remote-logs',
default=False,
action='store_true',
help='Download remote minion and runtests log files'
)
vm_actions.add_argument(
'--remote-log-path',
action='append',
default=[],
help='Provide additional log paths to download from remote minion'
)
testing_source_options = parser.add_argument_group(
'Testing Options',
'In case there\'s a need to provide a different repository and/or commit from which the tests suite should be '
'executed on'
)
testing_source_options.add_argument(
'--test-transport',
default='zeromq',
choices=('zeromq', 'raet'),
help='Set to raet to run integration tests with raet transport. Default: %default')
testing_source_options.add_argument(
'--test-git-url',
default=None,
help='The testing git repository url')
testing_source_options.add_argument(
'--test-git-commit',
default=None,
help='The testing git commit to track')
options = parser.parse_args()
if not options.vm_source and not options.vm_name:
parser.error('Unable to get VM name from environ nor generate it without --vm-source')
if options.lxc:
if not options.lxc_host:
parser.error('Need to provide where to deploy the LXC VM by passing it to --lxc-host')
options.cloud = False
if not options.vm_name:
options.vm_name = get_vm_name(options)
if options.echo_parseable_output:
echo_parseable_environment(options)
parser.exit()
if options.lxc:
options.cloud = False
if options.pre_mortem:
# Run any actions supposed to be executed right before killing the VM
if options.download_remote_reports:
# Download unittest reports
download_unittest_reports(options)
# Download coverage report
download_coverage_report(options)
else:
if options.download_unittest_reports:
download_unittest_reports(options)
if options.download_coverage_report:
download_coverage_report(options)
if options.download_remote_logs:
download_remote_logs(options)
if options.delete_vm:
delete_vm(options)
parser.exit()
# RUN IT!!!
cmd = []
minion_target = build_minion_target(options)
if options.peer:
cmd.extend(['salt-call', 'publish.runner'])
else:
cmd.append('salt-run')
if options.lxc:
cmd.append('lxc.init')
if options.peer:
cmd.append(
'arg="{0}"'.format(
to_cli_yaml([
options.vm_name,
'host={0}'.format(options.lxc_host),
'image={0}'.format(options.vm_source)
])
)
)
else:
cmd.extend([options.vm_name,
'host={0}'.format(options.lxc_host),
'image={0}'.format(options.vm_source)])
else:
cmd.append('cloud.profile')
if options.peer:
cmd.append(
'arg="{0}"'.format(
to_cli_yaml([options.vm_source, options.vm_name])
)
)
else:
cmd.extend([options.vm_source, options.vm_name])
if options.cloud:
if options.bootstrap_salt_commit is not None:
if options.bootstrap_salt_url is None:
options.bootstrap_salt_url = 'https://github.com/saltstack/salt.git'
cmd.append(
'script_args="-D -g {bootstrap_salt_url} -n git {bootstrap_salt_commit}"'
)
else:
cmd.append('script-args="-D"')
cmd = ' '.join(cmd).format(**options.__dict__)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to bootstrap VM. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
print('VM Bootstrapped. Exit code: {0}'.format(retcode))
sys.stdout.flush()
print('Sleeping for 5 seconds to allow the minion to breathe a little')
sys.stdout.flush()
time.sleep(5)
if options.scp:
prepare_ssh_access(options)
if options.cloud:
if options.bootstrap_salt_commit is not None:
# Let's find out if the installed version matches the passed in pillar
# information
print('Grabbing bootstrapped minion version information ... ')
cmd = []
if options.peer:
cmd.extend(['salt-call', '--out=json', 'publish.publish'])
else:
cmd.extend(['salt', '-t', '100', '--out=json'])
cmd.extend([minion_target, 'test.version'])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to get the bootstrapped minion version. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
if not stdout.strip():
print('Failed to get the bootstrapped minion version(no output). Exit code: {0}'.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
try:
version_info = json.loads(stdout.strip())
if options.peer:
version_info = version_info['local']
else:
version_info = version_info[options.vm_name]
bootstrap_salt_commit = options.bootstrap_salt_commit
if re.match('v[0-9]+', bootstrap_salt_commit):
# We've been passed a tag
bootstrap_salt_commit = bootstrap_salt_commit[1:]
else:
# Most likely a git SHA
bootstrap_salt_commit = bootstrap_salt_commit[:7]
if bootstrap_salt_commit not in version_info:
print('\n\nATTENTION!!!!\n')
print('The boostrapped minion version commit does not contain the desired commit:')
print(' {0!r} does not contain {1!r}'.format(version_info, bootstrap_salt_commit))
print('\n\n')
sys.stdout.flush()
#if options.delete_vm:
# delete_vm(options)
#sys.exit(retcode)
else:
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
# Run preparation SLS
time.sleep(3)
cmd = []
if options.peer:
cmd.append('salt-call')
if options.no_color:
cmd.append('--no-color')
cmd.append('publish.publish')
else:
cmd.extend(['salt', '-t', '1800'])
if options.no_color:
cmd.append('--no-color')
cmd.extend([
minion_target, 'state.sls', options.prep_sls, 'pillar="{0}"'.format(build_pillar_data(options))
])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('Failed to execute the preparation SLS file. Exit code: {0}'.format(proc.returncode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(proc.returncode)
if options.prep_sls_2 is not None:
time.sleep(3)
# Run the 2nd preparation SLS
cmd = []
if options.peer:
cmd.append('salt-call')
if options.no_color:
cmd.append('--no-color')
if options.peer:
cmd.append('publish.publish')
else:
cmd.extend(['salt', '-t', '1800'])
if options.no_color:
cmd.append('--no-color')
cmd.extend([
minion_target, 'state.sls', options.prep_sls_2, 'pillar="{0}"'.format(build_pillar_data(options))
])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('Failed to execute the 2nd preparation SLS file. Exit code: {0}'.format(proc.returncode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(proc.returncode)
# Run remote checks
if options.test_git_url is not None:
time.sleep(1)
# Let's find out if the cloned repository if checked out from the
# desired repository
print('Grabbing the cloned repository remotes information ... ')
cmd = []
if options.peer:
cmd.extend(['salt-call', '--out=json', 'publish.publish'])
else:
cmd.extend(['salt', '-t', '100', '--out=json'])
cmd.extend([minion_target, 'git.remote_get', '/testing'])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to get the cloned repository remote. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
if not stdout:
print('Failed to get the cloned repository remote(no output). Exit code: {0}'.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
try:
remotes_info = json.loads(stdout.strip())
if remotes_info is not None:
if options.peer:
remotes_info = remotes_info['local']
else:
remotes_info = remotes_info[options.vm_name]
if remotes_info is None or options.test_git_url not in remotes_info:
print('The cloned repository remote is not the desired one:')
print(' {0!r} is not in {1}'.format(options.test_git_url, remotes_info))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
if options.test_git_commit is not None:
time.sleep(1)
# Let's find out if the cloned repository is checked out at the desired
# commit
print('Grabbing the cloned repository commit information ... ')
cmd = []
if options.peer:
cmd.extend(['salt-call', '--out=json', 'publish.publish'])
else:
cmd.extend(['salt', '-t', '100', '--out=json'])
cmd.extend([minion_target, 'git.revision', '/testing'])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to get the cloned repository revision. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
if not stdout:
print('Failed to get the cloned repository revision(no output). Exit code: {0}'.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
try:
revision_info = json.loads(stdout.strip())
if options.peer:
revision_info = revision_info['local']
else:
revision_info = revision_info[options.vm_name]
if revision_info[:7] != options.test_git_commit[:7]:
print('The cloned repository commit is not the desired one:')
print(' {0!r} != {1!r}'.format(revision_info[:7], options.test_git_commit[:7]))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
# Run tests here
time.sleep(3)
cmd = []
if options.peer:
cmd.append('salt-call')
if options.no_color:
cmd.append('--no-color')
cmd.append('publish.publish')
else:
cmd.extend(['salt', '-t', '1800'])
if options.no_color:
cmd.append('--no-color')
cmd.extend([
minion_target,
'state.sls',
options.sls,
'pillar="{0}"'.format(build_pillar_data(options))
])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
try:
match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout)
retcode = int(match.group('exitcode'))
except AttributeError:
# No regex matching
retcode = 1
except ValueError:
# Not a number!?
retcode = 1
except TypeError:
# No output!?
retcode = 1
if stdout:
# Anything else, raise the exception
raise
if options.download_remote_reports:
# Download unittest reports
download_unittest_reports(options)
# Download coverage report
download_coverage_report(options)
if options.download_remote_logs:
download_remote_logs(options)
if options.delete_vm:
delete_vm(options)
parser.exit(proc.returncode)
def run(opts):
'''
RUN!
'''
vm_name = os.environ.get(
'JENKINS_SALTCLOUD_VM_NAME',
generate_vm_name(opts)
)
if opts.download_remote_reports:
if opts.test_without_coverage is False:
opts.download_coverage_report = vm_name
opts.download_unittest_reports = vm_name
opts.download_packages = vm_name
if opts.bootstrap_salt_commit is not None:
if opts.bootstrap_salt_url is None:
opts.bootstrap_salt_url = 'https://github.com/saltstack/salt.git'
cmd = (
'salt-cloud -l debug'
' --script-args "-D -g {bootstrap_salt_url} -n git {1}"'
' -p {provider}_{platform} {0}'.format(
vm_name,
os.environ.get(
'SALT_MINION_BOOTSTRAP_RELEASE',
opts.bootstrap_salt_commit
),
**opts.__dict__
)
)
else:
cmd = (
'salt-cloud -l debug'
' --script-args "-D -n git {1}" -p {provider}_{platform} {0}'.format(
vm_name,
os.environ.get(
'SALT_MINION_BOOTSTRAP_RELEASE',
opts.bootstrap_salt_commit
),
**opts.__dict__
)
)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to bootstrap VM. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print('VM Bootstrapped. Exit code: {0}'.format(retcode))
sys.stdout.flush()
print('Sleeping for 5 seconds to allow the minion to breathe a little')
sys.stdout.flush()
time.sleep(5)
if opts.bootstrap_salt_commit is not None:
# Let's find out if the installed version matches the passed in pillar
# information
print('Grabbing bootstrapped minion version information ... ')
cmd = 'salt -t 100 {0} --out json test.version'.format(build_minion_target(opts, vm_name))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to get the bootstrapped minion version. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if not stdout.strip():
print('Failed to get the bootstrapped minion version(no output). Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
version_info = json.loads(stdout.strip())
bootstrap_minion_version = os.environ.get(
'SALT_MINION_BOOTSTRAP_RELEASE',
opts.bootstrap_salt_commit[:7]
)
print('Minion reported salt version: {0}'.format(version_info))
if bootstrap_minion_version not in version_info[vm_name]:
print('\n\nATTENTION!!!!\n')
print('The boostrapped minion version commit does not contain the desired commit:')
print(' {0!r} does not contain {1!r}'.format(version_info[vm_name], bootstrap_minion_version))
print('\n\n')
sys.stdout.flush()
#if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
# delete_vm(opts)
#sys.exit(retcode)
else:
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
if opts.cloud_only:
# Run Cloud Provider tests preparation SLS
time.sleep(3)
cmd = (
'salt -t 900 {target} state.sls {cloud_prep_sls} pillar="{pillar}" '
'--no-color'.format(
target=build_minion_target(opts, vm_name),
cloud_prep_sls='cloud-only',
pillar=build_pillar_data(opts),
)
)
else:
# Run standard preparation SLS
time.sleep(3)
cmd = (
'salt -t 1800 {target} state.sls {prep_sls} pillar="{pillar}" '
'--no-color'.format(
target=build_minion_target(opts, vm_name),
prep_sls=opts.prep_sls,
pillar=build_pillar_data(opts),
)
)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to execute the preparation SLS file. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if opts.cloud_only:
time.sleep(3)
# Run Cloud Provider tests pillar preparation SLS
cmd = (
'salt -t 600 {target} state.sls {cloud_prep_sls} pillar="{pillar}" '
'--no-color'.format(
target=build_minion_target(opts, vm_name),
cloud_prep_sls='cloud-test-configs',
pillar=build_pillar_data(opts),
)
)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, stderr = proc.communicate()
if stdout:
# DO NOT print the state return here!
print('Cloud configuration files provisioned via pillar.')
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to execute the preparation SLS file. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if opts.prep_sls_2 is not None:
time.sleep(3)
# Run the 2nd preparation SLS
cmd = (
'salt -t 30 {target} state.sls {prep_sls_2} pillar="{pillar}" '
'--no-color'.format(
prep_sls_2=opts.prep_sls_2,
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
)
)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to execute the 2nd preparation SLS file. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
# Run remote checks
if opts.test_git_url is not None:
time.sleep(1)
# Let's find out if the cloned repository if checked out from the
# desired repository
print('Grabbing the cloned repository remotes information ... ')
cmd = 'salt -t 100 {0} --out json git.remote_get /testing'.format(build_minion_target(opts, vm_name))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to get the cloned repository remote. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if not stdout:
print('Failed to get the cloned repository remote(no output). Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
remotes_info = json.loads(stdout.strip())
if remotes_info is None or remotes_info[vm_name] is None or opts.test_git_url not in remotes_info[vm_name]:
print('The cloned repository remote is not the desired one:')
print(' {0!r} is not in {1}'.format(opts.test_git_url, remotes_info))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
if opts.test_git_commit is not None:
time.sleep(1)
# Let's find out if the cloned repository is checked out at the desired
# commit
print('Grabbing the cloned repository commit information ... ')
cmd = 'salt -t 100 {0} --out json git.revision /testing'.format(build_minion_target(opts, vm_name))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to get the cloned repository revision. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if not stdout:
print('Failed to get the cloned repository revision(no output). Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
revision_info = json.loads(stdout.strip())
if revision_info[vm_name][7:] != opts.test_git_commit[7:]:
print('The cloned repository commit is not the desired one:')
print(' {0!r} != {1!r}'.format(revision_info[vm_name][:7], opts.test_git_commit[:7]))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
# Run tests here
time.sleep(3)
cmd = (
'salt -t 1800 {target} state.sls {sls} pillar="{pillar}" --no-color'.format(
sls=opts.sls,
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
)
)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
try:
match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout)
retcode = int(match.group('exitcode'))
except AttributeError:
# No regex matching
retcode = 1
except ValueError:
# Not a number!?
retcode = 1
except TypeError:
# No output!?
retcode = 1
if stdout:
# Anything else, raise the exception
raise
if retcode == 0:
# Build packages
time.sleep(3)
cmd = (
'salt -t 1800 {target} state.sls buildpackage pillar="{pillar}" --no-color'.format(
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
)
)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
# Grab packages and log file (or just log file if build failed)
download_packages(opts)
if opts.download_remote_reports:
# Download unittest reports
download_unittest_reports(opts)
# Download coverage report
if opts.test_without_coverage is False:
download_coverage_report(opts)
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
return retcode
def test_deferred_write_on_sigint(tmp_path):
pyscript = dedent(r"""
import sys
import time
import signal
import logging
CODE_DIR = {!r}
if CODE_DIR in sys.path:
sys.path.remove(CODE_DIR)
sys.path.insert(0, CODE_DIR)
from salt._logging.handlers import DeferredStreamHandler
# Reset any logging handlers we might have already
logging.root.handlers[:] = []
handler = DeferredStreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
logging.root.addHandler(handler)
if signal.getsignal(signal.SIGINT) != signal.default_int_handler:
# Looking at you Debian based distros :/
signal.signal(signal.SIGINT, signal.default_int_handler)
log = logging.getLogger(__name__)
start_printed = False
while True:
try:
log.debug('Foo')
if start_printed is False:
sys.stdout.write('STARTED\n')
sys.stdout.write('SIGINT HANDLER: {{!r}}\n'.format(signal.getsignal(signal.SIGINT)))
sys.stdout.flush()
start_printed = True
time.sleep(0.125)
except (KeyboardInterrupt, SystemExit):
log.info('KeyboardInterrupt caught')
sys.stdout.write('KeyboardInterrupt caught\n')
sys.stdout.flush()
break
log.info('EXITING')
sys.stdout.write('EXITING\n')
sys.stdout.flush()
sys.exit(0)
""".format(RUNTIME_VARS.CODE_DIR))
script_path = tmp_path / "sigint_deferred_logging_test.py"
script_path.write_text(pyscript, encoding="utf-8")
proc = NonBlockingPopen(
[sys.executable, str(script_path)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out = b""
err = b""
# Test should take less than 20 seconds, way less
execution_time = 10
start = time.time()
max_time = time.time() + execution_time
try:
signalled = False
log.info("Starting Loop")
while True:
time.sleep(0.125)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if b"STARTED" in out and not signalled:
# Enough time has passed
proc.send_signal(signal.SIGINT)
signalled = True
log.debug("Sent SIGINT after: %s", time.time() - start)
if signalled is False:
if out:
pytest.fail(
"We have stdout output when there should be none: {}".
format(out))
if err:
pytest.fail(
"We have stderr output when there should be none: {}".
format(err))
if _out is None and _err is None:
log.info("_out and _err are None")
if b"Foo" not in err:
pytest.fail(
"No more output and 'Foo' should be in stderr and it's not: {}"
.format(err))
break
if proc.poll() is not None:
log.debug("poll() is not None")
if b"Foo" not in err:
pytest.fail(
"Process terminated and 'Foo' should be in stderr and it's not: {}"
.format(err))
break
if time.time() > max_time:
log.debug("Reached max time")
if b"Foo" not in err:
pytest.fail(
"'Foo' should be in stderr and it's not:\n{0}\nSTDERR:\n{0}\n{1}\n{0}\nSTDOUT:\n{0}\n{2}\n{0}"
.format("-" * 80, err, out))
finally:
terminate_process(proc.pid, kill_children=True)
log.debug("Test took %s seconds", time.time() - start)
def run(opts):
'''
RUN!
'''
vm_name = os.environ.get('JENKINS_SALTCLOUD_VM_NAME',
generate_vm_name(opts.platform))
if opts.download_remote_reports:
opts.download_coverage_report = vm_name
opts.download_unittest_reports = vm_name
cmd = ('salt-cloud -l debug --script-args "-D -n git {commit}" -p '
'{provider}_{platform} {0}'.format(vm_name, **opts.__dict__))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to bootstrap VM. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(vm_name)
sys.exit(retcode)
print('VM Bootstrapped. Exit code: {0}'.format(retcode))
sys.stdout.flush()
print('Sleeping for 5 seconds to allow the minion to breathe a little')
sys.stdout.flush()
time.sleep(5)
# Run tests here
cmd = ('salt -t 1800 {vm_name} state.sls {sls} pillar="{pillar}" '
'--no-color'.format(sls=opts.sls,
pillar=opts.pillar.format(commit=opts.commit),
vm_name=vm_name,
commit=opts.commit))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
#proc = NonBlockingPopen(
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
# stream_stds=True
)
#proc.poll_and_read_until_finish()
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
try:
match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout)
retcode = int(match.group('exitcode'))
except AttributeError:
# No regex matching
retcode = 1
except ValueError:
# Not a number!?
retcode = 1
except TypeError:
# No output!?
retcode = 1
if stdout:
# Anything else, raise the exception
raise
if opts.download_remote_reports:
# Download unittest reports
download_unittest_reports(opts)
# Download coverage report
download_coverage_report(opts)
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(vm_name)
return retcode
def scp_file(dest_path, contents, kwargs):
'''
Use scp to copy a file to a server
'''
tmpfh, tmppath = tempfile.mkstemp()
with salt.utils.fopen(tmppath, 'w') as tmpfile:
tmpfile.write(contents)
log.debug('Uploading {0} to {1} (scp)'.format(dest_path, kwargs['hostname']))
ssh_args = [
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, ie, non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none'
]
if 'key_filename' in kwargs:
# There should never be both a password and an ssh key passed in, so
ssh_args.extend([
# tell SSH to skip password authentication
'-oPasswordAuthentication=no',
'-oChallengeResponseAuthentication=no',
# Make sure public key authentication is enabled
'-oPubkeyAuthentication=yes',
# No Keyboard interaction!
'-oKbdInteractiveAuthentication=no',
# Also, specify the location of the key file
'-i {0}'.format(kwargs['key_filename'])
])
cmd = 'scp {0} {1} {2[username]}@{2[hostname]}:{3}'.format(
' '.join(ssh_args), tmppath, kwargs, dest_path
)
log.debug('SCP command: {0!r}'.format(cmd))
if 'password' in kwargs:
cmd = 'sshpass -p {0} {1}'.format(kwargs['password'], cmd)
try:
proc = NonBlockingPopen(
cmd,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
stream_stds=kwargs.get('display_ssh_output', True),
)
log.debug(
'Uploading file(PID {0}): {1!r}'.format(
proc.pid, dest_path
)
)
proc.poll_and_read_until_finish()
proc.communicate()
return proc.returncode
except Exception as err:
log.error(
'Failed to upload file {0!r}: {1}\n'.format(
dest_path, err
),
exc_info=True
)
# Signal an error
return 1
def test_deferred_write_on_atexit(tmp_path):
# Python will .flush() and .close() all logging handlers at interpreter shutdown.
# This should be enough to flush our deferred messages.
pyscript = dedent(r"""
import sys
import time
import logging
CODE_DIR = {!r}
if CODE_DIR in sys.path:
sys.path.remove(CODE_DIR)
sys.path.insert(0, CODE_DIR)
from salt._logging.handlers import DeferredStreamHandler
# Reset any logging handlers we might have already
logging.root.handlers[:] = []
handler = DeferredStreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
logging.root.addHandler(handler)
log = logging.getLogger(__name__)
sys.stdout.write('STARTED\n')
sys.stdout.flush()
log.debug('Foo')
sys.exit(0)
""".format(RUNTIME_VARS.CODE_DIR))
script_path = tmp_path / "atexit_deferred_logging_test.py"
script_path.write_text(pyscript, encoding="utf-8")
proc = NonBlockingPopen(
[sys.executable, str(script_path)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out = b""
err = b""
# This test should never take more than 5 seconds
execution_time = 5
max_time = time.time() + execution_time
try:
# Just loop consuming output
while True:
if time.time() > max_time:
pytest.fail("Script didn't exit after {} second".format(
execution_time))
time.sleep(0.125)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if _out is None and _err is None:
# The script exited
break
if proc.poll() is not None:
# The script exited
break
finally:
terminate_process(proc.pid, kill_children=True)
if b"Foo" not in err:
pytest.fail("'Foo' should be in stderr and it's not: {}".format(err))
def run(opts):
'''
RUN!
'''
vm_name = os.environ.get(
'JENKINS_SALTCLOUD_VM_NAME',
generate_vm_name(opts.platform)
)
if opts.download_remote_reports:
opts.download_coverage_report = vm_name
opts.download_unittest_reports = vm_name
cmd = (
'salt-cloud -l debug'
' --script-args "-D -g {salt_url} -n git {commit}"'
' -p {provider}_{platform} {0}'.format(vm_name, **opts.__dict__)
)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to bootstrap VM. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(vm_name)
sys.exit(retcode)
print('VM Bootstrapped. Exit code: {0}'.format(retcode))
sys.stdout.flush()
print('Sleeping for 5 seconds to allow the minion to breathe a little')
sys.stdout.flush()
time.sleep(5)
# Do we need extra setup?
if opts.salt_url != SALT_GIT_URL:
cmds = (
'salt -t 100 {vm_name} git.remote_set /testing name={0!r} url={1!r}'.format(
'upstream',
SALT_GIT_URL,
vm_name=vm_name
),
'salt -t 100 {vm_name} git.fetch /testing \'upstream --tags\''.format(
vm_name=vm_name
)
)
for cmd in cmds:
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, _ = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
# Run tests here
cmd = (
'salt -t 1800 {vm_name} state.sls {sls} pillar="{pillar}" '
'--no-color'.format(
sls=opts.sls,
pillar=opts.pillar.format(
commit=opts.commit,
salt_url=opts.salt_url
),
vm_name=vm_name,
commit=opts.commit
)
)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
#proc = NonBlockingPopen(
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
# stream_stds=True
)
#proc.poll_and_read_until_finish()
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
try:
match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout)
retcode = int(match.group('exitcode'))
except AttributeError:
# No regex matching
retcode = 1
except ValueError:
# Not a number!?
retcode = 1
except TypeError:
# No output!?
retcode = 1
if stdout:
# Anything else, raise the exception
raise
if opts.download_remote_reports:
# Download unittest reports
download_unittest_reports(opts)
# Download coverage report
download_coverage_report(opts)
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(vm_name)
return retcode
def root_cmd(command, tty, sudo, **kwargs):
'''
Wrapper for commands to be run as root
'''
if sudo:
command = 'sudo {0}'.format(command)
log.debug('Using sudo to run command {0}'.format(command))
ssh_args = []
if tty:
# Use double `-t` on the `ssh` command, it's necessary when `sudo` has
# `requiretty` enforced.
ssh_args.extend(['-t', '-t'])
ssh_args.extend([
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, ie, non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none'
])
if 'key_filename' in kwargs:
# There should never be both a password and an ssh key passed in, so
ssh_args.extend([
# tell SSH to skip password authentication
'-oPasswordAuthentication=no',
'-oChallengeResponseAuthentication=no',
# Make sure public key authentication is enabled
'-oPubkeyAuthentication=yes',
# No Keyboard interaction!
'-oKbdInteractiveAuthentication=no',
# Also, specify the location of the key file
'-i {0}'.format(kwargs['key_filename'])
])
cmd = 'ssh {0} {1[username]}@{1[hostname]} {2}'.format(
' '.join(ssh_args), kwargs, pipes.quote(command))
log.debug('SSH command: {0!r}'.format(cmd))
if 'password' in kwargs:
cmd = 'sshpass -p {0} {1}'.format(kwargs['password'], cmd)
try:
proc = NonBlockingPopen(
cmd,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
stream_stds=kwargs.get('display_ssh_output', True),
)
log.debug('Executing command(PID {0}): {1!r}'.format(
proc.pid, command))
proc.poll_and_read_until_finish()
proc.communicate()
return proc.returncode
except Exception as err:
log.error('Failed to execute command {0!r}: {1}\n'.format(
command, err),
exc_info=True)
# Signal an error
return 1
def run(opts):
'''
RUN!
'''
vm_name = os.environ.get('JENKINS_SALTCLOUD_VM_NAME',
generate_vm_name(opts))
if opts.download_remote_reports:
if opts.test_without_coverage is False:
opts.download_coverage_report = vm_name
opts.download_unittest_reports = vm_name
opts.download_packages = vm_name
if opts.bootstrap_salt_commit is not None:
if opts.bootstrap_salt_url is None:
opts.bootstrap_salt_url = 'https://github.com/saltstack/salt.git'
cmd = ('salt-cloud -l debug'
' --script-args "-D -g {bootstrap_salt_url} -n git {1}"'
' -p {provider}_{platform} {0}'.format(
vm_name,
os.environ.get('SALT_MINION_BOOTSTRAP_RELEASE',
opts.bootstrap_salt_commit),
**opts.__dict__))
else:
cmd = ('salt-cloud -l debug'
' --script-args "-D -n git {1}" -p {provider}_{platform} {0}'.
format(
vm_name,
os.environ.get('SALT_MINION_BOOTSTRAP_RELEASE',
opts.bootstrap_salt_commit),
**opts.__dict__))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to bootstrap VM. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print('VM Bootstrapped. Exit code: {0}'.format(retcode))
sys.stdout.flush()
print('Sleeping for 5 seconds to allow the minion to breathe a little')
sys.stdout.flush()
time.sleep(5)
if opts.bootstrap_salt_commit is not None:
# Let's find out if the installed version matches the passed in pillar
# information
print('Grabbing bootstrapped minion version information ... ')
cmd = 'salt -t 100 {0} --out json test.version'.format(
build_minion_target(opts, vm_name))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
retcode = proc.returncode
if retcode != 0:
print(
'Failed to get the bootstrapped minion version. Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if not stdout.strip():
print(
'Failed to get the bootstrapped minion version(no output). Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
version_info = json.loads(stdout.strip())
bootstrap_minion_version = os.environ.get(
'SALT_MINION_BOOTSTRAP_RELEASE',
opts.bootstrap_salt_commit[:7])
print('Minion reported salt version: {0}'.format(version_info))
if bootstrap_minion_version not in version_info[vm_name]:
print('\n\nATTENTION!!!!\n')
print(
'The boostrapped minion version commit does not contain the desired commit:'
)
print(' {0!r} does not contain {1!r}'.format(
version_info[vm_name], bootstrap_minion_version))
print('\n\n')
sys.stdout.flush()
#if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
# delete_vm(opts)
#sys.exit(retcode)
else:
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
if opts.cloud_only:
# Run Cloud Provider tests preparation SLS
time.sleep(3)
cmd = (
'salt -t 900 {target} state.sls {cloud_prep_sls} pillar="{pillar}" '
'--no-color'.format(
target=build_minion_target(opts, vm_name),
cloud_prep_sls='cloud-only',
pillar=build_pillar_data(opts),
))
else:
# Run standard preparation SLS
time.sleep(3)
cmd = ('salt -t 1800 {target} state.sls {prep_sls} pillar="{pillar}" '
'--no-color'.format(
target=build_minion_target(opts, vm_name),
prep_sls=opts.prep_sls,
pillar=build_pillar_data(opts),
))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to execute the preparation SLS file. Exit code: {0}'.
format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if opts.cloud_only:
time.sleep(3)
# Run Cloud Provider tests pillar preparation SLS
cmd = (
'salt -t 600 {target} state.sls {cloud_prep_sls} pillar="{pillar}" '
'--no-color'.format(
target=build_minion_target(opts, vm_name),
cloud_prep_sls='cloud-test-configs',
pillar=build_pillar_data(opts),
))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, stderr = proc.communicate()
if stdout:
# DO NOT print the state return here!
print('Cloud configuration files provisioned via pillar.')
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to execute the preparation SLS file. Exit code: {0}'.
format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if opts.prep_sls_2 is not None:
time.sleep(3)
# Run the 2nd preparation SLS
cmd = ('salt -t 30 {target} state.sls {prep_sls_2} pillar="{pillar}" '
'--no-color'.format(
prep_sls_2=opts.prep_sls_2,
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
retcode = proc.returncode
if retcode != 0:
print(
'Failed to execute the 2nd preparation SLS file. Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
# Run remote checks
if opts.test_git_url is not None:
time.sleep(1)
# Let's find out if the cloned repository if checked out from the
# desired repository
print('Grabbing the cloned repository remotes information ... ')
cmd = 'salt -t 100 {0} --out json git.remote_get /testing'.format(
build_minion_target(opts, vm_name))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to get the cloned repository remote. Exit code: {0}'.
format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if not stdout:
print(
'Failed to get the cloned repository remote(no output). Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
remotes_info = json.loads(stdout.strip())
if remotes_info is None or remotes_info[
vm_name] is None or opts.test_git_url not in remotes_info[
vm_name]:
print('The cloned repository remote is not the desired one:')
print(' {0!r} is not in {1}'.format(opts.test_git_url,
remotes_info))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
if opts.test_git_commit is not None:
time.sleep(1)
# Let's find out if the cloned repository is checked out at the desired
# commit
print('Grabbing the cloned repository commit information ... ')
cmd = 'salt -t 100 {0} --out json git.revision /testing'.format(
build_minion_target(opts, vm_name))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print(
'Failed to get the cloned repository revision. Exit code: {0}'.
format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if not stdout:
print(
'Failed to get the cloned repository revision(no output). Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
revision_info = json.loads(stdout.strip())
if revision_info[vm_name][7:] != opts.test_git_commit[7:]:
print('The cloned repository commit is not the desired one:')
print(' {0!r} != {1!r}'.format(revision_info[vm_name][:7],
opts.test_git_commit[:7]))
sys.stdout.flush()
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
# Run tests here
time.sleep(3)
cmd = ('salt -t 1800 {target} state.sls {sls} pillar="{pillar}" --no-color'
.format(
sls=opts.sls,
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
try:
match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout)
retcode = int(match.group('exitcode'))
except AttributeError:
# No regex matching
retcode = 1
except ValueError:
# Not a number!?
retcode = 1
except TypeError:
# No output!?
retcode = 1
if stdout:
# Anything else, raise the exception
raise
if retcode == 0:
# Build packages
time.sleep(3)
cmd = (
'salt -t 1800 {target} state.sls buildpackage pillar="{pillar}" --no-color'
.format(
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
))
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
if stderr:
print(stderr)
sys.stderr.flush()
# Grab packages and log file (or just log file if build failed)
download_packages(opts)
if opts.download_remote_reports:
# Download unittest reports
download_unittest_reports(opts)
# Download coverage report
if opts.test_without_coverage is False:
download_coverage_report(opts)
if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
delete_vm(opts)
return retcode
def run(opts):
"""
RUN!
"""
vm_name = os.environ.get("JENKINS_SALTCLOUD_VM_NAME",
generate_vm_name(opts))
if opts.download_remote_reports:
if opts.test_without_coverage is False:
opts.download_coverage_report = vm_name
opts.download_unittest_reports = vm_name
opts.download_packages = vm_name
if opts.bootstrap_salt_commit is not None:
if opts.bootstrap_salt_url is None:
opts.bootstrap_salt_url = "https://github.com/saltstack/salt.git"
cmd = ("salt-cloud -l debug"
' --script-args "-D -g {bootstrap_salt_url} -n git {1}"'
" -p {provider}_{platform} {0}".format(
vm_name,
os.environ.get("SALT_MINION_BOOTSTRAP_RELEASE",
opts.bootstrap_salt_commit),
**opts.__dict__))
else:
cmd = ("salt-cloud -l debug"
' --script-args "-D -n git {1}" -p {provider}_{platform} {0}'.
format(
vm_name,
os.environ.get("SALT_MINION_BOOTSTRAP_RELEASE",
opts.bootstrap_salt_commit),
**opts.__dict__))
if opts.splay is not None:
# Sleep a random number of seconds
cloud_downtime = random.randint(0, opts.splay)
print("Sleeping random period before calling salt-cloud: {0}".format(
cloud_downtime))
time.sleep(cloud_downtime)
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True,
)
proc.poll_and_read_until_finish(interval=0.5)
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print("Failed to bootstrap VM. Exit code: {0}".format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print("VM Bootstrapped. Exit code: {0}".format(retcode))
sys.stdout.flush()
# Sleep a random number of seconds
bootstrap_downtime = random.randint(0, opts.splay)
print("Sleeping for {0} seconds to allow the minion to breathe a little".
format(bootstrap_downtime))
sys.stdout.flush()
time.sleep(bootstrap_downtime)
if opts.bootstrap_salt_commit is not None:
# Let's find out if the installed version matches the passed in pillar
# information
print("Grabbing bootstrapped minion version information ... ")
cmd = "salt -t 100 {0} --out json test.version".format(
build_minion_target(opts, vm_name))
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
retcode = proc.returncode
if retcode != 0:
print(
"Failed to get the bootstrapped minion version. Exit code: {0}"
.format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
outstr = salt.utils.stringutils.to_str(stdout).strip()
if not outstr:
print(
"Failed to get the bootstrapped minion version(no output). Exit code: {0}"
.format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
version_info = salt.utils.json.loads(outstr)
bootstrap_minion_version = os.environ.get(
"SALT_MINION_BOOTSTRAP_RELEASE",
opts.bootstrap_salt_commit[:7])
print("Minion reported salt version: {0}".format(version_info))
if bootstrap_minion_version not in version_info[vm_name]:
print("\n\nATTENTION!!!!\n")
print(
"The boostrapped minion version commit does not contain the desired commit:"
)
print(" '{0}' does not contain '{1}'".format(
version_info[vm_name], bootstrap_minion_version))
print("\n\n")
sys.stdout.flush()
# if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ:
# delete_vm(opts)
# sys.exit(retcode)
else:
print("matches!")
except ValueError:
print("Failed to load any JSON from '{0}'".format(outstr))
if opts.cloud_only:
# Run Cloud Provider tests preparation SLS
cloud_provider_downtime = random.randint(3, opts.splay)
time.sleep(cloud_provider_downtime)
cmd = (
'salt -t 900 {target} state.sls {cloud_prep_sls} pillar="{pillar}" '
"--no-color".format(
target=build_minion_target(opts, vm_name),
cloud_prep_sls="cloud-only",
pillar=build_pillar_data(opts),
))
else:
# Run standard preparation SLS
standard_sls_downtime = random.randint(3, opts.splay)
time.sleep(standard_sls_downtime)
cmd = ('salt -t 1800 {target} state.sls {prep_sls} pillar="{pillar}" '
"--no-color".format(
target=build_minion_target(opts, vm_name),
prep_sls=opts.prep_sls,
pillar=build_pillar_data(opts),
))
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
if stdout:
print(salt.utils.stringutils.to_str(stdout))
if stderr:
print(salt.utils.stringutils.to_str(stderr))
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print("Failed to execute the preparation SLS file. Exit code: {0}".
format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if opts.cloud_only:
cloud_provider_pillar = random.randint(3, opts.splay)
time.sleep(cloud_provider_pillar)
# Run Cloud Provider tests pillar preparation SLS
cmd = (
'salt -t 600 {target} state.sls {cloud_prep_sls} pillar="{pillar}" '
"--no-color".format(
target=build_minion_target(opts, vm_name),
cloud_prep_sls="cloud-test-configs",
pillar=build_pillar_data(opts),
))
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, stderr = proc.communicate()
if stdout:
# DO NOT print the state return here!
print("Cloud configuration files provisioned via pillar.")
if stderr:
print(salt.utils.stringutils.to_str(stderr))
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print("Failed to execute the preparation SLS file. Exit code: {0}".
format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if opts.prep_sls_2 is not None:
sls_2_downtime = random.randint(3, opts.splay)
time.sleep(sls_2_downtime)
# Run the 2nd preparation SLS
cmd = ('salt -t 30 {target} state.sls {prep_sls_2} pillar="{pillar}" '
"--no-color".format(
prep_sls_2=opts.prep_sls_2,
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
))
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, stderr = proc.communicate()
if stdout:
print(salt.utils.stringutils.to_str(stdout))
if stderr:
print(salt.utils.stringutils.to_str(stderr))
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print(
"Failed to execute the 2nd preparation SLS file. Exit code: {0}"
.format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
# Run remote checks
if opts.test_git_url is not None:
test_git_downtime = random.randint(1, opts.splay)
time.sleep(test_git_downtime)
# Let's find out if the cloned repository if checked out from the
# desired repository
print("Grabbing the cloned repository remotes information ... ")
cmd = "salt -t 100 {0} --out json git.remote_get /testing".format(
build_minion_target(opts, vm_name))
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
retcode = proc.returncode
if retcode != 0:
print("Failed to get the cloned repository remote. Exit code: {0}".
format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if not stdout:
print(
"Failed to get the cloned repository remote(no output). Exit code: {0}"
.format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
remotes_info = salt.utils.json.loads(stdout.strip())
if (remotes_info is None or remotes_info[vm_name] is None
or opts.test_git_url not in remotes_info[vm_name]):
print("The cloned repository remote is not the desired one:")
print(" '{0}' is not in {1}".format(opts.test_git_url,
remotes_info))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print("matches!")
except ValueError:
print("Failed to load any JSON from '{0}'".format(
salt.utils.stringutils.to_str(stdout).strip()))
if opts.test_git_commit is not None:
test_git_commit_downtime = random.randint(1, opts.splay)
time.sleep(test_git_commit_downtime)
# Let's find out if the cloned repository is checked out at the desired
# commit
print("Grabbing the cloned repository commit information ... ")
cmd = "salt -t 100 {0} --out json git.revision /testing".format(
build_minion_target(opts, vm_name))
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print(
"Failed to get the cloned repository revision. Exit code: {0}".
format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
if not stdout:
print(
"Failed to get the cloned repository revision(no output). Exit code: {0}"
.format(retcode))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
try:
revision_info = salt.utils.json.loads(stdout.strip())
if revision_info[vm_name][7:] != opts.test_git_commit[7:]:
print("The cloned repository commit is not the desired one:")
print(" '{0}' != '{1}'".format(revision_info[vm_name][:7],
opts.test_git_commit[:7]))
sys.stdout.flush()
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
sys.exit(retcode)
print("matches!")
except ValueError:
print("Failed to load any JSON from '{0}'".format(
salt.utils.stringutils.to_str(stdout).strip()))
# Run tests here
test_begin_downtime = random.randint(3, opts.splay)
time.sleep(test_begin_downtime)
cmd = 'salt -t 1800 {target} state.sls {sls} pillar="{pillar}" --no-color'.format(
sls=opts.sls,
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
)
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
outstr = salt.utils.stringutils.to_str(stdout)
if outstr:
print(outstr)
if stderr:
print(salt.utils.stringutils.to_str(stderr))
sys.stdout.flush()
try:
match = re.search(r"Test Suite Exit Code: (?P<exitcode>[\d]+)", outstr)
retcode = int(match.group("exitcode"))
except AttributeError:
# No regex matching
retcode = 1
except ValueError:
# Not a number!?
retcode = 1
except TypeError:
# No output!?
retcode = 1
if outstr:
# Anything else, raise the exception
raise
if retcode == 0:
# Build packages
time.sleep(3)
cmd = 'salt -t 1800 {target} state.sls buildpackage pillar="{pillar}" --no-color'.format(
pillar=build_pillar_data(opts),
target=build_minion_target(opts, vm_name),
)
print("Running CMD: {0}".format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = proc.communicate()
if stdout:
print(salt.utils.stringutils.to_str(stdout))
if stderr:
print(salt.utils.stringutils.to_str(stderr))
sys.stdout.flush()
# Grab packages and log file (or just log file if build failed)
download_packages(opts)
if opts.download_remote_reports:
# Download unittest reports
download_unittest_reports(opts)
# Download coverage report
if opts.test_without_coverage is False:
download_coverage_report(opts)
if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ:
delete_vm(opts)
return retcode
def download_remote_logs(options):
'''
Download the generated logs from minion
'''
print('Downloading remote logs...')
sys.stdout.flush()
for fname in ('salt-runtests.log', 'minion.log'):
if os.path.isfile(os.path.join(options.workspace, fname)):
os.unlink(os.path.join(options.workspace, fname))
if not options.remote_log_path:
options.remote_log_path = [
'/tmp/salt-runtests.log', '/var/log/salt/minion'
]
cmds = []
if options.scp:
for remote_log in options.remote_log_path:
cmds.append(' '.join(
build_scp_command(
options, '-r',
'root@{0}:{1}'.format(get_minion_external_address(options),
remote_log),
os.path.join(
options.workspace, '{0}{1}'.format(
os.path.basename(remote_log),
'' if remote_log.endswith('.log') else '.log')))))
else:
for remote_log in options.remote_log_path:
cmds.extend([
'{{0}} {{1}} archive.gzip {0}'.format(remote_log),
'{{0}} {{1}} cp.push {0}.gz'.format(remote_log),
'gunzip /var/cache/salt/master/minions/{{2}}/files{0}.gz'.
format(remote_log),
'mv /var/cache/salt/master/minions/{{2}}/files{0} {{3}}/{1}'.
format(
remote_log, '{0}{1}'.format(
os.path.basename(remote_log),
'' if remote_log.endswith('.log') else '.log'))
])
for cmd in cmds:
cmd = cmd.format(
'salt-call publish.publish' if options.lxc else 'salt',
build_minion_target(options), options.vm_name, options.workspace)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('\nFailed to execute command. Exit code: {0}'.format(
proc.returncode))
time.sleep(0.25)
def download_remote_logs(options):
'''
Download the generated logs from minion
'''
print('Downloading remote logs...')
sys.stdout.flush()
for fname in ('salt-runtests.log', 'minion.log'):
if os.path.isfile(os.path.join(options.workspace, fname)):
os.unlink(os.path.join(options.workspace, fname))
if not options.remote_log_path:
options.remote_log_path = [
'/tmp/salt-runtests.log',
'/var/log/salt/minion'
]
cmds = []
if options.scp:
for remote_log in options.remote_log_path:
cmds.append(
' '.join(build_scp_command(options,
'-r',
'root@{0}:{1}'.format(
get_minion_external_address(options),
remote_log
),
os.path.join(
options.workspace,
'{0}{1}'.format(
os.path.basename(remote_log),
'' if remote_log.endswith('.log') else '.log'
)
)))
)
else:
for remote_log in options.remote_log_path:
cmds.extend([
'{{0}} {{1}} archive.gzip {0}'.format(remote_log),
'{{0}} {{1}} cp.push {0}.gz'.format(remote_log),
'gunzip /var/cache/salt/master/minions/{{2}}/files{0}.gz'.format(remote_log),
'mv /var/cache/salt/master/minions/{{2}}/files{0} {{3}}/{1}'.format(
remote_log,
'{0}{1}'.format(
os.path.basename(remote_log),
'' if remote_log.endswith('.log') else '.log'
)
)
])
for cmd in cmds:
cmd = cmd.format(
'salt-call publish.publish' if options.lxc else 'salt',
build_minion_target(options),
options.vm_name,
options.workspace
)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'\nFailed to execute command. Exit code: {0}'.format(
proc.returncode
)
)
time.sleep(0.25)
def main():
'''
Main script execution
'''
parser = argparse.ArgumentParser(description='Jenkins execution helper')
parser.add_argument('-w',
'--workspace',
default=os.path.abspath(
os.environ.get('WORKSPACE', os.getcwd())),
help='Path to the execution workspace')
# Output Options
output_group = parser.add_argument_group('Output Options')
output_group.add_argument('--no-color',
'--no-colour',
action='store_true',
default=False,
help='Don\'t use colors')
output_group.add_argument(
'--echo-parseable-output',
action='store_true',
default=False,
help='Print Jenkins related environment variables and exit')
output_group.add_argument(
'--pull-request',
type=int,
action=GetPullRequestAction,
default=None,
help='Include the Pull Request information in parseable output')
# Deployment Selection
deployment_group = parser.add_argument_group('Deployment Selection')
deployment_group.add_argument(
'--pre-mortem',
action='store_true',
default=False,
help='Don\'t try to deploy an new VM. Consider a VM deployed and only '
'execute post test suite execution commands. Right before killing the VM'
)
deployment_group_mutually_exclusive = deployment_group.add_mutually_exclusive_group(
)
deployment_group_mutually_exclusive.add_argument(
'--cloud',
action='store_true',
default=True,
help='Salt Cloud Deployment')
deployment_group_mutually_exclusive.add_argument(
'--lxc',
action='store_true',
default=False,
help='Salt LXC Deployment')
deployment_group.add_argument('--lxc-host',
default=None,
help='The host where to deploy the LXC VM')
# Execution Selections
execution_group = parser.add_argument_group('Execution Selection')
execution_group.add_argument(
'--peer',
action='store_true',
default=False,
help='Run salt commands through the peer system')
execution_group.add_argument('--scp',
action='store_true',
default=False,
help='Download logs and reports using SCP')
bootstrap_script_options = parser.add_argument_group(
'Bootstrap Script Options',
'In case there\'s a need to provide the bootstrap script from an alternate URL and/or from a specific commit.'
)
bootstrap_script_options.add_argument(
'--bootstrap-salt-url',
default=None,
help='The salt git repository url used to bootstrap a minion')
bootstrap_script_options.add_argument(
'--bootstrap-salt-commit',
default=None,
help='The salt git commit used to bootstrap a minion')
vm_options_group = parser.add_argument_group('VM Options')
vm_options_group.add_argument('vm_name',
nargs='?',
help='Virtual machine name')
vm_options_group.add_argument(
'--vm-prefix',
default=os.environ.get('JENKINS_VM_NAME_PREFIX', 'ZJENKINS'),
help='The bootstrapped machine name prefix. Default: %(default)r')
vm_options_group.add_argument(
'--vm-source',
default=os.environ.get('JENKINS_VM_SOURCE', None),
help=
'The VM source. In case of --cloud usage, the could profile name. In case of --lxc usage, the image name.'
)
vm_options_group.add_argument(
'--grain-target',
action='append',
default=[],
help=
'Match minions using compound matchers, the minion ID, plus the passed grain.'
)
vm_preparation_group = parser.add_argument_group(
'VM Preparation Options',
'Salt SLS selection to prepare the VM. The defaults are based on the salt-jenkins repository. See '
'https://github.com/saltstack/salt-jenkins')
vm_preparation_group.add_argument(
'--prep-sls',
default='git.salt',
help=
'The sls file to execute to prepare the system. Default: %(default)r')
vm_preparation_group.add_argument(
'--prep-sls-2',
default=None,
help='An optional 2nd system preparation SLS')
vm_preparation_group.add_argument('--sls',
default='testrun-no-deps',
help='The final sls file to execute.')
vm_preparation_group.add_argument(
'--pillar',
action='append',
nargs=2,
help=
'Pillar (key, value)s to pass to the sls file. Example: \'--pillar pillar_key pillar_value\''
)
vm_actions = parser.add_argument_group(
'VM Actions', 'Action to execute on a running VM')
vm_actions.add_argument('--delete-vm',
action='store_true',
default=False,
help='Delete a running VM')
vm_actions.add_argument(
'--download-remote-reports',
default=False,
action='store_true',
help='Download remote reports when running remote \'testrun\' state')
vm_actions.add_argument('--download-unittest-reports',
default=False,
action='store_true',
help='Download the XML unittest results')
vm_actions.add_argument('--download-coverage-report',
default=False,
action='store_true',
help='Download the XML coverage reports')
vm_actions.add_argument(
'--download-remote-logs',
default=False,
action='store_true',
help='Download remote minion and runtests log files')
vm_actions.add_argument(
'--remote-log-path',
action='append',
default=[],
help='Provide additional log paths to download from remote minion')
testing_source_options = parser.add_argument_group(
'Testing Options',
'In case there\'s a need to provide a different repository and/or commit from which the tests suite should be '
'executed on')
testing_source_options.add_argument(
'--test-transport',
default='zeromq',
choices=('zeromq', 'raet'),
help=
'Set to raet to run integration tests with raet transport. Default: %default'
)
testing_source_options.add_argument('--test-git-url',
default=None,
help='The testing git repository url')
testing_source_options.add_argument('--test-git-commit',
default=None,
help='The testing git commit to track')
options = parser.parse_args()
if not options.vm_source and not options.vm_name:
parser.error(
'Unable to get VM name from environ nor generate it without --vm-source'
)
if options.lxc:
if not options.lxc_host:
parser.error(
'Need to provide where to deploy the LXC VM by passing it to --lxc-host'
)
options.cloud = False
if not options.vm_name:
options.vm_name = get_vm_name(options)
if options.echo_parseable_output:
echo_parseable_environment(options)
parser.exit()
if options.lxc:
options.cloud = False
if options.pre_mortem:
# Run any actions supposed to be executed right before killing the VM
if options.download_remote_reports:
# Download unittest reports
download_unittest_reports(options)
# Download coverage report
download_coverage_report(options)
else:
if options.download_unittest_reports:
download_unittest_reports(options)
if options.download_coverage_report:
download_coverage_report(options)
if options.download_remote_logs:
download_remote_logs(options)
if options.delete_vm:
delete_vm(options)
parser.exit()
# RUN IT!!!
cmd = []
minion_target = build_minion_target(options)
if options.peer:
cmd.extend(['salt-call', 'publish.runner'])
else:
cmd.append('salt-run')
if options.lxc:
cmd.append('lxc.init')
if options.peer:
cmd.append('arg="{0}"'.format(
to_cli_yaml([
options.vm_name, 'host={0}'.format(options.lxc_host),
'image={0}'.format(options.vm_source)
])))
else:
cmd.extend([
options.vm_name, 'host={0}'.format(options.lxc_host),
'image={0}'.format(options.vm_source)
])
else:
cmd.append('cloud.profile')
if options.peer:
cmd.append('arg="{0}"'.format(
to_cli_yaml([options.vm_source, options.vm_name])))
else:
cmd.extend([options.vm_source, options.vm_name])
if options.cloud:
if options.bootstrap_salt_commit is not None:
if options.bootstrap_salt_url is None:
options.bootstrap_salt_url = 'https://github.com/saltstack/salt.git'
cmd.append(
'script_args="-D -g {bootstrap_salt_url} -n git {bootstrap_salt_commit}"'
)
else:
cmd.append('script-args="-D"')
cmd = ' '.join(cmd).format(**options.__dict__)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
retcode = proc.returncode
if retcode != 0:
print('Failed to bootstrap VM. Exit code: {0}'.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
print('VM Bootstrapped. Exit code: {0}'.format(retcode))
sys.stdout.flush()
print('Sleeping for 5 seconds to allow the minion to breathe a little')
sys.stdout.flush()
time.sleep(5)
if options.scp:
prepare_ssh_access(options)
if options.cloud:
if options.bootstrap_salt_commit is not None:
# Let's find out if the installed version matches the passed in pillar
# information
print('Grabbing bootstrapped minion version information ... ')
cmd = []
if options.peer:
cmd.extend(['salt-call', '--out=json', 'publish.publish'])
else:
cmd.extend(['salt', '-t', '100', '--out=json'])
cmd.extend([minion_target, 'test.version'])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
retcode = proc.returncode
if retcode != 0:
print(
'Failed to get the bootstrapped minion version. Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
if not stdout.strip():
print(
'Failed to get the bootstrapped minion version(no output). Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
try:
version_info = json.loads(stdout.strip())
if options.peer:
version_info = version_info['local']
else:
version_info = version_info[options.vm_name]
bootstrap_salt_commit = options.bootstrap_salt_commit
if re.match('v[0-9]+', bootstrap_salt_commit):
# We've been passed a tag
bootstrap_salt_commit = bootstrap_salt_commit[1:]
else:
# Most likely a git SHA
bootstrap_salt_commit = bootstrap_salt_commit[:7]
if bootstrap_salt_commit not in version_info:
print('\n\nATTENTION!!!!\n')
print(
'The boostrapped minion version commit does not contain the desired commit:'
)
print(' {0!r} does not contain {1!r}'.format(
version_info, bootstrap_salt_commit))
print('\n\n')
sys.stdout.flush()
#if options.delete_vm:
# delete_vm(options)
#sys.exit(retcode)
else:
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(
stdout.strip()))
# Run preparation SLS
time.sleep(3)
cmd = []
if options.peer:
cmd.append('salt-call')
if options.no_color:
cmd.append('--no-color')
cmd.append('publish.publish')
else:
cmd.extend(['salt', '-t', '1800'])
if options.no_color:
cmd.append('--no-color')
cmd.extend([
minion_target, 'state.sls', options.prep_sls,
'pillar="{0}"'.format(build_pillar_data(options))
])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print('Failed to execute the preparation SLS file. Exit code: {0}'.
format(proc.returncode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(proc.returncode)
if options.prep_sls_2 is not None:
time.sleep(3)
# Run the 2nd preparation SLS
cmd = []
if options.peer:
cmd.append('salt-call')
if options.no_color:
cmd.append('--no-color')
if options.peer:
cmd.append('publish.publish')
else:
cmd.extend(['salt', '-t', '1800'])
if options.no_color:
cmd.append('--no-color')
cmd.extend([
minion_target, 'state.sls', options.prep_sls_2,
'pillar="{0}"'.format(build_pillar_data(options))
])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode != 0:
print(
'Failed to execute the 2nd preparation SLS file. Exit code: {0}'
.format(proc.returncode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(proc.returncode)
# Run remote checks
if options.test_git_url is not None:
time.sleep(1)
# Let's find out if the cloned repository if checked out from the
# desired repository
print('Grabbing the cloned repository remotes information ... ')
cmd = []
if options.peer:
cmd.extend(['salt-call', '--out=json', 'publish.publish'])
else:
cmd.extend(['salt', '-t', '100', '--out=json'])
cmd.extend([minion_target, 'git.remote_get', '/testing'])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print('Failed to get the cloned repository remote. Exit code: {0}'.
format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
if not stdout:
print(
'Failed to get the cloned repository remote(no output). Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
try:
remotes_info = json.loads(stdout.strip())
if remotes_info is not None:
if options.peer:
remotes_info = remotes_info['local']
else:
remotes_info = remotes_info[options.vm_name]
if remotes_info is None or options.test_git_url not in remotes_info:
print('The cloned repository remote is not the desired one:')
print(' {0!r} is not in {1}'.format(options.test_git_url,
remotes_info))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
if options.test_git_commit is not None:
time.sleep(1)
# Let's find out if the cloned repository is checked out at the desired
# commit
print('Grabbing the cloned repository commit information ... ')
cmd = []
if options.peer:
cmd.extend(['salt-call', '--out=json', 'publish.publish'])
else:
cmd.extend(['salt', '-t', '100', '--out=json'])
cmd.extend([minion_target, 'git.revision', '/testing'])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
sys.stdout.flush()
retcode = proc.returncode
if retcode != 0:
print(
'Failed to get the cloned repository revision. Exit code: {0}'.
format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
if not stdout:
print(
'Failed to get the cloned repository revision(no output). Exit code: {0}'
.format(retcode))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
try:
revision_info = json.loads(stdout.strip())
if options.peer:
revision_info = revision_info['local']
else:
revision_info = revision_info[options.vm_name]
if revision_info[:7] != options.test_git_commit[:7]:
print('The cloned repository commit is not the desired one:')
print(' {0!r} != {1!r}'.format(revision_info[:7],
options.test_git_commit[:7]))
sys.stdout.flush()
if options.delete_vm:
delete_vm(options)
sys.exit(retcode)
print('matches!')
except ValueError:
print('Failed to load any JSON from {0!r}'.format(stdout.strip()))
# Run tests here
time.sleep(3)
cmd = []
if options.peer:
cmd.append('salt-call')
if options.no_color:
cmd.append('--no-color')
cmd.append('publish.publish')
else:
cmd.extend(['salt', '-t', '1800'])
if options.no_color:
cmd.append('--no-color')
cmd.extend([
minion_target, 'state.sls', options.sls,
'pillar="{0}"'.format(build_pillar_data(options))
])
if options.peer and ' ' in minion_target:
cmd.append('expr_form="compound"')
cmd = ' '.join(cmd)
print('Running CMD: {0!r}'.format(cmd))
sys.stdout.flush()
proc = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, _ = proc.communicate()
if stdout:
print(stdout)
sys.stdout.flush()
try:
match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout)
retcode = int(match.group('exitcode'))
except AttributeError:
# No regex matching
retcode = 1
except ValueError:
# Not a number!?
retcode = 1
except TypeError:
# No output!?
retcode = 1
if stdout:
# Anything else, raise the exception
raise
if options.download_remote_reports:
# Download unittest reports
download_unittest_reports(options)
# Download coverage report
download_coverage_report(options)
if options.download_remote_logs:
download_remote_logs(options)
if options.delete_vm:
delete_vm(options)
parser.exit(proc.returncode)
def root_cmd(command, tty, sudo, **kwargs):
'''
Wrapper for commands to be run as root
'''
if sudo:
command = 'sudo {0}'.format(command)
log.debug('Using sudo to run command {0}'.format(command))
ssh_args = []
if tty:
# Use double `-t` on the `ssh` command, it's necessary when `sudo` has
# `requiretty` enforced.
ssh_args.extend(['-t', '-t'])
ssh_args.extend([
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, ie, non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none'
])
if 'key_filename' in kwargs:
# There should never be both a password and an ssh key passed in, so
ssh_args.extend([
# tell SSH to skip password authentication
'-oPasswordAuthentication=no',
'-oChallengeResponseAuthentication=no',
# Make sure public key authentication is enabled
'-oPubkeyAuthentication=yes',
# No Keyboard interaction!
'-oKbdInteractiveAuthentication=no',
# Also, specify the location of the key file
'-i {0}'.format(kwargs['key_filename'])
])
cmd = 'ssh {0} {1[username]}@{1[hostname]} {2}'.format(
' '.join(ssh_args), kwargs, pipes.quote(command)
)
log.debug('SSH command: {0!r}'.format(cmd))
if 'password' in kwargs:
cmd = 'sshpass -p {0} {1}'.format(kwargs['password'], cmd)
try:
proc = NonBlockingPopen(
cmd,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
stream_stds=kwargs.get('display_ssh_output', True),
)
log.debug(
'Executing command(PID {0}): {1!r}'.format(
proc.pid, command
)
)
proc.poll_and_read_until_finish()
proc.communicate()
return proc.returncode
except Exception as err:
log.error(
'Failed to execute command {0!r}: {1}\n'.format(
command, err
),
exc_info=True
)
# Signal an error
return 1
def run(platform, provider, commit, clean):
'''
RUN!
'''
htag = hashlib.md5(str(random.randint(1, 100000000))).hexdigest()[:6]
vm_name = 'ZZZ{0}{1}'.format(platform, htag)
cmd = 'salt-cloud -l debug --script-args "-D -n git {0}" -p {1}_{2} {3}'.format(
commit, provider, platform, vm_name)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
proc.communicate()
if proc.returncode > 0:
print('Failed to bootstrap VM. Exit code: {0}'.format(proc.returncode))
sys.stdout.flush()
cleanup(clean, vm_name)
sys.exit(proc.returncode)
print('VM Bootstrapped. Exit code: {0}'.format(proc.returncode))
sys.stdout.flush()
# Run tests here
cmd = 'salt -t 1800 {0} state.sls testrun pillar="{{git_commit: {1}}}" --no-color'.format(
vm_name,
commit)
print('Running CMD: {0}'.format(cmd))
sys.stdout.flush()
proc = NonBlockingPopen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stream_stds=True
)
proc.poll_and_read_until_finish()
stdout, stderr = proc.communicate()
if stderr:
print(stderr)
if stdout:
print(stdout)
sys.stdout.flush()
try:
match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout)
retcode = int(match.group('exitcode'))
except AttributeError:
# No regex matching
retcode = 1
except ValueError:
# Not a number!?
retcode = 1
except TypeError:
# No output!?
retcode = 1
if stdout:
# Anything else, raise the exception
raise
cleanup(clean, vm_name)
return retcode
