def delete_vm(options): ''' Stop a VM ''' cmd = [] if options.peer: cmd.extend(['salt-call', 'publish.runner']) else: cmd.append('salt-run') if options.lxc: cmd.append('lxc.purge') else: cmd.append('cloud.destroy') cmd.append(options.vm_name) print('Running CMD: {0!r}'.format(' '.join(cmd))) sys.stdout.flush() proc = NonBlockingPopen( ' '.join(cmd), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate()
def win_cmd(command, **kwargs): ''' Wrapper for commands to be run against Windows boxes ''' try: proc = NonBlockingPopen( command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stream_stds=kwargs.get('display_ssh_output', True), ) log.debug( 'Executing command(PID {0}): {1!r}'.format( proc.pid, command ) ) proc.poll_and_read_until_finish() proc.communicate() return proc.returncode except Exception as err: log.error( 'Failed to execute command {0!r}: {1}\n'.format( command, err ), exc_info=True ) # Signal an error return 1
def sync_minion(options): if 'salt_minion_synced' not in options: cmd = [] if options.peer: cmd.extend(['salt-call', 'publish.runner']) else: cmd.append('salt') cmd.extend([ build_minion_target(options), 'saltutil.sync_all' ]) print('Running CMD: {0!r}'.format(' '.join(cmd))) sys.stdout.flush() proc = NonBlockingPopen( ' '.join(cmd), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) sys.exit(proc.returncode) setattr(options, 'salt_minion_synced', 'yes')
def download_unittest_reports(options): ''' Download the generated unit test reports from minion ''' print('Downloading remote unittest reports...') sys.stdout.flush() xml_reports_path = os.path.join(options.workspace, 'xml-test-reports') if os.path.isdir(xml_reports_path): shutil.rmtree(xml_reports_path) if options.scp: cmds = ( ' '.join(build_scp_command(options, '-r', 'root@{0}:/tmp/xml-unitests-output/*'.format( get_minion_external_address(options) ), os.path.join(options.workspace, 'xml-test-reports'))), ) else: os.makedirs(xml_reports_path) cmds = ( '{0} {1} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/', '{0} {1} cp.push /tmp/xml-test-reports.tar.gz', 'mv -f /var/cache/salt/master/minions/{2}/files/tmp/xml-test-reports.tar.gz {3} && ' 'tar zxvf {3}/xml-test-reports.tar.gz -C {3}/xml-test-reports && ' 'rm -f {3}/xml-test-reports.tar.gz' ) for cmd in cmds: cmd = cmd.format( 'salt-call publish.publish' if options.lxc else 'salt', build_minion_target(options), options.vm_name, options.workspace ) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) time.sleep(0.25)
def download_remote_logs(options): print('Downloading remote logs...') sys.stdout.flush() workspace = options.workspace vm_name = options.download_remote_logs for fname in ('salt-runtests.log', 'minion.log'): if os.path.isfile(os.path.join(workspace, fname)): os.unlink(os.path.join(workspace, fname)) if not options.remote_log_path: options.remote_log_path = [ '/tmp/salt-runtests.log', '/var/log/salt/minion' ] cmds = [] for remote_log in options.remote_log_path: cmds.extend([ 'salt {{0}} archive.gzip {0}'.format(remote_log), 'salt {{0}} cp.push {0}.gz'.format(remote_log), 'gunzip /var/cache/salt/master/minions/{{1}}/files{0}.gz'.format(remote_log), 'mv /var/cache/salt/master/minions/{{1}}/files{0} {{2}}/{1}'.format( remote_log, '{0}{1}'.format( os.path.basename(remote_log), '' if remote_log.endswith('.log') else '.log' ) ) ]) for cmd in cmds: cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) time.sleep(0.25)
def test_master_startup(self): proc = NonBlockingPopen( [ sys.executable, self.get_script_path("master"), "-c", RUNTIME_VARS.TMP_CONF_DIR, "-l", "info", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) out = b"" err = b"" # Testing this should never be longer than 1 minute max_time = time.time() + 60 try: while True: if time.time() > max_time: assert False, "Max timeout ocurred" time.sleep(0.5) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if b"DeprecationWarning: object() takes no parameters" in out: self.fail( "'DeprecationWarning: object() takes no parameters' was seen in" " output" ) if b"TypeError: object() takes no parameters" in out: self.fail( "'TypeError: object() takes no parameters' was seen in output" ) if b"Setting up the master communication server" in out: # We got past the place we need, stop the process break if out is None and err is None: break if proc.poll() is not None: break finally: terminate_process(proc.pid, kill_children=True)
def download_coverage_report(options): ''' Download the generated coverage report from minion ''' print('Downloading remote coverage report...') sys.stdout.flush() if os.path.isfile(os.path.join(options.workspace, 'coverage.xml')): os.unlink(os.path.join(options.workspace, 'coverage.xml')) if options.scp: cmds = ( ' '.join(build_scp_command(options, 'root@{0}:/tmp/coverage.xml'.format( get_minion_external_address(options) ), os.path.join(options.workspace, 'coverage.xml'))), ) else: cmds = ( '{0} {1} archive.gzip /tmp/coverage.xml', '{0} {1} cp.push /tmp/coverage.xml.gz', 'gunzip /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml.gz', 'mv /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml {3}' ) for cmd in cmds: cmd = cmd.format( 'salt-call publish.publish' if options.lxc else 'salt', build_minion_target(options), options.vm_name, options.workspace ) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) time.sleep(0.25)
def prepare_ssh_access(options): ''' Generate a temporary SSH key, valid for one hour, and set it as an authorized key in the minion's root user account on the remote system. ''' print('Generating temporary SSH Key') ssh_key_path = os.path.join(options.workspace, 'jenkins_ssh_key_test') subprocess.call( 'ssh-keygen -t ecdsa -b 521 -C "$(whoami)@$(hostname)-$(date --rfc-3339=seconds)" ' '-f {0} -N \'\' -V -10m:+1h'.format(ssh_key_path), shell=True, ) cmd = [] if options.peer: cmd.extend(['salt-call', 'publish.publish']) else: cmd.append('salt') pub_key_contents = open('{0}.pub'.format(ssh_key_path)).read().strip() enc, key, comment = pub_key_contents.split(' ', 2) cmd.extend([ build_minion_target(options), 'ssh.set_auth_key', 'root', '{0!r}'.format(key), 'enc={0}'.format(enc), 'comment={0!r}'.format(comment) ]) cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) )
def test_master_startup(self): proc = NonBlockingPopen( [ self.get_script_path('master'), '-c', self.config_dir, '-l', 'info' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) out = six.b('') err = six.b('') # Testing this should never be longer than 1 minute max_time = time.time() + 60 try: while True: if time.time() > max_time: assert False, 'Max timeout ocurred' time.sleep(0.5) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if six.b('DeprecationWarning: object() takes no parameters') in out: self.fail('\'DeprecationWarning: object() takes no parameters\' was seen in output') if six.b('TypeError: object() takes no parameters') in out: self.fail('\'TypeError: object() takes no parameters\' was seen in output') if six.b('Setting up the master communication server') in out: # We got past the place we need, stop the process break if out is None and err is None: break if proc.poll() is not None: break finally: terminate_process(proc.pid, kill_children=True)
def cleanup(clean, vm_name): if not clean: return cmd = 'salt-cloud -d {0} -y'.format(vm_name) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate()
def delete_vm(options): ''' Stop a VM ''' cmd = 'salt-cloud -d {0} -y'.format(options.delete_vm) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish(interval=0.5) proc.communicate()
def download_packages(options): print('Downloading packages...') sys.stdout.flush() workspace = options.workspace vm_name = options.download_packages for fglob in ('salt-*.rpm', 'salt-*.deb', 'salt-*.pkg.xz', 'salt-buildpackage.log'): for fname in glob.glob(os.path.join(workspace, fglob)): if os.path.isfile(fname): os.unlink(fname) cmds = [ ('salt {{0}} archive.tar czf {0}.tar.gz sources=\'*.*\' cwd={0}' .format(options.package_artifact_dir)), 'salt {{0}} cp.push {0}.tar.gz'.format(options.package_artifact_dir), ('tar -C {{2}} -xzf /var/cache/salt/master/minions/{{1}}/files{0}.tar.gz' .format(options.package_artifact_dir)), ] for cmd in cmds: cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) time.sleep(0.25)
def download_remote_logs(options): print('Downloading remote logs...') sys.stdout.flush() workspace = options.workspace vm_name = options.download_remote_logs for fname in ('salt-runtests.log', 'minion.log'): if os.path.isfile(os.path.join(workspace, fname)): os.unlink(os.path.join(workspace, fname)) cmds = ( 'salt {0} archive.gzip /tmp/salt-runtests.log', 'salt {0} archive.gzip /var/log/salt/minion', 'salt {0} cp.push /tmp/salt-runtests.log.gz', 'salt {0} cp.push /var/log/salt/minion.gz', 'gunzip /var/cache/salt/master/minions/{0}/files/tmp/salt-runtests.log.gz', 'gunzip /var/cache/salt/master/minions/{0}/files/var/log/salt/minion.gz', 'mv /var/cache/salt/master/minions/{0}/files/tmp/salt-runtests.log {1}/salt-runtests.log', 'mv /var/cache/salt/master/minions/{0}/files/var/log/salt/minion {1}/minion.log' ) for cmd in cmds: cmd = cmd.format(vm_name, workspace) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) time.sleep(0.25)
def download_unittest_reports(options): print('Downloading remote unittest reports...') sys.stdout.flush() workspace = options.workspace xml_reports_path = os.path.join(workspace, 'xml-test-reports') if os.path.isdir(xml_reports_path): shutil.rmtree(xml_reports_path) os.makedirs(xml_reports_path) cmds = ( 'salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unittests-output/', 'salt {0} cp.push /tmp/xml-test-reports.tar.gz', 'mv -f /var/cache/salt/master/minions/{1}/files/tmp/xml-test-reports.tar.gz {2} && ' 'tar zxvf {2}/xml-test-reports.tar.gz -C {2}/xml-test-reports && ' 'rm -f {2}/xml-test-reports.tar.gz' ) vm_name = options.download_unittest_reports for cmd in cmds: cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) time.sleep(0.25)
def download_unittest_reports(options): print('Downloading remote unittest reports...') sys.stdout.flush() if os.path.isdir('xml-test-reports'): shutil.rmtree('xml-test-reports') os.makedirs('xml-test-reports') cmds = ( 'salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/', 'salt {0} cp.push /tmp/xml-test-reports.tar.gz', 'mv -f /var/cache/salt/master/minions/{0}/files/tmp/xml-test-reports.tar.gz {1}', 'tar zxvf {1}/xml-test-reports.tar.gz -C {1}/xml-test-reports', 'rm -f {1}/xml-test-reports.tar.gz') vm_name = options.download_unittest_reports workspace = options.workspace for cmd in cmds: cmd = cmd.format(vm_name, workspace) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('\nFailed to execute command. Exit code: {0}'.format( proc.returncode)) time.sleep(0.25)
def download_coverage_report(options): print('Downloading remote coverage report...') sys.stdout.flush() workspace = options.workspace vm_name = options.download_coverage_report if os.path.isfile(os.path.join(workspace, 'coverage.xml')): os.unlink(os.path.join(workspace, 'coverage.xml')) cmds = ( 'salt {0} archive.gzip /tmp/coverage.xml', 'salt {0} cp.push /tmp/coverage.xml.gz', 'gunzip /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml.gz', 'mv /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml {2}') for cmd in cmds: cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() if proc.returncode != 0: print('\nFailed to execute command. Exit code: {0}'.format( proc.returncode)) time.sleep(0.25)
def sync_minion(options): if 'salt_minion_synced' not in options: cmd = [] if options.peer: cmd.extend(['salt-call', 'publish.runner']) else: cmd.append('salt') cmd.extend([build_minion_target(options), 'saltutil.sync_all']) print('Running CMD: {0!r}'.format(' '.join(cmd))) sys.stdout.flush() proc = NonBlockingPopen(' '.join(cmd), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('\nFailed to execute command. Exit code: {0}'.format( proc.returncode)) sys.exit(proc.returncode) setattr(options, 'salt_minion_synced', 'yes')
def delete_vm(options): ''' Stop a VM ''' cmd = [] if options.peer: cmd.extend(['salt-call', 'publish.runner']) else: cmd.append('salt-run') if options.lxc: cmd.append('lxc.purge') else: cmd.append('cloud.destroy') cmd.append(options.vm_name) print('Running CMD: {0!r}'.format(' '.join(cmd))) sys.stdout.flush() proc = NonBlockingPopen(' '.join(cmd), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate()
def download_unittest_reports(options): print('Downloading remote unittest reports...') sys.stdout.flush() if os.path.isdir('xml-test-reports'): shutil.rmtree('xml-test-reports') os.makedirs('xml-test-reports') cmds = ( 'salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/', 'salt {0} cp.push /tmp/xml-test-reports.tar.gz', 'mv -f /var/cache/salt/master/minions/{0}/files/tmp/xml-test-reports.tar.gz {1}', 'tar zxvf {1}/xml-test-reports.tar.gz -C {1}/xml-test-reports', 'rm -f {1}/xml-test-reports.tar.gz' ) vm_name = options.download_unittest_reports workspace = options.workspace for cmd in cmds: cmd = cmd.format(vm_name, workspace) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) )
def download_coverage_report(options): print('Downloading remote coverage report...') sys.stdout.flush() workspace = options.workspace vm_name = options.download_coverage_report if os.path.isfile(os.path.join(workspace, 'coverage.xml')): os.unlink(os.path.join(workspace, 'coverage.xml')) cmds = ( 'salt {0} archive.gzip /tmp/coverage.xml', 'salt {0} cp.push /tmp/coverage.xml.gz', 'gunzip /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml.gz', 'mv /var/cache/salt/master/minions/{1}/files/tmp/coverage.xml {2}' ) for cmd in cmds: cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) time.sleep(0.25)
def scp_file(dest_path, contents, kwargs): ''' Use scp to copy a file to a server ''' tmpfh, tmppath = tempfile.mkstemp() with salt.utils.fopen(tmppath, 'w') as tmpfile: tmpfile.write(contents) log.debug('Uploading {0} to {1} (scp)'.format(dest_path, kwargs['hostname'])) ssh_args = [ # Don't add new hosts to the host key database '-oStrictHostKeyChecking=no', # Set hosts key database path to /dev/null, ie, non-existing '-oUserKnownHostsFile=/dev/null', # Don't re-use the SSH connection. Less failures. '-oControlPath=none' ] if 'key_filename' in kwargs: # There should never be both a password and an ssh key passed in, so ssh_args.extend([ # tell SSH to skip password authentication '-oPasswordAuthentication=no', '-oChallengeResponseAuthentication=no', # Make sure public key authentication is enabled '-oPubkeyAuthentication=yes', # No Keyboard interaction! '-oKbdInteractiveAuthentication=no', # Also, specify the location of the key file '-i {0}'.format(kwargs['key_filename']) ]) cmd = 'scp {0} {1} {2[username]}@{2[hostname]}:{3}'.format( ' '.join(ssh_args), tmppath, kwargs, dest_path) log.debug('SCP command: {0!r}'.format(cmd)) if 'password' in kwargs: cmd = 'sshpass -p {0} {1}'.format(kwargs['password'], cmd) try: proc = NonBlockingPopen( cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stream_stds=kwargs.get('display_ssh_output', True), ) log.debug('Uploading file(PID {0}): {1!r}'.format(proc.pid, dest_path)) proc.poll_and_read_until_finish() proc.communicate() return proc.returncode except Exception as err: log.error('Failed to upload file {0!r}: {1}\n'.format(dest_path, err), exc_info=True) # Signal an error return 1
def delete_vm(options): ''' Stop a VM ''' cmd = 'salt-cloud -d {0} -y'.format(options.delete_vm) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish(interval=0.5) proc.communicate()
def download_unittest_reports(options): ''' Download the generated unit test reports from minion ''' print('Downloading remote unittest reports...') sys.stdout.flush() xml_reports_path = os.path.join(options.workspace, 'xml-test-reports') if os.path.isdir(xml_reports_path): shutil.rmtree(xml_reports_path) if options.scp: cmds = (' '.join( build_scp_command( options, '-r', 'root@{0}:/tmp/xml-unitests-output/*'.format( get_minion_external_address(options)), os.path.join(options.workspace, 'xml-test-reports'))), ) else: os.makedirs(xml_reports_path) cmds = ( '{0} {1} archive.tar zcvf /tmp/xml-test-reports.tar.gz \'*.xml\' cwd=/tmp/xml-unitests-output/', '{0} {1} cp.push /tmp/xml-test-reports.tar.gz', 'mv -f /var/cache/salt/master/minions/{2}/files/tmp/xml-test-reports.tar.gz {3} && ' 'tar zxvf {3}/xml-test-reports.tar.gz -C {3}/xml-test-reports && ' 'rm -f {3}/xml-test-reports.tar.gz') for cmd in cmds: cmd = cmd.format( 'salt-call publish.publish' if options.lxc else 'salt', build_minion_target(options), options.vm_name, options.workspace) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('\nFailed to execute command. Exit code: {0}'.format( proc.returncode)) time.sleep(0.25)
def download_coverage_report(options): ''' Download the generated coverage report from minion ''' print('Downloading remote coverage report...') sys.stdout.flush() if os.path.isfile(os.path.join(options.workspace, 'coverage.xml')): os.unlink(os.path.join(options.workspace, 'coverage.xml')) if options.scp: cmds = (' '.join( build_scp_command( options, 'root@{0}:/tmp/coverage.xml'.format( get_minion_external_address(options)), os.path.join(options.workspace, 'coverage.xml'))), ) else: cmds = ( '{0} {1} archive.gzip /tmp/coverage.xml', '{0} {1} cp.push /tmp/coverage.xml.gz', 'gunzip /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml.gz', 'mv /var/cache/salt/master/minions/{2}/files/tmp/coverage.xml {3}') for cmd in cmds: cmd = cmd.format( 'salt-call publish.publish' if options.lxc else 'salt', build_minion_target(options), options.vm_name, options.workspace) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('\nFailed to execute command. Exit code: {0}'.format( proc.returncode)) time.sleep(0.25)
def download_unittest_reports(options): print("Downloading remote unittest reports...") sys.stdout.flush() workspace = options.workspace xml_reports_path = os.path.join(workspace, "xml-test-reports") if os.path.isdir(xml_reports_path): shutil.rmtree(xml_reports_path) os.makedirs(xml_reports_path) cmds = ( "salt {0} archive.tar zcvf /tmp/xml-test-reports.tar.gz '*.xml' cwd=/tmp/xml-unittests-output/", "salt {0} cp.push /tmp/xml-test-reports.tar.gz", "mv -f /var/cache/salt/master/minions/{1}/files/tmp/xml-test-reports.tar.gz {2} && " "tar zxvf {2}/xml-test-reports.tar.gz -C {2}/xml-test-reports && " "rm -f {2}/xml-test-reports.tar.gz", ) vm_name = options.download_unittest_reports for cmd in cmds: cmd = cmd.format(build_minion_target(options, vm_name), vm_name, workspace) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True, ) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() if proc.returncode != 0: print("\nFailed to execute command. Exit code: {0}".format( proc.returncode)) time.sleep(0.25)
def prepare_ssh_access(options): ''' Generate a temporary SSH key, valid for one hour, and set it as an authorized key in the minion's root user account on the remote system. ''' print('Generating temporary SSH Key') ssh_key_path = os.path.join(options.workspace, 'jenkins_ssh_key_test') subprocess.call( 'ssh-keygen -t ecdsa -b 521 -C "$(whoami)@$(hostname)-$(date --rfc-3339=seconds)" ' '-f {0} -N \'\' -V -10m:+1h'.format(ssh_key_path), shell=True, ) cmd = [] if options.peer: cmd.extend(['salt-call', 'publish.publish']) else: cmd.append('salt') pub_key_contents = open('{0}.pub'.format(ssh_key_path)).read().strip() enc, key, comment = pub_key_contents.split(' ', 2) cmd.extend([ build_minion_target(options), 'ssh.set_auth_key', 'root', '{0!r}'.format(key), 'enc={0}'.format(enc), 'comment={0!r}'.format(comment) ]) cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('\nFailed to execute command. Exit code: {0}'.format( proc.returncode))
def win_cmd(command, **kwargs): ''' Wrapper for commands to be run against Windows boxes ''' try: proc = NonBlockingPopen( command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stream_stds=kwargs.get('display_ssh_output', True), ) log.debug('Executing command(PID {0}): {1!r}'.format( proc.pid, command)) proc.poll_and_read_until_finish() proc.communicate() return proc.returncode except Exception as err: log.error('Failed to execute command {0!r}: {1}\n'.format( command, err), exc_info=True) # Signal an error return 1
def main(): ''' Main script execution ''' parser = argparse.ArgumentParser( description='Jenkins execution helper' ) parser.add_argument( '-w', '--workspace', default=os.path.abspath(os.environ.get('WORKSPACE', os.getcwd())), help='Path to the execution workspace' ) # Output Options output_group = parser.add_argument_group('Output Options') output_group.add_argument( '--no-color', '--no-colour', action='store_true', default=False, help='Don\'t use colors' ) output_group.add_argument( '--echo-parseable-output', action='store_true', default=False, help='Print Jenkins related environment variables and exit' ) output_group.add_argument( '--pull-request', type=int, action=GetPullRequestAction, default=None, help='Include the Pull Request information in parseable output' ) # Deployment Selection deployment_group = parser.add_argument_group('Deployment Selection') deployment_group.add_argument( '--pre-mortem', action='store_true', default=False, help='Don\'t try to deploy an new VM. Consider a VM deployed and only ' 'execute post test suite execution commands. Right before killing the VM' ) deployment_group_mutually_exclusive = deployment_group.add_mutually_exclusive_group() deployment_group_mutually_exclusive.add_argument( '--cloud', action='store_true', default=True, help='Salt Cloud Deployment' ) deployment_group_mutually_exclusive.add_argument( '--lxc', action='store_true', default=False, help='Salt LXC Deployment' ) deployment_group.add_argument( '--lxc-host', default=None, help='The host where to deploy the LXC VM' ) # Execution Selections execution_group = parser.add_argument_group('Execution Selection') execution_group.add_argument( '--peer', action='store_true', default=False, help='Run salt commands through the peer system' ) execution_group.add_argument( '--scp', action='store_true', default=False, help='Download logs and reports using SCP' ) bootstrap_script_options = parser.add_argument_group( 'Bootstrap Script Options', 'In case there\'s a need to provide the bootstrap script from an alternate URL and/or from a specific commit.' ) bootstrap_script_options.add_argument( '--bootstrap-salt-url', default=None, help='The salt git repository url used to bootstrap a minion' ) bootstrap_script_options.add_argument( '--bootstrap-salt-commit', default=None, help='The salt git commit used to bootstrap a minion' ) vm_options_group = parser.add_argument_group('VM Options') vm_options_group.add_argument('vm_name', nargs='?', help='Virtual machine name') vm_options_group.add_argument( '--vm-prefix', default=os.environ.get('JENKINS_VM_NAME_PREFIX', 'ZJENKINS'), help='The bootstrapped machine name prefix. Default: %(default)r' ) vm_options_group.add_argument( '--vm-source', default=os.environ.get('JENKINS_VM_SOURCE', None), help='The VM source. In case of --cloud usage, the could profile name. In case of --lxc usage, the image name.' ) vm_options_group.add_argument( '--grain-target', action='append', default=[], help='Match minions using compound matchers, the minion ID, plus the passed grain.' ) vm_preparation_group = parser.add_argument_group( 'VM Preparation Options', 'Salt SLS selection to prepare the VM. The defaults are based on the salt-jenkins repository. See ' 'https://github.com/saltstack/salt-jenkins' ) vm_preparation_group.add_argument( '--prep-sls', default='git.salt', help='The sls file to execute to prepare the system. Default: %(default)r' ) vm_preparation_group.add_argument( '--prep-sls-2', default=None, help='An optional 2nd system preparation SLS' ) vm_preparation_group.add_argument( '--sls', default='testrun-no-deps', help='The final sls file to execute.' ) vm_preparation_group.add_argument( '--pillar', action='append', nargs=2, help='Pillar (key, value)s to pass to the sls file. Example: \'--pillar pillar_key pillar_value\'' ) vm_actions = parser.add_argument_group( 'VM Actions', 'Action to execute on a running VM' ) vm_actions.add_argument( '--delete-vm', action='store_true', default=False, help='Delete a running VM' ) vm_actions.add_argument( '--download-remote-reports', default=False, action='store_true', help='Download remote reports when running remote \'testrun\' state' ) vm_actions.add_argument( '--download-unittest-reports', default=False, action='store_true', help='Download the XML unittest results' ) vm_actions.add_argument( '--download-coverage-report', default=False, action='store_true', help='Download the XML coverage reports' ) vm_actions.add_argument( '--download-remote-logs', default=False, action='store_true', help='Download remote minion and runtests log files' ) vm_actions.add_argument( '--remote-log-path', action='append', default=[], help='Provide additional log paths to download from remote minion' ) testing_source_options = parser.add_argument_group( 'Testing Options', 'In case there\'s a need to provide a different repository and/or commit from which the tests suite should be ' 'executed on' ) testing_source_options.add_argument( '--test-transport', default='zeromq', choices=('zeromq', 'raet'), help='Set to raet to run integration tests with raet transport. Default: %default') testing_source_options.add_argument( '--test-git-url', default=None, help='The testing git repository url') testing_source_options.add_argument( '--test-git-commit', default=None, help='The testing git commit to track') options = parser.parse_args() if not options.vm_source and not options.vm_name: parser.error('Unable to get VM name from environ nor generate it without --vm-source') if options.lxc: if not options.lxc_host: parser.error('Need to provide where to deploy the LXC VM by passing it to --lxc-host') options.cloud = False if not options.vm_name: options.vm_name = get_vm_name(options) if options.echo_parseable_output: echo_parseable_environment(options) parser.exit() if options.lxc: options.cloud = False if options.pre_mortem: # Run any actions supposed to be executed right before killing the VM if options.download_remote_reports: # Download unittest reports download_unittest_reports(options) # Download coverage report download_coverage_report(options) else: if options.download_unittest_reports: download_unittest_reports(options) if options.download_coverage_report: download_coverage_report(options) if options.download_remote_logs: download_remote_logs(options) if options.delete_vm: delete_vm(options) parser.exit() # RUN IT!!! cmd = [] minion_target = build_minion_target(options) if options.peer: cmd.extend(['salt-call', 'publish.runner']) else: cmd.append('salt-run') if options.lxc: cmd.append('lxc.init') if options.peer: cmd.append( 'arg="{0}"'.format( to_cli_yaml([ options.vm_name, 'host={0}'.format(options.lxc_host), 'image={0}'.format(options.vm_source) ]) ) ) else: cmd.extend([options.vm_name, 'host={0}'.format(options.lxc_host), 'image={0}'.format(options.vm_source)]) else: cmd.append('cloud.profile') if options.peer: cmd.append( 'arg="{0}"'.format( to_cli_yaml([options.vm_source, options.vm_name]) ) ) else: cmd.extend([options.vm_source, options.vm_name]) if options.cloud: if options.bootstrap_salt_commit is not None: if options.bootstrap_salt_url is None: options.bootstrap_salt_url = 'https://github.com/saltstack/salt.git' cmd.append( 'script_args="-D -g {bootstrap_salt_url} -n git {bootstrap_salt_commit}"' ) else: cmd.append('script-args="-D"') cmd = ' '.join(cmd).format(**options.__dict__) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to bootstrap VM. Exit code: {0}'.format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) print('VM Bootstrapped. Exit code: {0}'.format(retcode)) sys.stdout.flush() print('Sleeping for 5 seconds to allow the minion to breathe a little') sys.stdout.flush() time.sleep(5) if options.scp: prepare_ssh_access(options) if options.cloud: if options.bootstrap_salt_commit is not None: # Let's find out if the installed version matches the passed in pillar # information print('Grabbing bootstrapped minion version information ... ') cmd = [] if options.peer: cmd.extend(['salt-call', '--out=json', 'publish.publish']) else: cmd.extend(['salt', '-t', '100', '--out=json']) cmd.extend([minion_target, 'test.version']) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to get the bootstrapped minion version. Exit code: {0}'.format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) if not stdout.strip(): print('Failed to get the bootstrapped minion version(no output). Exit code: {0}'.format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) try: version_info = json.loads(stdout.strip()) if options.peer: version_info = version_info['local'] else: version_info = version_info[options.vm_name] bootstrap_salt_commit = options.bootstrap_salt_commit if re.match('v[0-9]+', bootstrap_salt_commit): # We've been passed a tag bootstrap_salt_commit = bootstrap_salt_commit[1:] else: # Most likely a git SHA bootstrap_salt_commit = bootstrap_salt_commit[:7] if bootstrap_salt_commit not in version_info: print('\n\nATTENTION!!!!\n') print('The boostrapped minion version commit does not contain the desired commit:') print(' {0!r} does not contain {1!r}'.format(version_info, bootstrap_salt_commit)) print('\n\n') sys.stdout.flush() #if options.delete_vm: # delete_vm(options) #sys.exit(retcode) else: print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) # Run preparation SLS time.sleep(3) cmd = [] if options.peer: cmd.append('salt-call') if options.no_color: cmd.append('--no-color') cmd.append('publish.publish') else: cmd.extend(['salt', '-t', '1800']) if options.no_color: cmd.append('--no-color') cmd.extend([ minion_target, 'state.sls', options.prep_sls, 'pillar="{0}"'.format(build_pillar_data(options)) ]) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('Failed to execute the preparation SLS file. Exit code: {0}'.format(proc.returncode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(proc.returncode) if options.prep_sls_2 is not None: time.sleep(3) # Run the 2nd preparation SLS cmd = [] if options.peer: cmd.append('salt-call') if options.no_color: cmd.append('--no-color') if options.peer: cmd.append('publish.publish') else: cmd.extend(['salt', '-t', '1800']) if options.no_color: cmd.append('--no-color') cmd.extend([ minion_target, 'state.sls', options.prep_sls_2, 'pillar="{0}"'.format(build_pillar_data(options)) ]) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('Failed to execute the 2nd preparation SLS file. Exit code: {0}'.format(proc.returncode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(proc.returncode) # Run remote checks if options.test_git_url is not None: time.sleep(1) # Let's find out if the cloned repository if checked out from the # desired repository print('Grabbing the cloned repository remotes information ... ') cmd = [] if options.peer: cmd.extend(['salt-call', '--out=json', 'publish.publish']) else: cmd.extend(['salt', '-t', '100', '--out=json']) cmd.extend([minion_target, 'git.remote_get', '/testing']) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() sys.stdout.flush() retcode = proc.returncode if retcode != 0: print('Failed to get the cloned repository remote. Exit code: {0}'.format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) if not stdout: print('Failed to get the cloned repository remote(no output). Exit code: {0}'.format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) try: remotes_info = json.loads(stdout.strip()) if remotes_info is not None: if options.peer: remotes_info = remotes_info['local'] else: remotes_info = remotes_info[options.vm_name] if remotes_info is None or options.test_git_url not in remotes_info: print('The cloned repository remote is not the desired one:') print(' {0!r} is not in {1}'.format(options.test_git_url, remotes_info)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) if options.test_git_commit is not None: time.sleep(1) # Let's find out if the cloned repository is checked out at the desired # commit print('Grabbing the cloned repository commit information ... ') cmd = [] if options.peer: cmd.extend(['salt-call', '--out=json', 'publish.publish']) else: cmd.extend(['salt', '-t', '100', '--out=json']) cmd.extend([minion_target, 'git.revision', '/testing']) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() sys.stdout.flush() retcode = proc.returncode if retcode != 0: print('Failed to get the cloned repository revision. Exit code: {0}'.format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) if not stdout: print('Failed to get the cloned repository revision(no output). Exit code: {0}'.format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) try: revision_info = json.loads(stdout.strip()) if options.peer: revision_info = revision_info['local'] else: revision_info = revision_info[options.vm_name] if revision_info[:7] != options.test_git_commit[:7]: print('The cloned repository commit is not the desired one:') print(' {0!r} != {1!r}'.format(revision_info[:7], options.test_git_commit[:7])) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) # Run tests here time.sleep(3) cmd = [] if options.peer: cmd.append('salt-call') if options.no_color: cmd.append('--no-color') cmd.append('publish.publish') else: cmd.extend(['salt', '-t', '1800']) if options.no_color: cmd.append('--no-color') cmd.extend([ minion_target, 'state.sls', options.sls, 'pillar="{0}"'.format(build_pillar_data(options)) ]) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() if stdout: print(stdout) sys.stdout.flush() try: match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout) retcode = int(match.group('exitcode')) except AttributeError: # No regex matching retcode = 1 except ValueError: # Not a number!? retcode = 1 except TypeError: # No output!? retcode = 1 if stdout: # Anything else, raise the exception raise if options.download_remote_reports: # Download unittest reports download_unittest_reports(options) # Download coverage report download_coverage_report(options) if options.download_remote_logs: download_remote_logs(options) if options.delete_vm: delete_vm(options) parser.exit(proc.returncode)
def run(opts): ''' RUN! ''' vm_name = os.environ.get( 'JENKINS_SALTCLOUD_VM_NAME', generate_vm_name(opts) ) if opts.download_remote_reports: if opts.test_without_coverage is False: opts.download_coverage_report = vm_name opts.download_unittest_reports = vm_name opts.download_packages = vm_name if opts.bootstrap_salt_commit is not None: if opts.bootstrap_salt_url is None: opts.bootstrap_salt_url = 'https://github.com/saltstack/salt.git' cmd = ( 'salt-cloud -l debug' ' --script-args "-D -g {bootstrap_salt_url} -n git {1}"' ' -p {provider}_{platform} {0}'.format( vm_name, os.environ.get( 'SALT_MINION_BOOTSTRAP_RELEASE', opts.bootstrap_salt_commit ), **opts.__dict__ ) ) else: cmd = ( 'salt-cloud -l debug' ' --script-args "-D -n git {1}" -p {provider}_{platform} {0}'.format( vm_name, os.environ.get( 'SALT_MINION_BOOTSTRAP_RELEASE', opts.bootstrap_salt_commit ), **opts.__dict__ ) ) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to bootstrap VM. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) print('VM Bootstrapped. Exit code: {0}'.format(retcode)) sys.stdout.flush() print('Sleeping for 5 seconds to allow the minion to breathe a little') sys.stdout.flush() time.sleep(5) if opts.bootstrap_salt_commit is not None: # Let's find out if the installed version matches the passed in pillar # information print('Grabbing bootstrapped minion version information ... ') cmd = 'salt -t 100 {0} --out json test.version'.format(build_minion_target(opts, vm_name)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to get the bootstrapped minion version. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if not stdout.strip(): print('Failed to get the bootstrapped minion version(no output). Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) try: version_info = json.loads(stdout.strip()) bootstrap_minion_version = os.environ.get( 'SALT_MINION_BOOTSTRAP_RELEASE', opts.bootstrap_salt_commit[:7] ) print('Minion reported salt version: {0}'.format(version_info)) if bootstrap_minion_version not in version_info[vm_name]: print('\n\nATTENTION!!!!\n') print('The boostrapped minion version commit does not contain the desired commit:') print(' {0!r} does not contain {1!r}'.format(version_info[vm_name], bootstrap_minion_version)) print('\n\n') sys.stdout.flush() #if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: # delete_vm(opts) #sys.exit(retcode) else: print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) if opts.cloud_only: # Run Cloud Provider tests preparation SLS time.sleep(3) cmd = ( 'salt -t 900 {target} state.sls {cloud_prep_sls} pillar="{pillar}" ' '--no-color'.format( target=build_minion_target(opts, vm_name), cloud_prep_sls='cloud-only', pillar=build_pillar_data(opts), ) ) else: # Run standard preparation SLS time.sleep(3) cmd = ( 'salt -t 1800 {target} state.sls {prep_sls} pillar="{pillar}" ' '--no-color'.format( target=build_minion_target(opts, vm_name), prep_sls=opts.prep_sls, pillar=build_pillar_data(opts), ) ) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() retcode = proc.returncode if retcode != 0: print('Failed to execute the preparation SLS file. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if opts.cloud_only: time.sleep(3) # Run Cloud Provider tests pillar preparation SLS cmd = ( 'salt -t 600 {target} state.sls {cloud_prep_sls} pillar="{pillar}" ' '--no-color'.format( target=build_minion_target(opts, vm_name), cloud_prep_sls='cloud-test-configs', pillar=build_pillar_data(opts), ) ) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) stdout, stderr = proc.communicate() if stdout: # DO NOT print the state return here! print('Cloud configuration files provisioned via pillar.') sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() retcode = proc.returncode if retcode != 0: print('Failed to execute the preparation SLS file. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if opts.prep_sls_2 is not None: time.sleep(3) # Run the 2nd preparation SLS cmd = ( 'salt -t 30 {target} state.sls {prep_sls_2} pillar="{pillar}" ' '--no-color'.format( prep_sls_2=opts.prep_sls_2, pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), ) ) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() retcode = proc.returncode if retcode != 0: print('Failed to execute the 2nd preparation SLS file. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) # Run remote checks if opts.test_git_url is not None: time.sleep(1) # Let's find out if the cloned repository if checked out from the # desired repository print('Grabbing the cloned repository remotes information ... ') cmd = 'salt -t 100 {0} --out json git.remote_get /testing'.format(build_minion_target(opts, vm_name)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to get the cloned repository remote. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if not stdout: print('Failed to get the cloned repository remote(no output). Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) try: remotes_info = json.loads(stdout.strip()) if remotes_info is None or remotes_info[vm_name] is None or opts.test_git_url not in remotes_info[vm_name]: print('The cloned repository remote is not the desired one:') print(' {0!r} is not in {1}'.format(opts.test_git_url, remotes_info)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) if opts.test_git_commit is not None: time.sleep(1) # Let's find out if the cloned repository is checked out at the desired # commit print('Grabbing the cloned repository commit information ... ') cmd = 'salt -t 100 {0} --out json git.revision /testing'.format(build_minion_target(opts, vm_name)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() sys.stdout.flush() retcode = proc.returncode if retcode != 0: print('Failed to get the cloned repository revision. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if not stdout: print('Failed to get the cloned repository revision(no output). Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) try: revision_info = json.loads(stdout.strip()) if revision_info[vm_name][7:] != opts.test_git_commit[7:]: print('The cloned repository commit is not the desired one:') print(' {0!r} != {1!r}'.format(revision_info[vm_name][:7], opts.test_git_commit[:7])) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) # Run tests here time.sleep(3) cmd = ( 'salt -t 1800 {target} state.sls {sls} pillar="{pillar}" --no-color'.format( sls=opts.sls, pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), ) ) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() try: match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout) retcode = int(match.group('exitcode')) except AttributeError: # No regex matching retcode = 1 except ValueError: # Not a number!? retcode = 1 except TypeError: # No output!? retcode = 1 if stdout: # Anything else, raise the exception raise if retcode == 0: # Build packages time.sleep(3) cmd = ( 'salt -t 1800 {target} state.sls buildpackage pillar="{pillar}" --no-color'.format( pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), ) ) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() # Grab packages and log file (or just log file if build failed) download_packages(opts) if opts.download_remote_reports: # Download unittest reports download_unittest_reports(opts) # Download coverage report if opts.test_without_coverage is False: download_coverage_report(opts) if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) return retcode
def test_deferred_write_on_sigint(tmp_path): pyscript = dedent(r""" import sys import time import signal import logging CODE_DIR = {!r} if CODE_DIR in sys.path: sys.path.remove(CODE_DIR) sys.path.insert(0, CODE_DIR) from salt._logging.handlers import DeferredStreamHandler # Reset any logging handlers we might have already logging.root.handlers[:] = [] handler = DeferredStreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) logging.root.addHandler(handler) if signal.getsignal(signal.SIGINT) != signal.default_int_handler: # Looking at you Debian based distros :/ signal.signal(signal.SIGINT, signal.default_int_handler) log = logging.getLogger(__name__) start_printed = False while True: try: log.debug('Foo') if start_printed is False: sys.stdout.write('STARTED\n') sys.stdout.write('SIGINT HANDLER: {{!r}}\n'.format(signal.getsignal(signal.SIGINT))) sys.stdout.flush() start_printed = True time.sleep(0.125) except (KeyboardInterrupt, SystemExit): log.info('KeyboardInterrupt caught') sys.stdout.write('KeyboardInterrupt caught\n') sys.stdout.flush() break log.info('EXITING') sys.stdout.write('EXITING\n') sys.stdout.flush() sys.exit(0) """.format(RUNTIME_VARS.CODE_DIR)) script_path = tmp_path / "sigint_deferred_logging_test.py" script_path.write_text(pyscript, encoding="utf-8") proc = NonBlockingPopen( [sys.executable, str(script_path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) out = b"" err = b"" # Test should take less than 20 seconds, way less execution_time = 10 start = time.time() max_time = time.time() + execution_time try: signalled = False log.info("Starting Loop") while True: time.sleep(0.125) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if b"STARTED" in out and not signalled: # Enough time has passed proc.send_signal(signal.SIGINT) signalled = True log.debug("Sent SIGINT after: %s", time.time() - start) if signalled is False: if out: pytest.fail( "We have stdout output when there should be none: {}". format(out)) if err: pytest.fail( "We have stderr output when there should be none: {}". format(err)) if _out is None and _err is None: log.info("_out and _err are None") if b"Foo" not in err: pytest.fail( "No more output and 'Foo' should be in stderr and it's not: {}" .format(err)) break if proc.poll() is not None: log.debug("poll() is not None") if b"Foo" not in err: pytest.fail( "Process terminated and 'Foo' should be in stderr and it's not: {}" .format(err)) break if time.time() > max_time: log.debug("Reached max time") if b"Foo" not in err: pytest.fail( "'Foo' should be in stderr and it's not:\n{0}\nSTDERR:\n{0}\n{1}\n{0}\nSTDOUT:\n{0}\n{2}\n{0}" .format("-" * 80, err, out)) finally: terminate_process(proc.pid, kill_children=True) log.debug("Test took %s seconds", time.time() - start)
def run(opts): ''' RUN! ''' vm_name = os.environ.get('JENKINS_SALTCLOUD_VM_NAME', generate_vm_name(opts.platform)) if opts.download_remote_reports: opts.download_coverage_report = vm_name opts.download_unittest_reports = vm_name cmd = ('salt-cloud -l debug --script-args "-D -n git {commit}" -p ' '{provider}_{platform} {0}'.format(vm_name, **opts.__dict__)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to bootstrap VM. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(vm_name) sys.exit(retcode) print('VM Bootstrapped. Exit code: {0}'.format(retcode)) sys.stdout.flush() print('Sleeping for 5 seconds to allow the minion to breathe a little') sys.stdout.flush() time.sleep(5) # Run tests here cmd = ('salt -t 1800 {vm_name} state.sls {sls} pillar="{pillar}" ' '--no-color'.format(sls=opts.sls, pillar=opts.pillar.format(commit=opts.commit), vm_name=vm_name, commit=opts.commit)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() #proc = NonBlockingPopen( proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, # stream_stds=True ) #proc.poll_and_read_until_finish() stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() try: match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout) retcode = int(match.group('exitcode')) except AttributeError: # No regex matching retcode = 1 except ValueError: # Not a number!? retcode = 1 except TypeError: # No output!? retcode = 1 if stdout: # Anything else, raise the exception raise if opts.download_remote_reports: # Download unittest reports download_unittest_reports(opts) # Download coverage report download_coverage_report(opts) if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(vm_name) return retcode
def scp_file(dest_path, contents, kwargs): ''' Use scp to copy a file to a server ''' tmpfh, tmppath = tempfile.mkstemp() with salt.utils.fopen(tmppath, 'w') as tmpfile: tmpfile.write(contents) log.debug('Uploading {0} to {1} (scp)'.format(dest_path, kwargs['hostname'])) ssh_args = [ # Don't add new hosts to the host key database '-oStrictHostKeyChecking=no', # Set hosts key database path to /dev/null, ie, non-existing '-oUserKnownHostsFile=/dev/null', # Don't re-use the SSH connection. Less failures. '-oControlPath=none' ] if 'key_filename' in kwargs: # There should never be both a password and an ssh key passed in, so ssh_args.extend([ # tell SSH to skip password authentication '-oPasswordAuthentication=no', '-oChallengeResponseAuthentication=no', # Make sure public key authentication is enabled '-oPubkeyAuthentication=yes', # No Keyboard interaction! '-oKbdInteractiveAuthentication=no', # Also, specify the location of the key file '-i {0}'.format(kwargs['key_filename']) ]) cmd = 'scp {0} {1} {2[username]}@{2[hostname]}:{3}'.format( ' '.join(ssh_args), tmppath, kwargs, dest_path ) log.debug('SCP command: {0!r}'.format(cmd)) if 'password' in kwargs: cmd = 'sshpass -p {0} {1}'.format(kwargs['password'], cmd) try: proc = NonBlockingPopen( cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stream_stds=kwargs.get('display_ssh_output', True), ) log.debug( 'Uploading file(PID {0}): {1!r}'.format( proc.pid, dest_path ) ) proc.poll_and_read_until_finish() proc.communicate() return proc.returncode except Exception as err: log.error( 'Failed to upload file {0!r}: {1}\n'.format( dest_path, err ), exc_info=True ) # Signal an error return 1
def test_deferred_write_on_atexit(tmp_path): # Python will .flush() and .close() all logging handlers at interpreter shutdown. # This should be enough to flush our deferred messages. pyscript = dedent(r""" import sys import time import logging CODE_DIR = {!r} if CODE_DIR in sys.path: sys.path.remove(CODE_DIR) sys.path.insert(0, CODE_DIR) from salt._logging.handlers import DeferredStreamHandler # Reset any logging handlers we might have already logging.root.handlers[:] = [] handler = DeferredStreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) logging.root.addHandler(handler) log = logging.getLogger(__name__) sys.stdout.write('STARTED\n') sys.stdout.flush() log.debug('Foo') sys.exit(0) """.format(RUNTIME_VARS.CODE_DIR)) script_path = tmp_path / "atexit_deferred_logging_test.py" script_path.write_text(pyscript, encoding="utf-8") proc = NonBlockingPopen( [sys.executable, str(script_path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) out = b"" err = b"" # This test should never take more than 5 seconds execution_time = 5 max_time = time.time() + execution_time try: # Just loop consuming output while True: if time.time() > max_time: pytest.fail("Script didn't exit after {} second".format( execution_time)) time.sleep(0.125) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if _out is None and _err is None: # The script exited break if proc.poll() is not None: # The script exited break finally: terminate_process(proc.pid, kill_children=True) if b"Foo" not in err: pytest.fail("'Foo' should be in stderr and it's not: {}".format(err))
def run(opts): ''' RUN! ''' vm_name = os.environ.get( 'JENKINS_SALTCLOUD_VM_NAME', generate_vm_name(opts.platform) ) if opts.download_remote_reports: opts.download_coverage_report = vm_name opts.download_unittest_reports = vm_name cmd = ( 'salt-cloud -l debug' ' --script-args "-D -g {salt_url} -n git {commit}"' ' -p {provider}_{platform} {0}'.format(vm_name, **opts.__dict__) ) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to bootstrap VM. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(vm_name) sys.exit(retcode) print('VM Bootstrapped. Exit code: {0}'.format(retcode)) sys.stdout.flush() print('Sleeping for 5 seconds to allow the minion to breathe a little') sys.stdout.flush() time.sleep(5) # Do we need extra setup? if opts.salt_url != SALT_GIT_URL: cmds = ( 'salt -t 100 {vm_name} git.remote_set /testing name={0!r} url={1!r}'.format( 'upstream', SALT_GIT_URL, vm_name=vm_name ), 'salt -t 100 {vm_name} git.fetch /testing \'upstream --tags\''.format( vm_name=vm_name ) ) for cmd in cmds: print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) stdout, _ = proc.communicate() if stdout: print(stdout) sys.stdout.flush() # Run tests here cmd = ( 'salt -t 1800 {vm_name} state.sls {sls} pillar="{pillar}" ' '--no-color'.format( sls=opts.sls, pillar=opts.pillar.format( commit=opts.commit, salt_url=opts.salt_url ), vm_name=vm_name, commit=opts.commit ) ) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() #proc = NonBlockingPopen( proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, # stream_stds=True ) #proc.poll_and_read_until_finish() stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() try: match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout) retcode = int(match.group('exitcode')) except AttributeError: # No regex matching retcode = 1 except ValueError: # Not a number!? retcode = 1 except TypeError: # No output!? retcode = 1 if stdout: # Anything else, raise the exception raise if opts.download_remote_reports: # Download unittest reports download_unittest_reports(opts) # Download coverage report download_coverage_report(opts) if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(vm_name) return retcode
def root_cmd(command, tty, sudo, **kwargs): ''' Wrapper for commands to be run as root ''' if sudo: command = 'sudo {0}'.format(command) log.debug('Using sudo to run command {0}'.format(command)) ssh_args = [] if tty: # Use double `-t` on the `ssh` command, it's necessary when `sudo` has # `requiretty` enforced. ssh_args.extend(['-t', '-t']) ssh_args.extend([ # Don't add new hosts to the host key database '-oStrictHostKeyChecking=no', # Set hosts key database path to /dev/null, ie, non-existing '-oUserKnownHostsFile=/dev/null', # Don't re-use the SSH connection. Less failures. '-oControlPath=none' ]) if 'key_filename' in kwargs: # There should never be both a password and an ssh key passed in, so ssh_args.extend([ # tell SSH to skip password authentication '-oPasswordAuthentication=no', '-oChallengeResponseAuthentication=no', # Make sure public key authentication is enabled '-oPubkeyAuthentication=yes', # No Keyboard interaction! '-oKbdInteractiveAuthentication=no', # Also, specify the location of the key file '-i {0}'.format(kwargs['key_filename']) ]) cmd = 'ssh {0} {1[username]}@{1[hostname]} {2}'.format( ' '.join(ssh_args), kwargs, pipes.quote(command)) log.debug('SSH command: {0!r}'.format(cmd)) if 'password' in kwargs: cmd = 'sshpass -p {0} {1}'.format(kwargs['password'], cmd) try: proc = NonBlockingPopen( cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stream_stds=kwargs.get('display_ssh_output', True), ) log.debug('Executing command(PID {0}): {1!r}'.format( proc.pid, command)) proc.poll_and_read_until_finish() proc.communicate() return proc.returncode except Exception as err: log.error('Failed to execute command {0!r}: {1}\n'.format( command, err), exc_info=True) # Signal an error return 1
def run(opts): ''' RUN! ''' vm_name = os.environ.get('JENKINS_SALTCLOUD_VM_NAME', generate_vm_name(opts)) if opts.download_remote_reports: if opts.test_without_coverage is False: opts.download_coverage_report = vm_name opts.download_unittest_reports = vm_name opts.download_packages = vm_name if opts.bootstrap_salt_commit is not None: if opts.bootstrap_salt_url is None: opts.bootstrap_salt_url = 'https://github.com/saltstack/salt.git' cmd = ('salt-cloud -l debug' ' --script-args "-D -g {bootstrap_salt_url} -n git {1}"' ' -p {provider}_{platform} {0}'.format( vm_name, os.environ.get('SALT_MINION_BOOTSTRAP_RELEASE', opts.bootstrap_salt_commit), **opts.__dict__)) else: cmd = ('salt-cloud -l debug' ' --script-args "-D -n git {1}" -p {provider}_{platform} {0}'. format( vm_name, os.environ.get('SALT_MINION_BOOTSTRAP_RELEASE', opts.bootstrap_salt_commit), **opts.__dict__)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to bootstrap VM. Exit code: {0}'.format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) print('VM Bootstrapped. Exit code: {0}'.format(retcode)) sys.stdout.flush() print('Sleeping for 5 seconds to allow the minion to breathe a little') sys.stdout.flush() time.sleep(5) if opts.bootstrap_salt_commit is not None: # Let's find out if the installed version matches the passed in pillar # information print('Grabbing bootstrapped minion version information ... ') cmd = 'salt -t 100 {0} --out json test.version'.format( build_minion_target(opts, vm_name)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() retcode = proc.returncode if retcode != 0: print( 'Failed to get the bootstrapped minion version. Exit code: {0}' .format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if not stdout.strip(): print( 'Failed to get the bootstrapped minion version(no output). Exit code: {0}' .format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) try: version_info = json.loads(stdout.strip()) bootstrap_minion_version = os.environ.get( 'SALT_MINION_BOOTSTRAP_RELEASE', opts.bootstrap_salt_commit[:7]) print('Minion reported salt version: {0}'.format(version_info)) if bootstrap_minion_version not in version_info[vm_name]: print('\n\nATTENTION!!!!\n') print( 'The boostrapped minion version commit does not contain the desired commit:' ) print(' {0!r} does not contain {1!r}'.format( version_info[vm_name], bootstrap_minion_version)) print('\n\n') sys.stdout.flush() #if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: # delete_vm(opts) #sys.exit(retcode) else: print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) if opts.cloud_only: # Run Cloud Provider tests preparation SLS time.sleep(3) cmd = ( 'salt -t 900 {target} state.sls {cloud_prep_sls} pillar="{pillar}" ' '--no-color'.format( target=build_minion_target(opts, vm_name), cloud_prep_sls='cloud-only', pillar=build_pillar_data(opts), )) else: # Run standard preparation SLS time.sleep(3) cmd = ('salt -t 1800 {target} state.sls {prep_sls} pillar="{pillar}" ' '--no-color'.format( target=build_minion_target(opts, vm_name), prep_sls=opts.prep_sls, pillar=build_pillar_data(opts), )) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() retcode = proc.returncode if retcode != 0: print('Failed to execute the preparation SLS file. Exit code: {0}'. format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if opts.cloud_only: time.sleep(3) # Run Cloud Provider tests pillar preparation SLS cmd = ( 'salt -t 600 {target} state.sls {cloud_prep_sls} pillar="{pillar}" ' '--no-color'.format( target=build_minion_target(opts, vm_name), cloud_prep_sls='cloud-test-configs', pillar=build_pillar_data(opts), )) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) stdout, stderr = proc.communicate() if stdout: # DO NOT print the state return here! print('Cloud configuration files provisioned via pillar.') sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() retcode = proc.returncode if retcode != 0: print('Failed to execute the preparation SLS file. Exit code: {0}'. format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if opts.prep_sls_2 is not None: time.sleep(3) # Run the 2nd preparation SLS cmd = ('salt -t 30 {target} state.sls {prep_sls_2} pillar="{pillar}" ' '--no-color'.format( prep_sls_2=opts.prep_sls_2, pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), )) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() retcode = proc.returncode if retcode != 0: print( 'Failed to execute the 2nd preparation SLS file. Exit code: {0}' .format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) # Run remote checks if opts.test_git_url is not None: time.sleep(1) # Let's find out if the cloned repository if checked out from the # desired repository print('Grabbing the cloned repository remotes information ... ') cmd = 'salt -t 100 {0} --out json git.remote_get /testing'.format( build_minion_target(opts, vm_name)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to get the cloned repository remote. Exit code: {0}'. format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if not stdout: print( 'Failed to get the cloned repository remote(no output). Exit code: {0}' .format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) try: remotes_info = json.loads(stdout.strip()) if remotes_info is None or remotes_info[ vm_name] is None or opts.test_git_url not in remotes_info[ vm_name]: print('The cloned repository remote is not the desired one:') print(' {0!r} is not in {1}'.format(opts.test_git_url, remotes_info)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) if opts.test_git_commit is not None: time.sleep(1) # Let's find out if the cloned repository is checked out at the desired # commit print('Grabbing the cloned repository commit information ... ') cmd = 'salt -t 100 {0} --out json git.revision /testing'.format( build_minion_target(opts, vm_name)) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() sys.stdout.flush() retcode = proc.returncode if retcode != 0: print( 'Failed to get the cloned repository revision. Exit code: {0}'. format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) if not stdout: print( 'Failed to get the cloned repository revision(no output). Exit code: {0}' .format(retcode)) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) try: revision_info = json.loads(stdout.strip()) if revision_info[vm_name][7:] != opts.test_git_commit[7:]: print('The cloned repository commit is not the desired one:') print(' {0!r} != {1!r}'.format(revision_info[vm_name][:7], opts.test_git_commit[:7])) sys.stdout.flush() if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) sys.exit(retcode) print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) # Run tests here time.sleep(3) cmd = ('salt -t 1800 {target} state.sls {sls} pillar="{pillar}" --no-color' .format( sls=opts.sls, pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), )) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() try: match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout) retcode = int(match.group('exitcode')) except AttributeError: # No regex matching retcode = 1 except ValueError: # Not a number!? retcode = 1 except TypeError: # No output!? retcode = 1 if stdout: # Anything else, raise the exception raise if retcode == 0: # Build packages time.sleep(3) cmd = ( 'salt -t 1800 {target} state.sls buildpackage pillar="{pillar}" --no-color' .format( pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), )) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() if stdout: print(stdout) sys.stdout.flush() if stderr: print(stderr) sys.stderr.flush() # Grab packages and log file (or just log file if build failed) download_packages(opts) if opts.download_remote_reports: # Download unittest reports download_unittest_reports(opts) # Download coverage report if opts.test_without_coverage is False: download_coverage_report(opts) if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: delete_vm(opts) return retcode
def run(opts): """ RUN! """ vm_name = os.environ.get("JENKINS_SALTCLOUD_VM_NAME", generate_vm_name(opts)) if opts.download_remote_reports: if opts.test_without_coverage is False: opts.download_coverage_report = vm_name opts.download_unittest_reports = vm_name opts.download_packages = vm_name if opts.bootstrap_salt_commit is not None: if opts.bootstrap_salt_url is None: opts.bootstrap_salt_url = "https://github.com/saltstack/salt.git" cmd = ("salt-cloud -l debug" ' --script-args "-D -g {bootstrap_salt_url} -n git {1}"' " -p {provider}_{platform} {0}".format( vm_name, os.environ.get("SALT_MINION_BOOTSTRAP_RELEASE", opts.bootstrap_salt_commit), **opts.__dict__)) else: cmd = ("salt-cloud -l debug" ' --script-args "-D -n git {1}" -p {provider}_{platform} {0}'. format( vm_name, os.environ.get("SALT_MINION_BOOTSTRAP_RELEASE", opts.bootstrap_salt_commit), **opts.__dict__)) if opts.splay is not None: # Sleep a random number of seconds cloud_downtime = random.randint(0, opts.splay) print("Sleeping random period before calling salt-cloud: {0}".format( cloud_downtime)) time.sleep(cloud_downtime) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True, ) proc.poll_and_read_until_finish(interval=0.5) proc.communicate() retcode = proc.returncode if retcode != 0: print("Failed to bootstrap VM. Exit code: {0}".format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) print("VM Bootstrapped. Exit code: {0}".format(retcode)) sys.stdout.flush() # Sleep a random number of seconds bootstrap_downtime = random.randint(0, opts.splay) print("Sleeping for {0} seconds to allow the minion to breathe a little". format(bootstrap_downtime)) sys.stdout.flush() time.sleep(bootstrap_downtime) if opts.bootstrap_salt_commit is not None: # Let's find out if the installed version matches the passed in pillar # information print("Grabbing bootstrapped minion version information ... ") cmd = "salt -t 100 {0} --out json test.version".format( build_minion_target(opts, vm_name)) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() retcode = proc.returncode if retcode != 0: print( "Failed to get the bootstrapped minion version. Exit code: {0}" .format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) outstr = salt.utils.stringutils.to_str(stdout).strip() if not outstr: print( "Failed to get the bootstrapped minion version(no output). Exit code: {0}" .format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) try: version_info = salt.utils.json.loads(outstr) bootstrap_minion_version = os.environ.get( "SALT_MINION_BOOTSTRAP_RELEASE", opts.bootstrap_salt_commit[:7]) print("Minion reported salt version: {0}".format(version_info)) if bootstrap_minion_version not in version_info[vm_name]: print("\n\nATTENTION!!!!\n") print( "The boostrapped minion version commit does not contain the desired commit:" ) print(" '{0}' does not contain '{1}'".format( version_info[vm_name], bootstrap_minion_version)) print("\n\n") sys.stdout.flush() # if opts.clean and 'JENKINS_SALTCLOUD_VM_NAME' not in os.environ: # delete_vm(opts) # sys.exit(retcode) else: print("matches!") except ValueError: print("Failed to load any JSON from '{0}'".format(outstr)) if opts.cloud_only: # Run Cloud Provider tests preparation SLS cloud_provider_downtime = random.randint(3, opts.splay) time.sleep(cloud_provider_downtime) cmd = ( 'salt -t 900 {target} state.sls {cloud_prep_sls} pillar="{pillar}" ' "--no-color".format( target=build_minion_target(opts, vm_name), cloud_prep_sls="cloud-only", pillar=build_pillar_data(opts), )) else: # Run standard preparation SLS standard_sls_downtime = random.randint(3, opts.splay) time.sleep(standard_sls_downtime) cmd = ('salt -t 1800 {target} state.sls {prep_sls} pillar="{pillar}" ' "--no-color".format( target=build_minion_target(opts, vm_name), prep_sls=opts.prep_sls, pillar=build_pillar_data(opts), )) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() if stdout: print(salt.utils.stringutils.to_str(stdout)) if stderr: print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() retcode = proc.returncode if retcode != 0: print("Failed to execute the preparation SLS file. Exit code: {0}". format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) if opts.cloud_only: cloud_provider_pillar = random.randint(3, opts.splay) time.sleep(cloud_provider_pillar) # Run Cloud Provider tests pillar preparation SLS cmd = ( 'salt -t 600 {target} state.sls {cloud_prep_sls} pillar="{pillar}" ' "--no-color".format( target=build_minion_target(opts, vm_name), cloud_prep_sls="cloud-test-configs", pillar=build_pillar_data(opts), )) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) stdout, stderr = proc.communicate() if stdout: # DO NOT print the state return here! print("Cloud configuration files provisioned via pillar.") if stderr: print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() retcode = proc.returncode if retcode != 0: print("Failed to execute the preparation SLS file. Exit code: {0}". format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) if opts.prep_sls_2 is not None: sls_2_downtime = random.randint(3, opts.splay) time.sleep(sls_2_downtime) # Run the 2nd preparation SLS cmd = ('salt -t 30 {target} state.sls {prep_sls_2} pillar="{pillar}" ' "--no-color".format( prep_sls_2=opts.prep_sls_2, pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), )) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) stdout, stderr = proc.communicate() if stdout: print(salt.utils.stringutils.to_str(stdout)) if stderr: print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() retcode = proc.returncode if retcode != 0: print( "Failed to execute the 2nd preparation SLS file. Exit code: {0}" .format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) # Run remote checks if opts.test_git_url is not None: test_git_downtime = random.randint(1, opts.splay) time.sleep(test_git_downtime) # Let's find out if the cloned repository if checked out from the # desired repository print("Grabbing the cloned repository remotes information ... ") cmd = "salt -t 100 {0} --out json git.remote_get /testing".format( build_minion_target(opts, vm_name)) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() retcode = proc.returncode if retcode != 0: print("Failed to get the cloned repository remote. Exit code: {0}". format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) if not stdout: print( "Failed to get the cloned repository remote(no output). Exit code: {0}" .format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) try: remotes_info = salt.utils.json.loads(stdout.strip()) if (remotes_info is None or remotes_info[vm_name] is None or opts.test_git_url not in remotes_info[vm_name]): print("The cloned repository remote is not the desired one:") print(" '{0}' is not in {1}".format(opts.test_git_url, remotes_info)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) print("matches!") except ValueError: print("Failed to load any JSON from '{0}'".format( salt.utils.stringutils.to_str(stdout).strip())) if opts.test_git_commit is not None: test_git_commit_downtime = random.randint(1, opts.splay) time.sleep(test_git_commit_downtime) # Let's find out if the cloned repository is checked out at the desired # commit print("Grabbing the cloned repository commit information ... ") cmd = "salt -t 100 {0} --out json git.revision /testing".format( build_minion_target(opts, vm_name)) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() sys.stdout.flush() retcode = proc.returncode if retcode != 0: print( "Failed to get the cloned repository revision. Exit code: {0}". format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) if not stdout: print( "Failed to get the cloned repository revision(no output). Exit code: {0}" .format(retcode)) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) try: revision_info = salt.utils.json.loads(stdout.strip()) if revision_info[vm_name][7:] != opts.test_git_commit[7:]: print("The cloned repository commit is not the desired one:") print(" '{0}' != '{1}'".format(revision_info[vm_name][:7], opts.test_git_commit[:7])) sys.stdout.flush() if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) sys.exit(retcode) print("matches!") except ValueError: print("Failed to load any JSON from '{0}'".format( salt.utils.stringutils.to_str(stdout).strip())) # Run tests here test_begin_downtime = random.randint(3, opts.splay) time.sleep(test_begin_downtime) cmd = 'salt -t 1800 {target} state.sls {sls} pillar="{pillar}" --no-color'.format( sls=opts.sls, pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), ) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() outstr = salt.utils.stringutils.to_str(stdout) if outstr: print(outstr) if stderr: print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() try: match = re.search(r"Test Suite Exit Code: (?P<exitcode>[\d]+)", outstr) retcode = int(match.group("exitcode")) except AttributeError: # No regex matching retcode = 1 except ValueError: # Not a number!? retcode = 1 except TypeError: # No output!? retcode = 1 if outstr: # Anything else, raise the exception raise if retcode == 0: # Build packages time.sleep(3) cmd = 'salt -t 1800 {target} state.sls buildpackage pillar="{pillar}" --no-color'.format( pillar=build_pillar_data(opts), target=build_minion_target(opts, vm_name), ) print("Running CMD: {0}".format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = proc.communicate() if stdout: print(salt.utils.stringutils.to_str(stdout)) if stderr: print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() # Grab packages and log file (or just log file if build failed) download_packages(opts) if opts.download_remote_reports: # Download unittest reports download_unittest_reports(opts) # Download coverage report if opts.test_without_coverage is False: download_coverage_report(opts) if opts.clean and "JENKINS_SALTCLOUD_VM_NAME" not in os.environ: delete_vm(opts) return retcode
def download_remote_logs(options): ''' Download the generated logs from minion ''' print('Downloading remote logs...') sys.stdout.flush() for fname in ('salt-runtests.log', 'minion.log'): if os.path.isfile(os.path.join(options.workspace, fname)): os.unlink(os.path.join(options.workspace, fname)) if not options.remote_log_path: options.remote_log_path = [ '/tmp/salt-runtests.log', '/var/log/salt/minion' ] cmds = [] if options.scp: for remote_log in options.remote_log_path: cmds.append(' '.join( build_scp_command( options, '-r', 'root@{0}:{1}'.format(get_minion_external_address(options), remote_log), os.path.join( options.workspace, '{0}{1}'.format( os.path.basename(remote_log), '' if remote_log.endswith('.log') else '.log'))))) else: for remote_log in options.remote_log_path: cmds.extend([ '{{0}} {{1}} archive.gzip {0}'.format(remote_log), '{{0}} {{1}} cp.push {0}.gz'.format(remote_log), 'gunzip /var/cache/salt/master/minions/{{2}}/files{0}.gz'. format(remote_log), 'mv /var/cache/salt/master/minions/{{2}}/files{0} {{3}}/{1}'. format( remote_log, '{0}{1}'.format( os.path.basename(remote_log), '' if remote_log.endswith('.log') else '.log')) ]) for cmd in cmds: cmd = cmd.format( 'salt-call publish.publish' if options.lxc else 'salt', build_minion_target(options), options.vm_name, options.workspace) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('\nFailed to execute command. Exit code: {0}'.format( proc.returncode)) time.sleep(0.25)
def download_remote_logs(options): ''' Download the generated logs from minion ''' print('Downloading remote logs...') sys.stdout.flush() for fname in ('salt-runtests.log', 'minion.log'): if os.path.isfile(os.path.join(options.workspace, fname)): os.unlink(os.path.join(options.workspace, fname)) if not options.remote_log_path: options.remote_log_path = [ '/tmp/salt-runtests.log', '/var/log/salt/minion' ] cmds = [] if options.scp: for remote_log in options.remote_log_path: cmds.append( ' '.join(build_scp_command(options, '-r', 'root@{0}:{1}'.format( get_minion_external_address(options), remote_log ), os.path.join( options.workspace, '{0}{1}'.format( os.path.basename(remote_log), '' if remote_log.endswith('.log') else '.log' ) ))) ) else: for remote_log in options.remote_log_path: cmds.extend([ '{{0}} {{1}} archive.gzip {0}'.format(remote_log), '{{0}} {{1}} cp.push {0}.gz'.format(remote_log), 'gunzip /var/cache/salt/master/minions/{{2}}/files{0}.gz'.format(remote_log), 'mv /var/cache/salt/master/minions/{{2}}/files{0} {{3}}/{1}'.format( remote_log, '{0}{1}'.format( os.path.basename(remote_log), '' if remote_log.endswith('.log') else '.log' ) ) ]) for cmd in cmds: cmd = cmd.format( 'salt-call publish.publish' if options.lxc else 'salt', build_minion_target(options), options.vm_name, options.workspace ) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print( '\nFailed to execute command. Exit code: {0}'.format( proc.returncode ) ) time.sleep(0.25)
def main(): ''' Main script execution ''' parser = argparse.ArgumentParser(description='Jenkins execution helper') parser.add_argument('-w', '--workspace', default=os.path.abspath( os.environ.get('WORKSPACE', os.getcwd())), help='Path to the execution workspace') # Output Options output_group = parser.add_argument_group('Output Options') output_group.add_argument('--no-color', '--no-colour', action='store_true', default=False, help='Don\'t use colors') output_group.add_argument( '--echo-parseable-output', action='store_true', default=False, help='Print Jenkins related environment variables and exit') output_group.add_argument( '--pull-request', type=int, action=GetPullRequestAction, default=None, help='Include the Pull Request information in parseable output') # Deployment Selection deployment_group = parser.add_argument_group('Deployment Selection') deployment_group.add_argument( '--pre-mortem', action='store_true', default=False, help='Don\'t try to deploy an new VM. Consider a VM deployed and only ' 'execute post test suite execution commands. Right before killing the VM' ) deployment_group_mutually_exclusive = deployment_group.add_mutually_exclusive_group( ) deployment_group_mutually_exclusive.add_argument( '--cloud', action='store_true', default=True, help='Salt Cloud Deployment') deployment_group_mutually_exclusive.add_argument( '--lxc', action='store_true', default=False, help='Salt LXC Deployment') deployment_group.add_argument('--lxc-host', default=None, help='The host where to deploy the LXC VM') # Execution Selections execution_group = parser.add_argument_group('Execution Selection') execution_group.add_argument( '--peer', action='store_true', default=False, help='Run salt commands through the peer system') execution_group.add_argument('--scp', action='store_true', default=False, help='Download logs and reports using SCP') bootstrap_script_options = parser.add_argument_group( 'Bootstrap Script Options', 'In case there\'s a need to provide the bootstrap script from an alternate URL and/or from a specific commit.' ) bootstrap_script_options.add_argument( '--bootstrap-salt-url', default=None, help='The salt git repository url used to bootstrap a minion') bootstrap_script_options.add_argument( '--bootstrap-salt-commit', default=None, help='The salt git commit used to bootstrap a minion') vm_options_group = parser.add_argument_group('VM Options') vm_options_group.add_argument('vm_name', nargs='?', help='Virtual machine name') vm_options_group.add_argument( '--vm-prefix', default=os.environ.get('JENKINS_VM_NAME_PREFIX', 'ZJENKINS'), help='The bootstrapped machine name prefix. Default: %(default)r') vm_options_group.add_argument( '--vm-source', default=os.environ.get('JENKINS_VM_SOURCE', None), help= 'The VM source. In case of --cloud usage, the could profile name. In case of --lxc usage, the image name.' ) vm_options_group.add_argument( '--grain-target', action='append', default=[], help= 'Match minions using compound matchers, the minion ID, plus the passed grain.' ) vm_preparation_group = parser.add_argument_group( 'VM Preparation Options', 'Salt SLS selection to prepare the VM. The defaults are based on the salt-jenkins repository. See ' 'https://github.com/saltstack/salt-jenkins') vm_preparation_group.add_argument( '--prep-sls', default='git.salt', help= 'The sls file to execute to prepare the system. Default: %(default)r') vm_preparation_group.add_argument( '--prep-sls-2', default=None, help='An optional 2nd system preparation SLS') vm_preparation_group.add_argument('--sls', default='testrun-no-deps', help='The final sls file to execute.') vm_preparation_group.add_argument( '--pillar', action='append', nargs=2, help= 'Pillar (key, value)s to pass to the sls file. Example: \'--pillar pillar_key pillar_value\'' ) vm_actions = parser.add_argument_group( 'VM Actions', 'Action to execute on a running VM') vm_actions.add_argument('--delete-vm', action='store_true', default=False, help='Delete a running VM') vm_actions.add_argument( '--download-remote-reports', default=False, action='store_true', help='Download remote reports when running remote \'testrun\' state') vm_actions.add_argument('--download-unittest-reports', default=False, action='store_true', help='Download the XML unittest results') vm_actions.add_argument('--download-coverage-report', default=False, action='store_true', help='Download the XML coverage reports') vm_actions.add_argument( '--download-remote-logs', default=False, action='store_true', help='Download remote minion and runtests log files') vm_actions.add_argument( '--remote-log-path', action='append', default=[], help='Provide additional log paths to download from remote minion') testing_source_options = parser.add_argument_group( 'Testing Options', 'In case there\'s a need to provide a different repository and/or commit from which the tests suite should be ' 'executed on') testing_source_options.add_argument( '--test-transport', default='zeromq', choices=('zeromq', 'raet'), help= 'Set to raet to run integration tests with raet transport. Default: %default' ) testing_source_options.add_argument('--test-git-url', default=None, help='The testing git repository url') testing_source_options.add_argument('--test-git-commit', default=None, help='The testing git commit to track') options = parser.parse_args() if not options.vm_source and not options.vm_name: parser.error( 'Unable to get VM name from environ nor generate it without --vm-source' ) if options.lxc: if not options.lxc_host: parser.error( 'Need to provide where to deploy the LXC VM by passing it to --lxc-host' ) options.cloud = False if not options.vm_name: options.vm_name = get_vm_name(options) if options.echo_parseable_output: echo_parseable_environment(options) parser.exit() if options.lxc: options.cloud = False if options.pre_mortem: # Run any actions supposed to be executed right before killing the VM if options.download_remote_reports: # Download unittest reports download_unittest_reports(options) # Download coverage report download_coverage_report(options) else: if options.download_unittest_reports: download_unittest_reports(options) if options.download_coverage_report: download_coverage_report(options) if options.download_remote_logs: download_remote_logs(options) if options.delete_vm: delete_vm(options) parser.exit() # RUN IT!!! cmd = [] minion_target = build_minion_target(options) if options.peer: cmd.extend(['salt-call', 'publish.runner']) else: cmd.append('salt-run') if options.lxc: cmd.append('lxc.init') if options.peer: cmd.append('arg="{0}"'.format( to_cli_yaml([ options.vm_name, 'host={0}'.format(options.lxc_host), 'image={0}'.format(options.vm_source) ]))) else: cmd.extend([ options.vm_name, 'host={0}'.format(options.lxc_host), 'image={0}'.format(options.vm_source) ]) else: cmd.append('cloud.profile') if options.peer: cmd.append('arg="{0}"'.format( to_cli_yaml([options.vm_source, options.vm_name]))) else: cmd.extend([options.vm_source, options.vm_name]) if options.cloud: if options.bootstrap_salt_commit is not None: if options.bootstrap_salt_url is None: options.bootstrap_salt_url = 'https://github.com/saltstack/salt.git' cmd.append( 'script_args="-D -g {bootstrap_salt_url} -n git {bootstrap_salt_commit}"' ) else: cmd.append('script-args="-D"') cmd = ' '.join(cmd).format(**options.__dict__) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() retcode = proc.returncode if retcode != 0: print('Failed to bootstrap VM. Exit code: {0}'.format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) print('VM Bootstrapped. Exit code: {0}'.format(retcode)) sys.stdout.flush() print('Sleeping for 5 seconds to allow the minion to breathe a little') sys.stdout.flush() time.sleep(5) if options.scp: prepare_ssh_access(options) if options.cloud: if options.bootstrap_salt_commit is not None: # Let's find out if the installed version matches the passed in pillar # information print('Grabbing bootstrapped minion version information ... ') cmd = [] if options.peer: cmd.extend(['salt-call', '--out=json', 'publish.publish']) else: cmd.extend(['salt', '-t', '100', '--out=json']) cmd.extend([minion_target, 'test.version']) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() retcode = proc.returncode if retcode != 0: print( 'Failed to get the bootstrapped minion version. Exit code: {0}' .format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) if not stdout.strip(): print( 'Failed to get the bootstrapped minion version(no output). Exit code: {0}' .format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) try: version_info = json.loads(stdout.strip()) if options.peer: version_info = version_info['local'] else: version_info = version_info[options.vm_name] bootstrap_salt_commit = options.bootstrap_salt_commit if re.match('v[0-9]+', bootstrap_salt_commit): # We've been passed a tag bootstrap_salt_commit = bootstrap_salt_commit[1:] else: # Most likely a git SHA bootstrap_salt_commit = bootstrap_salt_commit[:7] if bootstrap_salt_commit not in version_info: print('\n\nATTENTION!!!!\n') print( 'The boostrapped minion version commit does not contain the desired commit:' ) print(' {0!r} does not contain {1!r}'.format( version_info, bootstrap_salt_commit)) print('\n\n') sys.stdout.flush() #if options.delete_vm: # delete_vm(options) #sys.exit(retcode) else: print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format( stdout.strip())) # Run preparation SLS time.sleep(3) cmd = [] if options.peer: cmd.append('salt-call') if options.no_color: cmd.append('--no-color') cmd.append('publish.publish') else: cmd.extend(['salt', '-t', '1800']) if options.no_color: cmd.append('--no-color') cmd.extend([ minion_target, 'state.sls', options.prep_sls, 'pillar="{0}"'.format(build_pillar_data(options)) ]) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print('Failed to execute the preparation SLS file. Exit code: {0}'. format(proc.returncode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(proc.returncode) if options.prep_sls_2 is not None: time.sleep(3) # Run the 2nd preparation SLS cmd = [] if options.peer: cmd.append('salt-call') if options.no_color: cmd.append('--no-color') if options.peer: cmd.append('publish.publish') else: cmd.extend(['salt', '-t', '1800']) if options.no_color: cmd.append('--no-color') cmd.extend([ minion_target, 'state.sls', options.prep_sls_2, 'pillar="{0}"'.format(build_pillar_data(options)) ]) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode != 0: print( 'Failed to execute the 2nd preparation SLS file. Exit code: {0}' .format(proc.returncode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(proc.returncode) # Run remote checks if options.test_git_url is not None: time.sleep(1) # Let's find out if the cloned repository if checked out from the # desired repository print('Grabbing the cloned repository remotes information ... ') cmd = [] if options.peer: cmd.extend(['salt-call', '--out=json', 'publish.publish']) else: cmd.extend(['salt', '-t', '100', '--out=json']) cmd.extend([minion_target, 'git.remote_get', '/testing']) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() sys.stdout.flush() retcode = proc.returncode if retcode != 0: print('Failed to get the cloned repository remote. Exit code: {0}'. format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) if not stdout: print( 'Failed to get the cloned repository remote(no output). Exit code: {0}' .format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) try: remotes_info = json.loads(stdout.strip()) if remotes_info is not None: if options.peer: remotes_info = remotes_info['local'] else: remotes_info = remotes_info[options.vm_name] if remotes_info is None or options.test_git_url not in remotes_info: print('The cloned repository remote is not the desired one:') print(' {0!r} is not in {1}'.format(options.test_git_url, remotes_info)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) if options.test_git_commit is not None: time.sleep(1) # Let's find out if the cloned repository is checked out at the desired # commit print('Grabbing the cloned repository commit information ... ') cmd = [] if options.peer: cmd.extend(['salt-call', '--out=json', 'publish.publish']) else: cmd.extend(['salt', '-t', '100', '--out=json']) cmd.extend([minion_target, 'git.revision', '/testing']) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() sys.stdout.flush() retcode = proc.returncode if retcode != 0: print( 'Failed to get the cloned repository revision. Exit code: {0}'. format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) if not stdout: print( 'Failed to get the cloned repository revision(no output). Exit code: {0}' .format(retcode)) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) try: revision_info = json.loads(stdout.strip()) if options.peer: revision_info = revision_info['local'] else: revision_info = revision_info[options.vm_name] if revision_info[:7] != options.test_git_commit[:7]: print('The cloned repository commit is not the desired one:') print(' {0!r} != {1!r}'.format(revision_info[:7], options.test_git_commit[:7])) sys.stdout.flush() if options.delete_vm: delete_vm(options) sys.exit(retcode) print('matches!') except ValueError: print('Failed to load any JSON from {0!r}'.format(stdout.strip())) # Run tests here time.sleep(3) cmd = [] if options.peer: cmd.append('salt-call') if options.no_color: cmd.append('--no-color') cmd.append('publish.publish') else: cmd.extend(['salt', '-t', '1800']) if options.no_color: cmd.append('--no-color') cmd.extend([ minion_target, 'state.sls', options.sls, 'pillar="{0}"'.format(build_pillar_data(options)) ]) if options.peer and ' ' in minion_target: cmd.append('expr_form="compound"') cmd = ' '.join(cmd) print('Running CMD: {0!r}'.format(cmd)) sys.stdout.flush() proc = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, _ = proc.communicate() if stdout: print(stdout) sys.stdout.flush() try: match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout) retcode = int(match.group('exitcode')) except AttributeError: # No regex matching retcode = 1 except ValueError: # Not a number!? retcode = 1 except TypeError: # No output!? retcode = 1 if stdout: # Anything else, raise the exception raise if options.download_remote_reports: # Download unittest reports download_unittest_reports(options) # Download coverage report download_coverage_report(options) if options.download_remote_logs: download_remote_logs(options) if options.delete_vm: delete_vm(options) parser.exit(proc.returncode)
def root_cmd(command, tty, sudo, **kwargs): ''' Wrapper for commands to be run as root ''' if sudo: command = 'sudo {0}'.format(command) log.debug('Using sudo to run command {0}'.format(command)) ssh_args = [] if tty: # Use double `-t` on the `ssh` command, it's necessary when `sudo` has # `requiretty` enforced. ssh_args.extend(['-t', '-t']) ssh_args.extend([ # Don't add new hosts to the host key database '-oStrictHostKeyChecking=no', # Set hosts key database path to /dev/null, ie, non-existing '-oUserKnownHostsFile=/dev/null', # Don't re-use the SSH connection. Less failures. '-oControlPath=none' ]) if 'key_filename' in kwargs: # There should never be both a password and an ssh key passed in, so ssh_args.extend([ # tell SSH to skip password authentication '-oPasswordAuthentication=no', '-oChallengeResponseAuthentication=no', # Make sure public key authentication is enabled '-oPubkeyAuthentication=yes', # No Keyboard interaction! '-oKbdInteractiveAuthentication=no', # Also, specify the location of the key file '-i {0}'.format(kwargs['key_filename']) ]) cmd = 'ssh {0} {1[username]}@{1[hostname]} {2}'.format( ' '.join(ssh_args), kwargs, pipes.quote(command) ) log.debug('SSH command: {0!r}'.format(cmd)) if 'password' in kwargs: cmd = 'sshpass -p {0} {1}'.format(kwargs['password'], cmd) try: proc = NonBlockingPopen( cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stream_stds=kwargs.get('display_ssh_output', True), ) log.debug( 'Executing command(PID {0}): {1!r}'.format( proc.pid, command ) ) proc.poll_and_read_until_finish() proc.communicate() return proc.returncode except Exception as err: log.error( 'Failed to execute command {0!r}: {1}\n'.format( command, err ), exc_info=True ) # Signal an error return 1
def run(platform, provider, commit, clean): ''' RUN! ''' htag = hashlib.md5(str(random.randint(1, 100000000))).hexdigest()[:6] vm_name = 'ZZZ{0}{1}'.format(platform, htag) cmd = 'salt-cloud -l debug --script-args "-D -n git {0}" -p {1}_{2} {3}'.format( commit, provider, platform, vm_name) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() proc.communicate() if proc.returncode > 0: print('Failed to bootstrap VM. Exit code: {0}'.format(proc.returncode)) sys.stdout.flush() cleanup(clean, vm_name) sys.exit(proc.returncode) print('VM Bootstrapped. Exit code: {0}'.format(proc.returncode)) sys.stdout.flush() # Run tests here cmd = 'salt -t 1800 {0} state.sls testrun pillar="{{git_commit: {1}}}" --no-color'.format( vm_name, commit) print('Running CMD: {0}'.format(cmd)) sys.stdout.flush() proc = NonBlockingPopen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stream_stds=True ) proc.poll_and_read_until_finish() stdout, stderr = proc.communicate() if stderr: print(stderr) if stdout: print(stdout) sys.stdout.flush() try: match = re.search(r'Test Suite Exit Code: (?P<exitcode>[\d]+)', stdout) retcode = int(match.group('exitcode')) except AttributeError: # No regex matching retcode = 1 except ValueError: # Not a number!? retcode = 1 except TypeError: # No output!? retcode = 1 if stdout: # Anything else, raise the exception raise cleanup(clean, vm_name) return retcode