def test_provision(ubuntu_codename): logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s') if ubuntu_codename not in AMIS: logging.error('ubuntu codename %r not understood', ubuntu_codename) sys.exit(1) ami = AMIS[ubuntu_codename] build_number = os.environ.get('BUILD_NUMBER', time.time()) hostname = 'bootstrap-test-%s-%s.krxd.net' % (ubuntu_codename, build_number) logging.info('Starting instance %s', hostname) proc = subprocess.Popen( '/bin/bash', stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) proc.stdin.write( 'start_instance.py -t c3.large -H %s -s krux-ops-dev %s s_basic' % (hostname, ami)) proc.stdin.close() multiproc.run_subproc(proc=proc, prefix='[start_instance.py] ', output_func=logging.info) if proc.returncode != 0: logging.error('Starting instance failed! See above for details') sys.exit(1) ec2_conn = boto.ec2.connect_to_region('us-east-1') reservations = ec2_conn.get_all_instances(filters={'tag:Name': hostname}) if len(reservations) == 0 or len(reservations[0].instances) == 0: logging.error( 'Something has gone horribly wrong and we can\'t find the instance we just started') sys.exit(1) instance = reservations[0].instances[0] start = time.time() while instance.tags.get('krux-status') not in FINISHED_STATUSES: if time.time() - start > TIMEOUT: logging.error( 'Instance [%s] bootstrap failed to complete in %r seconds. Current status is %r.', instance.id, TIMEOUT, instance.tags.get('krux-status')) sys.exit(1) logging.info('Waiting on instance %s, current status: %r', instance.id, instance.tags.get('krux-status')) time.sleep(5) instance.update() logging.info('Bootstrap finished with status %r', instance.tags.get('krux-status')) if instance.tags['krux-status'] == 'bootstrap_failed': logging.error('Bootstrap failed, log into %s [%s] to debug, then terminate it.', hostname, instance.id) sys.exit(1) logging.info('Bootstrap completed successfully in %rs.', time.time() - start) logging.info('Terminating test instance %s [%s].', hostname, instance.id) proc = subprocess.Popen( '/bin/bash', stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) proc.stdin.write( 'delete_instance.py -y %s' % (hostname,)) proc.stdin.close() multiproc.run_subproc(proc=proc, prefix='[delete_instance.py] ', output_func=logging.info) if proc.returncode != 0: logging.error('Terminating instance failed! See above for details') sys.exit(1) logging.info('Instance terminated.')
def start( self, command, cwd=None, output_running=False, stdin=__SENTINEL, close_stdin=True, force_tty=False, capture_output=False, ): proc = self._start('/bin/bash -c', command, cwd, output_running, stdin, close_stdin, force_tty) (stdout, stderr, threads) = multiproc.run_subproc( proc, output_func=self.puts, wait=False, capture_output=capture_output, use_color=self.use_color ) return (proc, threads, stdout, stderr)
def _run( self, executable, command, cwd=None, output_running=False, stdin=__SENTINEL, capture_output=False ): """ Runs command via ssh and executable. """ ssh = self._start(executable, command, cwd, output_running, stdin) (stdout, stderr) = multiproc.run_subproc( ssh, output_func=self.puts, capture_output=capture_output, use_color=self.use_color ) if ssh.returncode: raise SSHException("ssh return code was %r" % (ssh.returncode,)) return (stdout, stderr)
def start( self, command, cwd=None, output_running=False, stdin=__SENTINEL, close_stdin=True, force_tty=False, capture_output=False, ): proc = self._start( "/bin/bash -c", command, cwd, output_running, stdin, close_stdin, force_tty ) (stdout, stderr, threads) = multiproc.run_subproc( proc, output_func=self.puts, wait=False, capture_output=capture_output, use_color=self.use_color, ) return (proc, threads, stdout, stderr)
def _run( self, executable, command, cwd=None, output_running=False, stdin=__SENTINEL, capture_output=False, ): """ Runs command via ssh and executable. """ ssh = self._start(executable, command, cwd, output_running, stdin) (stdout, stderr) = multiproc.run_subproc( ssh, output_func=self.puts, capture_output=capture_output, use_color=self.use_color, ) if ssh.returncode: raise SSHException("ssh return code was %r" % (ssh.returncode,)) return (stdout, stderr)
def generateSpritesheets(self, pc_tsid, actuals, base_hash): event = Event() # store the data in the shared dict for eleven.http to use self.shared[pc_tsid] = { 'tsid': pc_tsid, 'actuals': actuals, 'avatar_hash': base_hash, 'event': event, } self.log.info('generateSpritesheets started for %r', pc_tsid) self.log.debug('generateSpritesheets data %r %r %r', pc_tsid, actuals, base_hash) # We need to loop here in case the displays are already being used. We loop until # we find a free display. xvfb = xvfb_threads = browser = browser_threads = None try: self.log.info('Starting Xvfb for %s' % (pc_tsid,)) for DISPLAY_NUM in xrange(0, 100): xvfb = subprocess.Popen( ['Xvfb', ':%i' % (DISPLAY_NUM,)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) xvfb.stdin.close() (_, _, xvfb_threads) = multiproc.run_subproc(xvfb, '[xvfb] ', wait=False) # sleep for 1s to see if it's going to die time.sleep(1) if xvfb.returncode is None: # Xvfb is running break # we got a returncode, so Xvfb exited before it should have, terminate and # try the next display number multiproc.terminate_subproc(xvfb, xvfb_threads) self.log.debug('Display number %i appears to be in use, trying the next one', DISPLAY_NUM) if xvfb.returncode is not None: raise Error('No free display found for use with Xvfb') tsid_signed = URLSafeSerializer(self.secret_key).dumps(pc_tsid) # grab the existing environment variables and add/change DISPLAY to the one Xvfb # is running on. env = os.environ.copy() env['DISPLAY'] = ':%i' % (DISPLAY_NUM,) url = 'http://127.0.0.1:%i/generate/%s' % (self.http_port, tsid_signed) self.log.info('Loading URL %s for %s' % (url, pc_tsid)) browser = subprocess.Popen( ['arora', url], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, ) browser.stdin.close() (_, _, browser_threads) = multiproc.run_subproc(browser, '[browser] ', wait=False) start = time.time() while time.time() - start < self.task_timeout: # wait for the http worker to send us an event saying that the browser hit the done API if event.ready(): break time.sleep(0.05) if not event.ready(): raise Exception('Spritesheet generation exceeded task timeout %is' % (self.task_timeout,)) self.log.info('generateSpritesheets done for %r', pc_tsid) finally: if browser is not None: self.log.info('terminating browser proc') browser.terminate() multiproc.terminate_subproc(browser, browser_threads) if xvfb is not None: self.log.info('terminating xvfb proc') xvfb.terminate() multiproc.terminate_subproc(xvfb, xvfb_threads) del self.shared[pc_tsid] self.log.info('finished')