def test_provision(ubuntu_codename):
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(message)s')
if ubuntu_codename not in AMIS:
logging.error('ubuntu codename %r not understood', ubuntu_codename)
sys.exit(1)
ami = AMIS[ubuntu_codename]
build_number = os.environ.get('BUILD_NUMBER', time.time())
hostname = 'bootstrap-test-%s-%s.krxd.net' % (ubuntu_codename, build_number)
logging.info('Starting instance %s', hostname)
proc = subprocess.Popen(
'/bin/bash',
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
proc.stdin.write(
'start_instance.py -t c3.large -H %s -s krux-ops-dev %s s_basic' % (hostname, ami))
proc.stdin.close()
multiproc.run_subproc(proc=proc, prefix='[start_instance.py] ', output_func=logging.info)
if proc.returncode != 0:
logging.error('Starting instance failed! See above for details')
sys.exit(1)
ec2_conn = boto.ec2.connect_to_region('us-east-1')
reservations = ec2_conn.get_all_instances(filters={'tag:Name': hostname})
if len(reservations) == 0 or len(reservations[0].instances) == 0:
logging.error(
'Something has gone horribly wrong and we can\'t find the instance we just started')
sys.exit(1)
instance = reservations[0].instances[0]
start = time.time()
while instance.tags.get('krux-status') not in FINISHED_STATUSES:
if time.time() - start > TIMEOUT:
logging.error(
'Instance [%s] bootstrap failed to complete in %r seconds. Current status is %r.',
instance.id, TIMEOUT, instance.tags.get('krux-status'))
sys.exit(1)
logging.info('Waiting on instance %s, current status: %r',
instance.id, instance.tags.get('krux-status'))
time.sleep(5)
instance.update()
logging.info('Bootstrap finished with status %r', instance.tags.get('krux-status'))
if instance.tags['krux-status'] == 'bootstrap_failed':
logging.error('Bootstrap failed, log into %s [%s] to debug, then terminate it.',
hostname, instance.id)
sys.exit(1)
logging.info('Bootstrap completed successfully in %rs.', time.time() - start)
logging.info('Terminating test instance %s [%s].', hostname, instance.id)
proc = subprocess.Popen(
'/bin/bash',
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
proc.stdin.write(
'delete_instance.py -y %s' % (hostname,))
proc.stdin.close()
multiproc.run_subproc(proc=proc, prefix='[delete_instance.py] ', output_func=logging.info)
if proc.returncode != 0:
logging.error('Terminating instance failed! See above for details')
sys.exit(1)
logging.info('Instance terminated.')
def start(
self, command,
cwd=None, output_running=False, stdin=__SENTINEL, close_stdin=True, force_tty=False, capture_output=False,
):
proc = self._start('/bin/bash -c', command, cwd, output_running, stdin, close_stdin, force_tty)
(stdout, stderr, threads) = multiproc.run_subproc(
proc, output_func=self.puts, wait=False, capture_output=capture_output, use_color=self.use_color
)
return (proc, threads, stdout, stderr)
def _run(
self, executable, command,
cwd=None, output_running=False, stdin=__SENTINEL, capture_output=False
):
"""
Runs command via ssh and executable.
"""
ssh = self._start(executable, command, cwd, output_running, stdin)
(stdout, stderr) = multiproc.run_subproc(
ssh, output_func=self.puts, capture_output=capture_output, use_color=self.use_color
)
if ssh.returncode:
raise SSHException("ssh return code was %r" % (ssh.returncode,))
return (stdout, stderr)
def start(
self,
command,
cwd=None,
output_running=False,
stdin=__SENTINEL,
close_stdin=True,
force_tty=False,
capture_output=False,
):
proc = self._start(
"/bin/bash -c", command, cwd, output_running, stdin, close_stdin, force_tty
)
(stdout, stderr, threads) = multiproc.run_subproc(
proc,
output_func=self.puts,
wait=False,
capture_output=capture_output,
use_color=self.use_color,
)
return (proc, threads, stdout, stderr)
def _run(
self,
executable,
command,
cwd=None,
output_running=False,
stdin=__SENTINEL,
capture_output=False,
):
"""
Runs command via ssh and executable.
"""
ssh = self._start(executable, command, cwd, output_running, stdin)
(stdout, stderr) = multiproc.run_subproc(
ssh,
output_func=self.puts,
capture_output=capture_output,
use_color=self.use_color,
)
if ssh.returncode:
raise SSHException("ssh return code was %r" % (ssh.returncode,))
return (stdout, stderr)
def generateSpritesheets(self, pc_tsid, actuals, base_hash):
event = Event()
# store the data in the shared dict for eleven.http to use
self.shared[pc_tsid] = {
'tsid': pc_tsid,
'actuals': actuals,
'avatar_hash': base_hash,
'event': event,
}
self.log.info('generateSpritesheets started for %r', pc_tsid)
self.log.debug('generateSpritesheets data %r %r %r', pc_tsid, actuals, base_hash)
# We need to loop here in case the displays are already being used. We loop until
# we find a free display.
xvfb = xvfb_threads = browser = browser_threads = None
try:
self.log.info('Starting Xvfb for %s' % (pc_tsid,))
for DISPLAY_NUM in xrange(0, 100):
xvfb = subprocess.Popen(
['Xvfb', ':%i' % (DISPLAY_NUM,)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
xvfb.stdin.close()
(_, _, xvfb_threads) = multiproc.run_subproc(xvfb, '[xvfb] ', wait=False)
# sleep for 1s to see if it's going to die
time.sleep(1)
if xvfb.returncode is None:
# Xvfb is running
break
# we got a returncode, so Xvfb exited before it should have, terminate and
# try the next display number
multiproc.terminate_subproc(xvfb, xvfb_threads)
self.log.debug('Display number %i appears to be in use, trying the next one', DISPLAY_NUM)
if xvfb.returncode is not None:
raise Error('No free display found for use with Xvfb')
tsid_signed = URLSafeSerializer(self.secret_key).dumps(pc_tsid)
# grab the existing environment variables and add/change DISPLAY to the one Xvfb
# is running on.
env = os.environ.copy()
env['DISPLAY'] = ':%i' % (DISPLAY_NUM,)
url = 'http://127.0.0.1:%i/generate/%s' % (self.http_port, tsid_signed)
self.log.info('Loading URL %s for %s' % (url, pc_tsid))
browser = subprocess.Popen(
['arora', url],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
)
browser.stdin.close()
(_, _, browser_threads) = multiproc.run_subproc(browser, '[browser] ', wait=False)
start = time.time()
while time.time() - start < self.task_timeout:
# wait for the http worker to send us an event saying that the browser hit the done API
if event.ready():
break
time.sleep(0.05)
if not event.ready():
raise Exception('Spritesheet generation exceeded task timeout %is' % (self.task_timeout,))
self.log.info('generateSpritesheets done for %r', pc_tsid)
finally:
if browser is not None:
self.log.info('terminating browser proc')
browser.terminate()
multiproc.terminate_subproc(browser, browser_threads)
if xvfb is not None:
self.log.info('terminating xvfb proc')
xvfb.terminate()
multiproc.terminate_subproc(xvfb, xvfb_threads)
del self.shared[pc_tsid]
self.log.info('finished')
