def do_finalize(shutit):
"""Runs finalize phase; run after all builds are complete and all modules
have been stopped.
"""
cfg = shutit.cfg
# Stop all the modules
if cfg['build']['interactive'] >= 3:
print('\nStopping all modules before finalize phase' + util.colour('31',
'\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
stop_all(shutit)
# Finalize in reverse order
shutit.log('PHASE: finalize', code='31')
if cfg['build']['interactive'] >= 3:
print('\nNow doing finalize phase, which we do when all builds are ' +
'complete and modules are stopped' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
for module_id in module_ids(shutit, rev=True):
module = shutit.shutit_map[module_id]
# Only finalize if it's thought to be installed.
if is_built(shutit, shutit.shutit_map[module_id]):
shutit.login(prompt_prefix=module_id)
if not shutit.shutit_map[module_id].finalize(shutit):
shutit.fail(module_id + ' failed on finalize',
child=shutit.pexpect_children['container_child'])
shutit.logout()
def do_finalize(shutit):
"""Runs finalize phase; run after all builds are complete and all modules
have been stopped.
"""
cfg = shutit.cfg
# Stop all the modules
if cfg['build']['interactive'] >= 3:
print('\nStopping all modules before finalize phase' + util.colour('31',
'\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
stop_all(shutit)
# Finalize in reverse order
shutit.log('PHASE: finalize', code='31')
if cfg['build']['interactive'] >= 3:
print('\nNow doing finalize phase, which we do when all builds are ' +
'complete and modules are stopped' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
for module_id in module_ids(shutit, rev=True):
# Only finalize if it's thought to be installed.
if is_built(shutit, shutit.shutit_map[module_id]):
shutit.login()
if not shutit.shutit_map[module_id].finalize(shutit):
shutit.fail(module_id + ' failed on finalize',
child=shutit.pexpect_children['container_child'])
shutit.logout()
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='31')
shutit.log(util.print_config(shutit.cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
for module_id in module_ids(shutit):
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='31')
if cfg[module.module_id]['shutit.core.module.build']:
if module.is_installed(shutit):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' +
str(module.run_order))
else:
# We move to the module directory to perform the build, returning immediately afterwards.
revert_dir = os.getcwd()
os.chdir(os.path.dirname(module.__module_file))
shutit.login()
build_module(shutit, module)
shutit.logout()
os.chdir(revert_dir)
if is_built(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['container_child'])
def do_test(shutit):
"""Runs test phase, erroring if any return false.
"""
cfg = shutit.cfg
if not cfg['build']['dotest']:
shutit.log('Tests configured off, not running')
return
# Test in reverse order
shutit.log('PHASE: test', code='31')
if cfg['build']['interactive'] >= 3:
print '\nNow doing test phase' + util.colour('31',
'\n\n[Hit return to continue]\n')
util.util_raw_input(shutit=shutit)
stop_all(shutit)
start_all(shutit)
for module_id in module_ids(shutit, rev=True):
module = shutit.shutit_map[module_id]
# Only test if it's thought to be installed.
if is_built(shutit, shutit.shutit_map[module_id]):
shutit.log('RUNNING TEST ON: ' + module_id, code='31')
shutit.login(prompt_prefix=module_id)
if not shutit.shutit_map[module_id].test(shutit):
shutit.fail(module_id + ' failed on test',
child=shutit.pexpect_children['container_child'])
shutit.logout()
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='31')
shutit.log(util.print_config(shutit.cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
util.colour('31', '\n[Hit return to continue]'))
raw_input('')
for module_id in module_ids(shutit):
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='31')
if cfg[module.module_id]['build']:
if module.is_installed(shutit):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' +
str(module.run_order))
else:
revert_dir = os.getcwd()
os.chdir(os.path.dirname(module.__module_file))
build_module(shutit, module)
os.chdir(revert_dir)
if is_built(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['container_child'])
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='31')
shutit.log(util.print_config(shutit.cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
for module_id in module_ids(shutit):
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='31')
if cfg[module.module_id]['shutit.core.module.build']:
if is_installed(shutit,module):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' + str(module.run_order))
else:
# We move to the module directory to perform the build, returning immediately afterwards.
revert_dir = os.getcwd()
os.chdir(os.path.dirname(module.__module_file))
shutit.login(prompt_prefix=module_id)
build_module(shutit, module)
shutit.logout()
os.chdir(revert_dir)
if is_to_be_built_or_is_installed(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['target'])
def build(self, shutit):
"""Sets up the machine ready for building.
"""
cfg = shutit.cfg
ssh_host = cfg[self.module_id]['ssh_host']
ssh_port = cfg[self.module_id]['ssh_port']
ssh_user = cfg[self.module_id]['ssh_user']
ssh_pass = cfg[self.module_id]['password']
ssh_key = cfg[self.module_id]['ssh_key']
ssh_cmd = cfg[self.module_id]['ssh_cmd']
opts = [
'-t', '-o', 'UserKnownHostsFile=/dev/null', '-o',
'StrictHostKeyChecking=no'
]
if ssh_pass == '':
opts += ['-o', 'PasswordAuthentication=no']
if ssh_port != '':
opts += ['-p', ssh_port]
if ssh_key != '':
opts += ['-i', ssh_key]
host_arg = ssh_host
if host_arg == '':
shutit.fail('No host specified for sshing', throw_exception=False)
if ssh_user != '':
host_arg = ssh_user + '@' + host_arg
cmd_arg = ssh_cmd
if cmd_arg == '':
cmd_arg = 'sudo su -s /bin/bash -'
ssh_command = ['ssh'] + opts + [host_arg, cmd_arg]
if cfg['build']['interactive'] >= 3:
print('\n\nAbout to connect to host.' + '\n\n' +
util.colour('31', '\n[Hit return to continue]'))
util.util_raw_input(shutit=shutit)
shutit.cfg['build']['ssh_command'] = ' '.join(ssh_command)
shutit.log('\n\nCommand being run is:\n\n' +
shutit.cfg['build']['ssh_command'],
force_stdout=True,
prefix=False)
container_child = pexpect.spawn(ssh_command[0], ssh_command[1:])
expect = ['assword', cfg['expect_prompts']['base_prompt'].strip()]
res = container_child.expect(expect, 10)
while True:
shutit.log(container_child.before + container_child.after,
prefix=False,
force_stdout=True)
if res == 0:
shutit.log('...')
res = shutit.send(ssh_pass,
child=container_child,
expect=expect,
timeout=10,
check_exit=False,
fail_on_empty_before=False)
elif res == 1:
shutit.log('Prompt found, breaking out')
break
self._setup_prompts(shutit, container_child)
self._add_begin_build_info(shutit, ssh_command)
return True
def conn_container(shutit):
"""Connect to the container.
"""
assert len(shutit.conn_modules) == 1
# Set up the container in pexpect.
if shutit.cfg['build']['interactive'] >= 3:
print('\nRunning the conn module (' +
shutit.shutit_main_dir + '/setup.py)' + util.colour('31','\n[Hit return to continue]'))
raw_input('')
list(shutit.conn_modules)[0].build(shutit)
def build(self, shutit):
"""Sets up the machine ready for building.
"""
cfg = shutit.cfg
ssh_host = cfg[self.module_id]['ssh_host']
ssh_port = cfg[self.module_id]['ssh_port']
ssh_user = cfg[self.module_id]['ssh_user']
ssh_pass = cfg[self.module_id]['password']
ssh_key = cfg[self.module_id]['ssh_key']
ssh_cmd = cfg[self.module_id]['ssh_cmd']
opts = [
'-t',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no'
]
if ssh_pass == '':
opts += ['-o', 'PasswordAuthentication=no']
if ssh_port != '':
opts += ['-p', ssh_port]
if ssh_key != '':
opts += ['-i', ssh_key]
host_arg = ssh_host
if host_arg == '':
shutit.fail('No host specified for sshing', throw_exception=False)
if ssh_user != '':
host_arg = ssh_user + '@' + host_arg
cmd_arg = ssh_cmd
if cmd_arg == '':
cmd_arg = 'sudo su -s /bin/bash -'
ssh_command = ['ssh'] + opts + [host_arg, cmd_arg]
if cfg['build']['interactive'] >= 3:
print('\n\nAbout to connect to host.' +
'\n\n' + util.colour('31', '\n[Hit return to continue]'))
util.util_raw_input(shutit=shutit)
shutit.cfg['build']['ssh_command'] = ' '.join(ssh_command)
shutit.log('\n\nCommand being run is:\n\n' + shutit.cfg['build']['ssh_command'],
force_stdout=True, prefix=False)
target_child = pexpect.spawn(ssh_command[0], ssh_command[1:])
expect = ['assword', cfg['expect_prompts']['base_prompt'].strip()]
res = target_child.expect(expect, 10)
while True:
shutit.log(target_child.before + target_child.after, prefix=False,
force_stdout=True)
if res == 0:
shutit.log('...')
res = shutit.send(ssh_pass,
child=target_child, expect=expect, timeout=10,
check_exit=False, fail_on_empty_before=False)
elif res == 1:
shutit.log('Prompt found, breaking out')
break
self._setup_prompts(shutit, target_child)
self._add_begin_build_info(shutit, ssh_command)
return True
def init_shutit_map(shutit):
"""Initializes the module map of shutit based on the modules we have gathered.
Checks we have core modules
Checks for duplicate module details.
Sets up common config.
Sets up map of modules.
"""
cfg = shutit.cfg
shutit_map = shutit.shutit_map
modules = shutit.shutit_modules
# Have we got anything to process outside of special modules?
if len([mod for mod in modules if mod.run_order > 0]) < 1:
shutit.log(modules)
path = ':'.join(cfg['host']['shutit_module_paths'])
if path == '':
shutit.fail('No modules aside from core ones found and no ShutIt module path given. Did you set --shutit_module_path/-m wrongly?')
elif path == '.':
shutit.fail('No modules aside from core ones found and no ShutIt module path given apart from default (.). Did you set --shutit_module_path/-m? Is there a STOP file in your . dir?')
else:
shutit.fail('No modules aside from core ones found and no ShutIt modules in path:\n\n' + path +
'\n\nor their subfolders. Check you set --shutit_module_path/-m setting and check that there are ShutIt modules below without STOP files in any relevant directories.')
shutit.log('PHASE: base setup',code='31')
if cfg['build']['interactive'] >= 3:
shutit.log('\nChecking to see whether there are duplicate module ids or run orders in the visible modules.',force_stdout=True)
shutit.log('\nModules I see are:\n',force_stdout=True)
for m in modules:
shutit.log(m.module_id,force_stdout=True,code='31')
shutit.log('\n',force_stdout=True)
run_orders = {}
has_core_module = False
for m in modules:
assert isinstance(m, ShutItModule)
if m.module_id in shutit_map:
shutit.fail('Duplicated module id: ' + m.module_id)
if m.run_order in run_orders:
shutit.fail('Duplicate run order: ' + str(m.run_order) + ' for ' +
m.module_id + ' and ' + run_orders[m.run_order].module_id)
if m.run_order == 0:
has_core_module = True
shutit_map[m.module_id] = run_orders[m.run_order] = m
if not has_core_module:
shutit.fail('No module with run_order=0 specified! This is required.')
if cfg['build']['interactive'] >= 3:
print(util.colour('31','Module id and run order checks OK\n[Hit return to continue]'))
raw_input('')
def pause_point(self,msg,child=None,print_input=True,level=1):
"""Inserts a pause in the build session, which allows the user to try things out before continuing.
Ignored if we are not in an interactive mode, or the interactive level is less than the passed-in one.
Designed to help debug the build, or drop to on failure so the situation can be debugged.
- msg - Message to display to user on pause point.
- child - See send()
- print_input - Whether to take input at this point (ie interact), or simply pause pending any input.
- level - Minimum level to invoke the pause_point at
"""
child = child or self.get_default_child()
if not self.cfg['build']['interactive'] or self.cfg['build']['interactive'] < level:
return
if child and print_input:
print util.colour('31','\n\nPause point:\n\n') + msg + util.colour('31','\n\nYou can now type in commands and alter the state of the container.\nHit return to see the prompt\nHit CTRL and ] at the same time to continue with build\n\nHit CTRL and [ to save the state\n\n')
oldlog = child.logfile_send
child.logfile_send = None
child.interact(input_filter=self._pause_input_filter)
child.logfile_send = oldlog
else:
print msg
print util.colour('31','\n\n[Hit return to continue]\n')
raw_input('')
def stop_all(shutit, run_order=-1):
"""Runs stop method on all modules less than the passed-in run_order.
Used when container is exporting itself mid-build, so we clean up state before committing run files etc.
"""
cfg = shutit.cfg
shutit_map = shutit.shutit_map
if cfg['build']['interactive'] >= 3:
print('\nRunning stop on all modules' + util.colour('31','\n[Hit return to continue]'))
raw_input('')
# sort them to it's stopped in reverse order)
for mid in module_ids(shutit, rev=True):
shutit_module_obj = shutit_map[mid]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if is_built(shutit,shutit_module_obj):
if not shutit_module_obj.stop(shutit):
shutit.fail('failed to stop: ' + mid,child=shutit.pexpect_children['container_child'])
def start_all(shutit, run_order=-1):
"""Runs start method on all modules less than the passed-in run_order.
Used when container is exporting itself mid-build, so we can export a clean
container and still depended-on modules running if necessary.
"""
cfg = shutit.cfg
shutit_map = shutit.shutit_map
if cfg['build']['interactive'] >= 3:
print('\nRunning start on all modules' + util.colour('31','\n[Hit return to continue]'))
raw_input('')
# sort them to they're started in order)
for mid in module_ids(shutit):
shutit_module_obj = shutit_map[mid]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if is_built(shutit,shutit_module_obj):
if not shutit_module_obj.start(shutit):
shutit.fail('failed to start: ' + mid,child=shutit.pexpect_children['container_child'])
def start_all(shutit, run_order=-1):
"""Runs start method on all modules less than the passed-in run_order.
Used when container is exporting itself mid-build, so we can export a clean
container and still depended-on modules running if necessary.
"""
cfg = shutit.cfg
if cfg['build']['interactive'] >= 3:
print('\nRunning start on all modules' +
util.colour('31', '\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
# sort them to they're started in order)
for module_id in module_ids(shutit):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if is_built(shutit, shutit_module_obj):
if not shutit_module_obj.start(shutit):
shutit.fail('failed to start: ' + module_id, \
child=shutit.pexpect_children['container_child'])
def stop_all(shutit, run_order=-1):
"""Runs stop method on all modules less than the passed-in run_order.
Used when container is exporting itself mid-build, so we clean up state
before committing run files etc.
"""
cfg = shutit.cfg
if cfg['build']['interactive'] >= 3:
print('\nRunning stop on all modules' + \
util.colour('31', '\n[Hit return to continue]'))
util.util_raw_input(shutit=shutit)
# sort them to it's stopped in reverse order)
for module_id in module_ids(shutit, rev=True):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if is_built(shutit, shutit_module_obj):
if not shutit_module_obj.stop(shutit):
shutit.fail('failed to stop: ' + \
module_id, child=shutit.pexpect_children['container_child'])
def stop_all(shutit, run_order=-1):
"""Runs stop method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we clean up state
before committing run files etc.
"""
cfg = shutit.cfg
if cfg['build']['interactive'] >= 3:
print('\nRunning stop on all modules' + \
util.colour('31', '\n\n[Hit return to continue]'))
util.util_raw_input(shutit=shutit)
# sort them so they're stopped in reverse order
for module_id in module_ids(shutit, rev=True):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
# TODO: cache this - it's a big time-waster when libraries get big
if is_to_be_built_or_is_installed(shutit, shutit_module_obj):
if not shutit_module_obj.stop(shutit):
shutit.fail('failed to stop: ' + \
module_id, child=shutit.pexpect_children['target_child'])
def conn_container(shutit):
"""Connect to the container.
"""
conn_module = None
for mod in shutit.conn_modules:
if mod.module_id == shutit.cfg['build']['conn_module']:
conn_module = mod
break
if conn_module is None:
shutit.fail('Couldn\'t find conn_module ' + shutit.cfg['build']['conn_module'])
# Set up the target in pexpect.
if shutit.cfg['build']['interactive'] >= 3:
print('\nRunning the conn module (' +
shutit.shutit_main_dir + '/setup.py)' + util.colour('31',
'\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
conn_module.get_config(shutit)
conn_module.build(shutit)
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='31')
shutit.log(util.print_config(shutit.cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
module_id_list = module_ids(shutit)
if cfg['build']['deps_only']:
module_id_list_build_only = filter(lambda x: cfg[x]['shutit.core.module.build'], module_id_list)
for module_id in module_id_list:
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='31')
if cfg[module.module_id]['shutit.core.module.build']:
if is_installed(shutit,module):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' + str(module.run_order))
else:
# We move to the module directory to perform the build, returning immediately afterwards.
if cfg['build']['deps_only'] and module_id == module_id_list_build_only[-1]:
# If this is the last module, and we are only building deps, stop here.
cfg['build']['report'] = (cfg['build']['report'] + '\nSkipping: ' +
module.module_id + ' with run order: ' + str(module.run_order) +
'\n\tas this is the final module and we are building dependencies only')
else:
revert_dir = os.getcwd()
cfg['target']['module_root_dir'] = os.path.dirname(module.__module_file)
shutit.chdir(cfg['target']['module_root_dir'])
shutit.login(prompt_prefix=module_id)
build_module(shutit, module)
shutit.logout()
shutit.chdir(revert_dir)
if is_installed(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['target_child'])
def start_all(shutit, run_order=-1):
"""Runs start method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we can export a clean
target and still depended-on modules running if necessary.
"""
cfg = shutit.cfg
if cfg['build']['interactive'] >= 3:
print('\nRunning start on all modules' +
util.colour('31', '\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
# sort them so they're started in order
for module_id in module_ids(shutit):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
# TODO: cache this - it's a big time-waster when libraries get big
if is_to_be_built_or_is_installed(shutit, shutit_module_obj):
if not shutit_module_obj.start(shutit):
shutit.fail('failed to start: ' + module_id, \
child=shutit.pexpect_children['target_child'])
def conn_container(shutit):
"""Connect to the container.
"""
conn_module = None
for mod in shutit.conn_modules:
if mod.module_id == shutit.cfg['build']['conn_module']:
conn_module = mod
break
if conn_module is None:
shutit.fail('Couldn\'t find conn_module ' + shutit.cfg['build']['conn_module'])
# Set up the target in pexpect.
if shutit.cfg['build']['interactive'] >= 3:
print('\nRunning the conn module (' +
shutit.shutit_main_dir + '/setup.py)' + util.colour('31',
'\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
conn_module.get_config(shutit)
conn_module.build(shutit)
def log(self, msg, code=None, pause=0, prefix=True, force_stdout=False):
"""Logging function.
- code - Colour code for logging. Ignored if we are in serve mode.
- pause - Length of time to pause after logging (default: 0)
- prefix - Whether to output logging prefix (LOG: <time>) (default: True)
- force_stdout - If we are not in debug, put this in stdout anyway (default: False)
"""
if prefix:
prefix = 'LOG: ' + time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
msg = prefix + ' ' + str(msg)
# Don't colour message if we are in serve mode.
if code != None and not self.cfg['action']['serve']:
msg = util.colour(code, msg)
if self.cfg['build']['debug'] or force_stdout:
print >> sys.stdout, msg
sys.stdout.flush()
if self.cfg['build']['build_log']:
print >> cfg['build']['build_log'], msg
self.cfg['build']['build_log'].flush()
time.sleep(pause)
def do_test(shutit):
"""Runs test phase, erroring if any return false.
"""
cfg = shutit.cfg
if not cfg['build']['dotest']:
shutit.log('Tests configured off, not running')
return
# Test in reverse order
shutit.log('PHASE: test', code='31')
if cfg['build']['interactive'] >= 3:
print '\nNow doing test phase' + util.colour('31',
'\n\n[Hit return to continue]\n')
util.util_raw_input(shutit=shutit)
stop_all(shutit)
start_all(shutit)
for module_id in module_ids(shutit, rev=True):
# Only test if it's thought to be installed.
if is_built(shutit, shutit.shutit_map[module_id]):
shutit.log('RUNNING TEST ON: ' + module_id, code='31')
shutit.login()
if not shutit.shutit_map[module_id].test(shutit):
shutit.fail(module_id + ' failed on test',
child=shutit.pexpect_children['container_child'])
shutit.logout()
def build(self,shutit):
"""Sets up the container ready for building.
"""
# Uncomment for testing for "failure" cases.
#sys.exit(1)
while not self._check_docker(shutit):
pass
cfg = shutit.cfg
docker = cfg['host']['docker_executable'].split(' ')
# Always-required options
cfg['build']['cidfile'] = '/tmp/' + cfg['host']['username'] + '_cidfile_' + cfg['build']['build_id']
cidfile_arg = '--cidfile=' + cfg['build']['cidfile']
# Singly-specified options
privileged_arg = ''
lxc_conf_arg = ''
name_arg = ''
hostname_arg = ''
volume_arg = ''
rm_arg = ''
net_arg = ''
if cfg['build']['privileged']:
privileged_arg = '--privileged=true'
else:
# TODO: put in to ensure serve always works.
# Need better solution in place, eg refresh builder when build needs privileged
privileged_arg = '--privileged=true'
if cfg['build']['lxc_conf'] != '':
lxc_conf_arg = '--lxc-conf=' + cfg['build']['lxc_conf']
if cfg['container']['name'] != '':
name_arg = '--name=' + cfg['container']['name']
if cfg['container']['hostname'] != '':
hostname_arg = '-h=' + cfg['container']['hostname']
if cfg['host']['resources_dir'] != '':
volume_arg = '-v=' + cfg['host']['resources_dir'] + ':/resources'
if cfg['build']['net'] != '':
net_arg = '--net="' + cfg['build']['net'] + '"'
# Incompatible with do_repository_work
if cfg['container']['rm']:
rm_arg = '--rm=true'
# Multiply-specified options
port_args = []
dns_args = []
ports_list = cfg['container']['ports'].strip().split()
dns_list = cfg['host']['dns'].strip().split()
for portmap in ports_list:
port_args.append('-p=' + portmap)
for dns in dns_list:
dns_args.append('--dns=' + dns)
docker_command = docker + [
arg for arg in [
'run',
cidfile_arg,
privileged_arg,
lxc_conf_arg,
name_arg,
hostname_arg,
volume_arg,
rm_arg,
net_arg,
] + port_args + dns_args + [
'-t',
'-i',
cfg['container']['docker_image'],
'/bin/bash'
] if arg != ''
]
if cfg['build']['interactive'] >= 2:
print('\n\nAbout to start container. ' +
'Ports mapped will be: ' + ', '.join(port_args) +
' (from\n\n[host]\nports:<value>\n\nconfig, building on the ' +
'configurable base image passed in in:\n\n\t--image <image>\n' +
'\nor config:\n\n\t[container]\n\tdocker_image:<image>)\n\nBase ' +
'image in this case is:\n\n\t' + cfg['container']['docker_image'] +
'\n\n' + util.colour('31','[Hit return to continue]'))
raw_input('')
shutit.log('\n\nCommand being run is:\n\n' + ' '.join(docker_command),force_stdout=True,prefix=False)
shutit.log('\n\nThis may download the image, please be patient\n\n',force_stdout=True,prefix=False)
container_child = pexpect.spawn(docker_command[0], docker_command[1:])
expect = ['assword',cfg['expect_prompts']['base_prompt'].strip(),'Waiting','ulling','endpoint','Download']
res = container_child.expect(expect,9999)
while True:
shutit.log(""">>>\n""" + container_child.before + container_child.after + """\n<<<""")
if res == 0:
shutit.log('...')
res = shutit.send(cfg['host']['password'],child=container_child,expect=expect,timeout=9999,check_exit=False,fail_on_empty_before=False)
elif res == 1:
shutit.log('Prompt found, breaking out')
break
else:
res = container_child.expect(expect,9999)
continue
# Get the cid
time.sleep(5) # cidfile creation is sometimes slow...
shutit.log('Slept')
cid = open(cfg['build']['cidfile']).read()
shutit.log('Opening file')
if cid == '' or re.match('^[a-z0-9]+$', cid) == None:
shutit.fail('Could not get container_id - quitting. Check whether ' +
'other containers may be clashing on port allocation or name.' +
'\nYou might want to try running: sudo docker kill ' +
cfg['container']['name'] + '; sudo docker rm ' +
cfg['container']['name'] + '\nto resolve a name clash or: ' +
cfg['host']['docker_executable'] + ' ps -a | grep ' +
cfg['container']['ports'] + ' | awk \'{print $1}\' | ' +
'xargs ' + cfg['host']['docker_executable'] + ' kill\nto + '
'resolve a port clash\n')
shutit.log('cid: ' + cid)
cfg['container']['container_id'] = cid
# Now let's have a host_child
shutit.log('Creating host child')
shutit.log('Spawning host child')
host_child = pexpect.spawn('/bin/bash')
shutit.log('Spawning done')
# Some pexpect settings
shutit.pexpect_children['host_child'] = host_child
shutit.pexpect_children['container_child'] = container_child
shutit.log('Setting default expect')
shutit.set_default_expect(cfg['expect_prompts']['base_prompt'])
shutit.log('Setting default expect done')
host_child.logfile_send = container_child.logfile_send = sys.stdout
host_child.logfile_read = container_child.logfile_read = sys.stdout
host_child.maxread = container_child.maxread = 2000
host_child.searchwindowsize = container_child.searchwindowsize = 1024
delay = cfg['build']['command_pause']
host_child.delaybeforesend = container_child.delaybeforesend = delay
# Set up prompts and let the user do things before the build
# host child
shutit.log('Setting default child')
shutit.set_default_child(host_child)
shutit.log('Setting default child done')
shutit.log('Setting up default prompt on host child')
shutit.log('Setting up prompt')
shutit.setup_prompt('real_user_prompt',prefix='REAL_USER')
shutit.log('Setting up prompt done')
# container child
shutit.set_default_child(container_child)
shutit.log('Setting up default prompt on container child')
shutit.setup_prompt('pre_build', prefix='PRE_BUILD')
shutit.send('export HOME=/root')
shutit.get_distro_info()
shutit.setup_prompt('root_prompt', prefix='ROOT')
# Create the build directory and put the config in it.
shutit.send('mkdir -p ' + shutit.cfg ['build']['build_db_dir'] + '/' + shutit.cfg['build']['build_id'])
# Record the command we ran and the python env.
# TODO: record the image id we ran against - wait for "docker debug" command
shutit.send_file(shutit.cfg['build']['build_db_dir'] + '/' + shutit.cfg['build']['build_id'] + '/python_env.sh',str(sys.__dict__),log=False)
shutit.send_file(shutit.cfg['build']['build_db_dir'] + '/' + shutit.cfg['build']['build_id'] + '/docker_command.sh',' '.join(docker_command),log=False)
shutit.pause_point('Anything you want to do now the container is connected to?', level=2)
return True
def build(self, shutit):
"""Sets up the target ready for building.
"""
# Uncomment for testing for "failure" cases.
#sys.exit(1)
while not self._check_docker(shutit):
pass
cfg = shutit.cfg
docker = cfg['host']['docker_executable'].split(' ')
# Always-required options
if not os.path.exists('/tmp/shutit/cidfiles'):
os.makedirs('/tmp/shutit/cidfiles')
cfg['build']['cidfile'] = '/tmp/shutit/cidfiles' + cfg['host']['username'] +\
'_cidfile_' + cfg['build']['build_id']
cidfile_arg = '--cidfile=' + cfg['build']['cidfile']
# Singly-specified options
privileged_arg = ''
lxc_conf_arg = ''
name_arg = ''
hostname_arg = ''
volume_arg = ''
rm_arg = ''
net_arg = ''
if cfg['build']['privileged']:
privileged_arg = '--privileged=true'
else:
# TODO: put in to ensure serve always works. --cap-add is now an option.
# Need better solution in place, eg refresh builder when build
# needs privileged
privileged_arg = '--privileged=true'
if cfg['build']['lxc_conf'] != '':
lxc_conf_arg = '--lxc-conf=' + cfg['build']['lxc_conf']
if cfg['target']['name'] != '':
name_arg = '--name=' + cfg['target']['name']
if cfg['target']['hostname'] != '':
hostname_arg = '-h=' + cfg['target']['hostname']
if cfg['host']['artifacts_dir'] != '':
volume_arg = '-v=' + cfg['host']['artifacts_dir'] + ':/artifacts'
if cfg['build']['net'] != '':
net_arg = '--net="' + cfg['build']['net'] + '"'
# Incompatible with do_repository_work
if cfg['target']['rm']:
rm_arg = '--rm=true'
# Multiply-specified options
port_args = []
dns_args = []
ports_list = cfg['target']['ports'].strip().split()
dns_list = cfg['host']['dns'].strip().split()
for portmap in ports_list:
port_args.append('-p=' + portmap)
for dns in dns_list:
dns_args.append('--dns=' + dns)
docker_command = docker + [
arg for arg in [
'run',
cidfile_arg,
privileged_arg,
lxc_conf_arg,
name_arg,
hostname_arg,
volume_arg,
rm_arg,
net_arg,
] + port_args + dns_args + [
'-t',
'-i',
cfg['target']['docker_image'],
'/bin/bash'
] if arg != ''
]
if cfg['build']['interactive'] >= 3:
print('\n\nAbout to start container. ' +
'Ports mapped will be: ' + ', '.join(port_args) +
'\n\n[host]\nports:<value>\n\nconfig, building on the ' +
'configurable base image passed in in:\n\n --image <image>\n' +
'\nor config:\n\n [target]\n docker_image:<image>)\n\n' +
'Base image in this case is:\n\n ' +
cfg['target']['docker_image'] +
'\n\n' + util.colour('31', '\n[Hit return to continue]'))
util.util_raw_input(shutit=shutit)
shutit.cfg['build']['docker_command'] = ' '.join(docker_command)
shutit.log('\n\nCommand being run is:\n\n' + shutit.cfg['build']['docker_command'],
force_stdout=True, prefix=False)
shutit.log('\n\nThis may download the image, please be patient\n\n',
force_stdout=True, prefix=False)
target_child = pexpect.spawn(docker_command[0], docker_command[1:])
expect = ['assword', cfg['expect_prompts']['base_prompt'].strip(), \
'Waiting', 'ulling', 'endpoint', 'Download']
res = target_child.expect(expect, 9999)
while True:
shutit.log(target_child.before + target_child.after, prefix=False,
force_stdout=True)
if res == 0:
shutit.log('...')
res = shutit.send(cfg['host']['password'], \
child=target_child, expect=expect, timeout=9999, \
check_exit=False, fail_on_empty_before=False)
elif res == 1:
shutit.log('Prompt found, breaking out')
break
else:
res = target_child.expect(expect, 9999)
continue
# Get the cid
time.sleep(1) # cidfile creation is sometimes slow...
shutit.log('Slept')
cid = open(cfg['build']['cidfile']).read()
shutit.log('Opening file')
if cid == '' or re.match('^[a-z0-9]+$', cid) == None:
shutit.fail('Could not get container_id - quitting. ' +
'Check whether ' +
'other containers may be clashing on port allocation or name.' +
'\nYou might want to try running: sudo docker kill ' +
cfg['target']['name'] + '; sudo docker rm ' +
cfg['target']['name'] + '\nto resolve a name clash or: ' +
cfg['host']['docker_executable'] + ' ps -a | grep ' +
cfg['target']['ports'] + ' | awk \'{print $1}\' | ' +
'xargs ' + cfg['host']['docker_executable'] + ' kill\nto + '
'resolve a port clash\n')
shutit.log('cid: ' + cid)
cfg['target']['container_id'] = cid
self._setup_prompts(shutit, target_child)
self._add_begin_build_info(shutit, docker_command)
return True
def prompt_cfg(self,msg,sec,name,ispass=False):
"""Prompt for a config value, optionally saving it to the user-level cfg.
Only runs if we are in an interactive mode.
msg - Message to display to user.
sec - Section of config to add to.
name - Config item name.
ispass - Hide the input from the terminal.
"""
cfg = self.cfg
cfgstr = '[%s]/%s' % (sec, name)
cp = cfg['config_parser']
usercfg = os.path.join(cfg['shutit_home'], 'config')
if not cfg['build']['interactive']:
shutit.fail('ShutIt is not in interactive mode so cannnot prompt ' +
'for values.')
print util.colour('34', '\nPROMPTING FOR CONFIG: %s' % (cfgstr,))
print util.colour('34', '\n' + msg + '\n')
if cp.has_option(sec, name):
whereset = cp.whereset(sec, name)
if usercfg == whereset:
self.fail(cfgstr + ' has already been set in the user ' +
'config, edit ' + usercfg + ' directly to change it')
for subcp, filename, fp in reversed(cp.layers):
# Is the config file loaded after the user config file?
if filename == whereset:
self.fail(cfgstr + ' is being set in ' + filename + ', ' +
'unable to override on a user config level')
elif filename == usercfg:
break
else:
# The item is not currently set so we're fine to do so
pass
if ispass:
val = getpass.getpass('>> ')
else:
val = raw_input('>> ')
is_excluded = (
cp.has_option('save_exclude', sec) and
name in cp.get('save_exclude', sec).split()
)
# TODO: ideally we would remember the prompted config item for this
# invocation of shutit
if not is_excluded:
usercp = [
subcp for subcp, filename, fp in cp.layers
if filename == usercfg
][0]
if raw_input(util.colour('34',
'Do you want to save this to your user settings? y/n: ')) == 'y':
sec_toset, name_toset, val_toset = sec, name, val
else:
# Never save it
if cp.has_option('save_exclude', sec):
excluded = cp.get('save_exclude', sec).split()
else:
excluded = []
excluded.append(name)
excluded = ' '.join(excluded)
sec_toset, name_toset, val_toset = 'save_exclude', sec, excluded
if not usercp.has_section(sec_toset):
usercp.add_section(sec_toset)
usercp.set(sec_toset, name_toset, val_toset)
usercp.write(open(usercfg, 'w'))
cp.reload()
return val
def build(self, shutit):
"""Sets up the container ready for building.
"""
# Uncomment for testing for "failure" cases.
#sys.exit(1)
while not self._check_docker(shutit):
pass
cfg = shutit.cfg
docker = cfg['host']['docker_executable'].split(' ')
# Always-required options
if not os.path.exists('/tmp/shutit/cidfiles'):
os.makedirs('/tmp/shutit/cidfiles')
cfg['build']['cidfile'] = '/tmp/shutit/cidfiles' + cfg['host']['username'] +\
'_cidfile_' + cfg['build']['build_id']
cidfile_arg = '--cidfile=' + cfg['build']['cidfile']
# Singly-specified options
privileged_arg = ''
lxc_conf_arg = ''
name_arg = ''
hostname_arg = ''
volume_arg = ''
rm_arg = ''
net_arg = ''
if cfg['build']['privileged']:
privileged_arg = '--privileged=true'
else:
# TODO: put in to ensure serve always works. --cap-add is now an option.
# Need better solution in place, eg refresh builder when build
# needs privileged
privileged_arg = '--privileged=true'
if cfg['build']['lxc_conf'] != '':
lxc_conf_arg = '--lxc-conf=' + cfg['build']['lxc_conf']
if cfg['container']['name'] != '':
name_arg = '--name=' + cfg['container']['name']
if cfg['container']['hostname'] != '':
hostname_arg = '-h=' + cfg['container']['hostname']
if cfg['host']['resources_dir'] != '':
volume_arg = '-v=' + cfg['host']['resources_dir'] + ':/resources'
if cfg['build']['net'] != '':
net_arg = '--net="' + cfg['build']['net'] + '"'
# Incompatible with do_repository_work
if cfg['container']['rm']:
rm_arg = '--rm=true'
# Multiply-specified options
port_args = []
dns_args = []
ports_list = cfg['container']['ports'].strip().split()
dns_list = cfg['host']['dns'].strip().split()
for portmap in ports_list:
port_args.append('-p=' + portmap)
for dns in dns_list:
dns_args.append('--dns=' + dns)
docker_command = docker + [
arg for arg in [
'run',
cidfile_arg,
privileged_arg,
lxc_conf_arg,
name_arg,
hostname_arg,
volume_arg,
rm_arg,
net_arg,
] + port_args + dns_args +
['-t', '-i', cfg['container']['docker_image'], '/bin/bash']
if arg != ''
]
if cfg['build']['interactive'] >= 3:
print(
'\n\nAbout to start container. ' + 'Ports mapped will be: ' +
', '.join(port_args) +
'\n\n[host]\nports:<value>\n\nconfig, building on the ' +
'configurable base image passed in in:\n\n --image <image>\n'
+
'\nor config:\n\n [container]\n docker_image:<image>)\n\n'
+ 'Base image in this case is:\n\n ' +
cfg['container']['docker_image'] + '\n\n' +
util.colour('31', '\n[Hit return to continue]'))
util.util_raw_input(shutit=shutit)
shutit.cfg['build']['docker_command'] = ' '.join(docker_command)
shutit.log('\n\nCommand being run is:\n\n' +
shutit.cfg['build']['docker_command'],
force_stdout=True,
prefix=False)
shutit.log('\n\nThis may download the image, please be patient\n\n',
force_stdout=True,
prefix=False)
container_child = pexpect.spawn(docker_command[0], docker_command[1:])
expect = ['assword', cfg['expect_prompts']['base_prompt'].strip(), \
'Waiting', 'ulling', 'endpoint', 'Download']
res = container_child.expect(expect, 9999)
while True:
shutit.log(container_child.before + container_child.after,
prefix=False,
force_stdout=True)
if res == 0:
shutit.log('...')
res = shutit.send(cfg['host']['password'], \
child=container_child, expect=expect, timeout=9999, \
check_exit=False, fail_on_empty_before=False)
elif res == 1:
shutit.log('Prompt found, breaking out')
break
else:
res = container_child.expect(expect, 9999)
continue
# Get the cid
time.sleep(5) # cidfile creation is sometimes slow...
shutit.log('Slept')
cid = open(cfg['build']['cidfile']).read()
shutit.log('Opening file')
if cid == '' or re.match('^[a-z0-9]+$', cid) == None:
shutit.fail(
'Could not get container_id - quitting. ' + 'Check whether ' +
'other containers may be clashing on port allocation or name.'
+ '\nYou might want to try running: sudo docker kill ' +
cfg['container']['name'] + '; sudo docker rm ' +
cfg['container']['name'] + '\nto resolve a name clash or: ' +
cfg['host']['docker_executable'] + ' ps -a | grep ' +
cfg['container']['ports'] + ' | awk \'{print $1}\' | ' +
'xargs ' + cfg['host']['docker_executable'] + ' kill\nto + '
'resolve a port clash\n')
shutit.log('cid: ' + cid)
cfg['container']['container_id'] = cid
self._setup_prompts(shutit, container_child)
self._add_begin_build_info(shutit, docker_command)
return True
def init_shutit_map(shutit):
"""Initializes the module map of shutit based on the modules
we have gathered.
Checks we have core modules
Checks for duplicate module details.
Sets up common config.
Sets up map of modules.
"""
cfg = shutit.cfg
modules = shutit.shutit_modules
# Have we got anything to process outside of special modules?
if len([mod for mod in modules if mod.run_order > 0]) < 1:
shutit.log(modules)
path = ':'.join(cfg['host']['shutit_module_path'])
if path == '':
shutit.fail('No modules aside from core ones found and no ShutIt' +
' module path given. ' +
'\nDid you set --shutit_module_path/-m' +
' wrongly?')
elif path == '.':
shutit.fail('No modules aside from core ones found and no ShutIt' +
' module path given apart from default (.).\nDid you' +
' set--shutit_module_path/-m? Is there a STOP* file' +
' in your . dir?')
else:
shutit.fail('No modules aside from core ones found and no ShutIt ' +
'modules in path:\n\n' + path +
'\n\nor their subfolders. Check your ' +
'--shutit_module_path/-m setting and check that there are ' +
'ShutItmodules below without STOP* files in any relevant ' +
'directories.')
shutit.log('PHASE: base setup', code='31')
if cfg['build']['interactive'] >= 3:
shutit.log('\nChecking to see whether there are duplicate module ids ' +
'or run orders in the visible modules.', force_stdout=True)
shutit.log('\nModules I see are:\n', force_stdout=True)
for module in modules:
shutit.log(module.module_id, force_stdout=True, code='31')
shutit.log('\n', force_stdout=True)
run_orders = {}
has_core_module = False
for module in modules:
assert isinstance(module, ShutItModule)
if module.module_id in shutit.shutit_map:
shutit.fail('Duplicated module id: ' + module.module_id +
'\n\nYou may want to check your --shutit_module_path setting')
if module.run_order in run_orders:
shutit.fail('Duplicate run order: ' + str(module.run_order) +
' for ' + module.module_id + ' and ' +
run_orders[module.run_order].module_id +
'\n\nYou may want to check your --shutit_module_path setting')
if module.run_order == 0:
has_core_module = True
shutit.shutit_map[module.module_id] = run_orders[module.run_order] = module
if not has_core_module:
shutit.fail('No module with run_order=0 specified! This is required.')
if cfg['build']['interactive'] >= 3:
print(util.colour('31', 'Module id and run order checks OK' +
'\n\n[Hit return to continue]\n'))
util.util_raw_input(shutit=shutit)
