def build(self, shutit, loglevel=logging.DEBUG):
"""Sets up the machine ready for building.
"""
cfg = shutit.cfg
ssh_host = cfg[self.module_id]['ssh_host']
ssh_port = cfg[self.module_id]['ssh_port']
ssh_user = cfg[self.module_id]['ssh_user']
ssh_pass = cfg[self.module_id]['password']
ssh_key = cfg[self.module_id]['ssh_key']
ssh_cmd = cfg[self.module_id]['ssh_cmd']
opts = [
'-t', '-o', 'UserKnownHostsFile=/dev/null', '-o',
'StrictHostKeyChecking=no'
]
if ssh_pass == '':
opts += ['-o', 'PasswordAuthentication=no']
if ssh_port != '':
opts += ['-p', ssh_port]
if ssh_key != '':
opts += ['-i', ssh_key]
host_arg = ssh_host
if host_arg == '':
shutit.fail('No host specified for sshing', throw_exception=False)
if ssh_user != '':
host_arg = ssh_user + '@' + host_arg
cmd_arg = ssh_cmd
if cmd_arg == '':
cmd_arg = 'sudo su -s /bin/bash -'
ssh_command = ['ssh'] + opts + [host_arg, cmd_arg]
if shutit.build['interactive'] >= 3:
print('\n\nAbout to connect to host.' + '\n\n' +
shutit_util.colourise('32', '\n[Hit return to continue]'))
shutit_util.util_raw_input()
shutit.build['ssh_command'] = ' '.join(ssh_command)
shutit.log('Command being run is: ' + shutit.build['ssh_command'],
level=logging.INFO)
shutit_pexpect_session = shutit_pexpect.ShutItPexpectSession(
'target_child', ssh_command[0], ssh_command[1:])
target_child = shutit_pexpect_session.pexpect_child
expect = ['assword', shutit.expect_prompts['base_prompt'].strip()]
res = shutit.child_expect(target_child, expect, timeout=10)
while True:
shutit.log(target_child.before + target_child.after,
level=logging.DEBUG)
if res == 0:
shutit.log('...', level=logging.DEBUG)
res = shutit.send(ssh_pass,
shutit_pexpect_child=target_child,
expect=expect,
timeout=10,
check_exit=False,
fail_on_empty_before=False,
echo=False,
loglevel=loglevel)
elif res == 1:
shutit.log('Prompt found, breaking out', level=logging.DEBUG)
break
self.setup_host_child()
self.setup_target_child(target_child)
return True
def build_module(shutit, module, loglevel=logging.DEBUG):
"""Build passed-in module.
"""
cfg = shutit.cfg
shutit.log('Building ShutIt module: ' + module.module_id + ' with run order: ' + str(module.run_order), level=logging.INFO)
cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding ShutIt module: ' + module.module_id + ' with run order: ' + str(module.run_order))
if not module.build(shutit):
shutit.fail(module.module_id + ' failed on build', shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').pexpect_child)
else:
if cfg['build']['delivery'] in ('docker','dockerfile'):
# Create a directory and files to indicate this has been built.
shutit.send(' mkdir -p ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + ' && touch ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + '/built && rm -f ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + '/removed', loglevel=loglevel)
# Put it into "installed" cache
cfg['environment'][cfg['build']['current_environment_id']]['modules_installed'].append(module.module_id)
# Remove from "not installed" cache
if module.module_id in cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed']:
cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed'].remove(module.module_id)
shutit.pause_point('\nPausing to allow inspect of build for: ' + module.module_id, print_input=True, level=2)
cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' + module.module_id)
if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['interactive'] >= 3:
shutit.log(shutit_util.build_report(shutit, '#Module:' + module.module_id), level=logging.DEBUG)
if (not cfg[module.module_id]['shutit.core.module.tag'] and cfg['build']['interactive'] >= 2):
print ("\n\nDo you want to save state now we\'re at the " + "end of this module? (" + module.module_id + ") (input y/n)")
cfg[module.module_id]['shutit.core.module.tag'] = (shutit_util.util_raw_input(shutit=shutit,default='y') == 'y')
if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['tag_modules']:
shutit.log(module.module_id + ' configured to be tagged, doing repository work',level=logging.INFO)
# Stop all before we tag to avoid file changing errors, and clean up pid files etc..
stop_all(shutit, module.run_order)
shutit.do_repository_work(str(module.module_id) + '_' + str(module.run_order), password=cfg['host']['password'], docker_executable=cfg['host']['docker_executable'], force=True)
# Start all after we tag to ensure services are up as expected.
start_all(shutit, module.run_order)
if cfg['build']['interactive'] >= 2:
print ("\n\nDo you want to stop interactive mode? (input y/n)\n")
if shutit_util.util_raw_input(shutit=shutit,default='y') == 'y':
cfg['build']['interactive'] = 0
def do_finalize(shutit):
"""Runs finalize phase; run after all builds are complete and all modules
have been stopped.
"""
cfg = shutit.cfg
# Stop all the modules
if cfg['build']['interactive'] >= 3:
print('\nStopping all modules before finalize phase' + shutit_util.colour('32',
'\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
stop_all(shutit)
# Finalize in reverse order
shutit.log('PHASE: finalize', code='32')
if cfg['build']['interactive'] >= 3:
print('\nNow doing finalize phase, which we do when all builds are ' +
'complete and modules are stopped' +
shutit_util.colour('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
# Login at least once to get the exports.
for module_id in shutit_util.module_ids(shutit, rev=True):
# Only finalize if it's thought to be installed.
if shutit_util.is_installed(shutit, shutit.shutit_map[module_id]):
shutit.login(prompt_prefix=module_id,command='bash')
if not shutit.shutit_map[module_id].finalize(shutit):
shutit.fail(module_id + ' failed on finalize',
child=shutit.pexpect_children['target_child'])
shutit.logout()
def do_test(shutit):
"""Runs test phase, erroring if any return false.
"""
cfg = shutit.cfg
if not cfg['build']['dotest']:
shutit.log('Tests configured off, not running')
return
# Test in reverse order
shutit.log('PHASE: test', code='32')
if cfg['build']['interactive'] >= 3:
print '\nNow doing test phase' + shutit_util.colour(
'32', '\n\n[Hit return to continue]\n')
shutit_util.util_raw_input(shutit=shutit)
stop_all(shutit)
start_all(shutit)
for module_id in module_ids(shutit, rev=True):
module = shutit.shutit_map[module_id]
# Only test if it's installed.
if is_installed(shutit, shutit.shutit_map[module_id]):
shutit.log('RUNNING TEST ON: ' + module_id, code='32')
shutit.login(prompt_prefix=module_id, command='bash')
if not shutit.shutit_map[module_id].test(shutit):
shutit.fail(module_id + ' failed on test',
child=shutit.pexpect_children['target_child'])
shutit.logout()
def do_finalize():
"""Runs finalize phase; run after all builds are complete and all modules
have been stopped.
"""
shutit = shutit_global.shutit
# Stop all the modules
if shutit.build['interactive'] >= 3:
print('\nStopping all modules before finalize phase' +
shutit_util.colourise('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input()
stop_all()
# Finalize in reverse order
shutit.log('PHASE: finalize', level=logging.DEBUG)
if shutit.build['interactive'] >= 3:
print('\nNow doing finalize phase, which we do when all builds are ' +
'complete and modules are stopped' +
shutit_util.colourise('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input()
# Login at least once to get the exports.
for module_id in shutit_util.module_ids(rev=True):
# Only finalize if it's thought to be installed.
if shutit_util.is_installed(shutit.shutit_map[module_id]):
shutit.login(prompt_prefix=module_id, command='bash')
if not shutit.shutit_map[module_id].finalize(shutit):
shutit.fail(module_id + ' failed on finalize',
shutit_pexpect_child=shutit.
get_shutit_pexpect_session_from_id(
'target_child').pexpect_child)
shutit.logout()
def do_finalize(shutit):
"""Runs finalize phase; run after all builds are complete and all modules
have been stopped.
"""
cfg = shutit.cfg
# Stop all the modules
if cfg["build"]["interactive"] >= 3:
print (
"\nStopping all modules before finalize phase" + shutit_util.colour("32", "\n\n[Hit return to continue]\n")
)
shutit_util.util_raw_input(shutit=shutit)
stop_all(shutit)
# Finalize in reverse order
shutit.log("PHASE: finalize", code="32")
if cfg["build"]["interactive"] >= 3:
print (
"\nNow doing finalize phase, which we do when all builds are "
+ "complete and modules are stopped"
+ shutit_util.colour("32", "\n\n[Hit return to continue]\n")
)
shutit_util.util_raw_input(shutit=shutit)
# Login at least once to get the exports.
for module_id in module_ids(shutit, rev=True):
# Only finalize if it's thought to be installed.
if is_installed(shutit, shutit.shutit_map[module_id]):
shutit.login(prompt_prefix=module_id, command="bash")
if not shutit.shutit_map[module_id].finalize(shutit):
shutit.fail(module_id + " failed on finalize", child=shutit.pexpect_children["target_child"])
shutit.logout()
def do_test():
"""Runs test phase, erroring if any return false.
"""
shutit = shutit_global.shutit
if not shutit.build['dotest']:
shutit.log('Tests configured off, not running', level=logging.DEBUG)
return
# Test in reverse order
shutit.log('PHASE: test', level=logging.DEBUG)
if shutit.build['interactive'] >= 3:
print '\nNow doing test phase' + shutit_util.colourise(
'32', '\n\n[Hit return to continue]\n')
shutit_util.util_raw_input()
stop_all()
start_all()
for module_id in shutit_util.module_ids(rev=True):
# Only test if it's installed.
if shutit_util.is_installed(shutit.shutit_map[module_id]):
shutit.log('RUNNING TEST ON: ' + module_id, level=logging.DEBUG)
shutit.login(prompt_prefix=module_id, command='bash')
if not shutit.shutit_map[module_id].test(shutit):
shutit.fail(module_id + ' failed on test',
shutit_pexpect_child=shutit.
get_shutit_pexpect_session_from_id(
'target_child').pexpect_child)
shutit.logout()
def do_finalize(shutit):
"""Runs finalize phase; run after all builds are complete and all modules
have been stopped.
"""
cfg = shutit.cfg
# Stop all the modules
if cfg['build']['interactive'] >= 3:
print('\nStopping all modules before finalize phase' +
shutit_util.colour('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
stop_all(shutit)
# Finalize in reverse order
shutit.log('PHASE: finalize', code='32')
if cfg['build']['interactive'] >= 3:
print('\nNow doing finalize phase, which we do when all builds are ' +
'complete and modules are stopped' +
shutit_util.colour('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
# Login at least once to get the exports.
for module_id in module_ids(shutit, rev=True):
# Only finalize if it's thought to be installed.
if is_installed(shutit, shutit.shutit_map[module_id]):
shutit.login(prompt_prefix=module_id, command='bash')
if not shutit.shutit_map[module_id].finalize(shutit):
shutit.fail(module_id + ' failed on finalize',
child=shutit.pexpect_children['target_child'])
shutit.logout()
def setup_shutit_path(cfg):
# try the current directory, the .. directory, or the ../shutit directory, the ~/shutit
if not cfg["host"]["add_shutit_to_path"]:
return
res = shutit_util.util_raw_input(
prompt="shutit appears not to be on your path - should try and we find it and add it to your ~/.bashrc (Y/n)?"
)
if res in ["n", "N"]:
with open(os.path.join(cfg["shutit_home"], "config"), "a") as f:
f.write("\n[host]\nadd_shutit_to_path: no\n")
return
path_to_shutit = ""
for d in [".", "..", "~", "~/shutit"]:
path = os.path.abspath(d + "/shutit")
if not os.path.isfile(path):
continue
path_to_shutit = path
while path_to_shutit == "":
d = shutit_util.util_raw_input(prompt="cannot auto-find shutit - please input the path to your shutit dir\n")
path = os.path.abspath(d + "/shutit")
if not os.path.isfile(path):
continue
path_to_shutit = path
if path_to_shutit != "":
bashrc = os.path.expanduser("~/.bashrc")
with open(bashrc, "a") as myfile:
# http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch
myfile.write('\nexport PATH="$PATH:' + os.path.dirname(path_to_shutit) + '"\n')
shutit_util.util_raw_input(prompt="\nPath set up - please open new terminal and re-run command\n")
sys.exit()
def do_test(shutit):
"""Runs test phase, erroring if any return false.
"""
cfg = shutit.cfg
if not cfg['build']['dotest']:
shutit.log('Tests configured off, not running')
return
# Test in reverse order
shutit.log('PHASE: test', code='32')
if cfg['build']['interactive'] >= 3:
print '\nNow doing test phase' + shutit_util.colour('32',
'\n\n[Hit return to continue]\n')
shutit_util.util_raw_input(shutit=shutit)
stop_all(shutit)
start_all(shutit)
for module_id in module_ids(shutit, rev=True):
module = shutit.shutit_map[module_id]
# Only test if it's installed.
if is_installed(shutit, shutit.shutit_map[module_id]):
shutit.log('RUNNING TEST ON: ' + module_id, code='32')
shutit.login(prompt_prefix=module_id,command='bash')
if not shutit.shutit_map[module_id].test(shutit):
shutit.fail(module_id + ' failed on test',
child=shutit.pexpect_children['target_child'])
shutit.logout()
def setup_shutit_path(cfg):
# try the current directory, the .. directory, or the ../shutit directory, the ~/shutit
if not cfg['host']['add_shutit_to_path']:
return
res = shutit_util.util_raw_input(prompt='shutit appears not to be on your path - should try and we find it and add it to your ~/.bashrc (Y/n)?')
if res in ['n','N']:
with open(os.path.join(cfg['shutit_home'], 'config'), 'a') as f:
f.write('\n[host]\nadd_shutit_to_path: no\n')
return
path_to_shutit = ''
for d in ['.','..','~','~/shutit']:
path = os.path.abspath(d + '/shutit')
if not os.path.isfile(path):
continue
path_to_shutit = path
while path_to_shutit == '':
d = shutit_util.util_raw_input(prompt='cannot auto-find shutit - please input the path to your shutit dir\n')
path = os.path.abspath(d + '/shutit')
if not os.path.isfile(path):
continue
path_to_shutit = path
if path_to_shutit != '':
bashrc = os.path.expanduser('~/.bashrc')
with open(bashrc, "a") as myfile:
#http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch
myfile.write('\nexport PATH="$PATH:' + os.path.dirname(path_to_shutit) + '"\n')
shutit_util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n')
sys.exit()
def build(self, shutit):
"""Sets up the machine ready for building.
"""
cfg = shutit.cfg
ssh_host = cfg[self.module_id]['ssh_host']
ssh_port = cfg[self.module_id]['ssh_port']
ssh_user = cfg[self.module_id]['ssh_user']
ssh_pass = cfg[self.module_id]['password']
ssh_key = cfg[self.module_id]['ssh_key']
ssh_cmd = cfg[self.module_id]['ssh_cmd']
opts = [
'-t',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no'
]
if ssh_pass == '':
opts += ['-o', 'PasswordAuthentication=no']
if ssh_port != '':
opts += ['-p', ssh_port]
if ssh_key != '':
opts += ['-i', ssh_key]
host_arg = ssh_host
if host_arg == '':
shutit.fail('No host specified for sshing', throw_exception=False)
if ssh_user != '':
host_arg = ssh_user + '@' + host_arg
cmd_arg = ssh_cmd
if cmd_arg == '':
cmd_arg = 'sudo su -s /bin/bash -'
ssh_command = ['ssh'] + opts + [host_arg, cmd_arg]
if cfg['build']['interactive'] >= 3:
print('\n\nAbout to connect to host.' +
'\n\n' + shutit_util.colour('32', '\n[Hit return to continue]'))
shutit_util.util_raw_input(shutit=shutit)
cfg['build']['ssh_command'] = ' '.join(ssh_command)
shutit.log('\n\nCommand being run is:\n\n' + cfg['build']['ssh_command'],
force_stdout=True, prefix=False)
target_child = pexpect.spawn(ssh_command[0], ssh_command[1:])
expect = ['assword', cfg['expect_prompts']['base_prompt'].strip()]
res = target_child.expect(expect, 10)
while True:
shutit.log(target_child.before + target_child.after, prefix=False,
force_stdout=True)
if res == 0:
shutit.log('...')
res = shutit.send(ssh_pass,
child=target_child, expect=expect, timeout=10,
check_exit=False, fail_on_empty_before=False)
elif res == 1:
shutit.log('Prompt found, breaking out')
break
self._setup_prompts(shutit, target_child)
self._add_begin_build_info(shutit, ssh_command)
return True
def build(self, shutit, loglevel=logging.DEBUG):
"""Sets up the machine ready for building.
"""
cfg = shutit.cfg
ssh_host = cfg[self.module_id]['ssh_host']
ssh_port = cfg[self.module_id]['ssh_port']
ssh_user = cfg[self.module_id]['ssh_user']
ssh_pass = cfg[self.module_id]['password']
ssh_key = cfg[self.module_id]['ssh_key']
ssh_cmd = cfg[self.module_id]['ssh_cmd']
opts = [
'-t',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no'
]
if ssh_pass == '':
opts += ['-o', 'PasswordAuthentication=no']
if ssh_port != '':
opts += ['-p', ssh_port]
if ssh_key != '':
opts += ['-i', ssh_key]
host_arg = ssh_host
if host_arg == '':
shutit.fail('No host specified for sshing', throw_exception=False)
if ssh_user != '':
host_arg = ssh_user + '@' + host_arg
cmd_arg = ssh_cmd
if cmd_arg == '':
cmd_arg = 'sudo su -s /bin/bash -'
ssh_command = ['ssh'] + opts + [host_arg, cmd_arg]
if shutit.build['interactive'] >= 3:
print('\n\nAbout to connect to host.' + '\n\n' + shutit_util.colourise('32', '\n[Hit return to continue]'))
shutit_util.util_raw_input()
shutit.build['ssh_command'] = ' '.join(ssh_command)
shutit.log('Command being run is: ' + shutit.build['ssh_command'],level=logging.INFO)
shutit_pexpect_session = shutit_pexpect.ShutItPexpectSession('target_child', ssh_command[0], ssh_command[1:])
target_child = shutit_pexpect_session.pexpect_child
expect = ['assword', shutit.expect_prompts['base_prompt'].strip()]
res = shutit.child_expect(target_child,expect, timeout=10)
while True:
shutit.log(target_child.before + target_child.after,level=logging.DEBUG)
if res == 0:
shutit.log('...',level=logging.DEBUG)
res = shutit.send(ssh_pass, shutit_pexpect_child=target_child, expect=expect, timeout=10, check_exit=False, fail_on_empty_before=False, echo=False, loglevel=loglevel)
elif res == 1:
shutit.log('Prompt found, breaking out',level=logging.DEBUG)
break
self.setup_host_child()
self.setup_target_child(target_child)
return True
def init_shutit_map(shutit):
"""Initializes the module map of shutit based on the modules
we have gathered.
Checks we have core modules
Checks for duplicate module details.
Sets up common config.
Sets up map of modules.
"""
cfg = shutit.cfg
modules = shutit.shutit_modules
# Have we got anything to process outside of special modules?
if len([mod for mod in modules if mod.run_order > 0]) < 1:
shutit.log(modules,level=logging.DEBUG)
path = ':'.join(cfg['host']['shutit_module_path'])
shutit.log('\nIf you are new to ShutIt, see:\n\n\thttp://ianmiell.github.io/shutit/\n\nor try running\n\n\tshutit skeleton\n\n',code=32,level=logging.INFO)
if path == '':
shutit.fail('No ShutIt modules aside from core ones found and no ShutIt module path given.\nDid you set --shutit_module_path/-m wrongly?\n')
elif path == '.':
shutit.fail('No modules aside from core ones found and no ShutIt module path given apart from default (.).\n\n- Did you set --shutit_module_path/-m?\n- Is there a STOP* file in your . dir?')
else:
shutit.fail('No modules aside from core ones found and no ShutIt modules in path:\n\n' + path + '\n\nor their subfolders. Check your --shutit_module_path/-m setting and check that there are ShutIt modules below without STOP* files in any relevant directories.')
shutit.log('PHASE: base setup', level=logging.DEBUG)
if cfg['build']['interactive'] >= 3:
shutit.log('\nChecking to see whether there are duplicate module ids or run orders in the visible modules.\nModules I see are:\n',level=logging.DEBUG)
for module in modules:
shutit.log(module.module_id, level=logging.DEBUG)
shutit.log('\n',level=logging.DEBUG)
run_orders = {}
has_core_module = False
for module in modules:
assert isinstance(module, ShutItModule)
if module.module_id in shutit.shutit_map:
shutit.fail('Duplicated module id: ' + module.module_id + '\n\nYou may want to check your --shutit_module_path setting')
if module.run_order in run_orders:
shutit.fail('Duplicate run order: ' + str(module.run_order) + ' for ' + module.module_id + ' and ' + run_orders[module.run_order].module_id + '\n\nYou may want to check your --shutit_module_path setting')
if module.run_order == 0:
has_core_module = True
shutit.shutit_map[module.module_id] = run_orders[module.run_order] = module
if not has_core_module:
shutit.fail('No module with run_order=0 specified! This is required.')
if cfg['build']['interactive'] >= 3:
print(shutit_util.colour('32', 'Module id and run order checks OK\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
def stop_all(run_order=-1):
"""Runs stop method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we clean up state
before committing run files etc.
"""
shutit = shutit_global.shutit
if shutit.build['interactive'] >= 3:
print('\nRunning stop on all modules' + shutit_util.colourise('32', '\n\n[Hit return to continue]'))
shutit_util.util_raw_input()
# sort them so they're stopped in reverse order
for module_id in shutit_util.module_ids(rev=True):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if shutit_util.is_installed(shutit_module_obj):
if not shutit_module_obj.stop(shutit):
shutit.fail('failed to stop: ' + module_id, shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').shutit_pexpect_child)
def stop_all(shutit, run_order=-1):
"""Runs stop method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we clean up state
before committing run files etc.
"""
cfg = shutit.cfg
if cfg["build"]["interactive"] >= 3:
print ("\nRunning stop on all modules" + shutit_util.colour("32", "\n\n[Hit return to continue]"))
shutit_util.util_raw_input(shutit=shutit)
# sort them so they're stopped in reverse order
for module_id in module_ids(shutit, rev=True):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if is_installed(shutit, shutit_module_obj):
if not shutit_module_obj.stop(shutit):
shutit.fail("failed to stop: " + module_id, child=shutit.pexpect_children["target_child"])
def start_all(shutit, run_order=-1):
"""Runs start method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we can export a clean
target and still depended-on modules running if necessary.
"""
cfg = shutit.cfg
if cfg["build"]["interactive"] >= 3:
print ("\nRunning start on all modules" + shutit_util.colour("32", "\n\n[Hit return to continue]\n"))
shutit_util.util_raw_input(shutit=shutit)
# sort them so they're started in order
for module_id in module_ids(shutit):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if is_installed(shutit, shutit_module_obj):
if not shutit_module_obj.start(shutit):
shutit.fail("failed to start: " + module_id, child=shutit.pexpect_children["target_child"])
def do_phone_home(
shutit,
msg=None,
question='Error seen - would you like to inform the maintainers?'):
"""Report message home.
msg - message to send home
question - question to ask - assumes Y/y for send message, else no
"""
if msg is None:
msg = {}
if shutit.build['interactive'] == 0:
return
msg.update({
'shutitrunstatus': 'fail',
'pwd': os.getcwd(),
'user': os.environ.get('LOGNAME', '')
})
if question != '' and shutit_util.util_raw_input(
shutit, prompt=question + ' (Y/n)\n') not in ('y', 'Y', ''):
return
try:
urllib.urlopen("http://shutit.tk?" + urllib.urlencode(msg))
except Exception as e:
shutit.log('failed to send message: ' + str(e.message),
level=logging.ERROR)
def start_all(run_order=-1):
"""Runs start method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we can export a clean
target and still depended-on modules running if necessary.
"""
shutit = shutit_global.shutit
if shutit.build['interactive'] >= 3:
print('\nRunning start on all modules' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input()
# sort them so they're started in order
for module_id in shutit_util.module_ids():
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if shutit_util.is_installed(shutit_module_obj):
if not shutit_module_obj.start(shutit):
shutit.fail('failed to start: ' + module_id, shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').shutit_pexpect_child)
def conn_target(shutit):
"""Connect to the target.
"""
conn_module = None
for mod in shutit.conn_modules:
if mod.module_id == shutit.build['conn_module']:
conn_module = mod
break
if conn_module is None:
shutit.fail('Couldn\'t find conn_module ' + shutit.build['conn_module'])
# Set up the target in pexpect.
if shutit.build['interactive'] >= 3:
print('\nRunning the conn module (' + shutit.shutit_main_dir + '/shutit_setup.py)' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input()
conn_module.get_config(shutit)
conn_module.build(shutit)
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='32')
shutit.log(shutit_util.print_config(cfg))
if cfg['build']['interactive'] >= 3:
print ('\nNow building any modules that need building' +
shutit_util.colour('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
module_id_list = shutit_util.module_ids(shutit)
if cfg['build']['deps_only']:
module_id_list_build_only = filter(lambda x: cfg[x]['shutit.core.module.build'], module_id_list)
for module_id in module_id_list:
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='32')
if cfg[module.module_id]['shutit.core.module.build']:
if cfg['build']['delivery'] not in module.ok_delivery_methods:
shutit.fail('Module: ' + module.module_id + ' can only be built with one of these --delivery methods: ' + str(module.ok_delivery_methods) + '\nSee shutit build -h for more info, or try adding: --delivery <method> to your shutit invocation')
if shutit_util.is_installed(shutit,module):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' + module.module_id +
' with run order: ' + str(module.run_order))
else:
# We move to the module directory to perform the build, returning immediately afterwards.
if cfg['build']['deps_only'] and module_id == module_id_list_build_only[-1]:
# If this is the last module, and we are only building deps, stop here.
cfg['build']['report'] = (cfg['build']['report'] + '\nSkipping: ' +
module.module_id + ' with run order: ' + str(module.run_order) +
'\n\tas this is the final module and we are building dependencies only')
else:
revert_dir = os.getcwd()
cfg['environment'][cfg['build']['current_environment_id']]['module_root_dir'] = os.path.dirname(module.__module_file)
shutit.chdir(cfg['environment'][cfg['build']['current_environment_id']]['module_root_dir'])
shutit.login(prompt_prefix=module_id,command='bash')
build_module(shutit, module)
shutit.logout()
shutit.chdir(revert_dir)
if shutit_util.is_installed(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['target_child'])
def start_all(shutit, run_order=-1):
"""Runs start method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we can export a clean
target and still depended-on modules running if necessary.
"""
cfg = shutit.cfg
if cfg['build']['interactive'] >= 3:
print('\nRunning start on all modules' +
shutit_util.colour('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
# sort them so they're started in order
for module_id in module_ids(shutit):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if is_installed(shutit, shutit_module_obj):
if not shutit_module_obj.start(shutit):
shutit.fail('failed to start: ' + module_id, \
child=shutit.pexpect_children['target_child'])
def stop_all(shutit, run_order=-1):
"""Runs stop method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we clean up state
before committing run files etc.
"""
cfg = shutit.cfg
if cfg['build']['interactive'] >= 3:
print('\nRunning stop on all modules' + \
shutit_util.colour('32', '\n\n[Hit return to continue]'))
shutit_util.util_raw_input(shutit=shutit)
# sort them so they're stopped in reverse order
for module_id in module_ids(shutit, rev=True):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if is_installed(shutit, shutit_module_obj):
if not shutit_module_obj.stop(shutit):
shutit.fail('failed to stop: ' + \
module_id, child=shutit.pexpect_children['target_child'])
def conn_target(shutit):
"""Connect to the target.
"""
conn_module = None
for mod in shutit.conn_modules:
if mod.module_id == shutit.build['conn_module']:
conn_module = mod
break
if conn_module is None:
shutit.fail('Couldn\'t find conn_module ' +
shutit.build['conn_module'])
# Set up the target in pexpect.
if shutit.build['interactive'] >= 3:
print('\nRunning the conn module (' + shutit.shutit_main_dir +
'/shutit_setup.py)' +
shutit_util.colourise('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input()
conn_module.get_config(shutit)
conn_module.build(shutit)
def stop_all(run_order=-1):
"""Runs stop method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we clean up state
before committing run files etc.
"""
shutit = shutit_global.shutit
if shutit.build['interactive'] >= 3:
print('\nRunning stop on all modules' +
shutit_util.colourise('32', '\n\n[Hit return to continue]'))
shutit_util.util_raw_input()
# sort them so they're stopped in reverse order
for module_id in shutit_util.module_ids(rev=True):
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if shutit_util.is_installed(shutit_module_obj):
if not shutit_module_obj.stop(shutit):
shutit.fail('failed to stop: ' + module_id,
shutit_pexpect_child=shutit.
get_shutit_pexpect_session_from_id(
'target_child').shutit_pexpect_child)
def start_all(run_order=-1):
"""Runs start method on all modules less than the passed-in run_order.
Used when target is exporting itself mid-build, so we can export a clean
target and still depended-on modules running if necessary.
"""
shutit = shutit_global.shutit
if shutit.build['interactive'] >= 3:
print('\nRunning start on all modules' +
shutit_util.colourise('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input()
# sort them so they're started in order
for module_id in shutit_util.module_ids():
shutit_module_obj = shutit.shutit_map[module_id]
if run_order == -1 or shutit_module_obj.run_order <= run_order:
if shutit_util.is_installed(shutit_module_obj):
if not shutit_module_obj.start(shutit):
shutit.fail('failed to start: ' + module_id,
shutit_pexpect_child=shutit.
get_shutit_pexpect_session_from_id(
'target_child').shutit_pexpect_child)
def setup_shutit_path(shutit):
# try the current directory, the .. directory, or the ../shutit directory, the ~/shutit
if not shutit.host['add_shutit_to_path']:
return
res = shutit_util.util_raw_input(
shutit,
prompt=
'shutit appears not to be on your path - should try and we find it and add it to your ~/.bashrc (Y/n)?'
)
if res in ['n', 'N']:
with open(os.path.join(shutit.host['shutit_path'], 'config'),
'a') as f:
f.write('\n[host]\nadd_shutit_to_path: no\n')
return
path_to_shutit = ''
for d in ['.', '..', '~', '~/shutit']:
path = os.path.abspath(d + '/shutit')
if not os.path.isfile(path):
continue
path_to_shutit = path
while path_to_shutit == '':
d = shutit_util.util_raw_input(
shutit,
prompt=
'cannot auto-find shutit - please input the path to your shutit dir\n'
)
path = os.path.abspath(d + '/shutit')
if not os.path.isfile(path):
continue
path_to_shutit = path
if path_to_shutit != '':
bashrc = os.path.expanduser('~/.bashrc')
with open(bashrc, "a") as myfile:
#http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch
myfile.write('\nexport PATH="$PATH:' +
os.path.dirname(path_to_shutit) + '"\n')
shutit_util.util_raw_input(
shutit,
prompt=
'\nPath set up - please open new terminal and re-run command\n')
shutit_util.handle_exit(shutit=shutit)
def do_test():
"""Runs test phase, erroring if any return false.
"""
shutit = shutit_global.shutit
if not shutit.build['dotest']:
shutit.log('Tests configured off, not running',level=logging.DEBUG)
return
# Test in reverse order
shutit.log('PHASE: test', level=logging.DEBUG)
if shutit.build['interactive'] >= 3:
print '\nNow doing test phase' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')
shutit_util.util_raw_input()
stop_all()
start_all()
for module_id in shutit_util.module_ids(rev=True):
# Only test if it's installed.
if shutit_util.is_installed(shutit.shutit_map[module_id]):
shutit.log('RUNNING TEST ON: ' + module_id, level=logging.DEBUG)
shutit.login(prompt_prefix=module_id,command='bash')
if not shutit.shutit_map[module_id].test(shutit):
shutit.fail(module_id + ' failed on test', shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').pexpect_child)
shutit.logout()
def do_finalize():
"""Runs finalize phase; run after all builds are complete and all modules
have been stopped.
"""
shutit = shutit_global.shutit
# Stop all the modules
if shutit.build['interactive'] >= 3:
print('\nStopping all modules before finalize phase' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input()
stop_all()
# Finalize in reverse order
shutit.log('PHASE: finalize', level=logging.DEBUG)
if shutit.build['interactive'] >= 3:
print('\nNow doing finalize phase, which we do when all builds are ' + 'complete and modules are stopped' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input()
# Login at least once to get the exports.
for module_id in shutit_util.module_ids(rev=True):
# Only finalize if it's thought to be installed.
if shutit_util.is_installed(shutit.shutit_map[module_id]):
shutit.login(prompt_prefix=module_id,command='bash')
if not shutit.shutit_map[module_id].finalize(shutit):
shutit.fail(module_id + ' failed on finalize', shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').pexpect_child)
shutit.logout()
def do_test(shutit):
"""Runs test phase, erroring if any return false.
"""
cfg = shutit.cfg
if not cfg["build"]["dotest"]:
shutit.log("Tests configured off, not running")
return
# Test in reverse order
shutit.log("PHASE: test", code="32")
if cfg["build"]["interactive"] >= 3:
print "\nNow doing test phase" + shutit_util.colour("32", "\n\n[Hit return to continue]\n")
shutit_util.util_raw_input(shutit=shutit)
stop_all(shutit)
start_all(shutit)
for module_id in module_ids(shutit, rev=True):
module = shutit.shutit_map[module_id]
# Only test if it's installed.
if is_installed(shutit, shutit.shutit_map[module_id]):
shutit.log("RUNNING TEST ON: " + module_id, code="32")
shutit.login(prompt_prefix=module_id, command="bash")
if not shutit.shutit_map[module_id].test(shutit):
shutit.fail(module_id + " failed on test", child=shutit.pexpect_children["target_child"])
shutit.logout()
def conn_target(shutit):
"""Connect to the target.
"""
cfg = shutit.cfg
conn_module = None
cfg = shutit.cfg
for mod in shutit.conn_modules:
if mod.module_id == cfg["build"]["conn_module"]:
conn_module = mod
break
if conn_module is None:
shutit.fail("Couldn't find conn_module " + cfg["build"]["conn_module"])
# Set up the target in pexpect.
if cfg["build"]["interactive"] >= 3:
print (
"\nRunning the conn module ("
+ shutit.shutit_main_dir
+ "/shutit_setup.py)"
+ shutit_util.colour("32", "\n\n[Hit return to continue]\n")
)
shutit_util.util_raw_input(shutit=shutit)
conn_module.get_config(shutit)
conn_module.build(shutit)
def do_phone_home(msg=None,question='Error seen - would you like to inform the maintainers?'):
"""Report message home.
msg - message to send home
question - question to ask - assumes Y/y for send message, else no
"""
if msg is None:
msg = {}
if shutit_global.shutit.shutit.build['interactive'] == 0:
return
msg.update({'shutitrunstatus':'fail','pwd':os.getcwd(),'user':os.environ.get('LOGNAME', '')})
if question != '' and shutit_util.util_raw_input(prompt=question + ' (Y/n)\n') not in ('y','Y',''):
return
try:
urllib.urlopen("http://shutit.tk?" + urllib.urlencode(msg))
except Exception as e:
shutit_global.shutit.log('failed to send message: ' + str(e.message),level=logging.ERROR)
def do_phone_home(msg=None, question="Error seen - would you like to inform the maintainers?"):
"""Report message home.
msg - message to send home
question - question to ask - assumes Y/y for send message, else no
"""
cfg = shutit.cfg
if msg is None:
msg = {}
if shutit_global.cfg["build"]["interactive"] == 0:
return
msg.update({"shutitrunstatus": "fail", "pwd": os.getcwd(), "user": os.environ.get("LOGNAME", "")})
if question != "" and shutit_util.util_raw_input(prompt=question + " (Y/n)\n") not in ("y", "Y", ""):
return
try:
urllib.urlopen("http://shutit.tk?" + urllib.urlencode(msg))
except Exception as e:
shutit_global.shutit.log("failed to send message: " + str(e.message))
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log("PHASE: build, repository work", code="32")
shutit.log(shutit_util.print_config(cfg))
if cfg["build"]["interactive"] >= 3:
print (
"\nNow building any modules that need building" + shutit_util.colour("32", "\n\n[Hit return to continue]\n")
)
shutit_util.util_raw_input(shutit=shutit)
module_id_list = module_ids(shutit)
if cfg["build"]["deps_only"]:
module_id_list_build_only = filter(lambda x: cfg[x]["shutit.core.module.build"], module_id_list)
for module_id in module_id_list:
module = shutit.shutit_map[module_id]
shutit.log("considering whether to build: " + module.module_id, code="32")
if cfg[module.module_id]["shutit.core.module.build"]:
if cfg["build"]["delivery"] not in module.ok_delivery_methods:
shutit.fail(
"Module: "
+ module.module_id
+ " can only be built with one of these --delivery methods: "
+ str(module.ok_delivery_methods)
+ "\nSee shutit build -h for more info, or try adding: --delivery <method> to your shutit invocation"
)
if is_installed(shutit, module):
cfg["build"]["report"] = (
cfg["build"]["report"]
+ "\nBuilt already: "
+ module.module_id
+ " with run order: "
+ str(module.run_order)
)
else:
# We move to the module directory to perform the build, returning immediately afterwards.
if cfg["build"]["deps_only"] and module_id == module_id_list_build_only[-1]:
# If this is the last module, and we are only building deps, stop here.
cfg["build"]["report"] = (
cfg["build"]["report"]
+ "\nSkipping: "
+ module.module_id
+ " with run order: "
+ str(module.run_order)
+ "\n\tas this is the final module and we are building dependencies only"
)
else:
revert_dir = os.getcwd()
cfg["environment"][cfg["build"]["current_environment_id"]]["module_root_dir"] = os.path.dirname(
module.__module_file
)
shutit.chdir(cfg["environment"][cfg["build"]["current_environment_id"]]["module_root_dir"])
shutit.login(prompt_prefix=module_id, command="bash")
build_module(shutit, module)
shutit.logout()
shutit.chdir(revert_dir)
if is_installed(shutit, module):
shutit.log("Starting module")
if not module.start(shutit):
shutit.fail(module.module_id + " failed on start", child=shutit.pexpect_children["target_child"])
def build_module(shutit, module):
"""Build passed-in module.
"""
cfg = shutit.cfg
shutit.log("building: " + module.module_id + " with run order: " + str(module.run_order), code="32")
cfg["build"]["report"] = (
cfg["build"]["report"] + "\nBuilding: " + module.module_id + " with run order: " + str(module.run_order)
)
if not module.build(shutit):
shutit.fail(module.module_id + " failed on build", child=shutit.pexpect_children["target_child"])
else:
if cfg["build"]["delivery"] in ("docker", "dockerfile"):
# Create a directory and files to indicate this has been built.
shutit.send(
" mkdir -p "
+ cfg["build"]["build_db_dir"]
+ "/module_record/"
+ module.module_id
+ " && touch "
+ cfg["build"]["build_db_dir"]
+ "/module_record/"
+ module.module_id
+ "/built && rm -f "
+ cfg["build"]["build_db_dir"]
+ "/module_record/"
+ module.module_id
+ "/removed"
)
# Put it into "installed" cache
cfg["environment"][cfg["build"]["current_environment_id"]]["modules_installed"].append(module.module_id)
# Remove from "not installed" cache
if module.module_id in cfg["environment"][cfg["build"]["current_environment_id"]]["modules_not_installed"]:
cfg["environment"][cfg["build"]["current_environment_id"]]["modules_not_installed"].remove(module.module_id)
shutit.pause_point("\nPausing to allow inspect of build for: " + module.module_id, print_input=True, level=2)
cfg["build"]["report"] = cfg["build"]["report"] + "\nCompleted module: " + module.module_id
if cfg[module.module_id]["shutit.core.module.tag"] or cfg["build"]["interactive"] >= 3:
shutit.log(shutit_util.build_report(shutit, "#Module:" + module.module_id), code="32")
if not cfg[module.module_id]["shutit.core.module.tag"] and cfg["build"]["interactive"] >= 2:
shutit.log(
"\n\nDo you want to save state now we're at the "
+ "end of this module? ("
+ module.module_id
+ ") (input y/n)",
force_stdout=True,
code="32",
)
cfg[module.module_id]["shutit.core.module.tag"] = shutit_util.util_raw_input(shutit=shutit, default="y") == "y"
if cfg[module.module_id]["shutit.core.module.tag"] or cfg["build"]["tag_modules"]:
shutit.log(module.module_id + " configured to be tagged, doing repository work", force_stdout=True)
# Stop all before we tag to avoid file changing errors,
# and clean up pid files etc..
stop_all(shutit, module.run_order)
shutit.do_repository_work(
str(module.module_id) + "_" + str(module.run_order),
password=cfg["host"]["password"],
docker_executable=cfg["host"]["docker_executable"],
force=True,
)
# Start all after we tag to ensure services are up as expected.
start_all(shutit, module.run_order)
if cfg["build"]["interactive"] >= 2:
shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n", force_stdout=True, code="32")
if shutit_util.util_raw_input(shutit=shutit, default="y") == "y":
cfg["build"]["interactive"] = 0
def init_shutit_map(shutit):
"""Initializes the module map of shutit based on the modules
we have gathered.
Checks we have core modules
Checks for duplicate module details.
Sets up common config.
Sets up map of modules.
"""
cfg = shutit.cfg
modules = shutit.shutit_modules
# Have we got anything to process outside of special modules?
if len([mod for mod in modules if mod.run_order > 0]) < 1:
shutit.log(modules)
path = ":".join(cfg["host"]["shutit_module_path"])
shutit.log(
"\nIf you are new to ShutIt, see:\n\n\thttp://ianmiell.github.io/shutit/\n\nor try running\n\n\tshutit skeleton\n\n",
code=31,
prefix=False,
force_stdout=True,
)
if path == "":
shutit.fail(
"No ShutIt modules aside from core ones found and no ShutIt"
+ " module path given. "
+ "\nDid you set --shutit_module_path/-m wrongly?\n"
)
elif path == ".":
shutit.fail(
"No modules aside from core ones found and no ShutIt"
+ " module path given apart from default (.).\n\n- Did you"
+ " set --shutit_module_path/-m?\n- Is there a STOP* file"
+ " in your . dir?\n"
)
else:
shutit.fail(
"No modules aside from core ones found and no ShutIt "
+ "modules in path:\n\n"
+ path
+ "\n\nor their subfolders. Check your "
+ "--shutit_module_path/-m setting and check that there are "
+ "ShutIt modules below without STOP* files in any relevant "
+ "directories.\n"
)
shutit.log("PHASE: base setup", code="32")
if cfg["build"]["interactive"] >= 3:
shutit.log(
"\nChecking to see whether there are duplicate module ids " + "or run orders in the visible modules.",
force_stdout=True,
)
shutit.log("\nModules I see are:\n", force_stdout=True)
for module in modules:
shutit.log(module.module_id, force_stdout=True, code="32")
shutit.log("\n", force_stdout=True)
run_orders = {}
has_core_module = False
for module in modules:
assert isinstance(module, ShutItModule)
if module.module_id in shutit.shutit_map:
shutit.fail(
"Duplicated module id: "
+ module.module_id
+ "\n\nYou may want to check your --shutit_module_path setting"
)
if module.run_order in run_orders:
shutit.fail(
"Duplicate run order: "
+ str(module.run_order)
+ " for "
+ module.module_id
+ " and "
+ run_orders[module.run_order].module_id
+ "\n\nYou may want to check your --shutit_module_path setting"
)
if module.run_order == 0:
has_core_module = True
shutit.shutit_map[module.module_id] = run_orders[module.run_order] = module
if not has_core_module:
shutit.fail("No module with run_order=0 specified! This is required.")
if cfg["build"]["interactive"] >= 3:
print (shutit_util.colour("32", "Module id and run order checks OK" + "\n\n[Hit return to continue]\n"))
shutit_util.util_raw_input(shutit=shutit)
def build_module(shutit, module):
"""Build passed-in module.
"""
cfg = shutit.cfg
shutit.log('building: ' + module.module_id + ' with run order: ' +
str(module.run_order),
code='32')
cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding: ' +
module.module_id + ' with run order: ' +
str(module.run_order))
if not module.build(shutit):
shutit.fail(module.module_id + ' failed on build',
child=shutit.pexpect_children['target_child'])
else:
if cfg['build']['delivery'] in ('docker', 'dockerfile'):
# Create a directory and files to indicate this has been built.
shutit.send(' mkdir -p ' + cfg['build']['build_db_dir'] +
'/module_record/' + module.module_id + ' && touch ' +
cfg['build']['build_db_dir'] + '/module_record/' +
module.module_id + '/built && rm -f ' +
cfg['build']['build_db_dir'] + '/module_record/' +
module.module_id + '/removed')
# Put it into "installed" cache
cfg['environment'][cfg['build']['current_environment_id']][
'modules_installed'].append(module.module_id)
# Remove from "not installed" cache
if module.module_id in cfg['environment'][cfg['build'][
'current_environment_id']]['modules_not_installed']:
cfg['environment'][cfg['build']['current_environment_id']][
'modules_not_installed'].remove(module.module_id)
shutit.pause_point('\nPausing to allow inspect of build for: ' +
module.module_id,
print_input=True,
level=2)
cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' +
module.module_id)
if cfg[module.module_id][
'shutit.core.module.tag'] or cfg['build']['interactive'] >= 3:
shutit.log(shutit_util.build_report(shutit,
'#Module:' + module.module_id),
code='32')
if (not cfg[module.module_id]['shutit.core.module.tag']
and cfg['build']['interactive'] >= 2):
shutit.log("\n\nDo you want to save state now we\'re at the " +
"end of this module? (" + module.module_id +
") (input y/n)",
force_stdout=True,
code='32')
cfg[module.module_id]['shutit.core.module.tag'] = (
shutit_util.util_raw_input(shutit=shutit, default='y') == 'y')
if cfg[module.
module_id]['shutit.core.module.tag'] or cfg['build']['tag_modules']:
shutit.log(module.module_id +
' configured to be tagged, doing repository work',
force_stdout=True)
# Stop all before we tag to avoid file changing errors,
# and clean up pid files etc..
stop_all(shutit, module.run_order)
shutit.do_repository_work(
str(module.module_id) + '_' + str(module.run_order),
password=cfg['host']['password'],
docker_executable=cfg['host']['docker_executable'],
force=True)
# Start all after we tag to ensure services are up as expected.
start_all(shutit, module.run_order)
if cfg['build']['interactive'] >= 2:
shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n",
force_stdout=True,
code='32')
if shutit_util.util_raw_input(shutit=shutit, default='y') == 'y':
cfg['build']['interactive'] = 0
def start_container(self, shutit_session_name, loglevel=logging.DEBUG):
shutit = shutit_global.shutit
docker = shutit.host['docker_executable'].split(' ')
# Always-required options
if not os.path.exists(shutit.build['shutit_state_dir'] + '/cidfiles'):
os.makedirs(shutit.build['shutit_state_dir'] + '/cidfiles')
shutit.build['cidfile'] = shutit.build[
'shutit_state_dir'] + '/cidfiles/' + shutit.host[
'username'] + '_cidfile_' + shutit.build['build_id']
cidfile_arg = '--cidfile=' + shutit.build['cidfile']
# Singly-specified options
privileged_arg = ''
name_arg = ''
hostname_arg = ''
rm_arg = ''
net_arg = ''
mount_docker_arg = ''
shell_arg = '/bin/bash'
if shutit.build['privileged']:
privileged_arg = '--privileged=true'
if shutit.target['name'] != '':
name_arg = '--name=' + shutit.target['name']
if shutit.target['hostname'] != '':
hostname_arg = '-h=' + shutit.target['hostname']
if shutit.build['net'] != '':
net_arg = '--net="' + shutit.build['net'] + '"'
if shutit.build['mount_docker']:
mount_docker_arg = '-v=/var/run/docker.sock:/var/run/docker.sock'
# Incompatible with do_repository_work
if shutit.target['rm']:
rm_arg = '--rm=true'
if shutit.build['base_image'] in ('alpine', 'busybox'):
shell_arg = '/bin/ash'
# Multiply-specified options
port_args = []
dns_args = []
volume_args = []
volumes_from_args = []
volumes_list = shutit.target['volumes'].strip().split()
volumes_from_list = shutit.target['volumes_from'].strip().split()
ports_list = shutit.target['ports'].strip().split()
dns_list = shutit.host['dns'].strip().split()
for portmap in ports_list:
port_args.append('-p=' + portmap)
for dns in dns_list:
dns_args.append('--dns=' + dns)
for volume in volumes_list:
volume_args.append('-v=' + volume)
for volumes_from in volumes_from_list:
volumes_from_args.append('--volumes-from=' + volumes_from)
docker_command = docker + [
arg for arg in [
'run',
cidfile_arg,
privileged_arg,
name_arg,
hostname_arg,
rm_arg,
net_arg,
mount_docker_arg,
] + volume_args + volumes_from_args + port_args + dns_args +
['-t', '-i', shutit.target['docker_image'], shell_arg] if arg != ''
]
if shutit.build['interactive'] >= 3:
print(
'\n\nAbout to start container. Ports mapped will be: ' +
', '.join(port_args) +
'\n\n[host]\nports:<value>\n\nconfig, building on the configurable base image passed in in:\n\n --image <image>\n\nor config:\n\n [target]\n docker_image:<image>)\n\nBase image in this case is:\n\n '
+ shutit.target['docker_image'] + '\n\n' +
shutit_util.colourise('32', '\n[Hit return to continue]'))
shutit_util.util_raw_input()
shutit.build['docker_command'] = ' '.join(docker_command)
shutit.log('Command being run is: ' + shutit.build['docker_command'],
level=logging.DEBUG)
shutit.log('Downloading image, please be patient', level=logging.INFO)
was_sent = string.join(docker_command, ' ')
shutit_pexpect_session = shutit_pexpect.ShutItPexpectSession(
shutit_session_name, docker_command[0], docker_command[1:])
target_child = shutit_pexpect_session.pexpect_child
expect = [
'assword', shutit.expect_prompts['base_prompt'].strip(), 'Waiting',
'ulling', 'endpoint', 'Download', 'o such file'
]
res = shutit_pexpect_session.expect(expect, timeout=9999)
while True:
try:
shutit.log(target_child.before + target_child.after,
level=loglevel)
except:
pass
if res == 0:
res = shutit.send(shutit.host['password'],
shutit_pexpect_child=target_child,
expect=expect,
timeout=9999,
check_exit=False,
fail_on_empty_before=False,
echo=False,
loglevel=loglevel)
elif res == 1:
shutit.log('Prompt found, breaking out', level=logging.DEBUG)
break
elif res == 6:
shutit.fail(
'Docker not installed. Is this a mac? If so, install Docker Toolbox - see https://docker.com'
)
break
else:
res = shutit_pexpect_session.expect(expect, timeout=9999)
continue
# Did the pull work?
if not shutit_pexpect_session.check_last_exit_values(was_sent):
shutit_global.shutit.pause_point(
'Command:\n\n' + was_sent +
'\n\nfailed, you have a shell to try rectifying the problem before continuing.'
)
# Get the cid
while True:
try:
cid = open(shutit.build['cidfile']).read()
break
except Exception:
time.sleep(1)
if cid == '' or re.match('^[a-z0-9]+$', cid) == None:
shutit.fail(
'Could not get container_id - quitting. Check whether other containers may be clashing on port allocation or name.\nYou might want to try running: sudo docker kill '
+ shutit.target['name'] + '; sudo docker rm ' +
shutit.target['name'] + '\nto resolve a name clash or: ' +
shutit.host['docker_executable'] + ' ps -a | grep ' +
shutit.target['ports'] + " | awk '{print $1}' | " + 'xargs ' +
shutit.host['docker_executable'] + ' kill\nto ' +
'resolve a port clash\n')
shutit.log('cid: ' + cid, level=logging.DEBUG)
shutit.target['container_id'] = cid
return target_child
def do_interactive_modules(shutit):
cfg = shutit.cfg
errs = []
while True:
shutit_util.list_modules(shutit,
long_output=False,
sort_order='run_order')
# Which module do you want to toggle?
module_id = shutit_util.util_raw_input(
shutit,
prompt=
'Which module id do you want to toggle?\n(just hit return to continue with build)\n(you can enter a substring if it is uniquely matching)\n'
)
if module_id:
try:
_ = cfg[module_id]
except NameError:
matched_to = []
for m in cfg.keys():
if re.match('.*' + module_id + '.*', m):
matched_to.append(m)
if len(matched_to) > 1:
print(
'Please input a uniquely matchable module id. Matches were: '
+ str(matched_to))
continue
elif len(matched_to) == 0:
print('Please input a valid module id')
else:
module_id = matched_to[0]
cfg[module_id]['shutit.core.module.build'] = not cfg[module_id][
'shutit.core.module.build']
if not shutit_util.config_collection_for_built(shutit,
throw_error=False):
cfg[module_id]['shutit.core.module.build'] = not cfg[
module_id]['shutit.core.module.build']
shutit_util.util_raw_input(shutit,
prompt='Hit return to continue.\n')
continue
# If true, set up config for that module
if cfg[module_id]['shutit.core.module.build']:
# TODO: does this catch all the ones switched on? Once done, get configs for all those.
newcfg_list = []
while True:
print(
shutit_util.print_config(shutit,
cfg,
module_id=module_id))
name = shutit_util.util_raw_input(
shutit,
prompt=
'Above is the config for that module. Hit return to continue, or a config item you want to update.\n'
)
if name:
doing_list = False
while True:
if doing_list:
val_type = shutit_util.util_raw_input(
shutit,
prompt=
'Input the type for the next list item: b(oolean), s(tring).\n'
)
if val_type not in ('b', 's', ''):
continue
else:
val_type = shutit_util.util_raw_input(
shutit,
prompt=
'Input the type for that config item: b(oolean), s(tring), l(ist).\n'
)
if val_type not in ('b', 's', 'l', ''):
continue
if val_type == 's':
val = shutit_util.util_raw_input(
shutit,
prompt=
'Input the value new for that config item.\n'
)
if doing_list:
newcfg_list.append(val)
else:
break
elif val_type == 'b':
val = shutit_util.util_raw_input(
shutit,
prompt=
'Input the value new for the boolean (t/f).\n'
)
if doing_list:
if val == 't':
newcfg_list.append(True)
elif val == 'f':
newcfg_list.append(False)
else:
print('Input t or f please')
continue
else:
break
elif val_type == 'l':
doing_list = True
newcfg_list = []
elif val_type == '':
break
# TODO: handle blank/None
if doing_list:
cfg[module_id][name] = newcfg_list
else:
cfg[module_id][name] = val
else:
break
else:
pass
# TODO: if removing, get any that depend on it, and remove those too
else:
break
return errs
def setup_vagrant_pattern(shutit,
skel_path,
skel_delivery,
skel_domain,
skel_module_name,
skel_shutitfiles,
skel_domain_hash,
skel_depends,
skel_vagrant_num_machines,
skel_vagrant_machine_prefix,
skel_vagrant_ssh_access,
skel_vagrant_docker,
skel_vagrant_snapshot,
skel_vagrant_upload,
skel_vagrant_image_name):
################################################################################
# BEGIN ARGUMENT HANDLING
################################################################################
# Gather requirements for multinode vagrant setup:
options = []
if skel_vagrant_num_machines is None:
options.append({'name':'num_machines','question':'How many machines do you want?','value':'3','ok_values':[]})
else:
num_machines = skel_vagrant_num_machines
if skel_vagrant_machine_prefix is None:
options.append({'name':'machine_prefix','question':'What do you want to call the machines (eg superserver)?','value':'machine','ok_values':[]})
else:
machine_prefix = skel_vagrant_machine_prefix
if skel_vagrant_ssh_access is None:
options.append({'name':'ssh_access','question':'Do you want to have open ssh access between machines (yes or no)?','value':'yes','ok_values':['yes','no']})
else:
ssh_access = skel_vagrant_ssh_access
if skel_vagrant_docker is None:
options.append({'name':'docker','question':'Do you want Docker on the machine (yes or no)?','value':'no','ok_values':['yes','no']})
else:
docker = skel_vagrant_docker
if skel_vagrant_snapshot is None:
options.append({'name':'snapshot','question':'Do you want to snapshot the machine on completion (yes or no)?','value':'no','ok_values':['yes','no']})
else:
snapshot = skel_vagrant_snapshot
if skel_vagrant_upload is None:
options.append({'name':'upload','question':'Do you want to upload the snapshot on completion (yes or no)?','value':'no','ok_values':['yes','no']})
else:
upload = skel_vagrant_upload
if skel_vagrant_image_name is None:
options.append({'name':'image_name','question':'What base vagrant image you want?','value':'ubuntu/xenial64','ok_values':[]})
else:
image_name = skel_vagrant_image_name
options.append({'name':'sudo_password','question':'Input sudo password to save time (will be saved in readonly-by-you file)','value':'','ok_values':[]})
if options:
while True:
table = texttable.Texttable()
count = 1
rows = [['No.','Option','Current value']]
for opt in options:
rows.append([str(count),opt['question'],opt['value']])
count += 1
#table.set_deco(texttable.Texttable.HEADER)
table.set_cols_dtype(['i','a','a'])
table.set_cols_align(['r', "l", "r"])
table.add_rows(rows)
shutit.shutit_global_object.shutit_print(table.draw() + '\n')
readline.set_startup_hook(lambda: readline.insert_text(''))
for choice_li in ('5','4','3','2','1'):
readline.add_history(choice_li)
choice = shutit_util.util_raw_input(prompt='''
Choose an item to change if you want to change the default.
Input 'c' to continue to the build.
If you want to change a config, choose the number: ''')
readline.set_startup_hook()
if choice in ('c','1','2','3','4','5','6','7'):
if choice == 'c':
break
try:
choice = int(choice)
except ValueError:
shutit.shutit_global_object.shutit_print('Bad value, ignoring')
continue
else:
shutit.shutit_global_object.shutit_print('Bad value, ignoring')
continue
# Print out the actual choice....
shutit.shutit_global_object.shutit_print(rows[choice][1])
# off by one
choice -= 1
item = options[choice]
value = shutit_util.get_input(msg='Input the value: ',color=None)
if item['ok_values'] and value not in item['ok_values']:
shutit.shutit_global_object.shutit_print('Bad value, ignoring')
continue
item['value'] = value
for opt in options:
if opt['name'] == 'num_machines':
num_machines = int(opt['value'])
if opt['name'] == 'machine_prefix':
machine_prefix = opt['value']
if opt['name'] == 'ssh_access':
if opt['value'] == 'no':
ssh_access = False
elif opt['value'] == 'yes':
ssh_access = True
else:
shutit.fail('Bad value for ssh_access')
if opt['name'] == 'docker':
if opt['value'] == 'no':
docker = False
elif opt['value'] == 'yes':
docker = True
else:
shutit.fail('Bad value for docker')
if opt['name'] == 'snapshot':
if opt['value'] == 'no':
snapshot = False
elif opt['value'] == 'yes':
snapshot = True
else:
shutit.fail('Bad value for snapshot')
if opt['name'] == 'upload':
if opt['value'] == 'no':
upload = False
elif opt['value'] == 'yes':
upload = True
else:
shutit.fail('Bad value for upload')
if opt['name'] == 'image_name':
image_name = opt['value']
if opt['name'] == 'sudo_password':
sudo_password = opt['value']
num_machines = int(num_machines)
################################################################################
# END ARGUMENT HANDLING
################################################################################
################################################################################
# BEGIN TEXT GOBBETS
################################################################################
# Set up Vagrantfile data for the later
machine_stanzas = ''
vagrant_up_section = ''
machines_update = '''
# machines is a dict of dicts containing information about each machine for you to use.
machines = {}'''
for m in range(1,num_machines+1):
machine_name = machine_prefix + str(m)
machine_fqdn = machine_name + '.vagrant.test'
# vagrant_image is calculated within the code later
machine_stanzas += ('''
config.vm.define "''' + machine_name + '''" do |''' + machine_name + '''|
''' + machine_name + """.vm.box = ''' + '"' + vagrant_image + '"' + '''
""" + machine_name + '''.vm.hostname = "''' + machine_fqdn + '''"''' +
'''
config.vm.provider :virtualbox do |vb|
vb.name = "''' + skel_module_name + '_' + str(m) + '''"
end
end''')
machines_update += """
machines.update({'""" + machine_name + """':{'fqdn':'""" + machine_fqdn + """'}})"""
vagrant_up_section += '''
try:
pw = open('secret').read().strip()
except IOError:
pw = ''
if pw == '':
shutit.log("""You can get round this manual step by creating a 'secret' with your password: '******'""",level=logging.CRITICAL)
pw = shutit.get_env_pass()
import time
time.sleep(10)'''
vagrant_dir_section_1 = """
if shutit.build['vagrant_run_dir'] is None:
shutit.build['vagrant_run_dir'] = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda:0))) + '/vagrant_run'
timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
shutit.build['module_name'] = '""" + skel_module_name + """_' + timestamp
shutit.build['this_vagrant_run_dir'] = shutit.build['vagrant_run_dir'] + '/' + timestamp
shutit.send(' command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])"""
vagrant_dir_section_n = """
shutit.send(' command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])"""
vagrant_setup = r"""
# Set up the sessions
shutit_sessions = {}
for machine in sorted(machines.keys()):
shutit_sessions.update({machine:shutit.create_session('bash')})
# Set up and validate landrush
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
shutit_session.send('cd ' + shutit.build['this_vagrant_run_dir'])
# Remove any existing landrush entry.
shutit_session.send('vagrant landrush rm ' + machines[machine]['fqdn'])
# Needs to be done serially for stability reasons.
try:
shutit_session.multisend('vagrant up --provider ' + shutit.cfg['shutit-library.virtualization.virtualization.virtualization']['virt_method'] + ' ' + machine,{'assword for':pw,'assword:':pw})
except NameError:
shutit.multisend('vagrant up ' + machine,{'assword for':pw,'assword:':pw},timeout=99999)
if shutit.send_and_get_output("vagrant status 2> /dev/null | grep -w ^" + machine + " | awk '{print $2}'") != 'running':
shutit.pause_point("machine: " + machine + " appears not to have come up cleanly")
ip = shutit.send_and_get_output('''vagrant landrush ls 2> /dev/null | grep -w ^''' + machines[machine]['fqdn'] + ''' | awk '{print $2}' ''')
machines.get(machine).update({'ip':ip})
shutit_session.login(command='vagrant ssh ' + machine)
shutit_session.login(command='sudo su - ')
shutit_session.send('sysctl -w net.ipv4.conf.all.route_localnet=1')
# Correct /etc/hosts
shutit_session.send(r'''cat <(echo $(ip -4 -o addr show scope global | grep -v 10.0.2.15 | head -1 | awk '{print $4}' | sed 's/\(.*\)\/.*/\1/') $(hostname)) <(cat /etc/hosts | grep -v $(hostname -s)) > /tmp/hosts && mv -f /tmp/hosts /etc/hosts''')
# Correct any broken ip addresses.
if shutit.send_and_get_output('''vagrant landrush ls | grep ''' + machine + ''' | grep 10.0.2.15 | wc -l''') != '0':
shutit_session.log('A 10.0.2.15 landrush ip was detected for machine: ' + machine + ', correcting.',level=logging.WARNING)
# This beaut gets all the eth0 addresses from the machine and picks the first one that it not 10.0.2.15.
while True:
ipaddr = shutit_session.send_and_get_output(r'''ip -4 -o addr show scope global | grep -v 10.0.2.15 | head -1 | awk '{print $4}' | sed 's/\(.*\)\/.*/\1/' ''')
if ipaddr[0] not in ('1','2','3','4','5','6','7','8','9'):
time.sleep(10)
else:
break
# Send this on the host (shutit, not shutit_session)
shutit.send('vagrant landrush set ' + machines[machine]['fqdn'] + ' ' + ipaddr)
# Check that the landrush entry is there.
shutit.send('vagrant landrush ls | grep -w ' + machines[machine]['fqdn'])
# Gather landrush info
for machine in sorted(machines.keys()):
ip = shutit.send_and_get_output('''vagrant landrush ls 2> /dev/null | grep -w ^''' + machines[machine]['fqdn'] + ''' | awk '{print $2}' ''')
machines.get(machine).update({'ip':ip})"""
if ssh_access:
copy_keys_code = '''
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
root_password = '******'
shutit_session.install('net-tools') # netstat needed
if not shutit_session.command_available('host'):
shutit_session.install('bind-utils') # host needed
shutit_session.multisend('passwd',{'assword:':root_password})
shutit_session.send("""sed -i 's/.*PermitRootLogin.*/PermitRootLogin yes/g' /etc/ssh/sshd_config""")
shutit_session.send("""sed -i 's/.*PasswordAuthentication.*/PasswordAuthentication yes/g' /etc/ssh/sshd_config""")
shutit_session.send('service ssh restart || systemctl restart sshd')
shutit_session.multisend('ssh-keygen',{'Enter':'','verwrite':'n'})
for machine in sorted(machines.keys()):
for copy_to_machine in machines:
for item in ('fqdn','ip'):
shutit_session.multisend('ssh-copy-id root@' + machines[copy_to_machine][item],{'assword:':root_password,'ontinue conn':'yes'})'''
else:
copy_keys_code = ''
if docker:
docker_code = '''
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
# Workaround for docker networking issues + landrush.
shutit_session.install('docker')
shutit_session.insert_text('Environment=GODEBUG=netdns=cgo','/lib/systemd/system/docker.service',pattern='.Service.')
shutit_session.send('mkdir -p /etc/docker',note='Create the docker config folder')
shutit_session.send_file('/etc/docker/daemon.json',"""{
"dns": ["8.8.8.8"]
}""",note='Use the google dns server rather than the vagrant one. Change to the value you want if this does not work, eg if google dns is blocked.')
shutit_session.send('systemctl daemon-reload && systemctl restart docker')'''
else:
docker_code = ''
user_code = '''
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
shutit_session.run_script(r\'\'\'#!/bin/sh
# See https://raw.githubusercontent.com/ianmiell/vagrant-swapfile/master/vagrant-swapfile.sh
fallocate -l \'\'\' + shutit.cfg[self.module_id]['swapsize'] + r\'\'\' /swapfile
ls -lh /swapfile
chown root:root /swapfile
chmod 0600 /swapfile
ls -lh /swapfile
mkswap /swapfile
swapon /swapfile
swapon -s
grep -i --color swap /proc/meminfo
echo "\n/swapfile none swap sw 0 0" >> /etc/fstab\'\'\')
shutit_session.multisend('adduser person',{'Enter new UNIX password':'******','Retype new UNIX password:'******'person','Full Name':'','Phone':'','Room':'','Other':'','Is the information correct':'Y'})'''
machine_seed_code = '''
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
shutit_session.send('hostname')'''
if snapshot:
# TODO: add 'copy to snapshot folder function'
# TODO: create snapshot subfolder
snapshot_code = '''
for machine in sorted(machines.keys()):
shutit.send('vagrant snapshot save ' + machine,note='Snapshot the vagrant machine')'''
else:
snapshot_code = ''
if upload:
upload_code = '''
# Create a stable box name for this particular build
boxname_base = shutit.build['module_name'] + '_' + str(int(time.time()))
for machine in sorted(machines.keys()):
boxname = boxname_base + '_' + machine + '.box'
shutit.send('vagrant package --output ' + boxname + ' --vagrantfile Vagrantfile ' + machine,note='Package the vagrant machine')
shutit.send('mvn deploy:deploy-file -DgroupId=com.meirionconsulting -DartifactId=' + boxname + ' -Dversion=0.0.0.1 -Dpackaging=tar.gz -DrepositoryId=nexus.meirionconsulting.com -Durl=http://nexus.meirionconsulting.com/repository/maven-releases -Dfile=' + boxname,note='Push the vagrant box')'''
else:
upload_code = ''
# TODO get this info out
# print("""
#
# shutit.log('''********************************************************************************
#
## Vagrantfile created in: ''' + shutit.build['vagrant_run_dir'] + '''
## Run:
#
#cd ''' + shutit.build['vagrant_run_dir'] + ''' && vagrant status && vagrant landrush ls
#
## to get information about your machines' setup.
#
#********************************************************************************''')""")
#
# if snapshot:
# print("""
# shutit.log('''********************************************************************************
#
#Your VM images have been snapshotted in the folder ''' + shutit.build['vagrant_run_dir'] + '''
#
#********************************************************************************
#''')""")
get_config_section = """
def get_config(self, shutit):
shutit.get_config(self.module_id,'vagrant_image',default='""" + image_name + """')
shutit.get_config(self.module_id,'vagrant_provider',default='virtualbox')
shutit.get_config(self.module_id,'gui',default='false')
shutit.get_config(self.module_id,'memory',default='1024')
shutit.get_config(self.module_id,'swapsize',default='2G')
return True"""
header = '# Generated by shutit skeleton\n'
shared_imports = '''import random
import datetime
import logging
import string
import os
import inspect'''
################################################################################
# END TEXT GOBBETS
################################################################################
################################################################################
# FILE SETUP BEGIN
################################################################################
# Set up files:
# .gitignore
gitignore_filename = skel_path + '/.gitignore'
gitignore_file = open(gitignore_filename,'w+')
gitignore_file.write('''*pyc
vagrant_run
secret''')
gitignore_file.close()
os.chmod(gitignore_filename,0o700)
secretfile_filename = skel_path + '/secret'
secretfile_file = open(secretfile_filename,'w+')
secretfile_file.write(sudo_password)
secretfile_file.close()
os.chmod(secretfile_filename,0o400)
# README.md
readme_filename = skel_path + '/README.md'
readme_file = open(readme_filename,'w+')
readme_file.write('''
## Install
- virtualbox
- vagrant
- git
- python-pip
## Run
```
git clone --recursive [this repo]
cd [this repo file]
./run.sh
```
''')
readme_file.close()
os.chmod(readme_filename,0o700)
# run.sh
runsh_filename = skel_path + '/run.sh'
runsh_file = open(runsh_filename,'w+')
runsh_file.write('''#!/bin/bash
set -e
[[ -z "$SHUTIT" ]] && SHUTIT="$1/shutit"
[[ ! -a "$SHUTIT" ]] || [[ -z "$SHUTIT" ]] && SHUTIT="$(which shutit)"
if [[ ! -a "$SHUTIT" ]]
then
echo "Must have shutit on path, eg export PATH=$PATH:/path/to/shutit_dir"
exit 1
fi
./destroy_vms.sh
$SHUTIT build --echo -d bash -m shutit-library/vagrant -m shutit-library/virtualization -l debug "$@"
if [[ $? != 0 ]]
then
exit 1
fi''')
runsh_file.close()
os.chmod(runsh_filename,0o755)
# destroy_vms.sh
destroyvmssh_filename = skel_path + '/destroy_vms.sh'
destroyvmssh_file = open(destroyvmssh_filename,'w+')
destroyvmssh_file_contents = '''#!/bin/bash'''
if snapshot:
destroyvmssh_file_contents += '''
FOLDER=$( ls $( cd $( dirname "${BASH_SOURCE[0]}" ) && pwd )/vagrant_run 2> /dev/null)
ANSWER='y'
if [[ $FOLDER != '' ]]
then
echo "This is snapshotted - sure you want to continue deleting? (y/n)"
echo See folder: vagrant_run/${FOLDER}
read ANSWER
fi
if [[ ${ANSWER} != 'y' ]]
then
echo Refusing to continue
exit 1
fi'''
destroyvmssh_file_contents += '''
MODULE_NAME=''' + skel_module_name + '''
rm -rf $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/vagrant_run/*
XARGS_FLAG='--no-run-if-empty'
if ! echo '' | xargs --no-run-if-empty >/dev/null 2>&1
then
XARGS_FLAG=''
fi
if [[ $(command -v VBoxManage) != '' ]]
then
while true
do
VBoxManage list runningvms | grep ${MODULE_NAME} | awk '{print $1}' | xargs $XARGS_FLAG -IXXX VBoxManage controlvm 'XXX' poweroff && VBoxManage list vms | grep ''' + skel_module_name + ''' | awk '{print $1}' | xargs -IXXX VBoxManage unregistervm 'XXX' --delete
# The xargs removes whitespace
if [[ $(VBoxManage list vms | grep ${MODULE_NAME} | wc -l | xargs) -eq '0' ]]
then
break
else
ps -ef | grep virtualbox | grep ${MODULE_NAME} | awk '{print $2}' | xargs kill
sleep 10
fi
done
fi
if [[ $(command -v virsh) ]] && [[ $(kvm-ok 2>&1 | command grep 'can be used') != '' ]]
then
if [[ $(id -u) != '0' ]]
then
echo If using kvm, then you may need to be root or give perms to this user to destroy the pre-existing machines
fi
virsh list | grep ${MODULE_NAME} | awk '{print $1}' | xargs $XARGS_FLAG -n1 virsh destroy
fi'''
destroyvmssh_file.write(destroyvmssh_file_contents)
destroyvmssh_file.close()
os.chmod(destroyvmssh_filename,0o755)
# build.cnf
os.system('mkdir -p ' + skel_path + '/configs')
# git setup
os.system('git init')
os.system('git submodule init')
os.system('git submodule add https://github.com/ianmiell/shutit-library')
os.system('git submodule update')
# User message
log_message = '''
# Run:
cd ''' + skel_path + ''' && ./run.sh
# to run.
'''
if upload:
log_message += r'''
As you have chosen to upload, you may want to install maven and set your
~/.m2/settings.xml file to contain these settings:
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
https://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository/>
<interactiveMode/>
<usePluginRegistry/>
<offline/>
<pluginGroups/>
<servers>
<server>
<id>nexus.meirionconsulting.com</id>
<username>uploader</username>
<password>uploader</password>
</server>
</servers>
<mirrors/>
<proxies/>
<profiles/>
<activeProfiles/>
</settings>
so you can upload vagrant boxes.
'''
shutit.log(log_message,transient=True)
################################################################################
# FILE SETUP END
################################################################################
################################################################################
# BEGIN MODULE SETUP
################################################################################
################################################################################
# BEGIN SHUTITFILE HANDLING
################################################################################
# Handle shutitfiles. If there are no shutitfiles, handle separately.
# If there are more than one, you need to treat the first one differently.
if skel_shutitfiles:
_total = len(skel_shutitfiles)
_count = 0
for skel_shutitfile in skel_shutitfiles:
_count += 1
module_modifier = '_' + str(_count)
new_module_filename = skel_path + '/' + os.path.join(skel_module_name + module_modifier + '.py')
shutit.cfg['skeleton']['module_modifier'] = module_modifier
(sections, skel_module_id, skel_module_name, _, _) = shutitfile.shutitfile_to_shutit_module(shutit, skel_shutitfile,skel_path,skel_domain,skel_module_name,skel_domain_hash,skel_delivery,skel_depends,_count,_total,module_modifier)
shutit.cfg['skeleton']['header_section'] = sections['header_section']
shutit.cfg['skeleton']['config_section'] = sections['config_section']
shutit.cfg['skeleton']['build_section'] = sections['build_section']
shutit.cfg['skeleton']['finalize_section'] = sections['finalize_section']
shutit.cfg['skeleton']['test_section'] = sections['test_section']
shutit.cfg['skeleton']['isinstalled_section'] = sections['isinstalled_section']
shutit.cfg['skeleton']['start_section'] = sections['start_section']
shutit.cfg['skeleton']['stop_section'] = sections['stop_section']
shutit.cfg['skeleton']['final_section'] = sections['final_section']
module_file = open(new_module_filename,'w+')
# We only write out the heavy stuff for vagrant on the first time round
if _count == 1:
module_file.write(header + shared_imports + """
""" + shutit.cfg['skeleton']['header_section'] + """
def build(self, shutit):
vagrant_image = shutit.cfg[self.module_id]['vagrant_image']
vagrant_provider = shutit.cfg[self.module_id]['vagrant_provider']
gui = shutit.cfg[self.module_id]['gui']
memory = shutit.cfg[self.module_id]['memory']
""" + vagrant_dir_section_1 + """
if shutit.send_and_get_output('vagrant plugin list | grep landrush') == '':
shutit.send('vagrant plugin install landrush')
shutit.send('vagrant init ' + vagrant_image)
shutit.send_file(shutit.build['this_vagrant_run_dir'] + '/Vagrantfile','''Vagrant.configure("2") do |config|
config.landrush.enabled = true
config.vm.provider "virtualbox" do |vb|
vb.gui = ''' + gui + '''
vb.memory = "''' + memory + '''"
end
""" + machine_stanzas + """
end''')
""" + machines_update + """
""" + vagrant_up_section + """
""" + vagrant_setup + """
""" + copy_keys_code + """
""" + docker_code + """
""" + user_code + """
""" + shutit.cfg['skeleton']['build_section'] + """
""" + snapshot_code + """
""" + upload_code + """
return True
""" + get_config_section + """
""" + shutit.cfg['skeleton']['config_section'] + """ return True
def test(self, shutit):
""" + shutit.cfg['skeleton']['test_section'] + """ return True
def finalize(self, shutit):
""" + shutit.cfg['skeleton']['finalize_section'] + """ return True
def is_installed(self, shutit):
""" + shutit.cfg['skeleton']['isinstalled_section'] + """
return False
def start(self, shutit):
""" + shutit.cfg['skeleton']['start_section'] + """ return True
def stop(self, shutit):
""" + shutit.cfg['skeleton']['stop_section'] + """ return True
def module():
return """ + skel_module_name + module_modifier + """(
'""" + skel_module_id + """', """ + skel_domain_hash + """.000""" + str(_count) + """,
description='',
maintainer='',
delivery_methods=['bash'],
depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant']
)""")
# In the non-first one, we don't have all the setup stuff (but we do have some!)
else:
module_file.write(header + shared_imports + """
""" + shutit.cfg['skeleton']['header_section'] + """
def build(self, shutit):
""" + vagrant_dir_section_n + """
""" + shutit.cfg['skeleton']['build_section'] + """
""" + shutit.cfg['skeleton']['config_section'] + """ return True
def test(self, shutit):
""" + shutit.cfg['skeleton']['test_section'] + """ return True
def finalize(self, shutit):
""" + shutit.cfg['skeleton']['finalize_section'] + """ return True
def is_installed(self, shutit):
""" + shutit.cfg['skeleton']['isinstalled_section'] + """
return False
def start(self, shutit):
""" + shutit.cfg['skeleton']['start_section'] + """ return True
def stop(self, shutit):
""" + shutit.cfg['skeleton']['stop_section'] + """ return True
def module():
return """ + skel_module_name + module_modifier + """(
'""" + skel_module_id + """',""" + skel_domain_hash + """.000""" + str(_count) + """,
description='',
maintainer='',
delivery_methods=['bash'],
depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant']
)""")
module_file.close()
# Set up build.cnf
build_cnf_filename = skel_path + '/configs/build.cnf'
if _count == 1:
build_cnf_file = open(build_cnf_filename,'w+')
build_cnf_file.write('''###############################################################################
# PLEASE NOTE: This file should be changed only by the maintainer.
# PLEASE NOTE: This file is only sourced if the "shutit build" command is run
# and this file is in the relative path: configs/build.cnf
# This is to ensure it is only sourced if _this_ module is the
# target.
###############################################################################
# When this module is the one being built, which modules should be built along with it by default?
# This feeds into automated testing of each module.
[''' + skel_module_id + ''']
shutit.core.module.build:yes''')
build_cnf_file.close()
else:
build_cnf_file = open(build_cnf_filename,'a')
build_cnf_file.write('''
[''' + skel_domain + '''.''' + skel_module_name + module_modifier + ''']
shutit.core.module.build:yes''')
build_cnf_file.close()
os.chmod(build_cnf_filename,0o400)
################################################################################
# END SHUTITFILE HANDLING
################################################################################
else:
# No shutitfiles to consider, so simpler logic here.
shutit.cfg['skeleton']['header_section'] = 'from shutit_module import ShutItModule\n\nclass ' + skel_module_name + '(ShutItModule):\n'
new_module_filename = skel_path + '/' + skel_module_name + '.py'
module_file = open(new_module_filename,'w+')
module_file.write(header + shared_imports + """
""" + shutit.cfg['skeleton']['header_section'] + """
def build(self, shutit):
vagrant_image = shutit.cfg[self.module_id]['vagrant_image']
vagrant_provider = shutit.cfg[self.module_id]['vagrant_provider']
gui = shutit.cfg[self.module_id]['gui']
memory = shutit.cfg[self.module_id]['memory']
shutit.build['vagrant_run_dir'] = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda:0))) + '/vagrant_run'
shutit.build['module_name'] = '""" + skel_module_name + """_' + ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6))
shutit.build['this_vagrant_run_dir'] = shutit.build['vagrant_run_dir'] + '/' + shutit.build['module_name']
shutit.send(' command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])
shutit.send('command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])
if shutit.send_and_get_output('vagrant plugin list | grep landrush') == '':
shutit.send('vagrant plugin install landrush')
shutit.send('vagrant init ' + vagrant_image)
shutit.send_file(shutit.build['this_vagrant_run_dir'] + '/Vagrantfile','''Vagrant.configure("2") do |config|
config.landrush.enabled = true
config.vm.provider "virtualbox" do |vb|
vb.gui = ''' + gui + '''
vb.memory = "''' + memory + '''"
end
""" + machine_stanzas + """
end''')
""" + machines_update + """
""" + vagrant_up_section + """
""" + vagrant_setup + """
""" + copy_keys_code + """
""" + docker_code + """
""" + user_code + """
""" + machine_seed_code + """
""" + snapshot_code + """
""" + upload_code + """
return True
""" + get_config_section + """
def test(self, shutit):
return True
def finalize(self, shutit):
return True
def is_installed(self, shutit):
return False
def start(self, shutit):
return True
def stop(self, shutit):
return True
def module():
return """ + skel_module_name + """(
'""" + skel_domain + '''.''' + skel_module_name + """', """ + skel_domain_hash + """.0001,
description='',
maintainer='',
delivery_methods=['bash'],
depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant']
)""")
module_file.close()
build_cnf_filename = skel_path + '/configs/build.cnf'
build_cnf_file = open(build_cnf_filename,'w+')
build_cnf_file.write('''###############################################################################
# PLEASE NOTE: This file should be changed only by the maintainer.
# PLEASE NOTE: This file is only sourced if the "shutit build" command is run
# and this file is in the relative path: configs/build.cnf
# This is to ensure it is only sourced if _this_ module is the
# target.
###############################################################################
# When this module is the one being built, which modules should be built along with it by default?
# This feeds into automated testing of each module.
['''+skel_domain+'''.'''+skel_module_name+''']
shutit.core.module.build:yes''')
build_cnf_file.close()
os.chmod(build_cnf_filename,0o400)
def do_interactive_modules():
shutit = shutit_global.shutit
cfg = shutit.cfg
errs = []
while True:
shutit_util.list_modules(long_output=False,sort_order='run_order')
# Which module do you want to toggle?
module_id = shutit_util.util_raw_input(prompt='Which module id do you want to toggle?\n(just hit return to continue with build)\n(you can enter a substring if it is uniquely matching)\n')
if module_id:
try:
_=cfg[module_id]
except:
matched_to = []
for m in cfg.keys():
if re.match('.*'+module_id+'.*',m):
matched_to.append(m)
if len(matched_to) > 1:
print 'Please input a uniquely matchable module id. Matches were: ' + str(matched_to)
continue
elif len(matched_to) == 0:
print 'Please input a valid module id'
else:
module_id = matched_to[0]
cfg[module_id]['shutit.core.module.build'] = not cfg[module_id]['shutit.core.module.build']
if not shutit_util.config_collection_for_built(throw_error=False):
cfg[module_id]['shutit.core.module.build'] = not cfg[module_id]['shutit.core.module.build']
shutit_util.util_raw_input(prompt='Hit return to continue.\n')
continue
# If true, set up config for that module
if cfg[module_id]['shutit.core.module.build']:
# TODO: does this catch all the ones switched on? Once done, get configs for all those.
newcfg_list = []
while True:
print shutit_util.print_config(cfg,module_id=module_id)
name = shutit_util.util_raw_input(prompt='Above is the config for that module. Hit return to continue, or a config item you want to update.\n')
if name:
doing_list = False
while True:
if doing_list:
val_type = shutit_util.util_raw_input(prompt='Input the type for the next list item: b(oolean), s(tring).\n')
if val_type not in ('b','s',''):
continue
else:
val_type = shutit_util.util_raw_input(prompt='Input the type for that config item: b(oolean), s(tring), l(ist).\n')
if val_type not in ('b','s','l',''):
continue
if val_type == 's':
val = shutit_util.util_raw_input(prompt='Input the value new for that config item.\n')
if doing_list:
newcfg_list.append(val)
else:
break
elif val_type == 'b':
val = shutit_util.util_raw_input(prompt='Input the value new for the boolean (t/f).\n')
if doing_list:
if val == 't':
newcfg_list.append(True)
elif val == 'f':
newcfg_list.append(False)
else:
print 'Input t or f please'
continue
else:
break
elif val_type == 'l':
doing_list = True
newcfg_list = []
elif val_type == '':
break
# TODO: handle blank/None
if doing_list:
cfg[module_id][name] = newcfg_list
else:
cfg[module_id][name] = val
else:
break
else:
pass
# TODO: if removing, get any that depend on it, and remove those too
else:
break
return errs
def start_container(self, shutit, shutit_session_name, loglevel=logging.DEBUG):
cfg = shutit.cfg
docker = cfg['host']['docker_executable'].split(' ')
# Always-required options
if not os.path.exists(cfg['build']['shutit_state_dir'] + '/cidfiles'):
os.makedirs(cfg['build']['shutit_state_dir'] + '/cidfiles')
cfg['build']['cidfile'] = cfg['build']['shutit_state_dir'] + '/cidfiles/' + cfg['host']['username'] + '_cidfile_' + cfg['build']['build_id']
cidfile_arg = '--cidfile=' + cfg['build']['cidfile']
# Singly-specified options
privileged_arg = ''
name_arg = ''
hostname_arg = ''
rm_arg = ''
net_arg = ''
mount_docker_arg = ''
shell_arg = '/bin/bash'
if cfg['build']['privileged']:
privileged_arg = '--privileged=true'
if cfg['target']['name'] != '':
name_arg = '--name=' + cfg['target']['name']
if cfg['target']['hostname'] != '':
hostname_arg = '-h=' + cfg['target']['hostname']
if cfg['build']['net'] != '':
net_arg = '--net="' + cfg['build']['net'] + '"'
if cfg['build']['mount_docker']:
mount_docker_arg = '-v=/var/run/docker.sock:/var/run/docker.sock'
# Incompatible with do_repository_work
if cfg['target']['rm']:
rm_arg = '--rm=true'
if cfg['build']['base_image'] in ('alpine','busybox'):
shell_arg = '/bin/ash'
# Multiply-specified options
port_args = []
dns_args = []
volume_args = []
volumes_from_args = []
volumes_list = cfg['target']['volumes'].strip().split()
volumes_from_list = cfg['target']['volumes_from'].strip().split()
ports_list = cfg['target']['ports'].strip().split()
dns_list = cfg['host']['dns'].strip().split()
for portmap in ports_list:
port_args.append('-p=' + portmap)
for dns in dns_list:
dns_args.append('--dns=' + dns)
for volume in volumes_list:
volume_args.append('-v=' + volume)
for volumes_from in volumes_from_list:
volumes_from_args.append('--volumes-from=' + volumes_from)
docker_command = docker + [
arg for arg in [
'run',
cidfile_arg,
privileged_arg,
name_arg,
hostname_arg,
rm_arg,
net_arg,
mount_docker_arg,
] + volume_args + volumes_from_args + port_args + dns_args + [
'-t',
'-i',
cfg['target']['docker_image'],
shell_arg
] if arg != ''
]
if cfg['build']['interactive'] >= 3:
print('\n\nAbout to start container. Ports mapped will be: ' + ', '.join(port_args) + '\n\n[host]\nports:<value>\n\nconfig, building on the configurable base image passed in in:\n\n --image <image>\n\nor config:\n\n [target]\n docker_image:<image>)\n\nBase image in this case is:\n\n ' + cfg['target']['docker_image'] + '\n\n' + shutit_util.colour('32', '\n[Hit return to continue]'))
shutit_util.util_raw_input(shutit=shutit)
cfg['build']['docker_command'] = ' '.join(docker_command)
shutit.log('Command being run is: ' + cfg['build']['docker_command'],level=logging.DEBUG)
shutit.log('Downloading image, please be patient',level=logging.INFO)
shutit_pexpect_session = shutit_pexpect.ShutItPexpectSession(shutit_session_name, docker_command[0], docker_command[1:])
target_child = shutit_pexpect_session.pexpect_child
expect = ['assword', cfg['expect_prompts']['base_prompt'].strip(), 'Waiting', 'ulling', 'endpoint', 'Download']
res = shutit_pexpect_session.expect(expect, timeout=9999)
while True:
shutit.log(target_child.before + target_child.after,level=loglevel)
if res == 0:
res = shutit.send(cfg['host']['password'], child=target_child, expect=expect, timeout=9999, check_exit=False, fail_on_empty_before=False, echo=False, loglevel=loglevel)
elif res == 1:
shutit.log('Prompt found, breaking out',level=logging.DEBUG)
break
else:
res = shutit_pexpect_session.expect(expect, timeout=9999)
continue
# Get the cid
while True:
try:
cid = open(cfg['build']['cidfile']).read()
break
except Exception:
time.sleep(1)
if cid == '' or re.match('^[a-z0-9]+$', cid) == None:
shutit.fail('Could not get container_id - quitting. Check whether other containers may be clashing on port allocation or name.\nYou might want to try running: sudo docker kill ' + cfg['target']['name'] + '; sudo docker rm ' + cfg['target']['name'] + '\nto resolve a name clash or: ' + cfg['host']['docker_executable'] + ' ps -a | grep ' + cfg['target']['ports'] + " | awk '{print $1}' | " + 'xargs ' + cfg['host']['docker_executable'] + ' kill\nto ' + 'resolve a port clash\n')
shutit.log('cid: ' + cid,level=logging.DEBUG)
cfg['target']['container_id'] = cid
return target_child
def build_module(module, loglevel=logging.DEBUG):
"""Build passed-in module.
"""
shutit = shutit_global.shutit
cfg = shutit.cfg
shutit.log('Building ShutIt module: ' + module.module_id +
' with run order: ' + str(module.run_order),
level=logging.INFO)
shutit.build['report'] = (shutit.build['report'] +
'\nBuilding ShutIt module: ' + module.module_id +
' with run order: ' + str(module.run_order))
if not module.build(shutit):
shutit.fail(
module.module_id + ' failed on build',
shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id(
'target_child').pexpect_child)
else:
if shutit.build['delivery'] in ('docker', 'dockerfile'):
# Create a directory and files to indicate this has been built.
shutit.send(' mkdir -p ' + shutit.build['build_db_dir'] +
'/module_record/' + module.module_id + ' && touch ' +
shutit.build['build_db_dir'] + '/module_record/' +
module.module_id + '/built && rm -f ' +
shutit.build['build_db_dir'] + '/module_record/' +
module.module_id + '/removed',
loglevel=loglevel)
# Put it into "installed" cache
shutit_global.shutit.get_current_shutit_pexpect_session_environment(
).modules_installed.append(module.module_id)
# Remove from "not installed" cache
if module.module_id in shutit_global.shutit.get_current_shutit_pexpect_session_environment(
).modules_not_installed:
shutit_global.shutit.get_current_shutit_pexpect_session_environment(
).modules_not_installed.remove(module.module_id)
shutit.pause_point('\nPausing to allow inspect of build for: ' +
module.module_id,
print_input=True,
level=2)
shutit.build['report'] = (shutit.build['report'] + '\nCompleted module: ' +
module.module_id)
if cfg[module.module_id][
'shutit.core.module.tag'] or shutit.build['interactive'] >= 3:
shutit.log(shutit_util.build_report('#Module:' + module.module_id),
level=logging.DEBUG)
if (not cfg[module.module_id]['shutit.core.module.tag']
and shutit.build['interactive'] >= 2):
print("\n\nDo you want to save state now we\'re at the " +
"end of this module? (" + module.module_id + ") (input y/n)")
cfg[module.module_id]['shutit.core.module.tag'] = (
shutit_util.util_raw_input(default='y') == 'y')
if cfg[module.
module_id]['shutit.core.module.tag'] or shutit.build['tag_modules']:
shutit.log(module.module_id +
' configured to be tagged, doing repository work',
level=logging.INFO)
# Stop all before we tag to avoid file changing errors, and clean up pid files etc..
stop_all(module.run_order)
shutit.do_repository_work(
str(module.module_id) + '_' + str(module.run_order),
password=shutit.host['password'],
docker_executable=shutit.host['docker_executable'],
force=True)
# Start all after we tag to ensure services are up as expected.
start_all(module.run_order)
if shutit.build['interactive'] >= 2:
print("\n\nDo you want to stop interactive mode? (input y/n)\n")
if shutit_util.util_raw_input(default='y') == 'y':
shutit.build['interactive'] = 0
def build(self, shutit):
"""Sets up the target ready for building.
"""
# Uncomment for testing for "failure" cases.
#sys.exit(1)
while not self._check_docker(shutit):
pass
cfg = shutit.cfg
docker = cfg['host']['docker_executable'].split(' ')
# Always-required options
if not os.path.exists(cfg['build']['shutit_state_dir'] + '/cidfiles'):
os.makedirs(cfg['build']['shutit_state_dir'] + '/cidfiles')
cfg['build']['cidfile'] = cfg['build']['shutit_state_dir'] + '/cidfiles/' + cfg['host']['username'] +\
'_cidfile_' + cfg['build']['build_id']
cidfile_arg = '--cidfile=' + cfg['build']['cidfile']
# Singly-specified options
privileged_arg = ''
lxc_conf_arg = ''
name_arg = ''
hostname_arg = ''
volume_arg = ''
rm_arg = ''
net_arg = ''
mount_docker_arg = ''
shell_arg = '/bin/bash'
if cfg['build']['privileged']:
privileged_arg = '--privileged=true'
if cfg['build']['lxc_conf'] != '':
lxc_conf_arg = '--lxc-conf=' + cfg['build']['lxc_conf']
if cfg['target']['name'] != '':
name_arg = '--name=' + cfg['target']['name']
if cfg['target']['hostname'] != '':
hostname_arg = '-h=' + cfg['target']['hostname']
if cfg['host']['artifacts_dir'] != '':
volume_arg = '-v=' + cfg['host']['artifacts_dir'] + ':/artifacts'
if cfg['build']['net'] != '':
net_arg = '--net="' + cfg['build']['net'] + '"'
if cfg['build']['mount_docker']:
mount_docker_arg = '-v=/var/run/docker.sock:/var/run/docker.sock'
# Incompatible with do_repository_work
if cfg['target']['rm']:
rm_arg = '--rm=true'
if cfg['build']['base_image'] in ('alpine'):
shell_arg = '/bin/ash'
# Multiply-specified options
port_args = []
dns_args = []
ports_list = cfg['target']['ports'].strip().split()
dns_list = cfg['host']['dns'].strip().split()
for portmap in ports_list:
port_args.append('-p=' + portmap)
for dns in dns_list:
dns_args.append('--dns=' + dns)
docker_command = docker + [
arg for arg in [
'run',
cidfile_arg,
privileged_arg,
lxc_conf_arg,
name_arg,
hostname_arg,
volume_arg,
rm_arg,
net_arg,
mount_docker_arg,
] + port_args + dns_args + [
'-t',
'-i',
cfg['target']['docker_image'],
shell_arg
] if arg != ''
]
if cfg['build']['interactive'] >= 3:
print('\n\nAbout to start container. ' +
'Ports mapped will be: ' + ', '.join(port_args) +
'\n\n[host]\nports:<value>\n\nconfig, building on the ' +
'configurable base image passed in in:\n\n --image <image>\n' +
'\nor config:\n\n [target]\n docker_image:<image>)\n\n' +
'Base image in this case is:\n\n ' +
cfg['target']['docker_image'] +
'\n\n' + shutit_util.colour('32', '\n[Hit return to continue]'))
shutit_util.util_raw_input(shutit=shutit)
cfg['build']['docker_command'] = ' '.join(docker_command)
shutit.log('\n\nCommand being run is:\n\n' + cfg['build']['docker_command'],
force_stdout=True, prefix=False)
shutit.log('\n\nThis may download the image, please be patient\n\n',
force_stdout=True, prefix=False)
target_child = pexpect.spawn(docker_command[0], docker_command[1:])
expect = ['assword', cfg['expect_prompts']['base_prompt'].strip(), \
'Waiting', 'ulling', 'endpoint', 'Download']
res = target_child.expect(expect, 9999)
while True:
shutit.log(target_child.before + target_child.after, prefix=False,
force_stdout=True)
if res == 0:
shutit.log('...')
res = shutit.send(cfg['host']['password'], \
child=target_child, expect=expect, timeout=9999, \
check_exit=False, fail_on_empty_before=False)
elif res == 1:
shutit.log('Prompt found, breaking out')
break
else:
res = target_child.expect(expect, 9999)
continue
# Get the cid
while True:
try:
cid = open(cfg['build']['cidfile']).read()
break
except:
sleep(1)
if cid == '' or re.match('^[a-z0-9]+$', cid) == None:
shutit.fail('Could not get container_id - quitting. ' +
'Check whether ' +
'other containers may be clashing on port allocation or name.' +
'\nYou might want to try running: sudo docker kill ' +
cfg['target']['name'] + '; sudo docker rm ' +
cfg['target']['name'] + '\nto resolve a name clash or: ' +
cfg['host']['docker_executable'] + ' ps -a | grep ' +
cfg['target']['ports'] + ' | awk \'{print $1}\' | ' +
'xargs ' + cfg['host']['docker_executable'] + ' kill\nto + '
'resolve a port clash\n')
shutit.log('cid: ' + cid)
cfg['target']['container_id'] = cid
self._setup_prompts(shutit, target_child)
self._add_begin_build_info(shutit, docker_command)
return True
def setup_vagrant_pattern(shutit,
skel_path,
skel_delivery,
skel_domain,
skel_module_name,
skel_shutitfiles,
skel_domain_hash,
skel_depends,
skel_vagrant_num_machines,
skel_vagrant_machine_prefix,
skel_vagrant_ssh_access,
skel_vagrant_docker,
skel_vagrant_snapshot,
skel_vagrant_upload,
skel_vagrant_image_name):
################################################################################
# BEGIN ARGUMENT HANDLING
################################################################################
# Gather requirements for multinode vagrant setup:
options = []
if skel_vagrant_num_machines is None:
options.append({'name':'num_machines','question':'How many machines do you want?','value':'3','ok_values':[]})
else:
num_machines = skel_vagrant_num_machines
if skel_vagrant_machine_prefix is None:
options.append({'name':'machine_prefix','question':'What do you want to call the machines (eg superserver)?','value':'machine','ok_values':[]})
else:
machine_prefix = skel_vagrant_machine_prefix
if skel_vagrant_ssh_access is None:
options.append({'name':'ssh_access','question':'Do you want to have open ssh access between machines (yes or no)?','value':'yes','ok_values':['yes','no']})
else:
ssh_access = skel_vagrant_ssh_access
if skel_vagrant_docker is None:
options.append({'name':'docker','question':'Do you want Docker on the machine (yes or no)?','value':'no','ok_values':['yes','no']})
else:
docker = skel_vagrant_docker
if skel_vagrant_snapshot is None:
options.append({'name':'snapshot','question':'Do you want to snapshot the machine on completion (yes or no)?','value':'no','ok_values':['yes','no']})
else:
snapshot = skel_vagrant_snapshot
if skel_vagrant_upload is None:
options.append({'name':'upload','question':'Do you want to upload the snapshot on completion (yes or no)?','value':'no','ok_values':['yes','no']})
else:
upload = skel_vagrant_upload
if skel_vagrant_image_name is None:
options.append({'name':'image_name','question':'What base vagrant image you want?','value':'ubuntu/xenial64','ok_values':[]})
else:
image_name = skel_vagrant_image_name
options.append({'name':'sudo_password','question':'Input sudo password to save time (will be saved in readonly-by-you file)','value':'','ok_values':[]})
if options:
while True:
table = texttable.Texttable()
count = 1
rows = [['No.','Option','Current value']]
for opt in options:
rows.append([str(count),opt['question'],opt['value']])
count += 1
#table.set_deco(texttable.Texttable.HEADER)
table.set_cols_dtype(['i','a','a'])
table.set_cols_align(['r', "l", "r"])
table.add_rows(rows)
shutit.shutit_global_object.shutit_print(table.draw() + '\n')
readline.set_startup_hook(lambda: readline.insert_text(''))
for choice_li in ('5','4','3','2','1'):
readline.add_history(choice_li)
choice = shutit_util.util_raw_input(prompt='''
Choose an item to change if you want to change the default.
Input 'c' to continue to the build.
If you want to change a config, choose the number: ''')
readline.set_startup_hook()
if choice in ('c','1','2','3','4','5','6','7'):
if choice == 'c':
break
try:
choice = int(choice)
except ValueError:
shutit.shutit_global_object.shutit_print('Bad value, ignoring')
continue
else:
shutit.shutit_global_object.shutit_print('Bad value, ignoring')
continue
# Print out the actual choice....
shutit.shutit_global_object.shutit_print(rows[choice][1])
# off by one
choice -= 1
item = options[choice]
value = shutit_util.get_input(msg='Input the value: ',color=None)
if item['ok_values'] and value not in item['ok_values']:
shutit.shutit_global_object.shutit_print('Bad value, ignoring')
continue
item['value'] = value
for opt in options:
if opt['name'] == 'num_machines':
num_machines = int(opt['value'])
if opt['name'] == 'machine_prefix':
machine_prefix = opt['value']
if opt['name'] == 'ssh_access':
if opt['value'] == 'no':
ssh_access = False
elif opt['value'] == 'yes':
ssh_access = True
else:
shutit.fail('Bad value for ssh_access')
if opt['name'] == 'docker':
if opt['value'] == 'no':
docker = False
elif opt['value'] == 'yes':
docker = True
else:
shutit.fail('Bad value for docker')
if opt['name'] == 'snapshot':
if opt['value'] == 'no':
snapshot = False
elif opt['value'] == 'yes':
snapshot = True
else:
shutit.fail('Bad value for snapshot')
if opt['name'] == 'upload':
if opt['value'] == 'no':
upload = False
elif opt['value'] == 'yes':
upload = True
else:
shutit.fail('Bad value for upload')
if opt['name'] == 'image_name':
image_name = opt['value']
if opt['name'] == 'sudo_password':
sudo_password = opt['value']
num_machines = int(num_machines)
################################################################################
# END ARGUMENT HANDLING
################################################################################
################################################################################
# BEGIN TEXT GOBBETS
################################################################################
# Set up Vagrantfile data for the later
machine_stanzas = ''
vagrant_up_section = ''
machines_update = '''
# machines is a dict of dicts containing information about each machine for you to use.
machines = {}'''
for m in range(1,num_machines+1):
machine_name = machine_prefix + str(m)
machine_fqdn = machine_name + '.vagrant.test'
# vagrant_image is calculated within the code later
machine_stanzas += ('''
config.vm.define "''' + machine_name + '''" do |''' + machine_name + '''|
''' + machine_name + """.vm.box = ''' + '"' + vagrant_image + '"' + '''
""" + machine_name + '''.vm.hostname = "''' + machine_fqdn + '''"''' +
'''
config.vm.provider :virtualbox do |vb|
vb.name = "''' + skel_module_name + '_' + str(m) + '''"
end
end''')
machines_update += """
machines.update({'""" + machine_name + """':{'fqdn':'""" + machine_fqdn + """'}})"""
vagrant_up_section += '''
try:
pw = open('secret').read().strip()
except IOError:
pw = ''
if pw == '':
shutit.log("""You can get round this manual step by creating a 'secret' with your password: '******'""",level=logging.CRITICAL)
pw = shutit.get_env_pass()
import time
time.sleep(10)'''
vagrant_dir_section_1 = """
if shutit.build['vagrant_run_dir'] is None:
shutit.build['vagrant_run_dir'] = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda:0))) + '/vagrant_run'
timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
shutit.build['module_name'] = '""" + skel_module_name + """_' + timestamp
shutit.build['this_vagrant_run_dir'] = shutit.build['vagrant_run_dir'] + '/' + timestamp
shutit.send(' command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])"""
vagrant_dir_section_n = """
shutit.send(' command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])"""
vagrant_setup = r"""
# Set up the sessions
shutit_sessions = {}
for machine in sorted(machines.keys()):
shutit_sessions.update({machine:shutit.create_session('bash')})
# Set up and validate landrush
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
shutit_session.send('cd ' + shutit.build['this_vagrant_run_dir'])
# Remove any existing landrush entry.
shutit_session.send('vagrant landrush rm ' + machines[machine]['fqdn'])
# Needs to be done serially for stability reasons.
try:
shutit_session.multisend('vagrant up --provider ' + shutit.cfg['shutit-library.virtualization.virtualization.virtualization']['virt_method'] + machine_name,{'assword for':pw,'assword:':pw})
except NameError:
shutit.multisend('vagrant up ' + machine,{'assword for':pw,'assword:':pw},timeout=99999)
if shutit.send_and_get_output("vagrant status 2> /dev/null | grep -w ^" + machine + " | awk '{print $2}'") != 'running':
shutit.pause_point("machine: " + machine + " appears not to have come up cleanly")
ip = shutit.send_and_get_output('''vagrant landrush ls 2> /dev/null | grep -w ^''' + machines[machine]['fqdn'] + ''' | awk '{print $2}' ''')
machines.get(machine).update({'ip':ip})
shutit_session.login(command='vagrant ssh ' + machine)
shutit_session.login(command='sudo su - ')
# Correct /etc/hosts
shutit_session.send(r'''cat <(echo $(ip -4 -o addr show scope global | grep -v 10.0.2.15 | head -1 | awk '{print $4}' | sed 's/\(.*\)\/.*/\1/') $(hostname)) <(cat /etc/hosts | grep -v $(hostname -s)) > /tmp/hosts && mv -f /tmp/hosts /etc/hosts''')
# Correct any broken ip addresses.
if shutit.send_and_get_output('''vagrant landrush ls | grep ''' + machine + ''' | grep 10.0.2.15 | wc -l''') != '0':
shutit_session.log('A 10.0.2.15 landrush ip was detected for machine: ' + machine + ', correcting.',level=logging.WARNING)
# This beaut gets all the eth0 addresses from the machine and picks the first one that it not 10.0.2.15.
while True:
ipaddr = shutit_session.send_and_get_output(r'''ip -4 -o addr show scope global | grep -v 10.0.2.15 | head -1 | awk '{print $4}' | sed 's/\(.*\)\/.*/\1/' ''')
if ipaddr[0] not in ('1','2','3','4','5','6','7','8','9'):
time.sleep(10)
else:
break
# Send this on the host (shutit, not shutit_session)
shutit.send('vagrant landrush set ' + machines[machine]['fqdn'] + ' ' + ipaddr)
# Check that the landrush entry is there.
shutit.send('vagrant landrush ls | grep -w ' + machines[machine]['fqdn'])
# Gather landrush info
for machine in sorted(machines.keys()):
ip = shutit.send_and_get_output('''vagrant landrush ls 2> /dev/null | grep -w ^''' + machines[machine]['fqdn'] + ''' | awk '{print $2}' ''')
machines.get(machine).update({'ip':ip})"""
if ssh_access:
copy_keys_code = '''
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
root_password = '******'
shutit_session.install('net-tools') # netstat needed
if not shutit_session.command_available('host'):
shutit_session.install('bind-utils') # host needed
shutit_session.multisend('passwd',{'assword:':root_password})
shutit_session.send("""sed -i 's/.*PermitRootLogin.*/PermitRootLogin yes/g' /etc/ssh/sshd_config""")
shutit_session.send("""sed -i 's/.*PasswordAuthentication.*/PasswordAuthentication yes/g' /etc/ssh/sshd_config""")
shutit_session.send('service ssh restart || systemctl restart sshd')
shutit_session.multisend('ssh-keygen',{'Enter':'','verwrite':'n'})
for machine in sorted(machines.keys()):
for copy_to_machine in machines:
for item in ('fqdn','ip'):
shutit_session.multisend('ssh-copy-id root@' + machines[copy_to_machine][item],{'assword:':root_password,'ontinue conn':'yes'})'''
else:
copy_keys_code = ''
if docker:
docker_code = '''
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
# Workaround for docker networking issues + landrush.
shutit_session.install('docker')
shutit_session.insert_text('Environment=GODEBUG=netdns=cgo','/lib/systemd/system/docker.service',pattern='.Service.')
shutit_session.send('mkdir -p /etc/docker',note='Create the docker config folder')
shutit_session.send_file('/etc/docker/daemon.json',"""{
"dns": ["8.8.8.8"]
}""",note='Use the google dns server rather than the vagrant one. Change to the value you want if this does not work, eg if google dns is blocked.')
shutit_session.send('systemctl daemon-reload && systemctl restart docker')'''
else:
docker_code = ''
user_code = '''
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
shutit_session.run_script(r\'\'\'#!/bin/sh
# See https://raw.githubusercontent.com/ianmiell/vagrant-swapfile/master/vagrant-swapfile.sh
fallocate -l \'\'\' + shutit.cfg[self.module_id]['swapsize'] + r\'\'\' /swapfile
ls -lh /swapfile
chown root:root /swapfile
chmod 0600 /swapfile
ls -lh /swapfile
mkswap /swapfile
swapon /swapfile
swapon -s
grep -i --color swap /proc/meminfo
echo "\n/swapfile none swap sw 0 0" >> /etc/fstab\'\'\')
shutit_session.multisend('adduser person',{'Enter new UNIX password':'******','Retype new UNIX password:'******'person','Full Name':'','Phone':'','Room':'','Other':'','Is the information correct':'Y'})'''
machine_seed_code = '''
for machine in sorted(machines.keys()):
shutit_session = shutit_sessions[machine]
shutit_session.send('hostname')'''
if snapshot:
# TODO: add 'copy to snapshot folder function'
# TODO: create snapshot subfolder
snapshot_code = '''
for machine in sorted(machines.keys()):
shutit.send('vagrant snapshot save ' + machine,note='Snapshot the vagrant machine')'''
else:
snapshot_code = ''
if upload:
upload_code = '''
# Create a stable box name for this particular build
boxname_base = shutit.build['module_name'] + '_' + str(int(time.time()))
for machine in sorted(machines.keys()):
boxname = boxname_base + '_' + machine + '.box'
shutit.send('vagrant package --output ' + boxname + ' --vagrantfile Vagrantfile ' + machine,note='Package the vagrant machine')
shutit.send('mvn deploy:deploy-file -DgroupId=com.meirionconsulting -DartifactId=' + boxname + ' -Dversion=0.0.0.1 -Dpackaging=tar.gz -DrepositoryId=nexus.meirionconsulting.com -Durl=http://nexus.meirionconsulting.com/repository/maven-releases -Dfile=' + boxname,note='Push the vagrant box')'''
else:
upload_code = ''
print("""
shutit.log('''********************************************************************************
# Vagrantfile created in: ''' + shutit.build['vagrant_run_dir'] + '''
# Run:
cd ''' + shutit.build['vagrant_run_dir'] + ''' && vagrant status && vagrant landrush ls
# to get information about your machines' setup.
********************************************************************************''')""")
if snapshot:
print("""
shutit.log('''********************************************************************************
Your VM images have been snapshotted in the folder ''' + shutit.build['vagrant_run_dir'] + '''
********************************************************************************
''')""")
get_config_section = """
def get_config(self, shutit):
shutit.get_config(self.module_id,'vagrant_image',default='""" + image_name + """')
shutit.get_config(self.module_id,'vagrant_provider',default='virtualbox')
shutit.get_config(self.module_id,'gui',default='false')
shutit.get_config(self.module_id,'memory',default='1024')
shutit.get_config(self.module_id,'swapsize',default='2G')
return True"""
header = '# Generated by shutit skeleton\n'
shared_imports = '''import random
import datetime
import logging
import string
import os
import inspect'''
################################################################################
# END TEXT GOBBETS
################################################################################
################################################################################
# FILE SETUP BEGIN
################################################################################
# Set up files:
# .gitignore
gitignore_filename = skel_path + '/.gitignore'
gitignore_file = open(gitignore_filename,'w+')
gitignore_file.write('''*pyc
vagrant_run
secret''')
gitignore_file.close()
os.chmod(gitignore_filename,0o700)
secretfile_filename = skel_path + '/secret'
secretfile_file = open(secretfile_filename,'w+')
secretfile_file.write(sudo_password)
secretfile_file.close()
os.chmod(secretfile_filename,0o400)
# README.md
readme_filename = skel_path + '/README.md'
readme_file = open(readme_filename,'w+')
readme_file.write('''
## Install
- virtualbox
- vagrant
- git
- python-pip
## Run
```
git clone --recursive [this repo]
cd [this repo file]
./run.sh
```
''')
readme_file.close()
os.chmod(readme_filename,0o700)
# run.sh
runsh_filename = skel_path + '/run.sh'
runsh_file = open(runsh_filename,'w+')
runsh_file.write('''#!/bin/bash
set -e
[[ -z "$SHUTIT" ]] && SHUTIT="$1/shutit"
[[ ! -a "$SHUTIT" ]] || [[ -z "$SHUTIT" ]] && SHUTIT="$(which shutit)"
if [[ ! -a "$SHUTIT" ]]
then
echo "Must have shutit on path, eg export PATH=$PATH:/path/to/shutit_dir"
exit 1
fi
./destroy_vms.sh
$SHUTIT build --echo -d bash -m shutit-library/vagrant -m shutit-library/virtualization -l debug "$@"
if [[ $? != 0 ]]
then
exit 1
fi''')
runsh_file.close()
os.chmod(runsh_filename,0o755)
# destroy_vms.sh
destroyvmssh_filename = skel_path + '/destroy_vms.sh'
destroyvmssh_file = open(destroyvmssh_filename,'w+')
destroyvmssh_file_contents = '''#!/bin/bash'''
if snapshot:
destroyvmssh_file_contents += '''
FOLDER=$( ls $( cd $( dirname "${BASH_SOURCE[0]}" ) && pwd )/vagrant_run 2> /dev/null)
ANSWER='y'
if [[ $FOLDER != '' ]]
then
echo "This is snapshotted - sure you want to continue deleting? (y/n)"
echo See folder: vagrant_run/${FOLDER}
read ANSWER
fi
if [[ ${ANSWER} != 'y' ]]
then
echo Refusing to continue
exit 1
fi'''
destroyvmssh_file_contents += '''
MODULE_NAME=''' + skel_module_name + '''
rm -rf $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/vagrant_run/*
XARGS_FLAG='--no-run-if-empty'
if ! echo '' | xargs --no-run-if-empty >/dev/null 2>&1
then
XARGS_FLAG=''
fi
if [[ $(command -v VBoxManage) != '' ]]
then
while true
do
VBoxManage list runningvms | grep ${MODULE_NAME} | awk '{print $1}' | xargs $XARGS_FLAG -IXXX VBoxManage controlvm 'XXX' poweroff && VBoxManage list vms | grep ''' + skel_module_name + ''' | awk '{print $1}' | xargs -IXXX VBoxManage unregistervm 'XXX' --delete
# The xargs removes whitespace
if [[ $(VBoxManage list vms | grep ${MODULE_NAME} | wc -l | xargs) -eq '0' ]]
then
break
else
ps -ef | grep virtualbox | grep ${MODULE_NAME} | awk '{print $2}' | xargs kill
sleep 10
fi
done
fi
if [[ $(command -v virsh) ]] && [[ $(kvm-ok 2>&1 | command grep 'can be used') != '' ]]
then
virsh list | grep ${MODULE_NAME} | awk '{print $1}' | xargs $XARGS_FLAG -n1 virsh destroy
fi'''
destroyvmssh_file.write(destroyvmssh_file_contents)
destroyvmssh_file.close()
os.chmod(destroyvmssh_filename,0o755)
# build.cnf
os.system('mkdir -p ' + skel_path + '/configs')
# git setup
os.system('git init')
os.system('git submodule init')
os.system('git submodule add https://github.com/ianmiell/shutit-library')
os.system('git submodule update')
# User message
log_message = '''
# Run:
cd ''' + skel_path + ''' && ./run.sh
# to run.
'''
if upload:
log_message += r'''
As you have chosen to upload, you may want to install maven and set your
~/.m2/settings.xml file to contain these settings:
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
https://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository/>
<interactiveMode/>
<usePluginRegistry/>
<offline/>
<pluginGroups/>
<servers>
<server>
<id>nexus.meirionconsulting.com</id>
<username>uploader</username>
<password>uploader</password>
</server>
</servers>
<mirrors/>
<proxies/>
<profiles/>
<activeProfiles/>
</settings>
so you can upload vagrant boxes.
'''
shutit.log(log_message,transient=True)
################################################################################
# FILE SETUP END
################################################################################
################################################################################
# BEGIN MODULE SETUP
################################################################################
################################################################################
# BEGIN SHUTITFILE HANDLING
################################################################################
# Handle shutitfiles. If there are no shutitfiles, handle separately.
# If there are more than one, you need to treat the first one differently.
if skel_shutitfiles:
_total = len(skel_shutitfiles)
_count = 0
for skel_shutitfile in skel_shutitfiles:
_count += 1
module_modifier = '_' + str(_count)
new_module_filename = skel_path + '/' + os.path.join(skel_module_name + module_modifier + '.py')
shutit.cfg['skeleton']['module_modifier'] = module_modifier
(sections, skel_module_id, skel_module_name, _, _) = shutitfile.shutitfile_to_shutit_module(shutit, skel_shutitfile,skel_path,skel_domain,skel_module_name,skel_domain_hash,skel_delivery,skel_depends,_count,_total,module_modifier)
shutit.cfg['skeleton']['header_section'] = sections['header_section']
shutit.cfg['skeleton']['config_section'] = sections['config_section']
shutit.cfg['skeleton']['build_section'] = sections['build_section']
shutit.cfg['skeleton']['finalize_section'] = sections['finalize_section']
shutit.cfg['skeleton']['test_section'] = sections['test_section']
shutit.cfg['skeleton']['isinstalled_section'] = sections['isinstalled_section']
shutit.cfg['skeleton']['start_section'] = sections['start_section']
shutit.cfg['skeleton']['stop_section'] = sections['stop_section']
shutit.cfg['skeleton']['final_section'] = sections['final_section']
module_file = open(new_module_filename,'w+')
# We only write out the heavy stuff for vagrant on the first time round
if _count == 1:
module_file.write(header + shared_imports + """
""" + shutit.cfg['skeleton']['header_section'] + """
def build(self, shutit):
vagrant_image = shutit.cfg[self.module_id]['vagrant_image']
vagrant_provider = shutit.cfg[self.module_id]['vagrant_provider']
gui = shutit.cfg[self.module_id]['gui']
memory = shutit.cfg[self.module_id]['memory']
""" + vagrant_dir_section_1 + """
if shutit.send_and_get_output('vagrant plugin list | grep landrush') == '':
shutit.send('vagrant plugin install landrush')
shutit.send('vagrant init ' + vagrant_image)
shutit.send_file(shutit.build['this_vagrant_run_dir'] + '/Vagrantfile','''Vagrant.configure("2") do |config|
config.landrush.enabled = true
config.vm.provider "virtualbox" do |vb|
vb.gui = ''' + gui + '''
vb.memory = "''' + memory + '''"
end
""" + machine_stanzas + """
end''')
""" + machines_update + """
""" + vagrant_up_section + """
""" + vagrant_setup + """
""" + copy_keys_code + """
""" + docker_code + """
""" + user_code + """
""" + shutit.cfg['skeleton']['build_section'] + """
""" + snapshot_code + """
""" + upload_code + """
return True
""" + get_config_section + """
""" + shutit.cfg['skeleton']['config_section'] + """ return True
def test(self, shutit):
""" + shutit.cfg['skeleton']['test_section'] + """ return True
def finalize(self, shutit):
""" + shutit.cfg['skeleton']['finalize_section'] + """ return True
def is_installed(self, shutit):
""" + shutit.cfg['skeleton']['isinstalled_section'] + """
return False
def start(self, shutit):
""" + shutit.cfg['skeleton']['start_section'] + """ return True
def stop(self, shutit):
""" + shutit.cfg['skeleton']['stop_section'] + """ return True
def module():
return """ + skel_module_name + module_modifier + """(
'""" + skel_module_id + """', """ + skel_domain_hash + """.000""" + str(_count) + """,
description='',
maintainer='',
delivery_methods=['bash'],
depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant']
)""")
# In the non-first one, we don't have all the setup stuff (but we do have some!)
else:
module_file.write(header + shared_imports + """
""" + shutit.cfg['skeleton']['header_section'] + """
def build(self, shutit):
""" + vagrant_dir_section_n + """
""" + shutit.cfg['skeleton']['build_section'] + """
""" + shutit.cfg['skeleton']['config_section'] + """ return True
def test(self, shutit):
""" + shutit.cfg['skeleton']['test_section'] + """ return True
def finalize(self, shutit):
""" + shutit.cfg['skeleton']['finalize_section'] + """ return True
def is_installed(self, shutit):
""" + shutit.cfg['skeleton']['isinstalled_section'] + """
return False
def start(self, shutit):
""" + shutit.cfg['skeleton']['start_section'] + """ return True
def stop(self, shutit):
""" + shutit.cfg['skeleton']['stop_section'] + """ return True
def module():
return """ + skel_module_name + module_modifier + """(
'""" + skel_module_id + """',""" + skel_domain_hash + """.000""" + str(_count) + """,
description='',
maintainer='',
delivery_methods=['bash'],
depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant']
)""")
module_file.close()
# Set up build.cnf
build_cnf_filename = skel_path + '/configs/build.cnf'
if _count == 1:
build_cnf_file = open(build_cnf_filename,'w+')
build_cnf_file.write('''###############################################################################
# PLEASE NOTE: This file should be changed only by the maintainer.
# PLEASE NOTE: This file is only sourced if the "shutit build" command is run
# and this file is in the relative path: configs/build.cnf
# This is to ensure it is only sourced if _this_ module is the
# target.
###############################################################################
# When this module is the one being built, which modules should be built along with it by default?
# This feeds into automated testing of each module.
[''' + skel_module_id + ''']
shutit.core.module.build:yes''')
build_cnf_file.close()
else:
build_cnf_file = open(build_cnf_filename,'a')
build_cnf_file.write('''
[''' + skel_domain + '''.''' + skel_module_name + module_modifier + ''']
shutit.core.module.build:yes''')
build_cnf_file.close()
os.chmod(build_cnf_filename,0o400)
################################################################################
# END SHUTITFILE HANDLING
################################################################################
else:
# No shutitfiles to consider, so simpler logic here.
shutit.cfg['skeleton']['header_section'] = 'from shutit_module import ShutItModule\n\nclass ' + skel_module_name + '(ShutItModule):\n'
new_module_filename = skel_path + '/' + skel_module_name + '.py'
module_file = open(new_module_filename,'w+')
module_file.write(header + shared_imports + """
""" + shutit.cfg['skeleton']['header_section'] + """
def build(self, shutit):
vagrant_image = shutit.cfg[self.module_id]['vagrant_image']
vagrant_provider = shutit.cfg[self.module_id]['vagrant_provider']
gui = shutit.cfg[self.module_id]['gui']
memory = shutit.cfg[self.module_id]['memory']
shutit.build['vagrant_run_dir'] = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda:0))) + '/vagrant_run'
shutit.build['module_name'] = '""" + skel_module_name + """_' + ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6))
shutit.build['this_vagrant_run_dir'] = shutit.build['vagrant_run_dir'] + '/' + shutit.build['module_name']
shutit.send(' command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])
shutit.send('command rm -rf ' + shutit.build['this_vagrant_run_dir'] + ' && command mkdir -p ' + shutit.build['this_vagrant_run_dir'] + ' && command cd ' + shutit.build['this_vagrant_run_dir'])
if shutit.send_and_get_output('vagrant plugin list | grep landrush') == '':
shutit.send('vagrant plugin install landrush')
shutit.send('vagrant init ' + vagrant_image)
shutit.send_file(shutit.build['this_vagrant_run_dir'] + '/Vagrantfile','''Vagrant.configure("2") do |config|
config.landrush.enabled = true
config.vm.provider "virtualbox" do |vb|
vb.gui = ''' + gui + '''
vb.memory = "''' + memory + '''"
end
""" + machine_stanzas + """
end''')
""" + machines_update + """
""" + vagrant_up_section + """
""" + vagrant_setup + """
""" + copy_keys_code + """
""" + docker_code + """
""" + user_code + """
""" + machine_seed_code + """
""" + snapshot_code + """
""" + upload_code + """
return True
""" + get_config_section + """
def test(self, shutit):
return True
def finalize(self, shutit):
return True
def is_installed(self, shutit):
return False
def start(self, shutit):
return True
def stop(self, shutit):
return True
def module():
return """ + skel_module_name + """(
'""" + skel_domain + '''.''' + skel_module_name + """', """ + skel_domain_hash + """.0001,
description='',
maintainer='',
delivery_methods=['bash'],
depends=['""" + skel_depends + """','shutit-library.virtualization.virtualization.virtualization','tk.shutit.vagrant.vagrant.vagrant']
)""")
module_file.close()
build_cnf_filename = skel_path + '/configs/build.cnf'
build_cnf_file = open(build_cnf_filename,'w+')
build_cnf_file.write('''###############################################################################
# PLEASE NOTE: This file should be changed only by the maintainer.
# PLEASE NOTE: This file is only sourced if the "shutit build" command is run
# and this file is in the relative path: configs/build.cnf
# This is to ensure it is only sourced if _this_ module is the
# target.
###############################################################################
# When this module is the one being built, which modules should be built along with it by default?
# This feeds into automated testing of each module.
['''+skel_domain+'''.'''+skel_module_name+''']
shutit.core.module.build:yes''')
build_cnf_file.close()
os.chmod(build_cnf_filename,0o400)
def do_interactive_modules():
shutit = shutit_global.shutit
cfg = shutit.cfg
errs = []
while True:
shutit_util.list_modules(long_output=False, sort_order='run_order')
# Which module do you want to toggle?
module_id = shutit_util.util_raw_input(
prompt=
'Which module id do you want to toggle?\n(just hit return to continue with build)\n'
)
if module_id:
try:
_ = cfg[module_id]
except:
print 'Please input a valid module id'
continue
cfg[module_id]['shutit.core.module.build'] = not cfg[module_id][
'shutit.core.module.build']
if not shutit_util.config_collection_for_built(throw_error=False):
cfg[module_id]['shutit.core.module.build'] = not cfg[
module_id]['shutit.core.module.build']
shutit_util.util_raw_input(prompt='Hit return to continue.\n')
continue
# If true, set up config for that module
if cfg[module_id]['shutit.core.module.build']:
# TODO: does this catch all the ones switched on? Once done, get configs for all those.
newcfg_list = []
while True:
print shutit_util.print_config(cfg, module_id=module_id)
name = shutit_util.util_raw_input(
prompt=
'Above is the config for that module. Hit return to continue, or a config item you want to update.\n'
)
if name:
doing_list = False
while True:
if doing_list:
val_type = shutit_util.util_raw_input(
prompt=
'Input the type for the next list item: b(oolean), s(tring).\n'
)
if val_type not in ('b', 's', ''):
continue
else:
val_type = shutit_util.util_raw_input(
prompt=
'Input the type for that config item: b(oolean), s(tring), l(ist).\n'
)
if val_type not in ('b', 's', 'l', ''):
continue
if val_type == 's':
val = shutit_util.util_raw_input(
prompt=
'Input the value new for that config item.\n'
)
if doing_list:
newcfg_list.append(val)
else:
break
elif val_type == 'b':
val = shutit_util.util_raw_input(
prompt=
'Input the value new for the boolean (t/f).\n'
)
if doing_list:
if val == 't':
newcfg_list.append(True)
elif val == 'f':
newcfg_list.append(False)
else:
print 'Input t or f please'
continue
else:
break
elif val_type == 'l':
doing_list = True
newcfg_list = []
elif val_type == '':
break
# TODO: handle blank/None
if doing_list:
cfg[module_id][name] = newcfg_list
else:
cfg[module_id][name] = val
else:
break
else:
pass
# TODO: if removing, get any that depend on it, and remove those too
else:
break
return errs
def init_shutit_map(shutit):
"""Initializes the module map of shutit based on the modules
we have gathered.
Checks we have core modules
Checks for duplicate module details.
Sets up common config.
Sets up map of modules.
"""
cfg = shutit.cfg
modules = shutit.shutit_modules
# Have we got anything to process outside of special modules?
if len([mod for mod in modules if mod.run_order > 0]) < 1:
shutit.log(modules)
path = ':'.join(cfg['host']['shutit_module_path'])
shutit.log(
'\nIf you are new to ShutIt, see:\n\n\thttp://ianmiell.github.io/shutit/\n\nor try running\n\n\tshutit skeleton\n\n',
code=31,
prefix=False,
force_stdout=True)
if path == '':
shutit.fail(
'No ShutIt modules aside from core ones found and no ShutIt' +
' module path given. ' +
'\nDid you set --shutit_module_path/-m wrongly?\n')
elif path == '.':
shutit.fail(
'No modules aside from core ones found and no ShutIt' +
' module path given apart from default (.).\n\n- Did you' +
' set --shutit_module_path/-m?\n- Is there a STOP* file' +
' in your . dir?\n')
else:
shutit.fail(
'No modules aside from core ones found and no ShutIt ' +
'modules in path:\n\n' + path +
'\n\nor their subfolders. Check your ' +
'--shutit_module_path/-m setting and check that there are ' +
'ShutIt modules below without STOP* files in any relevant ' +
'directories.\n')
shutit.log('PHASE: base setup', code='32')
if cfg['build']['interactive'] >= 3:
shutit.log(
'\nChecking to see whether there are duplicate module ids ' +
'or run orders in the visible modules.',
force_stdout=True)
shutit.log('\nModules I see are:\n', force_stdout=True)
for module in modules:
shutit.log(module.module_id, force_stdout=True, code='32')
shutit.log('\n', force_stdout=True)
run_orders = {}
has_core_module = False
for module in modules:
assert isinstance(module, ShutItModule)
if module.module_id in shutit.shutit_map:
shutit.fail(
'Duplicated module id: ' + module.module_id +
'\n\nYou may want to check your --shutit_module_path setting')
if module.run_order in run_orders:
shutit.fail(
'Duplicate run order: ' + str(module.run_order) + ' for ' +
module.module_id + ' and ' +
run_orders[module.run_order].module_id +
'\n\nYou may want to check your --shutit_module_path setting')
if module.run_order == 0:
has_core_module = True
shutit.shutit_map[module.module_id] = run_orders[
module.run_order] = module
if not has_core_module:
shutit.fail('No module with run_order=0 specified! This is required.')
if cfg['build']['interactive'] >= 3:
print(
shutit_util.colour(
'32', 'Module id and run order checks OK' +
'\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
def do_build(shutit):
"""Runs build phase, building any modules that we've determined
need building.
"""
cfg = shutit.cfg
shutit.log('PHASE: build, repository work', code='32')
shutit.log(shutit_util.print_config(cfg))
if cfg['build']['interactive'] >= 3:
print('\nNow building any modules that need building' +
shutit_util.colour('32', '\n\n[Hit return to continue]\n'))
shutit_util.util_raw_input(shutit=shutit)
module_id_list = module_ids(shutit)
if cfg['build']['deps_only']:
module_id_list_build_only = filter(
lambda x: cfg[x]['shutit.core.module.build'], module_id_list)
for module_id in module_id_list:
module = shutit.shutit_map[module_id]
shutit.log('considering whether to build: ' + module.module_id,
code='32')
if cfg[module.module_id]['shutit.core.module.build']:
if cfg['build']['delivery'] not in module.ok_delivery_methods:
shutit.fail(
'Module: ' + module.module_id +
' can only be built with one of these --delivery methods: '
+ str(module.ok_delivery_methods) +
'\nSee shutit build -h for more info, or try adding: --delivery <method> to your shutit invocation'
)
if is_installed(shutit, module):
cfg['build']['report'] = (cfg['build']['report'] +
'\nBuilt already: ' +
module.module_id +
' with run order: ' +
str(module.run_order))
else:
# We move to the module directory to perform the build, returning immediately afterwards.
if cfg['build'][
'deps_only'] and module_id == module_id_list_build_only[
-1]:
# If this is the last module, and we are only building deps, stop here.
cfg['build']['report'] = (
cfg['build']['report'] + '\nSkipping: ' +
module.module_id + ' with run order: ' +
str(module.run_order) +
'\n\tas this is the final module and we are building dependencies only'
)
else:
revert_dir = os.getcwd()
cfg['environment'][cfg['build']['current_environment_id']][
'module_root_dir'] = os.path.dirname(
module.__module_file)
shutit.chdir(cfg['environment'][cfg['build'][
'current_environment_id']]['module_root_dir'])
shutit.login(prompt_prefix=module_id, command='bash')
build_module(shutit, module)
shutit.logout()
shutit.chdir(revert_dir)
if is_installed(shutit, module):
shutit.log('Starting module')
if not module.start(shutit):
shutit.fail(module.module_id + ' failed on start',
child=shutit.pexpect_children['target_child'])
