def do_finalize(): """Runs finalize phase; run after all builds are complete and all modules have been stopped. """ shutit = shutit_global.shutit # Stop all the modules if shutit.build['interactive'] >= 3: print('\nStopping all modules before finalize phase' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() stop_all() # Finalize in reverse order shutit.log('PHASE: finalize', level=logging.DEBUG) if shutit.build['interactive'] >= 3: print('\nNow doing finalize phase, which we do when all builds are ' + 'complete and modules are stopped' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() # Login at least once to get the exports. for module_id in shutit_util.module_ids(rev=True): # Only finalize if it's thought to be installed. if shutit_util.is_installed(shutit.shutit_map[module_id]): shutit.login(prompt_prefix=module_id, command='bash') if not shutit.shutit_map[module_id].finalize(shutit): shutit.fail(module_id + ' failed on finalize', shutit_pexpect_child=shutit. get_shutit_pexpect_session_from_id( 'target_child').pexpect_child) shutit.logout()
def do_test(): """Runs test phase, erroring if any return false. """ shutit = shutit_global.shutit if not shutit.build['dotest']: shutit.log('Tests configured off, not running', level=logging.DEBUG) return # Test in reverse order shutit.log('PHASE: test', level=logging.DEBUG) if shutit.build['interactive'] >= 3: print '\nNow doing test phase' + shutit_util.colourise( '32', '\n\n[Hit return to continue]\n') shutit_util.util_raw_input() stop_all() start_all() for module_id in shutit_util.module_ids(rev=True): # Only test if it's installed. if shutit_util.is_installed(shutit.shutit_map[module_id]): shutit.log('RUNNING TEST ON: ' + module_id, level=logging.DEBUG) shutit.login(prompt_prefix=module_id, command='bash') if not shutit.shutit_map[module_id].test(shutit): shutit.fail(module_id + ' failed on test', shutit_pexpect_child=shutit. get_shutit_pexpect_session_from_id( 'target_child').pexpect_child) shutit.logout()
def do_final_messages(self): # Show final report messages (ie messages to show after standard report). if self.report_final_messages != '': self.shutit_objects[0].log(shutit_util.colourise( 31, '\r\n\r\n' + self.report_final_messages + '\r\n\r\n'), level=logging.INFO, transient=True)
def build(self, shutit, loglevel=logging.DEBUG): """Sets up the machine ready for building. """ cfg = shutit.cfg ssh_host = cfg[self.module_id]['ssh_host'] ssh_port = cfg[self.module_id]['ssh_port'] ssh_user = cfg[self.module_id]['ssh_user'] ssh_pass = cfg[self.module_id]['password'] ssh_key = cfg[self.module_id]['ssh_key'] ssh_cmd = cfg[self.module_id]['ssh_cmd'] opts = [ '-t', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no' ] if ssh_pass == '': opts += ['-o', 'PasswordAuthentication=no'] if ssh_port != '': opts += ['-p', ssh_port] if ssh_key != '': opts += ['-i', ssh_key] host_arg = ssh_host if host_arg == '': shutit.fail('No host specified for sshing', throw_exception=False) if ssh_user != '': host_arg = ssh_user + '@' + host_arg cmd_arg = ssh_cmd if cmd_arg == '': cmd_arg = 'sudo su -s /bin/bash -' ssh_command = ['ssh'] + opts + [host_arg, cmd_arg] if shutit.build['interactive'] >= 3: print('\n\nAbout to connect to host.' + '\n\n' + shutit_util.colourise('32', '\n[Hit return to continue]')) shutit_util.util_raw_input() shutit.build['ssh_command'] = ' '.join(ssh_command) shutit.log('Command being run is: ' + shutit.build['ssh_command'], level=logging.INFO) shutit_pexpect_session = shutit_pexpect.ShutItPexpectSession( 'target_child', ssh_command[0], ssh_command[1:]) target_child = shutit_pexpect_session.pexpect_child expect = ['assword', shutit.expect_prompts['base_prompt'].strip()] res = shutit.child_expect(target_child, expect, timeout=10) while True: shutit.log(target_child.before + target_child.after, level=logging.DEBUG) if res == 0: shutit.log('...', level=logging.DEBUG) res = shutit.send(ssh_pass, shutit_pexpect_child=target_child, expect=expect, timeout=10, check_exit=False, fail_on_empty_before=False, echo=False, loglevel=loglevel) elif res == 1: shutit.log('Prompt found, breaking out', level=logging.DEBUG) break self.setup_host_child() self.setup_target_child(target_child) return True
def do_final_messages(shutit): # Show final report messages (ie messages to show after standard report). if shutit.build['report_final_messages'] != '': shutit.log(shutit_util.colourise( 31, '\r\n\r\n' + shutit.build['report_final_messages'] + '\r\n\r\n'), level=logging.INFO, transient=True)
def build(self, shutit, loglevel=logging.DEBUG): """Sets up the machine ready for building. """ cfg = shutit.cfg ssh_host = cfg[self.module_id]['ssh_host'] ssh_port = cfg[self.module_id]['ssh_port'] ssh_user = cfg[self.module_id]['ssh_user'] ssh_pass = cfg[self.module_id]['password'] ssh_key = cfg[self.module_id]['ssh_key'] ssh_cmd = cfg[self.module_id]['ssh_cmd'] opts = [ '-t', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no' ] if ssh_pass == '': opts += ['-o', 'PasswordAuthentication=no'] if ssh_port != '': opts += ['-p', ssh_port] if ssh_key != '': opts += ['-i', ssh_key] host_arg = ssh_host if host_arg == '': shutit.fail('No host specified for sshing', throw_exception=False) if ssh_user != '': host_arg = ssh_user + '@' + host_arg cmd_arg = ssh_cmd if cmd_arg == '': cmd_arg = 'sudo su -s /bin/bash -' ssh_command = ['ssh'] + opts + [host_arg, cmd_arg] if shutit.build['interactive'] >= 3: print('\n\nAbout to connect to host.' + '\n\n' + shutit_util.colourise('32', '\n[Hit return to continue]')) shutit_util.util_raw_input() shutit.build['ssh_command'] = ' '.join(ssh_command) shutit.log('Command being run is: ' + shutit.build['ssh_command'],level=logging.INFO) shutit_pexpect_session = shutit_pexpect.ShutItPexpectSession('target_child', ssh_command[0], ssh_command[1:]) target_child = shutit_pexpect_session.pexpect_child expect = ['assword', shutit.expect_prompts['base_prompt'].strip()] res = shutit.child_expect(target_child,expect, timeout=10) while True: shutit.log(target_child.before + target_child.after,level=logging.DEBUG) if res == 0: shutit.log('...',level=logging.DEBUG) res = shutit.send(ssh_pass, shutit_pexpect_child=target_child, expect=expect, timeout=10, check_exit=False, fail_on_empty_before=False, echo=False, loglevel=loglevel) elif res == 1: shutit.log('Prompt found, breaking out',level=logging.DEBUG) break self.setup_host_child() self.setup_target_child(target_child) return True
def start_all(run_order=-1): """Runs start method on all modules less than the passed-in run_order. Used when target is exporting itself mid-build, so we can export a clean target and still depended-on modules running if necessary. """ shutit = shutit_global.shutit if shutit.build['interactive'] >= 3: print('\nRunning start on all modules' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() # sort them so they're started in order for module_id in shutit_util.module_ids(): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if shutit_util.is_installed(shutit_module_obj): if not shutit_module_obj.start(shutit): shutit.fail('failed to start: ' + module_id, shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').shutit_pexpect_child)
def stop_all(run_order=-1): """Runs stop method on all modules less than the passed-in run_order. Used when target is exporting itself mid-build, so we clean up state before committing run files etc. """ shutit = shutit_global.shutit if shutit.build['interactive'] >= 3: print('\nRunning stop on all modules' + shutit_util.colourise('32', '\n\n[Hit return to continue]')) shutit_util.util_raw_input() # sort them so they're stopped in reverse order for module_id in shutit_util.module_ids(rev=True): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if shutit_util.is_installed(shutit_module_obj): if not shutit_module_obj.stop(shutit): shutit.fail('failed to stop: ' + module_id, shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').shutit_pexpect_child)
def init_shutit_map(shutit): """Initializes the module map of shutit based on the modules we have gathered. Checks we have core modules Checks for duplicate module details. Sets up common config. Sets up map of modules. """ modules = shutit.shutit_modules # Have we got anything to process outside of special modules? if len([mod for mod in modules if mod.run_order > 0]) < 1: shutit.log(modules,level=logging.DEBUG) path = ':'.join(shutit.host['shutit_module_path']) shutit.log('\nIf you are new to ShutIt, see:\n\n\thttp://ianmiell.github.io/shutit/\n\nor try running\n\n\tshutit skeleton\n\n',level=logging.INFO) if path == '': shutit.fail('No ShutIt modules aside from core ones found and no ShutIt module path given.\nDid you set --shutit_module_path/-m wrongly?\n') elif path == '.': shutit.fail('No modules aside from core ones found and no ShutIt module path given apart from default (.).\n\n- Did you set --shutit_module_path/-m?\n- Is there a STOP* file in your . dir?') else: shutit.fail('No modules aside from core ones found and no ShutIt modules in path:\n\n' + path + '\n\nor their subfolders. Check your --shutit_module_path/-m setting and check that there are ShutIt modules below without STOP* files in any relevant directories.') shutit.log('PHASE: base setup', level=logging.DEBUG) if shutit.build['interactive'] >= 3: shutit.log('\nChecking to see whether there are duplicate module ids or run orders in the visible modules.\nModules I see are:\n',level=logging.DEBUG) for module in modules: shutit.log(module.module_id, level=logging.DEBUG) shutit.log('\n',level=logging.DEBUG) run_orders = {} has_core_module = False for module in modules: assert isinstance(module, ShutItModule) if module.module_id in shutit.shutit_map: shutit.fail('Duplicated module id: ' + module.module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order in run_orders: shutit.fail('Duplicate run order: ' + str(module.run_order) + ' for ' + module.module_id + ' and ' + run_orders[module.run_order].module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order == 0: has_core_module = True shutit.shutit_map[module.module_id] = run_orders[module.run_order] = module if not has_core_module: shutit.fail('No module with run_order=0 specified! This is required.') if shutit.build['interactive'] >= 3: print(shutit_util.colourise('32', 'Module id and run order checks OK\n\n[Hit return to continue]\n')) shutit_util.util_raw_input()
def conn_target(shutit): """Connect to the target. """ conn_module = None for mod in shutit.conn_modules: if mod.module_id == shutit.build['conn_module']: conn_module = mod break if conn_module is None: shutit.fail('Couldn\'t find conn_module ' + shutit.build['conn_module']) # Set up the target in pexpect. if shutit.build['interactive'] >= 3: print('\nRunning the conn module (' + shutit.shutit_main_dir + '/shutit_setup.py)' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() conn_module.get_config(shutit) conn_module.build(shutit)
def stop_all(run_order=-1): """Runs stop method on all modules less than the passed-in run_order. Used when target is exporting itself mid-build, so we clean up state before committing run files etc. """ shutit = shutit_global.shutit if shutit.build['interactive'] >= 3: print('\nRunning stop on all modules' + shutit_util.colourise('32', '\n\n[Hit return to continue]')) shutit_util.util_raw_input() # sort them so they're stopped in reverse order for module_id in shutit_util.module_ids(rev=True): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if shutit_util.is_installed(shutit_module_obj): if not shutit_module_obj.stop(shutit): shutit.fail('failed to stop: ' + module_id, shutit_pexpect_child=shutit. get_shutit_pexpect_session_from_id( 'target_child').shutit_pexpect_child)
def start_all(run_order=-1): """Runs start method on all modules less than the passed-in run_order. Used when target is exporting itself mid-build, so we can export a clean target and still depended-on modules running if necessary. """ shutit = shutit_global.shutit if shutit.build['interactive'] >= 3: print('\nRunning start on all modules' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() # sort them so they're started in order for module_id in shutit_util.module_ids(): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if shutit_util.is_installed(shutit_module_obj): if not shutit_module_obj.start(shutit): shutit.fail('failed to start: ' + module_id, shutit_pexpect_child=shutit. get_shutit_pexpect_session_from_id( 'target_child').shutit_pexpect_child)
def do_test(): """Runs test phase, erroring if any return false. """ shutit = shutit_global.shutit if not shutit.build['dotest']: shutit.log('Tests configured off, not running',level=logging.DEBUG) return # Test in reverse order shutit.log('PHASE: test', level=logging.DEBUG) if shutit.build['interactive'] >= 3: print '\nNow doing test phase' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n') shutit_util.util_raw_input() stop_all() start_all() for module_id in shutit_util.module_ids(rev=True): # Only test if it's installed. if shutit_util.is_installed(shutit.shutit_map[module_id]): shutit.log('RUNNING TEST ON: ' + module_id, level=logging.DEBUG) shutit.login(prompt_prefix=module_id,command='bash') if not shutit.shutit_map[module_id].test(shutit): shutit.fail(module_id + ' failed on test', shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').pexpect_child) shutit.logout()
def do_build(): """Runs build phase, building any modules that we've determined need building. """ shutit = shutit_global.shutit cfg = shutit.cfg shutit.log('PHASE: build, repository work', level=logging.DEBUG) if shutit.build['interactive'] >= 3: print ('\nNow building any modules that need building' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() module_id_list = shutit_util.module_ids() if shutit.build['deps_only']: module_id_list_build_only = filter(lambda x: cfg[x]['shutit.core.module.build'], module_id_list) for module_id in module_id_list: module = shutit.shutit_map[module_id] shutit.log('Considering whether to build: ' + module.module_id, level=logging.INFO) if cfg[module.module_id]['shutit.core.module.build']: if shutit.build['delivery'] not in module.ok_delivery_methods: shutit.fail('Module: ' + module.module_id + ' can only be built with one of these --delivery methods: ' + str(module.ok_delivery_methods) + '\nSee shutit build -h for more info, or try adding: --delivery <method> to your shutit invocation') if shutit_util.is_installed(module): shutit.build['report'] = (shutit.build['report'] + '\nBuilt already: ' + module.module_id + ' with run order: ' + str(module.run_order)) else: # We move to the module directory to perform the build, returning immediately afterwards. if shutit.build['deps_only'] and module_id == module_id_list_build_only[-1]: # If this is the last module, and we are only building deps, stop here. shutit.build['report'] = (shutit.build['report'] + '\nSkipping: ' + module.module_id + ' with run order: ' + str(module.run_order) + '\n\tas this is the final module and we are building dependencies only') else: revert_dir = os.getcwd() shutit_global.shutit.get_current_shutit_pexpect_session_environment().module_root_dir = os.path.dirname(module.__module_file) shutit.chdir(shutit_global.shutit.get_current_shutit_pexpect_session_environment().module_root_dir) shutit.login(prompt_prefix=module_id,command='bash') build_module(module) shutit.logout() shutit.chdir(revert_dir) if shutit_util.is_installed(module): shutit.log('Starting module',level=logging.DEBUG) if not module.start(shutit): shutit.fail(module.module_id + ' failed on start', shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').pexpect_child)
def do_finalize(): """Runs finalize phase; run after all builds are complete and all modules have been stopped. """ shutit = shutit_global.shutit # Stop all the modules if shutit.build['interactive'] >= 3: print('\nStopping all modules before finalize phase' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() stop_all() # Finalize in reverse order shutit.log('PHASE: finalize', level=logging.DEBUG) if shutit.build['interactive'] >= 3: print('\nNow doing finalize phase, which we do when all builds are ' + 'complete and modules are stopped' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() # Login at least once to get the exports. for module_id in shutit_util.module_ids(rev=True): # Only finalize if it's thought to be installed. if shutit_util.is_installed(shutit.shutit_map[module_id]): shutit.login(prompt_prefix=module_id,command='bash') if not shutit.shutit_map[module_id].finalize(shutit): shutit.fail(module_id + ' failed on finalize', shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').pexpect_child) shutit.logout()
def start_container(self, shutit_session_name, loglevel=logging.DEBUG): shutit = shutit_global.shutit docker = shutit.host['docker_executable'].split(' ') # Always-required options if not os.path.exists(shutit.build['shutit_state_dir'] + '/cidfiles'): os.makedirs(shutit.build['shutit_state_dir'] + '/cidfiles') shutit.build['cidfile'] = shutit.build[ 'shutit_state_dir'] + '/cidfiles/' + shutit.host[ 'username'] + '_cidfile_' + shutit.build['build_id'] cidfile_arg = '--cidfile=' + shutit.build['cidfile'] # Singly-specified options privileged_arg = '' name_arg = '' hostname_arg = '' rm_arg = '' net_arg = '' mount_docker_arg = '' shell_arg = '/bin/bash' if shutit.build['privileged']: privileged_arg = '--privileged=true' if shutit.target['name'] != '': name_arg = '--name=' + shutit.target['name'] if shutit.target['hostname'] != '': hostname_arg = '-h=' + shutit.target['hostname'] if shutit.build['net'] != '': net_arg = '--net="' + shutit.build['net'] + '"' if shutit.build['mount_docker']: mount_docker_arg = '-v=/var/run/docker.sock:/var/run/docker.sock' # Incompatible with do_repository_work if shutit.target['rm']: rm_arg = '--rm=true' if shutit.build['base_image'] in ('alpine', 'busybox'): shell_arg = '/bin/ash' # Multiply-specified options port_args = [] dns_args = [] volume_args = [] volumes_from_args = [] volumes_list = shutit.target['volumes'].strip().split() volumes_from_list = shutit.target['volumes_from'].strip().split() ports_list = shutit.target['ports'].strip().split() dns_list = shutit.host['dns'].strip().split() for portmap in ports_list: port_args.append('-p=' + portmap) for dns in dns_list: dns_args.append('--dns=' + dns) for volume in volumes_list: volume_args.append('-v=' + volume) for volumes_from in volumes_from_list: volumes_from_args.append('--volumes-from=' + volumes_from) docker_command = docker + [ arg for arg in [ 'run', cidfile_arg, privileged_arg, name_arg, hostname_arg, rm_arg, net_arg, mount_docker_arg, ] + volume_args + volumes_from_args + port_args + dns_args + ['-t', '-i', shutit.target['docker_image'], shell_arg] if arg != '' ] if shutit.build['interactive'] >= 3: print( '\n\nAbout to start container. Ports mapped will be: ' + ', '.join(port_args) + '\n\n[host]\nports:<value>\n\nconfig, building on the configurable base image passed in in:\n\n --image <image>\n\nor config:\n\n [target]\n docker_image:<image>)\n\nBase image in this case is:\n\n ' + shutit.target['docker_image'] + '\n\n' + shutit_util.colourise('32', '\n[Hit return to continue]')) shutit_util.util_raw_input() shutit.build['docker_command'] = ' '.join(docker_command) shutit.log('Command being run is: ' + shutit.build['docker_command'], level=logging.DEBUG) shutit.log('Downloading image, please be patient', level=logging.INFO) was_sent = string.join(docker_command, ' ') shutit_pexpect_session = shutit_pexpect.ShutItPexpectSession( shutit_session_name, docker_command[0], docker_command[1:]) target_child = shutit_pexpect_session.pexpect_child expect = [ 'assword', shutit.expect_prompts['base_prompt'].strip(), 'Waiting', 'ulling', 'endpoint', 'Download', 'o such file' ] res = shutit_pexpect_session.expect(expect, timeout=9999) while True: try: shutit.log(target_child.before + target_child.after, level=loglevel) except: pass if res == 0: res = shutit.send(shutit.host['password'], shutit_pexpect_child=target_child, expect=expect, timeout=9999, check_exit=False, fail_on_empty_before=False, echo=False, loglevel=loglevel) elif res == 1: shutit.log('Prompt found, breaking out', level=logging.DEBUG) break elif res == 6: shutit.fail( 'Docker not installed. Is this a mac? If so, install Docker Toolbox - see https://docker.com' ) break else: res = shutit_pexpect_session.expect(expect, timeout=9999) continue # Did the pull work? if not shutit_pexpect_session.check_last_exit_values(was_sent): shutit_global.shutit.pause_point( 'Command:\n\n' + was_sent + '\n\nfailed, you have a shell to try rectifying the problem before continuing.' ) # Get the cid while True: try: cid = open(shutit.build['cidfile']).read() break except Exception: time.sleep(1) if cid == '' or re.match('^[a-z0-9]+$', cid) == None: shutit.fail( 'Could not get container_id - quitting. Check whether other containers may be clashing on port allocation or name.\nYou might want to try running: sudo docker kill ' + shutit.target['name'] + '; sudo docker rm ' + shutit.target['name'] + '\nto resolve a name clash or: ' + shutit.host['docker_executable'] + ' ps -a | grep ' + shutit.target['ports'] + " | awk '{print $1}' | " + 'xargs ' + shutit.host['docker_executable'] + ' kill\nto ' + 'resolve a port clash\n') shutit.log('cid: ' + cid, level=logging.DEBUG) shutit.target['container_id'] = cid return target_child
def start_container(self, shutit_session_name, loglevel=logging.DEBUG): shutit = shutit_global.shutit cfg = shutit.cfg docker = shutit.host['docker_executable'].split(' ') # Always-required options if not os.path.exists(shutit.build['shutit_state_dir'] + '/cidfiles'): os.makedirs(shutit.build['shutit_state_dir'] + '/cidfiles') shutit.build['cidfile'] = shutit.build['shutit_state_dir'] + '/cidfiles/' + shutit.host['username'] + '_cidfile_' + shutit.build['build_id'] cidfile_arg = '--cidfile=' + shutit.build['cidfile'] # Singly-specified options privileged_arg = '' name_arg = '' hostname_arg = '' rm_arg = '' net_arg = '' mount_docker_arg = '' shell_arg = '/bin/bash' if shutit.build['privileged']: privileged_arg = '--privileged=true' if shutit.target['name'] != '': name_arg = '--name=' + shutit.target['name'] if shutit.target['hostname'] != '': hostname_arg = '-h=' + shutit.target['hostname'] if shutit.build['net'] != '': net_arg = '--net="' + shutit.build['net'] + '"' if shutit.build['mount_docker']: mount_docker_arg = '-v=/var/run/docker.sock:/var/run/docker.sock' # Incompatible with do_repository_work if shutit.target['rm']: rm_arg = '--rm=true' if shutit.build['base_image'] in ('alpine','busybox'): shell_arg = '/bin/ash' # Multiply-specified options port_args = [] dns_args = [] volume_args = [] volumes_from_args = [] volumes_list = shutit.target['volumes'].strip().split() volumes_from_list = shutit.target['volumes_from'].strip().split() ports_list = shutit.target['ports'].strip().split() dns_list = shutit.host['dns'].strip().split() for portmap in ports_list: port_args.append('-p=' + portmap) for dns in dns_list: dns_args.append('--dns=' + dns) for volume in volumes_list: volume_args.append('-v=' + volume) for volumes_from in volumes_from_list: volumes_from_args.append('--volumes-from=' + volumes_from) docker_command = docker + [ arg for arg in [ 'run', cidfile_arg, privileged_arg, name_arg, hostname_arg, rm_arg, net_arg, mount_docker_arg, ] + volume_args + volumes_from_args + port_args + dns_args + [ '-t', '-i', shutit.target['docker_image'], shell_arg ] if arg != '' ] if shutit.build['interactive'] >= 3: print('\n\nAbout to start container. Ports mapped will be: ' + ', '.join(port_args) + '\n\n[host]\nports:<value>\n\nconfig, building on the configurable base image passed in in:\n\n --image <image>\n\nor config:\n\n [target]\n docker_image:<image>)\n\nBase image in this case is:\n\n ' + shutit.target['docker_image'] + '\n\n' + shutit_util.colourise('32', '\n[Hit return to continue]')) shutit_util.util_raw_input() shutit.build['docker_command'] = ' '.join(docker_command) shutit.log('Command being run is: ' + shutit.build['docker_command'],level=logging.DEBUG) shutit.log('Downloading image, please be patient',level=logging.INFO) was_sent = string.join(docker_command,' ') shutit_pexpect_session = shutit_pexpect.ShutItPexpectSession(shutit_session_name, docker_command[0], docker_command[1:]) target_child = shutit_pexpect_session.pexpect_child expect = ['assword', shutit.expect_prompts['base_prompt'].strip(), 'Waiting', 'ulling', 'endpoint', 'Download','o such file'] res = shutit_pexpect_session.expect(expect, timeout=9999) while True: try: shutit.log(target_child.before + target_child.after,level=loglevel) except: pass if res == 0: res = shutit.send(shutit.host['password'], shutit_pexpect_child=target_child, expect=expect, timeout=9999, check_exit=False, fail_on_empty_before=False, echo=False, loglevel=loglevel) elif res == 1: shutit.log('Prompt found, breaking out',level=logging.DEBUG) break elif res == 6: shutit.fail('Docker not installed. Is this a mac? If so, install Docker Toolbox - see https://docker.com') break else: res = shutit_pexpect_session.expect(expect, timeout=9999) continue # Did the pull work? if not shutit_pexpect_session.check_last_exit_values(was_sent): shutit_global.shutit.pause_point('Command:\n\n' + was_sent + '\n\nfailed, you have a shell to try rectifying the problem before continuing.') # Get the cid while True: try: cid = open(shutit.build['cidfile']).read() break except Exception: time.sleep(1) if cid == '' or re.match('^[a-z0-9]+$', cid) == None: shutit.fail('Could not get container_id - quitting. Check whether other containers may be clashing on port allocation or name.\nYou might want to try running: sudo docker kill ' + shutit.target['name'] + '; sudo docker rm ' + shutit.target['name'] + '\nto resolve a name clash or: ' + shutit.host['docker_executable'] + ' ps -a | grep ' + shutit.target['ports'] + " | awk '{print $1}' | " + 'xargs ' + shutit.host['docker_executable'] + ' kill\nto ' + 'resolve a port clash\n') shutit.log('cid: ' + cid,level=logging.DEBUG) shutit.target['container_id'] = cid return target_child
def init_shutit_map(shutit): """Initializes the module map of shutit based on the modules we have gathered. Checks we have core modules Checks for duplicate module details. Sets up common config. Sets up map of modules. """ modules = shutit.shutit_modules # Have we got anything to process outside of special modules? if len([mod for mod in modules if mod.run_order > 0]) < 1: shutit.log(modules, level=logging.DEBUG) path = ':'.join(shutit.host['shutit_module_path']) shutit.log( '\nIf you are new to ShutIt, see:\n\n\thttp://ianmiell.github.io/shutit/\n\nor try running\n\n\tshutit skeleton\n\n', level=logging.INFO) if path == '': shutit.fail( 'No ShutIt modules aside from core ones found and no ShutIt module path given.\nDid you set --shutit_module_path/-m wrongly?\n' ) elif path == '.': shutit.fail( 'No modules aside from core ones found and no ShutIt module path given apart from default (.).\n\n- Did you set --shutit_module_path/-m?\n- Is there a STOP* file in your . dir?' ) else: shutit.fail( 'No modules aside from core ones found and no ShutIt modules in path:\n\n' + path + '\n\nor their subfolders. Check your --shutit_module_path/-m setting and check that there are ShutIt modules below without STOP* files in any relevant directories.' ) shutit.log('PHASE: base setup', level=logging.DEBUG) if shutit.build['interactive'] >= 3: shutit.log( '\nChecking to see whether there are duplicate module ids or run orders in the visible modules.\nModules I see are:\n', level=logging.DEBUG) for module in modules: shutit.log(module.module_id, level=logging.DEBUG) shutit.log('\n', level=logging.DEBUG) run_orders = {} has_core_module = False for module in modules: assert isinstance(module, ShutItModule) if module.module_id in shutit.shutit_map: shutit.fail( 'Duplicated module id: ' + module.module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order in run_orders: shutit.fail( 'Duplicate run order: ' + str(module.run_order) + ' for ' + module.module_id + ' and ' + run_orders[module.run_order].module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order == 0: has_core_module = True shutit.shutit_map[module.module_id] = run_orders[ module.run_order] = module if not has_core_module: shutit.fail('No module with run_order=0 specified! This is required.') if shutit.build['interactive'] >= 3: print( shutit_util.colourise( '32', 'Module id and run order checks OK\n\n[Hit return to continue]\n' )) shutit_util.util_raw_input()
def do_build(): """Runs build phase, building any modules that we've determined need building. """ shutit = shutit_global.shutit cfg = shutit.cfg shutit.log('PHASE: build, repository work', level=logging.DEBUG) if shutit.build['interactive'] >= 3: print('\nNow building any modules that need building' + shutit_util.colourise('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input() module_id_list = shutit_util.module_ids() if shutit.build['deps_only']: module_id_list_build_only = filter( lambda x: cfg[x]['shutit.core.module.build'], module_id_list) for module_id in module_id_list: module = shutit.shutit_map[module_id] shutit.log('Considering whether to build: ' + module.module_id, level=logging.INFO) if cfg[module.module_id]['shutit.core.module.build']: if shutit.build['delivery'] not in module.ok_delivery_methods: shutit.fail( 'Module: ' + module.module_id + ' can only be built with one of these --delivery methods: ' + str(module.ok_delivery_methods) + '\nSee shutit build -h for more info, or try adding: --delivery <method> to your shutit invocation' ) if shutit_util.is_installed(module): shutit.build['report'] = (shutit.build['report'] + '\nBuilt already: ' + module.module_id + ' with run order: ' + str(module.run_order)) else: # We move to the module directory to perform the build, returning immediately afterwards. if shutit.build[ 'deps_only'] and module_id == module_id_list_build_only[ -1]: # If this is the last module, and we are only building deps, stop here. shutit.build['report'] = ( shutit.build['report'] + '\nSkipping: ' + module.module_id + ' with run order: ' + str(module.run_order) + '\n\tas this is the final module and we are building dependencies only' ) else: revert_dir = os.getcwd() shutit_global.shutit.get_current_shutit_pexpect_session_environment( ).module_root_dir = os.path.dirname(module.__module_file) shutit.chdir( shutit_global. shutit.get_current_shutit_pexpect_session_environment( ).module_root_dir) shutit.login(prompt_prefix=module_id, command='bash') build_module(module) shutit.logout() shutit.chdir(revert_dir) if shutit_util.is_installed(module): shutit.log('Starting module', level=logging.DEBUG) if not module.start(shutit): shutit.fail(module.module_id + ' failed on start', shutit_pexpect_child=shutit. get_shutit_pexpect_session_from_id( 'target_child').pexpect_child)