def do_build(shutit): """Runs build phase, building any modules that we've determined need building. """ cfg = shutit.cfg shutit.log('PHASE: build, repository work', code='31') shutit.log(util.print_config(shutit.cfg)) if cfg['build']['interactive'] >= 3: print ('\nNow building any modules that need building' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) for module_id in module_ids(shutit): module = shutit.shutit_map[module_id] shutit.log('considering whether to build: ' + module.module_id, code='31') if cfg[module.module_id]['shutit.core.module.build']: if module.is_installed(shutit): cfg['build']['report'] = (cfg['build']['report'] + '\nBuilt already: ' + module.module_id + ' with run order: ' + str(module.run_order)) else: # We move to the module directory to perform the build, returning immediately afterwards. revert_dir = os.getcwd() os.chdir(os.path.dirname(module.__module_file)) shutit.login(prompt_prefix=module_id) build_module(shutit, module) shutit.logout() os.chdir(revert_dir) if is_built(shutit, module): shutit.log('Starting module') if not module.start(shutit): shutit.fail(module.module_id + ' failed on start', child=shutit.pexpect_children['container_child'])
def do_finalize(shutit): """Runs finalize phase; run after all builds are complete and all modules have been stopped. """ cfg = shutit.cfg # Stop all the modules if cfg['build']['interactive'] >= 3: print('\nStopping all modules before finalize phase' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) stop_all(shutit) # Finalize in reverse order shutit.log('PHASE: finalize', code='31') if cfg['build']['interactive'] >= 3: print('\nNow doing finalize phase, which we do when all builds are ' + 'complete and modules are stopped' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) for module_id in module_ids(shutit, rev=True): # Only finalize if it's thought to be installed. if is_built(shutit, shutit.shutit_map[module_id]): shutit.login() if not shutit.shutit_map[module_id].finalize(shutit): shutit.fail(module_id + ' failed on finalize', child=shutit.pexpect_children['container_child']) shutit.logout()
def do_test(shutit): """Runs test phase, erroring if any return false. """ cfg = shutit.cfg if not cfg['build']['dotest']: shutit.log('Tests configured off, not running') return # Test in reverse order shutit.log('PHASE: test', code='31') if cfg['build']['interactive'] >= 3: print '\nNow doing test phase' + util.colour('31', '\n\n[Hit return to continue]\n') util.util_raw_input(shutit=shutit) stop_all(shutit) start_all(shutit) for module_id in module_ids(shutit, rev=True): module = shutit.shutit_map[module_id] # Only test if it's thought to be installed. if is_built(shutit, shutit.shutit_map[module_id]): shutit.log('RUNNING TEST ON: ' + module_id, code='31') shutit.login(prompt_prefix=module_id) if not shutit.shutit_map[module_id].test(shutit): shutit.fail(module_id + ' failed on test', child=shutit.pexpect_children['container_child']) shutit.logout()
def do_build(shutit): """Runs build phase, building any modules that we've determined need building. """ cfg = shutit.cfg shutit.log('PHASE: build, repository work', code='31') shutit.log(util.print_config(shutit.cfg)) if cfg['build']['interactive'] >= 3: print ('\nNow building any modules that need building' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) for module_id in module_ids(shutit): module = shutit.shutit_map[module_id] shutit.log('considering whether to build: ' + module.module_id, code='31') if cfg[module.module_id]['shutit.core.module.build']: if module.is_installed(shutit): cfg['build']['report'] = (cfg['build']['report'] + '\nBuilt already: ' + module.module_id + ' with run order: ' + str(module.run_order)) else: # We move to the module directory to perform the build, returning immediately afterwards. revert_dir = os.getcwd() os.chdir(os.path.dirname(module.__module_file)) shutit.login() build_module(shutit, module) shutit.logout() os.chdir(revert_dir) if is_built(shutit, module): shutit.log('Starting module') if not module.start(shutit): shutit.fail(module.module_id + ' failed on start', child=shutit.pexpect_children['container_child'])
def do_finalize(shutit): """Runs finalize phase; run after all builds are complete and all modules have been stopped. """ cfg = shutit.cfg # Stop all the modules if cfg['build']['interactive'] >= 3: print('\nStopping all modules before finalize phase' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) stop_all(shutit) # Finalize in reverse order shutit.log('PHASE: finalize', code='31') if cfg['build']['interactive'] >= 3: print('\nNow doing finalize phase, which we do when all builds are ' + 'complete and modules are stopped' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) for module_id in module_ids(shutit, rev=True): module = shutit.shutit_map[module_id] # Only finalize if it's thought to be installed. if is_built(shutit, shutit.shutit_map[module_id]): shutit.login(prompt_prefix=module_id) if not shutit.shutit_map[module_id].finalize(shutit): shutit.fail(module_id + ' failed on finalize', child=shutit.pexpect_children['container_child']) shutit.logout()
def setup_shutit_path(cfg): # try the current directory, the .. directory, or the ../shutit directory, the ~/shutit if not cfg['host']['add_shutit_to_path']: return res = util.util_raw_input(prompt='shutit appears not to be on your path - should try and we find it and add it to your ~/.bashrc (Y/n)?') if res in ['n','N']: with open(os.path.join(cfg['shutit_home'], 'config'), 'a') as f: f.write('\n[host]\nadd_shutit_to_path: no\n') return path_to_shutit = '' for d in ['.','..','~','~/shutit']: path = os.path.abspath(d + '/shutit') if not os.path.isfile(path): continue path_to_shutit = path while path_to_shutit == '': d = util.util_raw_input(prompt='cannot auto-find shutit - please input the path to your shutit dir\n') path = os.path.abspath(d + '/shutit') if not os.path.isfile(path): continue path_to_shutit = path if path_to_shutit != '': bashrc = os.path.expanduser('~/.bashrc') with open(bashrc, "a") as myfile: #http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch myfile.write('\nexport PATH="$PATH:' + os.path.dirname(path_to_shutit) + '"\n') util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n') sys.exit()
def build(self, shutit): """Sets up the machine ready for building. """ cfg = shutit.cfg ssh_host = cfg[self.module_id]['ssh_host'] ssh_port = cfg[self.module_id]['ssh_port'] ssh_user = cfg[self.module_id]['ssh_user'] ssh_pass = cfg[self.module_id]['password'] ssh_key = cfg[self.module_id]['ssh_key'] ssh_cmd = cfg[self.module_id]['ssh_cmd'] opts = [ '-t', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no' ] if ssh_pass == '': opts += ['-o', 'PasswordAuthentication=no'] if ssh_port != '': opts += ['-p', ssh_port] if ssh_key != '': opts += ['-i', ssh_key] host_arg = ssh_host if host_arg == '': shutit.fail('No host specified for sshing', throw_exception=False) if ssh_user != '': host_arg = ssh_user + '@' + host_arg cmd_arg = ssh_cmd if cmd_arg == '': cmd_arg = 'sudo su -s /bin/bash -' ssh_command = ['ssh'] + opts + [host_arg, cmd_arg] if cfg['build']['interactive'] >= 3: print('\n\nAbout to connect to host.' + '\n\n' + util.colour('31', '\n[Hit return to continue]')) util.util_raw_input(shutit=shutit) shutit.cfg['build']['ssh_command'] = ' '.join(ssh_command) shutit.log('\n\nCommand being run is:\n\n' + shutit.cfg['build']['ssh_command'], force_stdout=True, prefix=False) container_child = pexpect.spawn(ssh_command[0], ssh_command[1:]) expect = ['assword', cfg['expect_prompts']['base_prompt'].strip()] res = container_child.expect(expect, 10) while True: shutit.log(container_child.before + container_child.after, prefix=False, force_stdout=True) if res == 0: shutit.log('...') res = shutit.send(ssh_pass, child=container_child, expect=expect, timeout=10, check_exit=False, fail_on_empty_before=False) elif res == 1: shutit.log('Prompt found, breaking out') break self._setup_prompts(shutit, container_child) self._add_begin_build_info(shutit, ssh_command) return True
def build(self, shutit): """Sets up the machine ready for building. """ cfg = shutit.cfg ssh_host = cfg[self.module_id]['ssh_host'] ssh_port = cfg[self.module_id]['ssh_port'] ssh_user = cfg[self.module_id]['ssh_user'] ssh_pass = cfg[self.module_id]['password'] ssh_key = cfg[self.module_id]['ssh_key'] ssh_cmd = cfg[self.module_id]['ssh_cmd'] opts = [ '-t', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no' ] if ssh_pass == '': opts += ['-o', 'PasswordAuthentication=no'] if ssh_port != '': opts += ['-p', ssh_port] if ssh_key != '': opts += ['-i', ssh_key] host_arg = ssh_host if host_arg == '': shutit.fail('No host specified for sshing', throw_exception=False) if ssh_user != '': host_arg = ssh_user + '@' + host_arg cmd_arg = ssh_cmd if cmd_arg == '': cmd_arg = 'sudo su -s /bin/bash -' ssh_command = ['ssh'] + opts + [host_arg, cmd_arg] if cfg['build']['interactive'] >= 3: print('\n\nAbout to connect to host.' + '\n\n' + util.colour('31', '\n[Hit return to continue]')) util.util_raw_input(shutit=shutit) shutit.cfg['build']['ssh_command'] = ' '.join(ssh_command) shutit.log('\n\nCommand being run is:\n\n' + shutit.cfg['build']['ssh_command'], force_stdout=True, prefix=False) target_child = pexpect.spawn(ssh_command[0], ssh_command[1:]) expect = ['assword', cfg['expect_prompts']['base_prompt'].strip()] res = target_child.expect(expect, 10) while True: shutit.log(target_child.before + target_child.after, prefix=False, force_stdout=True) if res == 0: shutit.log('...') res = shutit.send(ssh_pass, child=target_child, expect=expect, timeout=10, check_exit=False, fail_on_empty_before=False) elif res == 1: shutit.log('Prompt found, breaking out') break self._setup_prompts(shutit, target_child) self._add_begin_build_info(shutit, ssh_command) return True
def build_module(shutit, module): """Build passed-in module. """ cfg = shutit.cfg shutit.log('building: ' + module.module_id + ' with run order: ' + str(module.run_order), code='31') cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding: ' + module.module_id + ' with run order: ' + str(module.run_order)) if not module.build(shutit): shutit.fail(module.module_id + ' failed on build', child=shutit.pexpect_children['target_child']) else: # Create a directory and files to indicate this has been built. shutit.send('mkdir -p /root/shutit_build/module_record/' + module.module_id + ' && touch /root/shutit_build/module_record/' + module.module_id + '/built && rm -f /root/shutit_build/module_record/' + module.module_id + '/removed') # Put it into "installed" cache shutit.cfg['target']['modules_installed'].append(module.module_id) # Remove from "not installed" cache shutit.cfg['target']['modules_not_installed'].remove(module.module_id) shutit.pause_point('\nPausing to allow inspect of build for: ' + module.module_id, print_input=True, level=2) cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' + module.module_id) if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['interactive'] >= 3: shutit.log(util.build_report(shutit, '#Module:' + module.module_id), code='31') if (not cfg[module.module_id]['shutit.core.module.tag'] and cfg['build']['interactive'] >= 2): shutit.log("\n\nDo you want to save state now we\'re at the " + "end of this module? (" + module.module_id + ") (input y/n)", force_stdout=True, code='31') cfg[module.module_id]['shutit.core.module.tag'] = (util.util_raw_input(shutit=shutit,default='y') == 'y') if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['tag_modules']: shutit.log(module.module_id + ' configured to be tagged, doing repository work', force_stdout=True) # Stop all before we tag to avoid file changing errors, # and clean up pid files etc.. stop_all(shutit, module.run_order) shutit.do_repository_work(str(module.module_id) + '_' + str(module.run_order), password=cfg['host']['password'], docker_executable=cfg['host']['docker_executable'], force=True) # Start all after we tag to ensure services are up as expected. start_all(shutit, module.run_order) if cfg['build']['interactive'] >= 2: shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n", force_stdout=True,code='31') if util.util_raw_input(shutit=shutit,default='y') == 'y': cfg['build']['interactive'] = 0
def start_all(shutit, run_order=-1): """Runs start method on all modules less than the passed-in run_order. Used when container is exporting itself mid-build, so we can export a clean container and still depended-on modules running if necessary. """ cfg = shutit.cfg if cfg['build']['interactive'] >= 3: print('\nRunning start on all modules' + util.colour('31', '\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) # sort them to they're started in order) for module_id in module_ids(shutit): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if is_built(shutit, shutit_module_obj): if not shutit_module_obj.start(shutit): shutit.fail('failed to start: ' + module_id, \ child=shutit.pexpect_children['container_child'])
def do_phone_home(msg,question=''): """Report message home. msg - message to send home question - question to ask - assumes Y/y for send message, else no """ if question != '': if util.util_raw_input(prompt=question + ' (Y/n)\n') not in ('y','Y',''): return urllib.urlopen("http://shutit.tk?" + urllib.urlencode(msg))
def stop_all(shutit, run_order=-1): """Runs stop method on all modules less than the passed-in run_order. Used when container is exporting itself mid-build, so we clean up state before committing run files etc. """ cfg = shutit.cfg if cfg['build']['interactive'] >= 3: print('\nRunning stop on all modules' + \ util.colour('31', '\n[Hit return to continue]')) util.util_raw_input(shutit=shutit) # sort them to it's stopped in reverse order) for module_id in module_ids(shutit, rev=True): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if is_built(shutit, shutit_module_obj): if not shutit_module_obj.stop(shutit): shutit.fail('failed to stop: ' + \ module_id, child=shutit.pexpect_children['container_child'])
def start_all(shutit, run_order=-1): """Runs start method on all modules less than the passed-in run_order. Used when container is exporting itself mid-build, so we can export a clean container and still depended-on modules running if necessary. """ cfg = shutit.cfg if cfg['build']['interactive'] >= 3: print('\nRunning start on all modules' + util.colour('31', '\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) # sort them to they're started in order) for module_id in module_ids(shutit): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if is_built(shutit, shutit_module_obj): if not shutit_module_obj.start(shutit): shutit.fail('failed to start: ' + module_id, \ child=shutit.pexpect_children['container_child'])
def stop_all(shutit, run_order=-1): """Runs stop method on all modules less than the passed-in run_order. Used when container is exporting itself mid-build, so we clean up state before committing run files etc. """ cfg = shutit.cfg if cfg['build']['interactive'] >= 3: print('\nRunning stop on all modules' + \ util.colour('31', '\n[Hit return to continue]')) util.util_raw_input(shutit=shutit) # sort them to it's stopped in reverse order) for module_id in module_ids(shutit, rev=True): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if is_built(shutit, shutit_module_obj): if not shutit_module_obj.stop(shutit): shutit.fail('failed to stop: ' + \ module_id, child=shutit.pexpect_children['container_child'])
def do_phone_home(msg,question=''): """Report message home. msg - message to send home question - question to ask - assumes Y/y for send message, else no """ if question != '': if util.util_raw_input(prompt=question + ' (Y/n)\n') not in ('y','Y',''): return urllib.urlopen("http://shutit.tk?" + urllib.urlencode(msg))
def build_module(shutit, module): """Build passed-in module. """ cfg = shutit.cfg shutit.log('building: ' + module.module_id + ' with run order: ' + str(module.run_order), code='31') cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding: ' + module.module_id + ' with run order: ' + str(module.run_order)) if not module.build(shutit): shutit.fail(module.module_id + ' failed on build', child=shutit.pexpect_children['container_child']) shutit.pause_point('\nPausing to allow inspect of build for: ' + module.module_id, print_input=True, level=2) cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' + module.module_id) if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['interactive'] >= 3: shutit.log(util.build_report(shutit, '#Module:' + module.module_id), code='31') if (not cfg[module.module_id]['shutit.core.module.tag'] and cfg['build']['interactive'] >= 2): shutit.log("\n\nDo you want to save state now we\'re at the " + "end of this module? (" + module.module_id + ") (input y/n)", force_stdout=True, code='31') cfg[module.module_id]['shutit.core.module.tag'] = (util.util_raw_input(shutit=shutit,default='y') == 'y') if cfg[module.module_id]['shutit.core.module.tag']: shutit.log(module.module_id + ' configured to be tagged, doing repository work', force_stdout=True) # Stop all before we tag to avoid file changing errors, # and clean up pid files etc.. stop_all(shutit, module.run_order) shutit.do_repository_work(str(module.module_id) + '_' + str(module.run_order), password=cfg['host']['password'], docker_executable=cfg['host']['docker_executable'], force=True) # Start all after we tag to ensure services are up as expected. start_all(shutit, module.run_order) if cfg['build']['interactive'] >= 2: shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n", force_stdout=True,code='31') if util.util_raw_input(shutit=shutit,default='y') == 'y': cfg['build']['interactive'] = 0
def stop_all(shutit, run_order=-1): """Runs stop method on all modules less than the passed-in run_order. Used when target is exporting itself mid-build, so we clean up state before committing run files etc. """ cfg = shutit.cfg if cfg['build']['interactive'] >= 3: print('\nRunning stop on all modules' + \ util.colour('31', '\n\n[Hit return to continue]')) util.util_raw_input(shutit=shutit) # sort them so they're stopped in reverse order for module_id in module_ids(shutit, rev=True): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: # TODO: cache this - it's a big time-waster when libraries get big if is_to_be_built_or_is_installed(shutit, shutit_module_obj): if not shutit_module_obj.stop(shutit): shutit.fail('failed to stop: ' + \ module_id, child=shutit.pexpect_children['target_child'])
def conn_container(shutit): """Connect to the container. """ conn_module = None for mod in shutit.conn_modules: if mod.module_id == shutit.cfg['build']['conn_module']: conn_module = mod break if conn_module is None: shutit.fail('Couldn\'t find conn_module ' + shutit.cfg['build']['conn_module']) # Set up the target in pexpect. if shutit.cfg['build']['interactive'] >= 3: print('\nRunning the conn module (' + shutit.shutit_main_dir + '/setup.py)' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) conn_module.get_config(shutit) conn_module.build(shutit)
def do_build(shutit): """Runs build phase, building any modules that we've determined need building. """ cfg = shutit.cfg shutit.log('PHASE: build, repository work', code='31') shutit.log(util.print_config(shutit.cfg)) if cfg['build']['interactive'] >= 3: print ('\nNow building any modules that need building' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) module_id_list = module_ids(shutit) if cfg['build']['deps_only']: module_id_list_build_only = filter(lambda x: cfg[x]['shutit.core.module.build'], module_id_list) for module_id in module_id_list: module = shutit.shutit_map[module_id] shutit.log('considering whether to build: ' + module.module_id, code='31') if cfg[module.module_id]['shutit.core.module.build']: if is_installed(shutit,module): cfg['build']['report'] = (cfg['build']['report'] + '\nBuilt already: ' + module.module_id + ' with run order: ' + str(module.run_order)) else: # We move to the module directory to perform the build, returning immediately afterwards. if cfg['build']['deps_only'] and module_id == module_id_list_build_only[-1]: # If this is the last module, and we are only building deps, stop here. cfg['build']['report'] = (cfg['build']['report'] + '\nSkipping: ' + module.module_id + ' with run order: ' + str(module.run_order) + '\n\tas this is the final module and we are building dependencies only') else: revert_dir = os.getcwd() cfg['target']['module_root_dir'] = os.path.dirname(module.__module_file) shutit.chdir(cfg['target']['module_root_dir']) shutit.login(prompt_prefix=module_id) build_module(shutit, module) shutit.logout() shutit.chdir(revert_dir) if is_installed(shutit, module): shutit.log('Starting module') if not module.start(shutit): shutit.fail(module.module_id + ' failed on start', child=shutit.pexpect_children['target_child'])
def start_all(shutit, run_order=-1): """Runs start method on all modules less than the passed-in run_order. Used when target is exporting itself mid-build, so we can export a clean target and still depended-on modules running if necessary. """ cfg = shutit.cfg if cfg['build']['interactive'] >= 3: print('\nRunning start on all modules' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) # sort them so they're started in order for module_id in module_ids(shutit): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: # TODO: cache this - it's a big time-waster when libraries get big if is_to_be_built_or_is_installed(shutit, shutit_module_obj): if not shutit_module_obj.start(shutit): shutit.fail('failed to start: ' + module_id, \ child=shutit.pexpect_children['target_child'])
def conn_container(shutit): """Connect to the container. """ conn_module = None for mod in shutit.conn_modules: if mod.module_id == shutit.cfg['build']['conn_module']: conn_module = mod break if conn_module is None: shutit.fail('Couldn\'t find conn_module ' + shutit.cfg['build']['conn_module']) # Set up the target in pexpect. if shutit.cfg['build']['interactive'] >= 3: print('\nRunning the conn module (' + shutit.shutit_main_dir + '/setup.py)' + util.colour('31', '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit) conn_module.get_config(shutit) conn_module.build(shutit)
def do_test(shutit): """Runs test phase, erroring if any return false. """ cfg = shutit.cfg if not cfg['build']['dotest']: shutit.log('Tests configured off, not running') return # Test in reverse order shutit.log('PHASE: test', code='31') if cfg['build']['interactive'] >= 3: print '\nNow doing test phase' + util.colour('31', '\n\n[Hit return to continue]\n') util.util_raw_input(shutit=shutit) stop_all(shutit) start_all(shutit) for module_id in module_ids(shutit, rev=True): # Only test if it's thought to be installed. if is_built(shutit, shutit.shutit_map[module_id]): shutit.log('RUNNING TEST ON: ' + module_id, code='31') shutit.login() if not shutit.shutit_map[module_id].test(shutit): shutit.fail(module_id + ' failed on test', child=shutit.pexpect_children['container_child']) shutit.logout()
def do_phone_home(msg=None,question='Error seen - would you like to inform the maintainers?'): """Report message home. msg - message to send home question - question to ask - assumes Y/y for send message, else no """ if msg is None: msg = {} if shutit_global.shutit.cfg['build']['interactive'] == 0: return msg.update({'shutitrunstatus':'fail','pwd':os.getcwd(),'user':os.environ.get('LOGNAME', '')}) if question != '' and util.util_raw_input(prompt=question + ' (Y/n)\n') not in ('y','Y',''): return try: urllib.urlopen("http://shutit.tk?" + urllib.urlencode(msg)) except Exception as e: shutit_global.shutit.log('failed to send message: ' + str(e.message))
def init_shutit_map(shutit): """Initializes the module map of shutit based on the modules we have gathered. Checks we have core modules Checks for duplicate module details. Sets up common config. Sets up map of modules. """ cfg = shutit.cfg modules = shutit.shutit_modules # Have we got anything to process outside of special modules? if len([mod for mod in modules if mod.run_order > 0]) < 1: shutit.log(modules) path = ':'.join(cfg['host']['shutit_module_path']) if path == '': shutit.fail('No modules aside from core ones found and no ShutIt' + ' module path given. ' + '\nDid you set --shutit_module_path/-m' + ' wrongly?') elif path == '.': shutit.fail('No modules aside from core ones found and no ShutIt' + ' module path given apart from default (.).\nDid you' + ' set--shutit_module_path/-m? Is there a STOP* file' + ' in your . dir?') else: shutit.fail('No modules aside from core ones found and no ShutIt ' + 'modules in path:\n\n' + path + '\n\nor their subfolders. Check your ' + '--shutit_module_path/-m setting and check that there are ' + 'ShutItmodules below without STOP* files in any relevant ' + 'directories.') shutit.log('PHASE: base setup', code='31') if cfg['build']['interactive'] >= 3: shutit.log('\nChecking to see whether there are duplicate module ids ' + 'or run orders in the visible modules.', force_stdout=True) shutit.log('\nModules I see are:\n', force_stdout=True) for module in modules: shutit.log(module.module_id, force_stdout=True, code='31') shutit.log('\n', force_stdout=True) run_orders = {} has_core_module = False for module in modules: assert isinstance(module, ShutItModule) if module.module_id in shutit.shutit_map: shutit.fail('Duplicated module id: ' + module.module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order in run_orders: shutit.fail('Duplicate run order: ' + str(module.run_order) + ' for ' + module.module_id + ' and ' + run_orders[module.run_order].module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order == 0: has_core_module = True shutit.shutit_map[module.module_id] = run_orders[module.run_order] = module if not has_core_module: shutit.fail('No module with run_order=0 specified! This is required.') if cfg['build']['interactive'] >= 3: print(util.colour('31', 'Module id and run order checks OK' + '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit)
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - sc - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if sys.stdout.isatty(): if spawn.find_executable('shutit') is None: # try the current directory, the .. directory, or the ../shutit directory, the ~/shutit pwd = os.getcwd() path_to_shutit = '' for d in ('.','..','~','~/shutit'): if os.path.isfile(os.path.expanduser(d) + '/shutit'): path_to_shutit = d + '/shutit' res = util.util_raw_input(prompt='shutit appears not to be on your path - would you like me to add it to your ~/.bashrc (Y/n)? ') if res not in ('n','N'): bashrc = os.path.expanduser('~/') + '.bashrc' if os.path.isfile(bashrc): with open(bashrc, "a") as myfile: #http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch myfile.write('export PATH="$PATH:' + path_to_shutit + '"\n') break if path_to_shutit == '': while True: res = util.util_raw_input(prompt='shutit appears not to be on your path - please input the path to your shutit dir\n') if os.path.isfile(os.path.expanduser(res) + '/shutit'): path_to_shutit = res + '/shutit' bashrc = os.path.expanduser('~/') + '.bashrc' if os.path.isfile(bashrc): with open(bashrc, "a") as myfile: myfile.write('\nexport PATH="$PATH:' + path_to_shutit + '"\n') break if path_to_shutit != '': util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n') sys.exit() shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(cfg) if cfg['action']['skeleton']: util.create_skeleton(shutit) return if cfg['action']['serve']: import shutit_srv shutit_srv.start() return util.load_configs(shutit) shutit_module_init(shutit) conn_container(shutit) errs = [] errs.extend(check_deps(shutit)) # Show dependency graph if cfg['action']['show_depgraph']: digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' shutit.log(digraph, force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['show_config']: shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Check for conflicts now. errs.extend(check_conflicts(shutit)) errs.extend(check_ready(shutit)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_container(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a container ' + 'called ' + shutit.cfg['container']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/default.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' + 'default test runner in test.sh\n\n' + 'You can inspect the details of the build in the container\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31') # Mark the build as completed shutit.cfg['build']['completed'] = True
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - sc - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if sys.stdout.isatty(): if spawn.find_executable('shutit') is None: # try the current directory, the .. directory, or the ../shutit directory, the ~/shutit pwd = os.getcwd() path_to_shutit = '' for d in ('.','..','~','~/shutit'): if os.path.isfile(os.path.expanduser(d) + '/shutit'): path_to_shutit = d + '/shutit' res = util.util_raw_input(prompt='shutit appears not to be on your path - would you like me to add it to your ~/.bashrc (Y/n)? ') if res not in ('n','N'): bashrc = os.path.expanduser('~/') + '.bashrc' if os.path.isfile(bashrc): with open(bashrc, "a") as myfile: #http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch myfile.write('export PATH="$PATH:' + path_to_shutit + '"\n') break if path_to_shutit == '': while True: res = util.util_raw_input(prompt='shutit appears not to be on your path - please input the path to your shutit dir\n') if os.path.isfile(os.path.expanduser(res) + '/shutit'): path_to_shutit = res + '/shutit' bashrc = os.path.expanduser('~/') + '.bashrc' if os.path.isfile(bashrc): with open(bashrc, "a") as myfile: myfile.write('\nexport PATH="$PATH:' + path_to_shutit + '"\n') break if path_to_shutit != '': util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n') sys.exit() shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(cfg) if cfg['action']['skeleton']: util.create_skeleton(shutit) return if cfg['action']['serve']: import shutit_srv shutit_srv.start() return util.load_configs(shutit) shutit_module_init(shutit) conn_container(shutit) errs = [] errs.extend(check_deps(shutit)) # Show dependency graph if cfg['action']['show_depgraph']: digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' shutit.log(digraph, force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['show_config']: shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Check for conflicts now. errs.extend(check_conflicts(shutit)) errs.extend(check_ready(shutit)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_container(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a container ' + 'called ' + shutit.cfg['container']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/default.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' + 'default test runner in test.sh\n\n' + 'You can inspect the details of the build in the container\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31') # Mark the build as completed shutit.cfg['build']['completed'] = True
def init_shutit_map(shutit): """Initializes the module map of shutit based on the modules we have gathered. Checks we have core modules Checks for duplicate module details. Sets up common config. Sets up map of modules. """ cfg = shutit.cfg modules = shutit.shutit_modules # Have we got anything to process outside of special modules? if len([mod for mod in modules if mod.run_order > 0]) < 1: shutit.log(modules) path = ':'.join(cfg['host']['shutit_module_path']) if path == '': shutit.fail('No modules aside from core ones found and no ShutIt' + ' module path given. ' + '\nDid you set --shutit_module_path/-m' + ' wrongly?') elif path == '.': shutit.fail('No modules aside from core ones found and no ShutIt' + ' module path given apart from default (.).\nDid you' + ' set--shutit_module_path/-m? Is there a STOP* file' + ' in your . dir?') else: shutit.fail('No modules aside from core ones found and no ShutIt ' + 'modules in path:\n\n' + path + '\n\nor their subfolders. Check your ' + '--shutit_module_path/-m setting and check that there are ' + 'ShutItmodules below without STOP* files in any relevant ' + 'directories.') shutit.log('PHASE: base setup', code='31') if cfg['build']['interactive'] >= 3: shutit.log('\nChecking to see whether there are duplicate module ids ' + 'or run orders in the visible modules.', force_stdout=True) shutit.log('\nModules I see are:\n', force_stdout=True) for module in modules: shutit.log(module.module_id, force_stdout=True, code='31') shutit.log('\n', force_stdout=True) run_orders = {} has_core_module = False for module in modules: assert isinstance(module, ShutItModule) if module.module_id in shutit.shutit_map: shutit.fail('Duplicated module id: ' + module.module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order in run_orders: shutit.fail('Duplicate run order: ' + str(module.run_order) + ' for ' + module.module_id + ' and ' + run_orders[module.run_order].module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order == 0: has_core_module = True shutit.shutit_map[module.module_id] = run_orders[module.run_order] = module if not has_core_module: shutit.fail('No module with run_order=0 specified! This is required.') if cfg['build']['interactive'] >= 3: print(util.colour('31', 'Module id and run order checks OK' + '\n\n[Hit return to continue]\n')) util.util_raw_input(shutit=shutit)
def build(self, shutit): """Sets up the target ready for building. """ # Uncomment for testing for "failure" cases. #sys.exit(1) while not self._check_docker(shutit): pass cfg = shutit.cfg docker = cfg['host']['docker_executable'].split(' ') # Always-required options if not os.path.exists('/tmp/shutit/cidfiles'): os.makedirs('/tmp/shutit/cidfiles') cfg['build']['cidfile'] = '/tmp/shutit/cidfiles' + cfg['host']['username'] +\ '_cidfile_' + cfg['build']['build_id'] cidfile_arg = '--cidfile=' + cfg['build']['cidfile'] # Singly-specified options privileged_arg = '' lxc_conf_arg = '' name_arg = '' hostname_arg = '' volume_arg = '' rm_arg = '' net_arg = '' if cfg['build']['privileged']: privileged_arg = '--privileged=true' else: # TODO: put in to ensure serve always works. --cap-add is now an option. # Need better solution in place, eg refresh builder when build # needs privileged privileged_arg = '--privileged=true' if cfg['build']['lxc_conf'] != '': lxc_conf_arg = '--lxc-conf=' + cfg['build']['lxc_conf'] if cfg['target']['name'] != '': name_arg = '--name=' + cfg['target']['name'] if cfg['target']['hostname'] != '': hostname_arg = '-h=' + cfg['target']['hostname'] if cfg['host']['artifacts_dir'] != '': volume_arg = '-v=' + cfg['host']['artifacts_dir'] + ':/artifacts' if cfg['build']['net'] != '': net_arg = '--net="' + cfg['build']['net'] + '"' # Incompatible with do_repository_work if cfg['target']['rm']: rm_arg = '--rm=true' # Multiply-specified options port_args = [] dns_args = [] ports_list = cfg['target']['ports'].strip().split() dns_list = cfg['host']['dns'].strip().split() for portmap in ports_list: port_args.append('-p=' + portmap) for dns in dns_list: dns_args.append('--dns=' + dns) docker_command = docker + [ arg for arg in [ 'run', cidfile_arg, privileged_arg, lxc_conf_arg, name_arg, hostname_arg, volume_arg, rm_arg, net_arg, ] + port_args + dns_args + [ '-t', '-i', cfg['target']['docker_image'], '/bin/bash' ] if arg != '' ] if cfg['build']['interactive'] >= 3: print('\n\nAbout to start container. ' + 'Ports mapped will be: ' + ', '.join(port_args) + '\n\n[host]\nports:<value>\n\nconfig, building on the ' + 'configurable base image passed in in:\n\n --image <image>\n' + '\nor config:\n\n [target]\n docker_image:<image>)\n\n' + 'Base image in this case is:\n\n ' + cfg['target']['docker_image'] + '\n\n' + util.colour('31', '\n[Hit return to continue]')) util.util_raw_input(shutit=shutit) shutit.cfg['build']['docker_command'] = ' '.join(docker_command) shutit.log('\n\nCommand being run is:\n\n' + shutit.cfg['build']['docker_command'], force_stdout=True, prefix=False) shutit.log('\n\nThis may download the image, please be patient\n\n', force_stdout=True, prefix=False) target_child = pexpect.spawn(docker_command[0], docker_command[1:]) expect = ['assword', cfg['expect_prompts']['base_prompt'].strip(), \ 'Waiting', 'ulling', 'endpoint', 'Download'] res = target_child.expect(expect, 9999) while True: shutit.log(target_child.before + target_child.after, prefix=False, force_stdout=True) if res == 0: shutit.log('...') res = shutit.send(cfg['host']['password'], \ child=target_child, expect=expect, timeout=9999, \ check_exit=False, fail_on_empty_before=False) elif res == 1: shutit.log('Prompt found, breaking out') break else: res = target_child.expect(expect, 9999) continue # Get the cid time.sleep(1) # cidfile creation is sometimes slow... shutit.log('Slept') cid = open(cfg['build']['cidfile']).read() shutit.log('Opening file') if cid == '' or re.match('^[a-z0-9]+$', cid) == None: shutit.fail('Could not get container_id - quitting. ' + 'Check whether ' + 'other containers may be clashing on port allocation or name.' + '\nYou might want to try running: sudo docker kill ' + cfg['target']['name'] + '; sudo docker rm ' + cfg['target']['name'] + '\nto resolve a name clash or: ' + cfg['host']['docker_executable'] + ' ps -a | grep ' + cfg['target']['ports'] + ' | awk \'{print $1}\' | ' + 'xargs ' + cfg['host']['docker_executable'] + ' kill\nto + ' 'resolve a port clash\n') shutit.log('cid: ' + cid) cfg['target']['container_id'] = cid self._setup_prompts(shutit, target_child) self._add_begin_build_info(shutit, docker_command) return True
def build(self, shutit): """Sets up the container ready for building. """ # Uncomment for testing for "failure" cases. #sys.exit(1) while not self._check_docker(shutit): pass cfg = shutit.cfg docker = cfg['host']['docker_executable'].split(' ') # Always-required options if not os.path.exists('/tmp/shutit/cidfiles'): os.makedirs('/tmp/shutit/cidfiles') cfg['build']['cidfile'] = '/tmp/shutit/cidfiles' + cfg['host']['username'] +\ '_cidfile_' + cfg['build']['build_id'] cidfile_arg = '--cidfile=' + cfg['build']['cidfile'] # Singly-specified options privileged_arg = '' lxc_conf_arg = '' name_arg = '' hostname_arg = '' volume_arg = '' rm_arg = '' net_arg = '' if cfg['build']['privileged']: privileged_arg = '--privileged=true' else: # TODO: put in to ensure serve always works. --cap-add is now an option. # Need better solution in place, eg refresh builder when build # needs privileged privileged_arg = '--privileged=true' if cfg['build']['lxc_conf'] != '': lxc_conf_arg = '--lxc-conf=' + cfg['build']['lxc_conf'] if cfg['container']['name'] != '': name_arg = '--name=' + cfg['container']['name'] if cfg['container']['hostname'] != '': hostname_arg = '-h=' + cfg['container']['hostname'] if cfg['host']['resources_dir'] != '': volume_arg = '-v=' + cfg['host']['resources_dir'] + ':/resources' if cfg['build']['net'] != '': net_arg = '--net="' + cfg['build']['net'] + '"' # Incompatible with do_repository_work if cfg['container']['rm']: rm_arg = '--rm=true' # Multiply-specified options port_args = [] dns_args = [] ports_list = cfg['container']['ports'].strip().split() dns_list = cfg['host']['dns'].strip().split() for portmap in ports_list: port_args.append('-p=' + portmap) for dns in dns_list: dns_args.append('--dns=' + dns) docker_command = docker + [ arg for arg in [ 'run', cidfile_arg, privileged_arg, lxc_conf_arg, name_arg, hostname_arg, volume_arg, rm_arg, net_arg, ] + port_args + dns_args + ['-t', '-i', cfg['container']['docker_image'], '/bin/bash'] if arg != '' ] if cfg['build']['interactive'] >= 3: print( '\n\nAbout to start container. ' + 'Ports mapped will be: ' + ', '.join(port_args) + '\n\n[host]\nports:<value>\n\nconfig, building on the ' + 'configurable base image passed in in:\n\n --image <image>\n' + '\nor config:\n\n [container]\n docker_image:<image>)\n\n' + 'Base image in this case is:\n\n ' + cfg['container']['docker_image'] + '\n\n' + util.colour('31', '\n[Hit return to continue]')) util.util_raw_input(shutit=shutit) shutit.cfg['build']['docker_command'] = ' '.join(docker_command) shutit.log('\n\nCommand being run is:\n\n' + shutit.cfg['build']['docker_command'], force_stdout=True, prefix=False) shutit.log('\n\nThis may download the image, please be patient\n\n', force_stdout=True, prefix=False) container_child = pexpect.spawn(docker_command[0], docker_command[1:]) expect = ['assword', cfg['expect_prompts']['base_prompt'].strip(), \ 'Waiting', 'ulling', 'endpoint', 'Download'] res = container_child.expect(expect, 9999) while True: shutit.log(container_child.before + container_child.after, prefix=False, force_stdout=True) if res == 0: shutit.log('...') res = shutit.send(cfg['host']['password'], \ child=container_child, expect=expect, timeout=9999, \ check_exit=False, fail_on_empty_before=False) elif res == 1: shutit.log('Prompt found, breaking out') break else: res = container_child.expect(expect, 9999) continue # Get the cid time.sleep(5) # cidfile creation is sometimes slow... shutit.log('Slept') cid = open(cfg['build']['cidfile']).read() shutit.log('Opening file') if cid == '' or re.match('^[a-z0-9]+$', cid) == None: shutit.fail( 'Could not get container_id - quitting. ' + 'Check whether ' + 'other containers may be clashing on port allocation or name.' + '\nYou might want to try running: sudo docker kill ' + cfg['container']['name'] + '; sudo docker rm ' + cfg['container']['name'] + '\nto resolve a name clash or: ' + cfg['host']['docker_executable'] + ' ps -a | grep ' + cfg['container']['ports'] + ' | awk \'{print $1}\' | ' + 'xargs ' + cfg['host']['docker_executable'] + ' kill\nto + ' 'resolve a port clash\n') shutit.log('cid: ' + cid) cfg['container']['container_id'] = cid self._setup_prompts(shutit, container_child) self._add_begin_build_info(shutit, docker_command) return True