def build_module(shutit, module): """Build passed-in module. """ cfg = shutit.cfg shutit.log('building: ' + module.module_id + ' with run order: ' + str(module.run_order),code='31') cfg['build']['report'] = cfg['build']['report'] + '\nBuilding: ' + module.module_id + ' with run order: ' + str(module.run_order) if not module.build(shutit): shutit.fail(module.module_id + ' failed on build',child=shutit.pexpect_children['container_child']) shutit.pause_point('\nPausing to allow inspect of build for: ' + module.module_id,print_input=True,level=2) cfg['build']['report'] = cfg['build']['report'] + '\nCompleted module: ' + module.module_id if cfg[module.module_id]['do_repository_work'] or cfg['build']['interactive'] >= 3: shutit.log(util.build_report(shutit,'#Module:' + module.module_id),code='31') if not cfg[module.module_id]['do_repository_work'] and cfg['build']['interactive'] >= 2: shutit.log("\n\nDo you want to save state now we\'re at the end of this module? (" + module.module_id + ") (input y/n)",force_stdout=True) cfg[module.module_id]['do_repository_work'] = (raw_input('') == 'y') if cfg[module.module_id]['do_repository_work']: shutit.log(module.module_id + ' configured to be tagged, doing repository work',force_stdout=True) # Stop all before we tag to avoid file changing errors, and clean up pid files etc.. stop_all(shutit, module.run_order) shutit.do_repository_work(str(module.module_id) + '_' + str(module.run_order),password=cfg['host']['password'],docker_executable=cfg['host']['docker_executable'],force=True) # Start all after we tag to ensure services are up as expected. start_all(shutit, module.run_order) if cfg['build']['interactive'] >= 2: shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n",force_stdout=True) if raw_input('') == 'y': cfg['build']['interactive'] = 0
def build_module(shutit, module): """Build passed-in module. """ cfg = shutit.cfg shutit.log('building: ' + module.module_id + ' with run order: ' + str(module.run_order), code='31') cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding: ' + module.module_id + ' with run order: ' + str(module.run_order)) if not module.build(shutit): shutit.fail(module.module_id + ' failed on build', child=shutit.pexpect_children['target_child']) else: # Create a directory and files to indicate this has been built. shutit.send('mkdir -p /root/shutit_build/module_record/' + module.module_id + ' && touch /root/shutit_build/module_record/' + module.module_id + '/built && rm -f /root/shutit_build/module_record/' + module.module_id + '/removed') # Put it into "installed" cache shutit.cfg['target']['modules_installed'].append(module.module_id) # Remove from "not installed" cache shutit.cfg['target']['modules_not_installed'].remove(module.module_id) shutit.pause_point('\nPausing to allow inspect of build for: ' + module.module_id, print_input=True, level=2) cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' + module.module_id) if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['interactive'] >= 3: shutit.log(util.build_report(shutit, '#Module:' + module.module_id), code='31') if (not cfg[module.module_id]['shutit.core.module.tag'] and cfg['build']['interactive'] >= 2): shutit.log("\n\nDo you want to save state now we\'re at the " + "end of this module? (" + module.module_id + ") (input y/n)", force_stdout=True, code='31') cfg[module.module_id]['shutit.core.module.tag'] = (util.util_raw_input(shutit=shutit,default='y') == 'y') if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['tag_modules']: shutit.log(module.module_id + ' configured to be tagged, doing repository work', force_stdout=True) # Stop all before we tag to avoid file changing errors, # and clean up pid files etc.. stop_all(shutit, module.run_order) shutit.do_repository_work(str(module.module_id) + '_' + str(module.run_order), password=cfg['host']['password'], docker_executable=cfg['host']['docker_executable'], force=True) # Start all after we tag to ensure services are up as expected. start_all(shutit, module.run_order) if cfg['build']['interactive'] >= 2: shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n", force_stdout=True,code='31') if util.util_raw_input(shutit=shutit,default='y') == 'y': cfg['build']['interactive'] = 0
def build_module(shutit, module): """Build passed-in module. """ cfg = shutit.cfg shutit.log('building: ' + module.module_id + ' with run order: ' + str(module.run_order), code='31') cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding: ' + module.module_id + ' with run order: ' + str(module.run_order)) if not module.build(shutit): shutit.fail(module.module_id + ' failed on build', child=shutit.pexpect_children['container_child']) shutit.pause_point('\nPausing to allow inspect of build for: ' + module.module_id, print_input=True, level=2) cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' + module.module_id) if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['interactive'] >= 3: shutit.log(util.build_report(shutit, '#Module:' + module.module_id), code='31') if (not cfg[module.module_id]['shutit.core.module.tag'] and cfg['build']['interactive'] >= 2): shutit.log("\n\nDo you want to save state now we\'re at the " + "end of this module? (" + module.module_id + ") (input y/n)", force_stdout=True, code='31') cfg[module.module_id]['shutit.core.module.tag'] = (util.util_raw_input(shutit=shutit,default='y') == 'y') if cfg[module.module_id]['shutit.core.module.tag']: shutit.log(module.module_id + ' configured to be tagged, doing repository work', force_stdout=True) # Stop all before we tag to avoid file changing errors, # and clean up pid files etc.. stop_all(shutit, module.run_order) shutit.do_repository_work(str(module.module_id) + '_' + str(module.run_order), password=cfg['host']['password'], docker_executable=cfg['host']['docker_executable'], force=True) # Start all after we tag to ensure services are up as expected. start_all(shutit, module.run_order) if cfg['build']['interactive'] >= 2: shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n", force_stdout=True,code='31') if util.util_raw_input(shutit=shutit,default='y') == 'y': cfg['build']['interactive'] = 0
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - sc - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(cfg) if cfg['action']['skeleton']: util.create_skeleton(shutit) return if cfg['action']['serve']: import shutit_srv shutit_srv.start() return util.load_configs(shutit) shutit_module_init(shutit) conn_container(shutit) errs = [] errs.extend(check_deps(shutit)) # Show dependency graph if cfg['action']['show_depgraph']: digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['build'] ]) digraph = digraph + '\n}' shutit.log(digraph, force_stdout=True) return # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['show_config']: shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']), force_stdout=True) return # Check for conflicts now. errs.extend(check_conflicts(shutit)) errs.extend(check_ready(shutit)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_container(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a container ' + 'called ' + shutit.cfg['container']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/default.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' + 'default test runner in test.sh\n\n' + 'You can inspect the details of the build in the container\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31')
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - sc - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if sys.stdout.isatty(): if spawn.find_executable('shutit') is None: # try the current directory, the .. directory, or the ../shutit directory, the ~/shutit pwd = os.getcwd() path_to_shutit = '' for d in ('.','..','~','~/shutit'): if os.path.isfile(os.path.expanduser(d) + '/shutit'): path_to_shutit = d + '/shutit' res = util.util_raw_input(prompt='shutit appears not to be on your path - would you like me to add it to your ~/.bashrc (Y/n)? ') if res not in ('n','N'): bashrc = os.path.expanduser('~/') + '.bashrc' if os.path.isfile(bashrc): with open(bashrc, "a") as myfile: #http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch myfile.write('export PATH="$PATH:' + path_to_shutit + '"\n') break if path_to_shutit == '': while True: res = util.util_raw_input(prompt='shutit appears not to be on your path - please input the path to your shutit dir\n') if os.path.isfile(os.path.expanduser(res) + '/shutit'): path_to_shutit = res + '/shutit' bashrc = os.path.expanduser('~/') + '.bashrc' if os.path.isfile(bashrc): with open(bashrc, "a") as myfile: myfile.write('\nexport PATH="$PATH:' + path_to_shutit + '"\n') break if path_to_shutit != '': util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n') sys.exit() shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(cfg) if cfg['action']['skeleton']: util.create_skeleton(shutit) return if cfg['action']['serve']: import shutit_srv shutit_srv.start() return util.load_configs(shutit) shutit_module_init(shutit) conn_container(shutit) errs = [] errs.extend(check_deps(shutit)) # Show dependency graph if cfg['action']['show_depgraph']: digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' shutit.log(digraph, force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['show_config']: shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Check for conflicts now. errs.extend(check_conflicts(shutit)) errs.extend(check_ready(shutit)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_container(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a container ' + 'called ' + shutit.cfg['container']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/default.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' + 'default test runner in test.sh\n\n' + 'You can inspect the details of the build in the container\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31') # Mark the build as completed shutit.cfg['build']['completed'] = True
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - sc - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if sys.stdout.isatty() and spawn.find_executable('shutit') is None: setup_shutit_path() shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(shutit) if cfg['action']['skeleton']: util.create_skeleton(shutit) cfg['build']['completed'] = True return if cfg['action']['serve']: import shutit_srv cfg['build']['interactive'] = 0 revert_dir = os.getcwd() os.chdir(sys.path[0]) shutit_srv.start() os.chdir(revert_dir) return util.load_configs(shutit) shutit_module_init(shutit) conn_target(shutit) errs = [] errs.extend(check_deps(shutit)) if cfg['action']['show_config']: # Show dependency graph digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' shutit.cfg['build']['depgraph'] = digraph digraph_all = 'digraph depgraph {\n' digraph_all = digraph_all + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() ]) digraph_all = digraph_all + '\n}' shutit.cfg['build']['depgraph_all'] = digraph_all shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True) shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) shutit.log('\n\n' + digraph, force_stdout=True) shutit.log('\n================================================================================\n' + digraph, force_stdout=True) shutit.log('\nAbove is the digraph for this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['show_config']: shutit.log(util.print_config(cfg, history=cfg['build']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True return # Check for conflicts now. errs.extend(check_conflicts(shutit)) errs.extend(check_ready(shutit)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_target(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a target ' + 'called ' + shutit.cfg['target']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/default.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh.\n\nThere\'s a ' + 'default test runner in test.sh\n\n' + 'You can inspect the details of the build in the target\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31') # Mark the build as completed shutit.cfg['build']['completed'] = True
def shutit_main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - list_configs - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') shutit = shutit_global.shutit cfg = shutit.cfg util.parse_args(shutit) if cfg['action']['skeleton']: util.create_skeleton(shutit) cfg['build']['completed'] = True return if cfg['action']['serve']: import shutit_srv cfg['build']['interactive'] = 0 revert_dir = os.getcwd() os.chdir(sys.path[0]) shutit_srv.start() os.chdir(revert_dir) return util.load_configs(shutit) # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if util.determine_interactive() and spawn.find_executable('shutit') is None: setup_shutit_path(cfg) util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'setup.py')) util.load_shutit_modules(shutit) if cfg['action']['list_modules']: util.list_modules(shutit) sys.exit(0) init_shutit_map(shutit) config_collection(shutit) conn_target(shutit) errs = [] errs.extend(check_deps(shutit)) if cfg['action']['list_deps']: # Show dependency graph digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in shutit.cfg and shutit.cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' f = file(cfg['build']['log_config_path'] + '/digraph.txt','w') f.write(digraph) f.close() digraph_all = 'digraph depgraph {\n' digraph_all = digraph_all + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() ]) digraph_all = digraph_all + '\n}' f = file(cfg['build']['log_config_path'] + '/digraph_all.txt','w') f.write(digraph_all) f.close() shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True) shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) shutit.log('\n\n' + digraph, force_stdout=True) shutit.log('\n================================================================================\n' + digraph, force_stdout=True) shutit.log('\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) # Exit now sys.exit(0) # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if cfg['action']['list_configs'] or cfg['build']['debug']: shutit.log(util.print_config(cfg, history=cfg['list_configs']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True f = file(cfg['build']['log_config_path'] + '/cfg.txt','w') f.write(util.print_config(cfg, history=cfg['list_configs']['cfghistory'])) f.close() shutit.log('================================================================================', force_stdout=True) shutit.log('Config details placed in: ' + cfg['build']['log_config_path'], force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('To render the digraph of this build into an image run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph.gv ' + cfg['build']['log_config_path'] + '/digraph.txt && dot -Tpdf -o digraph.pdf ' + cfg['build']['log_config_path'] + '/digraph.gv\n\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph_all.gv ' + cfg['build']['log_config_path'] + '/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' + cfg['build']['log_config_path'] + '/digraph_all.gv\n\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('\nConfiguration details have been written to the folder: ' + cfg['build']['log_config_path'] + '\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) if cfg['action']['list_configs']: return # Check for conflicts now. errs.extend(check_conflicts(shutit)) # Cache the results of check_ready at the start. errs.extend(check_ready(shutit, throw_error=False)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_target(shutit) shutit.log(util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='31') if cfg['build']['build_log']: shutit.cfg['build']['report_final_messages'] += "Build log file: " + cfg['host']['logfile'] # Show final report messages (ie messages to show after standard report). if shutit.cfg['build']['report_final_messages'] != '': shutit.log(shutit.cfg['build']['report_final_messages'], prefix=False, force_stdout=True, code='31') if shutit.cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a target ' + 'called ' + shutit.cfg['target']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/build.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh. These may need ' + 'tweaking for your particular environment, eg sudo\n\n' + 'You can inspect the details of the build in the target image\'s ' + '/root/shutit_build directory.', force_stdout=True, code='31') # Mark the build as completed shutit.cfg['build']['completed'] = True