def build_module(shutit, module, loglevel=logging.DEBUG):
"""Build passed-in module.
"""
cfg = shutit.cfg
shutit.log('Building ShutIt module: ' + module.module_id + ' with run order: ' + str(module.run_order), level=logging.INFO)
cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding ShutIt module: ' + module.module_id + ' with run order: ' + str(module.run_order))
if not module.build(shutit):
shutit.fail(module.module_id + ' failed on build', shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id('target_child').pexpect_child)
else:
if cfg['build']['delivery'] in ('docker','dockerfile'):
# Create a directory and files to indicate this has been built.
shutit.send(' mkdir -p ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + ' && touch ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + '/built && rm -f ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + '/removed', loglevel=loglevel)
# Put it into "installed" cache
cfg['environment'][cfg['build']['current_environment_id']]['modules_installed'].append(module.module_id)
# Remove from "not installed" cache
if module.module_id in cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed']:
cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed'].remove(module.module_id)
shutit.pause_point('\nPausing to allow inspect of build for: ' + module.module_id, print_input=True, level=2)
cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' + module.module_id)
if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['interactive'] >= 3:
shutit.log(shutit_util.build_report(shutit, '#Module:' + module.module_id), level=logging.DEBUG)
if (not cfg[module.module_id]['shutit.core.module.tag'] and cfg['build']['interactive'] >= 2):
print ("\n\nDo you want to save state now we\'re at the " + "end of this module? (" + module.module_id + ") (input y/n)")
cfg[module.module_id]['shutit.core.module.tag'] = (shutit_util.util_raw_input(shutit=shutit,default='y') == 'y')
if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['tag_modules']:
shutit.log(module.module_id + ' configured to be tagged, doing repository work',level=logging.INFO)
# Stop all before we tag to avoid file changing errors, and clean up pid files etc..
stop_all(shutit, module.run_order)
shutit.do_repository_work(str(module.module_id) + '_' + str(module.run_order), password=cfg['host']['password'], docker_executable=cfg['host']['docker_executable'], force=True)
# Start all after we tag to ensure services are up as expected.
start_all(shutit, module.run_order)
if cfg['build']['interactive'] >= 2:
print ("\n\nDo you want to stop interactive mode? (input y/n)\n")
if shutit_util.util_raw_input(shutit=shutit,default='y') == 'y':
cfg['build']['interactive'] = 0
def main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- list_configs - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail("Python version must be 2.7+")
shutit = shutit_global.shutit
cfg = shutit.cfg
shutit_util.parse_args(shutit)
if cfg["action"]["skeleton"]:
shutit_util.create_skeleton(shutit)
cfg["build"]["completed"] = True
return
if cfg["action"]["serve"]:
import shutit_srv
cfg["build"]["interactive"] = 0
revert_dir = os.getcwd()
os.chdir(sys.path[0])
shutit_srv.start()
os.chdir(revert_dir)
return
shutit_util.load_configs(shutit)
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if shutit_util.determine_interactive() and spawn.find_executable("shutit") is None:
setup_shutit_path(cfg)
shutit_util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, "shutit_setup.py"))
shutit_util.load_shutit_modules(shutit)
if cfg["action"]["list_modules"]:
shutit_util.list_modules(shutit)
sys.exit(0)
init_shutit_map(shutit)
config_collection(shutit)
conn_target(shutit)
errs = []
errs.extend(check_deps(shutit))
if cfg["action"]["list_deps"]:
# Show dependency graph
digraph = "digraph depgraph {\n"
digraph = digraph + "\n".join(
[
make_dep_graph(module)
for module_id, module in shutit.shutit_map.items()
if module_id in cfg and cfg[module_id]["shutit.core.module.build"]
]
)
digraph = digraph + "\n}"
f = file(cfg["build"]["log_config_path"] + "/digraph.txt", "w")
f.write(digraph)
f.close()
digraph_all = "digraph depgraph {\n"
digraph_all = digraph_all + "\n".join(
[make_dep_graph(module) for module_id, module in shutit.shutit_map.items()]
)
digraph_all = digraph_all + "\n}"
f = file(cfg["build"]["log_config_path"] + "/digraph_all.txt", "w")
f.write(digraph_all)
f.close()
shutit.log(
"\n================================================================================\n" + digraph_all,
force_stdout=True,
)
shutit.log(
"\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n",
force_stdout=True,
)
shutit.log(
"\n================================================================================\n", force_stdout=True
)
shutit.log("\n\n" + digraph, force_stdout=True)
shutit.log(
"\n================================================================================\n" + digraph,
force_stdout=True,
)
shutit.log(
"\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n",
force_stdout=True,
)
shutit.log(
"\n================================================================================\n", force_stdout=True
)
# Exit now
sys.exit(0)
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg["action"]["list_configs"] or cfg["build"]["debug"]:
shutit.log(shutit_util.print_config(cfg, history=cfg["list_configs"]["cfghistory"]), force_stdout=True)
# Set build completed
cfg["build"]["completed"] = True
f = file(cfg["build"]["log_config_path"] + "/cfg.txt", "w")
f.write(shutit_util.print_config(cfg, history=cfg["list_configs"]["cfghistory"]))
f.close()
shutit.log(
"================================================================================", force_stdout=True
)
shutit.log("Config details placed in: " + cfg["build"]["log_config_path"], force_stdout=True)
shutit.log(
"================================================================================", force_stdout=True
)
shutit.log(
"To render the digraph of this build into an image run eg:\n\ndot -Tgv -o "
+ cfg["build"]["log_config_path"]
+ "/digraph.gv "
+ cfg["build"]["log_config_path"]
+ "/digraph.txt && dot -Tpdf -o digraph.pdf "
+ cfg["build"]["log_config_path"]
+ "/digraph.gv\n\n",
force_stdout=True,
)
shutit.log(
"================================================================================", force_stdout=True
)
shutit.log(
"To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o "
+ cfg["build"]["log_config_path"]
+ "/digraph_all.gv "
+ cfg["build"]["log_config_path"]
+ "/digraph_all.txt && dot -Tpdf -o digraph_all.pdf "
+ cfg["build"]["log_config_path"]
+ "/digraph_all.gv\n\n",
force_stdout=True,
)
shutit.log(
"================================================================================", force_stdout=True
)
shutit.log(
"\nConfiguration details have been written to the folder: " + cfg["build"]["log_config_path"] + "\n",
force_stdout=True,
)
shutit.log(
"================================================================================", force_stdout=True
)
if cfg["action"]["list_configs"]:
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(shutit, throw_error=False))
if errs:
shutit.log(print_modules(shutit), code="31")
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code="31")
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_target(shutit)
shutit.log(shutit_util.build_report(shutit, "#Module: N/A (END)"), prefix=False, force_stdout=True, code="32")
if cfg["build"]["build_log"]:
cfg["build"]["report_final_messages"] += "Build log file: " + cfg["host"]["logfile"]
# Show final report messages (ie messages to show after standard report).
if cfg["build"]["report_final_messages"] != "":
shutit.log(cfg["build"]["report_final_messages"], prefix=False, force_stdout=True, code="31")
if cfg["build"]["interactive"] >= 3:
shutit.log(
"\n"
+ "The build is complete. You should now have a target "
+ "called "
+ cfg["target"]["name"]
+ " and a new image if you chose to commit it.\n\n"
+ "Look and play with the following files from the newly-created "
+ "module directory to dig deeper:\n\n configs/build.cnf\n "
+ "*.py\n\nYou can rebuild at any time by running the supplied "
+ "./build.sh and run with the supplied ./run.sh. These may need "
+ "tweaking for your particular environment, eg sudo\n\n"
+ "You can inspect the details of the build in the target image's "
+ cfg["build"]["build_db_dir"]
+ " directory.",
force_stdout=True,
code="32",
)
# Mark the build as completed
cfg["build"]["completed"] = True
def build_module(shutit, module):
"""Build passed-in module.
"""
cfg = shutit.cfg
shutit.log("building: " + module.module_id + " with run order: " + str(module.run_order), code="32")
cfg["build"]["report"] = (
cfg["build"]["report"] + "\nBuilding: " + module.module_id + " with run order: " + str(module.run_order)
)
if not module.build(shutit):
shutit.fail(module.module_id + " failed on build", child=shutit.pexpect_children["target_child"])
else:
if cfg["build"]["delivery"] in ("docker", "dockerfile"):
# Create a directory and files to indicate this has been built.
shutit.send(
" mkdir -p "
+ cfg["build"]["build_db_dir"]
+ "/module_record/"
+ module.module_id
+ " && touch "
+ cfg["build"]["build_db_dir"]
+ "/module_record/"
+ module.module_id
+ "/built && rm -f "
+ cfg["build"]["build_db_dir"]
+ "/module_record/"
+ module.module_id
+ "/removed"
)
# Put it into "installed" cache
cfg["environment"][cfg["build"]["current_environment_id"]]["modules_installed"].append(module.module_id)
# Remove from "not installed" cache
if module.module_id in cfg["environment"][cfg["build"]["current_environment_id"]]["modules_not_installed"]:
cfg["environment"][cfg["build"]["current_environment_id"]]["modules_not_installed"].remove(module.module_id)
shutit.pause_point("\nPausing to allow inspect of build for: " + module.module_id, print_input=True, level=2)
cfg["build"]["report"] = cfg["build"]["report"] + "\nCompleted module: " + module.module_id
if cfg[module.module_id]["shutit.core.module.tag"] or cfg["build"]["interactive"] >= 3:
shutit.log(shutit_util.build_report(shutit, "#Module:" + module.module_id), code="32")
if not cfg[module.module_id]["shutit.core.module.tag"] and cfg["build"]["interactive"] >= 2:
shutit.log(
"\n\nDo you want to save state now we're at the "
+ "end of this module? ("
+ module.module_id
+ ") (input y/n)",
force_stdout=True,
code="32",
)
cfg[module.module_id]["shutit.core.module.tag"] = shutit_util.util_raw_input(shutit=shutit, default="y") == "y"
if cfg[module.module_id]["shutit.core.module.tag"] or cfg["build"]["tag_modules"]:
shutit.log(module.module_id + " configured to be tagged, doing repository work", force_stdout=True)
# Stop all before we tag to avoid file changing errors,
# and clean up pid files etc..
stop_all(shutit, module.run_order)
shutit.do_repository_work(
str(module.module_id) + "_" + str(module.run_order),
password=cfg["host"]["password"],
docker_executable=cfg["host"]["docker_executable"],
force=True,
)
# Start all after we tag to ensure services are up as expected.
start_all(shutit, module.run_order)
if cfg["build"]["interactive"] >= 2:
shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n", force_stdout=True, code="32")
if shutit_util.util_raw_input(shutit=shutit, default="y") == "y":
cfg["build"]["interactive"] = 0
def main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- list_configs - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
cfg = shutit.cfg
shutit_util.parse_args(shutit)
if cfg['action']['skeleton']:
shutit_util.create_skeleton(shutit)
cfg['build']['completed'] = True
return
if cfg['action']['serve']:
import shutit_srv
cfg['build']['interactive'] = 0
revert_dir = os.getcwd()
os.chdir(sys.path[0])
shutit_srv.start()
os.chdir(revert_dir)
return
shutit_util.load_configs(shutit)
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if shutit_util.determine_interactive() and spawn.find_executable('shutit') is None:
setup_shutit_path(cfg)
shutit_util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'shutit_setup.py'))
shutit_util.load_shutit_modules(shutit)
if cfg['action']['list_modules']:
shutit_util.list_modules(shutit)
sys.exit(0)
init_shutit_map(shutit)
config_collection(shutit)
conn_target(shutit)
errs = []
errs.extend(check_deps(shutit))
if cfg['action']['list_deps']:
# Show dependency graph
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
if module_id in cfg and cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
f = file(cfg['build']['log_config_path'] + '/digraph.txt','w')
f.write(digraph)
f.close()
digraph_all = 'digraph depgraph {\n'
digraph_all = digraph_all + '\n'.join([
make_dep_graph(module) for module_id, module in shutit.shutit_map.items()
])
digraph_all = digraph_all + '\n}'
f = file(cfg['build']['log_config_path'] + '/digraph_all.txt','w')
f.write(digraph_all)
f.close()
shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True)
shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
shutit.log('\n\n' + digraph, force_stdout=True)
shutit.log('\n================================================================================\n' + digraph, force_stdout=True)
shutit.log('\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m library | dot -Tpng -o depgraph.png\n', force_stdout=True)
shutit.log('\n================================================================================\n', force_stdout=True)
# Exit now
sys.exit(0)
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['list_configs'] or cfg['build']['debug']:
shutit.log(shutit_util.print_config(cfg, history=cfg['list_configs']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
f = file(cfg['build']['log_config_path'] + '/cfg.txt','w')
f.write(shutit_util.print_config(cfg, history=cfg['list_configs']['cfghistory']))
f.close()
shutit.log('================================================================================', force_stdout=True)
shutit.log('Config details placed in: ' + cfg['build']['log_config_path'], force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('To render the digraph of this build into an image run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph.gv ' + cfg['build']['log_config_path'] + '/digraph.txt && dot -Tpdf -o digraph.pdf ' + cfg['build']['log_config_path'] + '/digraph.gv\n\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph_all.gv ' + cfg['build']['log_config_path'] + '/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' + cfg['build']['log_config_path'] + '/digraph_all.gv\n\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
shutit.log('\nConfiguration details have been written to the folder: ' + cfg['build']['log_config_path'] + '\n', force_stdout=True)
shutit.log('================================================================================', force_stdout=True)
if cfg['action']['list_configs']:
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(shutit, throw_error=False))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_target(shutit)
shutit.log(shutit_util.build_report(shutit, '#Module: N/A (END)'), prefix=False,
force_stdout=True, code='32')
if cfg['build']['build_log']:
cfg['build']['report_final_messages'] += "Build log file: " + cfg['host']['logfile']
# Show final report messages (ie messages to show after standard report).
if cfg['build']['report_final_messages'] != '':
shutit.log(cfg['build']['report_final_messages'], prefix=False,
force_stdout=True, code='31')
if cfg['build']['interactive'] >= 3:
shutit.log('\n' +
'The build is complete. You should now have a target ' +
'called ' + cfg['target']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/build.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh. These may need ' +
'tweaking for your particular environment, eg sudo\n\n' +
'You can inspect the details of the build in the target image\'s ' +
cfg['build']['build_db_dir'] + ' directory.', force_stdout=True, code='32')
# Mark the build as completed
cfg['build']['completed'] = True
def main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- list_configs - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
cfg = shutit.cfg
shutit.log('# ShutIt Started... ',transient=True)
shutit_util.parse_args()
shutit.log('# Loading configs...',transient=True)
shutit_util.load_configs()
if shutit.action['skeleton']:
shutit_skeleton.create_skeleton()
shutit.build['completed'] = True
return
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if shutit_util.determine_interactive() and spawn.find_executable('shutit') is None:
setup_shutit_path()
shutit_util.load_mod_from_file(os.path.join(shutit.shutit_main_dir, 'shutit_setup.py'))
shutit_util.load_shutit_modules()
shutit.log('ShutIt modules loaded',level=logging.INFO)
init_shutit_map(shutit)
shutit_util.config_collection()
shutit.log('Configuration loaded',level=logging.INFO)
if shutit.action['list_modules']:
shutit_util.list_modules()
shutit_util.handle_exit()
conn_target(shutit)
shutit.log('Connected to target',level=logging.INFO)
if shutit.build['interactive'] > 0 and shutit.build['choose_config']:
errs = do_interactive_modules()
else:
errs = []
errs.extend(check_deps())
if shutit.action['list_deps']:
# Show dependency graph
digraph = 'digraph depgraph {\n'
digraph += '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in cfg and cfg[module_id]['shutit.core.module.build'] ])
digraph += '\n}'
f = file(shutit.build['log_config_path'] + '/digraph.txt','w')
f.write(digraph)
f.close()
digraph_all = 'digraph depgraph {\n'
digraph_all += '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() ])
digraph_all += '\n}'
f = file(shutit.build['log_config_path'] + '/digraph_all.txt','w')
f.write(digraph_all)
f.close()
shutit.log('\n================================================================================\n' + digraph_all)
shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n')
shutit.log('\n================================================================================\n')
shutit.log('\n\n' + digraph)
shutit.log('\n================================================================================\n' + digraph)
shutit.log('\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n')
shutit.log('\n================================================================================\n')
# Exit now
shutit_util.handle_exit()
# Dependency validation done, now collect configs of those marked for build.
shutit_util.config_collection_for_built()
if shutit.action['list_configs'] or shutit.build['loglevel'] <= logging.DEBUG:
# Set build completed
shutit.build['completed'] = True
shutit.log('================================================================================')
shutit.log('Config details placed in: ' + shutit.build['log_config_path'])
shutit.log('================================================================================')
shutit.log('To render the digraph of this build into an image run eg:\n\ndot -Tgv -o ' + shutit.build['log_config_path'] + '/digraph.gv ' + shutit.build['log_config_path'] + '/digraph.txt && dot -Tpdf -o digraph.pdf ' + shutit.build['log_config_path'] + '/digraph.gv\n\n')
shutit.log('================================================================================')
shutit.log('To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o ' + shutit.build['log_config_path'] + '/digraph_all.gv ' + shutit.build['log_config_path'] + '/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' + shutit.build['log_config_path'] + '/digraph_all.gv\n\n')
shutit.log('================================================================================')
shutit.log('\nConfiguration details have been written to the folder: ' + shutit.build['log_config_path'] + '\n')
shutit.log('================================================================================')
if shutit.action['list_configs']:
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(throw_error=False))
if errs:
shutit.log(shutit_util.print_modules(), level=logging.ERROR)
child = None
for err in errs:
shutit.log(err[0], level=logging.ERROR)
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", shutit_pexpect_child=child)
do_remove()
do_build()
do_test()
do_finalize()
finalize_target()
shutit.log(shutit_util.build_report('#Module: N/A (END)'), level=logging.DEBUG)
# Show final report messages (ie messages to show after standard report).
if shutit.build['report_final_messages'] != '':
shutit.log(shutit.build['report_final_messages'], level=logging.INFO)
# Mark the build as completed
shutit.build['completed'] = True
shutit.log('ShutIt run finished',level=logging.INFO)
shutit_util.handle_exit(exit_code=0)
def main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- serve - run as a server
- list_configs - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
cfg = shutit.cfg
shutit_util.parse_args(shutit)
if cfg['action']['skeleton']:
shutit_util.create_skeleton(shutit)
cfg['build']['completed'] = True
return
if cfg['action']['serve']:
import shutit_srv
cfg['build']['interactive'] = 0
revert_dir = os.getcwd()
os.chdir(sys.path[0])
shutit_srv.start()
os.chdir(revert_dir)
return
shutit_util.load_configs(shutit)
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if shutit_util.determine_interactive(
) and spawn.find_executable('shutit') is None:
setup_shutit_path(cfg)
shutit_util.load_mod_from_file(
shutit, os.path.join(shutit.shutit_main_dir, 'shutit_setup.py'))
shutit_util.load_shutit_modules(shutit)
if cfg['action']['list_modules']:
shutit_util.list_modules(shutit)
sys.exit(0)
init_shutit_map(shutit)
config_collection(shutit)
conn_target(shutit)
errs = []
errs.extend(check_deps(shutit))
if cfg['action']['list_deps']:
# Show dependency graph
digraph = 'digraph depgraph {\n'
digraph = digraph + '\n'.join([
make_dep_graph(module)
for module_id, module in shutit.shutit_map.items()
if module_id in cfg and cfg[module_id]['shutit.core.module.build']
])
digraph = digraph + '\n}'
f = file(cfg['build']['log_config_path'] + '/digraph.txt', 'w')
f.write(digraph)
f.close()
digraph_all = 'digraph depgraph {\n'
digraph_all = digraph_all + '\n'.join([
make_dep_graph(module)
for module_id, module in shutit.shutit_map.items()
])
digraph_all = digraph_all + '\n}'
f = file(cfg['build']['log_config_path'] + '/digraph_all.txt', 'w')
f.write(digraph_all)
f.close()
shutit.log(
'\n================================================================================\n'
+ digraph_all,
force_stdout=True)
shutit.log(
'\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n',
force_stdout=True)
shutit.log(
'\n================================================================================\n',
force_stdout=True)
shutit.log('\n\n' + digraph, force_stdout=True)
shutit.log(
'\n================================================================================\n'
+ digraph,
force_stdout=True)
shutit.log(
'\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n',
force_stdout=True)
shutit.log(
'\n================================================================================\n',
force_stdout=True)
# Exit now
sys.exit(0)
# Dependency validation done, now collect configs of those marked for build.
config_collection_for_built(shutit)
if cfg['action']['list_configs'] or cfg['build']['debug']:
shutit.log(shutit_util.print_config(
cfg, history=cfg['list_configs']['cfghistory']),
force_stdout=True)
# Set build completed
cfg['build']['completed'] = True
f = file(cfg['build']['log_config_path'] + '/cfg.txt', 'w')
f.write(
shutit_util.print_config(
cfg, history=cfg['list_configs']['cfghistory']))
f.close()
shutit.log(
'================================================================================',
force_stdout=True)
shutit.log('Config details placed in: ' +
cfg['build']['log_config_path'],
force_stdout=True)
shutit.log(
'================================================================================',
force_stdout=True)
shutit.log(
'To render the digraph of this build into an image run eg:\n\ndot -Tgv -o '
+ cfg['build']['log_config_path'] + '/digraph.gv ' +
cfg['build']['log_config_path'] +
'/digraph.txt && dot -Tpdf -o digraph.pdf ' +
cfg['build']['log_config_path'] + '/digraph.gv\n\n',
force_stdout=True)
shutit.log(
'================================================================================',
force_stdout=True)
shutit.log(
'To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o '
+ cfg['build']['log_config_path'] + '/digraph_all.gv ' +
cfg['build']['log_config_path'] +
'/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' +
cfg['build']['log_config_path'] + '/digraph_all.gv\n\n',
force_stdout=True)
shutit.log(
'================================================================================',
force_stdout=True)
shutit.log(
'\nConfiguration details have been written to the folder: ' +
cfg['build']['log_config_path'] + '\n',
force_stdout=True)
shutit.log(
'================================================================================',
force_stdout=True)
if cfg['action']['list_configs']:
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(shutit, throw_error=False))
if errs:
shutit.log(print_modules(shutit), code='31')
child = None
for err in errs:
shutit.log(err[0], force_stdout=True, code='31')
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting", child=child)
shutit.record_config()
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_target(shutit)
shutit.log(shutit_util.build_report(shutit, '#Module: N/A (END)'),
prefix=False,
force_stdout=True,
code='32')
if cfg['build']['build_log']:
cfg['build']['report_final_messages'] += "Build log file: " + cfg[
'host']['logfile']
# Show final report messages (ie messages to show after standard report).
if cfg['build']['report_final_messages'] != '':
shutit.log(cfg['build']['report_final_messages'],
prefix=False,
force_stdout=True,
code='31')
if cfg['build']['interactive'] >= 3:
shutit.log(
'\n' + 'The build is complete. You should now have a target ' +
'called ' + cfg['target']['name'] +
' and a new image if you chose to commit it.\n\n' +
'Look and play with the following files from the newly-created ' +
'module directory to dig deeper:\n\n configs/build.cnf\n ' +
'*.py\n\nYou can rebuild at any time by running the supplied ' +
'./build.sh and run with the supplied ./run.sh. These may need ' +
'tweaking for your particular environment, eg sudo\n\n' +
'You can inspect the details of the build in the target image\'s '
+ cfg['build']['build_db_dir'] + ' directory.',
force_stdout=True,
code='32')
# Mark the build as completed
cfg['build']['completed'] = True
def build_module(shutit, module):
"""Build passed-in module.
"""
cfg = shutit.cfg
shutit.log('building: ' + module.module_id + ' with run order: ' +
str(module.run_order),
code='32')
cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding: ' +
module.module_id + ' with run order: ' +
str(module.run_order))
if not module.build(shutit):
shutit.fail(module.module_id + ' failed on build',
child=shutit.pexpect_children['target_child'])
else:
if cfg['build']['delivery'] in ('docker', 'dockerfile'):
# Create a directory and files to indicate this has been built.
shutit.send(' mkdir -p ' + cfg['build']['build_db_dir'] +
'/module_record/' + module.module_id + ' && touch ' +
cfg['build']['build_db_dir'] + '/module_record/' +
module.module_id + '/built && rm -f ' +
cfg['build']['build_db_dir'] + '/module_record/' +
module.module_id + '/removed')
# Put it into "installed" cache
cfg['environment'][cfg['build']['current_environment_id']][
'modules_installed'].append(module.module_id)
# Remove from "not installed" cache
if module.module_id in cfg['environment'][cfg['build'][
'current_environment_id']]['modules_not_installed']:
cfg['environment'][cfg['build']['current_environment_id']][
'modules_not_installed'].remove(module.module_id)
shutit.pause_point('\nPausing to allow inspect of build for: ' +
module.module_id,
print_input=True,
level=2)
cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' +
module.module_id)
if cfg[module.module_id][
'shutit.core.module.tag'] or cfg['build']['interactive'] >= 3:
shutit.log(shutit_util.build_report(shutit,
'#Module:' + module.module_id),
code='32')
if (not cfg[module.module_id]['shutit.core.module.tag']
and cfg['build']['interactive'] >= 2):
shutit.log("\n\nDo you want to save state now we\'re at the " +
"end of this module? (" + module.module_id +
") (input y/n)",
force_stdout=True,
code='32')
cfg[module.module_id]['shutit.core.module.tag'] = (
shutit_util.util_raw_input(shutit=shutit, default='y') == 'y')
if cfg[module.
module_id]['shutit.core.module.tag'] or cfg['build']['tag_modules']:
shutit.log(module.module_id +
' configured to be tagged, doing repository work',
force_stdout=True)
# Stop all before we tag to avoid file changing errors,
# and clean up pid files etc..
stop_all(shutit, module.run_order)
shutit.do_repository_work(
str(module.module_id) + '_' + str(module.run_order),
password=cfg['host']['password'],
docker_executable=cfg['host']['docker_executable'],
force=True)
# Start all after we tag to ensure services are up as expected.
start_all(shutit, module.run_order)
if cfg['build']['interactive'] >= 2:
shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n",
force_stdout=True,
code='32')
if shutit_util.util_raw_input(shutit=shutit, default='y') == 'y':
cfg['build']['interactive'] = 0
def main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- list_configs - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
cfg = shutit.cfg
shutit.log('ShutIt Started... ', transient=True, newline=False)
shutit_util.parse_args()
if shutit.action['skeleton']:
shutit_util.create_skeleton()
shutit.build['completed'] = True
return
shutit.log('Loading configs...', transient=True)
shutit_util.load_configs()
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if shutit_util.determine_interactive(
) and spawn.find_executable('shutit') is None:
setup_shutit_path(cfg)
shutit_util.load_mod_from_file(
os.path.join(shutit.shutit_main_dir, 'shutit_setup.py'))
shutit_util.load_shutit_modules()
shutit.log('ShutIt modules loaded', level=logging.INFO)
init_shutit_map(shutit)
shutit_util.config_collection()
shutit.log('Configuration loaded', level=logging.INFO)
if shutit.action['list_modules']:
shutit_util.list_modules()
shutit_util.handle_exit()
conn_target(shutit)
shutit.log('Connected to target', level=logging.INFO)
if shutit.build['interactive'] > 0 and shutit.build['choose_config']:
errs = do_interactive_modules()
else:
errs = []
errs.extend(check_deps())
if shutit.action['list_deps']:
# Show dependency graph
digraph = 'digraph depgraph {\n'
digraph += '\n'.join([
make_dep_graph(module)
for module_id, module in shutit.shutit_map.items()
if module_id in cfg and cfg[module_id]['shutit.core.module.build']
])
digraph += '\n}'
f = file(shutit.build['log_config_path'] + '/digraph.txt', 'w')
f.write(digraph)
f.close()
digraph_all = 'digraph depgraph {\n'
digraph_all += '\n'.join([
make_dep_graph(module)
for module_id, module in shutit.shutit_map.items()
])
digraph_all += '\n}'
f = file(shutit.build['log_config_path'] + '/digraph_all.txt', 'w')
f.write(digraph_all)
f.close()
shutit.log(
'\n================================================================================\n'
+ digraph_all)
shutit.log(
'\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n'
)
shutit.log(
'\n================================================================================\n'
)
shutit.log('\n\n' + digraph)
shutit.log(
'\n================================================================================\n'
+ digraph)
shutit.log(
'\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n'
)
shutit.log(
'\n================================================================================\n'
)
# Exit now
shutit_util.handle_exit()
# Dependency validation done, now collect configs of those marked for build.
shutit_util.config_collection_for_built()
if shutit.action[
'list_configs'] or shutit.build['loglevel'] <= logging.DEBUG:
# Set build completed
shutit.build['completed'] = True
shutit.log(
'================================================================================'
)
shutit.log('Config details placed in: ' +
shutit.build['log_config_path'])
shutit.log(
'================================================================================'
)
shutit.log(
'To render the digraph of this build into an image run eg:\n\ndot -Tgv -o '
+ shutit.build['log_config_path'] + '/digraph.gv ' +
shutit.build['log_config_path'] +
'/digraph.txt && dot -Tpdf -o digraph.pdf ' +
shutit.build['log_config_path'] + '/digraph.gv\n\n')
shutit.log(
'================================================================================'
)
shutit.log(
'To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o '
+ shutit.build['log_config_path'] + '/digraph_all.gv ' +
shutit.build['log_config_path'] +
'/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' +
shutit.build['log_config_path'] + '/digraph_all.gv\n\n')
shutit.log(
'================================================================================'
)
shutit.log(
'\nConfiguration details have been written to the folder: ' +
shutit.build['log_config_path'] + '\n')
shutit.log(
'================================================================================'
)
if shutit.action['list_configs']:
return
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(throw_error=False))
if errs:
shutit.log(shutit_util.print_modules(), level=logging.ERROR)
child = None
for err in errs:
shutit.log(err[0], level=logging.ERROR)
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting",
shutit_pexpect_child=child)
do_remove()
do_build()
do_test()
do_finalize()
finalize_target()
shutit.log(shutit_util.build_report('#Module: N/A (END)'),
level=logging.DEBUG)
# Show final report messages (ie messages to show after standard report).
if shutit.build['report_final_messages'] != '':
shutit.log(shutit.build['report_final_messages'], level=logging.INFO)
if shutit.build['interactive'] >= 3:
shutit.log(
'\n' +
'The build is complete. You should now have a target called ' +
shutit.target['name'] +
' and a new image if you chose to commit it.\n\nLook and play with the following files from the newly-created module directory to dig deeper:\n\n configs/build.cnf\n *.py\n\nYou can rebuild at any time by running the supplied ./build.sh and run with the supplied ./run.sh. These may need tweaking for your particular environment, eg sudo',
level=logging.DEBUG)
# Mark the build as completed
shutit.build['completed'] = True
shutit.log('ShutIt run finished', level=logging.INFO)
shutit_util.handle_exit(0)
def build_module(module, loglevel=logging.DEBUG):
"""Build passed-in module.
"""
shutit = shutit_global.shutit
cfg = shutit.cfg
shutit.log('Building ShutIt module: ' + module.module_id +
' with run order: ' + str(module.run_order),
level=logging.INFO)
shutit.build['report'] = (shutit.build['report'] +
'\nBuilding ShutIt module: ' + module.module_id +
' with run order: ' + str(module.run_order))
if not module.build(shutit):
shutit.fail(
module.module_id + ' failed on build',
shutit_pexpect_child=shutit.get_shutit_pexpect_session_from_id(
'target_child').pexpect_child)
else:
if shutit.build['delivery'] in ('docker', 'dockerfile'):
# Create a directory and files to indicate this has been built.
shutit.send(' mkdir -p ' + shutit.build['build_db_dir'] +
'/module_record/' + module.module_id + ' && touch ' +
shutit.build['build_db_dir'] + '/module_record/' +
module.module_id + '/built && rm -f ' +
shutit.build['build_db_dir'] + '/module_record/' +
module.module_id + '/removed',
loglevel=loglevel)
# Put it into "installed" cache
shutit_global.shutit.get_current_shutit_pexpect_session_environment(
).modules_installed.append(module.module_id)
# Remove from "not installed" cache
if module.module_id in shutit_global.shutit.get_current_shutit_pexpect_session_environment(
).modules_not_installed:
shutit_global.shutit.get_current_shutit_pexpect_session_environment(
).modules_not_installed.remove(module.module_id)
shutit.pause_point('\nPausing to allow inspect of build for: ' +
module.module_id,
print_input=True,
level=2)
shutit.build['report'] = (shutit.build['report'] + '\nCompleted module: ' +
module.module_id)
if cfg[module.module_id][
'shutit.core.module.tag'] or shutit.build['interactive'] >= 3:
shutit.log(shutit_util.build_report('#Module:' + module.module_id),
level=logging.DEBUG)
if (not cfg[module.module_id]['shutit.core.module.tag']
and shutit.build['interactive'] >= 2):
print("\n\nDo you want to save state now we\'re at the " +
"end of this module? (" + module.module_id + ") (input y/n)")
cfg[module.module_id]['shutit.core.module.tag'] = (
shutit_util.util_raw_input(default='y') == 'y')
if cfg[module.
module_id]['shutit.core.module.tag'] or shutit.build['tag_modules']:
shutit.log(module.module_id +
' configured to be tagged, doing repository work',
level=logging.INFO)
# Stop all before we tag to avoid file changing errors, and clean up pid files etc..
stop_all(module.run_order)
shutit.do_repository_work(
str(module.module_id) + '_' + str(module.run_order),
password=shutit.host['password'],
docker_executable=shutit.host['docker_executable'],
force=True)
# Start all after we tag to ensure services are up as expected.
start_all(module.run_order)
if shutit.build['interactive'] >= 2:
print("\n\nDo you want to stop interactive mode? (input y/n)\n")
if shutit_util.util_raw_input(default='y') == 'y':
shutit.build['interactive'] = 0
def setup_shutit_obj(shutit):
shutit_util.parse_args(shutit)
if not shutit.build['exam']:
shutit.log('# ShutIt Started... ', transient=True)
shutit.log('# Loading configs...', transient=True)
shutit_util.load_configs(shutit)
if shutit.action['skeleton']:
shutit_skeleton.create_skeleton(shutit)
shutit.build['completed'] = True
return
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if shutit_util.determine_interactive(
shutit) and spawn.find_executable('shutit') is None:
setup_shutit_path(shutit)
shutit_util.load_mod_from_file(
shutit, os.path.join(shutit.shutit_main_dir, 'shutit_setup.py'))
shutit_util.load_shutit_modules(shutit)
shutit.log('ShutIt modules loaded', level=logging.INFO)
init_shutit_map(shutit)
shutit_util.config_collection(shutit=shutit)
shutit.log('Configuration loaded', level=logging.INFO)
if shutit.action['list_modules']:
shutit_util.list_modules(shutit)
shutit_util.handle_exit(shutit=shutit)
if not shutit.action['list_deps'] and not shutit.action['list_modules']:
conn_target(shutit)
shutit.log('Connected to target', level=logging.INFO)
if shutit.build['interactive'] > 0 and shutit.build['choose_config']:
errs = do_interactive_modules(shutit)
else:
errs = []
errs.extend(check_deps(shutit))
do_lists(shutit)
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(shutit, throw_error=False))
if errs:
shutit.log(shutit_util.print_modules(shutit), level=logging.ERROR)
child = None
for err in errs:
shutit.log(err[0], level=logging.ERROR)
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting",
shutit_pexpect_child=child) # pragma: no cover
do_remove(shutit)
do_build(shutit)
do_test(shutit)
do_finalize(shutit)
finalize_target(shutit)
shutit.log(shutit_util.build_report(shutit, '#Module: N/A (END)'),
level=logging.DEBUG)
do_exam_output(shutit)
shutit_global.shutit_global_object.do_final_messages()
# Mark the build as completed
shutit.build['completed'] = True
shutit.log('ShutIt run finished', level=logging.INFO)
shutit_util.handle_exit(shutit=shutit, exit_code=0)
def main():
"""Main ShutIt function.
Handles the configured actions:
- skeleton - create skeleton module
- list_configs - output computed configuration
- depgraph - output digraph of module dependencies
"""
if sys.version_info.major == 2:
if sys.version_info.minor < 7:
shutit_global.shutit.fail('Python version must be 2.7+')
shutit = shutit_global.shutit
shutit_util.parse_args()
if not shutit.build['exam']:
shutit.log('# ShutIt Started... ', transient=True)
shutit.log('# Loading configs...', transient=True)
shutit_util.load_configs()
if shutit.action['skeleton']:
shutit_skeleton.create_skeleton()
shutit.build['completed'] = True
return
# Try and ensure shutit is on the path - makes onboarding easier
# Only do this if we're in a terminal
if shutit_util.determine_interactive(
) and spawn.find_executable('shutit') is None:
setup_shutit_path()
shutit_util.load_mod_from_file(
os.path.join(shutit.shutit_main_dir, 'shutit_setup.py'))
shutit_util.load_shutit_modules()
shutit.log('ShutIt modules loaded', level=logging.INFO)
init_shutit_map(shutit)
shutit_util.config_collection()
shutit.log('Configuration loaded', level=logging.INFO)
if shutit.action['list_modules']:
shutit_util.list_modules()
shutit_util.handle_exit()
if not shutit.action['list_deps'] and not shutit.action['list_modules']:
conn_target(shutit)
shutit.log('Connected to target', level=logging.INFO)
if shutit.build['interactive'] > 0 and shutit.build['choose_config']:
errs = do_interactive_modules()
else:
errs = []
errs.extend(check_deps())
do_lists(shutit)
# Check for conflicts now.
errs.extend(check_conflicts(shutit))
# Cache the results of check_ready at the start.
errs.extend(check_ready(throw_error=False))
if errs:
shutit.log(shutit_util.print_modules(), level=logging.ERROR)
child = None
for err in errs:
shutit.log(err[0], level=logging.ERROR)
if not child and len(err) > 1:
child = err[1]
shutit.fail("Encountered some errors, quitting",
shutit_pexpect_child=child)
do_remove()
do_build()
do_test()
do_finalize()
finalize_target()
shutit.log(shutit_util.build_report('#Module: N/A (END)'),
level=logging.DEBUG)
do_final_messages(shutit)
do_exam_output(shutit)
# Mark the build as completed
shutit.build['completed'] = True
shutit.log('ShutIt run finished', level=logging.INFO)
shutit_util.handle_exit(exit_code=0)
