def run(args, user, log_path):
"""Run the reimage for all the hosts in subproceesses."""
# Setup
phab_client = lib.get_phabricator_client()
lib.ensure_ipmi_password()
mgmts = lib.get_mgmts(args.hosts)
# Check that IPMI is working for all the hosts
for host in args.hosts:
lib.check_remote_ipmi(mgmts[host])
# Initialize data structures
procs = {}
retcodes = defaultdict(list)
# Validate hosts
if not args.new:
lib.validate_hosts(args.hosts, no_raise=args.no_verify)
# Update the Phabricator task
if args.phab_task_id is not None:
lib.phabricator_task_update(
phab_client, args.phab_task_id,
lib.PHAB_COMMENT_PRE.format(user=user,
hostname=socket.getfqdn(),
hosts=args.hosts,
log=log_path))
# Run the reimage for each host in a child process
try:
for host in args.hosts:
proc = reimage_host(host, mgmts[host], args)
if args.sequential:
retcodes[host] = proc.wait()
time.sleep(args.sleep)
else:
procs[host] = proc
if procs:
retcodes = wait_for_childrens(procs)
except KeyboardInterrupt:
# Terminate childrens
if procs:
for process in procs:
process.terminate()
else:
proc.terminate()
raise
# Comment on the Phabricator task
if args.phab_task_id is not None:
phabricator_message = lib.get_phabricator_post_message(retcodes)
lib.phabricator_task_update(phab_client, args.phab_task_id,
phabricator_message)
if max(retcodes.keys()) > 0:
return 1
return 0
def main():
"""Run the automated reimaging of a single host."""
# Setup
phab_client = None
args = parse_args()
lib.ensure_shell_mode()
user = lib.get_running_user()
log_path = setup_logging(user, args.host)
cumin_output_path = log_path.replace('.log', '_cumin.out')
if args.debug:
logger.setLevel(logging.DEBUG)
logger.info('wmf-auto-reimage-host called with args: {args}'.format(args=args))
lib.print_line('REIMAGE START | To monitor the full log and cumin output:', host=args.host)
lib.print_line('sudo tail -F {log}'.format(log=log_path), skip_time=True)
lib.print_line('sudo tail -F {log}'.format(log=cumin_output_path), skip_time=True)
try:
lib.ensure_ipmi_password()
lib.check_remote_ipmi(args.mgmt)
if args.rename_mgmt:
lib.check_remote_ipmi(args.rename_mgmt)
if args.phab_task_id is not None:
phab_client = lib.get_phabricator_client()
lib.phabricator_task_update(
phab_client, args.phab_task_id, lib.PHAB_COMMENT_PRE.format(
user=user, hostname=socket.getfqdn(), hosts=args.host, log=log_path))
try:
# This is needed due to a bug in tqdm and a limitation in Cumin
with open(cumin_output_path, 'w', 1) as cumin_output:
stderr = sys.stderr
stdout = sys.stdout
sys.stderr = cumin_output
sys.stdout = cumin_output
run(args, user, log_path)
retcode = 0
finally:
sys.stderr = stderr
sys.stdout = stdout
except BaseException as e:
message = 'Unable to run wmf-auto-reimage-host'
lib.print_line('{message}: {error}'.format(message=message, error=e), host=args.host)
logger.exception(message)
retcode = 2
finally:
lib.print_line('REIMAGE END | retcode={ret}'.format(ret=retcode), host=args.host)
# Comment on the Phabricator task
if args.phab_task_id is not None and phab_client is not None:
phabricator_message = lib.get_phabricator_post_message({retcode: [args.host]})
lib.phabricator_task_update(phab_client, args.phab_task_id, phabricator_message)
return retcode
def main():
"""Run the script."""
script_name = os.path.basename(__file__)
args = parse_args()
user = lib.get_running_user()
phab_client = lib.get_phabricator_client()
is_valid_host = lib.is_hostname_valid(args.host)
actions = []
if not is_valid_host and not args.force:
logger.error(
"{host} is not a valid hostname. Aborting.".format(host=args.host))
return 1
# Remove from Puppet and PuppetDB
lib.puppet_remove_host(args.host)
actions += ['Revoked Puppet certificate', 'Removed from PuppetDB']
# Downtime on Icinga both the host and the mgmt host, they will be removed by Puppet
if is_valid_host:
try:
lib.icinga_downtime(args.host,
user,
args.phab_task_id,
title=script_name)
actions.append('Downtimed host on Icinga')
except RuntimeError:
actions.append(
'Skipped downtime host on Icinga (likely already removed)')
mgmts = lib.get_mgmts([args.host])
try:
lib.icinga_downtime(mgmts[args.host],
user,
args.phab_task_id,
title=script_name)
actions.append('Downtimed mgmt interface on Icinga')
except RuntimeError:
actions.append(
'Skipped downtime mgmt interface on Icinga (likely already removed)'
)
# Remove from DebMonitor
lib.debmonitor_remove_host(args.host)
actions.append('Removed from DebMonitor')
message = (
'{script} was executed by {user} for {host} and performed the following actions:\n'
'- {actions}').format(script=script_name,
user=user,
host=args.host,
actions='\n- '.join(actions))
lib.phabricator_task_update(phab_client, args.phab_task_id, message)
return 0
