def run(args, user, log_path): """Run the reimage for all the hosts in subproceesses.""" # Setup phab_client = lib.get_phabricator_client() lib.ensure_ipmi_password() mgmts = lib.get_mgmts(args.hosts) # Check that IPMI is working for all the hosts for host in args.hosts: lib.check_remote_ipmi(mgmts[host]) # Initialize data structures procs = {} retcodes = defaultdict(list) # Validate hosts if not args.new: lib.validate_hosts(args.hosts, no_raise=args.no_verify) # Update the Phabricator task if args.phab_task_id is not None: lib.phabricator_task_update( phab_client, args.phab_task_id, lib.PHAB_COMMENT_PRE.format(user=user, hostname=socket.getfqdn(), hosts=args.hosts, log=log_path)) # Run the reimage for each host in a child process try: for host in args.hosts: proc = reimage_host(host, mgmts[host], args) if args.sequential: retcodes[host] = proc.wait() time.sleep(args.sleep) else: procs[host] = proc if procs: retcodes = wait_for_childrens(procs) except KeyboardInterrupt: # Terminate childrens if procs: for process in procs: process.terminate() else: proc.terminate() raise # Comment on the Phabricator task if args.phab_task_id is not None: phabricator_message = lib.get_phabricator_post_message(retcodes) lib.phabricator_task_update(phab_client, args.phab_task_id, phabricator_message) if max(retcodes.keys()) > 0: return 1 return 0
def main(): """Run the automated reimaging of a single host.""" # Setup phab_client = None args = parse_args() lib.ensure_shell_mode() user = lib.get_running_user() log_path = setup_logging(user, args.host) cumin_output_path = log_path.replace('.log', '_cumin.out') if args.debug: logger.setLevel(logging.DEBUG) logger.info('wmf-auto-reimage-host called with args: {args}'.format(args=args)) lib.print_line('REIMAGE START | To monitor the full log and cumin output:', host=args.host) lib.print_line('sudo tail -F {log}'.format(log=log_path), skip_time=True) lib.print_line('sudo tail -F {log}'.format(log=cumin_output_path), skip_time=True) try: lib.ensure_ipmi_password() lib.check_remote_ipmi(args.mgmt) if args.rename_mgmt: lib.check_remote_ipmi(args.rename_mgmt) if args.phab_task_id is not None: phab_client = lib.get_phabricator_client() lib.phabricator_task_update( phab_client, args.phab_task_id, lib.PHAB_COMMENT_PRE.format( user=user, hostname=socket.getfqdn(), hosts=args.host, log=log_path)) try: # This is needed due to a bug in tqdm and a limitation in Cumin with open(cumin_output_path, 'w', 1) as cumin_output: stderr = sys.stderr stdout = sys.stdout sys.stderr = cumin_output sys.stdout = cumin_output run(args, user, log_path) retcode = 0 finally: sys.stderr = stderr sys.stdout = stdout except BaseException as e: message = 'Unable to run wmf-auto-reimage-host' lib.print_line('{message}: {error}'.format(message=message, error=e), host=args.host) logger.exception(message) retcode = 2 finally: lib.print_line('REIMAGE END | retcode={ret}'.format(ret=retcode), host=args.host) # Comment on the Phabricator task if args.phab_task_id is not None and phab_client is not None: phabricator_message = lib.get_phabricator_post_message({retcode: [args.host]}) lib.phabricator_task_update(phab_client, args.phab_task_id, phabricator_message) return retcode
def main(): """Run the script.""" script_name = os.path.basename(__file__) args = parse_args() user = lib.get_running_user() phab_client = lib.get_phabricator_client() is_valid_host = lib.is_hostname_valid(args.host) actions = [] if not is_valid_host and not args.force: logger.error( "{host} is not a valid hostname. Aborting.".format(host=args.host)) return 1 # Remove from Puppet and PuppetDB lib.puppet_remove_host(args.host) actions += ['Revoked Puppet certificate', 'Removed from PuppetDB'] # Downtime on Icinga both the host and the mgmt host, they will be removed by Puppet if is_valid_host: try: lib.icinga_downtime(args.host, user, args.phab_task_id, title=script_name) actions.append('Downtimed host on Icinga') except RuntimeError: actions.append( 'Skipped downtime host on Icinga (likely already removed)') mgmts = lib.get_mgmts([args.host]) try: lib.icinga_downtime(mgmts[args.host], user, args.phab_task_id, title=script_name) actions.append('Downtimed mgmt interface on Icinga') except RuntimeError: actions.append( 'Skipped downtime mgmt interface on Icinga (likely already removed)' ) # Remove from DebMonitor lib.debmonitor_remove_host(args.host) actions.append('Removed from DebMonitor') message = ( '{script} was executed by {user} for {host} and performed the following actions:\n' '- {actions}').format(script=script_name, user=user, host=args.host, actions='\n- '.join(actions)) lib.phabricator_task_update(phab_client, args.phab_task_id, message) return 0