def commandline(cls, args): """Configure remote system""" if (args.parallel and args.parallel != 1) or args.jobs: if args.timestamp: cdist.log.setupTimestampingParallelLogging() else: cdist.log.setupParallelLogging() elif args.timestamp: cdist.log.setupTimestampingLogging() log = logging.getLogger("config") # No new child process if only one host at a time. if args.parallel == 1: log.debug("Only 1 parallel process, doing it sequentially") args.parallel = 0 if args.parallel: import signal signal.signal(signal.SIGTERM, mp_sig_handler) signal.signal(signal.SIGHUP, mp_sig_handler) cls._check_and_prepare_args(args) failed_hosts = [] time_start = time.time() cls.construct_remote_exec_copy_patterns(args) base_root_path = cls.create_base_root_path(args.out_path) hostcnt = 0 cfg = cdist.configuration.Configuration(args) configuration = cfg.get_config(section='GLOBAL') if args.tag or args.all_tagged_hosts: inventory.determine_default_inventory_dir(args, configuration) if args.all_tagged_hosts: inv_list = inventory.InventoryList( hosts=None, istag=True, hostfile=None, db_basedir=args.inventory_dir) else: inv_list = inventory.InventoryList( hosts=args.host, istag=True, hostfile=args.hostfile, db_basedir=args.inventory_dir, has_all_tags=args.has_all_tags) it = inv_list.entries() else: it = itertools.chain(cls.hosts(args.host), cls.hosts(args.hostfile)) process_args = [] if args.parallel: log.trace("Processing hosts in parallel") else: log.trace("Processing hosts sequentially") for entry in it: if isinstance(entry, tuple): # if configuring by specified tags host = entry[0] host_tags = entry[1] else: # if configuring by host then check inventory for tags host = entry inventory.determine_default_inventory_dir(args, configuration) inv_list = inventory.InventoryList( hosts=(host, ), db_basedir=args.inventory_dir) inv = tuple(inv_list.entries()) if inv: # host is present in inventory and has tags host_tags = inv[0][1] else: # host is not present in inventory or has no tags host_tags = None host_base_path, hostdir = cls.create_host_base_dirs( host, base_root_path) log.debug("Base root path for target host \"{}\" is \"{}\"".format( host, host_base_path)) hostcnt += 1 if args.parallel: pargs = (host, host_tags, host_base_path, hostdir, args, True, configuration) log.trace(("Args for multiprocessing operation " "for host {}: {}".format(host, pargs))) process_args.append(pargs) else: try: cls.onehost(host, host_tags, host_base_path, hostdir, args, parallel=False, configuration=configuration) except cdist.Error as e: failed_hosts.append(host) if args.parallel and len(process_args) == 1: log.debug("Only 1 host for parallel processing, doing it " "sequentially") try: cls.onehost(*process_args[0]) except cdist.Error as e: failed_hosts.append(host) elif args.parallel: log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) log.trace(("Starting multiprocessing Pool for {} " "parallel host operation".format(args.parallel))) results = mp_pool_run(cls.onehost, process_args, jobs=args.parallel) log.trace(("Multiprocessing for parallel host operation " "finished")) log.trace( "Multiprocessing for parallel host operation " "results: %s", results) failed_hosts = [host for host, result in results if not result] time_end = time.time() log.verbose("Total processing time for %s host(s): %s", hostcnt, (time_end - time_start)) if len(failed_hosts) > 0: raise cdist.Error("Failed to configure the following hosts: " + " ".join(failed_hosts)) elif not args.out_path: # If tmp out path created then remove it, but only if no failed # hosts. shutil.rmtree(base_root_path)
def commandline(cls, args): """Configure remote system""" # FIXME: Refactor relict - remove later log = logging.getLogger("cdist") # No new child process if only one host at a time. if args.parallel == 1: log.debug("Only 1 parallel process, doing it sequentially") args.parallel = 0 if args.parallel or args.jobs: # If parallel execution then also log process id cdist.log.setupParallelLogging() log = logging.getLogger("cdist") if args.parallel: import signal signal.signal(signal.SIGTERM, mp_sig_handler) signal.signal(signal.SIGHUP, mp_sig_handler) cls._check_and_prepare_args(args) failed_hosts = [] time_start = time.time() cls.construct_remote_exec_copy_patterns(args) base_root_path = cls.create_base_root_path(args.out_path) hostcnt = 0 cfg = cdist.configuration.Configuration(args) configuration = cfg.get_config(section='GLOBAL') if args.tag or args.all_tagged_hosts: inventory.determine_default_inventory_dir(args, configuration) if args.all_tagged_hosts: inv_list = inventory.InventoryList( hosts=None, istag=True, hostfile=None, db_basedir=args.inventory_dir) else: inv_list = inventory.InventoryList( hosts=args.host, istag=True, hostfile=args.hostfile, db_basedir=args.inventory_dir, has_all_tags=args.has_all_tags) it = inv_list.entries() else: it = itertools.chain(cls.hosts(args.host), cls.hosts(args.hostfile)) process_args = [] if args.parallel: log.trace("Processing hosts in parallel") else: log.trace("Processing hosts sequentially") for entry in it: if isinstance(entry, tuple): # if configuring by specified tags host = entry[0] host_tags = entry[1] else: # if configuring by host then check inventory for tags host = entry inventory.determine_default_inventory_dir(args, configuration) inv_list = inventory.InventoryList( hosts=(host,), db_basedir=args.inventory_dir) inv = tuple(inv_list.entries()) if inv: # host is present in inventory and has tags host_tags = inv[0][1] else: # host is not present in inventory or has no tags host_tags = None host_base_path, hostdir = cls.create_host_base_dirs( host, base_root_path) log.debug("Base root path for target host \"{}\" is \"{}\"".format( host, host_base_path)) hostcnt += 1 if args.parallel: pargs = (host, host_tags, host_base_path, hostdir, args, True, configuration) log.trace(("Args for multiprocessing operation " "for host {}: {}".format(host, pargs))) process_args.append(pargs) else: try: cls.onehost(host, host_tags, host_base_path, hostdir, args, parallel=False, configuration=configuration) except cdist.Error as e: failed_hosts.append(host) if args.parallel and len(process_args) == 1: log.debug("Only 1 host for parallel processing, doing it " "sequentially") try: cls.onehost(*process_args[0]) except cdist.Error as e: failed_hosts.append(host) elif args.parallel: log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) log.trace(("Starting multiprocessing Pool for {} " "parallel host operation".format(args.parallel))) results = mp_pool_run(cls.onehost, process_args, jobs=args.parallel) log.trace(("Multiprocessing for parallel host operation " "finished")) log.trace("Multiprocessing for parallel host operation " "results: %s", results) failed_hosts = [host for host, result in results if not result] time_end = time.time() log.verbose("Total processing time for %s host(s): %s", hostcnt, (time_end - time_start)) if len(failed_hosts) > 0: raise cdist.Error("Failed to configure the following hosts: " + " ".join(failed_hosts)) elif not args.out_path: # If tmp out path created then remove it, but only if no failed # hosts. shutil.rmtree(base_root_path)