def run(hosts, command, users=None, sudo=False, raw=False): assert hosts errors = 0 for host in hosts: lines = run_command(host=host, command=command, users=users, sudo=sudo) if lines is None: log.error("Command: '{}'\nFailed on host: '{}'".format( command, host)) errors += 1 continue host_colon = (host + ":").ljust(16) if lines == "": if not raw: print("{} '{}'".format(host_colon, command)) continue cmd = command lines = lines.replace("\r", "") for line in lines.split("\n"): if raw: print(line) elif cmd: print("{} '{}' -> '{}'".format(host_colon, cmd, line)) fill = " " * (len(cmd) + 7) cmd = None else: print("{}{}'{}'".format(host_colon, fill, line)) return errors
def _package_from_releases(tags, extension, version, edition, remote_download): releases = Releases(edition) release = releases.default if version: release = releases.pick_version(version) release.init_download() if not release.artifacts: log.error( f"The {version} {edition} release is empty, visit tracker.mender.io to file a bug report" ) return None artifacts = release.find(tags, extension) if not artifacts: log.error( "Could not find an appropriate package for host, please use --{}-package" .format("hub" if "hub" in tags else "client")) return None artifact = artifacts[-1] if remote_download: return artifact.url else: return download_package(artifact.url)
def deploy(hubs, masterfiles): if masterfiles.startswith(("http://", "https://")): urls = [masterfiles] paths = _download_urls(urls) assert len(paths) == 1 masterfiles = paths[0] log.debug(f"Deploying downloaded: {masterfiles}") else: masterfiles = os.path.abspath(os.path.expanduser(masterfiles)) log.debug(f"Deploy path expanded to: {masterfiles}") masterfiles = masterfiles.rstrip("/") if os.path.isfile(masterfiles): return deploy_tarball(hubs, masterfiles) if not os.path.isdir(masterfiles): log.error(f"'{masterfiles}' must be a directory") return 1 directory = masterfiles if not directory.endswith("/masterfiles"): log.error( "The masterfiles directory to deploy must be called 'masterfiles'") return 1 if os.path.isfile(f"{directory}/autogen.sh"): os.system(f"bash -c 'cd {directory} && ./autogen.sh 1>/dev/null 2>&1'") if not os.path.isfile(f"{directory}/promises.cf"): log.error( f"The autogen.sh script did not produce promises.cf in '{directory}'" ) return 1 elif os.path.isfile(f"{directory}/configure"): os.system(f"bash -c 'cd {directory} && ./configure 1>/dev/null 2>&1'") if not os.path.isfile(f"{directory}/promises.cf"): log.error( f"The configure script did not produce promises.cf in '{directory}'" ) return 1 else: log.debug( "No autogen.sh / configure found, assuming ready to install directory" ) if not os.path.isfile(f"{directory}/promises.cf"): log.error(f"No promises.cf in '{directory}'") return 1 assert (not cf_remote_dir().endswith("/")) tarball = cf_remote_dir() + "/masterfiles.tgz" above = directory[0:-len("/masterfiles")] os.system(f"rm -rf {tarball}") os.system(f"tar -czf {tarball} -C {above} masterfiles") return deploy_tarball(hubs, tarball)
def deploy_tarball(hubs, tarball): assert os.path.isfile(tarball) if not tarball.endswith((".tgz", ".tar.gz")): log.error( "The masterfiles directory must be in a gzipped tarball (.tgz or .tar.gz)" ) return 1 errors = 0 for hub in hubs: errors += deploy_masterfiles(hub, tarball) return errors
def ssh_cmd(connection, cmd, errors=False): assert connection try: log.debug("Running over SSH: '{}'".format(cmd)) result = connection.run(cmd, hide=True) output = result.stdout.replace("\r\n", "\n").strip("\n") log.debug("'{}' -> '{}'".format(cmd, output)) return output except UnexpectedExit as e: msg = "Non-sudo command unexpectedly exited: '{}'".format(cmd) if errors: print(e) log.error(msg) else: log.debug(str(e)) log.debug(msg) return None
def ssh_sudo(connection, cmd, errors=False): assert connection try: log.debug("Running(sudo) over SSH: '{}'".format(cmd)) result = connection.run('sudo bash -c "{}"'.format(cmd.replace('"', r'\"')), hide=True) output = result.stdout.strip("\n") log.debug("'{}' -> '{}'".format(cmd, output)) return output except UnexpectedExit as e: msg = "Sudo command unexpectedly exited: '{}'".format(cmd) if errors: print(e) log.error(msg) else: log.debug(str(e)) log.debug(msg) return None
def ssh_cmd(connection, cmd, errors=False): assert connection try: log.debug("Running over SSH: '{}'".format(cmd)) result = connection.run(cmd, hide=True) output = result.stdout.strip("\n") log.debug("'{}' -> '{}'".format(cmd, output)) return output except UnexpectedExit as e: msg = "Non-sudo command unexpectedly exited: '{}'".format(cmd) if errors: print(e) log.error(msg) else: log.debug(str(e)) log.debug(msg) return None
def _package_from_releases(tags, extension, version, edition, remote_download): releases = Releases(edition) release = releases.default if version: release = releases.pick_version(version) artifacts = release.find(tags, extension) if not artifacts: log.error( "Could not find an appropriate package for host, please use --{}-package" .format("hub" if hub else "client")) return 1 artifact = artifacts[-1] if remote_download: return artifact.url else: return download_package(artifact.url)
def ssh_sudo(connection, cmd, errors=False): assert connection try: log.debug(f"Running(sudo) over SSH: '{cmd}'") escaped = cmd.replace('"', r"\"") sudo_cmd = f'sudo bash -c "{escaped}"' result = connection.run(sudo_cmd, hide=True, pty=True) output = result.stdout.strip("\n") log.debug(f"'{cmd}' -> '{output}'") return output except UnexpectedExit as e: msg = f"Sudo command unexpectedly exited: '{cmd}'" if errors: print(e) log.error(msg) else: log.debug(str(e)) log.debug(msg) return None
def spawn_vms(vm_requests, creds, region, key_pair=None, security_groups=None, provider=Providers.AWS, size=None, network=None, role=None, spawned_cb=None): if provider not in (Providers.AWS, Providers.GCP): raise ValueError("Unsupported provider %s" % provider) if (provider == Providers.AWS) and (key_pair is None): raise ValueError("key pair ID required for AWS") if (provider == Providers.AWS) and (security_groups is None): raise ValueError("security groups required for AWS") ret = [] if provider == Providers.AWS: for req in vm_requests: vm = spawn_vm_in_aws(req.platform, creds, key_pair, security_groups, region, req.name, req.size, role) if spawned_cb is not None: spawned_cb(vm) ret.append(vm) else: tasks = [ GCPSpawnTask(spawned_cb, req.platform, creds, region, req.name, req.size, network, req.public_ip, role) for req in vm_requests ] with Pool(len(vm_requests)) as pool: pool.map(lambda x: x.run(), tasks) for task in tasks: if task.vm is None: for error in task.errors: log.error(str(error)) else: ret.append(task.vm) return ret
def uninstall_host(host, *, connection=None): data = get_info(host, connection=connection) print_info(data) if not data["agent_version"]: log.warning( "CFEngine does not seem to be installed on '{}' - attempting uninstall anyway" .format(host)) uninstall_cfengine(host, data, connection=connection) data = get_info(host, connection=connection) if (not data) or data["agent_version"]: log.error("Failed to uninstall CFEngine on '{}'".format(host)) return 1 print_info(data) print("Uninstallation successful on '{}'".format(host)) return 0
def deploy_masterfiles(host, tarball, *, connection=None): data = get_info(host, connection=connection) print_info(data) if not data["agent_version"]: log.error( f"Cannot deploy masterfiles on {host} - CFEngine not installed") return 1 scp(tarball, host, connection=connection) ssh_cmd(connection, f"tar -xzf masterfiles.tgz") commands = [ "systemctl stop cfengine3", "rm -rf /var/cfengine/masterfiles", "mv masterfiles /var/cfengine/masterfiles", "systemctl start cfengine3", "cf-agent -Kf update.cf", "cf-agent -K", ] combined = " && ".join(commands) print(f"Running: '{combined}'") ssh_sudo(connection, combined) return 0
def bootstrap_host(host_data, policy_server, *, connection=None, trust_server=True): host = host_data["ssh_host"] agent = host_data["agent"] print("Bootstrapping: '{}' -> '{}'".format(host, policy_server)) command = "{} --bootstrap {}".format(agent, policy_server) if not trust_server: command += " --trust-server=no" if host_data["os"] == "windows": output = ssh_cmd(connection, powershell(command)) else: output = ssh_sudo(connection, command) if output is None: sys.exit("Bootstrap failed on '{}'".format(host)) if output and "completed successfully" in output: print("Bootstrap successful: '{}' -> '{}'".format(host, policy_server)) return True else: log.error("Something went wrong while bootstrapping") return False
def install(hubs, clients, *, bootstrap=None, package=None, hub_package=None, client_package=None, version=None, demo=False, call_collect=False, edition=None, remote_download=False, trust_keys=None): assert hubs or clients assert not (hubs and clients and package) assert (trust_keys is None) or hasattr(trust_keys, "__iter__") # These assertions are checked/ensured in main.py # If there are URLs in any of the package strings and remote_download is FALSE, download and replace with path: packages = (package, hub_package, client_package) if remote_download: package, hub_package, client_package = _verify_package_urls(packages) else: package, hub_package, client_package = _download_urls(packages) # If any of these are folders, transform them to lists of the files inside those folders: package = _maybe_packages_in_folder(package) hub_package = _maybe_packages_in_folder(hub_package) client_package = _maybe_packages_in_folder(client_package) # If --hub-package or --client-pacakge are not specified, use --package argument: if not hub_package: hub_package = package if not client_package: client_package = package if bootstrap: if type(bootstrap) is str: bootstrap = [bootstrap] save_file(os.path.join(cf_remote_dir(), "policy_server.dat"), "\n".join(bootstrap + [""])) hub_jobs = [] if hubs: show_host_info = (len(hubs) == 1) if type(hubs) is str: hubs = [hubs] for index, hub in enumerate(hubs): log.debug("Installing {} hub package on '{}'".format(edition, hub)) hub_jobs.append( HostInstaller( hub, hub=True, packages=hub_package, bootstrap=bootstrap[index % len(bootstrap)] if bootstrap else None, version=version, demo=demo, call_collect=call_collect, edition=edition, show_info=show_host_info, remote_download=remote_download, trust_keys=trust_keys)) errors = 0 if hub_jobs: with Pool(len(hub_jobs)) as hubs_install_pool: hubs_install_pool.map(lambda job: job.run(), hub_jobs) errors = sum(job.errors for job in hub_jobs) if errors > 0: s = "s" if errors > 1 else "" log.error( f"{errors} error{s} encountered while installing hub packages, aborting..." ) return errors client_jobs = [] show_host_info = (clients and (len(clients) == 1)) for index, host in enumerate(clients or []): log.debug("Installing {} client package on '{}'".format(edition, host)) client_jobs.append( HostInstaller( host, hub=False, packages=client_package, bootstrap=bootstrap[index % len(bootstrap)] if bootstrap else None, version=version, demo=demo, edition=edition, show_info=show_host_info, remote_download=remote_download, trust_keys=trust_keys)) if client_jobs: with Pool(len(client_jobs)) as clients_install_pool: clients_install_pool.map(lambda job: job.run(), client_jobs) errors += sum(job.errors for job in client_jobs) if demo and hubs: for hub in hubs: print( "Your demo hub is ready: https://{}/ (Username: admin, Password: password)" .format(strip_user(hub))) if errors > 0: s = "s" if errors > 1 else "" log.error( f"{errors} error{s} encountered while installing client packages") return errors
def install_host(host, *, hub=False, packages=None, bootstrap=None, version=None, demo=False, call_collect=False, connection=None, edition=None, show_info=True, remote_download=False, trust_keys=None): data = get_info(host, connection=connection) if show_info: print_info(data) package = None if packages and type(packages) is str: package = packages elif packages and len(packages) == 1: package = packages[0] if not package: tags = [] if edition == "enterprise": tags.append("hub" if hub else "agent") tags.append("64" if data["arch"] in ["x86_64", "amd64"] else data["arch"]) if data["arch"] in ["i386", "i486", "i586", "i686"]: tags.append("32") extension = None if "package_tags" in data and "msi" in data["package_tags"]: extension = ".msi" data["package_tags"].remove("msi") elif "dpkg" in data["bin"]: extension = ".deb" elif "rpm" in data["bin"]: extension = ".rpm" if "package_tags" in data and data["package_tags"]: tags.extend(data["package_tags"]) if packages is None: # No commandd line argument given package = _package_from_releases(tags, extension, version, edition, remote_download) else: package = _package_from_list(tags, extension, packages) if not package: log.error("Installation failed - no package found!") return 1 if remote_download: print(f"Downloading '{package}' on '{host}' using curl") r = ssh_cmd(cmd="curl --fail -O {}".format(package), connection=connection, errors=True) if r is None: return 1 else: scp(package, host, connection=connection) package = basename(package) install_package(host, package, data, connection=connection) data = get_info(host, connection=connection) if data["agent_version"] and len(data["agent_version"]) > 0: print("CFEngine {} was successfully installed on '{}'".format( data["agent_version"], host)) else: log.error("Installation failed!") return 1 if trust_keys: for key in trust_keys: scp(key, host, connection=connection) run_command(host, "mv %s /var/cfengine/ppkeys/" % basename(key), connection=connection, sudo=True) if bootstrap: ret = bootstrap_host(data, policy_server=bootstrap, connection=connection, trust_server=(not trust_keys)) if not ret: return 1 if demo: if hub: demo_lib.install_def_json(host, connection=connection, call_collect=call_collect) demo_lib.agent_run(data, connection=connection) demo_lib.disable_password_dialog(host) demo_lib.agent_run(data, connection=connection) return 0
def install_host(host, *, hub=False, package=None, bootstrap=None, version=None, demo=False, call_collect=False, connection=None, edition=None): data = get_info(host, connection=connection) print_info(data) if not package: tags = [] if edition == "enterprise": tags.append("hub" if hub else "agent") tags.append("64" if data["arch"] in ["x86_64", "amd64"] else data["arch"]) if data["arch"] in ["i386", "i486", "i586", "i686"]: tags.append("32") extension = None if "package_tags" in data and "msi" in data["package_tags"]: extension = ".msi" data["package_tags"].remove("msi") elif "dpkg" in data["bin"]: extension = ".deb" elif "rpm" in data["bin"]: extension = ".rpm" releases = Releases(edition) release = releases.default if version: release = releases.pick_version(version) if "package_tags" in data and data["package_tags"]: tags.extend(data["package_tags"]) artifacts = release.find(tags, extension) if not artifacts: log.error( "Could not find an appropriate package for host, please use --{}-package" .format("hub" if hub else "client")) return 1 artifact = artifacts[-1] package = download_package(artifact.url) scp(package, host, connection=connection) package = basename(package) install_package(host, package, data, connection=connection) data = get_info(host, connection=connection) if data["agent_version"] and len(data["agent_version"]) > 0: print("CFEngine {} was successfully installed on '{}'".format( data["agent_version"], host)) else: log.error("Installation failed!") return 1 if bootstrap: bootstrap_host(data, policy_server=bootstrap, connection=connection) if demo: if hub: demo_lib.install_def_json(host, connection=connection, call_collect=call_collect) demo_lib.agent_run(data, connection=connection) demo_lib.disable_password_dialog(host) demo_lib.agent_run(data, connection=connection) return 0