def is_image_already_imported(fingerprint): try: run_command("lxc image info {} >/dev/null 2>&1".format(fingerprint), shell=True) except Exception: return False return True
def setup_and_run_ansible(cname, debug=False, no_install=False, drive_count=1, tiny_install=False, proxyfs=False): if not no_install: # get all the guest-executed stuff pushed over # lxc file push ./ansible/ $CNAME/root/ # unfortunately, lxc doesn't support directly pushing a whole directory # https://github.com/lxc/lxd/issues/1218 cmd = 'tar cf - ansible | lxc exec %s -- tar xf - -C /root/' % cname run_command(cmd, cwd=sys.path[0], shell=True) # install ansible cmd = 'lxc exec %s -- /bin/bash /root/ansible/install_ansible.sh%s' \ % (cname, (' --debug' if debug else '')) run_command(cmd) def available_mounts(): total = 0 for j in range(1, 80): # drives per server (an arbitrary big number) i = 1 for i in range(1, 5): # servers per guest total += 1 if total > 8: return yield '/srv/%d/node/d%d' % (i, total) extra_vars = {'no_install': no_install, 'tiny_install': tiny_install, 'proxyfs': proxyfs} drive_list = [] all_mounts = available_mounts() for i in range(drive_count): drive_letter = 'sd%c' % chr(ord('b') + i) mount_point = next(all_mounts) x = {'drive_letter': drive_letter, 'mount_point': mount_point, } drive_list.append(x) extra_vars['drive_list'] = drive_list extra_vars = json.dumps(extra_vars) # run bootstrap playbook # see http://docs.ansible.com/ansible/latest/playbooks_variables.html#passing-variables-on-the-command-line # for passing in a list of drives cmd = 'lxc exec %s -- ansible-playbook -i "localhost," -c local ' \ '--extra-vars \'%s\' /root/ansible/bootstrap.yaml' % (cname, extra_vars) run_command(cmd, shell=True) # create shared code folder cmd = 'lxc config device add %(cname)s sharedcomponents disk ' \ 'path=/home/swift/code source=%(dir)s/guest_workspaces/%(cname)s' \ % {'cname': cname, 'dir': sys.path[0]} run_command(cmd, cwd=sys.path[0]) # install if not no_install: cmd = 'lxc exec %s -- ansible-playbook -i "localhost," -c local --extra-vars \'%s\' ' \ '/root/ansible/master_playbook.yaml' % (cname, extra_vars) run_command(cmd)
def import_split_image(manifest, alias): metadata_tarball_path, rootfs_tarball_path = \ download_split_image_files(manifest) # There might be an older image with the same alias delete_image_with_alias(alias) run_command("lxc image import {} {} --alias {}".format( metadata_tarball_path, rootfs_tarball_path, alias)) os.unlink(metadata_tarball_path) os.unlink(rootfs_tarball_path)
def git_clone_component(self, section, logfile_path=None): section_options = self.components_options[section] git_cmd = "git clone" if "branch" in section_options: git_cmd += " -b {}".format(section_options["branch"]) git_cmd += " {}".format(section_options["url"]) if "dest_path" in section_options: dest_path = os.path.join(self.workspace_dir, section_options["dest_path"]) git_cmd += " {}".format(dest_path) run_command(git_cmd, cwd=self.workspace_dir, logfile_path=logfile_path)
def git_checkout_and_pull_component(self, section, dest_path, logfile_path=None): run_command("git fetch --all --tags", dest_path, logfile_path=logfile_path) section_options = self.components_options[section] git_cmd = "git checkout " if "tag" in section_options: git_cmd += "tags/{}".format(section_options["tag"]) elif "sha" in section_options: git_cmd += section_options["sha"] elif "branch" in section_options: git_cmd += section_options["branch"] run_command(git_cmd, dest_path, logfile_path=logfile_path) if "branch" in section_options: run_command("git pull", dest_path, logfile_path=logfile_path)
def import_unified_image(manifest, alias): tarball_path = download_unified_image_file(manifest) # There might be an older image with the same alias delete_image_with_alias(alias) run_command("lxc image import {} --alias {}".format(tarball_path, alias)) os.unlink(tarball_path)
def delete_image_with_alias(alias): try: run_command("lxc image delete {}".format(alias)) except Exception: pass
base_image = args.baseimage volume_size = args.volsize volume_count = args.volcount if is_swiftstack_hosted_image(distro): import_image_if_needed(distro) default_image = distro else: default_image = get_default_image(distro) if base_image is None: base_image = default_image try: # make a container profile that maps 8 block devices to the guest run_command("lxc profile create {}-profile".format(container_name)) run_command("./make_lxc_profile.py {} {} {} {} | " "lxc profile edit {}-profile".format( container_name, VG_NAME, volume_size, volume_count, container_name), cwd=SCRIPT_DIR, shell=True) # launch the new container print("Trying to launch container from base image " "{}".format(base_image)) run_command("lxc launch {} {} -p {}-profile || " "lxc launch {} {} -p {}-profile".format( base_image, container_name, container_name, default_image, container_name, container_name), shell=True)
vol_size = man.get_config_option('drive_size') base_image = 'runway-base-%s' % man.get_config_option('family') debug = man.get_config_option('debug') debug_string = " --debug" if debug else "" vol_count = int(man.get_config_option('drive_count')) distro = man.runway_options.get('distro', 'ubuntu') tiny_deploy = man.get_config_option('tiny') no_install = tiny_deploy or \ man.get_config_option('no_install') no_snap = no_install or \ man.get_config_option('no_snapshot') # starting from a base image doesn't work if the base image # has fewer drives than the current manifest you're loading run_command( "./make_base_container.py " "{} {} {} {} {}".format(distro, container_name, vol_size, vol_count, base_image), RUNWAY_DIR) setup_and_run_ansible_on_guest.setup_and_run_ansible( container_name, debug=debug, drive_count=vol_count, tiny_install=tiny_deploy) if not no_install: run_command("./generic_installer.py {}".format(container_name), RUNWAY_DIR) if not no_snap: run_command( "./snapshot_created_container.sh " "{} {}{}".format(container_name, base_image, debug_string),
def retrieve_components(self): logfile_path = os.path.abspath( os.path.join(self.workspace_dir, DOWNLOAD_LOG_FILE_NAME)) for section in self.sections: colorprint.info("Getting {}...".format(section), logfile_path) section_options = self.components_options[section] dest_path = self.get_absolute_dest_path_for_section(section) component_exists = os.path.isdir(dest_path) # Run any needed command BEFORE cloning if not component_exists and "pre_cmd" in section_options: run_command( section_options["pre_cmd"], cwd=self.workspace_dir, logfile_path=logfile_path, ) if not section_options["local"]: if not component_exists: self.git_clone_component(section, logfile_path=logfile_path) # Git checkout + pull in case "sha" or "tag" option is present # or if the component directory already existed. if (component_exists or "sha" in section_options or "tag" in section_options): self.git_checkout_and_pull_component( section, dest_path, logfile_path=logfile_path) self.git_submodule_update(dest_path, logfile_path=logfile_path) current_sha = self.get_current_sha(dest_path) if current_sha: colorprint.normal( "Using SHA {} for {}\n".format(current_sha, section), logfile_path, ) else: colorprint.error( "Couldn't get the SHA for the current commit. The SHA is only " "printed for informational purposes, but not being able to " "get it might be a symptom of something bad happening.\n" ) else: if not component_exists: colorprint.warning( "Component '{}' has been marked as " "local, but it doesn't exist. You'll " "most probably want to add it before " "doing anything else.".format(section), logfile_path, ) else: colorprint.normal( "Component '{}' is locally managed.".format(section), logfile_path, ) # Run any needed command AFTER cloning if not component_exists and "post_cmd" in section_options: run_command( section_options["post_cmd"], cwd=self.workspace_dir, logfile_path=logfile_path, ) # Just print a new line to keep components' output separated colorprint.normal("", logfile_path)
def git_submodule_update(self, dest_path, logfile_path=None): run_command( "git submodule update --init --recursive", dest_path, logfile_path=logfile_path, )
args = parser.parse_args() container_name = args.container_name distro = args.distro manifest_file = ( os.path.abspath(args.manifest) if args.manifest is not None else None ) workspace_name = args.workspace # Setup workspace cmd = "./{}".format(SETUP_WORKSPACE_SCRIPT) if manifest_file is not None: cmd += " -m {}".format(manifest_file) if workspace_name is not None: cmd += " -w {}".format(workspace_name) run_command(cmd, cwd=BIN_DIR) # We need to know the number of drives and the drive size in order to # create a Virtual Disk large enough for 2 containers when creating the VM if workspace_name is None: workspace_name = workspaces.get_last_workspace_name() provided_workspace_name = False else: provided_workspace_name = True manifest = get_manifest(workspace_name) vol_size = manifest.get_config_option("drive_size") vol_count = int(manifest.get_config_option("drive_count")) # Vagrant up if os.environ.get("CONTROLLER_NAME") is None:
default=None, help="Workspace name") args = parser.parse_args() distro = args.distro manifest_file = os.path.abspath(args.manifest) \ if args.manifest is not None else None workspace_name = args.workspace # Setup workspace cmd = "./{}".format(SETUP_WORKSPACE_SCRIPT) if manifest_file is not None: cmd += " -m {}".format(manifest_file) if workspace_name is not None: cmd += " -w {}".format(workspace_name) run_command(cmd, cwd=BIN_DIR) # We need to know the number of drives and the drive size in order to # create a Virtual Disk large enough for 2 containers when creating the VM if workspace_name is None: workspace_name = workspaces.get_last_workspace_name() manifest = get_manifest(workspace_name) vol_size = manifest.get_config_option("drive_size") vol_count = int(manifest.get_config_option('drive_count')) # Vagrant up if os.environ.get('CONTROLLER_NAME') is None: colorprint.warning("WARNING: CONTROLLER_NAME env var hasn't been set. " "If you fail to 'vagrant up' your VM, open " "VirtualBox, check the name of your SCSI "
for entry in os.scandir(workspace_path): if entry.is_dir() and entry.name not in excluded_components and \ os.path.isfile('{}/{}/install.sh'.format(workspace_path, entry.name)): commands.append(os.path.join(entry.name, "install.sh")) return commands if __name__ == "__main__": container_name = sys.argv[1] if len(sys.argv) > 2: workspace_name = sys.argv[2] else: workspace_name = container_name workspace_path = workspaces.get_workspace_path(workspace_name) logfile_path = os.path.abspath( os.path.join(workspace_path, INSTALL_LOG_FILE_NAME)) manifest = get_manifest(workspace_name, logfile_path) install_commands = get_install_commands(manifest, workspace_path, logfile_path) for install_command in install_commands: cmd = 'lxc exec {} -- /bin/bash /home/swift/code/{}'.format( container_name, install_command) try: run_command(cmd, logfile_path=logfile_path) except Exception as e: colorprint.error(str(e), logfile_path) sys.exit(1)