def is_image_already_imported(fingerprint):
try:
run_command("lxc image info {} >/dev/null 2>&1".format(fingerprint),
shell=True)
except Exception:
return False
return True
def setup_and_run_ansible(cname, debug=False, no_install=False, drive_count=1,
tiny_install=False, proxyfs=False):
if not no_install:
# get all the guest-executed stuff pushed over
# lxc file push ./ansible/ $CNAME/root/
# unfortunately, lxc doesn't support directly pushing a whole directory
# https://github.com/lxc/lxd/issues/1218
cmd = 'tar cf - ansible | lxc exec %s -- tar xf - -C /root/' % cname
run_command(cmd, cwd=sys.path[0], shell=True)
# install ansible
cmd = 'lxc exec %s -- /bin/bash /root/ansible/install_ansible.sh%s' \
% (cname, (' --debug' if debug else ''))
run_command(cmd)
def available_mounts():
total = 0
for j in range(1, 80): # drives per server (an arbitrary big number)
i = 1
for i in range(1, 5): # servers per guest
total += 1
if total > 8:
return
yield '/srv/%d/node/d%d' % (i, total)
extra_vars = {'no_install': no_install, 'tiny_install': tiny_install,
'proxyfs': proxyfs}
drive_list = []
all_mounts = available_mounts()
for i in range(drive_count):
drive_letter = 'sd%c' % chr(ord('b') + i)
mount_point = next(all_mounts)
x = {'drive_letter': drive_letter,
'mount_point': mount_point,
}
drive_list.append(x)
extra_vars['drive_list'] = drive_list
extra_vars = json.dumps(extra_vars)
# run bootstrap playbook
# see http://docs.ansible.com/ansible/latest/playbooks_variables.html#passing-variables-on-the-command-line
# for passing in a list of drives
cmd = 'lxc exec %s -- ansible-playbook -i "localhost," -c local ' \
'--extra-vars \'%s\' /root/ansible/bootstrap.yaml' % (cname, extra_vars)
run_command(cmd, shell=True)
# create shared code folder
cmd = 'lxc config device add %(cname)s sharedcomponents disk ' \
'path=/home/swift/code source=%(dir)s/guest_workspaces/%(cname)s' \
% {'cname': cname, 'dir': sys.path[0]}
run_command(cmd, cwd=sys.path[0])
# install
if not no_install:
cmd = 'lxc exec %s -- ansible-playbook -i "localhost," -c local --extra-vars \'%s\' ' \
'/root/ansible/master_playbook.yaml' % (cname, extra_vars)
run_command(cmd)
def import_split_image(manifest, alias):
metadata_tarball_path, rootfs_tarball_path = \
download_split_image_files(manifest)
# There might be an older image with the same alias
delete_image_with_alias(alias)
run_command("lxc image import {} {} --alias {}".format(
metadata_tarball_path, rootfs_tarball_path, alias))
os.unlink(metadata_tarball_path)
os.unlink(rootfs_tarball_path)
def git_clone_component(self, section, logfile_path=None):
section_options = self.components_options[section]
git_cmd = "git clone"
if "branch" in section_options:
git_cmd += " -b {}".format(section_options["branch"])
git_cmd += " {}".format(section_options["url"])
if "dest_path" in section_options:
dest_path = os.path.join(self.workspace_dir,
section_options["dest_path"])
git_cmd += " {}".format(dest_path)
run_command(git_cmd, cwd=self.workspace_dir, logfile_path=logfile_path)
def git_checkout_and_pull_component(self,
section,
dest_path,
logfile_path=None):
run_command("git fetch --all --tags",
dest_path,
logfile_path=logfile_path)
section_options = self.components_options[section]
git_cmd = "git checkout "
if "tag" in section_options:
git_cmd += "tags/{}".format(section_options["tag"])
elif "sha" in section_options:
git_cmd += section_options["sha"]
elif "branch" in section_options:
git_cmd += section_options["branch"]
run_command(git_cmd, dest_path, logfile_path=logfile_path)
if "branch" in section_options:
run_command("git pull", dest_path, logfile_path=logfile_path)
def import_unified_image(manifest, alias):
tarball_path = download_unified_image_file(manifest)
# There might be an older image with the same alias
delete_image_with_alias(alias)
run_command("lxc image import {} --alias {}".format(tarball_path, alias))
os.unlink(tarball_path)
def delete_image_with_alias(alias):
try:
run_command("lxc image delete {}".format(alias))
except Exception:
pass
base_image = args.baseimage
volume_size = args.volsize
volume_count = args.volcount
if is_swiftstack_hosted_image(distro):
import_image_if_needed(distro)
default_image = distro
else:
default_image = get_default_image(distro)
if base_image is None:
base_image = default_image
try:
# make a container profile that maps 8 block devices to the guest
run_command("lxc profile create {}-profile".format(container_name))
run_command("./make_lxc_profile.py {} {} {} {} | "
"lxc profile edit {}-profile".format(
container_name, VG_NAME, volume_size, volume_count,
container_name),
cwd=SCRIPT_DIR,
shell=True)
# launch the new container
print("Trying to launch container from base image "
"{}".format(base_image))
run_command("lxc launch {} {} -p {}-profile || "
"lxc launch {} {} -p {}-profile".format(
base_image, container_name, container_name,
default_image, container_name, container_name),
shell=True)
vol_size = man.get_config_option('drive_size')
base_image = 'runway-base-%s' % man.get_config_option('family')
debug = man.get_config_option('debug')
debug_string = " --debug" if debug else ""
vol_count = int(man.get_config_option('drive_count'))
distro = man.runway_options.get('distro', 'ubuntu')
tiny_deploy = man.get_config_option('tiny')
no_install = tiny_deploy or \
man.get_config_option('no_install')
no_snap = no_install or \
man.get_config_option('no_snapshot')
# starting from a base image doesn't work if the base image
# has fewer drives than the current manifest you're loading
run_command(
"./make_base_container.py "
"{} {} {} {} {}".format(distro, container_name, vol_size,
vol_count, base_image), RUNWAY_DIR)
setup_and_run_ansible_on_guest.setup_and_run_ansible(
container_name,
debug=debug,
drive_count=vol_count,
tiny_install=tiny_deploy)
if not no_install:
run_command("./generic_installer.py {}".format(container_name),
RUNWAY_DIR)
if not no_snap:
run_command(
"./snapshot_created_container.sh "
"{} {}{}".format(container_name, base_image, debug_string),
def retrieve_components(self):
logfile_path = os.path.abspath(
os.path.join(self.workspace_dir, DOWNLOAD_LOG_FILE_NAME))
for section in self.sections:
colorprint.info("Getting {}...".format(section), logfile_path)
section_options = self.components_options[section]
dest_path = self.get_absolute_dest_path_for_section(section)
component_exists = os.path.isdir(dest_path)
# Run any needed command BEFORE cloning
if not component_exists and "pre_cmd" in section_options:
run_command(
section_options["pre_cmd"],
cwd=self.workspace_dir,
logfile_path=logfile_path,
)
if not section_options["local"]:
if not component_exists:
self.git_clone_component(section,
logfile_path=logfile_path)
# Git checkout + pull in case "sha" or "tag" option is present
# or if the component directory already existed.
if (component_exists or "sha" in section_options
or "tag" in section_options):
self.git_checkout_and_pull_component(
section, dest_path, logfile_path=logfile_path)
self.git_submodule_update(dest_path, logfile_path=logfile_path)
current_sha = self.get_current_sha(dest_path)
if current_sha:
colorprint.normal(
"Using SHA {} for {}\n".format(current_sha, section),
logfile_path,
)
else:
colorprint.error(
"Couldn't get the SHA for the current commit. The SHA is only "
"printed for informational purposes, but not being able to "
"get it might be a symptom of something bad happening.\n"
)
else:
if not component_exists:
colorprint.warning(
"Component '{}' has been marked as "
"local, but it doesn't exist. You'll "
"most probably want to add it before "
"doing anything else.".format(section),
logfile_path,
)
else:
colorprint.normal(
"Component '{}' is locally managed.".format(section),
logfile_path,
)
# Run any needed command AFTER cloning
if not component_exists and "post_cmd" in section_options:
run_command(
section_options["post_cmd"],
cwd=self.workspace_dir,
logfile_path=logfile_path,
)
# Just print a new line to keep components' output separated
colorprint.normal("", logfile_path)
def git_submodule_update(self, dest_path, logfile_path=None):
run_command(
"git submodule update --init --recursive",
dest_path,
logfile_path=logfile_path,
)
args = parser.parse_args()
container_name = args.container_name
distro = args.distro
manifest_file = (
os.path.abspath(args.manifest) if args.manifest is not None else None
)
workspace_name = args.workspace
# Setup workspace
cmd = "./{}".format(SETUP_WORKSPACE_SCRIPT)
if manifest_file is not None:
cmd += " -m {}".format(manifest_file)
if workspace_name is not None:
cmd += " -w {}".format(workspace_name)
run_command(cmd, cwd=BIN_DIR)
# We need to know the number of drives and the drive size in order to
# create a Virtual Disk large enough for 2 containers when creating the VM
if workspace_name is None:
workspace_name = workspaces.get_last_workspace_name()
provided_workspace_name = False
else:
provided_workspace_name = True
manifest = get_manifest(workspace_name)
vol_size = manifest.get_config_option("drive_size")
vol_count = int(manifest.get_config_option("drive_count"))
# Vagrant up
if os.environ.get("CONTROLLER_NAME") is None:
default=None,
help="Workspace name")
args = parser.parse_args()
distro = args.distro
manifest_file = os.path.abspath(args.manifest) \
if args.manifest is not None else None
workspace_name = args.workspace
# Setup workspace
cmd = "./{}".format(SETUP_WORKSPACE_SCRIPT)
if manifest_file is not None:
cmd += " -m {}".format(manifest_file)
if workspace_name is not None:
cmd += " -w {}".format(workspace_name)
run_command(cmd, cwd=BIN_DIR)
# We need to know the number of drives and the drive size in order to
# create a Virtual Disk large enough for 2 containers when creating the VM
if workspace_name is None:
workspace_name = workspaces.get_last_workspace_name()
manifest = get_manifest(workspace_name)
vol_size = manifest.get_config_option("drive_size")
vol_count = int(manifest.get_config_option('drive_count'))
# Vagrant up
if os.environ.get('CONTROLLER_NAME') is None:
colorprint.warning("WARNING: CONTROLLER_NAME env var hasn't been set. "
"If you fail to 'vagrant up' your VM, open "
"VirtualBox, check the name of your SCSI "
for entry in os.scandir(workspace_path):
if entry.is_dir() and entry.name not in excluded_components and \
os.path.isfile('{}/{}/install.sh'.format(workspace_path,
entry.name)):
commands.append(os.path.join(entry.name, "install.sh"))
return commands
if __name__ == "__main__":
container_name = sys.argv[1]
if len(sys.argv) > 2:
workspace_name = sys.argv[2]
else:
workspace_name = container_name
workspace_path = workspaces.get_workspace_path(workspace_name)
logfile_path = os.path.abspath(
os.path.join(workspace_path, INSTALL_LOG_FILE_NAME))
manifest = get_manifest(workspace_name, logfile_path)
install_commands = get_install_commands(manifest, workspace_path,
logfile_path)
for install_command in install_commands:
cmd = 'lxc exec {} -- /bin/bash /home/swift/code/{}'.format(
container_name, install_command)
try:
run_command(cmd, logfile_path=logfile_path)
except Exception as e:
colorprint.error(str(e), logfile_path)
sys.exit(1)
