def delete_snapshot(self, snapshot_name):
"""Delete a snapshot of the instance.
Args:
snapshot_name: the name to delete
"""
self._log.debug('deleting snapshot %s/%s', self.name, snapshot_name)
subp(['lxc', 'delete', '%s/%s' % (self.name, snapshot_name)])
def edit(self, key, value):
"""Edit the config of the instance.
Args:
key: The config key to edit
value: The new value to set the key to
"""
self._log.debug('editing %s with %s=%s', self.name, key, value)
subp(['lxc', 'config', 'set', self.name, key, value])
def restore(self, snapshot_name):
"""Restore instance from a specific snapshot.
Args:
snapshot_name: Name of snapshot to restore from
"""
self._log.debug('restoring %s from snapshot %s', self.name,
snapshot_name)
subp(['lxc', 'restore', self.name, snapshot_name])
def delete_image(self, image_id):
"""Delete the image.
Args:
image_id: string, LXD image fingerprint
"""
self._log.debug("Deleting image: '%s'", image_id)
subp(['lxc', 'image', 'delete', image_id])
self._log.debug('Deleted %s', image_id)
def delete(self, wait=True):
"""Delete and purge the current instance.
Args:
wait: wait for delete
"""
if not wait:
raise ValueError(
'wait=False not supported for KVM instance delete')
self._log.debug('deleting %s', self.name)
subp(['multipass', 'delete', '--purge', self.name])
def delete(self, wait=True):
"""Delete and purge the current instance.
Args:
wait: wait for delete
"""
self._log.debug('deleting %s', self.name)
subp(['multipass', 'delete', '--purge', self.name])
if wait:
self.wait_for_delete()
def shutdown(self, wait=True):
"""Shutdown instance.
Args:
wait: boolean, wait for instance to shutdown
"""
if not wait:
raise ValueError(
'wait=False not supported for KVM instance shutdown')
if self.state == 'Stopped':
return
self._log.debug('shutting down %s', self.name)
subp(['multipass', 'stop', self.name])
def shutdown(self, wait=True):
"""Shutdown instance.
Args:
wait: boolean, wait for instance to shutdown
"""
if self.state == 'Stopped':
return
self._log.debug('shutting down %s', self.name)
subp(['multipass', 'stop', self.name])
if wait:
self.wait_for_stop()
def start(self, wait=True):
"""Start instance.
Args:
wait: boolean, wait for instance to fully start
"""
if self.state == 'Running':
return
self._log.debug('starting %s', self.name)
subp(['multipass', 'start', self.name])
if wait:
self.wait()
def snapshot(self, snapshot_name, stateful=False):
"""Create a snapshot from the instance.
Args:
snapshot_name: name to call snapshot
stateful: boolean, stateful snapshot or not
"""
self.clean()
self.shutdown()
cmd = ['lxc', 'snapshot', self.name, snapshot_name]
if stateful:
cmd.append('--stateful')
self._log.debug('creating snapshot %s', snapshot_name)
subp(cmd)
def delete(self, wait=True):
"""Delete the current instance.
By default this will use the '--force' option to prevent the
need to always stop the instance first. This makes it easier
to work with ephemeral instances as well, which are deleted
on stop.
Args:
wait: wait for delete
"""
self._log.debug('deleting %s', self.name)
subp(['lxc', 'delete', self.name, '--force'])
if wait:
self.wait_for_delete()
def create_profile(
self, profile_name, profile_config, force=False
):
"""Create a lxd profile.
Create a lxd profile and populate it with the given
profile config. If the profile already exists, we will
not recreate it, unless the force parameter is set to True.
Args:
profile_name: Name of the profile to be created
profile_config: Config to be added to the new profile
force: Force the profile creation if it already exists
"""
profile_yaml = subp(["lxc", "profile", "list", "--format", "yaml"])
profile_list = [
profile["name"] for profile in yaml.safe_load(profile_yaml)
]
if profile_name in profile_list and not force:
msg = "The profile named {} already exists".format(profile_name)
self._log.debug(msg)
print(msg)
return
if force:
self._log.debug(
"Deleting current profile %s ...", profile_name)
subp(["lxc", "profile", "delete", profile_name])
self._log.debug("Creating profile %s ...", profile_name)
subp(["lxc", "profile", "create", profile_name])
subp(["lxc", "profile", "edit", profile_name], data=profile_config)
def _run_command(self, command, stdin):
"""Run command in the instance."""
if self.execute_via_ssh:
return super()._run_command(command, stdin)
base_cmd = [
'lxc', 'exec', self.name, '--', 'sudo', '-u', self.username, '--'
]
return subp(base_cmd + list(command), rcs=None)
def clone(self, base, new_instance_name):
"""Create copy of an existing instance or snapshot.
Uses the `lxc copy` command to create a copy of an existing
instance or a snapshot. To clone a snapshot then the base
is `instance_name/snapshot_name` otherwise if base is only
an existing instance it will clone an instance.
Args:
base: base instance or instance/snapshot
new_instance_name: name of new instance
Returns:
The created LXD instance object
"""
self._log.debug('cloning %s to %s', base, new_instance_name)
subp(['lxc', 'copy', base, new_instance_name])
return LXDInstance(new_instance_name)
def wait_for_stop(self):
"""Wait for instance stop."""
self._log.debug('waiting for stop: %s', self.name)
for _ in range(100):
result = subp('lxc list {} -c s --format csv'.format(
self.name).split())
if result == 'STOPPED':
return
time.sleep(1)
raise TimeoutError
def local_snapshot(self, snapshot_name, stateful=False):
"""Create an LXD snapshot (not a launchable image).
Args:
snapshot_name: name to call snapshot
stateful: boolean, stateful snapshot or not
"""
self.clean()
self.shutdown()
if snapshot_name is None:
snapshot_name = '{}-snapshot'.format(self.name)
cmd = ['lxc', 'snapshot', self.name, snapshot_name]
if stateful:
cmd.append('--stateful')
self._log.debug('creating snapshot %s', snapshot_name)
subp(cmd)
return snapshot_name
def __init__(self,
tag,
timestamp_suffix=True,
credentials_path=None,
project=None,
region="us-west2",
zone="a",
service_account_email=None):
"""Initialize the connection to GCE.
Args:
tag: string used to name and tag resources with
timestamp_suffix: bool set True to append a timestamp suffix to the
tag
credentials_path: path to credentials file for GCE
project: GCE project
region: GCE region
zone: GCE zone
service_account_email: service account to bind launched
instances to
"""
super().__init__(tag, timestamp_suffix)
self._log.debug('logging into GCE')
if credentials_path:
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(
credentials_path)
if project:
os.environ["GOOGLE_CLOUD_PROJECT"] = str(project)
else:
command = ['gcloud', 'config', 'get-value', 'project']
exception_text = (
"Could not obtain GCE project id. Has the CLI client been "
"setup?\nCommand attempted: '{}'".format(' '.join(command)))
try:
result = subp(command, rcs=())
except FileNotFoundError as e:
raise Exception(exception_text) from e
if not result.ok:
exception_text += '\nstdout: {}\nstderr: {}'.format(
result.stdout, result.stderr)
raise Exception(exception_text)
project = result.stdout
# disable cache_discovery due to:
# https://github.com/google/google-api-python-client/issues/299
self.compute = googleapiclient.discovery.build('compute',
'v1',
cache_discovery=False)
self.project = project
self.region = region
self.zone = '%s-%s' % (region, zone)
self.instance_counter = count()
self.service_account_email = service_account_email
def shutdown(self, wait=True, force=False, **kwargs):
"""Shutdown instance.
Args:
wait: boolean, wait for instance to shutdown
force: boolean, force instance to shutdown
"""
if self.state == 'Stopped':
return
self._log.debug('shutting down %s', self.name)
cmd = ["lxc", "stop", self.name]
if force:
cmd.append("--force")
subp(cmd)
if wait:
self.wait_for_stop()
def ip(self):
"""Return IP address of instance.
Returns:
IP address assigned to instance.
"""
command = 'lxc list {} -c 4 --format csv'.format(self.name)
result = subp(command.split()).stdout
ip_address = result.split()[0]
return ip_address
def snapshot(self, snapshot_name):
"""Create an image snapshot.
Snapshot is a bit of a misnomer here. Since "snapshot" in the
context of clouds means "create a launchable container from
this instance", we actually need to do a publish here. If you
need the lxd "snapshot" functionality, use local_snapshot
Args:
snapshot_name: name to call snapshot
"""
self.clean()
self.shutdown()
if snapshot_name is None:
snapshot_name = '{}-snapshot'.format(self.name)
cmd = ['lxc', 'publish', self.name, '--alias', snapshot_name]
self._log.debug('Publishing snapshot %s', snapshot_name)
subp(cmd)
return "local:{}".format(snapshot_name)
def pull_file(self, remote_path, local_path):
"""Pull file from an instance.
Args:
remote_path: path to remote file to pull down
local_path: local path to put the file
"""
self._log.debug('pulling file %s to %s', remote_path, local_path)
result = subp(['multipass', 'transfer', '%s:%s' %
(self.name, remote_path), local_path])
if result.failed:
raise RuntimeError(result.stderr)
def delete(self, wait=True):
"""Delete the current instance.
By default this will use the '--force' option to prevent the
need to always stop the instance first. This makes it easier
to work with ephemeral instances as well, which are deleted
on stop.
Args:
wait: wait for delete
"""
# Delete the container in two stages (first stop, then delete)
# to workaround the "ZFS dataset is busy" problem.
# Upstream LXD bug: https://github.com/lxc/lxd/issues/4656
self.shutdown()
self._log.debug('deleting %s', self.name)
subp(['lxc', 'delete', self.name, '--force'])
if wait:
self.wait_for_delete()
def launch(self, name, release, inst_type=None, wait=True):
"""Set up and launch a container.
This will init and start a container with the provided settings.
If no remote is specified pycloudlib defaults to daily images.
Args:
name: string, what to call the instance
release: string, [<remote>:]image, what release to launch
inst_type: string, type to use
wait: boolean, wait for instance to start
Returns:
The created KVM instance object
"""
if ':' not in release:
release = self._daily_remote + ':' + release
self._log.debug("Full release to launch: '%s'", release)
cmd = ['multipass', 'launch', '--name', name]
if inst_type:
inst_types = self._get_instance_types()
if inst_type not in inst_types:
raise RuntimeError('Unknown instance type: %s' % inst_type)
inst_cpus = str(int(inst_types[inst_type]['cpu']))
inst_mem = str(int(inst_types[inst_type]['mem'] * 1024**3))
self._log.debug("Instance type '%s' => cpus=%s, mem=%s", inst_type,
inst_cpus, inst_mem)
cmd += ['--cpus', inst_cpus, '--mem', inst_mem]
cmd.append(release)
self._log.debug('Creating %s', name)
subp(cmd)
return KVMInstance(name)
def init(
self, name, image_id, ephemeral=False, network=None, storage=None,
inst_type=None, profile_list=None, user_data=None,
config_dict=None, execute_via_ssh=True):
"""Init a container.
This will initialize a container, but not launch or start it.
If no remote is specified pycloudlib default to daily images.
Args:
name: string, what to call the instance
image_id: string, [<remote>:]<image identifier>, the image to
launch
ephemeral: boolean, ephemeral, otherwise persistent
network: string, optional, network name to use
storage: string, optional, storage name to use
inst_type: string, optional, type to use
profile_list: list, optional, profile(s) to use
user_data: used by cloud-init to run custom scripts/configuration
config_dict: dict, optional, configuration values to pass
execute_via_ssh: bool, optional, execute commands on the instance
via SSH if True (the default)
Returns:
The created LXD instance object
"""
image_id = self._normalize_image_id(image_id)
cmd = self._prepare_command(
name=name,
image_id=image_id,
ephemeral=ephemeral,
network=network,
storage=storage,
inst_type=inst_type,
profile_list=profile_list,
user_data=user_data,
config_dict=config_dict
)
print(cmd)
result = subp(cmd)
if not name:
name = result.split('Instance name is: ')[1]
self._log.debug('Created %s', name)
return LXDInstance(
name, self.key_pair, execute_via_ssh=execute_via_ssh
)
def state(self):
"""Return current status of instance.
If unable to get status will return 'Unknown'.
Returns:
Reported status from lxc info
"""
result = subp(['multipass', 'info', '--format', 'json', self.name])
info = json.loads(result)
state = info['info'][self.name]['state']
return state
def console_log(self):
"""Return console log.
Uses the '--show-log' option of console to get the console log
from an instance.
Returns:
bytes of this instance's console
"""
self._log.debug('getting console log for %s', self.name)
result = subp(['lxc', 'console', self.name, '--show-log'])
return result
def push_file(self, local_path, remote_path):
"""Push file to an instance.
The remote path must be absolute path with LXD due to the way
files are pulled off. Specifically, the format is 'name/path'
with path assumed to start from '/'.
Args:
local_path: local path to file to push up
remote_path: path to push file
"""
self._log.debug('pushing file %s to %s', local_path, remote_path)
if remote_path[0] != '/':
remote_pwd = self.execute('pwd')
remote_path = remote_pwd + '/' + remote_path
self._log.debug("Absolute remote path: %s", remote_path)
subp([
'lxc', 'file', 'push', local_path,
'%s%s' % (self.name, remote_path)
])
def execute(self, command, stdin=None, description=None):
"""Execute command in instance, recording output, error and exit code.
Assumes functional networking and execution with the target filesystem
being available at /.
Args:
command: the command to execute as root inside the image. If
command is a string, then it will be executed as:
`['sh', '-c', command]`
stdin: bytes content for standard in
description: purpose of command
Returns:
Result object
"""
if isinstance(command, str):
command = ['sh', '-c', command]
self._log.info('executing: %s', shell_quote(command))
if description:
self._log.debug(description)
else:
self._log.debug('executing: %s', shell_quote(command))
if self._type == 'lxd':
base_cmd = ['lxc', 'exec', self.name, '--']
return subp(base_cmd + list(command))
# multipass handling of redirects is buggy, so we don't bind
# stdin to /dev/null for the moment (shortcircuit_stdin=False).
# See: https://github.com/CanonicalLtd/multipass/issues/667
if self._type == 'kvm':
base_cmd = ['multipass', 'exec', self.name, '--']
return subp(base_cmd + list(command), shortcircuit_stdin=False)
return self._ssh(list(command), stdin=stdin)
def __init__(
self,
tag,
timestamp_suffix=True,
compartment_id=None,
config_path='~/.oci/config',
):
"""
Initialize the connection to OCI.
OCI must be initialized on the CLI first:
https://github.com/cloud-init/qa-scripts/blob/master/doc/launching-oracle.md
Args:
tag: Name of instance
timestamp_suffix: bool set True to append a timestamp suffix to the
tag
compartment_id: A compartment found at
https://console.us-phoenix-1.oraclecloud.com/a/identity/compartments
config_path: Path of OCI config file
"""
super().__init__(tag, timestamp_suffix)
if not compartment_id:
command = ['oci', 'iam', 'compartment', 'get']
exception_text = (
"Could not obtain OCI compartment id. Has the CLI client been "
"setup?\nCommand attempted: '{}'".format(' '.join(command)))
try:
result = subp(command, rcs=())
except FileNotFoundError as e:
raise Exception(exception_text) from e
if not result.ok:
exception_text += '\nstdout: {}\nstderr: {}'.format(
result.stdout, result.stderr)
raise Exception(exception_text)
compartment_id = json.loads(result.stdout)['data']['id']
self.compartment_id = compartment_id
if not os.path.isfile(os.path.expanduser(config_path)):
raise ValueError(
'{} is not a valid config file. '
'Pass a valid config file or first setup your OCI client. '
'See https://github.com/cloud-init/qa-scripts/blob/master/'
'doc/launching-oracle.md'.format(config_path))
self.config_path = config_path
config = oci.config.from_file(str(config_path))
self._log.debug('Logging into OCI')
self.compute_client = oci.core.ComputeClient(config)
self.network_client = oci.core.VirtualNetworkClient(config)
def state(self):
"""Return current status of instance.
If unable to get status will return 'Unknown'.
Returns:
Reported status from lxc info
"""
result = subp(['lxc', 'info', self.name])
try:
return re.findall(r'Status: (.*)', result)[0]
except IndexError:
return 'Unknown'