def destroy(group_name=None):
if os.path.exists(CLOUD_CONFIG_FPATH):
creds_data = read_json(CLOUD_CONFIG_FPATH)
else:
print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH)
return 1
creds = AWSCredentials(creds_data["aws"]["key"],
creds_data["aws"]["secret"])
if not os.path.exists(CLOUD_STATE_FPATH):
print("No saved cloud state info")
return 1
vms_info = read_json(CLOUD_STATE_FPATH)
to_destroy = []
if group_name:
print("Destroying hosts in the '%s' group" % group_name)
if not group_name.startswith("@"):
group_name = "@" + group_name
if group_name not in vms_info:
print("Group '%s' not found" % group_name)
return 1
region = vms_info[group_name]["meta"]["region"]
driver = get_cloud_driver(Providers.AWS, creds, region)
for name, vm_info in vms_info[group_name].items():
if name == "meta":
continue
vm_uuid = vm_info["uuid"]
vm = VM.get_by_uuid(vm_uuid, driver)
if vm is not None:
to_destroy.append(vm)
else:
print("VM '%s' not found in the clouds" % vm_uuid)
del vms_info[group_name]
else:
print("Destroying all hosts")
for group_name in [
key for key in vms_info.keys() if key.startswith("@")
]:
region = vms_info[group_name]["meta"]["region"]
driver = get_cloud_driver(Providers.AWS, creds, region)
for name, vm_info in vms_info[group_name].items():
if name == "meta":
continue
vm_uuid = vm_info["uuid"]
vm = VM.get_by_uuid(vm_uuid, driver)
if vm is not None:
to_destroy.append(vm)
else:
print("VM '%s' not found in the clouds" % vm_uuid)
del vms_info[group_name]
destroy_vms(to_destroy)
write_json(CLOUD_STATE_FPATH, vms_info)
return 0
def spawn(platform, count, role, group_name, provider=Providers.AWS, region=None):
if os.path.exists(CLOUD_CONFIG_FPATH):
creds_data = read_json(CLOUD_CONFIG_FPATH)
else:
print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH)
return 1
if os.path.exists(CLOUD_STATE_FPATH):
vms_info = read_json(CLOUD_STATE_FPATH)
else:
vms_info = dict()
group_key = "@%s" % group_name
if group_key in vms_info:
print("Group '%s' already exists!" % group_key)
return 1
try:
creds = AWSCredentials(creds_data["aws"]["key"], creds_data["aws"]["secret"])
sec_groups = creds_data["aws"]["security_groups"]
key_pair = creds_data["aws"]["key_pair"]
except KeyError:
print("Incomplete AWS credential info") # TODO: report missing keys
return 1
region = region or creds_data["aws"].get("region", "eu-west-1")
requests = []
for i in range(count):
requests.append(VMRequest(platform=platform,
name=(platform + role + str(i)),
size=None))
print("Spawning VMs...", end="")
sys.stdout.flush()
vms = spawn_vms(requests, creds, region, key_pair,
security_groups=sec_groups,
provider=provider,
role=role)
print("DONE")
if not all(vm.public_ips for vm in vms):
print("Waiting for VMs to get IP addresses...", end="")
sys.stdout.flush() # STDOUT is line-buffered
while not all(vm.public_ips for vm in vms):
time.sleep(1)
print(".", end="")
sys.stdout.flush() # STDOUT is line-buffered
print("DONE")
vms_info[group_key] = dump_vms_info(vms)
write_json(CLOUD_STATE_FPATH, vms_info)
print("Details about the spawned VMs can be found in %s" % CLOUD_STATE_FPATH)
return 0
def get_cloud_hosts(name, private_ips=False):
if not os.path.exists(paths.CLOUD_STATE_FPATH):
return []
state = read_json(paths.CLOUD_STATE_FPATH)
group_name = None
hosts = []
if name.startswith("@") and name in state:
# @some_group given and exists
group_name = name
elif ("@" + name) in state:
# group_name given and @group_name exists
group_name = "@" + name
if group_name is not None:
for name, info in state[group_name].items():
if name == "meta":
continue
log.debug("found name '{}' in state, info='{}'".format(name, info))
hosts.append(info)
else:
if name in state:
# host_name given and exists at the top level
hosts.append(state[name])
else:
for group_name in [
key for key in state.keys() if key.startswith("@")
]:
if name in state[group_name]:
hosts.append(state[group_name][name])
ret = []
for host in hosts:
if private_ips:
key = "private_ips"
else:
key = "public_ips"
ips = host.get(key, [])
if len(ips) > 0:
if host.get("user"):
ret.append('{}@{}'.format(host.get("user"), ips[0]))
else:
ret.append(ips[0])
else:
ret.append(None)
return ret
def is_in_cloud_state(name):
if not os.path.exists(paths.CLOUD_STATE_FPATH):
return False
# else
state = read_json(paths.CLOUD_STATE_FPATH)
if name in state:
return True
if ("@" + name) in state:
return True
# search for a host in any of the groups
for group in [key for key in state.keys() if key.startswith("@")]:
if name in state[group]:
return True
return False
def destroy(group_name=None):
if os.path.exists(CLOUD_CONFIG_FPATH):
creds_data = read_json(CLOUD_CONFIG_FPATH)
else:
print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH)
return 1
aws_creds = None
try:
aws_creds = AWSCredentials(creds_data["aws"]["key"],
creds_data["aws"]["secret"])
except KeyError:
# missing/incomplete AWS credentials, may not be needed, though
pass
gcp_creds = None
try:
gcp_creds = GCPCredentials(creds_data["gcp"]["project_id"],
creds_data["gcp"]["service_account_id"],
creds_data["gcp"]["key_path"])
except KeyError:
# missing/incomplete GCP credentials, may not be needed, though
pass
if not os.path.exists(CLOUD_STATE_FPATH):
print("No saved cloud state info")
return 1
vms_info = read_json(CLOUD_STATE_FPATH)
to_destroy = []
if group_name:
print("Destroying hosts in the '%s' group" % group_name)
if not group_name.startswith("@"):
group_name = "@" + group_name
if group_name not in vms_info:
print("Group '%s' not found" % group_name)
return 1
region = vms_info[group_name]["meta"]["region"]
provider = vms_info[group_name]["meta"]["provider"]
if provider == "aws":
if aws_creds is None:
user_error("Missing/incomplete AWS credentials")
return 1
driver = get_cloud_driver(Providers.AWS, aws_creds, region)
if provider == "gcp":
if gcp_creds is None:
user_error("Missing/incomplete GCP credentials")
return 1
driver = get_cloud_driver(Providers.GCP, gcp_creds, region)
nodes = driver.list_nodes()
for name, vm_info in vms_info[group_name].items():
if name == "meta":
continue
vm_uuid = vm_info["uuid"]
vm = VM.get_by_uuid(vm_uuid, nodes=nodes)
if vm is not None:
to_destroy.append(vm)
else:
print("VM '%s' not found in the clouds" % vm_uuid)
del vms_info[group_name]
else:
print("Destroying all hosts")
for group_name in [
key for key in vms_info.keys() if key.startswith("@")
]:
region = vms_info[group_name]["meta"]["region"]
provider = vms_info[group_name]["meta"]["provider"]
if provider == "aws":
if aws_creds is None:
user_error("Missing/incomplete AWS credentials")
return 1
driver = get_cloud_driver(Providers.AWS, aws_creds, region)
if provider == "gcp":
if gcp_creds is None:
user_error("Missing/incomplete GCP credentials")
return 1
driver = get_cloud_driver(Providers.GCP, gcp_creds, region)
nodes = driver.list_nodes()
for name, vm_info in vms_info[group_name].items():
if name == "meta":
continue
vm_uuid = vm_info["uuid"]
vm = VM.get_by_uuid(vm_uuid, nodes=nodes)
if vm is not None:
to_destroy.append(vm)
else:
print("VM '%s' not found in the clouds" % vm_uuid)
del vms_info[group_name]
destroy_vms(to_destroy)
write_json(CLOUD_STATE_FPATH, vms_info)
return 0
def spawn(platform,
count,
role,
group_name,
provider=Providers.AWS,
region=None,
size=None,
network=None,
public_ip=True,
extend_group=False):
if os.path.exists(CLOUD_CONFIG_FPATH):
creds_data = read_json(CLOUD_CONFIG_FPATH)
else:
print("Cloud credentials not found at %s" % CLOUD_CONFIG_FPATH)
return 1
if os.path.exists(CLOUD_STATE_FPATH):
vms_info = read_json(CLOUD_STATE_FPATH)
else:
vms_info = dict()
group_key = "@%s" % group_name
group_exists = group_key in vms_info
if not extend_group and group_exists:
print("Group '%s' already exists!" % group_key)
return 1
creds = None
sec_groups = None
key_pair = None
if provider == Providers.AWS:
try:
creds = AWSCredentials(creds_data["aws"]["key"],
creds_data["aws"]["secret"])
sec_groups = creds_data["aws"]["security_groups"]
key_pair = creds_data["aws"]["key_pair"]
except KeyError:
print(
"Incomplete AWS credential info") # TODO: report missing keys
return 1
region = region or creds_data["aws"].get("region", "eu-west-1")
elif provider == Providers.GCP:
try:
creds = GCPCredentials(creds_data["gcp"]["project_id"],
creds_data["gcp"]["service_account_id"],
creds_data["gcp"]["key_path"])
except KeyError:
print(
"Incomplete GCP credential info") # TODO: report missing keys
return 1
region = region or creds_data["gcp"].get("region", "europe-west1-b")
# TODO: Do we have to complicate this instead of just assuming existing VMs
# were created by this code and thus follow the naming pattern from this
# code?
if group_exists:
range_start = len(
[key for key in vms_info[group_key].keys() if key != "meta"])
else:
range_start = 0
requests = []
for i in range(range_start, range_start + count):
vm_name = whoami()[0:2] + group_name + "-" + platform + role + str(i)
requests.append(
VMRequest(platform=platform,
name=vm_name,
size=size,
public_ip=public_ip))
print("Spawning VMs...", end="")
sys.stdout.flush()
vms = spawn_vms(requests,
creds,
region,
key_pair,
security_groups=sec_groups,
provider=provider,
network=network,
role=role,
spawned_cb=print_progress_dot)
print("DONE")
if public_ip and (not all(vm.public_ips for vm in vms)):
print("Waiting for VMs to get IP addresses...", end="")
sys.stdout.flush() # STDOUT is line-buffered
while not all(vm.public_ips for vm in vms):
time.sleep(1)
print_progress_dot()
print("DONE")
if group_exists:
vms_info[group_key].update(dump_vms_info(vms))
else:
vms_info[group_key] = dump_vms_info(vms)
write_json(CLOUD_STATE_FPATH, vms_info)
print("Details about the spawned VMs can be found in %s" %
CLOUD_STATE_FPATH)
return 0
