def callback(self, args):
file_prefix = os.path.splitext(args.private_key_file)[0]
try:
if args.vault_password_file is None:
vault_password = generate_random_password()
args.vault_password_file = "{}.vault_password".format(
file_prefix)
with open(args.vault_password_file, "w") as f:
f.write(vault_password)
elif os.path.exists(args.vault_password_file):
with open(args.vault_password_file, "r") as f:
vault_password = f.read().strip()
if vault_password is None:
raise YBOpsRuntimeError("Unable to read {}".format(
args.vault_password_file))
else:
raise YBOpsRuntimeError("Vault password file doesn't exist.")
if args.vault_file is None:
args.vault_file = "{}.vault".format(file_prefix)
rsa_key = validated_key_file(args.private_key_file)
except Exception:
self._cleanup_dir(os.path.dirname(args.private_key_file))
raise
# TODO: validate if the file provided is actually a private key file or not.
public_key = format_rsa_key(rsa_key, public_key=True)
private_key = format_rsa_key(rsa_key, public_key=False)
self.cluster_vault.update(id_rsa=private_key,
id_rsa_pub=public_key,
authorized_keys=public_key)
# These are saved for itest specific improvements.
aws_access_key = os.environ.get('AWS_ACCESS_KEY_ID', "")
aws_secret = os.environ.get('AWS_SECRET_ACCESS_KEY', "")
if aws_access_key and aws_secret:
self.cluster_vault.update(
AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'],
AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'])
vault_data = dict(cluster_server_vault=self.cluster_vault)
if args.has_sudo_password:
sudo_password = getpass.getpass("SUDO Password: "******"ansible_become_pass": sudo_password})
vault = Vault(vault_password)
vault.dump(vault_data, open(args.vault_file, 'w'))
print(
json.dumps({
"vault_file": args.vault_file,
"vault_password": args.vault_password_file
}))
def add_key_pair(self, args):
key_pair_name = args.key_pair_name
# If we were provided with a private key file, we use that to generate the public
# key using RSA. If not we will use the public key file (assuming the private key exists).
key_file = args.private_key_file if args.private_key_file else args.public_key_file
key_file_path = os.path.join(args.key_file_path, key_file)
# Make sure the key pair name doesn't exists already.
# TODO: may be add extra validation to see if the key exists in specific region
# if it doesn't exists in a region add them?. But only after validating the existing
# is the same key in other regions.
result = list(self.list_key_pair(args).values())[0]
if len(result) > 0:
raise YBOpsRuntimeError(
"KeyPair already exists {}".format(key_pair_name))
if not os.path.exists(key_file_path):
raise YBOpsRuntimeError(
"Key: {} file not found".format(key_file_path))
# This call would throw a exception if the file is not valid key file.
rsa_key = validated_key_file(key_file_path)
result = {}
for region, client in self._get_clients(args.region).items():
result[region] = client.import_key_pair(
KeyName=key_pair_name,
PublicKeyMaterial=format_rsa_key(rsa_key, public_key=True))
return result
def run_ansible_create(self, args):
server_type = args.type
can_ip_forward = (server_type.startswith('openvpn-server')
or server_type.startswith('ipsec-gateway'))
machine_image = args.machine_image if args.machine_image else \
self.cloud.get_image(args.region)["selfLink"]
ssh_keys = None
if args.private_key_file is not None:
rsa_key = validated_key_file(args.private_key_file)
public_key = format_rsa_key(rsa_key, public_key=True)
ssh_keys = "{}:{} {}".format(self.SSH_USER, public_key,
self.SSH_USER)
self.cloud.get_admin().create_instance(
args.region,
args.zone,
args.cloud_subnet,
args.search_pattern,
args.instance_type,
server_type,
args.use_preemptible,
can_ip_forward,
machine_image,
args.num_volumes,
args.volume_type,
args.volume_size,
args.boot_disk_size_gb,
args.assign_public_ip,
ssh_keys,
boot_script=args.boot_script,
auto_delete_boot_disk=args.auto_delete_boot_disk)
def callback(self, args):
file_prefix = os.path.splitext(args.private_key_file)[0]
if args.vault_password is None:
vault_password = generate_random_password()
args.vault_password = "******".format(file_prefix)
with file(args.vault_password, "w") as f:
f.write(vault_password)
elif os.path.exists(args.vault_password):
with file(args.vault_password) as f:
vault_password = f.read().strip()
if vault_password is None:
raise YBOpsRuntimeError("Unable to read {}".format(
args.vault_password))
else:
raise YBOpsRuntimeError("Vault password file doesn't exists.")
if args.vault_file is None:
args.vault_file = "{}.vault".format(file_prefix)
rsa_key = validated_key_file(args.private_key_file)
# TODO: validate if the file provided is actually a private key file or not.
public_key = format_rsa_key(rsa_key, public_key=True)
private_key = format_rsa_key(rsa_key, public_key=False)
self.cluster_vault.update(id_rsa=private_key,
id_rsa_pub=public_key,
authorized_keys=public_key)
vault_data = dict(cluster_server_vault=self.cluster_vault)
if args.has_sudo_password:
sudo_password = getpass.getpass("SUDO Password: "******"ansible_become_pass": sudo_password})
vault = Vault(vault_password)
vault.dump(vault_data, open(args.vault_file, 'w'))
print json.dumps({
"vault_file": args.vault_file,
"vault_password": args.vault_password
})
def run_ansible_create(self, args):
server_type = args.type
can_ip_forward = (
server_type.startswith('openvpn-server') or
server_type.startswith('ipsec-gateway'))
machine_image = args.machine_image if args.machine_image else \
self.cloud.get_image(args.region)["selfLink"]
ssh_keys = None
if args.private_key_file is not None:
rsa_key = validated_key_file(args.private_key_file)
public_key = format_rsa_key(rsa_key, public_key=True)
ssh_keys = "{}:{} {}".format(self.SSH_USER, public_key, self.SSH_USER)
self.cloud.create_instance(args, server_type, can_ip_forward, machine_image, ssh_keys)
def add_key_pair(self, args):
"""
Method to add key pair to AWS EC2.
True if new key pair with given name is added to AWS by Platform.
False if key pair with same name already exists and fingerprint is verified
Raises error if key is invalid, fingerprint generation fails, or fingerprint mismatches
"""
key_pair_name = args.key_pair_name
# If we were provided with a private key file, we use that to generate the public
# key using RSA. If not we will use the public key file (assuming the private key exists).
key_file = args.private_key_file if args.private_key_file else args.public_key_file
key_file_path = os.path.join(args.key_file_path, key_file)
if not os.path.exists(key_file_path):
raise YBOpsRuntimeError(
"Key: {} file not found".format(key_file_path))
# This call would throw a exception if the file is not valid key file.
rsa_key = validated_key_file(key_file_path)
# Validate the key pair if name already exists in AWS
result = list(self.list_key_pair(args).values())[0]
if len(result) > 0:
# Try to validate the keypair with KeyPair fingerprint in AWS
fingerprint = result[0][1]
possible_fingerprints = self._generate_fingerprints(key_file_path)
if fingerprint in possible_fingerprints:
return False
raise YBOpsRuntimeError(
"KeyPair {} already exists but fingerprint is invalid.".format(
key_pair_name))
result = {}
for region, client in self._get_clients(args.region).items():
result[region] = client.import_key_pair(
KeyName=key_pair_name,
PublicKeyMaterial=format_rsa_key(rsa_key, public_key=True))
return True
def create_vm(self, vmName, zone, numVolumes, subnet, private_key_file, volume_size,
instanceType, sshUser, image, nsg, pub, offer, sku, vnet, volType, serverType,
region, public_ip):
try:
return self.compute_client.virtual_machines.get(self.resource_group, vmName)
except CloudError:
pass
nic = self.create_nic(vmName, self.get_subnet_id(vnet, subnet),
zone, self.get_nsg_id(nsg), region, public_ip)
diskNames = [vmName + "-Disk-" + str(i) for i in range(1, numVolumes + 1)]
privateKey = validated_key_file(private_key_file)
vm_parameters = {
"location": region,
"os_profile": {
"computer_name": vmName,
"admin_username": sshUser,
"linux_configuration": {
"disable_password_authentication": True,
"ssh": {
"public_keys": [{
"path": "/home/{}/.ssh/authorized_keys".format(sshUser),
"key_data": format_rsa_key(privateKey, public_key=True)
}]
}
}
},
"hardware_profile": {
"vm_size": instanceType
},
"storage_profile": {
"osDisk": {
"createOption": "fromImage",
"managedDisk": {
"storageAccountType": "Standard_LRS"
}
},
"image_reference": {
"publisher": pub,
"offer": offer,
"sku": sku,
"version": image
}
},
"network_profile": {
"network_interfaces": [{
"id": nic.id
}]
},
"zones": [
zone
]
}
if (volType == "ultrassd_lrs"):
vm_parameters["additionalCapabilities"] = {"ultraSSDEnabled": True}
self.add_tag_resource(vm_parameters, "yb-server-type", serverType)
creation_result = self.compute_client.virtual_machines.create_or_update(
self.resource_group,
vmName,
vm_parameters
)
vm_result = creation_result.result()
vm = self.compute_client.virtual_machines.get(self.resource_group, vmName)
for idx, diskName in enumerate(diskNames):
self.appendDisk(vm, vmName, diskName, volume_size, idx, zone, volType, region)
return
def create_vm(self, vmName, zone, numVolumes, subnetId, private_key_file,
volume_size, instanceType, sshUser, image):
try:
return self.compute_client.virtual_machines.get(
RESOURCE_GROUP, vmName)
except CloudError:
pass
nic = self.create_nic(vmName, subnetId, zone)
diskNames = [
vmName + "-Disk-" + str(i) for i in range(1, numVolumes + 1)
]
privateKey = validated_key_file(private_key_file)
vm_parameters = {
'location': self.region,
'os_profile': {
'computer_name': vmName,
'admin_username': sshUser,
"linux_configuration": {
"disable_password_authentication": True,
"ssh": {
"public_keys": [{
"path":
"/home/{}/.ssh/authorized_keys".format(sshUser),
"key_data":
format_rsa_key(privateKey, public_key=True)
}]
}
}
},
'hardware_profile': {
'vm_size': instanceType
},
'storage_profile': {
'osDisk': {
'createOption': 'fromImage',
'managedDisk': {
'storageAccountType': 'Premium_LRS'
}
},
'image_reference': {
"publisher": "OpenLogic",
"offer": "CentOS",
"sku": "7_8",
"version": image
}
},
'network_profile': {
'network_interfaces': [{
'id': nic.id
}]
},
'zones': [zone]
}
creation_result = self.compute_client.virtual_machines.create_or_update(
RESOURCE_GROUP, vmName, vm_parameters)
vm_result = creation_result.result()
vm = self.compute_client.virtual_machines.get(RESOURCE_GROUP, vmName)
for idx, diskName in enumerate(diskNames):
self.appendDisk(vm, vmName, diskName, volume_size, idx, zone)
return
def create_vm(self, vm_name, zone, num_vols, private_key_file, volume_size,
instance_type, ssh_user, nsg, image, vol_type, server_type,
region, nic_id):
try:
return self.compute_client.virtual_machines.get(
RESOURCE_GROUP, vm_name)
except CloudError:
pass
disk_names = [
vm_name + "-Disk-" + str(i) for i in range(1, num_vols + 1)
]
private_key = validated_key_file(private_key_file)
shared_gallery_image_match = GALLERY_IMAGE_ID_REGEX.match(image)
if shared_gallery_image_match:
image_reference = {"id": image}
else:
# machine image URN - "OpenLogic:CentOS:7_8:7.8.2020051900"
pub, offer, sku, version = image.split(':')
image_reference = {
"publisher": pub,
"offer": offer,
"sku": sku,
"version": version
}
vm_parameters = {
"location": region,
"os_profile": {
"computer_name": vm_name,
"admin_username": ssh_user,
"linux_configuration": {
"disable_password_authentication": True,
"ssh": {
"public_keys": [{
"path":
"/home/{}/.ssh/authorized_keys".format(ssh_user),
"key_data":
format_rsa_key(private_key, public_key=True)
}]
}
}
},
"hardware_profile": {
"vm_size": instance_type
},
"storage_profile": {
"osDisk": {
"createOption": "fromImage",
"managedDisk": {
"storageAccountType": "Standard_LRS"
}
},
"image_reference": image_reference
},
"network_profile": {
"network_interfaces": [{
"id": nic_id
}]
}
}
if zone is not None:
vm_parameters["zones"] = [zone]
if (vol_type == "ultrassd_lrs"):
vm_parameters["additionalCapabilities"] = {"ultraSSDEnabled": True}
# Tag VM as cluster-server for ansible configure-{} script
self.add_tag_resource(vm_parameters, "yb-server-type", server_type)
creation_result = self.compute_client.virtual_machines.create_or_update(
RESOURCE_GROUP, vm_name, vm_parameters)
vm_result = creation_result.result()
vm = self.compute_client.virtual_machines.get(RESOURCE_GROUP, vm_name)
# Attach disks
for idx, disk_name in enumerate(disk_names):
self.appendDisk(vm, vm_name, disk_name, volume_size, idx, zone,
vol_type, region)
return
def create_or_update_vm(self,
vm_name,
zone,
num_vols,
private_key_file,
volume_size,
instance_type,
ssh_user,
nsg,
image,
vol_type,
server_type,
region,
nic_id,
tags,
disk_iops,
disk_throughput,
is_edit=False):
disk_names = [
vm_name + "-Disk-" + str(i) for i in range(1, num_vols + 1)
]
private_key = validated_key_file(private_key_file)
shared_gallery_image_match = GALLERY_IMAGE_ID_REGEX.match(image)
if shared_gallery_image_match:
image_reference = {"id": image}
else:
# machine image URN - "OpenLogic:CentOS:7_8:7.8.2020051900"
pub, offer, sku, version = image.split(':')
image_reference = {
"publisher": pub,
"offer": offer,
"sku": sku,
"version": version
}
vm_parameters = {
"location": region,
"os_profile": {
"computer_name": vm_name,
"admin_username": ssh_user,
"linux_configuration": {
"disable_password_authentication": True,
"ssh": {
"public_keys": [{
"path":
"/home/{}/.ssh/authorized_keys".format(ssh_user),
"key_data":
format_rsa_key(private_key, public_key=True)
}]
}
}
},
"hardware_profile": {
"vm_size": instance_type
},
"storage_profile": {
"osDisk": {
"createOption": "fromImage",
"managedDisk": {
"storageAccountType": "Standard_LRS"
}
},
"image_reference": image_reference
},
"network_profile": {
"network_interfaces": [{
"id": nic_id
}]
}
}
if zone is not None:
vm_parameters["zones"] = [zone]
if vol_type == ULTRASSD_LRS:
vm_parameters["additionalCapabilities"] = {"ultraSSDEnabled": True}
# Tag VM as cluster-server for ansible configure-{} script
self.add_tag_resource(vm_parameters, "yb-server-type", server_type)
for k in tags:
self.add_tag_resource(vm_parameters, k, tags[k])
creation_result = self.compute_client.virtual_machines.create_or_update(
RESOURCE_GROUP, vm_name, vm_parameters)
vm_result = creation_result.result()
vm = self.compute_client.virtual_machines.get(RESOURCE_GROUP, vm_name)
# Attach disks
if is_edit:
self.tag_disks(vm, vm_parameters["tags"])
else:
num_disks_attached = len(vm.storage_profile.data_disks)
for idx, disk_name in enumerate(disk_names):
# "Logical Unit Number" - where the data disk will be inserted. Add our disks
# after any existing ones.
lun = num_disks_attached + idx
self.append_disk(vm, vm_name, disk_name, volume_size, lun,
zone, vol_type, region, tags, disk_iops,
disk_throughput)
return
