def run(self, values): gce_svc = gce_service.GCEService(values.project, None, log) instance_config = instance_config_from_values( values, mode=INSTANCE_METAVISOR_MODE, cli_config=self.config) if values.startup_script: extra_items = [{ 'key': 'startup-script', 'value': values.startup_script }] else: extra_items = None brkt_userdata = instance_config.make_userdata() metadata = gce_service.gce_metadata_from_userdata( brkt_userdata, extra_items=extra_items) if not values.verbose: logging.getLogger('googleapiclient').setLevel(logging.ERROR) if values.instance_name: gce_service.validate_image_name(values.instance_name) encrypted_instance_id = launch_gce_image.launch(log, gce_svc, values.image, values.instance_name, values.zone, values.delete_boot, values.instance_type, values.network, values.subnetwork, metadata) print(encrypted_instance_id) return 0
def run(self, values, out=sys.stdout): instance_cfg = instance_config_from_values(values, mode=INSTANCE_METAVISOR_MODE) if values.make_user_data_brkt_files: _add_files_to_instance_config(instance_cfg, values.make_user_data_brkt_files) if values.make_user_data_guest_files: guest_files = [] for fname in values.make_user_data_guest_files: match = re.match('(.+):(.+)', fname) if match: guest_file = GuestFile(match.group(1), match.group(2), None) guest_files.append(guest_file) else: raise ValidationError('Unable to parse guest file and type: %s' % fname) _add_guest_files_to_instance_config(instance_cfg, guest_files) if values.make_user_data_guest_fqdn: vpn_config = 'fqdn: %s\n' % (values.make_user_data_guest_fqdn,) instance_cfg.add_brkt_file('vpn.yaml', vpn_config) out.write(instance_cfg.make_userdata()) return 0
def _get_brkt_config_for_cli_args(cli_args='', mode=INSTANCE_CREATOR_MODE, cli_config=None): values = instance_config_args_to_values(cli_args) ic = instance_config_from_values(values, mode=mode, cli_config=cli_config) ud = ic.make_userdata() brkt_config_json = get_mime_part_payload(ud, BRKT_CONFIG_CONTENT_TYPE) brkt_config = json.loads(brkt_config_json)['brkt'] return brkt_config
def test_ca_cert(self): domain = 'dummy.foo.com' # First make sure that you can't use --ca-cert without specifying endpoints cli_args = '--ca-cert dummy.crt' values = instance_config_args_to_values(cli_args) with self.assertRaises(ValidationError): ic = instance_config_from_values(values) # Now specify endpoint args but use a bogus cert endpoint_args = ('--brkt-env api.%s:7777,hsmproxy.%s:8888,' + 'network.%s:9999') % (domain, domain, domain) dummy_ca_cert = 'THIS IS NOT A CERTIFICATE' with tempfile.NamedTemporaryFile() as f: f.write(dummy_ca_cert) f.flush() cli_args = endpoint_args + ' --ca-cert %s' % f.name values = instance_config_args_to_values(cli_args) with self.assertRaises(ValidationError): ic = instance_config_from_values(values) # Now use endpoint args and a valid cert cli_args = endpoint_args + ' --ca-cert %s' % _get_ca_cert_filename() values = instance_config_args_to_values(cli_args) ic = instance_config_from_values(values) ud = ic.make_userdata() brkt_files = get_mime_part_payload(ud, BRKT_FILES_CONTENT_TYPE) self.assertTrue(brkt_files.startswith( "/var/brkt/ami_config/ca_cert.pem.dummy.foo.com: " + "{contents: '-----BEGIN CERTIFICATE-----")) # Make sure the --ca-cert arg is only recognized in 'creator' mode # prevent stderr message from parse_args sys.stderr = open(os.devnull, 'w') try: values = instance_config_args_to_values(cli_args, mode=INSTANCE_METAVISOR_MODE) except SystemExit: pass else: self.assertTrue(False, 'Did not get expected exception') sys.stderr.close() sys.stderr = sys.__stderr__
def run(self, values, out=sys.stdout): instance_cfg = instance_config_from_values(values, mode=INSTANCE_METAVISOR_MODE) if values.make_user_data_brkt_files: _add_files_to_instance_config(instance_cfg, values.make_user_data_brkt_files) if values.make_user_data_guest_fqdn: vpn_config = 'fqdn: %s\n' % (values.make_user_data_guest_fqdn,) instance_cfg.add_brkt_file('vpn.yaml', vpn_config) out.write(instance_cfg.make_userdata()) return 0
def test_brkt_env_update(self): """ Test that the Bracket environment is passed through to metavisor user data. """ aws_svc, encryptor_image, guest_image = build_aws_service() encrypted_ami_id = encrypt_ami.encrypt( aws_svc=aws_svc, enc_svc_cls=DummyEncryptorService, image_id=guest_image.id, encryptor_ami=encryptor_image.id ) api_host_port = 'api.example.com:777' hsmproxy_host_port = 'hsmproxy.example.com:888' network_host_port = 'network.example.com:999' cli_args = '--brkt-env %s,%s,%s' % (api_host_port, hsmproxy_host_port, network_host_port) values = instance_config_args_to_values(cli_args) ic = instance_config_from_values(values) def run_instance_callback(args): if args.image_id == encryptor_image.id: brkt_config = self._get_brkt_config_from_mime(args.user_data) d = json.loads(brkt_config) self.assertEquals( api_host_port, d['brkt']['api_host'] ) self.assertEquals( hsmproxy_host_port, d['brkt']['hsmproxy_host'] ) self.assertEquals( network_host_port, d['brkt']['network_host'] ) self.assertEquals( 'updater', d['brkt']['solo_mode'] ) aws_svc.run_instance_callback = run_instance_callback update_ami( aws_svc, encrypted_ami_id, encryptor_image.id, 'Test updated AMI', enc_svc_class=DummyEncryptorService, instance_config=ic )
def run_encrypt(values, config): session_id = util.make_nonce() gce_svc = gce_service.GCEService(values.project, session_id, log) check_args(values, gce_svc, config) encrypted_image_name = gce_service.get_image_name( values.encrypted_image_name, values.image) gce_service.validate_image_name(encrypted_image_name) if values.validate: gce_service.validate_images(gce_svc, encrypted_image_name, values.encryptor_image, values.image, values.image_project) if not values.verbose: logging.getLogger('googleapiclient').setLevel(logging.ERROR) log.info('Starting encryptor session %s', gce_svc.get_session_id()) encrypted_image_id = encrypt_gce_image.encrypt( gce_svc=gce_svc, enc_svc_cls=encryptor_service.EncryptorService, image_id=values.image, encryptor_image=values.encryptor_image, encrypted_image_name=encrypted_image_name, zone=values.zone, instance_config=instance_config_from_values( values, mode=INSTANCE_CREATOR_MODE, cli_config=config), image_project=values.image_project, keep_encryptor=values.keep_encryptor, image_file=values.image_file, image_bucket=values.bucket, network=values.network, subnetwork=values.subnetwork, status_port=values.status_port, cleanup=values.cleanup ) # Print the image name to stdout, in case the caller wants to process # the output. Log messages go to stderr. print(encrypted_image_id) return 0
def test_brkt_env_encrypt(self): """ Test that we parse the brkt_env value and pass the correct values to user_data when launching the encryptor instance. """ api_host_port = 'api.example.com:777' hsmproxy_host_port = 'hsmproxy.example.com:888' network_host_port = 'network.example.com:999' aws_svc, encryptor_image, guest_image = build_aws_service() def run_instance_callback(args): if args.image_id == encryptor_image.id: brkt_config = self._get_brkt_config_from_mime(args.user_data) d = json.loads(brkt_config) self.assertEquals( api_host_port, d['brkt']['api_host'] ) self.assertEquals( hsmproxy_host_port, d['brkt']['hsmproxy_host'] ) self.assertEquals( network_host_port, d['brkt']['network_host'] ) cli_args = '--brkt-env %s,%s,%s' % (api_host_port, hsmproxy_host_port, network_host_port) values = instance_config_args_to_values(cli_args) ic = instance_config_from_values(values) aws_svc.run_instance_callback = run_instance_callback encrypt_ami.encrypt( aws_svc=aws_svc, enc_svc_cls=DummyEncryptorService, image_id=guest_image.id, encryptor_ami=encryptor_image.id, instance_config=ic )
def run_update(values, config): session_id = util.make_nonce() gce_svc = gce_service.GCEService(values.project, session_id, log) check_args(values, gce_svc, config) encrypted_image_name = gce_service.get_image_name( values.encrypted_image_name, values.image) gce_service.validate_image_name(encrypted_image_name) if values.validate: gce_service.validate_images(gce_svc, encrypted_image_name, values.encryptor_image, values.image) if not values.verbose: logging.getLogger('googleapiclient').setLevel(logging.ERROR) log.info('Starting updater session %s', gce_svc.get_session_id()) updated_image_id = update_gce_image.update_gce_image( gce_svc=gce_svc, enc_svc_cls=encryptor_service.EncryptorService, image_id=values.image, encryptor_image=values.encryptor_image, encrypted_image_name=encrypted_image_name, zone=values.zone, instance_config=instance_config_from_values( values, mode=INSTANCE_UPDATER_MODE, cli_config=config), keep_encryptor=values.keep_encryptor, image_file=values.image_file, image_bucket=values.bucket, network=values.network, subnetwork=values.subnetwork, status_port=values.status_port, cleanup=values.cleanup ) print(updated_image_id) return 0
def test_proxy_config(self): cli_args = '--proxy %s' % (proxy_host_port) values = instance_config_args_to_values(cli_args) ic = instance_config_from_values(values) _verify_proxy_config_in_userdata(self, ic.make_userdata())
def run_update(values, config, verbose=False): nonce = util.make_nonce() aws_svc = aws_service.AWSService( nonce, retry_timeout=values.retry_timeout, retry_initial_sleep_seconds=values.retry_initial_sleep_seconds ) log.debug( 'Retry timeout=%.02f, initial sleep seconds=%.02f', aws_svc.retry_timeout, aws_svc.retry_initial_sleep_seconds) brkt_env = ( brkt_cli.brkt_env_from_values(values) or brkt_cli.get_prod_brkt_env() ) if values.validate: # Validate the region before connecting. _validate_region(aws_svc, values.region) if values.token: brkt_cli.check_jwt_auth(brkt_env, values.token) aws_svc.connect(values.region, key_name=values.key_name) encrypted_image = _validate_ami(aws_svc, values.ami) encryptor_ami = values.encryptor_ami or _get_encryptor_ami(values.region) default_tags = encrypt_ami.get_default_tags(nonce, encryptor_ami) default_tags.update(brkt_cli.parse_tags(values.tags)) aws_svc.default_tags = default_tags if values.validate: _validate_guest_encrypted_ami( aws_svc, encrypted_image.id, encryptor_ami) brkt_cli.validate_ntp_servers(values.ntp_servers) _validate(aws_svc, values, encryptor_ami) _validate_guest_encrypted_ami( aws_svc, encrypted_image.id, encryptor_ami) else: log.info('Skipping AMI validation.') mv_image = aws_svc.get_image(encryptor_ami) if (encrypted_image.virtualization_type != mv_image.virtualization_type): log.error( 'Virtualization type mismatch. %s is %s, but encryptor %s is ' '%s.', encrypted_image.id, encrypted_image.virtualization_type, mv_image.id, mv_image.virtualization_type ) return 1 encrypted_ami_name = values.encrypted_ami_name if encrypted_ami_name: # Check for name collision. filters = {'name': encrypted_ami_name} if aws_svc.get_images(filters=filters, owners=['self']): raise ValidationError( 'You already own image named %s' % encrypted_ami_name) else: encrypted_ami_name = _get_updated_image_name( encrypted_image.name, nonce) log.debug('Image name: %s', encrypted_ami_name) aws_service.validate_image_name(encrypted_ami_name) # Initial validation done log.info( 'Updating %s with new metavisor %s', encrypted_image.id, encryptor_ami ) instance_config = instance_config_from_values( values, mode=INSTANCE_UPDATER_MODE, cli_config=config) if verbose: with tempfile.NamedTemporaryFile( prefix='user-data-', delete=False ) as f: log.debug('Writing instance user data to %s', f.name) f.write(instance_config.make_userdata()) updated_ami_id = update_ami( aws_svc, encrypted_image.id, encryptor_ami, encrypted_ami_name, subnet_id=values.subnet_id, security_group_ids=values.security_group_ids, guest_instance_type=values.guest_instance_type, updater_instance_type=values.updater_instance_type, instance_config=instance_config, status_port=values.status_port, ) print(updated_ami_id) return 0
def run_encrypt(values, config, verbose=False): session_id = util.make_nonce() aws_svc = aws_service.AWSService( session_id, retry_timeout=values.retry_timeout, retry_initial_sleep_seconds=values.retry_initial_sleep_seconds ) log.debug( 'Retry timeout=%.02f, initial sleep seconds=%.02f', aws_svc.retry_timeout, aws_svc.retry_initial_sleep_seconds) brkt_env = ( brkt_cli.brkt_env_from_values(values) or brkt_cli.get_prod_brkt_env() ) if values.validate: # Validate the region before connecting. _validate_region(aws_svc, values.region) if values.token: brkt_cli.check_jwt_auth(brkt_env, values.token) aws_svc.connect(values.region, key_name=values.key_name) if values.validate: guest_image = _validate_guest_ami(aws_svc, values.ami) else: guest_image = aws_svc.get_image(values.ami) encryptor_ami = values.encryptor_ami or _get_encryptor_ami(values.region) default_tags = encrypt_ami.get_default_tags(session_id, encryptor_ami) default_tags.update(brkt_cli.parse_tags(values.tags)) aws_svc.default_tags = default_tags if values.validate: _validate(aws_svc, values, encryptor_ami) brkt_cli.validate_ntp_servers(values.ntp_servers) instance_config = instance_config_from_values( values, mode=INSTANCE_CREATOR_MODE, cli_config=config) if verbose: with tempfile.NamedTemporaryFile( prefix='user-data-', delete=False ) as f: log.debug('Writing instance user data to %s', f.name) f.write(instance_config.make_userdata()) encrypted_image_id = encrypt_ami.encrypt( aws_svc=aws_svc, enc_svc_cls=encryptor_service.EncryptorService, image_id=guest_image.id, encryptor_ami=encryptor_ami, encrypted_ami_name=values.encrypted_ami_name, subnet_id=values.subnet_id, security_group_ids=values.security_group_ids, guest_instance_type=values.guest_instance_type, instance_config=instance_config, status_port=values.status_port, save_encryptor_logs=values.save_encryptor_logs, terminate_encryptor_on_failure=( values.terminate_encryptor_on_failure) ) # Print the AMI ID to stdout, in case the caller wants to process # the output. Log messages go to stderr. print(encrypted_image_id) return 0
def run_encrypt(values, parsed_config, log, use_esx=False): session_id = util.make_nonce() if values.create_ovf or values.create_ova: # verify we have a valid output directory if values.target_path is None: raise ValidationError("Missing directory path to store " "final OVF/OVA images") if not os.path.exists(values.target_path): raise ValidationError("Target path %s not present", values.target_path) if values.create_ova: # verify ovftool is present try: cmd = [values.ovftool_path, '-v'] subprocess.check_call(cmd) except: raise ValidationError("OVFtool not present. " "Cannot create OVA") else: if use_esx is False and values.template_vm_name is None: raise ValidationError("Missing template-vm-name for the " "template VM") if (values.source_image_path is not None and values.image_name is None): raise ValidationError("Specify the Metavisor OVF file.") if use_esx: _check_env_vars_set('ESX_USER_NAME') else: _check_env_vars_set('VCENTER_USER_NAME') vcenter_password = _get_vcenter_password(use_esx) brkt_cli.validate_ntp_servers(values.ntp_servers) brkt_env = brkt_cli.brkt_env_from_values(values) if brkt_env is None: _, brkt_env = parsed_config.get_current_env() if not values.token: raise ValidationError('Must provide a token') proxy = None if values.http_proxy: proxy = _parse_proxies(values.http_proxy)[0] # Download images from S3 try: if (values.encryptor_vmdk is None and values.source_image_path is None): (ovf, file_list) = \ esx_service.download_ovf_from_s3( values.bucket_name, image_name=values.image_name, proxy=proxy ) if ovf is None: raise ValidationError("Did not find MV OVF images") except Exception as e: raise ValidationError("Failed to download MV image from S3: ", e) # Connect to vCenter try: vc_swc = esx_service.initialize_vcenter( host=values.vcenter_host, user=os.getenv('ESX_USER_NAME') if use_esx else os.getenv('VCENTER_USER_NAME'), password=vcenter_password, port=values.vcenter_port, datacenter_name=None if use_esx else values.vcenter_datacenter, datastore_name=values.vcenter_datastore, esx_host=use_esx, cluster_name=None if use_esx else values.vcenter_cluster, no_of_cpus=values.no_of_cpus, memory_gb=values.memory_gb, session_id=session_id, network_name=values.network_name, nic_type=values.nic_type, verify=False if use_esx else values.validate, ) except Exception as e: raise ValidationError("Failed to connect to vCenter: ", e) # Validate that template does not already exist if values.template_vm_name: if vc_swc.find_vm(values.template_vm_name): raise ValidationError("VM with the same name as requested " "template VM name %s already exists" % values.template_vm_name) # Set tear-down vc_swc.set_teardown(values.no_teardown) # Set the disk-type if values.disk_type == "thin": vc_swc.set_thin_disk(True) vc_swc.set_eager_scrub(False) elif values.disk_type == "thick-lazy-zeroed": vc_swc.set_thin_disk(False) vc_swc.set_eager_scrub(False) elif values.disk_type == "thick-eager-zeroed": vc_swc.set_thin_disk(False) vc_swc.set_eager_scrub(True) else: raise ValidationError("Disk Type %s not correct. Can only be " "thin, thick-lazy-zeroed or " "thick-eager-zeroed" % (values.disk_type,)) try: instance_config = instance_config_from_values( values, mode=INSTANCE_CREATOR_MODE, cli_config=parsed_config) user_data_str = vc_swc.create_userdata_str(instance_config, update=False, ssh_key_file=values.ssh_public_key_file) if (values.encryptor_vmdk is not None): # Create from MV VMDK encrypt_vmdk.encrypt_from_vmdk( vc_swc, encryptor_service.EncryptorService, values.vmdk, vm_name=values.template_vm_name, create_ovf=values.create_ovf, create_ova=values.create_ova, target_path=values.target_path, image_name=values.encrypted_ovf_name, ovftool_path=values.ovftool_path, metavisor_vmdk=values.encryptor_vmdk, user_data_str=user_data_str, serial_port_file_name=values.serial_port_file_name, status_port=values.status_port, ) elif (values.source_image_path is not None): # Create from MV OVF in local directory encrypt_vmdk.encrypt_from_local_ovf( vc_swc, encryptor_service.EncryptorService, values.vmdk, vm_name=values.template_vm_name, create_ovf=values.create_ovf, create_ova=values.create_ova, target_path=values.target_path, image_name=values.encrypted_ovf_name, ovftool_path=values.ovftool_path, source_image_path=values.source_image_path, ovf_image_name=values.image_name, user_data_str=user_data_str, serial_port_file_name=values.serial_port_file_name, status_port=values.status_port, ) else: # Create from MV OVF in S3 encrypt_vmdk.encrypt_from_s3( vc_swc, encryptor_service.EncryptorService, values.vmdk, vm_name=values.template_vm_name, create_ovf=values.create_ovf, create_ova=values.create_ova, target_path=values.target_path, image_name=values.encrypted_ovf_name, ovftool_path=values.ovftool_path, ovf_name=ovf, download_file_list=file_list, user_data_str=user_data_str, serial_port_file_name=values.serial_port_file_name, status_port=values.status_port, ) return 0 except Exception as e: log.error("Failed to encrypt the guest VMDK: %s", e) return 1
def run_update(values, parsed_config, log, use_esx=False): session_id = util.make_nonce() encrypted_ovf_name = None encrypted_ova_name = None if values.create_ovf or values.create_ova: # verify we have a valid input directory if values.target_path is None: raise ValidationError("Missing directory path to fetch " "encrypted OVF/OVA images from") if not os.path.exists(values.target_path): raise ValidationError("Target path %s not present", values.target_path) if values.create_ovf: name = os.path.join(values.target_path, values.encrypted_ovf_name + ".ovf") if (os.path.exists(name) is False): raise ValidationError("Encrypted OVF image not found at " "%s", name) encrypted_ovf_name = values.encrypted_ovf_name else: encrypted_ova_name = values.encrypted_ovf_name name = os.path.join(values.target_path, values.encrypted_ovf_name + ".ova") if (os.path.exists(name) is False): raise ValidationError("Encrypted OVA image not found at " "%s", name) # verify ovftool is present try: cmd = [values.ovftool_path, '-v'] subprocess.check_call(cmd) except: raise ValidationError("OVFtool not present. " "Cannot process OVA") else: if use_esx: raise ValidationError("Cannot use template VMs for " "updation on a single ESX host") if (values.template_vm_name is None): raise ValidationError("Encrypted image not provided") if (values.source_image_path is not None and values.image_name is None): raise ValidationError("Specify the Metavisor OVF file.") if use_esx: _check_env_vars_set('ESX_USER_NAME') else: _check_env_vars_set('VCENTER_USER_NAME') vcenter_password = _get_vcenter_password(use_esx) brkt_cli.validate_ntp_servers(values.ntp_servers) brkt_env = brkt_cli.brkt_env_from_values(values) if brkt_env is None: _, brkt_env = parsed_config.get_current_env() if not values.token: raise ValidationError('Must provide a token') proxy = None if values.http_proxy: proxy = _parse_proxies(values.http_proxy)[0] # Download images from S3 try: if (values.encryptor_vmdk is None and values.source_image_path is None): (ovf_name, download_file_list) = \ esx_service.download_ovf_from_s3( values.bucket_name, image_name=values.image_name, proxy=proxy ) if ovf_name is None: raise ValidationError("Did not find MV OVF images") except Exception as e: raise ValidationError("Failed to download MV image from S3: ", e) # Connect to vCenter try: vc_swc = esx_service.initialize_vcenter( host=values.vcenter_host, user=os.getenv('ESX_USER_NAME') if use_esx else os.getenv('VCENTER_USER_NAME'), password=vcenter_password, port=values.vcenter_port, datacenter_name=None if use_esx else values.vcenter_datacenter, datastore_name=values.vcenter_datastore, esx_host=use_esx, cluster_name=None if use_esx else values.vcenter_cluster, no_of_cpus=values.no_of_cpus, memory_gb=values.memory_gb, session_id=session_id, network_name=values.network_name, nic_type=values.nic_type, verify=False if use_esx else values.validate, ) except Exception as e: raise ValidationError("Failed to connect to vCenter: ", e) if values.template_vm_name: if vc_swc.find_vm(values.template_vm_name) is None: raise ValidationError("Template VM %s not found" % values.template_vm_name) try: instance_config = instance_config_from_values( values, mode=INSTANCE_UPDATER_MODE, cli_config=parsed_config) user_data_str = vc_swc.create_userdata_str(instance_config, update=True, ssh_key_file=values.ssh_public_key_file) if (values.encryptor_vmdk is not None): # Create from MV VMDK update_vmdk.update_from_vmdk( vc_swc, encryptor_service.EncryptorService, template_vm_name=values.template_vm_name, target_path=values.target_path, ovf_name=encrypted_ovf_name, ova_name=encrypted_ova_name, ovftool_path=values.ovftool_path, metavisor_vmdk=values.encryptor_vmdk, user_data_str=user_data_str, status_port=values.status_port, ) elif (values.source_image_path is not None): # Create from MV OVF in local directory update_vmdk.update_from_local_ovf( vc_swc, encryptor_service.EncryptorService, template_vm_name=values.template_vm_name, target_path=values.target_path, ovf_name=encrypted_ovf_name, ova_name=encrypted_ova_name, ovftool_path=values.ovftool_path, source_image_path=values.source_image_path, ovf_image_name=values.image_name, user_data_str=user_data_str, status_port=values.status_port, ) else: # Create from MV OVF in S3 update_vmdk.update_from_s3( vc_swc, encryptor_service.EncryptorService, template_vm_name=values.template_vm_name, target_path=values.target_path, ovf_name=encrypted_ovf_name, ova_name=encrypted_ova_name, ovftool_path=values.ovftool_path, mv_ovf_name=ovf_name, download_file_list=download_file_list, user_data_str=user_data_str, status_port=values.status_port, ) return 0 except: log.error("Failed to update encrypted VMDK"); return 1