def run_job_on_worker(worker_info, call_ids_range, job_payload): """ Install all the Lithops dependencies into the worker. Runs the job """ instance_name, ip_address, instance_id = worker_info logger.info('Going to setup {}, IP address {}'.format( instance_name, ip_address)) ssh_client = SSHClient(ip_address, STANDALONE_SSH_CREDNTIALS) wait_instance_ready(ssh_client) # upload zip lithops package logger.info('Uploading lithops files to VM instance {}'.format(ip_address)) ssh_client.upload_local_file('/opt/lithops/lithops_standalone.zip', '/tmp/lithops_standalone.zip') logger.info( 'Executing lithops installation process on VM instance {}'.format( ip_address)) vm_data = { 'instance_name': instance_name, 'ip_address': ip_address, 'instance_id': instance_id } script = get_worker_setup_script(STANDALONE_CONFIG, vm_data) ssh_client.run_remote_command(script, run_async=True) ssh_client.close() # Wait until the proxy is ready wait_proxy_ready(ip_address) dbr = job_payload['data_byte_ranges'] job_payload['call_ids'] = call_ids_range job_payload['data_byte_ranges'] = [ dbr[int(call_id)] for call_id in call_ids_range ] url = "http://{}:{}/run".format(ip_address, STANDALONE_SERVICE_PORT) r = requests.post(url, data=json.dumps(job_payload)) response = r.json() if 'activationId' in response: logger.info('Calls {} invoked. Activation ID: {}'.format( ', '.join(call_ids_range), response['activationId'])) else: logger.error('calls {} failed invocation: {}'.format( ', '.join(call_ids_range), response['error']))
class VMInstance: def __init__(self, config): self.ip_address = self.config['ip_address'] self.ssh_client = None self.ssh_credentials = { 'username': self.config.get('ssh_user', 'root'), 'password': self.config.get('ssh_password', None), 'key_filename': self.config.get('ssh_key_filename', None) } logger.debug('{} created'.format(self)) def __str__(self): return 'VM instance {}'.format(self.ip_address) def get_ssh_client(self): """ Creates an ssh client against the VM only if the Instance is the master """ if self.ip_address: if not self.ssh_client: self.ssh_client = SSHClient(self.ip_address, self.ssh_credentials) return self.ssh_client def del_ssh_client(self): """ Deletes the ssh client """ if self.ssh_client: self.ssh_client.close() self.ssh_client = None def create(self): pass def start(self): pass def stop(self): pass def delete(self): pass
class IBMVPCInstance: def __init__(self, name, ibm_vpc_config, ibm_vpc_client=None, public=False): """ Initialize a IBMVPCInstance instance VMs can have master role, this means they will have a public IP address """ self.name = name.lower() self.config = ibm_vpc_config self.delete_on_dismantle = self.config['delete_on_dismantle'] self.profile_name = self.config['profile_name'] self.ibm_vpc_client = ibm_vpc_client or self._create_vpc_client() self.public = public self.ssh_client = None self.instance_id = None self.instance_data = None self.ip_address = None self.public_ip = None self.ssh_credentials = { 'username': self.config['ssh_user'], 'password': self.config.get('ssh_password', None if public else SSH_PASSWD), 'key_filename': self.config.get('ssh_key_filename', None) } def __str__(self): return 'VM instance {} ({})'.format(self.name, self.public_ip or self.ip_address) def _create_vpc_client(self): """ Creates an IBM VPC python-sdk instance """ authenticator = IAMAuthenticator(self.iam_api_key) ibm_vpc_client = VpcV1('2021-01-19', authenticator=authenticator) ibm_vpc_client.set_service_url(self.config['endpoint'] + '/v1') return ibm_vpc_client def get_ssh_client(self): """ Creates an ssh client against the VM only if the Instance is the master """ if self.ip_address or self.public_ip: if not self.ssh_client: self.ssh_client = SSHClient(self.public_ip or self.ip_address, self.ssh_credentials) return self.ssh_client def del_ssh_client(self): """ Deletes the ssh client """ if self.ssh_client: try: self.ssh_client.close() except Exception: pass self.ssh_client = None def _create_instance(self): """ Creates a new VM instance """ logger.debug("Creating new VM instance {}".format(self.name)) security_group_identity_model = {'id': self.config['security_group_id']} subnet_identity_model = {'id': self.config['subnet_id']} primary_network_interface = { 'name': 'eth0', 'subnet': subnet_identity_model, 'security_groups': [security_group_identity_model] } boot_volume_profile = { 'capacity': 100, 'name': '{}-boot'.format(self.name), 'profile': {'name': self.config['volume_tier_name']}} boot_volume_attachment = { 'delete_volume_on_instance_delete': True, 'volume': boot_volume_profile } key_identity_model = {'id': self.config['key_id']} instance_prototype = {} instance_prototype['name'] = self.name instance_prototype['keys'] = [key_identity_model] instance_prototype['profile'] = {'name': self.profile_name} instance_prototype['resource_group'] = {'id': self.config['resource_group_id']} instance_prototype['vpc'] = {'id': self.config['vpc_id']} instance_prototype['image'] = {'id': self.config['image_id']} instance_prototype['zone'] = {'name': self.config['zone_name']} instance_prototype['boot_volume_attachment'] = boot_volume_attachment instance_prototype['primary_network_interface'] = primary_network_interface if not self.public: instance_prototype['user_data'] = CLOUD_CONFIG try: resp = self.ibm_vpc_client.create_instance(instance_prototype) except ApiException as e: if e.code == 400 and 'already exists' in e.message: return self.get_instance_data() elif e.code == 400 and 'over quota' in e.message: logger.debug("Create VM instance {} failed due to quota limit" .format(self.name)) else: logger.debug("Create VM instance {} failed with status code {}" .format(self.name, str(e.code))) raise e logger.debug("VM instance {} created successfully ".format(self.name)) return resp.result def _attach_floating_ip(self, instance): """ Attach a floating IP address only if the VM is the master instance """ fip = self.config['floating_ip'] fip_id = self.config['floating_ip_id'] # logger.debug('Attaching floating IP {} to VM instance {}'.format(fip, instance['id'])) # we need to check if floating ip is not attached already. if not, attach it to instance instance_primary_ni = instance['primary_network_interface'] if instance_primary_ni['primary_ipv4_address'] and instance_primary_ni['id'] == fip_id: # floating ip already atteched. do nothing logger.debug('Floating IP {} already attached to eth0'.format(fip)) else: self.ibm_vpc_client.add_instance_network_interface_floating_ip( instance['id'], instance['network_interfaces'][0]['id'], fip_id) def get_instance_data(self): """ Returns the instance information """ instances_data = self.ibm_vpc_client.list_instances(name=self.name).get_result() if len(instances_data['instances']) > 0: self.instance_data = instances_data['instances'][0] return self.instance_data return None def get_instance_id(self): """ Returns the instance ID """ instance_data = self.get_instance_data() if instance_data: self.instance_id = instance_data['id'] return self.instance_id logger.debug('VM instance {} does not exists'.format(self.name)) return None def _get_ip_address(self): """ Requests the the primary network IP address """ ip_address = None if self.instance_id: while not ip_address: instance_data = self.ibm_vpc_client.get_instance(self.instance_id).get_result() ip_address = instance_data['primary_network_interface']['primary_ipv4_address'] return ip_address def create(self, check_if_exists=False, start=True): """ Creates a new VM instance """ instance = None vsi_exists = True if self.instance_id else False if check_if_exists and not vsi_exists: logger.debug('Checking if VM instance {} already exists'.format(self.name)) instances_data = self.get_instance_data() if instances_data: logger.debug('VM instance {} already exists'.format(self.name)) vsi_exists = True self.instance_id = instances_data['id'] if not vsi_exists: instance = self._create_instance() self.instance_id = instance['id'] self.ip_address = self._get_ip_address() if self.public and instance: self._attach_floating_ip(instance) if start: # In IBM VPC, VM instances are automatically started on create if vsi_exists: self.start() return self.instance_id def start(self): logger.debug("Starting VM instance {}".format(self.name)) try: resp = self.ibm_vpc_client.create_instance_action(self.instance_id, 'start') except ApiException as e: if e.code == 404: pass else: raise e logger.debug("VM instance {} started successfully".format(self.name)) def _delete_instance(self): """ Deletes the VM instacne and the associated volume """ logger.debug("Deleting VM instance {}".format(self.name)) try: self.ibm_vpc_client.delete_instance(self.instance_id) except ApiException as e: if e.code == 404: pass else: raise e self.instance_id = None self.ip_address = None self.del_ssh_client() def _stop_instance(self): """ Stops the VM instacne and """ logger.debug("Stopping VM instance {}".format(self.name)) try: resp = self.ibm_vpc_client.create_instance_action(self.instance_id, 'stop') except ApiException as e: if e.code == 404: pass else: raise e def stop(self): if self.delete_on_dismantle: self._delete_instance() else: self._stop_instance() def delete(self): """ Deletes the VM instance """ self._delete_instance()
class IBMVPCInstance: def __init__(self, name, ibm_vpc_config, ibm_vpc_client=None, public=False): """ Initialize a IBMVPCInstance instance VMs can have master role, this means they will have a public IP address """ self.name = name.lower() self.config = ibm_vpc_config self.delete_on_dismantle = self.config['delete_on_dismantle'] self.profile_name = self.config['profile_name'] self.ibm_vpc_client = ibm_vpc_client or self._create_vpc_client() self.public = public self.ssh_client = None self.instance_id = None self.instance_data = None self.private_ip = None self.public_ip = None self.home_dir = '/root' self.ssh_credentials = { 'username': self.config['ssh_username'], 'password': self.config['ssh_password'], 'key_filename': self.config.get('ssh_key_filename', '~/.ssh/id_rsa') } self.validated = False def __str__(self): return f'VM instance {self.name} ({self.public_ip or self.private_ip})' def _create_vpc_client(self): """ Creates an IBM VPC python-sdk instance """ authenticator = IAMAuthenticator(self.iam_api_key) ibm_vpc_client = VpcV1(VPC_API_VERSION, authenticator=authenticator) ibm_vpc_client.set_service_url(self.config['endpoint'] + '/v1') # decorate instance public methods with except/retry logic decorate_instance(self.ibm_vpc_client, vpc_retry_on_except) return ibm_vpc_client def get_ssh_client(self): """ Creates an ssh client against the VM only if the Instance is the master """ if not self.validated and self.public and self.instance_id: # validate that private ssh key in ssh_credentials is a pair of public key on instance key_filename = self.ssh_credentials['key_filename'] key_filename = os.path.abspath(os.path.expanduser(key_filename)) if not os.path.exists(key_filename): raise LithopsValidationError( f"Private key file {key_filename} doesn't exist") initialization_data = self.ibm_vpc_client.get_instance_initialization( self.instance_id).get_result() private_res = paramiko.RSAKey(filename=key_filename).get_base64() key = None names = [] for k in initialization_data['keys']: public_res = self.ibm_vpc_client.get_key( k['id']).get_result()['public_key'].split(' ')[1] if public_res == private_res: self.validated = True break else: names.append(k['name']) if not self.validated: raise LithopsValidationError( f"No public key from keys: {names} on master {self} not a pair for private ssh key {key_filename}" ) if self.private_ip or self.public_ip: if not self.ssh_client: self.ssh_client = SSHClient(self.public_ip or self.private_ip, self.ssh_credentials) return self.ssh_client def del_ssh_client(self): """ Deletes the ssh client """ if self.ssh_client: try: self.ssh_client.close() except Exception: pass self.ssh_client = None def is_ready(self, verbose=False): """ Checks if the VM instance is ready to receive ssh connections """ login_type = 'password' if 'password' in self.ssh_credentials and \ not self.public else 'publickey' try: self.get_ssh_client().run_remote_command('id') except LithopsValidationError as e: raise e except Exception as e: if verbose: logger.debug( f'SSH to {self.private_ip} failed ({login_type}): {e}') self.del_ssh_client() return False return True def wait_ready(self, verbose=False): """ Waits until the VM instance is ready to receive ssh connections """ logger.debug(f'Waiting {self} to become ready') start = time.time() while (time.time() - start < INSTANCE_START_TIMEOUT): if self.is_ready(verbose=verbose): start_time = round(time.time() - start, 2) logger.debug(f'{self} ready in {start_time} seconds') return True time.sleep(5) raise TimeoutError(f'Readiness probe expired on {self}') def _create_instance(self, user_data): """ Creates a new VM instance """ logger.debug("Creating new VM instance {}".format(self.name)) security_group_identity_model = { 'id': self.config['security_group_id'] } subnet_identity_model = {'id': self.config['subnet_id']} primary_network_interface = { 'name': 'eth0', 'subnet': subnet_identity_model, 'security_groups': [security_group_identity_model] } boot_volume_data = { 'capacity': self.config['boot_volume_capacity'], 'name': '{}-{}-boot'.format(self.name, str(uuid.uuid4())[:4]), 'profile': { 'name': self.config['boot_volume_profile'] } } boot_volume_attachment = { 'delete_volume_on_instance_delete': True, 'volume': boot_volume_data } key_identity_model = {'id': self.config['key_id']} instance_prototype = {} instance_prototype['name'] = self.name instance_prototype['keys'] = [key_identity_model] instance_prototype['profile'] = {'name': self.profile_name} instance_prototype['resource_group'] = { 'id': self.config['resource_group_id'] } instance_prototype['vpc'] = {'id': self.config['vpc_id']} instance_prototype['image'] = {'id': self.config['image_id']} instance_prototype['zone'] = {'name': self.config['zone_name']} instance_prototype['boot_volume_attachment'] = boot_volume_attachment instance_prototype[ 'primary_network_interface'] = primary_network_interface if user_data: instance_prototype['user_data'] = user_data try: resp = self.ibm_vpc_client.create_instance(instance_prototype) except ApiException as e: if e.code == 400 and 'already exists' in e.message: return self.get_instance_data() elif e.code == 400 and 'over quota' in e.message: logger.debug( "Create VM instance {} failed due to quota limit".format( self.name)) else: logger.debug( "Create VM instance {} failed with status code {}: {}". format(self.name, str(e.code), e.message)) raise e logger.debug("VM instance {} created successfully ".format(self.name)) return resp.result def _attach_floating_ip(self, instance): """ Attach a floating IP address only if the VM is the master instance """ fip = self.config['floating_ip'] fip_id = self.config['floating_ip_id'] # logger.debug('Attaching floating IP {} to VM instance {}'.format(fip, instance['id'])) # we need to check if floating ip is not attached already. if not, attach it to instance instance_primary_ni = instance['primary_network_interface'] if instance_primary_ni['primary_ipv4_address'] and instance_primary_ni[ 'id'] == fip_id: # floating ip already atteched. do nothing logger.debug('Floating IP {} already attached to eth0'.format(fip)) else: self.ibm_vpc_client.add_instance_network_interface_floating_ip( instance['id'], instance['network_interfaces'][0]['id'], fip_id) def get_instance_data(self): """ Returns the instance information """ instances_data = self.ibm_vpc_client.list_instances( name=self.name).get_result() if len(instances_data['instances']) > 0: self.instance_data = instances_data['instances'][0] return self.instance_data return None def get_instance_id(self): """ Returns the instance ID """ instance_data = self.get_instance_data() if instance_data: self.instance_id = instance_data['id'] return self.instance_id logger.debug('VM instance {} does not exists'.format(self.name)) return None def get_private_ip(self): """ Requests the private IP address """ while not self.private_ip or self.private_ip == '0.0.0.0': time.sleep(1) instance_data = self.get_instance_data() self.private_ip = instance_data['primary_network_interface'][ 'primary_ipv4_address'] return self.private_ip def get_public_ip(self): """ Requests the public IP address """ if self.public and self.public_ip: return self.public_ip return None def create(self, check_if_exists=False, user_data=None): """ Creates a new VM instance """ instance = None vsi_exists = True if self.instance_id else False if check_if_exists and not vsi_exists: logger.debug('Checking if VM instance {} already exists'.format( self.name)) instances_data = self.get_instance_data() if instances_data: logger.debug('VM instance {} already exists'.format(self.name)) vsi_exists = True self.instance_id = instances_data['id'] if not vsi_exists: instance = self._create_instance(user_data=user_data) self.instance_id = instance['id'] self.private_ip = self.get_private_ip() else: self.start() if self.public and instance: self._attach_floating_ip(instance) return self.instance_id def start(self): logger.debug("Starting VM instance {}".format(self.name)) try: self.ibm_vpc_client.create_instance_action(self.instance_id, 'start') except ApiException as e: if e.code == 404: pass else: raise e logger.debug("VM instance {} started successfully".format(self.name)) def _delete_instance(self): """ Deletes the VM instacne and the associated volume """ logger.debug("Deleting VM instance {}".format(self.name)) try: self.ibm_vpc_client.delete_instance(self.instance_id) except ApiException as e: if e.code == 404: pass else: raise e self.instance_id = None self.private_ip = None self.del_ssh_client() def _stop_instance(self): """ Stops the VM instacne and """ logger.debug("Stopping VM instance {}".format(self.name)) try: self.ibm_vpc_client.create_instance_action(self.instance_id, 'stop') except ApiException as e: if e.code == 404: pass else: raise e def stop(self): if self.delete_on_dismantle: self._delete_instance() else: self._stop_instance() def delete(self): """ Deletes the VM instance """ self._delete_instance() def validate_capabilities(self): """ Validate hardware/os requirments specified in backend config """ if self.config.get('singlesocket'): cmd = "lscpu -p=socket|grep -v '#'" res = self.get_ssh_client().run_remote_command(cmd) sockets = set() for c in res: if c != '\n': sockets.add(c) if len(sockets) != 1: raise LithopsValidationError( f'Not using single CPU socket as specified, using {len(sockets)} sockets instead' )
class VMInstance: def __init__(self, config): self.public_ip = self.private_ip = self.config['ip_address'] self.ssh_client = None self.ssh_credentials = { 'username': self.config.get('ssh_user', 'root'), 'password': self.config.get('ssh_password', None), 'key_filename': self.config.get('ssh_key_filename', '~/.ssh/id_rsa') } logger.debug('{} created'.format(self)) def __str__(self): return 'VM instance {}'.format(self.ip_address) def get_ssh_client(self): """ Creates an ssh client against the VM only if the Instance is the master """ if self.public_ip: if not self.ssh_client: self.ssh_client = SSHClient(self.public_ip, self.ssh_credentials) return self.ssh_client def del_ssh_client(self): """ Deletes the ssh client """ if self.ssh_client: try: self.ssh_client.close() except Exception: pass self.ssh_client = None def is_ready(self, verbose=False): """ Checks if the VM is ready to receive ssh connections """ try: self.get_ssh_client().run_remote_command('id') except LithopsValidationError as e: raise e except Exception as e: if verbose: logger.debug(f'ssh to {self.private_ip} failed: {e}') self.del_ssh_client() return False return True def wait_ready(self, verbose=False): """ Waits until the VM is ready to receive ssh connections """ logger.debug(f'Waiting {self} to become ready') start = time.time() while (time.time() - start < INSTANCE_START_TIMEOUT): if self.is_ready(verbose=verbose): start_time = round(time.time() - start, 2) logger.debug(f'{self} ready in {start_time} seconds') return True time.sleep(5) raise TimeoutError(f'Readiness probe expired on {self}') def get_public_ip(self): """ Requests the the primary public IP address """ return self.public_ip def create(self, **kwargs): pass def start(self): pass def stop(self): pass def delete(self): pass
class EC2Instance: def __init__(self, name, ec2_config, ec2_client=None, public=False): """ Initialize a EC2Instance instance VMs can have master role, this means they will have a public IP address """ self.name = name.lower() self.config = ec2_config self.delete_on_dismantle = self.config['delete_on_dismantle'] self.instance_type = self.config['worker_instance_type'] self.region = self.config['region_name'] self.spot_instance = self.config['request_spot_instances'] self.ec2_client = ec2_client or self._create_ec2_client() self.public = public self.ssh_client = None self.instance_id = None self.instance_data = None self.private_ip = None self.public_ip = '0.0.0.0' self.fast_io = self.config.get('fast_io', False) self.home_dir = '/home/ubuntu' self.ssh_credentials = { 'username': self.config['ssh_username'], 'password': self.config['ssh_password'], 'key_filename': self.config.get('ssh_key_filename', '~/.ssh/id_rsa') } def __str__(self): ip = self.public_ip if self.public else self.private_ip if ip is None or ip == '0.0.0.0': return f'VM instance {self.name}' else: return f'VM instance {self.name} ({ip})' def _create_ec2_client(self): """ Creates an EC2 boto3 instance """ client_config = botocore.client.Config( user_agent_extra=self.config['user_agent'] ) ec2_client = boto3.client( 'ec2', aws_access_key_id=self.ec2_config['access_key_id'], aws_secret_access_key=self.ec2_config['secret_access_key'], config=client_config, region_name=self.region ) return ec2_client def get_ssh_client(self): """ Creates an ssh client against the VM only if the Instance is the master """ if self.public: if not self.ssh_client or self.ssh_client.ip_address != self.public_ip: self.ssh_client = SSHClient(self.public_ip, self.ssh_credentials) else: if not self.ssh_client or self.ssh_client.ip_address != self.private_ip: self.ssh_client = SSHClient(self.private_ip, self.ssh_credentials) return self.ssh_client def del_ssh_client(self): """ Deletes the ssh client """ if self.ssh_client: try: self.ssh_client.close() except Exception: pass self.ssh_client = None def is_ready(self, verbose=False): """ Checks if the VM instance is ready to receive ssh connections """ login_type = 'password' if 'password' in self.ssh_credentials and \ not self.public else 'publickey' try: self.get_ssh_client().run_remote_command('id') except LithopsValidationError as e: raise e except Exception as e: if verbose: logger.debug(f'SSH to {self.private_ip} failed ({login_type}): {e}') self.del_ssh_client() return False return True def wait_ready(self, verbose=False): """ Waits until the VM instance is ready to receive ssh connections """ logger.debug(f'Waiting {self} to become ready') start = time.time() while(time.time() - start < INSTANCE_START_TIMEOUT): if self.is_ready(verbose=verbose): start_time = round(time.time()-start, 2) logger.debug(f'{self} ready in {start_time} seconds') return True time.sleep(5) raise TimeoutError(f'Readiness probe expired on {self}') def _create_instance(self, user_data=None): """ Creates a new VM instance """ if self.fast_io: BlockDeviceMappings = [ { 'DeviceName': '/dev/xvda', 'Ebs': { 'VolumeSize': 100, 'DeleteOnTermination': True, 'VolumeType': 'gp2', # 'Iops' : 10000, }, }, ] else: BlockDeviceMappings = None LaunchSpecification = { "ImageId": self.config['target_ami'], "InstanceType": self.instance_type, "SecurityGroupIds": [self.config['security_group_id']], "EbsOptimized": False, "IamInstanceProfile": {'Name': self.config['iam_role']}, "Monitoring": {'Enabled': False} } if BlockDeviceMappings is not None: LaunchSpecification['BlockDeviceMappings'] = BlockDeviceMappings if 'key_name' in self.config: LaunchSpecification['KeyName'] = self.config['key_name'] if self.spot_instance and not self.public: logger.debug("Creating new VM instance {} (Spot)".format(self.name)) if user_data: # Allow master VM to access workers trough ssh password LaunchSpecification['UserData'] = b64s(user_data) spot_requests = self.ec2_client.request_spot_instances( SpotPrice=str(self.config['spot_price']), InstanceCount=1, LaunchSpecification=LaunchSpecification)['SpotInstanceRequests'] request_ids = [r['SpotInstanceRequestId'] for r in spot_requests] pending_request_ids = request_ids while pending_request_ids: time.sleep(3) spot_requests = self.ec2_client.describe_spot_instance_requests( SpotInstanceRequestIds=request_ids)['SpotInstanceRequests'] failed_requests = [r for r in spot_requests if r['State'] == 'failed'] if failed_requests: failure_reasons = {r['Status']['Code'] for r in failed_requests} logger.debug(failure_reasons) raise Exception( "The spot request failed for the following reason{s}: {reasons}" .format( s='' if len(failure_reasons) == 1 else 's', reasons=', '.join(failure_reasons))) pending_request_ids = [ r['SpotInstanceRequestId'] for r in spot_requests if r['State'] == 'open'] self.ec2_client.create_tags( Resources=[r['InstanceId'] for r in spot_requests], Tags=[{'Key': 'Name', 'Value': self.name}] ) filters = [{'Name': 'instance-id', 'Values': [r['InstanceId'] for r in spot_requests]}] resp = self.ec2_client.describe_instances(Filters=filters)['Reservations'][0] else: logger.debug("Creating new VM instance {}".format(self.name)) LaunchSpecification['MinCount'] = 1 LaunchSpecification['MaxCount'] = 1 LaunchSpecification["TagSpecifications"] = [{"ResourceType": "instance", "Tags": [{'Key': 'Name', 'Value': self.name}]}] LaunchSpecification["InstanceInitiatedShutdownBehavior"] = 'terminate' if self.delete_on_dismantle else 'stop' if user_data: LaunchSpecification['UserData'] = user_data # if not self.public: # LaunchSpecification['NetworkInterfaces'] = [{'AssociatePublicIpAddress': False, 'DeviceIndex': 0}] resp = self.ec2_client.run_instances(**LaunchSpecification) logger.debug("VM instance {} created successfully ".format(self.name)) return resp['Instances'][0] def get_instance_data(self): """ Returns the instance information """ if self.instance_id: instances = self.ec2_client.describe_instances(InstanceIds=[self.instance_id]) instances = instances['Reservations'][0]['Instances'] if len(instances) > 0: self.instance_data = instances[0] return self.instance_data else: filters = [{'Name': 'tag:Name', 'Values': [self.name]}] resp = self.ec2_client.describe_instances(Filters=filters) if len(resp['Reservations']) > 0: self.instance_data = resp['Reservations'][0]['Instances'][0] return self.instance_data return None def get_instance_id(self): """ Returns the instance ID """ if self.instance_id: return self.instance_id instance_data = self.get_instance_data() if instance_data: self.instance_id = instance_data['InstanceId'] return self.instance_id logger.debug('VM instance {} does not exists'.format(self.name)) return None def get_private_ip(self): """ Requests the private IP address """ while not self.private_ip: instance_data = self.get_instance_data() if instance_data and 'PrivateIpAddress' in instance_data: self.private_ip = instance_data['PrivateIpAddress'] else: time.sleep(1) return self.private_ip def get_public_ip(self): """ Requests the public IP address """ while self.public and (not self.public_ip or self.public_ip == '0.0.0.0'): instance_data = self.get_instance_data() if instance_data and 'PublicIpAddress' in instance_data: self.public_ip = instance_data['PublicIpAddress'] else: time.sleep(1) return self.public_ip def create(self, check_if_exists=False, user_data=None): """ Creates a new VM instance """ vsi_exists = True if self.instance_id else False if check_if_exists and not vsi_exists: logger.debug('Checking if VM instance {} already exists'.format(self.name)) instance_data = self.get_instance_data() if instance_data: logger.debug('VM instance {} already exists'.format(self.name)) vsi_exists = True self.instance_id = instance_data['InstanceId'] self.private_ip = instance_data['PrivateIpAddress'] if not vsi_exists: instance_data = self._create_instance(user_data=user_data) self.instance_id = instance_data['InstanceId'] self.private_ip = instance_data['PrivateIpAddress'] self.public_ip = self.get_public_ip() else: self.start() return self.instance_id def start(self): logger.info("Starting VM instance {}".format(self.name)) try: self.ec2_client.start_instances(InstanceIds=[self.instance_id]) self.public_ip = self.get_public_ip() except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'IncorrectInstanceState': time.sleep(20) return self.start() else: raise e logger.debug("VM instance {} started successfully".format(self.name)) def _delete_instance(self): """ Deletes the VM instance and the associated volume """ logger.debug("Deleting VM instance {}".format(self.name)) self.ec2_client.terminate_instances(InstanceIds=[self.instance_id]) self.instance_id = None self.private_ip = None self.public_ip = None self.del_ssh_client() def _stop_instance(self): """ Stops the VM instacne and """ logger.debug("Stopping VM instance {}".format(self.name)) self.ec2_client.stop_instances(InstanceIds=[self.instance_id]) def stop(self): if self.delete_on_dismantle: self._delete_instance() else: self._stop_instance() def delete(self): """ Deletes the VM instance """ self._delete_instance() def validate_capabilities(self): """ Validate hardware/os requirments specified in backend config """ pass