def _init_security_group(self):
"""
Initialize security group.
"""
# Checks if security group exists
security_group_id = None
security_group_name = self._security_group.lower()
with _exception_handler(gen_msg=('no_find', "security groups")):
for security_group in self._neutron_client.list_security_groups(
)['security_groups']:
if security_group['name'].lower() == security_group_name:
self._security_group = security_group['name']
security_group_id = security_group['id']
# Create security group if not exists
if security_group_id is None:
with _exception_handler(gen_msg=('created_failed',
"security groups")):
security_group_id = self._neutron_client.create_security_group(
{
'security_group': {
'name': self._security_group,
'description': _utl.gen_msg('accelize_generated'),
}
})['security_group']['id']
_get_logger().debug(
_utl.gen_msg('created_named', 'security group',
self._security_group))
# Verify rules associated to security group for host IP address
public_ip = _utl.get_host_public_ip()
# Create rule on SSH and HTTP
for port in self.ALLOW_PORTS:
with _exception_handler(filter_error_codes=(409, )):
self._neutron_client.create_security_group_rule({
'security_group_rule': {
'direction': 'ingress',
'port_range_min': str(port),
'port_range_max': str(port),
'remote_ip_prefix': public_ip,
'protocol': 'tcp',
'security_group_id': security_group_id
}
})
_get_logger().debug(
_utl.gen_msg('authorized_ip', public_ip, self._security_group))
def _init_role(self):
"""
Initialize RAM role.
This role allow to perform actions defined by policy.
"""
if self._request(
'CreateRole', domain='ram', RoleName=self._role,
AssumeRolePolicyDocument=_json.dumps(
self.ASSUME_ROLE_POLICY_DOCUMENT),
Description=_utl.gen_msg('accelize_generated'),
error_code_ignore='EntityAlreadyExists'):
_get_logger().debug(_utl.gen_msg(
'created_named', 'RAM role', self._role))
def _init_role(self):
"""
Initialize IAM role.
This role allow to perform actions defined by policy.
"""
with _exception_handler(filter_error_codes='EntityAlreadyExists'):
role = self._iam_resource.create_role(
RoleName=self._role,
AssumeRolePolicyDocument=_json_dumps(
self.ASSUME_ROLE_POLICY_DOCUMENT),
Description=_utl.gen_msg('accelize_generated'))
_get_logger().debug(_utl.gen_msg('created_named', 'IAM role',
role))
def _init_policy(self):
"""
Initialize RAM policy.
This policy allow instance to:
- Load FPGA bitstream.
- Access to OSS buckets objects for read and write.
"""
if self._request(
'CreatePolicy', domain='ram', PolicyName=self._policy,
PolicyDocument=_json.dumps(self.POLICY_DOCUMENT),
Description=_utl.gen_msg('accelize_generated'),
error_code_ignore='EntityAlreadyExists'):
_get_logger().debug(_utl.gen_msg(
'created_named', 'RAM policy', self._policy))
def _attach_instance_profile_role(self):
"""
Attach IAM role to IAM instance profile.
"""
# Attach role to instance profile
with _exception_handler(filter_error_codes='LimitExceeded'):
with _utl.Timeout(self.TIMEOUT,
sleep=self._TIMEOUT_SLEEP) as timeout:
while True:
try:
self._iam_client.add_role_to_instance_profile(
InstanceProfileName=self._instance_profile_name,
RoleName=self._role)
break
# Some time, instance_profile is not ready immediately
except _ClientError as exception:
if (exception.response['Error']['Code'] ==
'NoSuchEntityException'):
if timeout.reached():
raise _exc.HostRuntimeException(gen_msg=(
'timeout',
'IAM instance_profile and role attachment'
))
continue
raise
_get_logger().debug(
_utl.gen_msg('attached_to', 'IAM role', self._role,
'IAM instance profile',
self._instance_profile_name))
def _init_policy(self):
"""
Initialize IAM policy.
This policy allow instance to:
- Load FPGA bitstream.
- Access to S3 buckets objects for read and write.
Returns:
str: 'policy'
"""
# Create a policy
with _exception_handler(filter_error_codes='EntityAlreadyExists'):
self._iam_client.create_policy(PolicyName=self._policy,
PolicyDocument=_json_dumps(
self.POLICY_DOCUMENT))
_get_logger().debug(
_utl.gen_msg('created_named', 'policy', self._policy))
with _exception_handler():
response = self._iam_client.list_policies(Scope='Local',
OnlyAttached=False,
MaxItems=100)
for policy_item in response['Policies']:
if policy_item['PolicyName'] == self._policy:
self._policy_arn = policy_item['Arn']
# 'policy' returns str is used set future object ID
return 'policy'
raise _exc.HostConfigurationException(gen_msg=('created_failed_named',
'IAM policy',
self._policy))
def _init_key_pair(self):
"""
Initializes key pair.
"""
# Checks if Key pairs exists, needs to get the full pairs list
# and compare in lower case because Boto perform its checks case
# sensitive and AWS use case insensitive names.
with _exception_handler():
key_pairs = self._ec2_client.describe_key_pairs()
name_lower = self._key_pair.lower()
for key_pair in key_pairs['KeyPairs']:
key_pair_name = key_pair['KeyName']
if key_pair_name.lower() == name_lower:
self._key_pair = key_pair_name
return
# Key does not exist on the CSP, create it
with _exception_handler():
key_pair = self._ec2_resource.create_key_pair(
KeyName=self._key_pair)
_utl.create_key_pair_file(self._key_pair, key_pair.key_material)
_get_logger().debug(
_utl.gen_msg("created_named", "key pair", self._key_pair))
def _init_security_group(self):
"""
Initialize security group.
"""
response = self._request('DescribeSecurityGroups')
# Checks if security group exists
lower_name = self._security_group.lower()
for security_group in response['SecurityGroups']['SecurityGroup']:
security_group_name = security_group['SecurityGroupName']
if security_group_name.lower() == lower_name:
# Update security group name
self._security_group = security_group_name
self._security_group_id = security_group['SecurityGroupId']
break
# Creates security group if not exists
if not self._security_group_id:
response = self._request(
'CreateSecurityGroup',
SecurityGroupName=self._security_group,
Description=_utl.gen_msg('accelize_generated'))
self._security_group_id = response['SecurityGroupId']
_get_logger().debug(_utl.gen_msg(
'created_named', 'security group', self._security_group_id))
# Adds host IP to security group if not already done
public_ip = _utl.get_host_public_ip()
rules = list()
for port in self.ALLOW_PORTS:
rules.append(dict(
Priority=1, IpProtocol='tcp', PortRange='%s/%s' % (port, port),
SourceCidrIp=public_ip))
for rule in rules:
self._request(
'AuthorizeSecurityGroup',
SecurityGroupId=self._security_group_id, **rule)
_get_logger().debug(
_utl.gen_msg('authorized_ip', public_ip, self._security_group))
def _attach_role_policy(self):
"""
Attach IAM policy to IAM role.
"""
with _exception_handler(filter_error_codes='EntityAlreadyExists'):
self._iam_client.attach_role_policy(PolicyArn=self._policy_arn,
RoleName=self._role)
_get_logger().debug(
_utl.gen_msg('attached_to', 'IAM policy', self._policy_arn,
'IAM role', self._role))
def _attach_role_policy(self):
"""
Attach RAM policy to RAM role.
"""
if self._request(
'AttachPolicyToRole', domain='ram', PolicyType='Custom',
PolicyName=self._policy, RoleName=self._role,
error_code_ignore='EntityAlreadyExists'):
_get_logger().debug(_utl.gen_msg(
'attached_to', 'RAM policy', self._policy,
'RAM role', self._role))
def _init_instance_profile(self):
"""
Initialize IAM instance profile.
This instance_profile allow to perform actions defined by role.
"""
# Create instance profile
with _exception_handler(filter_error_codes='EntityAlreadyExists'):
self._iam_client.create_instance_profile(
InstanceProfileName=self._instance_profile_name)
_get_logger().debug(
_utl.gen_msg('created_named', 'IAM instance profile',
self._instance_profile_name))
def _start_new_instance(self):
"""
Start a new instance.
Returns:
object: Instance
str: Instance ID
"""
kwargs = dict(ImageId=self._image_id,
InstanceType=self._instance_type,
KeyName=self._key_pair,
SecurityGroups=[self._security_group],
IamInstanceProfile={'Name': 'AccelizeLoadFPGA'},
InstanceInitiatedShutdownBehavior='terminate',
TagSpecifications=[{
'ResourceType':
'instance',
'Tags': [{
'Key': 'Generated',
'Value': _utl.gen_msg('accelize_generated')
}, {
'Key': 'Name',
'Value': self._get_host_name()
}, {
'Key': 'Apyfal',
'Value': self._get_tag()
}]
}],
MinCount=1,
MaxCount=1,
UserData=self._user_data)
if self._block_devices:
kwargs['BlockDeviceMappings'] = self._block_devices
if self._spot_instance:
_utl.recursive_update(kwargs, self.SPOT_INSTANCE_MARKET_OPTIONS)
if self._spot_block_duration:
kwargs['InstanceMarketOptions']['SpotOptions'][
'BlockDurationMinutes'] = self._spot_block_duration
_utl.recursive_update(kwargs, self._boto3_create_instances_kwargs)
# Create instance
with _exception_handler():
instance = self._ec2_resource.create_instances(**kwargs)[0]
return instance, instance.id
def _start_new_instance(self):
"""
Starts a new instance.
Returns:
dict: Instance
str: Instance ID
"""
# Gets maximum Internet bandwidth
response = self._request('DescribeBandwidthLimitation',
InstanceType=self._instance_type)
max_bandwidth = response['Bandwidths']['Bandwidth'][0]['Max']
# Creates instance
kwargs = dict(
ImageId=self._image_id, InstanceType=self._instance_type,
SecurityGroupId=self._security_group_id,
InstanceName=self._get_host_name(),
Description=_utl.gen_msg('accelize_generated'),
InternetMaxBandwidthOut=max_bandwidth,
KeyPairName=self._key_pair, RamRoleName=self._role,
UserData=_b64encode(self._user_data).decode(),
parameters={'Tag.1.Key': 'Apyfal', 'Tag.1.Value': self._get_tag()})
_utl.recursive_update(kwargs, self._acs_create_instance_kwargs)
response = self._request('CreateInstance', **kwargs)
instance_id = response['InstanceId']
# Allocates public IP address
self._instance_request(
'AllocatePublicIpAddress', status_desc='allocating IP address',
InstanceId=instance_id)
# Starts instance
self._instance_request(
'StartInstance', status_desc='starting', InstanceId=instance_id)
# Return basic instance description as instance and instance ID
return {'InstanceId': instance_id}, instance_id
def _init_key_pair(self):
"""
Initializes key pair.
Returns:
bool: True if reuses existing key
"""
# Get key pair from CSP is exists
key_pair_name = self._key_pair.lower()
with _exception_handler(gen_msg=('no_find', "key pair")):
for key_pair in self._nova_client.keypairs.list():
if key_pair.name.lower() == key_pair_name:
self._key_pair = key_pair.name
return
# Create key pair if not exists
with _exception_handler(gen_msg=('created_failed', "key pair")):
key_pair = self._nova_client.keypairs.create_keypair(
name=self._key_pair)
_utl.create_key_pair_file(self._key_pair, key_pair.private_key)
_get_logger().debug(
_utl.gen_msg("created_named", "key pair", self._key_pair))
def start(self,
accelerator=None,
accel_parameters=None,
stop_mode=None,
image_id=None,
instance_type=None):
"""
Start instance if not already started. Create instance if necessary.
Needs "accel_client" or "accel_parameters".
Args:
accelerator (str): Name of the accelerator.
accel_parameters (dict): Can override parameters from accelerator
client.
image_id (str): Force the use of specified image ID.
instance_type (str): Force the use of specified instance type.
stop_mode (str or int): See "stop_mode" property for more
information.
"""
# Updates stop mode
self.stop_mode = stop_mode
# Starts instance only if not already started
if self._host_ip is None:
# Get parameters from accelerator
self._set_accelerator_requirements(
accelerator=accelerator,
accel_parameters=accel_parameters,
image_id=image_id,
instance_type=instance_type)
# Checks CSP credential
self._check_credential()
# Creates and starts instance if not exists
if self.instance_id is None:
_get_logger().info("Configuring host on %s instance...",
self._host_type)
with self._stop_silently_on_exception():
self._create_instance()
with self._stop_silently_on_exception():
self._instance, self._instance_id = \
self._start_new_instance()
_get_logger().debug(
_utl.gen_msg('created_named', 'instance',
self._instance_id))
# If exists, starts it directly
else:
self._start_existing_instance(self._status())
# Waiting for instance provisioning
with self._stop_silently_on_exception():
self._wait_instance_ready()
# Update instance URL
self._host_ip = self.host_ip
self._url = _utl.format_url(self._host_ip,
force_secure=bool(self._ssl_cert_crt))
# Waiting for the instance to boot
self._wait_instance_boot()
_get_logger().info("Host ready")
# If Host IP exists exists, checks if reachable
elif self.ALLOW_PORTS and not _utl.check_port(self.host_ip, 80):
raise _exc.HostRuntimeException(gen_msg=('unable_reach_port',
self.host_ip, 80))
def _init_security_group(self):
"""
Initialize security group.
"""
# Get list of security groups
# Checks if Key pairs exists, like for key pairs
# needs case insensitive names check
with _exception_handler():
security_groups = self._ec2_client.describe_security_groups()
name_lower = self._security_group.lower()
group_exists = False
security_group_id = ''
for security_group in security_groups['SecurityGroups']:
group_name = security_group['GroupName']
if group_name.lower() == name_lower:
# Update name
self._security_group = group_name
# Get group ID
security_group_id = security_group['GroupId']
# Mark as existing
group_exists = True
break
# Try to create security group if not exist
if not group_exists:
# Get VPC
with _exception_handler():
vpc_id = self._ec2_client.describe_vpcs().get('Vpcs',
[{}])[0].get(
'VpcId', '')
with _exception_handler():
response = self._ec2_client.create_security_group(
GroupName=self._security_group,
Description=_utl.gen_msg('accelize_generated'),
VpcId=vpc_id)
# Get group ID
security_group_id = response['GroupId']
_get_logger().debug(
_utl.gen_msg('created_named', 'security group',
security_group_id))
# Add host IP to security group if not already done
public_ip = _utl.get_host_public_ip()
ip_permissions = []
for port in self.ALLOW_PORTS:
ip_permissions.append({
'IpProtocol': 'tcp',
'FromPort': port,
'ToPort': port,
'IpRanges': [{
'CidrIp': public_ip
}]
})
with _exception_handler(
filter_error_codes='InvalidPermission.Duplicate'):
self._ec2_client.authorize_security_group_ingress(
GroupId=security_group_id, IpPermissions=ip_permissions)
_get_logger().debug(
_utl.gen_msg('authorized_ip', public_ip, self._security_group))