def _set_accelerator_requirements(
self, accelerator=None, accel_parameters=None,
request_to_server=True):
"""
Configures instance with accelerator client parameters.
Needs "accel_client" or "accel_parameters".
Args:
accelerator (str): Name of the accelerator
accel_parameters (dict): Can override parameters from accelerator
client.
request_to_server (bool): If True, get parameters from server.
Raises:
apyfal.exceptions.HostConfigurationException:
Parameters are not valid..
"""
# Gets parameters
self._config_env = dict()
if accelerator is not None and request_to_server:
_utl.recursive_update(
self._config_env, self._config.get_host_requirements(
self._host_type, accelerator))
if accel_parameters is not None:
_utl.recursive_update(
self._config_env, _deepcopy(accel_parameters))
# Gets accelerator name
self._accelerator = self._config_env.pop(
'accelerator', None) or accelerator
def stop(self, info_dict=None, full_stop=True):
"""
Stop accelerator.
Args:
full_stop (bool): If True, send stop request to accelerator
application. If False only clean up accelerator client
environment.
info_dict (dict or None): If a dict passed, this dict is updated
with extra information from current operation.
"""
self._stopped = True
# Stops
if full_stop:
response = self._stop()
if info_dict is not None and response:
_utl.recursive_update(info_dict, response)
# Clears temporary directory
try:
_rmtree(self._cache['tmp_dir'])
except (OSError, KeyError):
pass
# Clears cache
self._cache.clear()
def _start_new_instance(self):
"""
Start a new instance.
Returns:
object: Instance
str: Instance ID
"""
kwargs = dict(name=self._get_host_name(),
key_name=self._key_pair,
security_groups=[self._security_group],
userdata=self._user_data,
meta={'Apyfal': self._get_tag()})
with _exception_handler(gen_msg=('unable_to', "start")):
kwargs["image"] = self._nova_client.glance.find_image(
self._image_id)
kwargs["flavor"] = self._nova_client.flavors.get(
self._instance_type)
_utl.recursive_update(kwargs,
self._nova_client_create_server_kwargs)
instance = self._nova_client.servers.create(**kwargs)
return instance, instance.id
def _update_with_default_configuration(self, to_update, config_env):
"""
Get default configuration environment from metering server.
Args:
to_update (dict): Environment to update
config_env (dict): User environment
Returns:
dict: Default configuration environment
"""
# Updates credentials in configuration with local file
section = self._config['accelize']
section['client_id'] = (section.get('client_id')
or config_env.get('client_id')
or to_update.get('client_id'))
section['secret_id'] = (section.get('secret_id')
or config_env.get('client_secret')
or to_update.get('client_secret'))
# Gets default configuration
try:
_utl.recursive_update(
to_update,
self._config.get_host_configurations()[self._host_type][
self._name][self._region])
# Empty default configuration is possible
except KeyError:
return
def start(self,
src=None,
info_dict=None,
host_env=None,
reload=None,
reset=None,
**parameters):
"""
Configures accelerator.
Args:
src (path-like object or file-like object): Depending on the
accelerator, a configuration data need to be loaded before
a process can be run.
Path-like object can be path, URL or cloud object URL.
parameters (str, path-like object or dict):
Accelerator configuration specific
parameters Can also be a full configuration parameters
dictionary (Or JSON equivalent as str literal or apyfal.storage
URL to file) Parameters dictionary override default
configuration values, individuals specific parameters overrides
parameters dictionary values. Take a look to accelerator
documentation for more information on possible parameters.
Path-like object can be path, URL or cloud object URL.
reload (bool): Force reload of FPGA bitstream.
reset (bool): Force reset of FPGA logic.
info_dict (dict or None): If a dict passed, this dict is updated
with extra information from current operation.
host_env (dict): Overrides Accelerator "env".
"""
self._stopped = False
_get_logger().info("Configuring accelerator...")
# Configure start
parameters = self._get_parameters(parameters,
self._configuration_parameters)
parameters['env'].update(host_env or dict())
# Set FPGA reset and reload options
if reload is not None:
parameters['app']['reload'] = reload
if reset is not None:
parameters['app']['reset'] = reset
# Handle files
with self._data_file(src, parameters, ('src', 'datafile'),
mode='rb') as src:
# Starts
response = self._start(src, parameters)
# Check response status
self._raise_for_status(response, "Failed to configure accelerator: ")
_get_logger().info("Accelerator ready")
# Update info dict
if info_dict is not None and response:
_utl.recursive_update(info_dict, response)
def mount(self):
"""Mount storage."""
storage_parameters = _deepcopy(self.STORAGE_PARAMETERS)
self._update_parameter(storage_parameters)
_utl.recursive_update(storage_parameters, self._storage_parameters)
return _pycosio.mount(storage=self.STORAGE_NAME,
extra_root=self.EXTRA_ROOT,
storage_parameters=storage_parameters,
unsecure=self._unsecure)
def _get_parameters(parameters, default_parameters, copy=True):
"""
Gets parameters from different sources, and merge them together.
If 'parameters' contain a key named 'parameters', it will be
read as a full parameter dict, or JSON literal or JSON file URL.
Other keys from 'parameters' will be merged to the 'specific'
section of the result dict.
Args:
parameters (dict): parameters
default_parameters (dict): default parameters
copy (bool): If True return a copy of updated default_parameters,
else update directly.
Returns:
dict : parameters.
"""
# Takes default parameters as basis
if copy:
result = _deepcopy(default_parameters)
else:
result = default_parameters
# Gets parameters from included JSON file
json_parameters = parameters.pop('parameters', None)
if json_parameters:
# Reads JSON parameter from file or literal
if not isinstance(json_parameters, dict):
# JSON literal
if json_parameters.rstrip().startswith('{'):
json_parameters = _json.loads(json_parameters)
# JSON file
else:
with _srg.open(json_parameters, 'rt') as json_file:
json_parameters = _json.load(json_file)
# Merges to result
_utl.recursive_update(result, json_parameters)
# Merges other parameters to specific section of parameters
try:
specific = result['app']['specific']
except KeyError:
specific = result['app']['specific'] = dict()
_utl.recursive_update(specific, parameters)
return result
def process(self, src=None, dst=None, info_dict=None, **parameters):
"""
Processes with accelerator.
Args:
src (path-like object or file-like object):
Source data to process.
Path-like object can be path, URL or cloud object URL.
dst (path-like object or file-like object):
Processed data destination.
Path-like object can be path, URL or cloud object URL.
parameters (path-like object, str or dict): Accelerator process
specific parameters
Can also be a full process parameters dictionary
(Or JSON equivalent as str literal) Parameters dictionary
override default configuration
values, individuals specific parameters overrides parameters
dictionary values. Take a look to accelerator documentation for
more information on possible parameters.
Path-like object can be path, URL or cloud object URL.
info_dict (dict or None): If a dict passed, this dict is updated
with extra information from current operation.
Returns:
Result from process operation, depending used accelerator.
"""
# Configures processing
parameters = self._get_parameters(parameters, self._process_parameters)
# Handle files
with self._data_file(src, parameters, ('src', 'file_in'),
mode='rb') as src:
with self._data_file(dst,
parameters, ('dst', 'file_out'),
mode='wb') as dst:
# Processes
response = self._process(src, dst, parameters)
# Check response status
self._raise_for_status(response, "Processing failed: ")
# Get result from response
result = response['app'].pop('specific', None)
# Update info dict
if info_dict is not None and response:
_utl.recursive_update(info_dict, response)
# Returns result
return result
def _start_new_instance(self):
"""
Start a new instance.
Returns:
object: Instance
str: Instance ID
"""
kwargs = dict(ImageId=self._image_id,
InstanceType=self._instance_type,
KeyName=self._key_pair,
SecurityGroups=[self._security_group],
IamInstanceProfile={'Name': 'AccelizeLoadFPGA'},
InstanceInitiatedShutdownBehavior='terminate',
TagSpecifications=[{
'ResourceType':
'instance',
'Tags': [{
'Key': 'Generated',
'Value': _utl.gen_msg('accelize_generated')
}, {
'Key': 'Name',
'Value': self._get_host_name()
}, {
'Key': 'Apyfal',
'Value': self._get_tag()
}]
}],
MinCount=1,
MaxCount=1,
UserData=self._user_data)
if self._block_devices:
kwargs['BlockDeviceMappings'] = self._block_devices
if self._spot_instance:
_utl.recursive_update(kwargs, self.SPOT_INSTANCE_MARKET_OPTIONS)
if self._spot_block_duration:
kwargs['InstanceMarketOptions']['SpotOptions'][
'BlockDurationMinutes'] = self._spot_block_duration
_utl.recursive_update(kwargs, self._boto3_create_instances_kwargs)
# Create instance
with _exception_handler():
instance = self._ec2_resource.create_instances(**kwargs)[0]
return instance, instance.id
def _start_new_instance(self):
"""
Starts a new instance.
Returns:
dict: Instance
str: Instance ID
"""
# Gets maximum Internet bandwidth
response = self._request('DescribeBandwidthLimitation',
InstanceType=self._instance_type)
max_bandwidth = response['Bandwidths']['Bandwidth'][0]['Max']
# Creates instance
kwargs = dict(
ImageId=self._image_id, InstanceType=self._instance_type,
SecurityGroupId=self._security_group_id,
InstanceName=self._get_host_name(),
Description=_utl.gen_msg('accelize_generated'),
InternetMaxBandwidthOut=max_bandwidth,
KeyPairName=self._key_pair, RamRoleName=self._role,
UserData=_b64encode(self._user_data).decode(),
parameters={'Tag.1.Key': 'Apyfal', 'Tag.1.Value': self._get_tag()})
_utl.recursive_update(kwargs, self._acs_create_instance_kwargs)
response = self._request('CreateInstance', **kwargs)
instance_id = response['InstanceId']
# Allocates public IP address
self._instance_request(
'AllocatePublicIpAddress', status_desc='allocating IP address',
InstanceId=instance_id)
# Starts instance
self._instance_request(
'StartInstance', status_desc='starting', InstanceId=instance_id)
# Return basic instance description as instance and instance ID
return {'InstanceId': instance_id}, instance_id
def test_recursive_update():
"""Tests test_recursive_update"""
from apyfal._utilities import recursive_update
to_update = {'root1': {'key1': 1, 'key2': 2}, 'key3': 3}
update = {'root1': {'key1': 1.0, 'key4': 4.0}, 'key5': 5.0}
expected = {
'root1': {
'key1': 1.0,
'key2': 2,
'key4': 4.0
},
'key3': 3,
'key5': 5.0
}
assert recursive_update(to_update, update) == expected
def test_acceleratorclient_get_parameters(tmpdir):
"""Tests AcceleratorClient._get_parameters"""
from apyfal.client import AcceleratorClient
from apyfal._utilities import recursive_update
# Mocks some variables
default_parameters = {'app': {'specific': {}, "key0": 0, "key1": 1}}
# Mocks Client
class DummyClient(AcceleratorClient):
"""Dummy Client"""
def __del__(self):
"""Do nothing"""
def _start(self, *_):
"""Do nothing"""
def _process(self, *_):
"""Do nothing"""
def _stop(self, *_):
"""Do nothing"""
def function(self, **parameters):
"""Passe parameters to _get_parameters and return result"""
return self._get_parameters(parameters, default_parameters)
client = DummyClient('Dummy')
# Test: Pass specific parameters as keyword arguments
excepted_parameters = copy.deepcopy(default_parameters)
excepted_parameters['app']['specific'] = {'key0': 0, 'key1': 1}
assert client.function(key0=0, key1=1) == excepted_parameters
# Test: loads parameters dict
dummy_parameters = {'app': {'specific': {'key1': 1}, "key0": 1}}
excepted_parameters = copy.deepcopy(default_parameters)
recursive_update(excepted_parameters, dummy_parameters)
assert client.function(
parameters=dummy_parameters) == excepted_parameters
# Test: loads parameters dict as JSON literal
assert client.function(parameters=json.dumps(
dummy_parameters)) == excepted_parameters
# Test: loads parameters dict as JSON file
json_file = tmpdir.join('parameters.json')
json_file.write(json.dumps(dummy_parameters))
assert client.function(parameters=str(json_file)) == excepted_parameters
# Test: Simultaneous parameters dict + keyword arguments
excepted_parameters = copy.deepcopy(default_parameters)
recursive_update(excepted_parameters, dummy_parameters)
excepted_parameters['app']['specific'].update({'key0': 0, 'key1': 0})
assert client.function(
parameters=dummy_parameters, key0=0, key1=0) == excepted_parameters
# Test: Missing specific section in source
excepted_parameters = copy.deepcopy(default_parameters)
del default_parameters['app']['specific']
excepted_parameters['app']['specific'] = {'key0': 0, 'key1': 1}
assert client.function(key0=0, key1=1) == excepted_parameters