def _set_accelerator_requirements( self, accelerator=None, accel_parameters=None, request_to_server=True): """ Configures instance with accelerator client parameters. Needs "accel_client" or "accel_parameters". Args: accelerator (str): Name of the accelerator accel_parameters (dict): Can override parameters from accelerator client. request_to_server (bool): If True, get parameters from server. Raises: apyfal.exceptions.HostConfigurationException: Parameters are not valid.. """ # Gets parameters self._config_env = dict() if accelerator is not None and request_to_server: _utl.recursive_update( self._config_env, self._config.get_host_requirements( self._host_type, accelerator)) if accel_parameters is not None: _utl.recursive_update( self._config_env, _deepcopy(accel_parameters)) # Gets accelerator name self._accelerator = self._config_env.pop( 'accelerator', None) or accelerator
def stop(self, info_dict=None, full_stop=True): """ Stop accelerator. Args: full_stop (bool): If True, send stop request to accelerator application. If False only clean up accelerator client environment. info_dict (dict or None): If a dict passed, this dict is updated with extra information from current operation. """ self._stopped = True # Stops if full_stop: response = self._stop() if info_dict is not None and response: _utl.recursive_update(info_dict, response) # Clears temporary directory try: _rmtree(self._cache['tmp_dir']) except (OSError, KeyError): pass # Clears cache self._cache.clear()
def _start_new_instance(self): """ Start a new instance. Returns: object: Instance str: Instance ID """ kwargs = dict(name=self._get_host_name(), key_name=self._key_pair, security_groups=[self._security_group], userdata=self._user_data, meta={'Apyfal': self._get_tag()}) with _exception_handler(gen_msg=('unable_to', "start")): kwargs["image"] = self._nova_client.glance.find_image( self._image_id) kwargs["flavor"] = self._nova_client.flavors.get( self._instance_type) _utl.recursive_update(kwargs, self._nova_client_create_server_kwargs) instance = self._nova_client.servers.create(**kwargs) return instance, instance.id
def _update_with_default_configuration(self, to_update, config_env): """ Get default configuration environment from metering server. Args: to_update (dict): Environment to update config_env (dict): User environment Returns: dict: Default configuration environment """ # Updates credentials in configuration with local file section = self._config['accelize'] section['client_id'] = (section.get('client_id') or config_env.get('client_id') or to_update.get('client_id')) section['secret_id'] = (section.get('secret_id') or config_env.get('client_secret') or to_update.get('client_secret')) # Gets default configuration try: _utl.recursive_update( to_update, self._config.get_host_configurations()[self._host_type][ self._name][self._region]) # Empty default configuration is possible except KeyError: return
def start(self, src=None, info_dict=None, host_env=None, reload=None, reset=None, **parameters): """ Configures accelerator. Args: src (path-like object or file-like object): Depending on the accelerator, a configuration data need to be loaded before a process can be run. Path-like object can be path, URL or cloud object URL. parameters (str, path-like object or dict): Accelerator configuration specific parameters Can also be a full configuration parameters dictionary (Or JSON equivalent as str literal or apyfal.storage URL to file) Parameters dictionary override default configuration values, individuals specific parameters overrides parameters dictionary values. Take a look to accelerator documentation for more information on possible parameters. Path-like object can be path, URL or cloud object URL. reload (bool): Force reload of FPGA bitstream. reset (bool): Force reset of FPGA logic. info_dict (dict or None): If a dict passed, this dict is updated with extra information from current operation. host_env (dict): Overrides Accelerator "env". """ self._stopped = False _get_logger().info("Configuring accelerator...") # Configure start parameters = self._get_parameters(parameters, self._configuration_parameters) parameters['env'].update(host_env or dict()) # Set FPGA reset and reload options if reload is not None: parameters['app']['reload'] = reload if reset is not None: parameters['app']['reset'] = reset # Handle files with self._data_file(src, parameters, ('src', 'datafile'), mode='rb') as src: # Starts response = self._start(src, parameters) # Check response status self._raise_for_status(response, "Failed to configure accelerator: ") _get_logger().info("Accelerator ready") # Update info dict if info_dict is not None and response: _utl.recursive_update(info_dict, response)
def mount(self): """Mount storage.""" storage_parameters = _deepcopy(self.STORAGE_PARAMETERS) self._update_parameter(storage_parameters) _utl.recursive_update(storage_parameters, self._storage_parameters) return _pycosio.mount(storage=self.STORAGE_NAME, extra_root=self.EXTRA_ROOT, storage_parameters=storage_parameters, unsecure=self._unsecure)
def _get_parameters(parameters, default_parameters, copy=True): """ Gets parameters from different sources, and merge them together. If 'parameters' contain a key named 'parameters', it will be read as a full parameter dict, or JSON literal or JSON file URL. Other keys from 'parameters' will be merged to the 'specific' section of the result dict. Args: parameters (dict): parameters default_parameters (dict): default parameters copy (bool): If True return a copy of updated default_parameters, else update directly. Returns: dict : parameters. """ # Takes default parameters as basis if copy: result = _deepcopy(default_parameters) else: result = default_parameters # Gets parameters from included JSON file json_parameters = parameters.pop('parameters', None) if json_parameters: # Reads JSON parameter from file or literal if not isinstance(json_parameters, dict): # JSON literal if json_parameters.rstrip().startswith('{'): json_parameters = _json.loads(json_parameters) # JSON file else: with _srg.open(json_parameters, 'rt') as json_file: json_parameters = _json.load(json_file) # Merges to result _utl.recursive_update(result, json_parameters) # Merges other parameters to specific section of parameters try: specific = result['app']['specific'] except KeyError: specific = result['app']['specific'] = dict() _utl.recursive_update(specific, parameters) return result
def process(self, src=None, dst=None, info_dict=None, **parameters): """ Processes with accelerator. Args: src (path-like object or file-like object): Source data to process. Path-like object can be path, URL or cloud object URL. dst (path-like object or file-like object): Processed data destination. Path-like object can be path, URL or cloud object URL. parameters (path-like object, str or dict): Accelerator process specific parameters Can also be a full process parameters dictionary (Or JSON equivalent as str literal) Parameters dictionary override default configuration values, individuals specific parameters overrides parameters dictionary values. Take a look to accelerator documentation for more information on possible parameters. Path-like object can be path, URL or cloud object URL. info_dict (dict or None): If a dict passed, this dict is updated with extra information from current operation. Returns: Result from process operation, depending used accelerator. """ # Configures processing parameters = self._get_parameters(parameters, self._process_parameters) # Handle files with self._data_file(src, parameters, ('src', 'file_in'), mode='rb') as src: with self._data_file(dst, parameters, ('dst', 'file_out'), mode='wb') as dst: # Processes response = self._process(src, dst, parameters) # Check response status self._raise_for_status(response, "Processing failed: ") # Get result from response result = response['app'].pop('specific', None) # Update info dict if info_dict is not None and response: _utl.recursive_update(info_dict, response) # Returns result return result
def _start_new_instance(self): """ Start a new instance. Returns: object: Instance str: Instance ID """ kwargs = dict(ImageId=self._image_id, InstanceType=self._instance_type, KeyName=self._key_pair, SecurityGroups=[self._security_group], IamInstanceProfile={'Name': 'AccelizeLoadFPGA'}, InstanceInitiatedShutdownBehavior='terminate', TagSpecifications=[{ 'ResourceType': 'instance', 'Tags': [{ 'Key': 'Generated', 'Value': _utl.gen_msg('accelize_generated') }, { 'Key': 'Name', 'Value': self._get_host_name() }, { 'Key': 'Apyfal', 'Value': self._get_tag() }] }], MinCount=1, MaxCount=1, UserData=self._user_data) if self._block_devices: kwargs['BlockDeviceMappings'] = self._block_devices if self._spot_instance: _utl.recursive_update(kwargs, self.SPOT_INSTANCE_MARKET_OPTIONS) if self._spot_block_duration: kwargs['InstanceMarketOptions']['SpotOptions'][ 'BlockDurationMinutes'] = self._spot_block_duration _utl.recursive_update(kwargs, self._boto3_create_instances_kwargs) # Create instance with _exception_handler(): instance = self._ec2_resource.create_instances(**kwargs)[0] return instance, instance.id
def _start_new_instance(self): """ Starts a new instance. Returns: dict: Instance str: Instance ID """ # Gets maximum Internet bandwidth response = self._request('DescribeBandwidthLimitation', InstanceType=self._instance_type) max_bandwidth = response['Bandwidths']['Bandwidth'][0]['Max'] # Creates instance kwargs = dict( ImageId=self._image_id, InstanceType=self._instance_type, SecurityGroupId=self._security_group_id, InstanceName=self._get_host_name(), Description=_utl.gen_msg('accelize_generated'), InternetMaxBandwidthOut=max_bandwidth, KeyPairName=self._key_pair, RamRoleName=self._role, UserData=_b64encode(self._user_data).decode(), parameters={'Tag.1.Key': 'Apyfal', 'Tag.1.Value': self._get_tag()}) _utl.recursive_update(kwargs, self._acs_create_instance_kwargs) response = self._request('CreateInstance', **kwargs) instance_id = response['InstanceId'] # Allocates public IP address self._instance_request( 'AllocatePublicIpAddress', status_desc='allocating IP address', InstanceId=instance_id) # Starts instance self._instance_request( 'StartInstance', status_desc='starting', InstanceId=instance_id) # Return basic instance description as instance and instance ID return {'InstanceId': instance_id}, instance_id
def test_recursive_update(): """Tests test_recursive_update""" from apyfal._utilities import recursive_update to_update = {'root1': {'key1': 1, 'key2': 2}, 'key3': 3} update = {'root1': {'key1': 1.0, 'key4': 4.0}, 'key5': 5.0} expected = { 'root1': { 'key1': 1.0, 'key2': 2, 'key4': 4.0 }, 'key3': 3, 'key5': 5.0 } assert recursive_update(to_update, update) == expected
def test_acceleratorclient_get_parameters(tmpdir): """Tests AcceleratorClient._get_parameters""" from apyfal.client import AcceleratorClient from apyfal._utilities import recursive_update # Mocks some variables default_parameters = {'app': {'specific': {}, "key0": 0, "key1": 1}} # Mocks Client class DummyClient(AcceleratorClient): """Dummy Client""" def __del__(self): """Do nothing""" def _start(self, *_): """Do nothing""" def _process(self, *_): """Do nothing""" def _stop(self, *_): """Do nothing""" def function(self, **parameters): """Passe parameters to _get_parameters and return result""" return self._get_parameters(parameters, default_parameters) client = DummyClient('Dummy') # Test: Pass specific parameters as keyword arguments excepted_parameters = copy.deepcopy(default_parameters) excepted_parameters['app']['specific'] = {'key0': 0, 'key1': 1} assert client.function(key0=0, key1=1) == excepted_parameters # Test: loads parameters dict dummy_parameters = {'app': {'specific': {'key1': 1}, "key0": 1}} excepted_parameters = copy.deepcopy(default_parameters) recursive_update(excepted_parameters, dummy_parameters) assert client.function( parameters=dummy_parameters) == excepted_parameters # Test: loads parameters dict as JSON literal assert client.function(parameters=json.dumps( dummy_parameters)) == excepted_parameters # Test: loads parameters dict as JSON file json_file = tmpdir.join('parameters.json') json_file.write(json.dumps(dummy_parameters)) assert client.function(parameters=str(json_file)) == excepted_parameters # Test: Simultaneous parameters dict + keyword arguments excepted_parameters = copy.deepcopy(default_parameters) recursive_update(excepted_parameters, dummy_parameters) excepted_parameters['app']['specific'].update({'key0': 0, 'key1': 0}) assert client.function( parameters=dummy_parameters, key0=0, key1=0) == excepted_parameters # Test: Missing specific section in source excepted_parameters = copy.deepcopy(default_parameters) del default_parameters['app']['specific'] excepted_parameters['app']['specific'] = {'key0': 0, 'key1': 1} assert client.function(key0=0, key1=1) == excepted_parameters