def get_host_requirements(self, host_type, accelerator):
"""
Gets accelerators requirements to use with host.
Args:
host_type (str): Host type.
accelerator (str): Name of the accelerator
Returns:
dict: AcceleratorClient requirements for host.
"""
# Get host_type configuration
try:
provider_config = self.get_host_configurations()[host_type]
except KeyError:
raise _exc.ClientConfigurationException(
"Host '%s' is not supported. Available hosts are: %s" %
(host_type, ', '.join(self.get_host_configurations().keys())))
# Get accelerator configuration
try:
accelerator_config = provider_config[accelerator]
except KeyError:
raise _exc.ClientConfigurationException(
"AcceleratorClient '%s' is not supported on '%s'." %
(accelerator, host_type))
accelerator_config['accelerator'] = accelerator
return accelerator_config
def _process(self, src, dst, parameters):
"""
Client specific process implementation.
Args:
src (file-like object): Input data.
dst (file-like object): Output data.
parameters (dict): Parameters dict.
Returns:
dict: response dict.
"""
# Check if configuration was done
if self._configuration_url is None:
raise _exc.ClientConfigurationException(
"AcceleratorClient has not been configured. "
"Use 'start' function.")
# Post processing request
fields = {
'parameters': _json.dumps(parameters),
'configuration': self._configuration_url
}
if src:
fields['datafile'] = 'src', src, 'application/octet-stream'
multipart = _MultipartEncoder(fields=fields)
response = self._session.post(
self._endpoints['process'],
data=multipart,
headers={'Content-Type': multipart.content_type})
# Check response and append process ID to process URL
process_url = self._endpoints['process'] + str(
self._raise_for_error(response)['id'])
# Get result
try:
# Wait processing
while True:
response_dict = self._raise_for_error(
self._session.get(process_url))
if response_dict['processed']:
break
# Gets result file
if dst:
response = self._session.get(response_dict['datafileresult'],
stream=True)
_shutil.copyfileobj(response.raw, dst)
# Gets result dict
return response_dict['parametersresult']
finally:
# Deletes process result on server
self._session.delete(process_url)
def url(self, url):
"""
URL of the accelerator host.
Args:
url (str): URL.
"""
# Check URL
if not url:
raise _exc.ClientConfigurationException("Host URL is not valid.")
self._url = url = _utl.format_url(url,
force_secure=bool(
self._ssl_cert_crt))
# Updates endpoints
for route in self._REST_API:
self._endpoints[route] = url + self._REST_API[route]
def _configuration_url(self):
"""Last configuration URL"""
# Get last configuration, if any
try:
endpoint = self._endpoints['start']
except KeyError:
raise _exc.ClientConfigurationException(
'Unknown host URL, please run accelerator "start" method.')
response = self._session.get(endpoint)
try:
last_config = response.json()['results'][0]
except (KeyError, IndexError, ValueError):
return
# The last configuration URL should be keep in order to not request
# it to user.
if last_config['used'] != 0:
return last_config['url']
def __init__(self,
accelerator=None,
host_ip=None,
ssl_cert_crt=None,
*args,
**kwargs):
# Initialize client
_Client.__init__(self, accelerator=accelerator, *args, **kwargs)
# Initializes HTTP client
self._ssl_cert_crt = ssl_cert_crt
self._endpoints = {}
# Mandatory parameters
if not accelerator:
raise _exc.ClientConfigurationException(
"'accelerator' argument is mandatory.")
# Pass host URL if already defined.
if host_ip:
self.url = host_ip
def _checks_apyfal_version(self, config_env):
"""
Checks if client Apyfal version is compatible.
Args:
config_env (dict): environment.
Raises:
apyfal.exceptions.ClientConfigurationException:
Apyfal version is not compatible.
"""
try:
if _LooseVersion(config_env['apyfal_version']) < _LooseVersion(
self.APYFAL_MINIMUM_VERSION):
raise _exc.ClientConfigurationException(
'Apyfal version needs to be at least %s. Please upgrade it.'
% self.APYFAL_MINIMUM_VERSION)
# Version not available: Return, can come from REST API directly.
except KeyError:
return
def process_map(self,
srcs=None,
dsts=None,
timeout=None,
info_list=None,
**parameters):
"""
Map process execution on multiples files.
Args:
srcs (iterable of path-like object or file-like object):
Iterable of input data to process.
Must be an iterable of "src" parameters of the "process" method.
Path-like object can be path, URL or cloud object URL.
dsts (iterable of path-like object or file-like object):
Iterable of output data.
Must be an iterable of "dst" parameters of the "process" method.
Path-like object can be path, URL or cloud object URL.
timeout (float): The maximum number of seconds to wait. If None,
then there is no limit on the wait time.
parameters (path-like object, str or dict): Accelerator process
specific parameters
Can also be a full process parameters dictionary
(Or JSON equivalent as str literal) Parameters dictionary
override default configuration
values, individuals specific parameters overrides parameters
dictionary values. Take a look to accelerator documentation for
more information on possible parameters.
Path-like object can be path, URL or cloud object URL.
info_list (list): If a list passed, this list is updated
with "info_dict" extra information dicts for each process
operation.
Returns:
generator: Results.
Raises:
concurrent.futures.TimeoutError: "timeout" reached on at least one
task.
"""
# Based on "concurrent.futures.Executor.map"
# Initializes timeout
if timeout is not None:
end_time = timeout + time()
# Get file count
src = dst = None
if srcs is not None:
size_src = len(srcs)
else:
size_src = 0
if dsts is not None:
size_dst = len(dsts)
else:
size_dst = 0
if size_src and size_dst and size_src != size_dst:
raise _exc.ClientConfigurationException(
'"files_in" and "files_out" must contain the same count of'
' files.')
# Submit process
futures = []
for index in range(size_src or size_dst):
if size_src:
src = srcs[index]
if size_dst:
dst = dsts[index]
futures.append(
self.process_submit(src=src,
dst=dst,
info_dict=self._get_info_dict(info_list),
**parameters))
def result_iterator():
"""
Yield must be hidden in closure so that the futures are submitted
before the first iterator value is required.
"""
try:
# reverse to keep finishing order
futures.reverse()
while futures:
# Careful not to keep a reference to the popped future
if timeout is None:
yield futures.pop().result()
else:
yield futures.pop().result(end_time - time())
finally:
for future in futures:
future.cancel()
return result_iterator()
def _data_file(self, url, parameters, parameter_name, mode):
"""Get files with apyfal.storage.
Args:
url (str or file-like object): Input URL.
parameters (dict): Parameters dict.
parameter_name (str or tuple of str): Parameter name for input URL.
mode (str): Access mode. 'r' or 'w'.
Returns:
str or file-like object or None:
Local version of input path.
"""
# Apyfal 1.1.0 Compatibility
if isinstance(parameter_name, tuple):
parameter_name, ap110_name = parameter_name
else:
ap110_name = parameter_name
# No URL
if url is None:
# Get URL from parameters if not provided directly
try:
url = parameters['app']['specific'].pop(parameter_name)
# Still no URL, yields directly
except KeyError:
# Apyfal 1.1.0 Compatibility
url = parameters['app']['specific'].pop(ap110_name, None)
if url is None:
yield None
return
# Gets scheme and path from URL
scheme, path = _srg.parse_url(url, not self.REMOTE)
# File scheme: Check paths
if scheme == 'file':
path = _os_path.abspath(path)
# Only authorises files in whitelisted directories on host
if not self.REMOTE and url.startswith('host://'):
for authorized in self._authorized_host_dirs:
if not path.startswith(authorized):
raise _exc.ClientSecurityException(
"Unauthorized path: '%s'" % path)
# Checks input file exists
if 'r' in mode and not _os_path.isfile(path):
raise _exc.ClientConfigurationException(
gen_msg=('not_found_named', parameter_name, path))
# Ensures output parent directory exists
elif 'w' in mode:
_utl.makedirs(_os_path.dirname(path), exist_ok=True)
# Client side:
# Sends URL to host side as parameters and
# yields None to client
if self.REMOTE and scheme not in ('stream', 'file'):
parameters['app']['specific'][parameter_name] = url
# Apyfal 1.1.0 Compatibility
if ap110_name != parameter_name:
parameters['app']['specific'][ap110_name] = url
yield None
# Other case, yields file in expected format (file or stream)
else:
# As file
if self._PARAMETER_IO_FORMAT.get(parameter_name,
'stream') == 'file':
# Already a file
if scheme == 'file':
yield path
# Use temporary file
else:
with self.as_tmp_file(url, mode) as file:
yield file
# As stream
else:
with _srg.open(url, mode) as stream:
yield stream