def _init_accelize_drm(self):
"""Initialize Accelize DRM requirements"""
# Create configuration file from application
accelize_drm_enable = self._app('accelize_drm', 'use_service')
accelize_drm_conf = self._app('accelize_drm', 'conf')
if accelize_drm_enable and not accelize_drm_conf:
raise ConfigurationException(
'Application definition section "accelize_drm" require '
'"conf" value to be specified if "use_service" is '
'specified.')
json_write(accelize_drm_conf, self._accelize_drm_conf_json)
# Get credentials file from user configuration
for src in get_sources_dirs(self._user_config):
cred_path = join(src, 'cred.json')
if isfile(cred_path):
symlink(cred_path, self._accelize_drm_cred_json)
break
else:
raise ConfigurationException(
'No Accelize DRM credential found. Please, make sure to '
f'have your "cred.json" file installed in "{HOME_DIR}", '
f'current directory or path specified with the '
f'"user_config" argument.')
def _init_accelize_conf(self, accelize_drm_conf, accelize_drm_enable,
provider):
"""
Initialize Accelize DRM Configuration.
Args:
accelize_drm_conf (dict): conf.json content
accelize_drm_enable (bool): True if service is enabled
provider (str): Provider.
Returns:
str: Path to conf.json
"""
# Create configuration file from application
if accelize_drm_enable and not accelize_drm_conf:
raise ConfigurationException(
'Application definition section "accelize_drm" require '
'"conf" value to be specified if "use_service" is '
'specified.')
accelize_drm_conf_json = join(self._config_dir,
'accelize_drm_conf.json')
if provider:
# Set board type value to provider
try:
design = accelize_drm_conf['design']
except KeyError:
accelize_drm_conf['design'] = design = dict()
design['boardType'] = provider
json_write(accelize_drm_conf, accelize_drm_conf_json)
return accelize_drm_conf_json
def mock_terraform_provider(source_dir, **variables):
"""
Mock provider configuration
Args:
source_dir (py.path.local) Source directory.
variables: Override local variables.
Returns:
dict: Input variables.
"""
from accelpy._common import json_write
local_variables = {
"host_public_ip": "127.0.0.1",
"host_private_ip": "127.0.0.1",
"remote_user": '******',
"remote_os": "os",
"provider_required_driver": "driver",
"require_ask_pass": "******"
}
json_write({"locals": local_variables},
source_dir.join('common.testing.tf.json'))
if variables:
json_write({"locals": variables},
source_dir.join('common.testing_override.tf.json'))
local_variables.update(variables)
return local_variables
def create_configuration(self,
provider=None,
application_type=None,
variables=None,
user_config=None):
"""
Generate Terraform configuration.
Configuration is built using sources Terraform configurations files
found from:
- This module provided default configuration.
- Configuration from user home directory.
- Current working directory.
- Directory specified by "user_config" argument.
If multiples files with the same name are found, the last one found is
used. Directories are checked in the listed order to allow user to
override default configuration easily.
Args:
provider (str): Provider name.
user_config (path-like object): User configuration directory.
variables (dict): Terraform input variables.
"""
# Link configuration files matching provider and options
for name, src_path in self._list_sources(provider, application_type,
user_config):
dst_path = join(self._config_dir, name)
# Replace existing file
try:
remove(dst_path)
except OSError:
pass
# Create symbolic link to configuration file
symlink(src_path, dst_path)
# Add variables
tf_vars = {
key: value
for key, value in (variables or dict()).items()
if value is not None
}
json_write(tf_vars, join(self._config_dir,
'generated.auto.tfvars.json'))
# Initialize Terraform
self._exec('init',
self._no_color,
'-input=false',
pipe_stdout=True,
env=self._exec_env,
retries=3)
def _get_last_version(cls):
"""
Get last version information from HashiCorp checkpoint API.
Returns:
dict: Last version information.
"""
info_cache = join(cls._install_dir(), 'info.json')
# Get Last version information from HashiCorp checkpoint API
if not isfile(info_cache) or getmtime(info_cache) < time() - 3600.0:
# Lazy import: Only used on update
from platform import machine, system
# Update from the web
last_release = cls._download(
'https://checkpoint-api.hashicorp.com/v1/check/' +
cls._name()).json()
current_version = last_release['current_version']
download_url = last_release['current_download_url'].rstrip('/')
# Define platform specific utility executable and archive name
arch = machine().lower()
arch = {'x86_64': 'amd64'}.get(arch, arch)
last_release['archive_name'] = archive_name = \
f"{cls._name()}_{current_version}_{system().lower()}_{arch}.zip"
last_release['executable_name'] = \
f'{cls._name()}.exe' if system() == 'Windows' else cls._name()
# Define download URL
last_release['archive_url'] = f"{download_url}/{archive_name}"
last_release['checksum_url'] = checksum_url = \
f"{download_url}/{cls._name()}_{current_version}_SHA256SUMS"
last_release['signature_url'] = f"{checksum_url}.sig"
# Cache result
makedirs(cls._install_dir(), exist_ok=True)
json_write(last_release, info_cache)
else:
# Get cached version
last_release = json_read(info_cache)
return last_release
def test_ansible(tmpdir):
"""
Test Ansible handler
Args:
tmpdir (py.path.local) tmpdir pytest fixture
"""
from accelpy._ansible import Ansible
from accelpy._common import yaml_read, json_write
source_dir = tmpdir.join('source').ensure(dir=True)
config_dir = tmpdir.join('config').ensure(dir=True)
variables = dict(key='value')
# Ensure Accelize "cred.json" exists
json_write(dict(client_secret='', client_id=''),
source_dir.join('cred.json'))
# Test: Create configuration (With not specific provider and application)
ansible = Ansible(config_dir, variables=variables, user_config=source_dir)
ansible.create_configuration()
playbook = yaml_read(config_dir.join('playbook.yml'))[0]
assert 'pre_tasks' in playbook
assert playbook['vars'] == variables
assert playbook['roles'] == ['common.init']
assert config_dir.join('cred.json').isfile()
# Test: Re-create should not raise
ansible.create_configuration()
# Test: lint should not raise on basic configuration
ansible.lint()
# Test: Galaxy install role
ansible.galaxy_install(['dev-sec.os-hardening', 'dev-sec.ssh-hardening'])
# Test: Galaxy install should do nothing if no roles
ansible.galaxy_install([])
# Test: Create configuration (with application that requires dependencies)
ansible = Ansible(config_dir, application_type='container_service')
ansible.create_configuration()
playbook = yaml_read(config_dir.join('playbook.yml'))[0]
assert 'pre_tasks' in playbook
assert not playbook['vars']
assert 'container_service' in playbook['roles']
def create_configuration(self):
"""
Generate Terraform configuration.
Configuration is built using sources Terraform configurations files
found from:
- This module provided default configuration.
- Configuration from user home directory.
- Current working directory.
- Directory specified by "user_config" argument.
If multiples files with the same name are found, the last one found is
used. Directories are checked in the listed order to allow user to
override default configuration easily.
"""
# Lazy import: Only used if new configuration
from accelpy._ansible import Ansible
# Symlink common plugins dir to to avoid to re-download them each time
dot_dir = join(self._config_dir, '.terraform')
makedirs(dot_dir, exist_ok=True)
symlink(self._plugins_dir(), join(dot_dir, 'plugins'))
# Link configuration files matching provider and options
for name, src_path in self._list_sources():
dst_path = join(self._config_dir, name)
# Replace existing file
try:
remove(dst_path)
except OSError:
pass
# Create symbolic link to configuration file
symlink(src_path, dst_path)
# Add variables
tf_vars = {
key: value for key, value in self._variables.items()
if value is not None}
tf_vars['ansible'] = Ansible.playbook_exec()
json_write(
tf_vars, join(self._config_dir, 'generated.auto.tfvars.json'))
def create_configuration(self, provider=None, application_type=None,
variables=None, user_config=None):
"""
Generate packer configuration file.
Args:
provider (str): Provider name.
user_config (path-like object): User configuration directory.
vars (dict): Terraform input variables.
"""
# Lazy import, may not be used
from jinja2 import Environment
# Get template from this package and user directories
sources = dict(vars=dict(variables=variables or dict()))
for name, src_path in self._list_sources(
provider, application_type, user_config):
sources[name] = json_read(src_path)
# Generate the Packer template file
template = dict()
for key in sorted(sources):
recursive_update(template, sources[key])
# Evaluate variables that contain Jinja templates
variables = template['variables']
env = Environment(extensions=['jinja2.ext.loopcontrols'])
to_clean = set()
for key in sorted(variables):
value = variables[key]
if isinstance(value, str) and '{' in value:
variables[key] = env.from_string(value).render(variables)
# Mark for deletion, Packer does not accept non string as variables
elif not isinstance(value, str):
to_clean.add(key)
for key in to_clean:
del variables[key]
# Save template
json_write(template, self._template)
def mock_packer_provider(source_dir):
"""
Mock provider configuration
Args:
source_dir (py.path.local) Source directory.
Returns:
str: artifact
"""
from accelpy._common import json_write
artifact = "artifact.json"
json_write({
"builders": [{
"type": "file",
"content": "",
"target": artifact
}]}, source_dir.join('testing.json'))
return artifact
def create_configuration(self):
"""
Generate packer configuration file.
"""
# Lazy import: Only used on new configuration creation
from accelpy._ansible import Ansible
# Get template from this package and user directories
self._variables['ansible'] = Ansible.playbook_exec()
sources = dict(vars=dict(variables=self._variables))
for name, src_path in self._list_sources():
sources[name] = json_read(src_path)
# Generate the Packer template file
template = dict()
for key in sorted(sources):
recursive_update(template, sources[key])
json_write(template, self._template)
def test_json_read_write(tmpdir):
"""
Test json_read/json_write
Args:
tmpdir (py.path.local) tmpdir pytest fixture
"""
from accelpy._common import json_write, json_read
from accelpy.exceptions import ConfigurationException
json_file = tmpdir.join('file.json')
# Test: correct file
data = {'key': 'value'}
json_write(data, json_file)
assert json_read(json_file) == data
# Test: badly formatted file
json_file.write('{key: ')
with pytest.raises(ConfigurationException):
json_read(json_file)
def _create_config(self, application, provider, user_config):
"""
Create configuration.
Args:
application (str or path-like object):
Application or path to application definition file.
provider (str): Provider name.
user_config (path-like object): User configuration directory.
"""
# Ensure config is cleaned on creation error
keep_config = self._keep_config
self._keep_config = False
# Create target configuration directory and remove access to other
# users since Terraform state files may content sensible data and
# directory may contain SSH private key
makedirs(self._config_dir, exist_ok=True)
chmod(self._config_dir, 0o700)
# Save user parameters
user_config = fsdecode(user_config or HOME_DIR)
json_write(dict(provider=provider, user_config=user_config),
self._user_parameters_json)
# Get application and its definition
self._init_application_definition(application)
app = self._application[provider]
fpga_count = app['fpga']['count']
application_type = app['application']['type']
accelize_drm_enable = app['accelize_drm']['use_service']
name = self._name
# Check Accelize DRM Requirements
accelize_drm_cred_json = self._init_accelize_cred(user_config)
accelize_drm_conf_json = self._init_accelize_conf(
app['accelize_drm']['conf'], accelize_drm_enable, provider)
# Lazy import, because may not be always used
from concurrent.futures import ThreadPoolExecutor
from accelpy._ansible import Ansible
# Set Ansible variables
ansible_env = Ansible.environment()
ansible_exec = Ansible.playbook_exec()
ansible_variables = dict(
fpga_image=app['fpga']['image'],
fpga_driver=app['fpga']['driver'],
fpga_driver_version=app['fpga']['driver_version'],
fpga_slots=[slot for slot in range(fpga_count)],
firewall_rules=app['firewall_rules'],
app_packages=app['package'],
accelize_drm_disabled=not accelize_drm_enable,
accelize_drm_conf_src=accelize_drm_conf_json,
accelize_drm_cred_src=accelize_drm_cred_json)
ansible_variables.update(app['application']['variables'])
# Set Packer variables
packer_variables = {
f'provider_param_{index}': value
for index, value in enumerate((provider or '').split(','))
}
packer_variables.update(
dict(image_name=name,
ansible=ansible_exec,
fpga_count=str(fpga_count)))
# Set Terraform variables
terraform_variables = dict(
ansible=' '.join(
[f'{key}={value}'
for key, value in ansible_env.items()] + [ansible_exec]),
firewall_rules=app['firewall_rules'],
fpga_count=fpga_count,
package_vm_image=app['package'][0]['name']
if app['package'][0]['type'] == 'vm_image' else '',
host_name=name,
host_provider=provider)
# Initialize utilities configuration
futures = []
with ThreadPoolExecutor(max_workers=3) as executor:
for utility, variables in ((self._terraform, terraform_variables),
(self._ansible, ansible_variables),
(self._packer, packer_variables)):
futures.append(
executor.submit(getattr(utility, 'create_configuration'),
provider=provider,
application_type=application_type,
variables=variables,
user_config=user_config))
for future in futures:
future.result()
# Restore keep config flag once configuration si completed
self._keep_config = keep_config
def test_accelize_ws_session(tmpdir):
"""
Test Accelize Web Service session
Args:
tmpdir (py.path.local) tmpdir pytest fixture
"""
from io import BytesIO
from os import environ
from json import dumps
from time import time
from requests import Response, Session
import accelpy._common as common
from accelpy._common import _AccelizeWSSession, json_write
from accelpy.exceptions import AuthenticationException, WebServerException
# Mock cache
environ['ACCELPY_CLI'] = 'True'
common_cache_dir = common.CACHE_DIR
common.CACHE_DIR = str(tmpdir.join('cache_01').ensure(dir=True))
# Mock get_accelize_cred
cred_json = str(tmpdir.join('cred.json'))
json_write(dict(client_id='accelpy_testing', client_secret=''), cred_json)
common_get_accelize_cred = common.get_accelize_cred
def get_accelize_cred():
"""
Returns bad credentials.
Returns:
str: path
"""
return cred_json
common.get_accelize_cred = get_accelize_cred
# Test Session initialization
session = _AccelizeWSSession()
assert session._request
# Mock server response
resp_status_code = 200
resp_content = dict(key='value')
auth_status_code = 200
auth_expire_in = 9999
class _Session(Session):
"""Mocked requests.Session"""
raw_error = False
@classmethod
def request(cls, method, url, **_):
"""Returns mocked response"""
if '/o/token/' in url:
status_code = auth_status_code
content = dumps(dict(
access_token='access_token', expires_in=auth_expire_in))
else:
status_code = resp_status_code
content = dumps(resp_content) if resp_content else ''
if status_code != 200:
if cls.raw_error:
content = dumps(dict(detail='error'))
else:
# Test return a non JSON error once
cls.raw_error |= True
content = 'error'
response = Response()
response.raw = BytesIO(content.encode())
response.raw.seek(0)
response.status_code = status_code
return response
session._session = _Session()
session._session_request = session._session.request
# Tests
try:
# Test: invalid credentials
auth_status_code = 400
with pytest.raises(AuthenticationException):
session.request(path='')
# Test: get token from web service
auth_status_code = 200
session._token = ''
assert session.request(path='') == resp_content
# Test: get token from cache
auth_status_code = 500
session._token = ''
assert session.request(path='') == resp_content
# Test: use existing token
assert session.request(path='') == resp_content
# Test: renew token from web service if expired
common.CACHE_DIR = str(tmpdir.join('cache_02').ensure(dir=True))
session._token_expire = int(time()) - 10
with pytest.raises(AuthenticationException):
session.request(path='')
auth_status_code = 200
assert session.request(path='') == resp_content
# Test: Renew token if server reject it
resp_status_code = 401
with pytest.raises(WebServerException):
session.request(path='')
assert session._token
# Test: Empty response
resp_content = ''
resp_status_code = 200
assert session.request(path='') is None
# Restore mocked function
finally:
common.get_accelize_cred = common_get_accelize_cred
common.CACHE_DIR = common_cache_dir
del environ['ACCELPY_CLI']
def __init__(self,
name=None,
application=None,
provider=None,
user_config=None,
destroy_on_exit=False,
keep_config=True):
# Initialize some futures values
self._ansible_config = None
self._packer_config = None
self._terraform_config = None
self._terraform_output = None
self._application_definition = None
# If true, Terraform infrastructure is destroyed on exit
self._destroy_on_exit = destroy_on_exit
self._keep_config = keep_config
# Define name
if not name:
from uuid import uuid1
name = str(uuid1()).replace('-', '')
self._name = name
# Define configuration directory en files
self._config_dir = join(CONFIG_DIR, name)
user_parameters_json = join(self._config_dir, 'user_parameters.json')
self._output_json = join(self._config_dir, 'output.json')
self._accelize_drm_conf_json = join(self._config_dir,
'accelize_drm_conf.json')
self._accelize_drm_cred_json = join(self._config_dir, 'cred.json')
# Create a new configuration
config_exists = isdir(self._config_dir)
if not config_exists and application:
# Ensure config is cleaned on creation error
self._keep_config = False
# Create target configuration directory and remove access to other
# users since Terraform state files may content sensible data and
# directory may contain SSH private key
makedirs(self._config_dir, exist_ok=True)
chmod(self._config_dir, 0o700)
# Get user parameters used
self._provider = provider
self._user_config = fsdecode(user_config or HOME_DIR)
# Save user parameters
json_write(
dict(provider=self._provider, user_config=self._user_config),
user_parameters_json)
# Get application and add it as link with configuration
self._application_yaml = realpath(fsdecode(application))
# Check Accelize Requirements
self._init_accelize_drm()
# Add link to configuration
symlink(self._application_yaml,
join(self._config_dir, 'application.yml'))
# Initialize Terraform and Ansible configuration
self._terraform.create_configuration()
self._ansible.create_configuration()
self._packer.create_configuration()
self._keep_config = keep_config
# Load an existing configuration
elif config_exists:
# Retrieve application parameters
self._application_yaml = realpath(
join(self._config_dir, 'application.yml'))
# Retrieve user parameters
user_parameters = json_read(user_parameters_json)
self._provider = user_parameters['provider']
self._user_config = user_parameters['user_config']
# Unable to create configuration
else:
raise ConfigurationException(
'Require at least an existing host name, or an '
'application to create a new host.')