def run(schema, file, **_):
result = {'state': 'succeeded', 'debugInfo': None}
try:
if schema not in schemas:
debug_info = 'Schema "{}" not found. Use "faice schema list" for available schemas.'.format(
schema)
print(debug_info, file=sys.stderr)
result['debugInfo'] = debug_info
result['state'] = 'failed'
return result
data = load_and_read(file, 'FILE')
jsonschema.validate(data, schemas[schema])
except ValidationError as e:
where = '/'.join([str(s) for s in e.absolute_path
]) if e.absolute_path else '/'
debug_info = 'File "{}" does not comply with schema "{}":\n\tkey in red file: {}\n\treason: {}'\
.format(file, schema, where, e.message)
print_exception(RedValidationError(debug_info))
result['debugInfo'] = exception_format()
result['state'] = 'failed'
except Exception as e:
print_exception(e)
result['debugInfo'] = exception_format()
result['state'] = 'failed'
return result
def run(red_file, fmt, prefix, **_):
secret_values = None
result = {'state': 'succeeded', 'debugInfo': None}
try:
ext = file_extension(fmt)
red_data = load_and_read(red_file, 'REDFILE')
secret_values = get_secret_values(red_data)
red_validation(red_data, False)
if 'batches' not in red_data:
wrapped_print(['ERROR: REDFILE does not contain batches.'],
error=True)
return 1
for batch in range(len(red_data['batches'])):
batch_data = convert_batch_experiment(red_data, batch)
dumped_batch_file = '{}batch_{}.{}'.format(prefix, batch, ext)
dump(batch_data, fmt, dumped_batch_file)
except Exception as e:
print_exception(e, secret_values)
result['debugInfo'] = exception_format(secret_values)
result['state'] = 'failed'
return result
def run(file, fmt, **_):
result = {'state': 'succeeded', 'debugInfo': None}
try:
data = load_and_read(file, 'FILE')
dump_print(data, fmt)
except Exception as e:
print_exception(e)
result['debugInfo'] = exception_format()
result['state'] = 'failed'
return result
def run(red_file, fmt, **_):
result = {'state': 'succeeded', 'debugInfo': None}
try:
red_data = load_and_read(red_file, 'REDFILE')
except AgentError as e:
print_exception(e)
result['debugInfo'] = exception_format()
result['state'] = 'failed'
return result
if 'cli' not in red_data:
print_exception(
RedSpecificationError(
'ERROR: REDFILE does not contain cli section.'))
result['debugInfo'] = exception_format()
result['state'] = 'failed'
return result
cli = red_data['cli']
dump_print(cli, fmt)
return result
def run(red_data,
disable_pull=False,
leave_container=False,
preserve_environment=None,
insecure=False,
output_mode=OutputMode.Connectors,
gpu_ids=None,
disable_connector_validation=None):
"""
Executes a RED Experiment.
:param red_data: A dict containing red data that will be executed. This red data cannot contain template keys
:param disable_pull: If True the docker image is not pulled from an registry
:param leave_container: If set to True, the executed docker container will not be removed.
:param preserve_environment: List of environment variables to preserve inside the docker container.
:param insecure: Allow insecure capabilities
:param output_mode: Either Connectors or Directory. If Connectors, the restricted_red agent will try to execute the
output connectors. If Directory faice will copy the output files into the host output directory.
:param gpu_ids: A list of gpu ids, that should be used. If None all gpus are considered.
:type gpu_ids: List[int] or None
:param disable_connector_validation: If set, the execution engine will skip connector validation
:type disable_connector_validation: bool
:raise TemplateError: If template keys are found in the given red data
"""
result = {'containers': [], 'debugInfo': None, 'state': 'succeeded'}
secret_values = None
try:
# secrets
secret_values = get_secret_values(red_data)
# validation
red_validation(red_data,
output_mode == OutputMode.Directory,
container_requirement=True,
allow_variables=False)
engine_validation(red_data, 'container', ['docker'], optional=False)
# process red data
restricted_red_batches = convert_red_to_restricted_red(red_data)
# docker settings
docker_image = red_data['container']['settings']['image']['url']
ram = red_data['container']['settings'].get('ram')
environment = env_vars(preserve_environment)
# create docker manager
docker_manager = DockerManager()
# gpus
gpus = get_gpus(docker_manager,
red_data['container']['settings'].get('gpus'), gpu_ids)
if not disable_pull:
registry_auth = red_data['container']['settings']['image'].get(
'auth')
docker_manager.pull(docker_image, auth=registry_auth)
if len(restricted_red_batches) == 1:
host_outdir = Path('outputs')
else:
host_outdir = Path('outputs_{batch_index}')
for batch_index, restricted_red_batch in enumerate(
restricted_red_batches):
container_execution_result = run_restricted_red_batch(
restricted_red_batch=restricted_red_batch,
docker_manager=docker_manager,
docker_image=docker_image,
host_outdir=host_outdir,
output_mode=output_mode,
leave_container=leave_container,
batch_index=batch_index,
ram=ram,
gpus=gpus,
environment=environment,
insecure=insecure,
disable_connector_validation=disable_connector_validation)
# handle execution result
# noinspection PyTypeChecker
result['containers'].append(container_execution_result.to_dict())
result[
'furtherErrors'] = container_execution_result.get_further_errors(
)
container_execution_result.raise_for_state()
except Exception as e:
result['briefExceptionText'] = brief_exception_text(e, secret_values)
result['debugInfo'] = exception_format(secret_values)
result['state'] = 'failed'
return result
def run(red_file, non_interactive, fmt, insecure, keyring_service, **_):
secret_values = None
result = {'state': 'succeeded', 'debugInfo': None}
try:
red_data = load_and_read(red_file, 'REDFILE')
red_validation(red_data, False)
engine_validation(red_data, 'execution', ['ccfaice', 'ccagency'],
'faice exec')
secret_values = get_secret_values(red_data)
# exec via CC-FAICE
# equivalent to `faice agent red --debug --outputs`
if 'execution' not in red_data:
raise KeyError(
'The key "execution" is needed in red file for usage with faice exec.'
)
if red_data['execution']['engine'] == 'ccfaice':
# use connectors, if red file specifies outputs
if _has_outputs(red_data):
faice_output_mode = OutputMode.Connectors
else:
faice_output_mode = OutputMode.Directory
result = run_faice_agent_red(red_file=red_file,
disable_pull=False,
leave_container=False,
preserve_environment=[],
non_interactive=non_interactive,
insecure=insecure,
output_mode=faice_output_mode,
keyring_service=keyring_service,
gpu_ids=None)
return result
complete_red_templates(red_data, keyring_service, non_interactive)
red_data_normalized = deepcopy(red_data)
normalize_keys(red_data_normalized)
if 'access' not in red_data_normalized['execution']['settings']:
result['debugInfo'] = [
'ERROR: cannot send RED data to CC-Agency if access settings are not defined.'
]
result['state'] = 'failed'
return result
if 'auth' not in red_data_normalized['execution']['settings'][
'access']:
result['debugInfo'] = [
'ERROR: cannot send RED data to CC-Agency if auth is not defined in access '
'settings.'
]
result['state'] = 'failed'
return result
access = red_data_normalized['execution']['settings']['access']
r = requests.post('{}/red'.format(access['url'].strip('/')),
auth=(access['auth']['username'],
access['auth']['password']),
json=red_data)
if 400 <= r.status_code < 500:
try:
pprint(r.json())
except ValueError: # if the body does not contain json, we ignore it
pass
r.raise_for_status()
dump_print(r.json(), fmt)
except Exception as e:
print_exception(e, secret_values)
result['debugInfo'] = exception_format(secret_values)
result['state'] = 'failed'
return result
def run(red_file,
disable_pull,
leave_container,
preserve_environment,
non_interactive,
insecure,
output_mode,
keyring_service,
gpu_ids,
**_
):
"""
Executes a RED Experiment.
:param red_file: The path or URL to the RED File to execute
:param disable_pull: If True the docker image is not pulled from an registry
:param leave_container: If set to True, the executed docker container will not be removed.
:param preserve_environment: List of environment variables to preserve inside the docker container.
:param non_interactive: If True, unresolved template values are not asked interactively
:param insecure: Allow insecure capabilities
:param output_mode: Either Connectors or Directory. If Connectors, the blue agent will try to execute the output
connectors. If Directory faice will copy the output files into the host output directory.
:param keyring_service: The keyring service name to use for template substitution
:param gpu_ids: A list of gpu ids, that should be used. If None all gpus are considered.
:type gpu_ids: List[int] or None
"""
result = {
'containers': [],
'debugInfo': None,
'state': 'succeeded'
}
secret_values = None
try:
red_data = load_and_read(red_file, 'REDFILE')
# validation
red_validation(red_data, output_mode == OutputMode.Directory, container_requirement=True)
engine_validation(red_data, 'container', ['docker'], optional=False)
# templates and secrets
complete_red_templates(red_data, keyring_service, non_interactive)
secret_values = get_secret_values(red_data)
normalize_keys(red_data)
# process red data
blue_batches = convert_red_to_blue(red_data)
# docker settings
docker_image = red_data['container']['settings']['image']['url']
ram = red_data['container']['settings'].get('ram')
environment = env_vars(preserve_environment)
# create docker manager
docker_manager = DockerManager()
# gpus
gpus = get_gpus(docker_manager, red_data['container']['settings'].get('gpus'), gpu_ids)
if not disable_pull:
registry_auth = red_data['container']['settings']['image'].get('auth')
docker_manager.pull(docker_image, auth=registry_auth)
if len(blue_batches) == 1:
host_outdir = 'outputs'
else:
host_outdir = 'outputs_{batch_index}'
for batch_index, blue_batch in enumerate(blue_batches):
container_execution_result = run_blue_batch(
blue_batch=blue_batch,
docker_manager=docker_manager,
docker_image=docker_image,
host_outdir=host_outdir,
output_mode=output_mode,
leave_container=leave_container,
batch_index=batch_index,
ram=ram,
gpus=gpus,
environment=environment,
insecure=insecure
)
# handle execution result
result['containers'].append(container_execution_result.to_dict())
container_execution_result.raise_for_state()
except Exception as e:
print_exception(e, secret_values)
result['debugInfo'] = exception_format(secret_values)
result['state'] = 'failed'
return result
def post_red():
user = auth.verify_user(request.authorization, request.cookies,
request.remote_addr)
if not request.json:
raise BadRequest('Did not send RED data as JSON.')
data = request.json
disable_retry = str_to_bool(
request.args.get('disableRetry', default=False))
disable_connector_validation = str_to_bool(
request.args.get('disableConnectorValidation', default=False))
try:
red_validation(data, False)
except Exception as e:
raise BadRequest(
'Given RED data is invalid. Consider using the FAICE commandline tools for local validation.\n{}'
.format(str(e)))
template_keys = get_variable_keys(data)
if template_keys:
raise BadRequest(
'The given red data contains the following variables: "{}". Please resolve them before submitting'
' to agency. Consider using CC-FAICE (faice exec).'.format(
', '.join(map(str, template_keys))))
secret_values = get_secret_values(data)
if 'batches' in data:
for batch in data['batches']:
if 'outputs' not in batch:
raise BadRequest(
'CC-Agency requires all batches to have outputs defined. At least one batch does not comply.'
)
elif 'outputs' not in data:
raise BadRequest(
'CC-Agency requires outputs to be defined in RED data.')
try:
engine_validation(data, 'container', ['docker'])
except Exception:
raise BadRequest('\n'.join(
exception_format(secret_values=secret_values)))
if 'ram' not in data['container']['settings']:
raise BadRequest(
'CC-Agency requires \'ram\' to be defined in the container settings.'
)
try:
engine_validation(data, 'execution', ['ccagency'], optional=True)
normalize_keys(data)
_ = convert_red_to_restricted_red(data)
except Exception:
raise BadRequest('\n'.join(
exception_format(secret_values=secret_values)))
experiment, batches, secrets = _prepare_red_data(
data, user, disable_retry, disable_connector_validation)
response = trustee_client.store(secrets)
if response['state'] == 'failed':
raise InternalServerError('Trustee service failed:\n{}'.format(
response['debug_info']))
bson_experiment_id = mongo.db['experiments'].insert_one(
experiment).inserted_id
experiment_id = str(bson_experiment_id)
for batch in batches:
batch['experimentId'] = experiment_id
mongo.db['batches'].insert_many(batches)
controller.send_json({'destination': 'scheduler'})
return create_flask_response({'experimentId': experiment_id}, auth,
user.authentication_cookie)
def run(
red_file,
non_interactive,
keyring_service,
preserve_environment,
disable_pull,
leave_container,
insecure,
gpu_ids,
disable_retry,
disable_connector_validation,
**_
):
"""
Runs the RED Client.
:param red_file: The path or URL to the RED File to execute
:param non_interactive: If True, unresolved template values are not asked interactively
:type non_interactive: bool
:param keyring_service: The keyring service name to use for template substitution
:type keyring_service: str
:param preserve_environment: List of environment variables to preserve inside the docker container.
:type preserve_environment: list[str]
:param disable_pull: If True the docker image is not pulled from an registry
:type disable_pull: bool
:param leave_container: If set to True, the executed docker container will not be removed.
:type leave_container: bool
:param insecure: Allow insecure capabilities
:type insecure: bool
:param gpu_ids: A list of gpu ids, that should be used. If None all gpus are considered.
:type gpu_ids: List[int] or None
:param disable_retry: If True, the execution engine will not retry the experiment, if it fails.
:type disable_retry: bool
:param disable_connector_validation: If True, the execution engine will skip connector validation
:type disable_connector_validation: bool
:return: a dictionary containing debug information about the process
"""
secret_values = None
result = {
'state': 'succeeded',
'debugInfo': None
}
try:
red_data = load_and_read(red_file, 'REDFILE')
secret_values = get_secret_values(red_data)
red_validation(red_data, False)
engine_validation(red_data, 'execution', ['ccfaice', 'ccagency'], 'faice exec')
_check_execution_arguments(
red_data.get('execution', {}).get('engine', 'ccfaice'),
preserve_environment=preserve_environment,
disable_pull=disable_pull,
leave_container=leave_container,
insecure=insecure,
gpu_ids=gpu_ids,
disable_retry=disable_retry,
disable_connector_validation=disable_connector_validation
)
if 'execution' not in red_data:
raise KeyError('The key "execution" is needed in red file for usage with faice exec.')
complete_red_variables(red_data, keyring_service, non_interactive)
# exec via faice or agency
if red_data['execution']['engine'] == 'ccfaice':
return run_faice(
red_data, preserve_environment, disable_pull, leave_container, insecure, gpu_ids,
disable_connector_validation
)
elif red_data['execution']['engine'] == 'ccagency':
return run_agency(red_data, disable_retry, disable_connector_validation)
except Exception as e:
print_exception(e, secret_values)
result['debugInfo'] = exception_format(secret_values)
result['state'] = 'failed'
return result