def read(raw_data, var_name):
try:
data = yaml.load(raw_data)
except YAMLError as e:
raise AgentError('data for argument "{}" is neither json nor yaml formatted. Failed with the following message:'
'\n{}'.format(var_name, str(e)))
if not isinstance(data, dict):
raise AgentError('data for argument "{}" does not contain a dictionary.\ndata: "{}"'.format(var_name, data))
return data
def get_file_archive(container, file_path):
"""
Retrieves the given file path as tar-archive from the internal docker container.
:param container: The container to get the archive from
:type container: Container
:param file_path: A file path inside the docker container
:type file_path: str
:return: A tar archive, which corresponds to the given file path
:rtype: tarfile.TarFile
:raise AgentError: If the given file could not be fetched
"""
try:
bits, _ = container.get_archive(file_path)
output_archive_bytes = io.BytesIO()
for chunk in bits:
output_archive_bytes.write(chunk)
output_archive_bytes.seek(0)
except DockerException as e:
raise AgentError(str(e))
return tarfile.TarFile(fileobj=output_archive_bytes)
def raise_for_state(self):
"""
Raises an AgentError, if state is not successful.
:raise AgentError: If state is not successful
"""
if not self.successful():
raise AgentError(self.agent_std_err)
def dump(stream, dump_format, file_name):
if dump_format == 'json':
with open(file_name, 'w') as f:
json.dump(stream, f, indent=JSON_INDENT)
elif dump_format in ['yaml', 'yml']:
with open(file_name, 'w') as f:
yaml.dump(stream, f)
else:
raise AgentError('invalid dump format "{}"'.format(dump_format))
def dump_print(stream, dump_format, error=False):
if dump_format == 'json':
if error:
print(json.dumps(stream, indent=JSON_INDENT), file=sys.stderr)
else:
print(json.dumps(stream, indent=JSON_INDENT))
elif dump_format in ['yaml', 'yml']:
if error:
yaml.dump(stream, sys.stderr)
else:
yaml.dump(stream, sys.stdout)
elif dump_format != 'none':
raise AgentError('invalid dump format "{}"'.format(dump_format))
def get_agent_result_dict(self):
"""
This function parses the stdout only once.
:return: The result of the agent as dictionary
:rtype: Dict
:raise AgentError: If the stdout of the agent is not valid json
"""
if self._parsed_stdout is None:
try:
self._parsed_stdout = json.loads(self._stdout)
except json.JSONDecodeError:
raise AgentError('Could not parse stdout of agent.\n'
'Agent stdout:\n{}'
'\nAgent stderr:\n{}'.format(
self._stdout, self._stderr))
return self._parsed_stdout
def _handle_directory_outputs(host_outdir, outputs, container, docker_manager):
"""
Creates the host_outdir and retrieves the files given in outputs from the docker container. The retrieved files are
then stored in the created host_outdir.
:param host_outdir: The absolute path to the output directory of the host.
:type host_outdir: str
:param outputs: A dictionary mapping output_keys to file information.
:type outputs: Dict[str, Dict]
:param container: The container to get the outputs from
:type container: Container
:param docker_manager: The docker manager from which to retrieve the files
:type docker_manager: DockerManager
:raise AgentError: If a file given in outputs could not be retrieved by the docker manager
"""
os.makedirs(host_outdir, exist_ok=True)
for output_key, output_file_information in outputs.items():
container_file_path = output_file_information['path']
# continue, if the output file was not found
if container_file_path is None:
continue
file_path = os.path.join(CONTAINER_OUTPUT_DIR, container_file_path)
if not file_path:
continue
try:
file_archive = docker_manager.get_file_archive(container, file_path)
except AgentError as e:
raise AgentError(
'Could not retrieve output file "{}" with path "{}" from docker container. '
'Failed with the following message:\n{}'
.format(output_key, file_path, str(e))
)
file_archive.extractall(host_outdir)
file_archive.close()
def retrieve_file_archive(container, container_path):
"""
Retrieves the file given by container_path as TarFile object with only one member.
:param container: The container to retrieve the file from
:type container: Container
:param container_path: The path inside the container to retrieve. This should be an absolute path.
:type container_path: PurePosixPath
:return: A TarFile object with the only member being the specified file
:rtype: tarfile.TarFile
:raise AgentError: If the container path does not exists or if the connection to the docker container is
interrupted.
"""
try:
bits, _ = container.get_archive(container_path.as_posix())
except (DockerException, ConnectionError) as e:
raise AgentError(str(e))
return tarfile.open(fileobj=ContainerFileBitsWrapper(bits), mode='r|*')
def _transfer_file(container_path, container, host_outdir):
"""
Transfers a given file path from the given container into the given host output directory
:param container_path: The path inside the container where the source file is located. If absolute the absolute path
is taken. If relative it is relative to the cc CONTAINER_OUTPUT_DIR.
:type container_path: PurePosixPath
:param container: The docker container from which to transfer the file
:type container: Container
:param host_outdir: The host output directory where to copy the file
:type host_outdir: Path
:raise AgentError: If the given container file could not be retrieved
"""
abs_container_path = CONTAINER_OUTPUT_DIR / container_path
try:
with retrieve_file_archive(container,
abs_container_path) as file_archive:
file_archive.extractall(host_outdir)
except AgentError as e:
raise AgentError(
'Could not retrieve stdout file with path "{}" from docker container. '
'Failed with the following message:\n{}'.format(
abs_container_path, str(e)))
def file_extension(dump_format):
if dump_format == 'json':
return dump_format
if dump_format in ['yaml', 'yml']:
return 'yml'
raise AgentError('invalid dump format "{}"'.format(dump_format))
def load(location, var_name):
try:
with open(os.path.expanduser(location)) as f:
return f.read()
except:
raise AgentError('File "{}" for argument "{}" could not be loaded from file system'.format(location, var_name))