class JSONFileComm(FileComm.FileComm):
r"""Class for handling I/O from/to a JSON file on disk.
Args:
name (str): The environment variable where file path is stored.
**kwargs: Additional keywords arguments are passed to parent class.
"""
_filetype = 'json'
_schema_properties = inherit_schema(FileComm.FileComm._schema_properties,
**JSONSerialize._schema_properties)
_default_serializer = JSONSerialize
@classmethod
def get_testing_options(cls):
r"""Method to return a dictionary of testing options for this class.
Returns:
dict: Dictionary of variables to use for testing. Key/value pairs:
kwargs (dict): Keyword arguments for comms tested with the
provided content.
send (list): List of objects to send to test file.
recv (list): List of objects that will be received from a test
file that was sent the messages in 'send'.
contents (bytes): Bytes contents of test file created by sending
the messages in 'send'.
"""
out = super(JSONFileComm, cls).get_testing_options()
out['recv'] = out['send']
return out
class FileComm(CommBase.CommBase):
r"""Class for handling I/O from/to a file on disk.
>>> x = FileComm('test_send', address='test_file.txt', direction='send')
>>> x.send('Test message')
True
>>> with open('test_file.txt', 'r') as fd:
... print(fd.read())
Test message
>>> x = FileComm('test_recv', address='test_file.txt', direction='recv')
>>> x.recv()
(True, b'Test message')
Args:
name (str): The environment variable where communication address is
stored.
read_meth (str, optional): Method that should be used to read data
from the file. Defaults to 'read'. Ignored if direction is 'send'.
append (bool, optional): If True and writing, file is openned in append
mode. Defaults to False.
in_temp (bool, optional): If True, the path will be considered relative
to the platform temporary directory. Defaults to False.
open_as_binary (bool, optional): If True, the file is opened in binary
mode. Defaults to True.
newline (str, optional): String indicating a new line. Defaults to
serialize._default_newline.
is_series (bool, optional): If True, input/output will be done to
a series of files. If reading, each file will be processed until
the end is reached. If writing, each output will be to a new
file in the series. The addressed is assumed to contain a format
for the index of the file. Defaults to False.
wait_for_creation (float, optional): Time (in seconds) that should be
waited before opening for the file to be created if it dosn't exist.
Defaults to 0 s and file will attempt to be opened immediately.
**kwargs: Additional keywords arguments are passed to parent class.
Attributes:
fd (file): File that should be read/written.
read_meth (str): Method that should be used to read data from the file.
append (bool): If True and writing, file is openned in append mode.
in_temp (bool): If True, the path will be considered relative to the
platform temporary directory.
open_as_binary (bool): If True, the file is opened in binary mode.
newline (str): String indicating a new line.
is_series (bool): If True, input/output will be done to a series of
files. If reading, each file will be processed until the end is
reached. If writing, each output will be to a new file in the series.
platform_newline (str): String indicating a newline on the current
platform.
Raises:
ValueError: If the read_meth is not one of the supported values.
"""
_filetype = 'binary'
_datatype = {'type': 'bytes'}
_schema_type = 'file'
_schema_required = ['name', 'filetype', 'working_dir']
_schema_properties = inherit_schema(
CommBase.CommBase._schema_properties,
{'working_dir': {'type': 'string'},
'filetype': {'type': 'string', 'default': _filetype},
'append': {'type': 'boolean', 'default': False},
'in_temp': {'type': 'boolean', 'default': False},
'is_series': {'type': 'boolean', 'default': False},
'wait_for_creation': {'type': 'float', 'default': 0.0}},
remove_keys=['commtype', 'datatype'], **DirectSerialize._schema_properties)
_default_serializer = DirectSerialize
_attr_conv = ['newline', 'platform_newline']
_default_extension = '.txt'
is_file = True
_maxMsgSize = 0
def __init__(self, *args, **kwargs):
kwargs.setdefault('close_on_eof_send', True)
return super(FileComm, self).__init__(*args, **kwargs)
def _init_before_open(self, read_meth='read', open_as_binary=True, **kwargs):
r"""Get absolute path and set attributes."""
super(FileComm, self)._init_before_open(**kwargs)
# Process file class keywords
if not hasattr(self, '_fd'):
self._fd = None
if read_meth not in ['read', 'readline']:
raise ValueError("read_meth '%s' not supported." % read_meth)
self.read_meth = read_meth
self.platform_newline = platform._newline
if self.in_temp:
self.address = os.path.join(tempfile.gettempdir(), self.address)
self.address = os.path.abspath(self.address)
self.open_as_binary = open_as_binary
self._series_index = 0
# Put string attributes in the correct format
if self.open_as_binary:
func_conv = backwards.as_bytes
else:
func_conv = backwards.as_unicode
for k in self._attr_conv:
v = getattr(self, k)
if v is not None:
setattr(self, k, func_conv(v))
@classmethod
def get_testing_options(cls, read_meth='read', open_as_binary=True, **kwargs):
r"""Method to return a dictionary of testing options for this class.
Returns:
dict: Dictionary of variables to use for testing. Key/value pairs:
kwargs (dict): Keyword arguments for comms tested with the
provided content.
send (list): List of objects to send to test file.
recv (list): List of objects that will be received from a test
file that was sent the messages in 'send'.
contents (bytes): Bytes contents of test file created by sending
the messages in 'send'.
"""
out = super(FileComm, cls).get_testing_options(**kwargs)
out['kwargs']['read_meth'] = read_meth
out['kwargs']['open_as_binary'] = open_as_binary
if (read_meth == 'read') and isinstance(out['recv'][0], backwards.bytes_type):
out['recv'] = [b''.join(out['recv'])]
if not open_as_binary:
out['contents'] = out['contents'].replace(
backwards.match_stype(out['contents'], '\n'),
backwards.match_stype(out['contents'], platform._newline))
return out
@classmethod
def is_installed(cls, language=None):
r"""Determine if the necessary libraries are installed for this
communication class.
Args:
language (str, optional): Specific language that should be checked
for compatibility. Defaults to None and all languages supported
on the current platform will be checked.
Returns:
bool: Is the comm installed.
"""
# Filesystem is implied
return True
@classmethod
def underlying_comm_class(self):
r"""str: Name of underlying communication class."""
return 'FileComm'
@classmethod
def close_registry_entry(cls, value):
r"""Close a registry entry."""
out = False
if not value.closed: # pragma: debug
value.close()
out = True
return out
@classmethod
def new_comm_kwargs(cls, *args, **kwargs):
r"""Initialize communication with new queue."""
kwargs.setdefault('address', 'file.txt')
return args, kwargs
@property
def open_mode(self):
r"""str: Mode that should be used to open the file."""
if self.direction == 'recv':
io_mode = 'r'
elif self.append == 'ow':
io_mode = 'r+'
elif self.append:
io_mode = 'a'
else:
io_mode = 'w'
if self.open_as_binary:
io_mode += 'b'
return io_mode
def opp_comm_kwargs(self):
r"""Get keyword arguments to initialize communication with opposite
comm object.
Returns:
dict: Keyword arguments for opposite comm object.
"""
kwargs = super(FileComm, self).opp_comm_kwargs()
kwargs['newline'] = self.newline
kwargs['open_as_binary'] = self.open_as_binary
kwargs['is_series'] = self.is_series
return kwargs
@property
def registry_key(self):
r"""str: String used to register the socket."""
# return self.address
return '%s_%s_%s' % (self.address, self.direction, self.uuid)
def record_position(self):
r"""Record the current position in the file/series."""
_rec_pos = self.fd.tell()
_rec_ind = self._series_index
return _rec_pos, _rec_ind
def change_position(self, file_pos, series_index=None):
r"""Change the position in the file/series.
Args:
file_pos (int): Position that should be moved to in the file.
series_index (int, optinal): Index of the file in the series that
should be moved to. Defaults to None and will be set to the
current series index.
"""
if series_index is None:
series_index = self._series_index
self.advance_in_series(series_index)
self.advance_in_file(file_pos)
def advance_in_file(self, file_pos):
r"""Advance to a certain position in the current file.
Args:
file_pos (int): Position that should be moved to in the current.
file.
"""
if self.is_open:
try:
self.fd.seek(file_pos)
except (AttributeError, ValueError): # pragma: debug
if self.is_open:
raise
def advance_in_series(self, series_index=None):
r"""Advance to a certain file in a series.
Args:
series_index (int, optional): Index of file in the series that
should be moved to. Defaults to None and call will advance to
the next file in the series.
Returns:
bool: True if the file was advanced in the series, False otherwise.
"""
out = False
if self.is_series:
if series_index is None:
series_index = self._series_index + 1
if self._series_index != series_index:
if (((self.direction == 'send')
or os.path.isfile(self.get_series_address(series_index)))):
self._file_close()
self._series_index = series_index
self._open()
out = True
self.debug("Advanced to %d", series_index)
return out
def get_series_address(self, index=None):
r"""Get the address of a file in the series.
Args:
index (int, optional): Index in series to get address for.
Defaults to None and the current index is used.
Returns:
str: Address for the file in the series.
"""
if index is None:
index = self._series_index
return self.address % index
@property
def current_address(self):
r"""str: Address of file currently being used."""
if self.is_series:
address = self.get_series_address()
else:
address = self.address
return address
def _open(self):
address = self.current_address
if self.fd is None:
if (not os.path.isfile(address)) and (self.wait_for_creation > 0):
T = self.start_timeout(self.wait_for_creation)
while (not T.is_out) and (not os.path.isfile(address)):
self.sleep()
self.stop_timeout()
self._fd = open(address, self.open_mode)
T = self.start_timeout()
while (not T.is_out) and (not self.is_open): # pragma: debug
self.sleep()
self.stop_timeout()
if self.append == 'ow':
try:
self.fd.seek(0, os.SEEK_END)
except (AttributeError, ValueError): # pragma: debug
if self.is_open:
raise
def _file_close(self):
if self.is_open:
try:
self.fd.flush()
os.fsync(self.fd.fileno())
except OSError: # pragma: debug
pass
try:
self.fd.close()
except (AttributeError, ValueError): # pragma: debug
if self.is_open:
raise
self._fd = None
def open(self):
r"""Open the file."""
super(FileComm, self).open()
self._open()
self.register_comm(self.registry_key, self.fd)
def _close(self, *args, **kwargs):
r"""Close the file."""
self._file_close()
self.unregister_comm(self.registry_key)
super(FileComm, self)._close(*args, **kwargs)
def remove_file(self):
r"""Remove the file."""
assert(self.is_closed)
if self.is_series:
i = 0
while True:
address = self.get_series_address(i)
if not os.path.isfile(address):
break
os.remove(address)
i += 1
else:
if os.path.isfile(self.address):
os.remove(self.address)
@property
def is_open(self):
r"""bool: True if the connection is open."""
try:
return (self.fd is not None) and (not self.fd.closed)
except AttributeError: # pragma: debug
if self.fd is not None:
raise
return False
@property
def fd(self):
r"""Associated file identifier."""
return self._fd
@property
def remaining_bytes(self):
r"""int: Remaining bytes in the file."""
if self.is_closed or self.direction == 'send':
return 0
pos = self.record_position()
try:
curpos = self.fd.tell()
self.fd.seek(0, os.SEEK_END)
endpos = self.fd.tell()
out = endpos - curpos
except (ValueError, AttributeError): # pragma: debug
if self.is_open:
raise
out = 0
if self.is_series:
i = self._series_index + 1
while True:
fname = self.get_series_address(i)
if not os.path.isfile(fname):
break
out += os.path.getsize(fname)
i += 1
self.change_position(*pos)
return out
@property
def n_msg_recv(self):
r"""int: The number of messages in the file."""
if self.is_closed:
return 0
if self.read_meth == 'read':
return int(self.remaining_bytes > 0)
elif self.read_meth == 'readline':
pos = self.record_position()
try:
out = 0
flag, msg = self._recv()
while len(msg) != 0 and msg != self.eof_msg:
out += 1
flag, msg = self._recv()
except ValueError: # pragma: debug
out = 0
self.change_position(*pos)
else: # pragma: debug
self.error('Unsupported read_meth: %s', self.read_meth)
out = 0
return out
def on_send_eof(self):
r"""Close file when EOF to be sent.
Returns:
bool: False so that message not sent.
"""
flag, msg_s = super(FileComm, self).on_send_eof()
try:
self.fd.flush()
except (AttributeError, ValueError): # pragma: debug
if self.is_open:
raise
# self.close()
return flag, msg_s
def _send(self, msg):
r"""Write message to a file.
Args:
msg (bytes, str): Data to write to the file.
Returns:
bool: Success or failure of writing to the file.
"""
try:
if msg != self.eof_msg:
if not self.open_as_binary:
msg = backwards.as_unicode(msg)
self.fd.write(msg)
if self.append == 'ow':
self.fd.truncate()
self.fd.flush()
except (AttributeError, ValueError): # pragma: debug
if self.is_open:
raise
return False
if msg != self.eof_msg and self.is_series:
self.advance_in_series()
self.debug("Advanced to %d", self._series_index)
return True
def _recv(self, timeout=0):
r"""Reads message from a file.
Args:
timeout (float, optional): Time in seconds to wait for a message.
Defaults to self.recv_timeout. Unused.
Returns:
tuple (bool, str): Success or failure of reading from the file and
the read messages as bytes.
"""
flag = True
try:
if self.read_meth == 'read':
out = self.fd.read()
elif self.read_meth == 'readline':
out = self.fd.readline()
except BaseException: # pragma: debug
# Use this to catch case where close called during receive.
# In the future this should be handled via a lock.
out = ''
if len(out) == 0:
if self.advance_in_series():
self.debug("Advanced to %d", self._series_index)
flag, out = self._recv()
else:
out = self.eof_msg
else:
out = out.replace(self.platform_newline, self.newline)
if not self.open_as_binary:
out = backwards.as_bytes(out)
return (flag, out)
def purge(self):
r"""Purge all messages from the comm."""
if self.is_open and self.direction == 'recv':
try:
self.fd.seek(0, os.SEEK_END)
except (AttributeError, ValueError): # pragma: debug
if self.is_open:
raise
class MakeModelDriver(ModelDriver):
r"""Class for running make file compiled drivers. Before running the
make command, the necessary compiler & linker flags for the interface's
C/C++ library are stored the environment variables YGGCCFLAGS and YGGLDFLAGS
respectively. These should be used in the make file to correctly compile
with the interface's C/C++ libraries.
Args:
name (str): Driver name.
args (str, list): Executable that should be created (make target) and
any arguments for the executable.
make_command (str, optional): Command that should be used for make.
Defaults to 'make' on Linux/MacOS and 'nmake' on windows.
makefile (str, optional): Path to make file either absolute, relative to
makedir (if provided), or relative to working_dir. Defaults to
Makefile.
makedir (str, optional): Directory where make should be invoked from
if it is not the same as the directory containing the makefile.
Defaults to directory containing makefile if provided, otherwise
self.working_dir.
**kwargs: Additional keyword arguments are passed to parent class.
Attributes:
compiled (bool): True if the compilation was successful, False otherwise.
target (str): Name of executable that should be created and called.
make_command (str): Command that should be used for make.
makedir (str): Directory where make should be invoked from.
makefile (str): Path to make file either relative to makedir or absolute.
Raises:
RuntimeError: If neither the IPC or ZMQ C libraries are available.
"""
_language = 'make'
_schema_properties = inherit_schema(
ModelDriver._schema_properties, {
'make_command': {
'type': 'string',
'default': _default_make_command
},
'makefile': {
'type': 'string',
'default': _default_makefile
},
'makedir': {
'type': 'string'
}
}) # default will depend on makefile
def __init__(self, name, args, **kwargs):
super(MakeModelDriver, self).__init__(name, args, **kwargs)
if not self.is_installed(): # pragma: windows
raise RuntimeError(
"No library available for models written in C/C++.")
self.debug('')
self.compiled = False
self.target = self.args[0]
if not os.path.isabs(self.makefile):
if self.makedir is not None:
self.makefile = os.path.normpath(
os.path.join(self.makedir, self.makefile))
else:
self.makefile = os.path.normpath(
os.path.join(self.working_dir, self.makefile))
if self.makedir is None:
self.makedir = os.path.dirname(self.makefile)
self.target_file = os.path.join(self.makedir, self.target)
self.args[0] = self.target_file
# Set environment variables
self.debug("Setting environment variables.")
compile_flags = ['-DYGG_DEBUG=%d' % self.logger.getEffectiveLevel()]
setup_environ(compile_flags=compile_flags)
# Compile in a new process
self.debug("Making target.")
self.make_target(self.target)
@classmethod
def is_installed(self):
r"""Determine if this model driver is installed on the current
machine.
Returns:
bool: Truth of if this model driver can be run on the current
machine.
"""
return (len(tools.get_installed_comm(language='c')) > 0)
def make_target(self, target):
r"""Run the make command to make the target.
Args:
target (str): Target that should be made.
Raises:
RuntimeError: If there is an error in running the make.
"""
curdir = os.getcwd()
os.chdir(self.makedir)
if self.make_command == 'nmake': # pragma: windows
make_opts = ['/NOLOGO', '/F']
else:
make_opts = ['-f']
make_args = [self.make_command] + make_opts + [self.makefile, target]
self.debug(' '.join(make_args))
if not os.path.isfile(self.makefile):
os.chdir(curdir)
raise IOError("Makefile %s not found" % self.makefile)
comp_process = tools.popen_nobuffer(make_args)
output, err = comp_process.communicate()
exit_code = comp_process.returncode
os.chdir(curdir)
if exit_code != 0:
self.error(output)
raise RuntimeError("Make failed with code %d." % exit_code)
self.debug('Make complete')
def cleanup(self):
r"""Remove compile executable."""
if (self.target_file is not None) and os.path.isfile(self.target_file):
self.make_target('clean')
super(MakeModelDriver, self).cleanup()
class CMakeModelDriver(ModelDriver):
r"""Class for running cmake compiled drivers. Before running the
cmake command, the cmake commands for setting the necessary compiler & linker
flags for the interface's C/C++ library are written to a file called
'ygg_cmake.txt' that should be included in the CMakeLists.txt file (after
the target executable has been added).
Args:
name (str): Driver name.
args (str, list): Executable that should be created (cmake target) and
any arguments for the executable.
sourcedir (str, optional): Source directory to call cmake on. If not
provided it is set to self.working_dir. This should be the directory
containing the CMakeLists.txt file. It can be relative to
self.working_dir or absolute.
builddir (str, optional): Directory where the build should be saved.
Defaults to <sourcedir>/build. It can be relative to self.working_dir
or absolute.
cmakeargs (list, optional): Arguments that should be passed to cmake.
Defaults to [].
preserve_cache (bool, optional): If True the cmake cache will be kept
following the run, otherwise all files created by cmake will be
cleaned up. Defaults to False.
**kwargs: Additional keyword arguments are passed to parent class.
Attributes:
compiled (bool): True if the compilation was successful, False otherwise.
target (str): Name of executable that should be created and called.
sourcedir (str): Source directory to call cmake on.
builddir (str): Directory where the build should be saved.
cmakeargs (list): Arguments that should be passed to cmake.
preserve_cache (bool): If True the cmake cache will be kept following the
run, otherwise all files created by cmake will be cleaned up.
Raises:
RuntimeError: If neither the IPC or ZMQ C libraries are available.
"""
_language = 'cmake'
_schema_properties = inherit_schema(
ModelDriver._schema_properties, {
'sourcedir': {
'type': 'string'
},
'builddir': {
'type': 'string'
},
'cmakeargs': {
'type': 'array',
'default': [],
'items': {
'type': 'string'
}
}
})
def __init__(self, name, args, preserve_cache=False, **kwargs):
super(CMakeModelDriver, self).__init__(name, args, **kwargs)
if not self.is_installed(): # pragma: windows
raise RuntimeError(
"No library available for models written in C/C++.")
self.debug('')
self.compiled = False
self.target = self.args[0]
if self.sourcedir is None:
self.sourcedir = self.working_dir
elif not os.path.isabs(self.sourcedir):
self.sourcedir = os.path.realpath(
os.path.join(self.working_dir, self.sourcedir))
if self.builddir is None:
self.builddir = os.path.join(self.sourcedir, 'build')
elif not os.path.isabs(self.builddir):
self.builddir = os.path.realpath(
os.path.join(self.working_dir, self.builddir))
if isinstance(self.cmakeargs, backwards.string_types):
self.cmakeargs = [self.cmakeargs]
self.preserve_cache = preserve_cache
self.target_file = os.path.join(self.builddir, self.target)
self.include_file = os.path.join(self.sourcedir, 'ygg_cmake.txt')
self.args[0] = self.target_file
# Set environment variables
self.debug("Setting environment variables.")
create_include(self.include_file, self.target)
# Compile in a new process
self.debug("Making target.")
self.run_cmake(self.target)
@classmethod
def is_installed(self):
r"""Determine if this model driver is installed on the current
machine.
Returns:
bool: Truth of if this model driver can be run on the current
machine.
"""
return GCCModelDriver.GCCModelDriver.is_installed()
def run_cmake(self, target=None):
r"""Run the cmake command on the source.
Args:
target (str, optional): Target to build.
Raises:
RuntimeError: If there is an error in running cmake.
"""
curdir = os.getcwd()
os.chdir(self.sourcedir)
if not os.path.isfile('CMakeLists.txt'):
os.chdir(curdir)
self.cleanup()
raise IOError('No CMakeLists.txt file found in %s.' %
self.sourcedir)
# Configuration
if target != 'clean':
config_cmd = ['cmake'] + self.cmakeargs
config_cmd += ['-H.', self.sourcedir, '-B%s' % self.builddir]
self.debug(' '.join(config_cmd))
comp_process = tools.popen_nobuffer(config_cmd)
output, err = comp_process.communicate()
exit_code = comp_process.returncode
if exit_code != 0:
os.chdir(curdir)
self.cleanup()
self.error(backwards.as_unicode(output))
raise RuntimeError("CMake config failed with code %d." %
exit_code)
self.debug('Config output: \n%s' % output)
# Build
build_cmd = ['cmake', '--build', self.builddir, '--clean-first']
if self.target is not None:
build_cmd += ['--target', self.target]
self.info(' '.join(build_cmd))
comp_process = tools.popen_nobuffer(build_cmd)
output, err = comp_process.communicate()
exit_code = comp_process.returncode
if exit_code != 0: # pragma: debug
os.chdir(curdir)
self.error(backwards.as_unicode(output))
self.cleanup()
raise RuntimeError("CMake build failed with code %d." % exit_code)
self.debug('Build output: \n%s' % output)
self.debug('Make complete')
os.chdir(curdir)
def cleanup(self):
r"""Remove compile executable."""
# self.run_cmake('clean')
if not self.preserve_cache:
rmfiles = [
self.include_file, self.target_file,
os.path.join(self.builddir, 'Makefile'),
os.path.join(self.builddir, 'CMakeCache.txt'),
os.path.join(self.builddir, 'cmake_install.cmake'),
os.path.join(self.builddir, 'CMakeFiles')
]
for f in rmfiles:
if os.path.isdir(f):
shutil.rmtree(f)
elif os.path.isfile(f):
os.remove(f)
if os.path.isdir(
self.builddir) and (not os.listdir(self.builddir)):
os.rmdir(self.builddir)
super(CMakeModelDriver, self).cleanup()
class AsciiFileComm(FileComm):
r"""Class for handling I/O from/to a file on disk.
Args:
name (str): The environment variable where communication address is
stored.
comment (str, optional): String indicating a comment. If 'read_meth'
is 'readline' and this is provided, lines starting with a comment
will be skipped.
**kwargs: Additional keywords arguments are passed to parent class.
Attributes:
comment (str): String indicating a comment.
"""
_filetype = 'ascii'
_schema_properties = inherit_schema(
FileComm._schema_properties,
{'comment': {'type': 'string',
'default': backwards.as_str(serialize._default_comment)}})
_attr_conv = FileComm._attr_conv + ['comment']
def _init_before_open(self, **kwargs):
r"""Get absolute path and set attributes."""
kwargs.setdefault('read_meth', 'readline')
super(AsciiFileComm, self)._init_before_open(**kwargs)
@classmethod
def get_testing_options(cls, **kwargs):
r"""Method to return a dictionary of testing options for this class.
Returns:
dict: Dictionary of variables to use for testing. Key/value pairs:
kwargs (dict): Keyword arguments for comms tested with the
provided content.
send (list): List of objects to send to test file.
recv (list): List of objects that will be received from a test
file that was sent the messages in 'send'.
contents (bytes): Bytes contents of test file created by sending
the messages in 'send'.
"""
kwargs.setdefault('read_meth', 'readline')
out = super(AsciiFileComm, cls).get_testing_options(**kwargs)
comment = backwards.as_bytes(
cls._schema_properties['comment']['default'] + 'Comment\n')
out['send'].append(comment)
out['contents'] = b''.join(out['send'])
# out['contents'] = out['contents'].replace(b'\n', platform._newline)
out['dict'] = {'f0': out['msg']}
return out
def opp_comm_kwargs(self):
r"""Get keyword arguments to initialize communication with opposite
comm object.
Returns:
dict: Keyword arguments for opposite comm object.
"""
kwargs = super(AsciiFileComm, self).opp_comm_kwargs()
kwargs['comment'] = self.serializer.comment
return kwargs
def _recv(self, timeout=0):
r"""Reads message from a file.
Args:
timeout (float, optional): Time in seconds to wait for a message.
Defaults to self.recv_timeout. Unused.
Returns:
tuple (bool, str): Success or failure of reading from the file and
the read messages as bytes.
"""
flag, msg = super(AsciiFileComm, self)._recv()
if self.read_meth == 'readline':
while flag and msg.startswith(backwards.as_bytes(self.comment)):
flag, msg = super(AsciiFileComm, self)._recv()
return flag, msg
class AsciiTableComm(AsciiFileComm):
r"""Class for handling I/O from/to a file on disk.
Args:
name (str): The environment variable where communication address is
stored.
delimiter (str, optional): String that should be used to separate
columns. If not provided and format_str is not set prior to I/O,
this defaults to whitespace.
use_astropy (bool, optional): If True and the astropy package is
installed, it will be used to read/write the table. Defaults to
False.
**kwargs: Additional keywords arguments are passed to parent class.
"""
_filetype = 'table'
_schema_properties = inherit_schema(
AsciiFileComm._schema_properties, {
'as_array': {
'type': 'boolean',
'default': False
},
'field_names': {
'type': 'array',
'items': {
'type': 'string'
}
},
'field_units': {
'type': 'array',
'items': {
'type': 'string'
}
}
}, **AsciiTableSerialize._schema_properties)
_default_serializer = AsciiTableSerialize
_attr_conv = AsciiFileComm._attr_conv + ['delimiter', 'format_str']
def _init_before_open(self, **kwargs):
r"""Set up dataio and attributes."""
self.header_was_read = False
self.header_was_written = False
super(AsciiTableComm, self)._init_before_open(**kwargs)
if self.serializer.as_array:
self.read_meth = 'read'
else:
self.read_meth = 'readline'
if self.append:
self.header_was_written = True
@classmethod
def get_testing_options(cls, as_array=False, **kwargs):
r"""Method to return a dictionary of testing options for this class.
Returns:
dict: Dictionary of variables to use for testing. Key/value pairs:
kwargs (dict): Keyword arguments for comms tested with the
provided content.
send (list): List of objects to send to test file.
recv (list): List of objects that will be received from a test
file that was sent the messages in 'send'.
contents (bytes): Bytes contents of test file created by sending
the messages in 'send'.
"""
out = super(AsciiFileComm, cls).get_testing_options(as_array=as_array,
**kwargs)
field_names = [
backwards.as_str(x) for x in out['kwargs']['field_names']
]
field_units = [
backwards.as_str(x) for x in out['kwargs']['field_units']
]
if as_array:
lst = out['send'][0]
out['recv'] = [[
units.add_units(np.hstack([x[i] for x in out['send']]), u)
for i, (n, u) in enumerate(zip(field_names, field_units))
]]
out['dict'] = {k: l for k, l in zip(field_names, lst)}
out['msg_array'] = serialize.list2numpy(lst, names=field_names)
else:
out['recv'] = out['send']
out['dict'] = {k: v for k, v in zip(field_names, out['send'][0])}
out['field_names'] = field_names
out['field_units'] = field_units
return out
def read_header(self):
r"""Read header lines from the file and update serializer info."""
if self.header_was_read:
return
pos = self.record_position()
self.change_position(0)
serialize.discover_header(self.fd,
self.serializer,
newline=self.newline,
comment=self.comment,
delimiter=self.delimiter)
self.change_position(*pos)
self.header_was_read = True
def write_header(self):
r"""Write header lines to the file based on the serializer info."""
if self.header_was_written:
return
header_msg = serialize.format_header(
format_str=self.serializer.format_str,
field_names=self.serializer.get_field_names(as_bytes=True),
field_units=self.serializer.get_field_units(as_bytes=True),
comment=self.comment,
newline=self.newline,
delimiter=self.delimiter)
self.fd.write(header_msg)
self.header_was_written = True
def record_position(self):
r"""Record the current position in the file/series."""
pos, ind = super(AsciiTableComm, self).record_position()
return pos, ind, self.header_was_read, self.header_was_written
def change_position(self,
file_pos,
series_index=None,
header_was_read=None,
header_was_written=None):
r"""Change the position in the file/series.
Args:
file_pos (int): Position that should be moved to in the file.
series_index (int, optinal): Index of the file in the series that
should be moved to. Defaults to None and will be set to the
current series index.
header_was_read (bool, optional): Status of if header has been
read or not. Defaults to None and will be set to the current
value.
header_was_written (bool, optional): Status of if header has been
written or not. Defaults to None and will be set to the current
value.
"""
if header_was_read is None:
header_was_read = self.header_was_read
if header_was_written is None:
header_was_written = self.header_was_written
super(AsciiTableComm, self).change_position(file_pos, series_index)
self.header_was_read = header_was_read
self.header_was_written = header_was_written
def advance_in_series(self, *args, **kwargs):
r"""Advance to a certain file in a series.
Args:
index (int, optional): Index of file in the series that should be
moved to. Defaults to None and call will advance to the next
file in the series.
Returns:
bool: True if the file was advanced in the series, False otherwise.
"""
out = super(AsciiTableComm, self).advance_in_series(*args, **kwargs)
if out:
self.header_was_read = False
self.header_was_written = False
return out
def _send(self, msg):
r"""Write message to a file.
Args:
msg (bytes, str): Data to write to the file.
Returns:
bool: Success or failure of writing to the file.
"""
if msg != self.eof_msg:
self.write_header()
return super(AsciiTableComm, self)._send(msg)
def _recv(self, timeout=0, **kwargs):
r"""Reads message from a file.
Args:
timeout (float, optional): Time in seconds to wait for a message.
Defaults to self.recv_timeout. Unused.
Returns:
tuple(bool, str): Success or failure of reading from the file.
"""
self.read_header()
return super(AsciiTableComm, self)._recv(timeout=timeout, **kwargs)
class GCCModelDriver(ModelDriver):
r"""Class for running gcc compiled drivers.
Args:
name (str): Driver name.
args (str or list): Argument(s) for running the model on the command
line. If the first element ends with '.c', the driver attempts to
compile the code with the necessary interface include directories.
Additional arguments that start with '-I' are included in the
compile command. Others are assumed to be runtime arguments.
cc (str, optional): C/C++ Compiler that should be used. Defaults to
gcc/clang for '.c' files, and g++/clang++ for '.cpp' or '.cc' files
on Linux/MacOS. Defaults to cl on Windows.
overwrite (bool, optional): If True, any existing object or executable
files for the model are overwritten, otherwise they will only be
compiled if they do not exist. Defaults to True. Setting this to
False can be done to improve performance after debugging is complete,
but this dosn't check if the source files should be changed, so
users should make sure they recompile after any changes. The value
of this keyword also determines whether or not any compilation
products are cleaned up after a run.
**kwargs: Additional keyword arguments are passed to parent class.
Attributes (in additon to parent class's):
overwrite (bool): If True, any existing compilation products will be
overwritten by compilation and cleaned up following the run.
Otherwise, existing products will be used and will remain after
the run.
products (list): File created by the compilation.
compiled (bool): True if the compilation was succesful. False otherwise.
cfile (str): Source file.
cc (str): C/C++ Compiler that should be used.
flags (list): List of compiler flags.
efile (str): Compiled executable file.
Raises:
RuntimeError: If neither the IPC or ZMQ C libraries are available.
RuntimeError: If the compilation fails.
"""
_language = ['c', 'c++', 'cpp']
_schema_properties = inherit_schema(
ModelDriver._schema_properties,
{
'cc': {
'type': 'string'
}, # default will depend on whats being compiled
'overwrite': {
'type': 'boolean',
'default': True
}
})
def __init__(self, name, args, **kwargs):
super(GCCModelDriver, self).__init__(name, args, **kwargs)
if not self.is_installed(): # pragma: windows
raise RuntimeError(
"No library available for models written in C/C++.")
self.debug('')
# Prepare arguments to compile the file
self.parse_arguments(self.args)
self.debug("Compiling")
self.products = do_compile(self.src,
out=self.efile,
cc=self.cc,
ccflags=self.ccflags,
ldflags=self.ldflags,
overwrite=self.overwrite,
working_dir=self.working_dir)
self.efile = self.products[0]
assert (os.path.isfile(self.efile))
self.debug("Compiled %s", self.efile)
if platform._is_win: # pragma: windows
self.args = [os.path.splitext(self.efile)[0]]
else:
self.args = [os.path.join(".", self.efile)]
self.args += self.run_args
self.debug('Compiled executable with %s', self.cc)
@classmethod
def is_installed(self):
r"""Determine if this model driver is installed on the current
machine.
Returns:
bool: Truth of if this model driver can be run on the current
machine.
"""
return _c_installed
def parse_arguments(self, args):
r"""Sort arguments based on their syntax. Arguments ending with '.c' or
'.cpp' are considered source and the first one will be compiled to an
executable. Arguments starting with '-L' or '-l' are treated as linker
flags. Arguments starting with '-' are treated as compiler flags. Any
arguments that do not fall into one of the categories will be treated
as command line arguments for the compiled executable.
Args:
args (list): List of arguments provided.
Raises:
RuntimeError: If there is not a valid source file in the argument
list.
"""
self.src = []
self.ldflags = []
self.ccflags = []
self.ccflags.append('-DYGG_DEBUG=%d' % self.logger.getEffectiveLevel())
self.run_args = []
self.efile = None
is_object = False
is_link = False
for a in args:
if a.endswith('.c') or a.endswith('.cpp') or a.endswith('.cc'):
self.src.append(a)
elif a.lower().startswith('-l') or is_link:
if a.lower().startswith('/out:'): # pragma: windows
self.efile = a[5:]
elif a.lower().startswith(
'-l') and platform._is_win: # pragma: windows
a1 = '/LIBPATH:"%s"' % a[2:]
if a1 not in self.ldflags:
self.ldflags.append(a1)
elif a not in self.ldflags:
self.ldflags.append(a)
elif a == '-o':
# Next argument should be the name of the executable
is_object = True
elif a.lower() == '/link': # pragma: windows
# Following arguments should be linker options
is_link = True
elif a.startswith('-') or (platform._is_win and a.startswith('/')):
if a not in self.ccflags:
self.ccflags.append(a)
else:
if is_object:
# Previous argument was -o flag
self.efile = a
is_object = False
else:
self.run_args.append(a)
# Check source file
if len(self.src) == 0:
raise RuntimeError("Could not locate a source file in the " +
"provided arguments.")
def remove_products(self):
r"""Delete products produced during the compilation process."""
if getattr(self, 'products', None) is None: # pragma: debug
return
products = self.products
if platform._is_win: # pragma: windows
for x in copy.deepcopy(products):
base = os.path.splitext(x)[0]
products += [base + ext for ext in ['.ilk', '.pdb', '.obj']]
for p in products:
if os.path.isfile(p):
T = self.start_timeout()
while ((not T.is_out) and os.path.isfile(p)):
try:
os.remove(p)
except BaseException: # pragma: debug
if os.path.isfile(p):
self.sleep()
if T.is_out:
raise
self.stop_timeout()
if os.path.isfile(p): # pragma: debug
raise RuntimeError("Failed to remove product: %s" % p)
def cleanup(self):
r"""Remove compile executable."""
if self.overwrite:
self.remove_products()
super(GCCModelDriver, self).cleanup()