class AsciiFileComm(FileComm):
r"""Class for handling I/O from/to a file on disk.
Args:
name (str): The environment variable where communication address is
stored.
comment (str, optional): String indicating a comment. If 'read_meth'
is 'readline' and this is provided, lines starting with a comment
will be skipped.
**kwargs: Additional keywords arguments are passed to parent class.
Attributes:
comment (str): String indicating a comment.
"""
_filetype = 'ascii'
_schema = inherit_schema(FileComm._schema,
'filetype',
_filetype,
comment={
'type': 'string',
'required': False
})
def _init_before_open(self, comment=serialize._default_comment, **kwargs):
r"""Get absolute path and set attributes."""
self.comment = backwards.unicode2bytes(comment)
kwargs.setdefault('read_meth', 'readline')
super(AsciiFileComm, self)._init_before_open(**kwargs)
def opp_comm_kwargs(self):
r"""Get keyword arguments to initialize communication with opposite
comm object.
Returns:
dict: Keyword arguments for opposite comm object.
"""
kwargs = super(AsciiFileComm, self).opp_comm_kwargs()
kwargs['comment'] = self.comment
return kwargs
def _recv(self, timeout=0):
r"""Reads message from a file.
Args:
timeout (float, optional): Time in seconds to wait for a message.
Defaults to self.recv_timeout. Unused.
Returns:
tuple (bool, str): Success or failure of reading from the file and
the read messages as bytes.
"""
flag, msg = super(AsciiFileComm, self)._recv()
if self.read_meth == 'readline':
while flag and msg.startswith(self.comment):
flag, msg = super(AsciiFileComm, self)._recv()
return flag, msg
class PandasFileComm(FileComm):
r"""Class for handling I/O from/to a pandas csv file on disk.
Args:
name (str): The environment variable where communication address is
stored.
delimiter (str, optional): String that should be used to separate
columns. Defaults to '\t'.
**kwargs: Additional keywords arguments are passed to parent class.
"""
_filetype = 'pandas'
_schema = inherit_schema(FileComm._schema, 'filetype', _filetype,
delimiter={'type': 'string', 'required': False})
def _init_before_open(self, delimiter='\t', serializer_kwargs=None, **kwargs):
r"""Set up dataio and attributes."""
if serializer_kwargs is None:
serializer_kwargs = {}
serializer_kwargs.update(stype=6, delimiter=delimiter)
kwargs['serializer_kwargs'] = serializer_kwargs
super(PandasFileComm, self)._init_before_open(**kwargs)
self.read_meth = 'read'
if self.append:
self.serializer.write_header = False
def send_dict(self, args_dict, field_order=None, **kwargs):
r"""Send a message with fields specified in the input dictionary.
Args:
args_dict (dict): Dictionary with fields specifying output fields.
field_order (list, optional): List of fields in the order they
should be passed to send. If not provided, the fields from
the serializer are used. If the serializer dosn't have
field names an error will be raised.
**kwargs: Additiona keyword arguments are passed to send.
Returns:
bool: Success/failure of send.
Raises:
RuntimeError: If the field order can not be determined.
"""
if field_order is None:
if self.serializer.field_names is not None:
field_order = [
backwards.bytes2unicode(n) for n in self.serializer.field_names]
elif len(args_dict) <= 1:
field_order = [k for k in args_dict.keys()]
else: # pragma: debug
raise RuntimeError("Could not determine the field order.")
args = (serialize.dict2pandas(args_dict, order=field_order), )
return self.send(*args, **kwargs)
class CMakeModelDriver(ModelDriver):
r"""Class for running cmake compiled drivers. Before running the
cmake command, the cmake commands for setting the necessary compiler & linker
flags for the interface's C/C++ library are written to a file called
'cis_cmake.txt' that should be included in the CMakeLists.txt file (after
the target executable has been added).
Args:
name (str): Driver name.
args (str, list): Executable that should be created (cmake target) and
any arguments for the executable.
sourcedir (str, optional): Source directory to call cmake on. If not
provided it is set to self.working_dir. This should be the directory
containing the CMakeLists.txt file. It can be relative to
self.working_dir or absolute.
builddir (str, optional): Directory where the build should be saved.
Defaults to <sourcedir>/build. It can be relative to self.working_dir
or absolute.
cmakeargs (list, optional): Arguments that should be passed to cmake.
Defaults to [].
preserve_cache (bool, optional): If True the cmake cache will be kept
following the run, otherwise all files created by cmake will be
cleaned up. Defaults to False.
**kwargs: Additional keyword arguments are passed to parent class.
Attributes:
compiled (bool): True if the compilation was successful, False otherwise.
target (str): Name of executable that should be created and called.
sourcedir (str): Source directory to call cmake on.
builddir (str): Directory where the build should be saved.
cmakeargs (list): Arguments that should be passed to cmake.
preserve_cache (bool): If True the cmake cache will be kept following the
run, otherwise all files created by cmake will be cleaned up.
Raises:
RuntimeError: If neither the IPC or ZMQ C libraries are available.
"""
_language = 'cmake'
_schema = inherit_schema(ModelDriver._schema, 'language', _language,
sourcedir={'type': 'string', 'required': False},
builddir={'type': 'string', 'required': False},
cmakeargs={'type': 'list', 'required': False,
'schema': {'type': 'string'}})
def __init__(self, name, args, sourcedir=None, builddir=None,
cmakeargs=None, preserve_cache=False, **kwargs):
super(CMakeModelDriver, self).__init__(name, args, **kwargs)
if not tools._c_library_avail: # pragma: windows
raise RuntimeError("No library available for models written in C/C++.")
self.debug('')
self.compiled = False
self.target = self.args[0]
if sourcedir is None:
sourcedir = self.working_dir
elif not os.path.isabs(sourcedir):
sourcedir = os.path.realpath(os.path.join(self.working_dir, sourcedir))
self.sourcedir = sourcedir
if builddir is None:
builddir = os.path.join(sourcedir, 'build')
elif not os.path.isabs(builddir):
builddir = os.path.realpath(os.path.join(self.working_dir, builddir))
self.builddir = builddir
if cmakeargs is None:
cmakeargs = []
elif isinstance(cmakeargs, backwards.string_types):
cmakeargs = [cmakeargs]
self.cmakeargs = cmakeargs
self.preserve_cache = preserve_cache
self.target_file = os.path.join(self.builddir, self.target)
self.include_file = os.path.join(self.sourcedir, 'cis_cmake.txt')
self.args[0] = self.target_file
# Set environment variables
self.debug("Setting environment variables.")
create_include(self.include_file, self.target)
# Compile in a new process
self.debug("Making target.")
self.run_cmake(self.target)
def run_cmake(self, target=None):
r"""Run the cmake command on the source.
Args:
target (str, optional): Target to build.
Raises:
RuntimeError: If there is an error in running cmake.
"""
curdir = os.getcwd()
os.chdir(self.sourcedir)
if not os.path.isfile('CMakeLists.txt'):
os.chdir(curdir)
self.cleanup()
raise IOError('No CMakeLists.txt file found in %s.' % self.sourcedir)
# Configuration
if target != 'clean':
config_cmd = ['cmake'] + self.cmakeargs
config_cmd += ['-H.', self.sourcedir, '-B%s' % self.builddir]
self.debug(' '.join(config_cmd))
comp_process = tools.popen_nobuffer(config_cmd)
output, err = comp_process.communicate()
exit_code = comp_process.returncode
if exit_code != 0:
os.chdir(curdir)
self.cleanup()
self.error(backwards.bytes2unicode(output))
raise RuntimeError("CMake config failed with code %d." % exit_code)
self.debug('Config output: \n%s' % output)
# Build
build_cmd = ['cmake', '--build', self.builddir, '--clean-first']
if self.target is not None:
build_cmd += ['--target', self.target]
self.info(' '.join(build_cmd))
comp_process = tools.popen_nobuffer(build_cmd)
output, err = comp_process.communicate()
exit_code = comp_process.returncode
if exit_code != 0:
os.chdir(curdir)
self.error(backwards.bytes2unicode(output))
self.cleanup()
raise RuntimeError("CMake build failed with code %d." % exit_code)
self.debug('Build output: \n%s' % output)
self.debug('Make complete')
os.chdir(curdir)
def cleanup(self):
r"""Remove compile executable."""
# self.run_cmake('clean')
if not self.preserve_cache:
rmfiles = [self.include_file,
self.target_file,
os.path.join(self.builddir, 'Makefile'),
os.path.join(self.builddir, 'CMakeCache.txt'),
os.path.join(self.builddir, 'cmake_install.cmake'),
os.path.join(self.builddir, 'CMakeFiles')]
for f in rmfiles:
if os.path.isdir(f):
shutil.rmtree(f)
elif os.path.isfile(f):
os.remove(f)
if os.path.isdir(self.builddir) and (not os.listdir(self.builddir)):
os.rmdir(self.builddir)
super(CMakeModelDriver, self).cleanup()
class MakeModelDriver(ModelDriver):
r"""Class for running make file compiled drivers. Before running the
make command, the necessary compiler & linker flags for the interface's
C/C++ library are stored the environment variables CISCCFLAGS and CISLDFLAGS
respectively. These should be used in the make file to correctly compile
with the interface's C/C++ libraries.
Args:
name (str): Driver name.
args (str, list): Executable that should be created (make target) and
any arguments for the executable.
make_command (str, optional): Command that should be used for make.
Defaults to 'make' on Linux/MacOS and 'nmake' on windows.
makefile (str, optional): Path to make file either relative to makedir
or absolute. Defaults to Makefile.
makedir (str, optional): Directory where make should be invoked from
if it is not the same as the directory containing the makefile.
Defaults to directory containing makefile if an absolute path is
provided, otherwise self.working_dir.
**kwargs: Additional keyword arguments are passed to parent class.
Attributes:
compiled (bool): True if the compilation was successful, False otherwise.
target (str): Name of executable that should be created and called.
make_command (str): Command that should be used for make.
makedir (str): Directory where make should be invoked from.
makefile (str): Path to make file either relative to makedir or absolute.
Raises:
RuntimeError: If neither the IPC or ZMQ C libraries are available.
"""
_language = 'make'
_schema = inherit_schema(ModelDriver._schema,
'language',
_language,
make_command={
'type': 'string',
'required': False
},
makefile={
'type': 'string',
'required': False
},
makedir={
'type': 'string',
'required': False
})
def __init__(self,
name,
args,
make_command=None,
makedir=None,
makefile=None,
**kwargs):
super(MakeModelDriver, self).__init__(name, args, **kwargs)
if not self.is_installed(): # pragma: windows
raise RuntimeError(
"No library available for models written in C/C++.")
self.debug('')
self.compiled = False
if make_command is None:
if platform._is_win: # pragma: windows
make_command = 'nmake'
else:
make_command = 'make'
self.target = self.args[0]
if makedir is None:
if (makefile is not None) and os.path.isabs(makefile):
makedir = os.path.dirname(makefile)
else:
makedir = self.working_dir
if makefile is None:
makefile = 'Makefile'
self.make_command = make_command
self.makedir = makedir
self.makefile = makefile
self.target_file = os.path.join(self.makedir, self.target)
self.args[0] = self.target_file
# Set environment variables
self.debug("Setting environment variables.")
compile_flags = ['-DCIS_DEBUG=%d' % self.logger.getEffectiveLevel()]
setup_environ(compile_flags=compile_flags)
# Compile in a new process
self.debug("Making target.")
self.make_target(self.target)
@classmethod
def is_installed(self):
r"""Determine if this model driver is installed on the current
machine.
Returns:
bool: Truth of if this model driver can be run on the current
machine.
"""
return (len(tools.get_installed_comm(language='c')) > 0)
def make_target(self, target):
r"""Run the make command to make the target.
Args:
target (str): Target that should be made.
Raises:
RuntimeError: If there is an error in running the make.
"""
curdir = os.getcwd()
os.chdir(self.makedir)
if self.make_command == 'nmake': # pragma: windows
make_opts = ['/NOLOGO', '/F']
else:
make_opts = ['-f']
make_args = [self.make_command] + make_opts + [self.makefile, target]
self.debug(' '.join(make_args))
if not os.path.isfile(self.makefile):
os.chdir(curdir)
raise IOError("Makefile %s not found" % self.makefile)
comp_process = tools.popen_nobuffer(make_args)
output, err = comp_process.communicate()
exit_code = comp_process.returncode
os.chdir(curdir)
if exit_code != 0:
self.error(output)
raise RuntimeError("Make failed with code %d." % exit_code)
self.debug('Make complete')
def cleanup(self):
r"""Remove compile executable."""
if (self.target_file is not None) and os.path.isfile(self.target_file):
self.make_target('clean')
super(MakeModelDriver, self).cleanup()
class AsciiTableComm(AsciiFileComm):
r"""Class for handling I/O from/to a file on disk.
Args:
name (str): The environment variable where communication address is
stored.
delimiter (str, optional): String that should be used to separate
columns. If not provided and format_str is not set prior to I/O,
this defaults to whitespace.
use_astropy (bool, optional): If True and the astropy package is
installed, it will be used to read/write the table. Defaults to
False.
**kwargs: Additional keywords arguments are passed to parent class.
"""
_filetype = 'table'
_schema = inherit_schema(AsciiFileComm._schema,
'filetype',
_filetype,
delimiter={
'type': 'string',
'required': False
},
use_astropy={
'type': 'boolean',
'required': False
})
def _init_before_open(self,
delimiter=None,
use_astropy=False,
serializer_kwargs=None,
**kwargs):
r"""Set up dataio and attributes."""
if serializer_kwargs is None:
serializer_kwargs = {}
self.header_was_read = False
self.header_was_written = False
serializer_kwargs.update(stype=3, use_astropy=use_astropy)
kwargs['serializer_kwargs'] = serializer_kwargs
super(AsciiTableComm, self)._init_before_open(**kwargs)
if self.serializer.as_array:
self.read_meth = 'read'
else:
self.read_meth = 'readline'
if self.append:
self.header_was_written = True
if delimiter is None:
delimiter = serialize._default_delimiter
self.delimiter = backwards.unicode2bytes(delimiter)
def read_header(self):
r"""Read header lines from the file and update serializer info."""
if self.header_was_read:
return
pos = self.record_position()
self.change_position(0)
serialize.discover_header(self.fd,
self.serializer,
newline=self.newline,
comment=self.comment,
delimiter=self.delimiter)
self.delimiter = self.serializer.table_info['delimiter']
self.change_position(*pos)
self.header_was_read = True
def write_header(self):
r"""Write header lines to the file based on the serializer info."""
if self.header_was_written:
return
header_msg = serialize.format_header(
format_str=self.serializer.format_str,
field_names=self.serializer.field_names,
field_units=self.serializer.field_units,
comment=self.comment,
newline=self.newline,
delimiter=self.delimiter)
self.fd.write(header_msg)
self.header_was_written = True
def record_position(self):
r"""Record the current position in the file/series."""
pos, ind = super(AsciiTableComm, self).record_position()
return pos, ind, self.header_was_read, self.header_was_written
def change_position(self,
file_pos,
series_index=None,
header_was_read=None,
header_was_written=None):
r"""Change the position in the file/series.
Args:
file_pos (int): Position that should be moved to in the file.
series_index (int, optinal): Index of the file in the series that
should be moved to. Defaults to None and will be set to the
current series index.
header_was_read (bool, optional): Status of if header has been
read or not. Defaults to None and will be set to the current
value.
header_was_written (bool, optional): Status of if header has been
written or not. Defaults to None and will be set to the current
value.
"""
if header_was_read is None:
header_was_read = self.header_was_read
if header_was_written is None:
header_was_written = self.header_was_written
super(AsciiTableComm, self).change_position(file_pos, series_index)
self.header_was_read = header_was_read
self.header_was_written = header_was_written
def advance_in_series(self, *args, **kwargs):
r"""Advance to a certain file in a series.
Args:
index (int, optional): Index of file in the series that should be
moved to. Defaults to None and call will advance to the next
file in the series.
Returns:
bool: True if the file was advanced in the series, False otherwise.
"""
out = super(AsciiTableComm, self).advance_in_series(*args, **kwargs)
if out:
self.header_was_read = False
self.header_was_written = False
return out
def _send(self, msg):
r"""Write message to a file.
Args:
msg (bytes, str): Data to write to the file.
Returns:
bool: Success or failure of writing to the file.
"""
if msg != self.eof_msg:
self.write_header()
return super(AsciiTableComm, self)._send(msg)
def _recv(self, timeout=0, **kwargs):
r"""Reads message from a file.
Args:
timeout (float, optional): Time in seconds to wait for a message.
Defaults to self.recv_timeout. Unused.
Returns:
tuple(bool, str): Success or failure of reading from the file.
"""
self.read_header()
return super(AsciiTableComm, self)._recv(timeout=timeout, **kwargs)
class GCCModelDriver(ModelDriver):
r"""Class for running gcc compiled drivers.
Args:
name (str): Driver name.
args (str or list): Argument(s) for running the model on the command
line. If the first element ends with '.c', the driver attempts to
compile the code with the necessary interface include directories.
Additional arguments that start with '-I' are included in the
compile command. Others are assumed to be runtime arguments.
cc (str, optional): C/C++ Compiler that should be used. Defaults to
gcc for '.c' files, and g++ for '.cpp' or '.cc' files on Linux or
MacOS. Defaults to cl on Windows.
**kwargs: Additional keyword arguments are passed to parent class.
Attributes (in additon to parent class's):
compiled (bool): True if the compilation was succesful. False otherwise.
cfile (str): Source file.
cc (str): C/C++ Compiler that should be used.
flags (list): List of compiler flags.
efile (str): Compiled executable file.
Raises:
RuntimeError: If neither the IPC or ZMQ C libraries are available.
RuntimeError: If the compilation fails.
"""
_language = ['c', 'c++', 'cpp']
_schema = inherit_schema(ModelDriver._schema, 'language', _language,
cc={'type': 'string', 'required': False})
def __init__(self, name, args, cc=None, **kwargs):
super(GCCModelDriver, self).__init__(name, args, **kwargs)
if not self.is_installed(): # pragma: windows
raise RuntimeError("No library available for models written in C/C++.")
self.debug('')
self.cc = cc
# Prepare arguments to compile the file
self.parse_arguments(self.args)
self.debug("Compiling")
self.efile = do_compile(self.src, out=self.efile, cc=self.cc,
ccflags=self.ccflags, ldflags=self.ldflags,
working_dir=self.working_dir)
assert(os.path.isfile(self.efile))
self.debug("Compiled %s", self.efile)
if platform._is_win: # pragma: windows
self.args = [os.path.splitext(self.efile)[0]]
else:
self.args = [os.path.join(".", self.efile)]
self.args += self.run_args
self.debug('Compiled executable with %s', self.cc)
@classmethod
def is_installed(self):
r"""Determine if this model driver is installed on the current
machine.
Returns:
bool: Truth of if this model driver can be run on the current
machine.
"""
return (len(tools.get_installed_comm(language='c')) > 0)
def parse_arguments(self, args):
r"""Sort arguments based on their syntax. Arguments ending with '.c' or
'.cpp' are considered source and the first one will be compiled to an
executable. Arguments starting with '-L' or '-l' are treated as linker
flags. Arguments starting with '-' are treated as compiler flags. Any
arguments that do not fall into one of the categories will be treated
as command line arguments for the compiled executable.
Args:
args (list): List of arguments provided.
Raises:
RuntimeError: If there is not a valid source file in the argument
list.
"""
self.src = []
self.ldflags = []
self.ccflags = []
self.ccflags.append('-DCIS_DEBUG=%d' % self.logger.getEffectiveLevel())
self.run_args = []
self.efile = None
is_object = False
is_link = False
for a in args:
if a.endswith('.c') or a.endswith('.cpp') or a.endswith('.cc'):
self.src.append(a)
elif a.lower().startswith('-l') or is_link:
if a.lower().startswith('/out:'): # pragma: windows
self.efile = a[5:]
elif a.lower().startswith('-l') and platform._is_win: # pragma: windows
a1 = '/LIBPATH:"%s"' % a[2:]
if a1 not in self.ldflags:
self.ldflags.append(a1)
elif a not in self.ldflags:
self.ldflags.append(a)
elif a == '-o':
# Next argument should be the name of the executable
is_object = True
elif a.lower() == '/link': # pragma: windows
# Following arguments should be linker options
is_link = True
elif a.startswith('-') or (platform._is_win and a.startswith('/')):
if a not in self.ccflags:
self.ccflags.append(a)
else:
if is_object:
# Previous argument was -o flag
self.efile = a
is_object = False
else:
self.run_args.append(a)
# Check source file
if len(self.src) == 0:
raise RuntimeError("Could not locate a source file in the "
+ "provided arguments.")
def remove_products(self):
r"""Delete products produced during the compilation process."""
if self.efile is None: # pragma: debug
return
products = [self.efile]
if platform._is_win: # pragma: windows
base = os.path.splitext(self.efile)[0]
products = [base + ext for ext in ['.ilk', '.pdb', '.obj']]
for p in products:
if os.path.isfile(p):
os.remove(p)
def cleanup(self):
r"""Remove compile executable."""
self.remove_products()
super(GCCModelDriver, self).cleanup()