def __init__(self, data=None):
super(Data, self).__init__(data)
#: dictionary of data sources added to the Data layer
self.data_sources = {}
#: data source objects
self.data_obj = {}
#: data
self.data = DataRegistry()
def __init__(self, data=None):
super(Data, self).__init__(data)
#: dictionary of data sources added to the Data layer
self.data_sources = {}
#: data source objects
self.data_obj = {}
#: data
self.data = DataRegistry()
class Data(Layer):
"""
The Data layer of the model.
:param data: Dictionary of model data specific to the data layer.
:type data: dict
The :attr:`~Layer.layer` attribute is a dictionary of data sources names
as keys of dictionaries for each data source with the module and optionally
the package containing the module, the filename, which can be ``None``,
containing specific data for the data source and an optional path to the
data file. If the path is ``None``, then the default path for data internal
to Circus is used. External data files should specify the path.
"""
def __init__(self, data=None):
super(Data, self).__init__(data)
#: dictionary of data sources added to the Data layer
self.data_sources = {}
#: data source objects
self.data_obj = {}
#: data
self.data = DataRegistry()
# layers are initialized by the model
def add(self, data_source, module, package=None):
"""
Add data_source to model. Tries to import module, then looks for data
source class definition.
:param data_source: Name of data source to add.
:type data_source: str
:param module: Module in which data source resides. Can be absolute or
relative. See :func:`importlib.import_module`
:type module: str
:param package: Optional, but must be used if module is relative.
:type package: str
.. seealso::
:func:`importlib.import_module`
"""
# TODO: 3 ways to add a data source
# (1) this way
# (2) using class factory, inputs in JSON file
# (3) use DataSource or one of its subclasses
# import module
mod = importlib.import_module(module, package)
# get data source class definition from the module
if data_source.startswith('_'):
err_msg = 'No "%s" attribute in "%s".' % (data_source, mod)
raise AttributeError(err_msg)
self.data_sources[data_source] = getattr(mod, data_source)
# only update layer info if it is missing!
if data_source not in self.layer:
# copy data source parameters to :attr:`Layer.layer`
self.layer[data_source] = {'module': module, 'package': package}
# add a place holder for the data source object when it's constructed
self.data_obj[data_source] = None
def open(self, data_source, filename, path=None):
"""
Open filename to get data for data_source.
:param data_source: Data source for which the file contains data.
:type data_source: str
:param filename: Name of the file which contains data for the data
source.
:type filename: str
:param path: Path of file containting data. [../data]
:type path: str
"""
# only update layer info if it is missing!
if data_source not in self.layer:
# update path and filename to this layer of the model
self.layer[data_source] = {'path': path, 'filename': filename}
# filename can be a list or a string, concatenate list with os.pathsep
# and append the full path to strings.
if isinstance(filename, basestring):
filename = os.path.join(path, filename)
else:
file_list = [os.path.join(path, f) for f in filename]
filename = os.path.pathsep.join(file_list)
# call constructor of data source with filename argument
self.data_obj[data_source] = self.data_sources[data_source](filename)
# register data and uncertainty in registry
data_src_obj = self.data_obj[data_source]
self.data.register(data_src_obj.data, data_src_obj.uncertainty,
data_src_obj.isconstant, data_src_obj.timeseries,
data_src_obj.data_source)
def load(self):
"""
Add data_sources to layer and open files with data for the data_source.
"""
for k, v in self.layer.iteritems():
self.add(k, v['module'], v.get('package'))
if v.get('filename'):
self.open(k, v['filename'], v.get('path'))
def edit(self, data_src, value):
"""
Edit data layer.
:param data_src: Name of :class:`DataSource` to edit.
:type data_scr: str
:param value: Values to edit.
:type value: dict
"""
# check if opening file
if 'filename' in value:
items = [
k for k, v in self.data.data_source.iteritems()
if v == data_src
]
self.data.unregister(items) # remove items from Registry
# open file and register new data
self.open(data_src, value['filename'], value.get('path'))
self.layer[data_src].update(value) # update layer with new items
def delete(self, data_src):
"""
Delete data sources.
"""
items = self.data_obj[data_src].data.keys() # items to edit
self.data.unregister(items) # remove items from Registry
self.layer.pop(data_src) # remove data source from layer
self.data_obj.pop(data_src) # remove data_source object
self.data_sources.pop(data_src) # remove data_source object
class Data(Layer):
"""
The Data layer of the model.
:param data: Dictionary of model data specific to the data layer.
:type data: dict
The :attr:`~Layer.layer` attribute is a dictionary of data sources names
as keys of dictionaries for each data source with the module and optionally
the package containing the module, the filename, which can be ``None``,
containing specific data for the data source and an optional path to the
data file. If the path is ``None``, then the default path for data internal
to Circus is used. External data files should specify the path.
"""
def __init__(self, data=None):
super(Data, self).__init__(data)
#: dictionary of data sources added to the Data layer
self.data_sources = {}
#: data source objects
self.data_obj = {}
#: data
self.data = DataRegistry()
# layers are initialized by the model
def add(self, data_source, module, package=None):
"""
Add data_source to model. Tries to import module, then looks for data
source class definition.
:param data_source: Name of data source to add.
:type data_source: str
:param module: Module in which data source resides. Can be absolute or
relative. See :func:`importlib.import_module`
:type module: str
:param package: Optional, but must be used if module is relative.
:type package: str
.. seealso::
:func:`importlib.import_module`
"""
# TODO: 3 ways to add a data source
# (1) this way
# (2) using class factory, inputs in JSON file
# (3) use DataSource or one of its subclasses
# import module
mod = importlib.import_module(module, package)
# get data source class definition from the module
if data_source.startswith('_'):
err_msg = 'No "%s" attribute in "%s".' % (data_source, mod)
raise AttributeError(err_msg)
self.data_sources[data_source] = getattr(mod, data_source)
# only update layer info if it is missing!
if data_source not in self.layer:
# copy data source parameters to :attr:`Layer.layer`
self.layer[data_source] = {'module': module, 'package': package}
# add a place holder for the data source object when it's constructed
self.data_obj[data_source] = None
def open(self, data_source, filename, path=None):
"""
Open filename to get data for data_source.
:param data_source: Data source for which the file contains data.
:type data_source: str
:param filename: Name of the file which contains data for the data
source.
:type filename: str
:param path: Path of file containting data. [../data]
:type path: str
"""
# default path for data is in ../data
if not path:
path = os.path.join(_DATA, data_source)
# only update layer info if it is missing!
if data_source not in self.layer:
# update path and filename to this layer of the model
self.layer[data_source] = {'path': path, 'filename': filename}
# filename can be a list or a string, concatenate list with os.pathsep
# and append the full path to strings.
if isinstance(filename, basestring):
filename = os.path.join(path, filename)
else:
file_list = [os.path.join(path, f) for f in filename]
filename = os.path.pathsep.join(file_list)
# call constructor of data source with filename argument
self.data_obj[data_source] = self.data_sources[data_source](filename)
# register data and uncertainty in registry
data_src_obj = self.data_obj[data_source]
self.data.register(data_src_obj.data, data_src_obj.uncertainty,
data_src_obj.isconstant, data_src_obj.timeseries,
data_src_obj.data_source)
def load(self):
"""
Add data_sources to layer and open files with data for the data_source.
"""
for k, v in self.layer.iteritems():
self.add(k, v['module'], v.get('package'))
if v.get('filename'):
self.open(k, v['filename'], v.get('path'))
def edit(self, data_src, value):
"""
Edit data layer.
:param data_src: Name of :class:`DataSource` to edit.
:type data_scr: str
:param value: Values to edit.
:type value: dict
"""
# check if opening file
if 'filename' in value:
# items = self.data_obj[data_src].data.keys() # items to edit
items = [k for k, v in self.data.data_source.iteritems() if
v == data_src]
self.data.unregister(items) # remove items from Registry
# open file and register new data
self.open(data_src, value['filename'], value.get('path'))
self.layer[data_src].update(value) # update layer with new items
def delete(self, data_src):
"""
Delete data sources.
"""
items = self.data_obj[data_src].data.keys() # items to edit
self.data.unregister(items) # remove items from Registry
self.layer.pop(data_src) # remove data source from layer
self.data_obj.pop(data_src) # remove data_source object
self.data_sources.pop(data_src) # remove data_source object
