class ModelAggregator(Module):
"""Combine 1+ models into an aggregate model
"""
_settings = ModuleSettings(namespace='fitting')
_input_ports = [
IPort(name='models',
label='models to aggregate',
signature='basic:Variant')
]
_output_ports = [
OPort(name='aggregated_models', signature='basic:Variant')
]
def compute(self):
"""Mandatory override of parent `Module` class.
Loop over however many models are connected to the 'models' input port
and combine them into a single aggregate model
"""
models = self.get_input_list('models')
aggregated = models[0]
if len(models) > 2:
for model in models[1:]:
aggregated += model
self.set_output('aggregated_models', aggregated)
class FindData(Module):
_settings = ModuleSettings(namespace="io")
_input_ports = [
IPort(name="file name",
label="file name and extension to search for",
signature="basic:String"),
IPort(name="seed path",
label="path corresponding to the "
"search starting point. Defaults to "
"the user's home directory.",
default="~",
signature="basic:String"),
]
_output_ports = [OPort(name="file path", signature="basic:String")]
def compute(self):
seed_path = self.get_input("seed path")
file_name = self.get_input("file name")
print file_name
existing_files = []
for dir_tree in os.walk(os.path.expanduser(seed_path)):
if file_name in dir_tree[2]:
existing_files.append(os.path.join(dir_tree[0], file_name))
print existing_files
for path in existing_files:
if 'Demos' in path:
file_path = path
self.set_output("file path", file_path)
class NestedDictCell(SpreadsheetCell):
_settings = ModuleSettings(namespace="vis")
_input_ports = [
IPort(name="dict_list", label="Dictionary to display",
signature="basic:List"),
]
def compute(self):
dict_list = self.get_input("dict_list")
self.cellWidget = self.displayAndWait(NestedDictWidget, (dict_list,))
def gen_module_ufunc(input_ports,
output_ports,
docstring,
module_name,
library_func,
module_namespace,
dict_port=None):
if dict_port is not None:
raise NotImplementedError("Dict_port is not supported for ufuncs")
# can't unpack dicts into ufuncs, assume all are
# mandatory
input_ports = [IPort(**pdict) for pdict in input_ports]
output_ports = [OPort(**pdict) for pdict in output_ports]
mandatory = input_ports
arg_names = [m.name for m in mandatory]
if len(mandatory) != library_func.nin:
raise ValueError("wrap {} : \n".format(library_func.__name__) +
"the docstring parsing went wrong " +
"ufunc should have {} args".format(library_func.nin) +
" parsing docstring has {}".format(len(mandatory)))
if len(output_ports) != library_func.nout:
raise ValueError("wrap {} : \n".format(library_func.__name__) +
"the docstring parsing went wrong" +
"ufunc should have {} out".format(library_func.nout) +
" parsing docstring has {}".format(len(output_ports)))
def compute(self):
args = list()
for arg_name in arg_names:
args.append(self.get_input(arg_name))
ret = library_func(*args)
if len(output_ports) == 1:
self.set_output(output_ports[0].name, ret)
else:
for (out_port, ret_val) in zip(output_ports, ret):
self.set_output(out_port.name, ret_val)
_settings = ModuleSettings(namespace=module_namespace)
new_class = type(
str(module_name), (Module, ), {
'compute': compute,
'__module__': __name__,
'_settings': _settings,
'__doc__': docstring,
'__name__': module_name,
'_input_ports': input_ports,
'_output_ports': output_ports
})
return new_class
class ReadNumpy(Module):
_settings = ModuleSettings(namespace="io")
_input_ports = [
IPort(name="file", label="File to read in", signature="basic:List")
]
_output_ports = [OPort(name="data", signature="basic:List")]
def compute(self):
fnames = self.get_input('file')
data = []
data = [np.load(fname + '.npy') for fname in fnames]
self.set_output('data', data)
class ReadTiff(Module):
_settings = ModuleSettings(namespace="io")
_input_ports = [
IPort(name="files", label="List of files", signature="basic:List"),
]
_output_ports = [OPort(name="data", signature="basic:List")]
def compute(self):
files_list = self.get_input("files")
data_list = []
for file in files_list:
data_list.append(imread(file))
self.set_output("data", data_list)
class CalibrationParameters(Module):
_settings = ModuleSettings(namespace="broker")
_input_ports = [
IPort(name="run_header",
label="Run header from the data broker",
signature="basic:Dictionary"),
]
_output_ports = [
OPort(name='calib_dict', signature='basic:Dictionary'),
OPort(name='nested', signature='basic:Boolean'),
]
def compute(self):
header = self.get_input('run_header')
calib_dict, nested = get_calib_dict(header)
self.set_output('calib_dict', calib_dict)
self.set_output('nested', nested)
class Stack1DCell(SpreadsheetCell):
_settings = ModuleSettings(namespace="vis")
_input_ports = [
IPort(name="data", label="Data to display",signature="basic:List"),
IPort(name="keys", label="Names of the data",signature="basic:List"),
]
_output_ports = [
OPort(name="displayed_data", signature="basic:List"),
]
def compute(self):
data = self.get_input("data")
try:
keys = self.get_input("keys")
except ModuleError:
keys = range(len(data))
self.cellWidget = self.displayAndWait(Stack1DWidget, (data,keys,))
class SwapAxes(Module):
_settings = ModuleSettings(namespace="utility")
_input_ports = [
IPort(name='arr', label='N-D array', signature='basic:List'),
IPort(name='ax0', label='Axis to swap from',
signature='basic:Integer'),
IPort(name='ax1', label='Axis to swap from',
signature='basic:Integer'),
]
_output_ports = [OPort(name='out', signature='basic:List')]
def compute(self):
arr = self.get_input('arr')
ax0 = self.get_input('ax0')
ax1 = self.get_input('ax1')
arr = np.asarray(arr)
self.set_output('out', np.swapaxes(arr, ax0, ax1))
class Flatten(Module):
_settings = ModuleSettings(namespace="utility")
_input_ports = [
IPort(name="list_of_lists",
label="List of lists to flatten",
signature="basic:List"),
]
_output_ports = [
OPort(name="flattened", signature="basic:List"),
]
def compute(self):
# gather input
lists = self.get_input('list_of_lists')
raveled = [np.ravel(im) for im in lists]
flattened = [item for sublist in raveled for item in sublist]
self.set_output('flattened', flattened)
class BrokerQuery(Module):
_settings = ModuleSettings(namespace="broker")
_input_ports = [
IPort(name="unique_query_dict",
label="guaranteed unique query for the data broker",
signature="basic:Dictionary"),
IPort(name="query_dict",
label="Query for the data broker",
signature="basic:Dictionary"),
IPort(name="is_returning_data",
label="Return data with search results",
signature="basic:Boolean",
default=True)
]
_output_ports = [
OPort(name="query_result", signature="basic:Dictionary"),
]
def compute(self):
query = None
if self.has_input("query_dict"):
query = self.get_input("query_dict")
return_only_one = False
if self.has_input("unique_query_dict"):
query = self.get_input("unique_query_dict")
return_only_one = True
if query is None:
logger.debug("no search dictionary was passed in, search "
"cannot proceed")
return
logger.debug("broker_query: {0}".format(query))
data = self.get_input("is_returning_data")
query["data"] = data
result = search(**query)
if return_only_one:
keys = list(result)
result = result[keys[0]]
self.set_output("query_result", result)
logger.debug("result: {0}".format(list(result)))
class Average(Module):
_settings = ModuleSettings(namespace="utility")
_input_ports = [
IPort(name="input",
label="Iterable to compute the average of",
signature="basic:Variant"),
]
_output_ports = [
OPort(name="avg", signature="basic:Float"),
OPort(name="avg_str", signature="basic:String"),
]
def compute(self):
# gather input
input = self.get_input('input')
# np.average(input)
avg = np.average(input)
self.set_output('avg', avg)
self.set_output('avg_str', str(avg))
class DataGen(Module):
_settings = ModuleSettings(namespace="vis|test")
_input_ports = [
IPort(name="num_datasets",
label="Number of datasets to generate",
signature="basic:Integer"),
]
_output_ports = [
OPort(name="OneDimStack", signature="basic:List"),
OPort(name="TwoDimStack", signature="basic:List"),
OPort(name="DataLabels", signature="basic:List"),
]
def compute(self):
length = self.get_input("num_datasets")
self.set_output("OneDimStack", self.make_onedim(length))
self.set_output("TwoDimStack", self.make_twodim(length))
self.set_output("DataLabels", self.make_labels(length))
def make_onedim(self, length):
"""
Construct a one dimensional stack of height 'length'
Parameters
----------
length : int
number of datasets to generate
Returns
-------
data : list
list of 2d np.ndarray
"""
x_axis = np.arange(0, 25, .01)
data = []
for idx in range(length):
x = x_axis
y = np.sin(x_axis + idx)
data.append((x, y))
return data
def make_twodim(self, length):
"""
Construct a two dimensional stack of height 'length'
Parameters
----------
length : int
number of datasets to generate
Returns
-------
data : list
list of 2d np.ndarray
"""
x, y = [_ * 2 * np.pi / 500 for _ in np.ogrid[-500:500, -500:500]]
rep = int(np.sqrt(length))
data = []
for idx in range(length):
kx = idx // rep + 1
ky = idx % rep
data.append(np.sin(kx * x) * np.cos(ky * y) + 1.05)
return data
def make_labels(self, length):
return [str(_) for _ in range(length)]
def gen_module(input_ports, output_ports, docstring,
module_name, library_func, module_namespace,
dict_port=None):
mandatory = []
optional = []
# create the lists of mandatory and optional input ports
for port in input_ports:
if port == dict_port:
# since dict port must be in the input_ports list but we dont want
# to treat it as a normal input port, do not assign it as
# optional or mandatory
continue
if port.optional:
optional.append(port.name)
else:
mandatory.append(port.name)
def compute(self):
dict_from_port = {}
params_dict = {}
if dict_port is not None:
dict_from_port = self.get_input(dict_port.name)
for opt in optional:
if opt in dict_from_port:
# obtain the parameter from the passed in dict
params_dict[opt] = dict_from_port[opt]
if self.has_input(opt):
params_dict[opt] = self.get_input(opt)
for mand in mandatory:
if mand in dict_from_port:
params_dict[mand] = dict_from_port[mand]
try:
params_dict[mand] = self.get_input(mand)
except ModuleError as me:
if mand in params_dict:
# pass on this exception, as the dictionary on dict_port
# has taken care of this key
pass
else:
logger.debug('The mandatory port {0} does not have input'
'and the input dictionary is either not '
'present or doesn\'t contain this key'
''.format(mand))
raise ModuleError(__name__, me)
# check for the presence of a 'value' attribute on the incoming
# port values. This indicates that this is a NSLS2 port type
for name, val in six.iteritems(params_dict):
print('name [{0}] has attribute value [{1}]'.format(name, val))
if hasattr(val, 'value'):
print('name [{0}] has attribute value [{1}]'.format(name, val))
params_dict[name] = val.value
ret = library_func(**params_dict)
if len(output_ports) == 1:
self.set_output(output_ports[0].name, ret)
else:
for (out_port, ret_val) in zip(output_ports, ret):
self.set_output(out_port.name, ret_val)
_settings = ModuleSettings(namespace=module_namespace)
new_class = type(str(module_name),
(Module,), {'compute': compute,
'__module__': __name__,
'_settings': _settings,
'__doc__': docstring,
'__name__': module_name,
'_input_ports': input_ports,
'_output_ports': output_ports})
return new_class
def gen_module(input_ports,
output_ports,
docstring,
module_name,
library_func,
module_namespace,
dict_port=None):
"""
Parameters
----------
input_ports : list
List of input ports
output_ports : list
List of output ports
docstring : ?
module_name : str
The name of the module (as displayed in vistrails
library_func : callable
The callable object to be wrapped for VisTrails
module_namespace : str
Vistrails namespace to use
dict_port : ?
"""
# convert input/output specs into VT port objects
input_ports = [IPort(**pdict) for pdict in input_ports]
output_ports = [OPort(**pdict) for pdict in output_ports]
mandatory = []
optional = []
# create the lists of mandatory and optional input ports
for port in input_ports:
if port == dict_port:
# since dict port must be in the input_ports list but we dont want
# to treat it as a normal input port, do not assign it as
# optional or mandatory
continue
if port.optional:
optional.append(port.name)
else:
mandatory.append(port.name)
def compute(self):
dict_from_port = {}
params_dict = {}
if dict_port is not None:
dict_from_port = self.get_input(dict_port.name)
for opt in optional:
if opt in dict_from_port:
# obtain the parameter from the passed in dict
params_dict[opt] = dict_from_port[opt]
if self.has_input(opt):
if opt in vt_reserved:
p_name = '_' + opt
else:
p_name = opt
params_dict[opt] = self.get_input(p_name)
for mand in mandatory:
if mand in vt_reserved:
p_name = '_' + mand
else:
p_name = mand
if mand in dict_from_port:
params_dict[mand] = dict_from_port[mand]
try:
params_dict[mand] = self.get_input(p_name)
except ModuleError as me:
if mand in params_dict:
# pass on this exception, as the dictionary on dict_port
# has taken care of this key
pass
else:
logger.debug('The mandatory port {0} does not have input'
'and the input dictionary is either not '
'present or doesn\'t contain this key'
''.format(mand))
raise ModuleError(__name__, me)
# check for the presence of a 'value' attribute on the incoming
# port values. This indicates that this is a NSLS2 port type
for name, val in six.iteritems(params_dict):
if hasattr(val, 'value'):
params_dict[name] = val.value
ret = library_func(**params_dict)
if len(output_ports) == 1:
self.set_output(output_ports[0].name, ret)
else:
for (out_port, ret_val) in zip(output_ports, ret):
self.set_output(out_port.name, ret_val)
_settings = ModuleSettings(namespace=module_namespace)
new_class = type(
str(module_name), (Module, ), {
'compute': compute,
'__module__': __name__,
'_settings': _settings,
'__doc__': docstring,
'__name__': module_name,
'_input_ports': input_ports,
'_output_ports': output_ports
})
return new_class
class Listify(Module):
src = "Cannot display wrapped source code as it was not found."
try:
src = obj_src(listify)
doc = ('source for wrapped function '
'metadataStore.utilities.commands:listify:\n\n' +
six.text_type(src.__str__()))
except IOError as ie:
msg = 'original source cannot be found for Listify'
logger.debug(msg)
print(msg)
src = docstring_func(listify)
doc = ('docstring for wrapped function '
'metadataStore.utilities.commands:listify:\n\n' +
six.text_type(src.__str__()))
__doc__ = ('Transpose the run header events into data lists\n\n' + doc)
_settings = ModuleSettings(namespace="broker")
_input_ports = [
IPort(name="run_header",
label="Run header from the data broker",
signature="basic:Dictionary"),
IPort(name="data_key",
label="The data key to turn in to a list",
signature="basic:String"),
]
_output_ports = [
OPort(name="listified_data", signature="basic:List"),
OPort(name="listified_time", signature="basic:List"),
]
def compute(self):
# gather input
header = self.get_input("run_header")
key = None
if self.has_input("data_key"):
key = self.get_input("data_key")
# print('key input: {0}'.format(key))
data_dict = listify(data_keys=key, run_header=header)
# print('data_dict: {0}'.format(data_dict))
# remove time from the dictionary
time = data_dict.pop('time')
# stringify the datetime object that gets returned
time = [t.isoformat() for t in time]
# get the remaining keys
key = list(data_dict)
# verify that there is only one key in the dictionary
if len(key) != 1:
raise ValueError("The return value from "
"metadataStore.utilities.utility.listify had "
"more than one key. Keys: {0}".format(key))
key = key[0]
data = data_dict[key]
# check to see if data is a list of lists
if len(data) == 1 and isinstance(data[0], list):
data = data[0]
# check to see if the data name exists in the sig_map dict. If so, this
# means that the data elements should be formatted into VisTrails
# objects
if key in sig_map:
sig = sig_map[key].split(':')
mod = self.registry.get_module_by_name(
sig[0], sig[1])
data = [mod(d) for d in data]
# log the values set to the output ports at a debug level
logger.debug('data ', data)
logger.debug('time ', time)
# set the module's output
self.set_output("listified_data", data)
self.set_output("listified_time", time)
class Crop2D(Module):
"""Cropping Module
Create a binary mask for an image based on two points
+---------------------+
| |
| p1-> +----+ |
| | | |
| +----+ <- p2 |
| |
+---------------------+
p1 = (row, col)
p2 = (row, col)
Input ports are p1r, p1c, p2r, p2c. Any combination of input ports are
valid. If either of the p1 ports are not present their value will set to
zero. If either of the p2 ports are not present, their value will be set
to the number of rows or columns, respectively.
"""
_settings = ModuleSettings(namespace="utility")
_input_ports = [
IPort(name='num_rows',
label='Number of rows in the image',
signature='basic:Integer'),
IPort(name='num_cols',
label='Number of columns in the image',
signature='basic:Integer'),
IPort(name='top_left_row',
label='pixel coordinate of the row of the top left corner',
signature='basic:Integer'),
IPort(name='top_left_column',
label='pixel coordinate of the column of the top left corner',
signature='basic:Integer'),
IPort(name='bottom_right_row',
label='pixel coordinate of the row of the bottom right corner',
signature='basic:Integer'),
IPort(
name='bottom_right_column',
label='pixel coordinate of the column of the bottom right corner',
signature='basic:Integer'),
]
_output_ports = [OPort(name='bin_mask', signature='basic:Variant')]
def compute(self):
rows = self.get_input('num_rows')
cols = self.get_input('num_rows')
im = np.zeros((rows, cols), 'Bool')
p1c = 0
p1r = 0
p2c = cols
p2r = rows
if self.has_input('top_left_column'):
p1c = self.get_input('top_left_column')
if self.has_input('top_left_row'):
p1r = self.get_input('top_left_row')
if self.has_input('bottom_right_column'):
p2c = self.get_input('bottom_right_column')
if self.has_input('bottom_right_row'):
p2r = self.get_input('bottom_right_row')
im[p1r:p2r, p1c:p2c] = True
self.set_output('bin_mask', im)
