class FileWriter(Actor):
"""
Writes input 'data' to file 'basename' + some counter + '.' + suffix
End of stream token changes file
inputs:
data: data
"""
@manage(['basename', 'counter'])
def init(self, basename, suffix=""):
self.basename = basename
self.suffix = suffix
self.counter = 0
self.setup()
def setup(self):
self.use('calvinsys.io.filehandler', shorthand='file')
fname = new_filename(self.basename, self.counter, self.suffix)
self.counter += 1
self.file = self['file'].open(fname, "w")
def exception_handler(self, action, args, exceptions):
self['file'].close(self.file)
self.file = None
return ActionResult(production=())
@condition(action_input=['data'])
@guard(lambda self, _: not self.file)
def open(self, data):
self.setup()
self.file.write_line(data)
return ActionResult(production=())
@condition(action_input=['data'])
@guard(lambda self, _: self.file)
def write(self, data):
self.file.write_line(data)
return ActionResult(production=())
action_priority = (write, open)
requires = ['calvinsys.io.filehandler']
test_args = [absolute_basename('test_file'), 'testing']
test_data = ['line-1', 'line-2']
test_set = [{
'in': {
'data': [file_data[0], file_data[1],
EOSToken()]
},
'postcond': [verify_file]
}, {
'in': {
'data': [file_data[0], file_data[1],
EOSToken()]
},
'postcond': [verify_file]
}]
def exception_handler(self, action, args):
try:
e = args[0]
except:
e = ExceptionToken()
self.status = e
self.token = EOSToken()
class IsEOS(Actor):
"""
Return 'true' if token is EOS-token
Inputs:
token : any token
Outputs:
status : 'true' if input token is EOS-token, false otherwise
"""
def exception_handler(self, action, args, context):
self.token = type(args[0]) is EOSToken
return ActionResult()
@manage([])
def init(self):
self.token = None
@condition([], ['status'])
@guard(lambda self: self.token is not None)
def produce(self):
tok = self.token
self.token = None
return ActionResult(production=(tok, ))
@condition(['token'])
@guard(lambda self, tok: self.token is None)
def consume(self, tok):
self.token = False
return ActionResult()
action_priority = (produce, consume)
test_set = [
{ # normal token
'in': {
'token': 42
},
'out': {
'status': [False]
}
},
{ # Exception
'in': {
'token': ExceptionToken()
},
'out': {
'status': [False]
}
},
{ # Exception
'in': {
'token': EOSToken()
},
'out': {
'status': [True]
}
},
]
class ToString(Actor):
"""
Transform data to JSON-string
Exception tokens will produce "null" as output unless another value is supplied
through the optional 'exception_output' argument.
Inputs:
data : any kind of token
Outputs:
string : JSON-formatted string
"""
def exception_handler(self, action, args):
return (self.json.tostring(self.default),)
@manage(['default'])
def init(self, exception_output=None):
self.default = exception_output
self.setup()
def did_migrate(self):
self.setup()
def setup(self):
self.json = calvinlib.use("json")
@condition(['data'], ['string'])
def dump(self, value):
return (self.json.tostring(value),)
action_priority = (dump,)
requires = ['json']
test_set = [
{
'inports': {'data': [1]},
'outports': {'string': ['1']},
},
{
'inports': {'data': [{"a": 1}]},
'outports': {'string': ['{"a": 1}']},
},
{
'inports': {'data': [EOSToken()]},
'outports': {'string': ['null']},
},
{
'inports': {'data': [ExceptionToken()]},
'outports': {'string': ['null']},
},
{
'setup': [lambda self: self.init(exception_output={})],
'inports': {'data': [ExceptionToken()]},
'outports': {'string': ['{}']},
},
]
def exception_handler(self, action, args, context):
try:
e = args[context['exceptions']['token'][0]]
except:
e = ExceptionToken()
self.status = e
self.token = EOSToken()
return ActionResult()
class FiniteCounter(Actor):
"""
Produce next token in a sequence start, start+1, ..., start+steps-1, EOSToken
If repeat is True will repeat sequence
Outputs:
integer : Integer
"""
@manage(['count', 'ends', 'restart', 'start', 'replicate_mult', 'stopped'])
def init(self,
start=0,
steps=sys.maxint,
repeat=False,
replicate_mult=False,
stopped=False):
self.count = start
self.ends = start + steps
self.restart = start if repeat else self.ends + 1
self.start = start
self.replicate_mult = replicate_mult
self.stopped = stopped
def did_replicate(self, index):
diff = self.start * index if self.replicate_mult else 0
# Offset by diff for each new replica
self.count += diff
self.ends += diff
self.restart += diff
@stateguard(lambda self: not self.stopped and self.count < self.ends)
@condition(action_output=['integer'])
def cnt(self):
#_log.info("FinitCounter (%s, %s, %s) count:%s" % (self._name, self._id, self.outports['integer'].id, self.count))
self.count += 1
return (self.count - 1, )
@stateguard(lambda self: not self.stopped and self.count == self.ends)
@condition(action_output=['integer'])
def the_end(self):
self.count = self.restart
return (EOSToken(), )
action_priority = (cnt, the_end)
def report(self, **kwargs):
self.stopped = kwargs.get("stopped", self.stopped)
return self.count
test_args = []
test_set = [
{
'setup': [lambda self: self.init(steps=3)],
'inports': {},
'outports': {
'integer': [0, 1, 2, EOSToken().value]
}
},
]
class LineJoin(Actor):
"""
Join strings into a text token using delimiter 'delim' (defaults to '\\n')
Consume consecutive strings until an end-of-stream (EOSToken) is received,
which triggers an output of the joined lines. After receiving the EOFToken,
the LineJoin is reset and ready for new lines to join.
Inputs:
line : arbitrary string
Outputs:
text : strings joined by
"""
@manage(['lines', 'delim', 'text'])
def init(self, delim='\n'):
self.delim = delim
self.lines = []
self.text = None
def exception_handler(self, action, args):
# FIXME: Check that action is append and args is EOSToken
# if not, call super's exception_handler
# Similarly, if self.text is not None => raise
self.text = self.delim.join(self.lines)
self.lines = []
@stateguard(lambda self: self.text is not None)
@condition([], ['text'])
def produce(self):
text = self.text
self.text = None
return (text,)
@stateguard(lambda self: self.text is None)
@condition(['line'], [])
def append(self, token):
self.lines.append(token)
action_priority = (produce, append, )
test_kwargs = {'delim': ' '}
test_set = [
{
'inports': {'line': ["One", "lines", "off", "words", EOSToken()]},
'outports': {'text': ["One lines off words"]}
}
]
class WordCount(Actor):
"""
Count occurances of words in a stream of words.
Inputs:
in : a word
Outputs:
out : count for each word
"""
@manage([])
def init(self):
self.word_counts = defaultdict(int)
self.finished = False
def exception_handler(self, action, args, exceptions):
self.finished = True
return ActionResult()
@condition(['in'], [])
def count_word(self, word):
self.word_counts[word] = self.word_counts[word] + 1
return ActionResult()
@condition(action_output=['out'])
@guard(lambda self: self.finished is True)
def output_counts(self):
self.finished = False
return ActionResult(production=(self.word_counts, ))
action_priority = (count_word, output_counts)
test_set = [{
'in': {
'in': ['a', 'b', 'a', EOSToken()]
},
'out': {
'out': [{
'a': 2,
'b': 1
}]
}
}]
class List(Actor):
"""
Create a list.
Consumes 'n' tokens to produce a list, 'n' defaults to 1. If 'n' is zero or negative,
consumes tokens until EOS encountered (variable list length).
Will produce an ExceptionToken if EOS is encountered when n > 0, or if an ExceptionToken is
encountered regardless of value of 'n'.
Inputs:
item: items to append to list
Outputs:
list: a list of consumed items
"""
def exception_handler(self, action, args, context):
exception = args[context['exceptions']['item'][0]]
if self.n or type(exception) is not EOSToken:
self._list = ExceptionToken()
self.done = True
return ActionResult()
@manage(['n', '_list', 'done'])
def init(self, n=1):
self.n = n if n > 0 else 0
self._list = []
self.done = False
@condition(['item'], [])
@guard(lambda self, item: not self.n and not self.done)
def add_item_EOS(self, item):
self._list.append(item)
return ActionResult()
@condition(['item'], [])
@guard(lambda self, item: self.n and not self.done)
def add_item(self, item):
self._list.append(item)
if len(self._list) == self.n:
self.done = True
return ActionResult()
@condition([], ['list'])
@guard(lambda self: self.done)
def produce_list(self):
res = self._list
self.done = False
self._list = []
return ActionResult(production=(res, ))
action_priority = (produce_list, add_item, add_item_EOS)
test_args = []
test_kwargs = {}
test_set = [
{
'in': {'item': [1, 2]},
'out': {'list': [[1], [2]]},
},
{
'setup': [lambda self: self.init(n=2)],
'in': {'item': [1, 2]},
'out': {'list': [[1, 2]]},
},
{
'setup': [lambda self: self.init(n=0)],
'in': {'item': [1, 2, EOSToken()]},
'out': {'list': [[1, 2]]},
},
# Error conditions
{
'setup': [lambda self: self.init(n=2)],
'in': {'item': [1, EOSToken(), 3, 4]},
'out': {'list': ['Exception', [3, 4]]},
},
{
'setup': [lambda self: self.init(n=0)],
'in': {'item': [1, ExceptionToken(), 3, EOSToken()]},
'out': {'list': ['Exception', [3]]},
},
]
def exception_handler(self, action, args, exceptions):
self.token = EOSToken()
self.status = 1
return ActionResult()
class FromString(Actor):
"""
Transform JSON-formatted string to value
Invalid input will produce an Exception token as output unless another value is supplied
through the optional 'exception_output' argument.
N.B. Using 'null' for 'exception_output' will produce an ExceptionToken rather than 'null'.
Inputs:
string : JSON-formatted string
Outputs:
data : data read from input string
"""
def exception_handler(self, action, args):
return (self.default, )
@manage(['exception_output'])
def init(self, exception_output=None):
self.exception_output = exception_output
self.setup()
def did_migrate(self):
self.setup()
def setup(self):
self.default = ExceptionToken(
) if self.exception_output is None else self.exception_output
self.use('calvinsys.native.python-json', shorthand='json')
@condition(['string'], ['data'])
def load(self, string):
try:
res = self['json'].loads(string)
except:
res = self.default
return (res, )
action_priority = (load, )
require = ['calvinsys.native.python-json']
test_set = [
{
'in': {
'string': ['1']
},
'out': {
'data': [1]
},
},
{
'in': {
'string': ['{"a": 1}']
},
'out': {
'data': [{
"a": 1
}]
},
},
{
'in': {
'string': [EOSToken()]
},
'out': {
'data': ['Exception']
},
},
{
'in': {
'string': [None]
},
'out': {
'data': ['Exception']
},
},
{
'setup': [lambda self: self.init(exception_output={})],
'in': {
'string': [None]
},
'out': {
'data': [{}]
},
},
]
def exception_handler(self, action, args, exceptions):
return ActionResult(tokens_consumed=1,
tokens_produced=1,
production=(EOSToken(), ))
def send_eos(self):
self.eos = False
return ActionResult(production=(EOSToken(),))
def eof(self):
self.calvinsys.io.file.close(self.file)
self.file = None
return ActionResult(tokens_produced=1, production=(EOSToken(), ))
class ExceptionHandler(Actor):
"""
Scan tokens for Exceptions.
Any non-exception or EOS is simply passed on. Exceptions other than EOS are replaced
with an EOS token on the ouput 'token' port unless optional 'replace' argument is true,
in which case 'replacement' argument (defaults to null) is produced.
Any exception (including EOS) are produces its reason on the 'status' output port.
Inputs:
token : any token
Outputs:
token : input token or EOS/replacement on exception
status : reason for any exception tokens encountered (including EOS)
"""
def exception_handler(self, action, args):
try:
e = args[0]
except:
e = ExceptionToken()
self.status = e
self.token = EOSToken()
@manage(['status', 'token', 'replace', 'replacement'])
def init(self, replace=False, replacement=None):
self.replace = replace
self.replacement = replacement
self.status = None
self.token = None
@stateguard(lambda self: self.token is not None and self.status)
@condition([], ['token', 'status'])
def produce_with_exception(self):
tok = self.replacement if self.replace else self.token
status = self.status
self.token = None
self.status = None
return (tok, status.value)
@stateguard(lambda self: self.token is not None and not self.status)
@condition([], ['token'])
def produce(self):
tok = self.token
self.token = None
return (tok, )
@stateguard(lambda self: not self.status and self.token is None)
@condition(['token'])
def consume(self, tok):
self.token = tok
self.status = None
action_priority = (produce_with_exception, produce, consume)
test_set = [
{ # normal token
'inports': {
'token': 42
},
'outports': {
'token': [42],
'status': []
}
},
{ # Exception
'inports': {
'token': ExceptionToken()
},
'outports': {
'token': ['End of stream'],
'status': ['Exception']
}
},
{ # Exception
'inports': {
'token': EOSToken()
},
'outports': {
'token': ['End of stream'],
'status': ['End of stream']
}
},
{ # Long list with Exceptions in middle
'setup': [lambda self: self.init(replace=True, replacement="EOS")],
'inports': {
'token': [0, 1, 2,
EOSToken(), 0, 1, 2,
EOSToken(), 0, 1, 2]
},
'outports': {
'token': [0, 1, 2, 'EOS', 0, 1, 2, 'EOS', 0, 1, 2],
'status': ['End of stream', 'End of stream']
}
},
{ # Exception with replace (default)
'setup': [lambda self: self.init(replace=True)],
'inports': {
'token': EOSToken()
},
'outports': {
'token': [None],
'status': ['End of stream']
}
},
{ # Exception with replace
'setup': [lambda self: self.init(replace=True, replacement={})],
'inports': {
'token': EOSToken()
},
'outports': {
'token': [{}],
'status': ['End of stream']
}
},
{ # Exception with replace
'setup': [lambda self: self.init(replace=True, replacement={})],
'inports': {
'token': ExceptionToken()
},
'outports': {
'token': [{}],
'status': ['Exception']
}
},
]
def send_eos(self):
self.eos = False
return (EOSToken(),)
class List_to_FileWriter(Actor):
"""
Writes input 'data' to file 'basename' + some counter + '.' + suffix
End of stream token changes file
inputs:
data: data
"""
def init(self):
self.numbers=[]
@manage(['basename', 'counter', 'suffix'])
def init(self, basename, suffix=""):
self.basename = basename
self.suffix = suffix
self.counter = 0
self.file = None
self.setup()
def setup(self):
self.use('calvinsys.io.filehandler', shorthand='file')
fname = new_filename(self.basename, self.counter, self.suffix)
self.counter += 1
self.file = self['file'].open(fname, "w")
def exception_handler(self, action, args):
self['file'].close(self.file)
self.file = None
@stateguard(lambda self: not self.file)
@condition(action_input=['data'])
def openf(self, data):
self.file.write_line(data.encode('utf-8'))
@stateguard(lambda self: self.file)
@condition(action_input=['data'])
def writef(self, data):
numbers.append(data)
numbers.append("20 sticken")
if (len(numbers) == 20):
self.setup()
self.file.write_line(numbers.encode('utf-8'))
del numbers[:]
def did_migrate(self):
self.file = None
self.setup()
action_priority = (writef, openf)
requires = ['calvinsys.io.filehandler']
test_args = [absolute_basename('test_file'), 'testing']
test_data = ['line-1', 'line-2']
test_set = [
{
'in': {'data': [file_data[0], file_data[1], EOSToken()]},
'postcond': [verify_file]
},
{
'in': {'data': [file_data[0], file_data[1], EOSToken()]},
'postcond': [verify_file]
}
]
def file_not_found(self):
self.file_not_found = False # Only report once
return ActionResult(production=(EOSToken(), self.status_ERR))
class Dict(Actor):
"""
Create a dict
Consume 'n' key/value pairs to produce a dictionary, 'n' defaults to 1.
If 'n' is zero or negative, consume key/value pairs until EOS encountered
on both input ports. If EOS is only encountered on one port, produce an execption.
Inputs:
key: key must be string
value: can be any token
Outputs:
dict: dictionary or Exception
"""
@manage(['n', '_dict', 'done'])
def init(self, n=1):
self.n = n if n > 0 else 0
self._dict = {}
self.done = False
def _bail(self):
self._dict = ExceptionToken()
self.done = True
def exception_handler(self, action, args, context):
if self.n or not ('key' in context['exceptions']
and 'value' in context['exceptions']):
self._bail()
self.done = True
return ActionResult()
@condition(['key', 'value'], [])
@guard(lambda self, key, value: not self.n and not self.done)
def add_entry_EOS(self, key, value):
if isinstance(key, basestring):
self._dict[key] = value
else:
self._bail()
return ActionResult()
@condition(['key', 'value'], [])
@guard(lambda self, key, value: self.n and not self.done)
def add_entry(self, key, value):
if isinstance(key, basestring):
self._dict[key] = value
self.done = bool(len(self._dict) == self.n)
else:
self._bail()
return ActionResult()
@condition([], ['dict'])
@guard(lambda self: self.done)
def produce_dict(self):
res = self._dict
self.done = False
self._dict = {}
return ActionResult(production=(res, ))
action_priority = (produce_dict, add_entry, add_entry_EOS)
test_set = [
{
'in': {
'key': ["a", "b"],
'value': [1, 2]
},
'out': {
'dict': [{
"a": 1
}, {
"b": 2
}]
},
},
{
'setup': [lambda self: self.init(n=2)],
'in': {
'key': ["a", "b"],
'value': [1, 2]
},
'out': {
'dict': [{
"a": 1,
"b": 2
}]
},
},
{
'setup': [lambda self: self.init(n=0)],
'in': {
'key': ["a", "b", EOSToken()],
'value': [1, 2, EOSToken()]
},
'out': {
'dict': [{
"a": 1,
"b": 2
}]
},
},
# Error conditions
{
'setup': [lambda self: self.init(n=0)],
'in': {
'key': ["a", EOSToken()],
'value': [1, 2]
},
'out': {
'dict': ['Exception']
},
},
{
'setup': [lambda self: self.init(n=2)],
'in': {
'key': ["a", 1, "b", "c"],
'value': [10, 20, 30, 40]
},
'out': {
'dict': ['Exception', {
"b": 30,
"c": 40
}]
},
},
]
def eof(self):
self.calvinsys.io.file.close(self.file)
self.file = None
return ActionResult(production=(EOSToken(), self.status_OK))
class List(Actor):
"""
Create a list.
Consumes 'n' tokens to produce a list, 'n' defaults to 1. If 'n' is zero or negative,
consumes tokens until EOS encountered (variable list length).
The optional arguments pre_list and post_list are used to prepend and extend the list before
delivering the final list.
Will produce an ExceptionToken if EOS is encountered when n > 0, or if an ExceptionToken is
encountered regardless of value of 'n'.
Inputs:
item: items to append to list
Outputs:
list: a list of consumed items
"""
def exception_handler(self, action, args):
if self.n or type(args[0]) is not EOSToken:
self._list = ExceptionToken()
self.done = True
@manage(['n', '_list', 'done'])
def init(self, n=1, pre_list=None, post_list=None):
self.n = n if n > 0 else 0
self._list = []
self.pre_list = pre_list
self.post_list = post_list
self.done = False
@stateguard(lambda self: not self.n and not self.done)
@condition(['item'], [])
def add_item_EOS(self, item):
self._list.append(item)
@stateguard(lambda self: self.n and not self.done)
@condition(['item'], [])
def add_item(self, item):
self._list.append(item)
if len(self._list) == self.n:
self.done = True
@stateguard(lambda self: self.done)
@condition([], ['list'])
def produce_list(self):
if isinstance(self._list, list):
res = (self.pre_list if self.pre_list else []) + self._list + (
self.post_list if self.post_list else [])
else:
res = self._list
self.done = False
self._list = []
return (res, )
action_priority = (produce_list, add_item, add_item_EOS)
test_args = []
test_kwargs = {}
test_set = [
{
'in': {
'item': [1, 2]
},
'out': {
'list': [[1], [2]]
},
},
{
'setup': [lambda self: self.init(n=2)],
'in': {
'item': [1, 2]
},
'out': {
'list': [[1, 2]]
},
},
{
'setup': [lambda self: self.init(n=2, pre_list=[5, 7])],
'in': {
'item': [1, 2]
},
'out': {
'list': [[5, 7, 1, 2]]
},
},
{
'setup': [lambda self: self.init(n=2, post_list=[5, 7])],
'in': {
'item': [1, 2]
},
'out': {
'list': [[1, 2, 5, 7]]
},
},
{
'setup':
[lambda self: self.init(n=2, pre_list=[8, 9], post_list=[5, 7])],
'in': {
'item': [1, 2]
},
'out': {
'list': [[8, 9, 1, 2, 5, 7]]
},
},
{
'setup': [lambda self: self.init(n=0)],
'in': {
'item': [1, 2, EOSToken()]
},
'out': {
'list': [[1, 2]]
},
},
# Error conditions
{
'setup': [lambda self: self.init(n=2)],
'in': {
'item': [1, EOSToken(), 3, 4]
},
'out': {
'list': ['Exception', [3, 4]]
},
},
{
'setup': [lambda self: self.init(n=0)],
'in': {
'item': [1, ExceptionToken(), 3,
EOSToken()]
},
'out': {
'list': ['Exception', [3]]
},
},
]
def eof(self):
calvinsys.close(self.file)
self.file = None
self.filelen = 0
self.totalread = 0
return (EOSToken(), )
def eof(self):
self['file'].close(self.file)
self.file = None
return ActionResult(production=(EOSToken(), ))
def exception_handler(self, action, args, exceptions):
return ActionResult(production=(EOSToken(), ))
def exception_handler(self, action, args, exceptions):
self.token = EOSToken()
self.status = 1
return ActionResult(tokens_consumed=1, tokens_produced=0)
def the_end(self):
self.count = self.restart
return (EOSToken(), )
def exception_handler(self, action, args):
return (EOSToken(), )
def eof(self):
self['file'].close(self.file)
self.file = None
return (EOSToken(), )