def test_module_roundrobin():
actor_config = ActorConfig('roundrobin', 100, 1, {}, "")
roundrobin = RoundRobin(actor_config)
roundrobin.pool.queue.inbox.disableFallThrough()
roundrobin.pool.createQueue("one")
roundrobin.pool.queue.one.disableFallThrough()
roundrobin.pool.createQueue("two")
roundrobin.pool.queue.two.disableFallThrough()
roundrobin.start()
event_one = Event("one")
event_two = Event("two")
roundrobin.pool.queue.inbox.put(event_one)
roundrobin.pool.queue.inbox.put(event_two)
assert getter(roundrobin.pool.queue.one).get() in ["one", "two"]
assert getter(roundrobin.pool.queue.two).get() in ["one", "two"]
roundrobin.stop()
def test_module_funnel():
actor_config = ActorConfig('funnel', 100, 1, {}, "")
funnel = Funnel(actor_config)
funnel.pool.queue.outbox.disableFallThrough()
funnel.pool.createQueue("one")
funnel.pool.queue.one.disableFallThrough()
funnel.pool.createQueue("two")
funnel.pool.queue.two.disableFallThrough()
funnel.start()
event_one = Event("one")
event_two = Event("two")
funnel.pool.queue.one.put(event_one)
funnel.pool.queue.two.put(event_two)
assert getter(funnel.pool.queue.outbox).get() in ["one", "two"]
assert getter(funnel.pool.queue.outbox).get() in ["one", "two"]
funnel.stop()
def test_module_jsonvalidate():
actor_config = ActorConfig('jsonvalidate', 100, 1, {}, "")
with open("/tmp/jsonvalidate.jsonschema", "w") as j:
j.write('{"type": "object", "properties": {"one": { "type": "integer"}}}')
jsonvalidate = JSONValidate(actor_config, "/tmp/jsonvalidate.jsonschema")
jsonvalidate.pool.queue.inbox.disableFallThrough()
jsonvalidate.pool.queue.outbox.disableFallThrough()
jsonvalidate.pool.queue.failed.disableFallThrough()
jsonvalidate.start()
valid = Event({"one": 1})
invalid = Event({"one": "one"})
jsonvalidate.pool.queue.inbox.put(valid)
valid_event = getter(jsonvalidate.pool.queue.outbox)
jsonvalidate.pool.queue.inbox.put(invalid)
invalid_event = getter(jsonvalidate.pool.queue.failed)
os.remove("/tmp/jsonvalidate.jsonschema")
assert valid_event.get() == {"one": 1}
assert invalid_event.get() == {"one": "one"}
def test_module_switch_event():
actor_config = ActorConfig('switch', 100, 1, {}, "")
switch = Switch(actor_config, outgoing="one")
switch.pool.queue.inbox.disableFallThrough()
switch.pool.queue.outbox.disableFallThrough()
switch.pool.createQueue("one")
switch.pool.queue.one.disableFallThrough()
switch.pool.createQueue("two")
switch.pool.queue.two.disableFallThrough()
switch.start()
event_one = Event("one")
switch.pool.queue.inbox.put(event_one)
assert getter(switch.pool.queue.one).get() == "one"
event_two = Event("two")
switch_event = Event("two")
switch.pool.queue.switch.put(switch_event)
switch.pool.queue.inbox.put(event_two)
assert getter(switch.pool.queue.two).get() == "two"
switch.stop()
def test_coucdb_filter(db):
config = ActorConfig('couchdbfilter', 100, 1, {}, "")
module = CouchdbFilter(
actor_config=config,
couchdb_url="{}/{}".format(couchdb_url, DB_NAME),
view_path="test/all",
filter_key='._id',
filter_value='if .mode == "test" then "" else ._id end',
op="eq")
data = {"data": "data", "_id": "test_doc", "id": "test_doc"}
module.couchdb.save(data)
module.logging.debug = mock.MagicMock()
module.pool.queue.outbox.disableFallThrough()
module.pool.queue.inbox.disableFallThrough()
module.start()
e = Event(data)
module.pool.queue.inbox.put(e)
sleep(1)
module.logging.debug.assert_called_with("Event from inbox {}".format(e))
data = module.couchdb.get("test_doc")
data["mode"] = "test"
module.couchdb.save(data)
e = Event(data)
module.pool.queue.inbox.put(e)
sleep(1)
module.logging.debug.assert_called_with(
'Skipped test_doc by filter. Op: eq old: test_doc new ')
def test_module_http_dynamic_method():
from wishbone.lookup import Lookup
class GetMethod(Lookup):
def __init__(self):
self.a = cycle(["POST", "PUT"])
def lookup(self):
return next(self.a)
webserver = WebServer()
webserver.start()
actor_config = ActorConfig('httpoutclient', 100, 1, {"method": GetMethod().lookup}, "")
http = HTTPOutClient(actor_config, url="http://localhost:8088/", accept="monkeyballs", method="~~method()")
http.pool.queue.inbox.disableFallThrough()
http.start()
http.pool.queue.inbox.put(Event('{"one": 1}'))
sleep(1)
assert getter(webserver.q)["REQUEST_METHOD"] == "POST"
http.pool.queue.inbox.put(Event('{"one": 1}'))
sleep(1)
assert getter(webserver.q)["REQUEST_METHOD"] == "PUT"
http.stop()
webserver.stop()
def test_module_fanout():
actor_config = ActorConfig('fanout', 100, 1, {}, "")
fanout = Fanout(actor_config)
fanout.pool.queue.inbox.disableFallThrough()
fanout.pool.createQueue("one")
fanout.pool.queue.one.disableFallThrough()
fanout.pool.createQueue("two")
fanout.pool.queue.two.disableFallThrough()
fanout.start()
e = Event(["hello"])
fanout.pool.queue.inbox.put(e)
one = getter(fanout.pool.queue.one)
two = getter(fanout.pool.queue.two)
fanout.stop()
assert one.get() == ["hello"]
assert two.get() == ["hello"]
assert id(one.get()) != id(two.get())
def test_module_decode_service_checkresult():
data = re.sub(' {2,4}', "\t", service_checkresult)
actor_config = ActorConfig('pd', 100, 1, {}, "")
pd = CheckResult(actor_config)
pd.pool.queue.inbox.disableFallThrough()
pd.pool.queue.outbox.disableFallThrough()
pd.start()
e = Event(data)
pd.pool.queue.inbox.put(e)
sleep(1)
assert pd.pool.queue.outbox.size() == 8
one = getter(pd.pool.queue.outbox).get()
assert isinstance(one, Metric)
assert one.name == 'service1.time'
assert one.value == '0.02'
two = getter(pd.pool.queue.outbox).get()
assert isinstance(two, Metric)
assert two.name == 'service1.cat'
assert two.value == '20'
def test_module_copy_default():
a = get_actor({"copy": ["does.not.exist", "@tmp.copy", "default"]})
e = Event({"greeting": "hi"})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert "default" == one.get('@tmp.copy')
def test_module_del_item():
a = get_actor({"del_item": ["fubar", "@data"]})
e = Event(["one", "two", "three", "fubar"])
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert "fubar" not in one.get('@data')
def test_module_replace():
a = get_actor({"replace": ['\d', "X", "@data"]})
e = Event("hello 123 hello")
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('@data') == "hello XXX hello"
def test_module_join():
a = get_actor({"join": ['@data', ",", "@tmp.joined"]})
e = Event(["one", "two", "three"])
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('@tmp.joined') == "one,two,three"
def test_module_modify_set():
a = get_actor({"set": ["hi", "blah"]})
e = Event('hello')
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('blah') == "hi"
def test_module_lowercase():
a = get_actor({"lowercase": ["@data.lower"]})
e = Event({"lower": "HELLO"})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('@data.lower') == "hello"
def test_module_delete():
a = get_actor({"delete": ["@data.two"]})
e = Event({"one": 1, "two": 2})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert "two" not in one.get('@data').keys()
def generateDicts(self):
while self.loop():
d = self.getDict()
event = Event(d)
self.submit(event, self.pool.queue.outbox)
self.key_number = +1
sleep(self.kwargs.interval)
def test_module_copy():
a = get_actor({"copy": ["@data", "@tmp.copy", "n/a"]})
e = Event({"greeting": "hi"})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert "hi" == one.get('@tmp.copy')["greeting"]
assert id(one.get("@data")) != id(one.get("@tmp.copy"))
def timer(self):
while self.loop():
if self.cron.check_trigger(time.localtime(time.time())[:5]):
self.logging.info("Cron fired.")
e = Event()
e.set(self.kwargs.payload, self.kwargs.field)
self.submit(e, self.pool.queue.outbox)
sleep(60)
def test_module_template():
a = get_actor(
{"template": ["result", "Good day in {language} is {word}.", "@data"]})
e = Event({"language": "German", "word": "gutten Tag"})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('result') == "Good day in German is gutten Tag."
def test_fileread(module):
module.start()
module.pool.queue.inbox.put(Event(DATA))
#gevent.sleep(10)
#import pdb; pdb.set_trace()
with open(os.path.join(here, 'results', '1111.json')) as df:
data = json.load(df)
assert data == DATA
def test_module_merge():
a = get_actor({"merge": ['@tmp.one', '@tmp.two', '@data']})
e = Event()
e.set(["one"], "@tmp.one")
e.set(["two"], "@tmp.two")
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get() == ["one", "two"]
def test_fileread(module):
module.start()
path = os.path.join(here, 'data.xml')
e = Event(path)
module.pool.queue.inbox.put(e)
doc = getter(module.pool.queue.outbox).get()
if not doc:
assert False
assert doc == XML
def test_module_acknowledge_dropped():
# An event tries to pass through with an unacknowledged ack_id and
# therefor should be dropped.
actor_config = ActorConfig('acknowledge', 100, 1, {}, "")
acknowledge = Acknowledge(actor_config)
acknowledge.pool.queue.inbox.disableFallThrough()
acknowledge.pool.queue.outbox.disableFallThrough()
acknowledge.pool.queue.acknowledge.disableFallThrough()
acknowledge.pool.queue.dropped.disableFallThrough()
acknowledge.start()
acknowledge.pool.queue.inbox.put(Event("one"))
acknowledge.pool.queue.inbox.put(Event("one"))
assert getter(acknowledge.pool.queue.dropped).get() == "one"
acknowledge.stop()
def consume(self, event):
data = event.get(self.kwargs.source)
if isinstance(data, dict) or isinstance(data, list):
for name, value in self.recurseData(data):
metric = Metric(time(), self.kwargs.type, self.kwargs.metric_source, name, value, "", self.kwargs.tags)
self.submit(Event(metric), self.pool.queue.outbox)
else:
raise Exception("Dropped incoming data because not of type <dict>. Perhaps you forgot to feed the data through wishbone.decode.json first?")
def produce(self):
while self.loop():
for feed in self.couchdb.changes(
feed="continuous", since=self.since, include_docs=True
):
self.since = feed.get('seq', feed.get('last_seq', "now"))
self.logging.debug("Change event {}".format(feed))
if feed and 'doc' in feed:
self.submit(Event(feed['doc']), "outbox")
self.logging.info("Stopping changes feed from couchdb")
def test_module_extract():
a = get_actor({
"extract":
["destination", "(?P<one>.*?)\ (?P<two>.*)\ (?P<three>.*)", "@data"]
})
e = Event("een twee drie")
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('destination.one') == "een"
def test_xml2dict(module):
module.start()
e = Event(XML)
module.pool.queue.inbox.put(e)
doc = getter(module.pool.queue.outbox).get()
if not doc:
assert False
assert doc['mydocument']['@has'] == 'an attribute'
assert doc['mydocument']['and']['many'] == ['elements', 'more elements']
assert doc['mydocument']['plus']['@a'] == 'complex'
assert doc['mydocument']['plus']['#text'] == 'element as well'
def produce(self):
while self.loop():
message = self.generateMessage(self.kwargs.message)
event = Event(message)
for key, value in list(self.kwargs.additional_values.items()):
event.set(value, key)
self.submit(event, self.pool.queue.outbox)
self.sleep()
self.logging.info("Stopped producing events.")
def __log(self, level, message):
event = Event(Log(time(), level, getpid(), self.name, message))
try:
self.logs.put(event)
except QueueFull:
if not self.__queue_full_message:
print("Log queue full for module '%s'. Dropping messages" %
(self.name))
else:
self.__queue_full_message = True
def produce(self):
while self.loop():
for feed in self.couchdb.changes(feed="continuous",
since=self.since):
self.since = feed.get('seq', feed.get('last_seq', "now"))
self.logging.debug("Change event {}".format(feed))
if 'id' in feed:
doc = self._get_doc(feed['id'])
e = Event(doc)
self.submit(e, "outbox")
sleep(0)
self.logging.info("Stopping changes feed from couchdb")
