def test_module_fanout():
actor_config = ActorConfig('fanout', 100, 1, {})
fanout = Fanout(actor_config, deep_copy=True)
fanout.pool.queue.inbox.disableFallThrough()
fanout.pool.createQueue("one")
fanout.pool.queue.one.disableFallThrough()
fanout.pool.createQueue("two")
fanout.pool.queue.two.disableFallThrough()
fanout.start()
e = Event('test')
e.setData("hello")
fanout.pool.queue.inbox.put(e)
one = getter(fanout.pool.queue.one)
two = getter(fanout.pool.queue.two)
fanout.stop()
assert one.raw()["test"]["data"] == "hello"
assert two.raw()["test"]["data"] == "hello"
assert id(one) != id(two)
def test_module_http_dynamic_method():
from wishbone.lookup import Lookup
class GetMethod(Lookup):
def __init__(self):
self.a = cycle(["POST", "PUT"])
def lookup(self):
return next(self.a)
webserver = WebServer()
webserver.start()
actor_config = ActorConfig('httpoutclient', 100, 1, {"method": GetMethod().lookup}, "")
http = HTTPOutClient(actor_config, url="http://localhost:8088/", accept="monkeyballs", method="~~method()")
http.pool.queue.inbox.disableFallThrough()
http.start()
http.pool.queue.inbox.put(Event('{"one": 1}'))
sleep(1)
assert getter(webserver.q)["REQUEST_METHOD"] == "POST"
http.pool.queue.inbox.put(Event('{"one": 1}'))
sleep(1)
assert getter(webserver.q)["REQUEST_METHOD"] == "PUT"
http.stop()
webserver.stop()
def test_module_funnel():
actor_config = ActorConfig('funnel', 100, 1, {}, "")
funnel = Funnel(actor_config)
funnel.pool.queue.outbox.disableFallThrough()
funnel.pool.createQueue("one")
funnel.pool.queue.one.disableFallThrough()
funnel.pool.createQueue("two")
funnel.pool.queue.two.disableFallThrough()
funnel.start()
event_one = Event("one")
event_two = Event("two")
funnel.pool.queue.one.put(event_one)
funnel.pool.queue.two.put(event_two)
assert getter(funnel.pool.queue.outbox).get() in ["one", "two"]
assert getter(funnel.pool.queue.outbox).get() in ["one", "two"]
funnel.stop()
def test_coucdb_filter(db):
config = ActorConfig('couchdbfilter', 100, 1, {}, "")
module = CouchdbFilter(
actor_config=config,
couchdb_url="{}/{}".format(couchdb_url, DB_NAME),
view_path="test/all",
filter_key='._id',
filter_value='if .mode == "test" then "" else ._id end',
op="eq")
data = {"data": "data", "_id": "test_doc", "id": "test_doc"}
module.couchdb.save(data)
module.logging.debug = mock.MagicMock()
module.pool.queue.outbox.disableFallThrough()
module.pool.queue.inbox.disableFallThrough()
module.start()
e = Event(data)
module.pool.queue.inbox.put(e)
sleep(1)
module.logging.debug.assert_called_with("Event from inbox {}".format(e))
data = module.couchdb.get("test_doc")
data["mode"] = "test"
module.couchdb.save(data)
e = Event(data)
module.pool.queue.inbox.put(e)
sleep(1)
module.logging.debug.assert_called_with(
'Skipped test_doc by filter. Op: eq old: test_doc new ')
def test_event_appendBulk():
e = Event(bulk=True)
ee = Event({"one": 1})
e.appendBulk(ee)
assert e.dump()["data"][0]["uuid"] == ee.data["uuid"]
def test_module_roundrobin():
actor_config = ActorConfig('roundrobin', 100, 1, {}, "")
roundrobin = RoundRobin(actor_config)
roundrobin.pool.queue.inbox.disableFallThrough()
roundrobin.pool.createQueue("one")
roundrobin.pool.queue.one.disableFallThrough()
roundrobin.pool.createQueue("two")
roundrobin.pool.queue.two.disableFallThrough()
roundrobin.start()
event_one = Event("one")
event_two = Event("two")
roundrobin.pool.queue.inbox.put(event_one)
roundrobin.pool.queue.inbox.put(event_two)
assert getter(roundrobin.pool.queue.one).get() in ["one", "two"]
assert getter(roundrobin.pool.queue.two).get() in ["one", "two"]
roundrobin.stop()
def test_module_unpack():
actor_config = ActorConfig('unpack', 100, 1, {}, "")
unpack = Unpack(actor_config)
unpack.pool.queue.inbox.disableFallThrough()
unpack.pool.queue.outbox.disableFallThrough()
unpack.start()
bulk = Event(bulk=True)
for _ in range(0, 10):
bulk.appendBulk(Event())
unpack.pool.queue.inbox.put(bulk)
for _ in range(0, 10):
assert getter(unpack.pool.queue.outbox)
try:
getter(unpack.pool.queue.outbox)
except Exception:
assert True
else:
assert False
def test_module_switch_event():
actor_config = ActorConfig('switch', 100, 1, {}, "")
switch = Switch(actor_config, outgoing="one")
switch.pool.queue.inbox.disableFallThrough()
switch.pool.queue.outbox.disableFallThrough()
switch.pool.createQueue("one")
switch.pool.queue.one.disableFallThrough()
switch.pool.createQueue("two")
switch.pool.queue.two.disableFallThrough()
switch.start()
event_one = Event("one")
switch.pool.queue.inbox.put(event_one)
assert getter(switch.pool.queue.one).get() == "one"
event_two = Event("two")
switch_event = Event("two")
switch.pool.queue.switch.put(switch_event)
switch.pool.queue.inbox.put(event_two)
assert getter(switch.pool.queue.two).get() == "two"
switch.stop()
def test_module_jsonvalidate():
actor_config = ActorConfig('jsonvalidate', 100, 1, {}, "")
with open("/tmp/jsonvalidate.jsonschema", "w") as j:
j.write('{"type": "object", "properties": {"one": { "type": "integer"}}}')
jsonvalidate = JSONValidate(actor_config, "/tmp/jsonvalidate.jsonschema")
jsonvalidate.pool.queue.inbox.disableFallThrough()
jsonvalidate.pool.queue.outbox.disableFallThrough()
jsonvalidate.pool.queue.failed.disableFallThrough()
jsonvalidate.start()
valid = Event({"one": 1})
invalid = Event({"one": "one"})
jsonvalidate.pool.queue.inbox.put(valid)
valid_event = getter(jsonvalidate.pool.queue.outbox)
jsonvalidate.pool.queue.inbox.put(invalid)
invalid_event = getter(jsonvalidate.pool.queue.failed)
os.remove("/tmp/jsonvalidate.jsonschema")
assert valid_event.get() == {"one": 1}
assert invalid_event.get() == {"one": "one"}
def timer(self):
while self.loop():
if self.cron.check_trigger(time.localtime(time.time())[:5]):
self.logging.info("Cron fired.")
e = Event()
e.set(self.kwargs.payload, self.kwargs.field)
self.submit(e, self.pool.queue.outbox)
sleep(60)
def timer(self):
while self.loop():
if self.cron.check_trigger(time.localtime(time.time())[:5]):
self.logging.info("Cron fired.")
e = Event()
e.set(self.kwargs.payload, self.kwargs.field)
self.submit(e, self.pool.queue.outbox)
sleep(60)
def test_event_clone():
a = Event({"one": 1, "two": 2})
b = a.clone()
assert id(a.data) != id(b.data)
assert not a.data["cloned"]
assert b.data["cloned"]
assert b.data["uuid_previous"][0] == a.data["uuid"]
def test_event_get_error():
e = Event({"one": 1, "two": {"three": 3}})
try:
e.get("data.blah")
except KeyError:
assert True
else:
assert False
def test_module_merge():
a = get_actor({"merge": ['@tmp.one', '@tmp.two', '@data']})
e = Event()
e.set(["one"], "@tmp.one")
e.set(["two"], "@tmp.two")
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get() == ["one", "two"]
def test_module_merge():
a = get_actor({"merge": ['@tmp.one', '@tmp.two', '@data']})
e = Event()
e.set(["one"], "@tmp.one")
e.set(["two"], "@tmp.two")
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get() == ["one", "two"]
def test_merge_fail():
e = Event("hi")
try:
e.merge(["two"])
except InvalidData:
assert True
else:
assert False
def test_event_render_error():
e = Event({"one": 1, "two": 2})
try:
e.render("{{data.one} is a number and so is {{data.two}}")
except InvalidData:
assert True
else:
assert False
def produce(self):
while self.loop():
message = self.generateMessage(self.kwargs.message)
event = Event(message)
for key, value in list(self.kwargs.additional_values.items()):
event.set(value, key)
self.submit(event, self.pool.queue.outbox)
self.sleep()
self.logging.info("Stopped producing events.")
def test_event_delete():
a = Event({"one": 1, "two": 2})
a.delete("data.two")
try:
a.get("data.two")
except KeyError:
assert True
else:
assert False
def __log(self, level, message):
event = Event(self.name)
event.data = Log(time(), level, getpid(), self.name, message)
while True:
try:
self.logs.put(event)
break
except QueueFull:
sleep(0.1)
def test_extractBulkItems():
from wishbone.event import extractBulkItems
e = Event(bulk=True)
e.appendBulk(Event({"one": 1}))
e.appendBulk(Event({"two": 2}))
e.appendBulk(Event({"three": 3}))
for item in extractBulkItems(e):
assert isinstance(item, Event)
def produce(self):
while self.loop():
message = self.generateMessage(self.kwargs.message)
event = Event(message)
for key, value in list(self.kwargs.additional_values.items()):
event.set(value, key)
self.submit(event, self.pool.queue.outbox)
self.sleep()
self.logging.info("Stopped producing events.")
def test_extractBulkItemValues():
from wishbone.event import extractBulkItemValues
e = Event(bulk=True)
e.appendBulk(Event({"one": 1}))
e.appendBulk(Event({"two": 2}))
e.appendBulk(Event({"three": 3}))
for item in extractBulkItemValues(e, "data"):
assert item in [{"one": 1}, {"two": 2}, {"three": 3}]
def test_event_slurp_bad():
a = Event()
del(a.data["uuid"])
b = Event()
try:
b.slurp(a.dump())
except InvalidData:
assert True
else:
assert False
def processEvent(self, data, meta, queue):
'''
The callback executed for each Wishbone event to be created out of a
single http request.
'''
e = Event(data)
e.set(meta, 'tmp.%s' % (self.name))
e.renderKwargs(self.kwargs_template)
self.submit(e, queue)
return self.getResponse(e, queue)
def metricProducer(self):
'''A greenthread which collects the queue metrics at the defined interval.'''
self.__run.wait()
hostname = socket.gethostname()
while self.loop():
for queue in self.pool.listQueues(names=True):
for metric, value in self.pool.getQueue(queue).stats().iteritems():
event = Wishbone_Event(self.name)
event.data = Metric(time=time(), type="wishbone", source=hostname, name="module.%s.queue.%s.%s" % (self.name, queue, metric), value=value, unit="", tags=())
self.submit(event, self.pool.queue.metrics)
sleep(self.frequency)
def test_event_appendBulkFull():
e = Event(bulk=True, bulk_size=1)
ee = Event({"one": 1})
e.appendBulk(ee)
try:
e.appendBulk(ee)
except BulkFull:
assert True
else:
assert False
def test_event_dump():
from wishbone.event import EVENT_RESERVED
data = {"one": 1, "two": 2}
a = Event(data)
result = a.dump()
for key in EVENT_RESERVED:
assert key in result
assert result["data"] == data
assert isinstance(result["timestamp"], float)
def generateEvent(self, data={}, destination=None):
'''
Generates a new event.
This function can get overridden by
``wishbone.module.InputModule._generateNativeEvent``.
The provided ``data`` will be traversed in search of valid templates
which then will be rendered.
Args:
data (``data``): The payload to add to the event.
destination (None): The destination key to write the data to
Returns:
wishbone.event.Event: An event containing ``data`` as a payload.
'''
if destination in [None, "data"]:
event = Wishbone_Event(data)
event.renderField(destination, self.env_template)
else:
event = Wishbone_Event()
event.set(data, destination)
event.renderField(destination, self.env_template)
return event
def test_module_humanlogformat():
actor_config = ActorConfig('humanlogformat', 100, 1, {}, "")
humanlogformat = HumanLogFormat(actor_config, colorize=False, ident='setup.py')
humanlogformat.pool.queue.inbox.disableFallThrough()
humanlogformat.pool.queue.outbox.disableFallThrough()
humanlogformat.start()
e = Event('test')
e.set(Log(1367682301.430527, 6, 3342, 'Router', 'Received SIGINT. Shutting down.'))
humanlogformat.pool.queue.inbox.put(e)
one = getter(humanlogformat.pool.queue.outbox)
assert one.get() == "2013-05-04T17:45:01 setup.py[3342]: informational Router: Received SIGINT. Shutting down."
def test_module_msgpackencode():
actor_config = ActorConfig("msgpackencode", 100, 1, {})
msgpackencode = MSGPackEncode(actor_config)
msgpackencode.pool.queue.inbox.disableFallThrough()
msgpackencode.pool.queue.outbox.disableFallThrough()
msgpackencode.start()
e = Event("test")
e.setData([1, 2, 3])
msgpackencode.pool.queue.inbox.put(e)
one = getter(msgpackencode.pool.queue.outbox)
assert one.data == "\x93\x01\x02\x03"
def test_module_msgpackdecode():
actor_config = ActorConfig('msgpackdecode', 100, 1, {})
msgpackdecode = MSGPackDecode(actor_config)
msgpackdecode.pool.queue.inbox.disableFallThrough()
msgpackdecode.pool.queue.outbox.disableFallThrough()
msgpackdecode.start()
e = Event('test')
e.setData('\x93\x01\x02\x03')
msgpackdecode.pool.queue.inbox.put(e)
one = getter(msgpackdecode.pool.queue.outbox)
assert one.data == [1, 2, 3]
def test_module_jsonencode():
actor_config = ActorConfig('jsonencode', 100, 1, {})
jsonencode = JSONEncode(actor_config)
jsonencode.pool.queue.inbox.disableFallThrough()
jsonencode.pool.queue.outbox.disableFallThrough()
jsonencode.start()
e = Event('test')
e.setData(["one", "two", "three"])
jsonencode.pool.queue.inbox.put(e)
one = getter(jsonencode.pool.queue.outbox)
assert one.data == '["one", "two", "three"]'
def test_module_header():
actor_config = ActorConfig('header', 100, 1, {})
header = Header(actor_config, header={"greeting": "hello"})
header.pool.queue.inbox.disableFallThrough()
header.pool.queue.outbox.disableFallThrough()
header.start()
e = Event('test')
e.setData('hello')
header.pool.queue.inbox.put(e)
one = getter(header.pool.queue.outbox)
assert one.getHeaderValue("header", "greeting") == "hello"
def test_module_join():
a = get_actor({"join": ['@data', ",", "@tmp.joined"]})
e = Event(["one", "two", "three"])
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('@tmp.joined') == "one,two,three"
def test_module_del_item():
a = get_actor({"del_item": ["fubar", "@data"]})
e = Event(["one", "two", "three", "fubar"])
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert "fubar" not in one.get('@data')
def test_module_replace():
a = get_actor({"replace": ['\d', "X", "@data"]})
e = Event("hello 123 hello")
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('@data') == "hello XXX hello"
def test_module_copy_default():
a = get_actor({"copy": ["does.not.exist", "@tmp.copy", "default"]})
e = Event({"greeting": "hi"})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert "default" == one.get('@tmp.copy')
def test_module_fanout():
actor_config = ActorConfig('fanout', 100, 1, {}, "")
fanout = Fanout(actor_config)
fanout.pool.queue.inbox.disableFallThrough()
fanout.pool.createQueue("one")
fanout.pool.queue.one.disableFallThrough()
fanout.pool.createQueue("two")
fanout.pool.queue.two.disableFallThrough()
fanout.start()
e = Event(["hello"])
fanout.pool.queue.inbox.put(e)
one = getter(fanout.pool.queue.one)
two = getter(fanout.pool.queue.two)
fanout.stop()
assert one.get() == ["hello"]
assert two.get() == ["hello"]
assert id(one.get()) != id(two.get())
def test_module_delete():
a = get_actor({"delete": ["@data.two"]})
e = Event({"one": 1, "two": 2})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert "two" not in one.get('@data').keys()
def test_module_decode_service_checkresult():
data = re.sub(' {2,4}', "\t", service_checkresult)
actor_config = ActorConfig('pd', 100, 1, {}, "")
pd = CheckResult(actor_config)
pd.pool.queue.inbox.disableFallThrough()
pd.pool.queue.outbox.disableFallThrough()
pd.start()
e = Event(data)
pd.pool.queue.inbox.put(e)
sleep(1)
assert pd.pool.queue.outbox.size() == 8
one = getter(pd.pool.queue.outbox).get()
assert isinstance(one, Metric)
assert one.name == 'service1.time'
assert one.value == '0.02'
two = getter(pd.pool.queue.outbox).get()
assert isinstance(two, Metric)
assert two.name == 'service1.cat'
assert two.value == '20'
def test_smaller_than():
rule = {"smaller": {
"condition": [
{"smaller": "<:100"}
],
"queue": [
{"smaller": {}}
]
}}
actor = generate_actor(rule)
one = Event("test")
one.setData({"smaller": "99"})
actor.pool.queue.inbox.put(one)
assert int(getter(actor.pool.queue.smaller).data["smaller"]) < 100
def test_bigger_than():
rule = {"bigger": {
"condition": [
{"bigger": ">:10"}
],
"queue": [
{"bigger": {}}
]
}}
actor = generate_actor(rule)
e = Event("test")
e.setData({"bigger": "100"})
actor.pool.queue.inbox.put(e)
assert getter(actor.pool.queue.bigger).data["bigger"] == "100"
def test_equal_to():
rule = {"equal": {
"condition": [
{"equal": "=:100"}
],
"queue": [
{"equal": {}}
]
}}
actor = generate_actor(rule)
one = Event("test")
one.setData({"equal": "100"})
actor.pool.queue.inbox.put(one)
assert int(getter(actor.pool.queue.equal).data["equal"]) == 100
def test_negative_list_membership():
rule = {"list_membership": {
"condition": [
{"list_membership": "!in:test"}
],
"queue": [
{"list_membership": {}}
]
}}
actor = generate_actor(rule)
one = Event("test")
one.setData({"list_membership": ["one", "three", "two"]})
actor.pool.queue.inbox.put(one)
assert "test" not in getter(actor.pool.queue.list_membership).data["list_membership"]
def test_regex():
rule = {"regex": {
"condition": [
{"regex": "re:.*?two.*"}
],
"queue": [
{"regex": {}}
]
}}
actor = generate_actor(rule)
e = Event("test")
e.setData({"regex": "one two three"})
actor.pool.queue.inbox.put(e)
assert getter(actor.pool.queue.regex).data["regex"] == "one two three"
def test_module_modify_set():
a = get_actor({"set": ["hi", "blah"]})
e = Event('hello')
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('blah') == "hi"
def test_module_lowercase():
a = get_actor({"lowercase": ["@data.lower"]})
e = Event({"lower": "HELLO"})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('@data.lower') == "hello"
def test_module_template():
a = get_actor(
{"template": ["result", "Good day in {language} is {word}.", "@data"]})
e = Event({"language": "German", "word": "gutten Tag"})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert one.get('result') == "Good day in German is gutten Tag."
def test_module_copy():
a = get_actor({"copy": ["@data", "@tmp.copy", "n/a"]})
e = Event({"greeting": "hi"})
a.pool.queue.inbox.put(e)
one = getter(a.pool.queue.outbox)
assert "hi" == one.get('@tmp.copy')["greeting"]
assert id(one.get("@data")) != id(one.get("@tmp.copy"))
def test_module_http_ok_server_response():
webserver = WebServer()
webserver.start()
actor_config = ActorConfig('httpoutclient', 100, 1, {}, "")
http = HTTPOutClient(actor_config, url="http://*****:*****@tmp.httpoutclient.server_response_json")["message"] == "hello world!")
http.stop()
webserver.stop()
def test_fileread(module):
module.start()
module.pool.queue.inbox.put(Event(DATA))
#gevent.sleep(10)
#import pdb; pdb.set_trace()
with open(os.path.join(here, 'results', '1111.json')) as df:
data = json.load(df)
assert data == DATA
def generateDicts(self):
while self.loop():
d = self.getDict()
event = Event(d)
self.submit(event, self.pool.queue.outbox)
self.key_number = +1
sleep(self.kwargs.interval)
def test_not_equal_to():
rule = {
"equal": {
"condition": [{
"equal": "!=:100"
}],
"queue": [{
"equal": {}
}]
}
}
actor = generate_actor(rule)
one = Event("test")
one.set({"equal": "101"})
actor.pool.queue.inbox.put(one)
assert int(getter(actor.pool.queue.equal).get()["equal"]) == 101
def test_fileread(module):
module.start()
path = os.path.join(here, 'data.xml')
e = Event(path)
module.pool.queue.inbox.put(e)
doc = getter(module.pool.queue.outbox).get()
if not doc:
assert False
assert doc == XML
def test_bigger_than():
rule = {
"bigger": {
"condition": [{
"bigger": ">:10"
}],
"queue": [{
"bigger": {}
}]
}
}
actor = generate_actor(rule)
e = Event("test")
e.set({"bigger": "100"})
actor.pool.queue.inbox.put(e)
assert getter(actor.pool.queue.bigger).get()["bigger"] == "100"
def test_smaller_than():
rule = {
"smaller": {
"condition": [{
"smaller": "<:100"
}],
"queue": [{
"smaller": {}
}]
}
}
actor = generate_actor(rule)
one = Event("test")
one.set({"smaller": "99"})
actor.pool.queue.inbox.put(one)
assert int(getter(actor.pool.queue.smaller).get()["smaller"]) < 100
def consume(self, event):
data = event.get(self.kwargs.source)
if isinstance(data, dict) or isinstance(data, list):
for name, value in self.recurseData(data):
metric = Metric(time(), self.kwargs.type, self.kwargs.metric_source, name, value, "", self.kwargs.tags)
self.submit(Event(metric), self.pool.queue.outbox)
else:
raise Exception("Dropped incoming data because not of type <dict>. Perhaps you forgot to feed the data through wishbone.decode.json first?")
def produce(self):
while self.loop():
for feed in self.couchdb.changes(
feed="continuous", since=self.since, include_docs=True
):
self.since = feed.get('seq', feed.get('last_seq', "now"))
self.logging.debug("Change event {}".format(feed))
if feed and 'doc' in feed:
self.submit(Event(feed['doc']), "outbox")
self.logging.info("Stopping changes feed from couchdb")
