def test_subscription(self):
service = PubSub('test_subscription')
sub = Subscriber('sub')
service.subscribe(sub, sub.key, 'topic', sub.cb1)
subscriptions = service.subscriptions()
self.assertIn('topic', subscriptions)
self.assertIn((sub, sub.key, sub.cb1), subscriptions['topic'])
def test_persistent_subscriptions(self):
service = PubSub('test_pseristsent_subscriptions')
sub = Subscriber('sub')
service.subscribe(sub, sub.key, 'topic', sub.cb1)
del service
service = PubSub('test_pseristsent_subscriptions')
subscriptions = service.subscriptions()
self.assertIn('topic', subscriptions)
self.assertIn((sub, sub.key, sub.cb1), subscriptions['topic'])
def test_duplicate_subscription(self):
service = PubSub('test_duplicate_subscription')
sub = Subscriber('sub')
local_dict = WeakValueDictionary()
service.subscribe(sub, sub.key, 'topic', sub.cb1)
service.subscribe(sub, sub.key, 'topic', sub.cb1)
subscriptions = service.subscriptions()
self.assertIn('topic', subscriptions)
self.assertEqual(len(subscriptions['topic']), 1)
self.assertIn((sub, sub.key, sub.cb1), subscriptions['topic'])
def test_publish_filter(self):
service = PubSub('test_publish_filter')
subscribers = [Subscriber('sub%d' % i) for i in range(1, 5)]
for sub in subscribers:
service.subscribe(sub, sub.key, 'topic', sub.cb1)
service.publish('topic', 'event1')
for sub in subscribers:
self.assertIn((sub, Subscriber.cb1, 'topic', 'event1'),
log['callbacks'])
service.publish('topic', 'event2', subscribers[0].key)
for sub in subscribers[:1]:
self.assertNotIn((sub, Subscriber.cb1, 'topic', 'event2'),
log['callbacks'])
for sub in subscribers[1:]:
self.assertIn((sub, Subscriber.cb1, 'topic', 'event2'),
log['callbacks'])
clear_log()
service.publish('topic', 'event3',
subscribers[0].key, [s.key for s in subscribers[:3]])
for sub in [subscribers[0], subscribers[3]]:
self.assertNotIn((sub, Subscriber.cb1, 'topic', 'event3'),
log['callbacks'])
for sub in subscribers[1:3]:
self.assertIn((sub, Subscriber.cb1, 'topic', 'event3'),
log['callbacks'])
def test_publish(self):
pubsub = PubSub(10)
self.assertEqual(0, len(pubsub.channels))
pubsub.publish("channel", "data")
self.assertEqual(1, len(pubsub.channels))
self.assertTrue("channel" in pubsub.channels)
pubsub.publish("channel2", "data")
self.assertEqual(2, len(pubsub.channels))
self.assertTrue("channel2" in pubsub.channels)
def test_weak_subscriptions(self):
service = PubSub('test_weak_subscriptions')
sub = Subscriber('sub')
weaksub = weakref.ref(sub)
service.subscribe(sub, sub.key, 'topic', sub.cb1)
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 1)
self.assertIn('topic', subscriptions)
self.assertIn((sub, sub.key, sub.cb1), subscriptions['topic'])
del subscriptions
del sub
gc.collect()
self.assertEqual(weaksub(), None)
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 0)
def __init__(self, capacity):
self.capacity = capacity
self.pubsub = PubSub(CloudyException)
self.nodes = {
idx: CloudyGCounter(idx, self.pubsub)
for idx in range(capacity)
}
def testGeoFeed(self):
item = {
'topic': 'T',
'key': 'K',
'latitude': 39,
'longitude': -79
}
id = GeoFeed.publish(**item)
self.assertEqual(id, item['key'])
time.sleep(2)
item['key'] = 'L'
id = GeoFeed.publish(**item)
self.assertEqual(id, item['key'])
last_published = '9999'
for doc in GeoFeed.list(item['topic']):
self.assertLess(doc['published'], last_published)
last_published = doc['published']
for doc in GeoFeed.list('NOT_FOUND'):
self.assertFalse('Should never get here')
doc = GeoFeed.get(item['topic'], item['key'])
self.assertIsNotNone(doc)
#set up PubSub subscription so that a task is created when an item is published to the feed
sub_url = "/pubsub/task"
event=GeoFeed._indexname(item['topic'])
channel='ProcessNew%s' % item['topic']
sub_data = {'secret': 'SECRET', 'channel': channel, 'pubname': 'key'}
PubSub.subscribe (event, sub_url, sub_data)
# now publish a new item to the feed. This should trigger creation of a new task in the queue
id = GeoFeed.publish(**item)
# need to manually process the task queue because we're in test mode
response = self.executeTask() # /pubsub/notify
self.assertEqual (response.json['status'], 'OK')
response = self.executeTask() # /pubsub/task
self.assertEqual (response.json['status'], 'OK')
# Now make sure there is a task in the queue
queue = TaskQueue()
lease = queue.lease (channel=channel)
self.assertIsNotNone(lease)
self.assertEqual(lease['id'], response.json['id'])
self.assertEqual(lease['content']['pub_data']['key'], item['key'])
def demo():
'''Demo gCRDT'''
ps = PubSub()
nodes = [GCounter(idx, ps) for idx in range(3)]
for i in range(10):
node = random.choice(nodes)
node.increment()
for n in nodes:
print("Node %d says %d" % (n.idx, n.value()))
def test_integrated(self):
'''Test some nodes hooked up with a pub/sub'''
pubsub = PubSub()
nodes = [GCounter(idx, pubsub) for idx in range(3)]
for i in range(100):
# Pick a random node and increment it
node = random.choice(nodes)
node.increment()
# All .value() checks should have the same result
self.assertTrue(all(n.value() == node.value() for n in nodes))
def main():
import logging.config
logging.config.fileConfig('logging_config.ini',
disable_existing_loggers=False)
parser = argparse.ArgumentParser()
parser.add_argument('--port',
nargs='?',
type=int,
default=5556,
help='Port the server should listen on')
parser.add_argument('--pubsub',
nargs='?',
type=str,
default='tcp://143.215.207.2:3247',
help='PubSub address other camera should subscribe to')
parser.add_argument('--storage',
nargs='?',
type=str,
default='output',
help='Directory to store the video')
parser.add_argument('--cname',
nargs='?',
type=str,
default='ferst_state',
help='The name of the camera')
args = parser.parse_args()
try:
createEmptyDir(args.storage)
except Exception as e:
SLogger.fatal('Unable to create the output directory: %s' % e)
sys.exit(-1)
pubsub = PubSub(args.cname, args.pubsub, context)
SLogger.info('Successfully start pubsub service')
tgraph = TrajectoryGraph()
SLogger.info('Successfully connect to graph database')
time.sleep(5)
cand = []
sub_thread = threading.Thread(target=listen_candidates,
args=(pubsub, cand))
sub_thread.start()
matching_queue = queue.SimpleQueue()
match_thread = threading.Thread(target=matching,
args=(matching_queue, cand, tgraph))
match_thread.start()
listen(args.port, args.storage, pubsub, tgraph, matching_queue)
def publish (**kw):
issuer = SeqidIssuer(series=kw['topic'])
seqid = issuer.issueSeqids()[0]
doc = search.Document(
doc_id = kw['key'],
fields=[
search.AtomField(name='topic', value=kw['topic']),
search.AtomField(name='key', value=kw['key']),
search.GeoField(name='location',
value=search.GeoPoint(kw.get('latitude',0), kw.get('longitude',0))),
search.AtomField(name='published', value=seqid2str(seqid)),
search.AtomField(name='url', value=kw.get('url', '')),
search.TextField(name='content', value=kw.get('content', ''))
],
rank = seqid /1000 # rank is int32, so convert seqid from milliseconds to seconds
)
index_name = GeoFeed._indexname(kw['topic'])
index = search.Index(name=index_name)
id = index.put(doc)[0].id
PubSub.publish(event=index_name, pub_data={'key':kw['key']})
return id
def main():
import logging.config
logging.config.fileConfig('logging_config.ini',
disable_existing_loggers=False)
args = arg_parse()
context = zmq.Context()
socket = context.socket(zmq.PAIR)
socket.bind('tcp://*:%s' % args.port)
# No clean up code
vstore = VideoStorageClient(args.video_storage_addr, context)
pubsub = PubSub(args.cname, args.pubsub, context)
tgraph = TrajectoryGraph()
vt = VehicleTracking()
pool = CandidatePool()
listen_thread = threading.Thread(target=listener_func, args=(pubsub, pool))
listen_thread.start()
coldstart(tgraph)
frame_id = 0
fps = FPS()
while True:
try:
rawimage, image, bboxes = parse_load(socket)
except Exception as e:
logging.warn("Unable to parse: exception %s" % e)
continue
tracked_bboxes = vt.sort_update(bboxes)
logging.info("Track result: %s" % tracked_bboxes)
frame_storage(vstore, args.cname, frame_id, rawimage, tracked_bboxes)
leaving_vehicles = vt.status_update(frame_id, image)
for vehicle in leaving_vehicles:
logging.info("Vehicle: %d is leaving" % vehicle.id)
hist = feature_extraction_adaptive_histogram(vehicle)
vertexid = vertex_storage(tgraph, args.cname, vehicle)
messaging(pubsub, vertexid, vehicle.id, hist)
res = pool.matching(hist, args.dis_thres)
logging.info("Re-Id for vehicle %d: %s" % (vehicle.id, res))
edge_storage(tgraph, res, vertexid)
frame_id += 1
logging.debug("FPS: %.2f" % fps())
cleanup()
def test_mutliple_subscriptions(self):
service = PubSub('test_multiple_subscription')
sub1 = Subscriber('sub1')
sub2 = Subscriber('sub2')
service.subscribe(sub1, sub1.key, 'topic1', sub1.cb1)
service.subscribe(sub1, sub1.key, 'topic1', sub1.cb2)
service.subscribe(sub1, sub1.key, 'topic2', sub1.cb1)
service.subscribe(sub1, sub1.key, 'topic2', sub1.cb2)
service.subscribe(sub2, sub2.key, 'topic1', sub2.cb1)
service.subscribe(sub2, sub2.key, 'topic1', sub2.cb2)
service.subscribe(sub2, sub2.key, 'topic2', sub2.cb1)
service.subscribe(sub2, sub2.key, 'topic2', sub2.cb2)
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 2)
self.assertIn('topic1', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertIn('topic2', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic2'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic2'])
def main():
import logging.config
logging.config.fileConfig('logging_config.ini', disable_existing_loggers=False)
parser = argparse.ArgumentParser()
parser.add_argument('--port', nargs='?', default=5556, help='Port, communication between RPi1 and RPi2')
parser.add_argument('--pubsub', nargs='?', default='tcp://143.215.207.2:3247', help='PubSub address other camera should subscribe to')
parser.add_argument("--video_storage_addr", nargs='?', default="tcp://130.207.122.57:1429", help="address of videoStorageServer")
parser.add_argument('--cname', required=True, help='The name of the camera')
args = parser.parse_args()
camera_name = args.cname
context = zmq.Context()
socket = context.socket(zmq.PAIR)
socket.bind('tcp://*:%s' % args.port)
vstore = VideoStorageClient(args.video_storage_addr, context)
pubsub = PubSub(args.cname, args.pubsub, context)
tgraph = TrajectoryGraph()
pool = CandidatePool()
listen_thread = threading.Thread(target=listener_func, args=(pubsub, pool))
listen_thread.start()
frame_id = 0
fps = FPS()
vt = VehicleTracking()
while True:
rawimage, image, bboxes = parse_load(socket)
tracked_bboxes = vt.sort_update(bboxes)
frame_storage(vstore, camera_name, frame_id, rawimage,
tracked_bboxes)
leaving_vehicles = vt.status_update(frame_id, image)
for vehicle in leaving_vehicles:
hist = feature_extraction_adaptive_histogram(vehicle)
vertexid = vertex_storage(tgraph, camera_name, vehicle)
messaging(pubsub, vertexid, vehicle.id, hist)
res = pool.matching(hist, args.dis_thres)
logging.info("Re-Id for vehicle %d: %s" % (vehicle.id, res))
edge_storage(tgraph, res, vertexid)
frame_id += 1
logging.info('fps: %.2f' % fps())
def test_publish(self):
def callback(topic, event):
log['callbacks'].append((callback, topic, event))
pass
service = PubSub('test_publish')
sub = Subscriber('sub')
service.subscribe(sub, sub.key, 'topic', sub.cb1)
service.subscribe(sub, sub.key, 'topic', sub.cb2)
service.subscribe(sub, sub.key, 'topic2', sub.cb2)
service.subscribe(sub, sub.key, 'topic2', callback)
service.publish('topic', 'event')
service.publish('topic2', 'event2')
self.assertEqual(len(log['callbacks']), 4)
self.assertIn((sub, Subscriber.cb1, 'topic', 'event'),
log['callbacks'])
self.assertIn((sub, Subscriber.cb2, 'topic', 'event'),
log['callbacks'])
self.assertIn((sub, Subscriber.cb2, 'topic2', 'event2'),
log['callbacks'])
self.assertIn((callback, 'topic2', 'event2'), log['callbacks'])
# The MIT License (MIT) # Copyright (c) 2020 Jan Cespivo # octopusLAB pubsub example from pubsub import PubSub pubsub = PubSub(100) pubsub.start()
def testPubSub(self):
sub_data = {'secret': 'SECRET'}
id = PubSub.subscribe ('EVENT', 'not_an_url', sub_data)
self.assertTrue (PubSub.unsubscribe (id))
self.assertFalse (PubSub.unsubscribe (id))
pub_data = {'message': 123}
self.assertEqual(0, PubSub.publish ('EVENT', pub_data))
PubSub.subscribe ('EVENT', 'not_an_url', sub_data)
self.assertEqual(1, PubSub.publish ('EVENT', pub_data))
response = self.executeTask() # /pubsub/notify
self.assertEqual(response.status_int, 200)
response.mustcontain ('unknown url type')
url = "/pubsub/test"
PubSub.subscribe ('EVENT2', url, sub_data)
self.assertEqual(1, PubSub.publish ('EVENT2', pub_data))
response = self.executeTask() # /pubsub/notify
self.assertEqual (response.json['status'], 'OK')
response = self.executeTask() # /pubsub/test
self.assertEqual (response.json['pub_data']["message"], 123)
url = "/pubsub/task"
sub_data = {'secret': 'SECRET', 'channel': 'CHANNEL', 'taskname': 'NAME'}
PubSub.subscribe ('EVENT3', url, sub_data)
self.assertEqual(1, PubSub.publish ('EVENT3', pub_data))
response = self.executeTask() # /pubsub/notify
self.assertEqual (response.json['status'], 'OK')
response = self.executeTask() # /pubsub/task
self.assertEqual (response.json['status'], 'OK')
queue = TaskQueue()
lease = queue.lease (channel='CHANNEL')
self.assertIsNotNone(lease)
self.assertEqual(lease['id'], response.json['id'])
# The MIT License (MIT)
# Copyright (c) 2020 Jan Cespivo
# octopusLAB pubsub example
from pubsub import PubSub
pubsub = PubSub(100)
pubsub.start()
#######################################################################################
from machine import Pin
def irq_handler(pin):
pubsub.publish('button', pin)
button_0 = Pin(0, Pin.IN)
button_0.irq(trigger=Pin.IRQ_FALLING, handler=irq_handler)
#######################################################################################
from machine import Timer
tim = Timer(-1)
tim.init(period=2000,
mode=Timer.PERIODIC,
callback=lambda t: pubsub.publish('timer'))
#######################################################################################
class Counter:
# run in its own thread. Feeds post a specified number of messages, waiting a
# random interval between messages. Each inbox is polled for messages received,
# terminating when no messages are received for a wait limit.
import random
import threading
import time
import fdb
from pubsub import PubSub
fdb.api_version(22)
db = fdb.open()
ps = PubSub(db)
ps.clear_all_messages()
# Create the specified numbers of feeds and inboxes. Subscribe each inbox to a
# randomly selected subset of feeds.
def setup_topology(feeds, inboxes):
feed_map = {f: ps.create_feed('Alice ' + str(f)) for f in range(feeds)}
inbox_map = {}
for i in range(inboxes):
inbox_map[i] = ps.create_inbox('Bob ' + str(i))
for f in random.sample(xrange(feeds), random.randint(1, feeds)):
ps.create_subscription(inbox_map[i], feed_map[f])
return feed_map, inbox_map
def test_subscription_filter(self):
service = PubSub('test_subscription_filter')
sub1 = Subscriber('sub1')
sub2 = Subscriber('sub2')
service.subscribe(sub1, sub1.key, 'topic1', sub1.cb1)
service.subscribe(sub1, sub1.key, 'topic1', sub1.cb2)
service.subscribe(sub1, sub1.key, 'topic2', sub1.cb1)
service.subscribe(sub1, sub1.key, 'topic2', sub1.cb2)
service.subscribe(sub2, sub2.key, 'topic1', sub2.cb1)
service.subscribe(sub2, sub2.key, 'topic1', sub2.cb2)
service.subscribe(sub2, sub2.key, 'topic2', sub2.cb1)
service.subscribe(sub2, sub2.key, 'topic2', sub2.cb2)
subscriptions = service.subscriptions(subscriber=sub2)
self.assertEqual(len(subscriptions), 2)
self.assertIn('topic1', subscriptions)
self.assertNotIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertIn('topic2', subscriptions)
self.assertNotIn((sub1, sub1.key, sub1.cb1), subscriptions['topic2'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic2'])
subscriptions = service.subscriptions(key=sub1.key)
self.assertEqual(len(subscriptions), 2)
self.assertIn('topic1', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertNotIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertNotIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertIn('topic2', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic2'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic2'])
self.assertNotIn((sub2, sub2.key, sub2.cb1), subscriptions['topic2'])
self.assertNotIn((sub2, sub2.key, sub2.cb2), subscriptions['topic2'])
subscriptions = service.subscriptions(topic='topic1')
self.assertEqual(len(subscriptions), 1)
self.assertIn('topic1', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertNotIn('topic2', subscriptions)
subscriptions = service.subscriptions(callback=sub1.cb1)
self.assertEqual(len(subscriptions), 2)
self.assertIn('topic1', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertNotIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertNotIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertIn('topic2', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic2'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic2'])
self.assertNotIn((sub2, sub2.key, sub2.cb1), subscriptions['topic2'])
self.assertNotIn((sub2, sub2.key, sub2.cb2), subscriptions['topic2'])
# The MIT License (MIT) # Copyright (c) 2020 Jan Cespivo # octopusLAB pubsub example from pubsub import PubSub pubsub = PubSub()
def test_unsubscribe(self):
service = PubSub('test_unsubscribe')
sub1 = Subscriber('sub1')
sub2 = Subscriber('sub2')
service.subscribe(sub1, sub1.key, 'topic1', sub1.cb1)
service.subscribe(sub1, sub1.key, 'topic1', sub1.cb2)
service.subscribe(sub1, sub1.key, 'topic2', sub1.cb1)
service.subscribe(sub1, sub1.key, 'topic2', sub1.cb2)
service.subscribe(sub2, sub2.key, 'topic1', sub2.cb1)
service.subscribe(sub2, sub2.key, 'topic1', sub2.cb2)
service.subscribe(sub2, sub2.key, 'topic2', sub2.cb1)
service.subscribe(sub2, sub2.key, 'topic2', sub2.cb2)
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 2)
self.assertIn('topic1', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertIn('topic2', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic2'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
service.unsubscribe(sub2, sub2.key, 'topic2', sub2.cb2)
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 2)
self.assertIn('topic1', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertIn('topic2', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic2'])
self.assertIn((sub1, sub1.key, sub1.cb2), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic2'])
self.assertNotIn((sub2, sub2.key, sub2.cb2), subscriptions['topic2'])
service.unsubscribe(sub1, sub1.key, callback=sub1.cb2)
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 2)
self.assertIn('topic1', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertIn('topic2', subscriptions)
self.assertIn((sub1, sub1.key, sub1.cb1), subscriptions['topic2'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic2'])
self.assertNotIn((sub2, sub2.key, sub2.cb2), subscriptions['topic2'])
service.unsubscribe(callback=sub1.cb1)
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 2)
self.assertIn('topic1', subscriptions)
self.assertNotIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertIn('topic2', subscriptions)
self.assertNotIn((sub1, sub1.key, sub1.cb1), subscriptions['topic2'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic2'])
self.assertIn((sub2, sub2.key, sub2.cb1), subscriptions['topic2'])
self.assertNotIn((sub2, sub2.key, sub2.cb2), subscriptions['topic2'])
service.unsubscribe(callback=sub2.cb1)
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 1)
self.assertIn('topic1', subscriptions)
self.assertNotIn((sub1, sub1.key, sub1.cb1), subscriptions['topic1'])
self.assertNotIn((sub1, sub1.key, sub1.cb2), subscriptions['topic1'])
self.assertNotIn((sub2, sub2.key, sub2.cb1), subscriptions['topic1'])
self.assertIn((sub2, sub2.key, sub2.cb2), subscriptions['topic1'])
self.assertNotIn('topic2', subscriptions)
service.unsubscribe()
subscriptions = service.subscriptions()
self.assertEqual(len(subscriptions), 0)
def pubsub():
app = PubSub("test", "test")
return app
def pubsub_client(self):
return PubSub(self)
from celery import Celery
from pubsub import PubSub
app = Celery()
pubsub_app = PubSub('amqp://guest@localhost', 'sample.topic')
@app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
sender.add_periodic_task(3.0, pubsub_listen, name='Consume messages')
@pubsub_app.subscribe('booking.*')
def process_booking_event(body, _):
print(body['object'])
@app.task
def pubsub_listen():
pubsub_app.drain()
return "Done!"
def __init__(self, messageTimeout,
subscriptionTimeout, messageFormat):
self.pubsub = PubSub(messageTimeout)
self.subscriptionTimeout = subscriptionTimeout
self.messageFormat = messageFormat
def namespaced_pubsub():
app = PubSub("test", "test", namespace=_NAMESPACE)
return app
def main():
import logging.config
logging.config.fileConfig('logging_config.ini', disable_existing_loggers=False)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger('HttpUtil').setLevel(logging.INFO)
args = arg_parse()
if args.imageSeq is not None or args.live is not None:
target_labelIds = get_target_labelIds(args.labels)
engine = BasicEngine(args.model)
model_w, model_h, tensor_start_index = engine_info(engine)
else:
logging.fatal("No valid stream input source")
socket = None
context = None
if args.output is not None:
context = zmq.Context()
socket = contect.socket(zmq.PAIR)
socket.connect(args.output)
if args.imageSeq is not None:
stream = ImageSequenceStream(args.imageSeq)
elif args.live is not None:
stream = SingleCampusCameraStream(args.live, args.userconfig, args.cameraconfig)
stream.login()
logging.info("Successfully login into Campus Camera Stream --- %s" % args.live)
if context is None:
context = zmq.Context()
# No clean up code
vstore = VideoStorageClient(args.video_storage_addr, context)
pubsub = PubSub(args.cname, args.pubsub, context)
tgraph = TrajectoryGraph()
vt = VehicleTracking()
pool = CandidatePool()
listen_thread = threading.Thread(target=listener_func, args=(pubsub, pool))
listen_thread.start()
coldstart(tgraph)
def cleanup():
if args.live is not None:
stream.logout()
if socket is not None:
socket.close()
context.term()
def signal_handler(sig, frame):
cleanup()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
frame_id = 0
fps = FPS()
while True:
try:
frame = stream.fetch_frame()
except:
break
image = load_frame(frame)
image_w, image_h = image.size
resized_image = resize_frame(image, model_w, model_h)
raw_result = inference(engine, resized_image)
bboxes = post_inference(raw_result,
tensor_start_index,
target_labelIds,
args.threshold,
args.top_k, image_w, image_h)
logging.info("Detection result: %s" % bboxes)
tracked_bboxes = vt.sort_update(bboxes)
logging.info("Track result: %s" % tracked_bboxes)
frame_storage(vstore, args.cname, frame_id, frame, tracked_bboxes)
opencv_image = load_opencv_PIL(image)
leaving_vehicles = vt.status_update(frame_id, opencv_image)
for vehicle in leaving_vehicles:
logging.info("Vehicle: %d is leaving" % vehicle.id)
hist = feature_extraction_adaptive_histogram(vehicle)
vertexid = vertex_storage(tgraph, args.cname, vehicle)
messaging(pubsub, vertexid, vehicle.id, hist)
res = pool.matching(hist, args.dis_thres)
logging.info("Re-Id for vehicle %d: %s" % (vehicle.id, res))
edge_storage(tgraph, res, vertexid)
frame_id += 1
logging.debug("FPS: %.2f" % fps())
cleanup()
from flask import Flask, send_file
from flask_socketio import SocketIO
from pubsub import PubSub
import os
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
app.socket = SocketIO(app)
app.aio = PubSub()
app.publish = app.aio.publish
import routes
import socketio
@app.route('/<path:path>')
def static_file(path):
temp = os.path.dirname(os.path.realpath(__file__)) + '/public/' + path
return send_file(temp)
def handler(signum, frame):
logger.info('got term... raising TermException')
raise TermException()
signal.signal(signal.SIGTERM, handler)
def create_data():
logger.info("callback invoked")
return {"data": "here is some data from server"}
def make_query_callback(data):
logger.info('got request of: %s' % data)
ps.publish('return_result', create_data())
return
ps = PubSub('AWS', topic_arn=os.environ['AWS_SNS_ARN'], logger=logger)
make_query_subscription = ps.subscribe('make_query',
callback=make_query_callback)
try:
while True:
time.sleep(100)
except:
logger.info('terminating... dropping subscriber')
make_query_subscription.unsubscribe()
from pubsub import PubSub
app = web.Application()
app.router.add_static('/dist', './dist')
app.router.add_static('/static', './static')
app.router.add_get('/ws', WebSocketView)
app.router.add_get('/_debug', DebugView)
# This should go last since it wildcard matches on any route.
app.router.add_get('/{rest:.*}', RootView)
aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader('templates'))
services = {
'db': Database(app),
'ps': PubSub(app),
}
def get_sorted_services():
return sorted(services.values(), key=lambda s: s.init_order)
app['clients'] = {}
app.update(services)
async def on_startup(app):
for service in get_sorted_services():
await service.startup()
class PubSubFactory(protocol.ServerFactory):
protocol = PubSubProtocol
def __init__(self, messageTimeout,
subscriptionTimeout, messageFormat):
self.pubsub = PubSub(messageTimeout)
self.subscriptionTimeout = subscriptionTimeout
self.messageFormat = messageFormat
def subscribe(self, request):
urlComponents = urlparse.urlparse(request.uri)
query = urlparse.parse_qs(urlComponents.query)
if "channel" not in query:
self._reportError(request,
"expected 'channel' in the request")
return
channel = tryParseInt(query["channel"][0], 0)
if channel == 0:
return self._reportError(request,
"'channel' value should be number")
query.setdefault("time_from", [0])
timeFrom = tryParseInt(query["time_from"][0], 0)
query.setdefault("min_id", [0])
mid = tryParseInt(query["min_id"][0], 0)
notify = lambda message: self._notify(request,message)
print "parameters", timeFrom, mid
self.pubsub.subscribe(channel, notify,
request.finish,
timeFrom,
minId = mid,
timeoutSec = self.subscriptionTimeout)
# --- Utilities ---
def _notify(self, request, message):
try:
request.setResponseCode(200, "OK")
message = self._normalize(message)
print "MESSAGE: ", message
request.write(message)
request.finish()
except Exception as e:
print "Unhandled Error"
print e
def _normalize(self, messages):
if len(messages) == 0:
return self.messageFormat % ("[]", 0)
maxId = messages[-1][0]
messages = (m for id, m in messages)
messages = "[%s]" % ",".join(messages)
return self.messageFormat % (messages, maxId)
def _reportError(self, request, errorMessage):
request.setResponseCode(400, "error")
request.responseHeaders.addRawHeader("Content-Type",
"text/html")
request.write("<H1>Error</H1>")
request.write("<p>%s</p>" % errorMessage)
request.finish()
from wallet.transaction import Transaction
from wallet.transaction_pool import TransactionPool
from flask_cors import CORS
from blockchain.block import BlockHelper
from pubsub import PubSub
app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": "*"}})
# In future when app starts could broadcast a message saying node joined, then get a list of peers
blockchain = Blockchain()
wallet = Wallet(blockchain)
transaction_pool = TransactionPool()
pubsub = PubSub(blockchain, transaction_pool)
@app.route("/")
@app.route("/blockchain")
def get_blockchain():
return jsonify(blockchain.serialize())
@app.route("/blockchain/range")
def get_blockchain_range():
return jsonify(
blockchain.serialize()[::-1][
int(request.args.get("s")) : int(request.args.get("e"))
]
)