def __init__(self, capacity):
self.capacity = capacity
self.pubsub = PubSub(CloudyException)
self.nodes = {
idx: CloudyGCounter(idx, self.pubsub)
for idx in range(capacity)
}
def demo():
'''Demo gCRDT'''
ps = PubSub()
nodes = [GCounter(idx, ps) for idx in range(3)]
for i in range(10):
node = random.choice(nodes)
node.increment()
for n in nodes:
print("Node %d says %d" % (n.idx, n.value()))
def test_integrated(self):
'''Test some nodes hooked up with a pub/sub'''
pubsub = PubSub()
nodes = [GCounter(idx, pubsub) for idx in range(3)]
for i in range(100):
# Pick a random node and increment it
node = random.choice(nodes)
node.increment()
# All .value() checks should have the same result
self.assertTrue(all(n.value() == node.value() for n in nodes))
def main():
import logging.config
logging.config.fileConfig('logging_config.ini',
disable_existing_loggers=False)
parser = argparse.ArgumentParser()
parser.add_argument('--port',
nargs='?',
type=int,
default=5556,
help='Port the server should listen on')
parser.add_argument('--pubsub',
nargs='?',
type=str,
default='tcp://143.215.207.2:3247',
help='PubSub address other camera should subscribe to')
parser.add_argument('--storage',
nargs='?',
type=str,
default='output',
help='Directory to store the video')
parser.add_argument('--cname',
nargs='?',
type=str,
default='ferst_state',
help='The name of the camera')
args = parser.parse_args()
try:
createEmptyDir(args.storage)
except Exception as e:
SLogger.fatal('Unable to create the output directory: %s' % e)
sys.exit(-1)
pubsub = PubSub(args.cname, args.pubsub, context)
SLogger.info('Successfully start pubsub service')
tgraph = TrajectoryGraph()
SLogger.info('Successfully connect to graph database')
time.sleep(5)
cand = []
sub_thread = threading.Thread(target=listen_candidates,
args=(pubsub, cand))
sub_thread.start()
matching_queue = queue.SimpleQueue()
match_thread = threading.Thread(target=matching,
args=(matching_queue, cand, tgraph))
match_thread.start()
listen(args.port, args.storage, pubsub, tgraph, matching_queue)
def main():
import logging.config
logging.config.fileConfig('logging_config.ini',
disable_existing_loggers=False)
args = arg_parse()
context = zmq.Context()
socket = context.socket(zmq.PAIR)
socket.bind('tcp://*:%s' % args.port)
# No clean up code
vstore = VideoStorageClient(args.video_storage_addr, context)
pubsub = PubSub(args.cname, args.pubsub, context)
tgraph = TrajectoryGraph()
vt = VehicleTracking()
pool = CandidatePool()
listen_thread = threading.Thread(target=listener_func, args=(pubsub, pool))
listen_thread.start()
coldstart(tgraph)
frame_id = 0
fps = FPS()
while True:
try:
rawimage, image, bboxes = parse_load(socket)
except Exception as e:
logging.warn("Unable to parse: exception %s" % e)
continue
tracked_bboxes = vt.sort_update(bboxes)
logging.info("Track result: %s" % tracked_bboxes)
frame_storage(vstore, args.cname, frame_id, rawimage, tracked_bboxes)
leaving_vehicles = vt.status_update(frame_id, image)
for vehicle in leaving_vehicles:
logging.info("Vehicle: %d is leaving" % vehicle.id)
hist = feature_extraction_adaptive_histogram(vehicle)
vertexid = vertex_storage(tgraph, args.cname, vehicle)
messaging(pubsub, vertexid, vehicle.id, hist)
res = pool.matching(hist, args.dis_thres)
logging.info("Re-Id for vehicle %d: %s" % (vehicle.id, res))
edge_storage(tgraph, res, vertexid)
frame_id += 1
logging.debug("FPS: %.2f" % fps())
cleanup()
def main():
import logging.config
logging.config.fileConfig('logging_config.ini', disable_existing_loggers=False)
parser = argparse.ArgumentParser()
parser.add_argument('--port', nargs='?', default=5556, help='Port, communication between RPi1 and RPi2')
parser.add_argument('--pubsub', nargs='?', default='tcp://143.215.207.2:3247', help='PubSub address other camera should subscribe to')
parser.add_argument("--video_storage_addr", nargs='?', default="tcp://130.207.122.57:1429", help="address of videoStorageServer")
parser.add_argument('--cname', required=True, help='The name of the camera')
args = parser.parse_args()
camera_name = args.cname
context = zmq.Context()
socket = context.socket(zmq.PAIR)
socket.bind('tcp://*:%s' % args.port)
vstore = VideoStorageClient(args.video_storage_addr, context)
pubsub = PubSub(args.cname, args.pubsub, context)
tgraph = TrajectoryGraph()
pool = CandidatePool()
listen_thread = threading.Thread(target=listener_func, args=(pubsub, pool))
listen_thread.start()
frame_id = 0
fps = FPS()
vt = VehicleTracking()
while True:
rawimage, image, bboxes = parse_load(socket)
tracked_bboxes = vt.sort_update(bboxes)
frame_storage(vstore, camera_name, frame_id, rawimage,
tracked_bboxes)
leaving_vehicles = vt.status_update(frame_id, image)
for vehicle in leaving_vehicles:
hist = feature_extraction_adaptive_histogram(vehicle)
vertexid = vertex_storage(tgraph, camera_name, vehicle)
messaging(pubsub, vertexid, vehicle.id, hist)
res = pool.matching(hist, args.dis_thres)
logging.info("Re-Id for vehicle %d: %s" % (vehicle.id, res))
edge_storage(tgraph, res, vertexid)
frame_id += 1
logging.info('fps: %.2f' % fps())
from flask import Flask, send_file
from flask_socketio import SocketIO
from pubsub import PubSub
import os
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
app.socket = SocketIO(app)
app.aio = PubSub()
app.publish = app.aio.publish
import routes
import socketio
@app.route('/<path:path>')
def static_file(path):
temp = os.path.dirname(os.path.realpath(__file__)) + '/public/' + path
return send_file(temp)
def handler(signum, frame):
logger.info('got term... raising TermException')
raise TermException()
signal.signal(signal.SIGTERM, handler)
def create_data():
logger.info("callback invoked")
return {"data": "here is some data from server"}
def make_query_callback(data):
logger.info('got request of: %s' % data)
ps.publish('return_result', create_data())
return
ps = PubSub('AWS', topic_arn=os.environ['AWS_SNS_ARN'], logger=logger)
make_query_subscription = ps.subscribe('make_query',
callback=make_query_callback)
try:
while True:
time.sleep(100)
except:
logger.info('terminating... dropping subscriber')
make_query_subscription.unsubscribe()
def pubsub():
app = PubSub("test", "test")
return app
# The MIT License (MIT) # Copyright (c) 2020 Jan Cespivo # octopusLAB pubsub example from pubsub import PubSub pubsub = PubSub(100) pubsub.start()
def main():
import logging.config
logging.config.fileConfig('logging_config.ini', disable_existing_loggers=False)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger('HttpUtil').setLevel(logging.INFO)
args = arg_parse()
if args.imageSeq is not None or args.live is not None:
target_labelIds = get_target_labelIds(args.labels)
engine = BasicEngine(args.model)
model_w, model_h, tensor_start_index = engine_info(engine)
else:
logging.fatal("No valid stream input source")
socket = None
context = None
if args.output is not None:
context = zmq.Context()
socket = contect.socket(zmq.PAIR)
socket.connect(args.output)
if args.imageSeq is not None:
stream = ImageSequenceStream(args.imageSeq)
elif args.live is not None:
stream = SingleCampusCameraStream(args.live, args.userconfig, args.cameraconfig)
stream.login()
logging.info("Successfully login into Campus Camera Stream --- %s" % args.live)
if context is None:
context = zmq.Context()
# No clean up code
vstore = VideoStorageClient(args.video_storage_addr, context)
pubsub = PubSub(args.cname, args.pubsub, context)
tgraph = TrajectoryGraph()
vt = VehicleTracking()
pool = CandidatePool()
listen_thread = threading.Thread(target=listener_func, args=(pubsub, pool))
listen_thread.start()
coldstart(tgraph)
def cleanup():
if args.live is not None:
stream.logout()
if socket is not None:
socket.close()
context.term()
def signal_handler(sig, frame):
cleanup()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
frame_id = 0
fps = FPS()
while True:
try:
frame = stream.fetch_frame()
except:
break
image = load_frame(frame)
image_w, image_h = image.size
resized_image = resize_frame(image, model_w, model_h)
raw_result = inference(engine, resized_image)
bboxes = post_inference(raw_result,
tensor_start_index,
target_labelIds,
args.threshold,
args.top_k, image_w, image_h)
logging.info("Detection result: %s" % bboxes)
tracked_bboxes = vt.sort_update(bboxes)
logging.info("Track result: %s" % tracked_bboxes)
frame_storage(vstore, args.cname, frame_id, frame, tracked_bboxes)
opencv_image = load_opencv_PIL(image)
leaving_vehicles = vt.status_update(frame_id, opencv_image)
for vehicle in leaving_vehicles:
logging.info("Vehicle: %d is leaving" % vehicle.id)
hist = feature_extraction_adaptive_histogram(vehicle)
vertexid = vertex_storage(tgraph, args.cname, vehicle)
messaging(pubsub, vertexid, vehicle.id, hist)
res = pool.matching(hist, args.dis_thres)
logging.info("Re-Id for vehicle %d: %s" % (vehicle.id, res))
edge_storage(tgraph, res, vertexid)
frame_id += 1
logging.debug("FPS: %.2f" % fps())
cleanup()
from wallet.transaction import Transaction
from wallet.transaction_pool import TransactionPool
from flask_cors import CORS
from blockchain.block import BlockHelper
from pubsub import PubSub
app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": "*"}})
# In future when app starts could broadcast a message saying node joined, then get a list of peers
blockchain = Blockchain()
wallet = Wallet(blockchain)
transaction_pool = TransactionPool()
pubsub = PubSub(blockchain, transaction_pool)
@app.route("/")
@app.route("/blockchain")
def get_blockchain():
return jsonify(blockchain.serialize())
@app.route("/blockchain/range")
def get_blockchain_range():
return jsonify(
blockchain.serialize()[::-1][
int(request.args.get("s")) : int(request.args.get("e"))
]
)
def namespaced_pubsub():
app = PubSub("test", "test", namespace=_NAMESPACE)
return app
from pubsub import PubSub
app = web.Application()
app.router.add_static('/dist', './dist')
app.router.add_static('/static', './static')
app.router.add_get('/ws', WebSocketView)
app.router.add_get('/_debug', DebugView)
# This should go last since it wildcard matches on any route.
app.router.add_get('/{rest:.*}', RootView)
aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader('templates'))
services = {
'db': Database(app),
'ps': PubSub(app),
}
def get_sorted_services():
return sorted(services.values(), key=lambda s: s.init_order)
app['clients'] = {}
app.update(services)
async def on_startup(app):
for service in get_sorted_services():
await service.startup()
# run in its own thread. Feeds post a specified number of messages, waiting a
# random interval between messages. Each inbox is polled for messages received,
# terminating when no messages are received for a wait limit.
import random
import threading
import time
import fdb
from pubsub import PubSub
fdb.api_version(22)
db = fdb.open()
ps = PubSub(db)
ps.clear_all_messages()
# Create the specified numbers of feeds and inboxes. Subscribe each inbox to a
# randomly selected subset of feeds.
def setup_topology(feeds, inboxes):
feed_map = {f: ps.create_feed('Alice ' + str(f)) for f in range(feeds)}
inbox_map = {}
for i in range(inboxes):
inbox_map[i] = ps.create_inbox('Bob ' + str(i))
for f in random.sample(xrange(feeds), random.randint(1, feeds)):
ps.create_subscription(inbox_map[i], feed_map[f])
return feed_map, inbox_map
# The MIT License (MIT) # Copyright (c) 2020 Jan Cespivo # octopusLAB pubsub example from pubsub import PubSub pubsub = PubSub()
def pubsub_client(self):
return PubSub(self)
from celery import Celery
from pubsub import PubSub
app = Celery()
pubsub_app = PubSub('amqp://guest@localhost', 'sample.topic')
@app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
sender.add_periodic_task(3.0, pubsub_listen, name='Consume messages')
@pubsub_app.subscribe('booking.*')
def process_booking_event(body, _):
print(body['object'])
@app.task
def pubsub_listen():
pubsub_app.drain()
return "Done!"
