def __init__(self, api_name):
self.cmd = self.image_name = self.api_name = None
self.num_outstanding = self.mean_outstanding = self.timeout = 0
self.reset(api_name)
self.qdev = qdev = ThreadDevice(zmq.QUEUE, zmq.XREP, zmq.XREQ)
endpt_in, endpt_out = APIQueue.allocate_random_endpoints()
self.endpoints = endpt_in, endpt_out
qdev.bind_in(endpt_in)
qdev.bind_out(endpt_out)
if APIQueue._zmq_major_ver() > 2:
qdev.setsockopt_in(zmq.SNDHWM, APIQueue.BUFFER_SZ)
qdev.setsockopt_out(zmq.RCVHWM, APIQueue.BUFFER_SZ)
qdev.setsockopt_in(zmq.RCVHWM, APIQueue.BUFFER_SZ)
qdev.setsockopt_out(zmq.SNDHWM, APIQueue.BUFFER_SZ)
else:
qdev.setsockopt_in(zmq.HWM, APIQueue.BUFFER_SZ)
qdev.setsockopt_out(zmq.HWM, APIQueue.BUFFER_SZ)
qdev.start()
APIQueue.QUEUES[api_name] = self
self.log_debug("Created " + self.debug_str())
import tornado
from sockjs.tornado import SockJSConnection
import zmq
from zmq.eventloop.zmqstream import ZMQStream
from zmq.devices.basedevice import ThreadDevice
from ConfigParser import NoOptionError
import core
from config import config, save_config
import commands
import sys
context = zmq.Context()
## bind a forwarder here so multiple spools can connect without conflict
pr = ThreadDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
pr.bind_in("ipc:///tmp/progressIn.ipc")
pr.bind_out("ipc:///tmp/progressOut.ipc")
pr.setsockopt_in(zmq.SUBSCRIBE, "")
pr.start()
class Status(SockJSConnection):
remaining = 0
rTime = 0
pTime = 0
def on_open(self, info):
self.prog_socket = context.socket(zmq.SUB)
self.prog_socket.connect("ipc:///tmp/progressOut.ipc")
self.prog_socket.setsockopt(zmq.SUBSCRIBE, "")
def mpconnector(child_pipe, cfg, server_ts, mp_loggerqueue):
setproctitle("gzbx." + os.path.basename(__file__))
logger = mplogging.setup_logger(mp_loggerqueue, __file__)
logger.debug(whoami() + "starting mpconnector process")
sh = SigHandler_MPconnector(logger)
signal.signal(signal.SIGINT, sh.sighandler_mpconnector)
signal.signal(signal.SIGTERM, sh.sighandler_mpconnector)
# setup ports + zmq streamer device
worker_port = 37100
connections_port = worker_port + 1
while is_port_in_use(worker_port) or is_port_in_use(connections_port):
worker_port += 1
connections_port = worker_port + 1
# streamerdevice gathers results downloaded from usenet servers and
# keeps them ready to be delivered to mpconnector (which is the only worker).
# this is not working with ProcessDevice -> stuck on sys.exit!??
streamerdevice = ThreadDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % connections_port)
streamerdevice.bind_out("tcp://127.0.0.1:%d" % worker_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, b"PULL")
streamerdevice.setsockopt_out(zmq.IDENTITY, b"PUSH")
streamerdevice.start()
# setup socket for worker
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.connect("tcp://127.0.0.1:%d" % worker_port)
# poller to check if streamerdevice is empty
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
thr_articlequeue = deque()
ct = ConnectionThreads(cfg, thr_articlequeue, connections_port, server_ts, logger)
cmdlist = ("start", "stop", "pause", "resume", "reset_timestamps", "reset_timestamps_bdl",
"get_downloaded_per_server", "exit", "clearqueues", "connection_thread_health", "get_server_config",
"set_tmode_sanitycheck", "set_tmode_download", "get_level_servers", "clear_articlequeue",
"queues_empty", "clear_resultqueue", "len_articlequeue", "push_articlequeue", "pull_resultqueue",
"push_entire_articlequeue", "pull_entire_resultqueue", "get_bytesdownloaded")
quit_via_cmdexit = False
while not TERMINATED:
try:
cmd, param = child_pipe.recv()
except Exception as e:
logger.warning(whoami() + str(e))
continue
if cmd in cmdlist:
result = True
if cmd == "push_articlequeue":
try:
thr_articlequeue.append(param)
except Exception:
result = None
elif cmd == "push_entire_articlequeue":
try:
for article0 in param:
thr_articlequeue.append(article0)
except Exception:
result = None
elif cmd == "pull_entire_resultqueue":
result = get_from_streamingdevice(socket, onlyfirst=False)
logger.debug(whoami() + "len_articlequeue: " + str(len(ct.articlequeue)))
elif cmd == "pull_resultqueue":
result = get_from_streamingdevice(socket, onlyfirst=True)
elif cmd == "start":
ct.start_threads()
elif cmd == "queues_empty":
socks = dict(poller.poll(timeout=10))
streamerdevice_empty = True
if socket in socks and socks[socket] == zmq.POLLIN:
streamerdevice_empty = False
result = (len(ct.articlequeue) == 0) and streamerdevice_empty
elif cmd == "get_bytesdownloaded":
result = ct.get_bytesdownloaded()
elif cmd == "len_articlequeue":
try:
result = int(len(ct.articlequeue))
except Exception:
result = 0
elif cmd == "clear_articlequeue":
ct.articlequeue.clear()
elif cmd == "clear_resultqueue":
get_from_streamingdevice(socket, onlyfirst=False)
elif cmd == "stop":
ct.stop_threads()
elif cmd == "resume":
ct.resume_threads()
elif cmd == "pause":
ct.pause_threads()
elif cmd == "reset_timestamps":
ct.reset_timestamps()
elif cmd == "reset_timestamps_bdl":
ct.reset_timestamps_bdl()
elif cmd == "get_downloaded_per_server":
result = ct.get_downloaded_per_server()
elif cmd == "connection_thread_health":
result = ct.connection_thread_health()
elif cmd == "get_server_config":
result = ct.get_server_config()
elif cmd == "set_tmode_sanitycheck":
for t, _ in ct.threads:
t.mode = "sanitycheck"
elif cmd == "set_tmode_download":
for t, _ in ct.threads:
t.mode = "download"
elif cmd == "get_level_servers":
le_serv0 = []
try:
retention = param
for level, serverlist in ct.level_servers.items():
level_servers = serverlist
le_dic = {}
for le in level_servers:
_, _, _, _, _, _, _, _, age, _ = ct.servers.get_single_server_config(le)
le_dic[le] = age
les = [le for le in level_servers if le_dic[le] > retention * 0.9]
le_serv0.append(les)
except Exception:
pass
result = le_serv0
elif cmd == "exit":
quit_via_cmdexit = True
break
elif cmd == "clearqueues":
ct.articlequeue.clear()
get_from_streamingdevice(socket, onlyfirst=False)
child_pipe.send(result)
else:
child_pipe.send(None)
logger.debug(whoami() + "shutting down ...")
ct.stop_threads()
get_from_streamingdevice(socket, onlyfirst=False)
if quit_via_cmdexit:
child_pipe.send(result)
import tornado
from sockjs.tornado import SockJSConnection
import zmq
from zmq.eventloop.zmqstream import ZMQStream
from zmq.devices.basedevice import ThreadDevice
from ConfigParser import NoOptionError
import core
from config import config, save_config
import commands
import sys
context = zmq.Context()
## bind a forwarder here so multiple spools can connect without conflict
pr = ThreadDevice(zmq.FORWARDER, zmq.SUB, zmq.PUB)
pr.bind_in('ipc:///tmp/progressIn.ipc')
pr.bind_out('ipc:///tmp/progressOut.ipc')
pr.setsockopt_in(zmq.SUBSCRIBE, '')
pr.start()
class Status(SockJSConnection):
remaining = 0
rTime = 0
pTime = 0
def on_open(self, info):
self.prog_socket = context.socket(zmq.SUB)
self.prog_socket.connect('ipc:///tmp/progressOut.ipc')
self.prog_socket.setsockopt(zmq.SUBSCRIBE, '')
def main(global_config, **settings):
engine = engine_from_config(settings, 'sqlalchemy.')
Base.metadata.create_all(engine)
Request.set_db_engine(engine)
authentication_policy = AuthenticationPolicy(
realm=settings['authentication.realm'])
config = Configurator(settings=settings, request_factory=Request,
default_permission='admin',
authentication_policy=authentication_policy)
config.include(includeme)
log = logging.getLogger(__name__)
log.info("Starting zmq QUEUE (%s ==> |QUEUE| ==> %s)",
settings['zmq.queue_addr'], settings['zmq.daemon_addr'])
device = ThreadDevice(zmq.QUEUE, zmq.REP, zmq.REQ)
device.bind_in(settings['zmq.queue_addr'])
device.connect_out(settings['zmq.daemon_addr'])
device.setsockopt_in(zmq.IDENTITY, 'REP')
device.setsockopt_out(zmq.IDENTITY, 'REQ')
device.start()
return config.make_wsgi_app()
