def __init__(self, url, auth=None, workers=20, agent_id=None,
queue_depth=Config.queue_depth()):
if url.endswith("/schemas"):
url = url[0:len(url)-len("/schemas")]
self._url = url + "/subscribe"
self._auth = auth
self._workers = int(workers)
self._children = []
self._agent_id = agent_id
self._queue = Queue(queue_depth)
self._ping_queue = Queue(queue_depth)
type_manager.register_type(type_manager.PUBLISHER,
Publisher(url + "/publish", auth))
class EventClient:
def __init__(self, url, auth=None, workers=20, agent_id=None,
queue_depth=Config.queue_depth()):
if url.endswith("/schemas"):
url = url[0:len(url)-len("/schemas")]
self._url = url + "/subscribe"
self._auth = auth
self._workers = int(workers)
self._children = []
self._agent_id = agent_id
self._queue = Queue(queue_depth)
self._ping_queue = Queue(queue_depth)
type_manager.register_type(type_manager.PUBLISHER,
Publisher(url + "/publish", auth))
def _start_children(self):
pid = os.getpid()
for i in range(self._workers):
p = spawn(target=_worker, args=('worker{0}'.format(i),
self._queue, pid))
self._children.append(p)
p = spawn(target=_worker, args=('ping', self._ping_queue, pid))
self._children.append(p)
def run(self, events):
_check_ts()
run(self._run, events)
def _run(self, events):
ppid = os.environ.get("AGENT_PARENT_PID")
headers = {}
args = {
"data": _data(events, self._agent_id),
"stream": True,
"headers": headers,
"timeout": Config.event_read_timeout()
}
if self._auth is not None:
if isinstance(self._auth, basestring):
headers["Authorization", self._auth]
else:
args["auth"] = self._auth
try:
drop_count = 0
ping_drop = 0
r = requests.post(self._url, **args)
if r.status_code != 201:
raise Exception("{} : {}".format(r.status_code, r.text))
self._start_children()
for line in r.iter_lines(chunk_size=512):
line = line.strip()
try:
ping = '"ping' in line
if len(line) > 0:
# TODO Need a better approach here
if ping:
self._ping_queue.put(line, block=False)
ping_drop = 0
else:
self._queue.put(line, block=False)
except Full:
log.info("Dropping request %s" % line)
drop_count += 1
max = Config.max_dropped_requests()
if ping:
ping_drop += 1
max = Config.max_dropped_ping()
if drop_count > max:
log.error('Max dropped requests [%s] exceeded', max)
break
if not _should_run(ppid):
log.info("Parent process has died or stamp changed,"
" exiting")
break
finally:
for child in self._children:
if hasattr(child, "terminate"):
try:
child.terminate()
except:
pass
sys.exit(0)
class EventClient:
def __init__(self, url, auth=None, workers=20, agent_id=None,
queue_depth=Config.queue_depth()):
if url.endswith("/schemas"):
url = url[0:len(url)-len("/schemas")]
self._url = url + "/subscribe"
self._auth = auth
self._workers = int(workers)
self._children = []
self._agent_id = agent_id
self._queue = Queue(queue_depth)
self._ping_queue = Queue(queue_depth)
type_manager.register_type(type_manager.PUBLISHER,
Publisher(url + "/publish", auth))
def _start_children(self):
pid = os.getpid()
for i in range(self._workers):
p = spawn(target=_worker, args=('worker{0}'.format(i),
self._queue, pid))
self._children.append(p)
p = spawn(target=_worker, args=('ping', self._ping_queue, pid))
self._children.append(p)
def run(self, events):
_check_ts()
run(self._run, events)
def _run(self, events):
ppid = os.environ.get("AGENT_PARENT_PID")
headers = []
if self._auth is not None:
auth_header = 'Authorization: Basic ' + base64.b64encode(
('%s:%s' % self._auth).encode('latin1')).strip()
headers.append(auth_header)
subscribe_url = self._url.replace('http', 'ws')
query_string = _events_query_string(events, self._agent_id)
subscribe_url = subscribe_url + '?' + query_string
try:
drops = {
'drop_count': 0,
'ping_drop': 0,
}
self._start_children()
def on_message(ws, message):
line = message.strip()
try:
ping = '"ping' in line
if len(line) > 0:
# TODO Need a better approach here
if ping:
self._ping_queue.put(line, block=False)
drops['ping_drop'] = 0
else:
self._queue.put(line, block=False)
except Full:
log.info("Dropping request %s" % line)
drops['drop_count'] += 1
drop_max = Config.max_dropped_requests()
drop_type = 'overall'
drop_test = drops['drop_count']
if ping:
drops['ping_drop'] += 1
drop_type = 'ping'
drop_test = drops['ping_drop']
drop_max = Config.max_dropped_ping()
if drop_test > drop_max:
log.error('Max of [%s] dropped [%s] requests exceeded',
drop_max, drop_type)
ws.close()
if not _should_run(ppid):
log.info("Parent process has died or stamp changed,"
" exiting")
ws.close()
def on_error(ws, error):
raise Exception('Received websocket error: [%s]', error)
def on_close(ws):
log.info('Websocket connection closed.')
def on_open(ws):
log.info('Websocket connection opened')
ws = websocket.WebSocketApp(subscribe_url,
header=headers,
on_message=on_message,
on_error=on_error,
on_close=on_close,
on_open=on_open)
ws.run_forever()
finally:
for child in self._children:
if hasattr(child, "terminate"):
try:
child.terminate()
except:
pass
sys.exit(0)
class EventClient:
def __init__(self, url, auth=None, workers=20, agent_id=None,
queue_depth=Config.queue_depth()):
if url.endswith("/schemas"):
url = url[0:len(url)-len("/schemas")]
self._url = url + "/subscribe"
self._auth = auth
self._workers = int(workers)
self._children = []
self._agent_id = agent_id
self._queue = Queue(queue_depth)
self._ping_queue = Queue(queue_depth)
type_manager.register_type(type_manager.PUBLISHER,
Publisher(url + "/publish", auth))
def _start_children(self):
pid = os.getpid()
for i in range(self._workers):
p = spawn(target=_worker, args=(self._queue, pid))
self._children.append(p)
p = spawn(target=_worker, args=(self._ping_queue, pid))
self._children.append(p)
def run(self, events):
_check_ts()
run(self._run, events)
def _run(self, events):
ppid = os.environ.get("AGENT_PARENT_PID")
headers = {}
args = {
"data": _data(events, self._agent_id),
"stream": True,
"headers": headers,
"timeout": Config.event_read_timeout()
}
if self._auth is not None:
if isinstance(self._auth, basestring):
headers["Authorization", self._auth]
else:
args["auth"] = self._auth
try:
drop_count = 0
ping_drop = 0
r = requests.post(self._url, **args)
if r.status_code != 201:
raise Exception(r.text)
self._start_children()
for line in r.iter_lines(chunk_size=1):
try:
ping = '"ping' in line
if len(line) > 0:
# TODO Need a better approach here
if ping:
self._ping_queue.put(line, block=False)
ping_drop = 0
else:
self._queue.put(line, block=False)
except Full:
log.info("Dropping request %s" % line)
drop_count += 1
max = Config.max_dropped_requests()
if ping:
ping_drop += 1
max = Config.max_dropped_ping()
if drop_count > max:
log.error('Max dropped requests [%s] exceeded', max)
break
if not _should_run(ppid):
log.info("Parent process has died or stamp changed,"
" exiting")
break
finally:
for child in self._children:
if hasattr(child, "terminate"):
try:
child.terminate()
except:
pass
sys.exit(0)