def _listenhandler(self):
try:
sock = self.listen
# client_socket
client_socket, address = sock.accept()
client_socket.setblocking(False)
# sock = gsocket(_socket = client_socket)
except asy_socket.error as error:
TCLogger.warning("accept:%s", error)
return
TCLogger.info("new connect fd:%s ", client_socket.fileno())
client_in_event = self.loop.io(client_socket.fileno(), 1)
client_out_event = self.loop.io(client_socket.fileno(), 2)
client = self.Client(client_socket, client_in_event, client_out_event)
client_in_event.start(self.handleClientRead, client)
## say hello
if self.say_hello_cb is not None:
self.say_hello_cb(client)
## (socket,read_cv,write_cb)
self.clients[client_socket.fileno()] = client
def sendData(self, data):
try:
self.__socket.send(data)
except asy_socket.error as e:
if e.errno in (errno.EINPROGRESS, errno.EWOULDBLOCK):
pass
else:
TCLogger.warning("connect:%s error:%s" % (self.remote, str(e)))
def start(self):
try:
## start
self._connectPeerWithHello(self.remote)
except Timeout as t:
TCLogger.warning("remote:%s time out state:%d", str(self.remote),
self.state)
self.close()
def _generator_PstatMessage(self):
try:
while self.task_running:
ps = PStatMessage(agentStat=self._generAgentStat())
TCLogger.debug(ps)
yield ps
time.sleep(STAT_INTERVAL)
except Exception as e:
TCLogger.warning("_generator_PstatMessage catch %s", e)
def run(self):
while True:
try:
gevent.sleep(10)
except:
break
# break by a signal
self.app_management.stopAll()
self.php_agent.stop()
TCLogger.warning("collector agent exit with SIGNAL")
def _pingPPing(self):
while self.task_running:
ping = PPing()
TCLogger.debug("%s send ping", self)
yield ping
with self.exit_cv:
if not self.task_running or self.exit_cv.wait(
self.timeout * 10):
TCLogger.warning("generate ping exit")
break
def _stat_thread_main(self):
self.task_running = True
while self.task_running:
try:
self.meta_stub.SendAgentStat(self._generator_PstatMessage())
except Exception as e:
TCLogger.warning("SendAgentStat met:%s", e)
with self.exit_cv:
if self.exit_cv.wait(self.timeout):
break
def start(self):
self.__socket = asy_socket.socket(asy_socket.AF_INET,asy_socket.SOCK_DGRAM)
self.__socket.setblocking(False)
try:
self.state = E_CONNECTED
self.__socket.connect(self.remote)
except asy_socket.error as e:
if not (e.errno in (errno.EINPROGRESS, errno.EWOULDBLOCK)):
TCLogger.warning("connect:%s error:%s" % (self.remote, str(e)))
self.__socket.close()
raise Exception("create DgramLayer failed")
def close(self):
TCLogger.warning("close connection (%s)", str(self.remote))
self._read_watcher.stop()
self._write_watcher.stop()
self.timer.stop()
self.socket.close()
self.state = E_CLOSED
## reset send buffer
self.send_buf = b''
self.timer = self.loop.timer(self.timeout)
self.timer.start(self._connectPeerWithHello, self.remote)
def sendSpan(self, spanMesg):
try:
self.span_queue.put(spanMesg, False)
except Full as e:
self.dropped_span_count += 1
TCLogger.warning("span send queue is full")
return False
except Exception as e:
TCLogger.error("send span failed: %s", e)
return False
return True
def findApp(self, app_id, app_name, service_type):
if app_id in self.app_map:
app = self.app_map[app_id]
## check app_name
if app.app_name != app_name:
TCLogger.warning(" app_name can't change when using ")
app = self.default_app
## check service_type
else:
TCLogger.info("collector-agent try to create a new application agent.[%s@%s@%d]",app_id,app_name,service_type)
app = self.collector_conf.collector_implement(self.collector_conf, app_id, app_name,service_type)
app.start()
self.app_map[app_id] = app
return app
def _recvFromPeer(self):
recv_buf = self.pRBufStart[self.rest_data_sz:]
recv_total = self.rest_data_sz
assert recv_total < self.RECV_BUF_SIZE
try:
recv_total = self.socket.recv_into(
recv_buf, self.RECV_BUF_SIZE - recv_total)
if recv_total == 0:
TCLogger.warning("peer close socket")
return -1
except asy_socket.error as error:
if error.errno in [asy_socket.EAGAIN, asy_socket.EWOULDBLOCK]:
return 0
TCLogger.warning("peer error:%s ", error)
return -1
return recv_total + self.rest_data_sz
def _recvFromPeer(self):
recv_buf = self.pRBufStart[self.rest_data_sz:]
recv_total = self.rest_data_sz
assert recv_total < self.RECV_BUF_SIZE
try:
recv_total = self.socket.recv_into(recv_buf,
self.RECV_BUF_SIZE - recv_total)
if recv_total == 0:
TCLogger.warning("peer: [%s:%d] closed", self.peer[0],
self.peer[1])
return -1
except asy_socket.error as error:
if error.errno in [errno.EAGAIN]:
return 0
TCLogger.warning("peer error:%s ", error)
return -1
return recv_total
def handleClientRead(self, client):
'''
:param self.Client client:
:return:
'''
while True:
len = client.recv(self.in_msg_cb)
if len > 0: ## read again
continue
if len < 0:
TCLogger.warning(" client:%d disconnected",
client.socket.fileno())
fd = client.socket.fileno()
client.close()
del self.clients[fd]
break
def recv(self, in_msg_cb):
'''
:param self.Client client:
:return: if rLen < 0 error happens
'''
rLen = self._recvFromPeer()
if rLen > 0:
self.rest_data_sz = in_msg_cb(self, self.pRBufStart, rLen)
if self.rest_data_sz >= self.RECV_BUF_SIZE:
TCLogger.warning("data from peer is overflow size=%d",
self.rest_data_sz)
return -1
if self.rest_data_sz > 0:
self.pRBufStart[0:self.rest_data_sz] = self.pRBufStart[
rLen - self.rest_data_sz:rLen]
return rLen
def _connectPeerAsy(self, addr, time):
self.socket = asy_socket.socket(asy_socket.AF_INET,
asy_socket.SOCK_STREAM)
self.socket.setblocking(False)
self._read_watcher = self.loop.io(self.socket.fileno(), 1)
self._write_watcher = self.loop.io(self.socket.fileno(), 2)
try:
self.state = E_CONNETING
self.socket.connect(addr)
except asy_socket.error as e:
if not (e.errno in (errno.EINPROGRESS, errno.EWOULDBLOCK)):
TCLogger.warning("connect:%s error:%s" % (addr, str(e)))
self.close()
return False
self.timer = self.loop.timer(self.timeout)
self.timer.start(self._actionTimeOut, time)
self._write_watcher.start(self._connected, addr)
return True
def generator_cmd():
i = 0
while self.task_running:
try:
cmd_response = PCmdStreamResponse(responseId=requestId,
sequenceId=i)
cmd_response.message.value = 'hello'
threadCountRes = PCmdActiveThreadCountRes(
commonStreamResponse=cmd_response)
threadCountRes.histogramSchemaType = 2
for stat in self.get_req_stat():
threadCountRes.activeThreadCount.append(stat)
threadCountRes.timeStamp = int(time.time())
i += 1
yield threadCountRes
## it's a templated task, don't use exit_cv
time.sleep(1)
except Exception as e:
TCLogger.warning("catch exception %s", e)
break
def handleFrontAgentData(self, client, type, body):
content = body.decode('utf-8')
try:
TCLogger.debug(content)
stack = json.loads(content)
except Exception as e:
TCLogger.error("json is crash")
return
if 'appid' not in stack:
TCLogger.warning(" drop %s, as no appid", content)
return
else:
appid = stack['appid']
if 'appname' not in stack:
TCLogger.warning(" drop %s, as no appname", content)
return
else:
appname = stack['appname']
if 'FT' not in stack:
TCLogger.warning(" drop %s, as no FT", content)
return
else:
ft = stack['FT']
app = self.findApp(appid, appname, ft)
app.asynSendSpan(stack, body)
self.recv_count += 1
def generator_cmd():
i = 0
while True:
try:
cmd_response = PCmdStreamResponse(responseId=requestId,
sequenceId=i)
cmd_response.message.value = 'hello'
threadCountRes = PCmdActiveThreadCountRes(
commonStreamResponse=cmd_response)
threadCountRes.histogramSchemaType = 2
for stat in self.get_req_stat():
threadCountRes.activeThreadCount.append(stat)
threadCountRes.timeStamp = int(time.time())
i += 1
yield threadCountRes
with self.exit_cv:
if self.exit_cv.wait(1):
break
except Exception as e:
TCLogger.warning("catch exception %s", e)
break
def _registerAllMeta(self, sec):
TCLogger.info("register all meta data")
while self.snd_task_is_running:
try:
for key, value in self.sql_table.items():
self.meta_stub.RequestSqlMetaData(value[1])
# api
for key, value in self.api_table.items():
self.meta_stub.RequestApiMetaData(value[1])
# string
for key, value in self.string_table.items():
self.meta_stub.RequestStringMetaData(value[1])
self.snd_task_is_running = False
TCLogger.info("send all meta data is done")
break
except Exception as e:
TCLogger.warning("re-send all meta data, met exception:%s", e)
with self.exit_cv:
if not self.snd_task_is_running or self.exit_cv.wait(sec):
break
def findApp(self, app_id, app_name, service_type):
if app_id in self.app_map:
app = self.app_map[app_id]
## check app_name
if app.app_name != app_name:
TCLogger.warning(" app_name can't change when using ")
app = self.default_app
## check service_type
else:
if service_type == PHP:
TCLogger.info(
"collector-agent try to create a new application agent.[%s@%s]",
app_id, app_name)
app = self.collector_conf.collector_implement(
self.collector_conf, app_id, app_name)
app.start()
self.app_map[app_id] = app
else:
raise NotImplementedError("service_type:%d not support",
service_type)
return app
def _response(self, future):
if future.exception():
TCLogger.warning("register meta failed %s", future.exception)
return
self.must_snd_meta_now = False
