def getAndSendSpan(self):
self.task_running = True
spans = []
def get_n_span(queue, N):
i = 0
try:
while N > i:
spans.append(queue.get(timeout=0.005))
i += 1
except Empty as e:
pass
return True if i > 0 else False
client = GrpcSpan.SpanClient(self.address, self.meta)
while self.task_running:
try:
if not get_n_span(self.queue, 10240):
with self.exit_cv:
if not self.task_running or self.exit_cv.wait(5):
TCLogger.info("span send thread is done")
break
continue
self.send_span_count += len(spans)
client.sendSpans(iter(spans))
TCLogger.debug("send %d", self.send_span_count)
except Exception as e:
TCLogger.error("span channel catches an exception:%s", e)
client.reconnect()
finally:
spans.clear()
def _send_thread_count(self, requestId):
channel = grpc.insecure_channel(self.address)
stub = Service_pb2_grpc.ProfilerCommandServiceStub(channel)
def generator_cmd():
i = 0
while self.task_running:
try:
cmd_response = PCmdStreamResponse(responseId=requestId,
sequenceId=i)
cmd_response.message.value = 'hello'
threadCountRes = PCmdActiveThreadCountRes(
commonStreamResponse=cmd_response)
threadCountRes.histogramSchemaType = 2
for stat in self.get_req_stat():
threadCountRes.activeThreadCount.append(stat)
threadCountRes.timeStamp = int(time.time())
i += 1
yield threadCountRes
## it's a templated task, don't use exit_cv
time.sleep(1)
except Exception as e:
TCLogger.warning("catch exception %s", e)
break
try:
TCLogger.debug("new a thread for activeThreadCound %d", requestId)
stub.CommandStreamActiveThreadCount(generator_cmd(),
metadata=self.profile_meta)
TCLogger.debug("send activeThreadCound requestId: %d is done",
requestId)
channel.close()
except Exception as e:
TCLogger.error(
"CommandStreamActiveThreadCount, catch exception %s", e)
def handleFrontAgentData(self, client, type, body):
content = body.decode('utf-8')
try:
TCLogger.debug(content)
stack = json.loads(content)
except Exception as e:
TCLogger.error("json is crash")
return
if 'appid' not in stack:
TCLogger.warning(" drop %s, as no appid", content)
return
else:
appid = stack['appid']
if 'appname' not in stack:
TCLogger.warning(" drop %s, as no appname", content)
return
else:
appname = stack['appname']
if 'FT' not in stack:
TCLogger.warning(" drop %s, as no FT", content)
return
else:
ft = stack['FT']
app = self.findApp(appid, appname, ft)
app.asynSendSpan(stack, body)
self.recv_count += 1
def loopTheQueue(self):
while True:
try:
body = self.mpQueue.get()
except KeyboardInterrupt:
break
if body == None:
TCLogger.info("agent: %s stopping", self.agent_meta)
break
else:
content = body.decode('utf-8')
try:
TCLogger.debug(content)
stack = json.loads(content)
except Exception as e:
TCLogger.error("json is crash")
return
super().sendSpan(stack, body)
try:
pSpan = self.span_factory.makeSpan(stack)
spanMesg = PSpanMessage(span=pSpan)
except Exception as e:
TCLogger.warn("interrupted by %s", e)
continue
self.span_helper.sendSpan(spanMesg)
def _send_thread_count(self, requestId):
channel = grpc.insecure_channel(self.address)
stub = Service_pb2_grpc.ProfilerCommandServiceStub(channel)
def generator_cmd():
i = 0
while True:
try:
cmd_response = PCmdStreamResponse(responseId=requestId,
sequenceId=i)
cmd_response.message.value = 'hello'
threadCountRes = PCmdActiveThreadCountRes(
commonStreamResponse=cmd_response)
threadCountRes.histogramSchemaType = 2
for stat in self.get_req_stat():
threadCountRes.activeThreadCount.append(stat)
threadCountRes.timeStamp = int(time.time())
i += 1
yield threadCountRes
with self.exit_cv:
if self.exit_cv.wait(1):
break
except Exception as e:
TCLogger.warning("catch exception %s", e)
break
try:
stub.CommandStreamActiveThreadCount(generator_cmd(),
metadata=self.profile_meta)
TCLogger.debug("send req state requestId: %d done", requestId)
except Exception as e:
TCLogger.error(
"CommandStreamActiveThreadCount, catch exception %s", e)
def _registerAgent(self):
while self.task_running:
try:
TCLogger.debug("sending agentinfo %s", self.agentinfo)
self.stub.RequestAgentInfo(self.agentinfo)
except Exception as e:
TCLogger.error(
" pinpoint collector is not available. Try it again [%s] ",
self.agentinfo)
continue
finally:
with self.exit_cv:
if not self.task_running or self.exit_cv.wait(
self.timeout):
break
iter_response = self.stub.PingSession(self._pingPPing(),
metadata=self.ping_meta)
try:
for response in iter_response:
TCLogger.debug('get ping response:%s agentinfo:%s',
response, self.meta)
except Exception as e:
TCLogger.error("ping response abort with exception: [%s] %s",
self.agentinfo, e)
TCLogger.info('agent thread exit: %s', self.task_running)
def _handleCommand(self):
handshake = PCmdServiceHandshake()
handshake.supportCommandServiceKey.append(PCommandType.PING)
handshake.supportCommandServiceKey.append(PCommandType.PONG)
handshake.supportCommandServiceKey.append(
PCommandType.ACTIVE_THREAD_COUNT)
handshake.supportCommandServiceKey.append(
PCommandType.ACTIVE_THREAD_DUMP)
handshake.supportCommandServiceKey.append(
PCommandType.ACTIVE_THREAD_LIGHT_DUMP)
cmd = PCmdMessage(handshakeMessage=handshake)
while self.task_running:
self.cmd_pipe = GrpcAgent.HandStreamIterator(cmd)
# while self.task_running:
msg_iter = self.cmd_sub.HandleCommand(self.cmd_pipe,
metadata=self.profile_meta)
try:
for msg in msg_iter:
TCLogger.debug("command channel %s", msg)
self._handleCmd(msg, self.cmd_pipe)
TCLogger.debug('iter_response is over')
except Exception as e:
TCLogger.error("handleCommand channel %s error", e)
finally:
with self.exit_cv:
if self.exit_cv.wait(self.timeout):
break
def stop(self):
TCLogger.debug("agent thread try to stop")
self.task_running = False
with self.exit_cv:
self.exit_cv.notify_all()
self.agent_thread.join()
super().stop()
TCLogger.debug("agent thread exit")
def sendMeta(self, meta):
'''
:param APIMeta meta:
:return:
'''
TCLogger.debug("meta: %s", meta.name)
self.collector_cb(meta.toPacket().getSerializedData())
def toPacket(self):
stringMeta = TStringMetaData(agentId=self.agentId,
agentStartTime=self.agentStartTime,
stringId=self.apiId,
stringValue=self.value)
TCLogger.debug("generator stringMate:%s", stringMeta)
body = CollectorPro.obj2bin(stringMeta, STRING_META_DATA)
packet = Packet(PacketType.APPLICATION_REQUEST, CollectorPro.getCurReqCount(), len(body), body)
return packet
def _generator_PstatMessage(self):
try:
while self.task_running:
ps = PStatMessage(agentStat=self._generAgentStat())
TCLogger.debug(ps)
yield ps
time.sleep(STAT_INTERVAL)
except Exception as e:
TCLogger.warning("_generator_PstatMessage catch %s", e)
def sendData(self, outData=None):
if self.state in (E_CLOSED, E_CONNETING):
# self.write_wait_queue.append(data)
TCLogger.debug("current state is:%d, data drop", self.state)
else:
if outData is not None:
self.send_buf += outData
self._write_watcher.start(self._sendData)
def _channelStateChange(self, activity):
if activity == grpc.ChannelConnectivity.TRANSIENT_FAILURE:
self.channelSetError()
elif activity == grpc.ChannelConnectivity.READY:
self.channelSetReady()
elif activity == grpc.ChannelConnectivity.IDLE:
self.channelSetIdle()
TCLogger.debug("channel state change %s dst:%s", activity,
self.address)
def _pingPPing(self):
while self.task_running:
ping = PPing()
TCLogger.debug("%s send ping", self)
yield ping
with self.exit_cv:
if self.exit_cv.wait(self.timeout):
TCLogger.debug("generate ping exit")
break
def toPacket(self):
apiMeta = TApiMetaData(agentId=self.agentId,
agentStartTime=self.agentStartTime,
apiId=self.apiId,
type=self.type,
apiInfo=self.name)
TCLogger.debug("generator ThriftAPIMeta:%s", apiMeta)
body = CollectorPro.obj2bin(apiMeta, API_META_DATA)
packet = Packet(PacketType.APPLICATION_REQUEST, CollectorPro.getCurReqCount(), len(body), body)
return packet
def close(self):
TCLogger.debug("close client:%d(%s)", self.socket.fileno(),
self.address)
if self._read_watcher is not None:
self._read_watcher.stop()
if self._write_watcher is not None:
self._write_watcher.stop()
self.socket.close()
def updateSqlMeta(self, sql):
if sql in self.sql_table:
return self.sql_table[sql][0]
else:
id = self.id
meta = PSqlMetaData(sqlId=id, sql=sql)
self.sql_table[sql] = [id, meta]
self._sendSqlMeta(meta)
self.id += 1
TCLogger.debug("register sql meta id:%d -> sql:[%s]", id, sql)
return id
def updateStringMeta(self, value):
if value in self.string_table:
return self.string_table[value][0]
else:
id = self.id
meta = PStringMetaData(stringId=id, stringValue=value)
self.string_table[value] = [id, meta]
self._sendStringMeta(meta)
self.id += 1
TCLogger.debug("register string meta id:%d -> value:[%s]", id,
value)
return id
def _phpClientSayHello(self, client):
response = {}
for cb in self.hello_callback:
data = cb()
for key in data:
response[key] = data[key]
data = json.dumps(response)
buf = struct.pack('!ii', RESPONSE_AGENT_INFO,
len(data)) + data.encode()
client.sendData(buf)
TCLogger.debug("send hello:%d len:%d", client.socket.fileno(),
len(buf))
def _sendData(self):
try:
ret = self.socket.send(self.send_buf)
except asy_socket.error as e:
TCLogger.error("_sendData %s", e)
self.close()
return
### not ready try next time
if ret != len(self.send_buf):
self.send_buf = self.send_buf[ret:]
# self.sendData()
return
TCLogger.debug("send %d", ret)
self._write_watcher.stop()
def updateApiMeta(self, apiInfo, line, type):
row_str = ("%s-%d-%d" % (apiInfo, line, type))
if row_str in self.api_table:
return self.api_table[row_str][0]
else:
id = self.id
meta = PApiMetaData(apiId=id,
apiInfo=apiInfo,
line=line,
type=type)
self.api_table[row_str] = [id, meta]
self._sendApiMeta(meta)
self.id += 1
TCLogger.debug("register api meta id:%d -> api:[%s]", id, row_str)
return id
def __init__(self, agentId, startTimeStamp, host):
self.state = TAgentStat()
self.cup_load = TCpuLoad()
self.state.agentId = agentId
self.state.startTimestamp = startTimeStamp
self.stateBatch = TAgentStatBatch()
self.stateBatch.agentId = agentId
self.stateBatch.startTimestamp = startTimeStamp
self.stateBatch.agentStats = []
self.remote = host
self.trans_layer = DgramLayer(host, None)
self.trans_layer.start()
TrainLayer.registerTimers(self.sendState, 20)
TCLogger.debug("register state timer")
def _sendData(self):
try:
ret = self.socket.send(self.send_buf)
except asy_socket.error as e:
TCLogger.error("_sendData %s", e)
self.close()
return
if ret != len(self.send_buf):
self.send_buf = self.send_buf[ret:]
self.sendData()
return
TCLogger.debug("send %d", ret)
## write is done
self.state &= (~E_WRITING)
self._write_watcher.stop()
self.send_buf = b''
def handleFrontAgentData(self, client, type, body):
content = body.decode('utf-8')
try:
TCLogger.debug(content)
stack = json.loads(content)
except Exception as e:
TCLogger.error("json is crash")
return
if 'appid' not in stack:
appid = self.default_appid
else:
appid = stack['appid']
if 'appname' not in stack:
appname = self.default_appname
else:
appname = stack['appname']
ft = stack['FT']
app = self.findApp(appid, appname, ft)
app.sendSpan(stack,body)
self.recv_count+=1
def _sendSpan(self, spanMsg):
self.span_sender_list[0].sendSpan(spanMsg)
TCLogger.debug(spanMsg)
return True
def _test_hello_cb(n_layer):
TCLogger.debug("send hello")
n_layer.sendData(b"GET index.html \r\n\r\n")
def stop(self):
self.server.stop()
TCLogger.debug("close listen_socket:%d[%s]",
self.listen_socket.fileno(), self.address)
self.listen_socket.close()
def sendState(self, layer):
self._upDateCurState()
body = CollectorPro.obj2bin(self.stateBatch, AGENT_STAT_BATCH)
TCLogger.debug("send state:%s", self.stateBatch)
self.trans_layer.sendData(body)