def processMain(self):
from PinpointAgent import stop_front_agent
stop_front_agent()
self.span_helper = GrpcAgentImplement.SpanHelper(
self.span_addr, self.app_id, self.app_name, self.startTimeStamp,
self.max_pending_sz)
self.agent_client = GrpcAgent(agentHost.hostname,
agentHost.ip,
self.web_port,
os.getpid(),
self.agent_addr,
self.service_type,
self.agent_meta,
self.getReqStat,
isContainer=self.isContainer)
self.meta_client = GrpcMeta(self.agent_addr, self.agent_meta)
self.stat_client = GrpcStat(self.stat_addr, self.agent_meta,
self.getIntervalStat)
self.span_factory = GrpcSpanFactory(self)
self.agent_client.start()
self.meta_client.start()
self.stat_client.start()
self.span_helper.start()
self.loopTheQueue()
self.stopProcessMain()
def __init__(self, ac, app_id, app_name, serviceType=PHP):
assert ac.collector_type == SUPPORT_GRPC
super().__init__(app_id, app_name)
self.agent_meta = [('starttime', str(ac.startTimestamp)),
('agentid', app_id),
('applicationname', app_name)]
self.startTimeStamp = ac.startTimestamp
self.max_pending_sz = ac.max_pending_size
self.agent_addr = ac.CollectorAgentIp + ':' + str(ac.CollectorAgentPort)
self.stat_addr = ac.CollectorStatIp + ':' + str(ac.CollectorSpanPort)
self.span_addr = ac.CollectorSpanIp + ':' + str(ac.CollectorSpanPort)
import os
self.agentHost = AgentHost()
self.max_span_sender_size = 2
self.span_sender_list = []
self.sender_index = 0
self._startSpanSender()
self.agent_client = GrpcAgent(self.agentHost.hostname, self.agentHost.ip, ac.getWebPort(), os.getpid(),
self.agent_addr, self.agent_meta)
self.meta_client = GrpcMeta(self.agent_addr, self.agent_meta)
self.agent_client.start()
self.meta_client.start()
self.span_factory = GrpcSpanFactory(self)
def test_metaData(self):
apis = (['aa', 10, 0], ['ada', 12, 0], ['aa', 31, 2], ['aaf', 11, 3])
strings = ('aaaaaaaaaaaa', 'bbbbbbbbbbb', 'cccccccc')
sqls = ('ssss', 'bbbbb', 'ssssssss')
meta_client = GrpcMeta('localhost:9993', self.agent_meta)
id = 0
for api in apis:
id = meta_client.updateApiMeta(*api)
self.assertEqual(id, 4)
for string in strings:
id = meta_client.updateStringMeta(string)
self.assertEqual(id, 7)
for sql in sqls:
id = meta_client.updateSqlMeta(sql)
self.assertEqual(id, 10)
id = meta_client.updateSqlMeta('ssss')
self.assertEqual(id, 8)
id = meta_client.updateStringMeta('bbbbbbbbbbb')
self.assertEqual(id, 6)
time.sleep(1)
self.assertGreaterEqual(self.metaStat.count, 10)
class GrpcAgentImplement(PinpointAgent):
class SpanHelper(object):
def __init__(self, span_addr, appid, appname, starttime,
max_pending_sz):
self.agent_meta = [('starttime', str(starttime)),
('agentid', appid),
('applicationname', appname)]
self.agent_id = appid
self.agent_name = appname
self.span_addr = span_addr
self.max_pending_sz = max_pending_sz
self.span_queue = Queue(self.max_pending_sz)
self.span_client = GrpcSpan(self.span_addr, self.agent_meta,
self.span_queue)
self.dropped_span_count = 0
def start(self):
self.span_client.start()
def sendSpan(self, spanMesg):
try:
self.span_queue.put(spanMesg, False)
except Full as e:
self.dropped_span_count += 1
TCLogger.warning("span send queue is full")
return False
except Exception as e:
TCLogger.error("send span failed: %s", e)
return False
return True
def stop(self):
self.span_client.stop()
TCLogger.info("grpc agent dropped %d", self.dropped_span_count)
def __init__(self, ac, app_id, app_name, serviceType):
assert ac.collector_type == SUPPORT_GRPC
super().__init__(app_id, app_name)
self.agent_meta = [('starttime', str(ac.startTimestamp)),
('agentid', app_id), ('applicationname', app_name)]
self.startTimeStamp = ac.startTimestamp
self.service_type = serviceType
self.max_pending_sz = ac.max_pending_size
self.agent_addr = ac.CollectorAgentIp + ':' + str(
ac.CollectorAgentPort)
self.stat_addr = ac.CollectorStatIp + ':' + str(ac.CollectorStatPort)
self.span_addr = ac.CollectorSpanIp + ':' + str(ac.CollectorSpanPort)
self.web_port = ac.getWebPort()
self.isContainer = ac.agentIsDocker
self.max_span_sender_size = 2
self.sender_index = 0
def start(self):
self.mpQueue = multiprocessing.Queue()
self.process = multiprocessing.Process(target=self.processMain)
self.process.start()
def processMain(self):
from PinpointAgent import stop_front_agent
stop_front_agent()
self.span_helper = GrpcAgentImplement.SpanHelper(
self.span_addr, self.app_id, self.app_name, self.startTimeStamp,
self.max_pending_sz)
self.agent_client = GrpcAgent(agentHost.hostname,
agentHost.ip,
self.web_port,
os.getpid(),
self.agent_addr,
self.service_type,
self.agent_meta,
self.getReqStat,
isContainer=self.isContainer)
self.meta_client = GrpcMeta(self.agent_addr, self.agent_meta)
self.stat_client = GrpcStat(self.stat_addr, self.agent_meta,
self.getIntervalStat)
self.span_factory = GrpcSpanFactory(self)
self.agent_client.start()
self.meta_client.start()
self.stat_client.start()
self.span_helper.start()
self.loopTheQueue()
self.stopProcessMain()
def loopTheQueue(self):
while True:
try:
body = self.mpQueue.get()
except KeyboardInterrupt:
break
if body == None:
TCLogger.info("agent: %s stopping", self.agent_meta)
break
else:
content = body.decode('utf-8')
try:
stack = json.loads(content)
except Exception as e:
TCLogger.error("json is crash:[%s]", content)
return
super().sendSpan(stack, body)
try:
pSpan = self.span_factory.makeSpan(stack)
spanMesg = PSpanMessage(span=pSpan)
except Exception as e:
TCLogger.warn("interrupted by %s", e)
continue
self.span_helper.sendSpan(spanMesg)
def asynSendSpan(self, stack, body):
self.mpQueue.put(body, timeout=5, block=False)
def stop(self):
self.mpQueue.put(None, timeout=5)
self.process.join()
def stopProcessMain(self):
self.agent_client.stop()
self.meta_client.stop()
self.stat_client.stop()
self.span_helper.stop()
def updateApiMeta(self, name, type=API_DEFAULT):
return self.meta_client.updateApiMeta(name, -1, type)
def updateStringMeta(self, name):
return self.meta_client.updateStringMeta(name)
def updateSqlMeta(self, sql):
return self.meta_client.updateSqlMeta(sql)
class GrpcAgentImplement(PinpointAgent):
class SpanSender(object):
def __init__(self, span_addr, appid, appname, starttime,max_pending_sz):
self.agent_meta = [('starttime', str(starttime)), ('agentid', appid), ('applicationname', appname)]
self.agent_id = appid
self.agent_name = appname
self.span_addr = span_addr
self.max_pending_sz =max_pending_sz
self.span_queue = Queue(self.max_pending_sz)
self.span_client = GrpcSpan(self.span_addr, self.agent_meta)
self.dropped_span_count=0
TCLogger.info("Successfully create a Span Sender")
def start(self):
self.span_client.start(self.span_queue)
def sendSpan(self, spanMesg):
try:
self.span_queue.put(spanMesg, False)
except Full as e:
self.dropped_span_count+=1
return False
except Exception as e:
TCLogger.error("send span failed: %s", e)
return False
return True
def stopSelf(self):
self.span_client.stop()
TCLogger.info("grpc agent dropped %d",self.dropped_span_count)
def __init__(self, ac, app_id, app_name, serviceType=PHP):
assert ac.collector_type == SUPPORT_GRPC
super().__init__(app_id, app_name)
self.agent_meta = [('starttime', str(ac.startTimestamp)),
('agentid', app_id),
('applicationname', app_name)]
self.startTimeStamp = ac.startTimestamp
self.max_pending_sz = ac.max_pending_size
self.agent_addr = ac.CollectorAgentIp + ':' + str(ac.CollectorAgentPort)
self.stat_addr = ac.CollectorStatIp + ':' + str(ac.CollectorSpanPort)
self.span_addr = ac.CollectorSpanIp + ':' + str(ac.CollectorSpanPort)
import os
self.agentHost = AgentHost()
self.max_span_sender_size = 2
self.span_sender_list = []
self.sender_index = 0
self._startSpanSender()
self.agent_client = GrpcAgent(self.agentHost.hostname, self.agentHost.ip, ac.getWebPort(), os.getpid(),
self.agent_addr, self.agent_meta)
self.meta_client = GrpcMeta(self.agent_addr, self.agent_meta)
self.agent_client.start()
self.meta_client.start()
self.span_factory = GrpcSpanFactory(self)
def start(self):
pass
def _sendSpan(self, spanMsg):
self.span_sender_list[0].sendSpan(spanMsg)
return True
def sendSpan(self, stack, body):
try:
pSpan = self.span_factory.makeSpan(stack)
spanMesg = PSpanMessage(span=pSpan)
except Exception as e:
TCLogger.warn(" interrupted by %s",e)
return False
if self._sendSpan(spanMesg):
return True
else:
if len(self.span_sender_list) < self.max_span_sender_size:
TCLogger.warn("try to create a new span_sender")
self._startSpanSender()
else:
TCLogger.warn("span_processes extend to max size")
return True
def _startSpanSender(self):
spanSender = GrpcAgentImplement.SpanSender(self.span_addr, self.app_id, self.app_name, self.startTimeStamp,self.max_pending_sz)
spanSender.start()
self.span_sender_list.append(spanSender)
def stop(self):
self.agent_client.stop()
self.meta_client.stop()
for sender in self.span_sender_list:
sender.stopSelf()
def updateApiMeta(self, name, type=API_DEFAULT):
return self.meta_client.updateApiMeta(name, -1, type)
def updateStringMeta(self, name):
return self.meta_client.updateStringMeta(name)
def test_once_timer(self):
meta = GrpcMeta('dev-pinpoint:9992', self.agent_meta)
meta.start()
time.sleep(1)
meta.stop()