def log(self, category, message): if self._add_newlines: message = message + '\n' if self._bucket is not None: self._bucket.append( scribe.LogEntry(category=category, message=message)) else: self._client.Log( messages=[scribe.LogEntry(category=category, message=message)])
def log(self, category, message): if (not self.transport.isOpen()): self.transport.open() message = base64.b64encode(message).strip() log_entry = scribe.LogEntry(category, message) result = self.client.Log(messages=[log_entry]) return result # 0 for success
def sendMessages(self, category, nb): msgs = [] for i in range(nb): msgs.append(scribe.LogEntry(category, "This is message " + str(i))) self.scribe_client.Log(messages=msgs) # Throttle to make sure messages go all the way through and counters are updated time.sleep(2)
def _send(self, span): log_entry = scribe.LogEntry(CATEGORY, self._build_message(span)) try: self.scribe_client.Log([log_entry]) except Exception as e: msg = 'ZipkinClient send error %s' % str(e) warnings.warn(msg) self._connect()
def write(self, category, data): """Write data to scribe instance""" if not self._is_scribe_ready(): return if not isinstance(data, list): data = [data] category = category or self.default_category messages = [] for msg in data: try: entry = scribe.LogEntry(category=category, message=msg) except Exception, e: entry = scribe.LogEntry(dict(category=category, message=msg)) messages.append(entry)
def emit(self, record): """Emit a record via Scribe.""" fmt_record = self.format(record) self._log_buffer.append( scribe.LogEntry(category=self._category, message=fmt_record)) try: self.scribe_write(self._log_buffer) except self.ScribeHandlerException: if not self._buffer_enabled: self._log_buffer = [] else: self._log_buffer = []
def log(self, category, message): '''specify a category and send the message''' # message = time.strftime('%H:%M:%S') + '\t' + message # add timestamp before log log_entry = scribe.LogEntry(category=category, message=message) try: self.client.Log([log_entry]) except Exception, e: self._transObj.close() self._transObj.connect() if self._transObj.isOpen(): self.client.Log([log_entry]) else: pass
def emit(self, record): now = datetime.datetime.now() record.servertime = "%s,%0.3d" % (now.strftime("%Y-%m-%d %H:%M:%S"), now.microsecond / 1000) message = "\\n".join( [r for r in self.format(record).split("\n") if r != ""]) + "\n" try: self.transport.open() entry = scribe.LogEntry(category=self.category, message=message) self.client.Log([entry]) self.transport.close() except Exception as e: logging.exception(e)
class TailThread(threading.Thread): def __init__(self, handleFile): threading.Thread.__init__(self) self.host = host self.port = port self.handleFile = handleFile self.w = None def run(self): category_name = getLocalIp() + "%" + self.handleFile.split( '/')[-2] + "%" + self.handleFile.split('/')[-1] socket = TSocket.TSocket(host=self.host, port=self.port) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(iprot=protocol, oprot=protocol) try: transport.open() except Exception, e: syslog.syslog("%s\n" % e) sys.exit(2) if self.handleFile.endswith(logFileFormat): try: file = open(self.handleFile, 'r') except Exception, e: syslog.syslog("%s\n" % e) sys.exit(2) file.seek(0, 2) while True: where = file.tell() line = file.readline().split('\n')[0] if not line: if not os.path.exists(self.handleFile): sys.exit(2) else: time.sleep(1) file.seek(where) else: try: log_entry = scribe.LogEntry(category=category_name, message=line) print "f**k is : ", log_entry result = client.Log(messages=[log_entry]) #transport.close() except Exception, e: syslog.syslog( "disconnect from scribe server,error: %s\n" % e)
def emit(self, record): """ Emit a record. If a formatter is specified, it is used to format the record. The record is then logged to Scribe with a trailing newline. """ # apply formatting to the record. msg = self.format(record) # for backwards-compatibility, do not add in a line break if it # is already being manually added into each record. if not msg.startswith('\n') or msg.endswith('\n'): msg += '\n' if (self.client is None) or (self.transport is None): raise ScribeTransportError('No transport defined') # It looks like pypy Does not have logging.logRecord.processName # This is a hackish workaround. if hasattr(record, 'processName'): pn = record.processName else: pn = 'Unknown' category = self.category % { 'module': record.module, 'levelname': record.levelname, 'loggername': record.name, 'processName': pn, 'hostname': socket.gethostname(), } log_entry = scribe.LogEntry(category=category, message=msg) try: self.transport.open() for le in self.get_entries(log_entry): result = self.client.Log(messages=[le[1]]) if result != scribe.ResultCode.OK: raise ScribeLogError(result) self.pop_entry(le[0]) self.transport.close() except TTransportException: if self.file_buffer is not None: self.add_entry(log_entry) self._do_error(record) except: self._do_error(record)
class TailThread(threading.Thread): def __init__(self, handleFile): threading.Thread.__init__(self) self.host = host self.port = port self.handleFile = handleFile self.w = None def run(self): category_name = getLocalIp() + "%" + self.handleFile.split( '/')[-2] + "%" + self.handleFile.split('/')[-1] socket = TSocket.TSocket(host=self.host, port=self.port) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(iprot=protocol, oprot=protocol) print self.handleFile try: transport.open() except Exception, e: syslog.syslog("%s\n" % e) sys.exit(2) while True: try: file = open(self.handleFile, 'r') except Exception, e: print e sys.exit(2) file.seek(0, 2) if self.w: file.seek(self.w) #else: # file.seek(0,2) where = file.tell() line = file.readline() if not line: time.sleep(1) file.seek(where) else: print line, # already has newline log_entry = scribe.LogEntry(category=category_name, message=line) print "f**k is: ", log_entry result = client.Log(messages=[log_entry]) self.w = file.tell() file.close()
def scribe_error(*args, **kw): options = app.get_options() socket = TSocket.TSocket(host=options.twitter_common_scribe_host, port=options.twitter_common_scribe_port) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(iprot=protocol, oprot=protocol) value = BasicExceptionHandler.format(*args, **kw) log_entry = scribe.LogEntry(category=options.twitter_common_scribe_category, message=value) try: transport.open() result = client.Log(messages=[log_entry]) transport.close() if result != scribe.ResultCode.OK: AppScribeExceptionHandler.log_error('Failed to scribe exception!') except TTransport.TTransportException: AppScribeExceptionHandler.log_error('Could not connect to scribe!')
def screenshot_scribe(msg): try: socket = TSocket.TSocket(host=SCRIBE_HOST, port=SCRIBE_PORT) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(protocol) transport.open() category = SCRIBE_CATEGORY log_entry = scribe.LogEntry(category, msg) result = client.Log(messages=[log_entry]) if result == 0: return True else: return False except Exception as e: logging.error(e) return False
def log(self, message): """ forward the already formatted message to the target destination """ message = message.replace('/|', '@@') try: type, _, _, level, _, _ = message.split('|') except: return True #invalid message, can't extract type and level category = "%s-%s" % (type, level) log_entry = scribe.LogEntry(dict(category=category, message=message)) if self._client == None: return False try: self._client.Log(messages=[log_entry]) except: self.enabled = False self.close() return False return True
def emit(self, msg): if not msg.startswith('\n') or msg.endswith('\n'): msg += '\n' if (self.client is None) or (self.transport is None): raise ScribeTransportError('No transport defined') log_entry = scribe.LogEntry(category=self.category, message=msg) try: self.transport.open() result = self.client.Log(messages=[ log_entry, ]) if result != scribe.ResultCode.OK: raise ScribeLogError(result) self.transport.close() except TTransportException: raise
def send_bi(host, port, category, message): assert message assert category if (message is None) or (category is None): return "error args" result = 0 socket = TSocket.TSocket(host=host, port=port) socket.setTimeout(3000) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(iprot=protocol, oprot=protocol) try: transport.open() log_entry = scribe.LogEntry(category=category, message=message) result = client.Log(messages=[log_entry]) except Exception as e: result = "failed to write bilog" traceback.print_exc(file=sys.stdout) finally: try: transport.close() except Exception as e: result = "failed to close socket" traceback.print_exc(file=sys.stdout) if result == scribe.ResultCode.OK: result = "success for sending bi log" elif result == scribe.ResultCode.TRY_LATER: result = "try it later" else: result = "Unknown error code." return result
def create_event(src, msgFrom, msgTo, msgSubject, msgDate): """Create Thrift event from a message Keyword arguments: src -- unique identifier for the source of the page (e.g. Nagios, Pages, ...) msgFrom -- email FROM header msgTo -- email TO header msgSubject -- email subject msgDate -- time when the message was received (seconds since epoch) """ transportOut = TTransport.TMemoryBuffer() protocolOut = TBinaryProtocol.TBinaryProtocol(transportOut) timeInMilliSinceEpoch = msgDate * 1000 opsAlert = OpsAlert(src, msgFrom, msgTo, msgSubject, timeInMilliSinceEpoch) opsAlert.write(protocolOut) opsAlertInBytes = transportOut.getvalue() log.debug('Created: %s', str(opsAlert)) return scribe.LogEntry( "OpsAlert", str(timeInMilliSinceEpoch) + ":" + base64.b64encode(opsAlertInBytes))
# You should have received a copy of the GNU Affero General Public License # along with SwiftGate. If not, see <http://www.gnu.org/licenses/>. import sys sys.path.append('/usr/lib/python2.7/site-packages/') from flask import abort, Flask, make_response, request, abort from httplib import HTTPConnection from scribe import scribe from thrift.transport import TTransport, TSocket from thrift.protocol import TBinaryProtocol import json, pika, memcache application = Flask(__name__) log_entry = scribe.LogEntry('swiftgate', 'Started Up') socket = TSocket.TSocket(host='localhost', port=1463) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(iprot=protocol, oprot=protocol) transport.open() result = client.Log(messages=[log_entry]) transport.close() pika_parameters = pika.ConnectionParameters('localhost') pika_connection = pika.BlockingConnection(pika_parameters) pika_channel = pika_connection.channel() pika_channel.queue_declare(queue='swiftgate', durable=True)
except Exception, e: syslog.syslog("file have to flushed, reopen it: %s"%e) else: try: t = old_md5 except Exception,e: file.seek(0,2) else: if new_md5 == old_md5: file.seek(0,2) max_wait = 0 else: old_md5 = commands.getoutput("md5sum %s"%self.handleFile).split()[0] max_wait = 0 try: log_entry = scribe.LogEntry(category=category_name, message=line) result = client.Log(messages=[log_entry]) #transport.close() except Exception, e: syslog.syslog("disconnect from scribe server,error: %s\n"%e) while True: try: transport.open() log_entry = scribe.LogEntry(category=category_name, message=line) result = client.Log(messages=[log_entry]) except Exception, e: syslog.syslog("can't conn scribe server.") time.sleep(3) else: break file.seek(where) else:
def send(self, msg): log_entry = scribe.LogEntry(category=self.category, message=msg) return self.client.Log(messages=[log_entry])
# Simple script to test the Thrift endpoint of the collector # import time from scribe import scribe from thrift.transport import TTransport, TSocket from thrift.protocol import TBinaryProtocol NB_MSGS = 20 transportOut = TTransport.TMemoryBuffer() protocolOut = TBinaryProtocol.TBinaryProtocol(transportOut) timeInMilliSinceEpoch = int(time.time()) * 1000 logEntry = scribe.LogEntry("hello", str(timeInMilliSinceEpoch) + ":world") socket = TSocket.TSocket(host='127.0.0.1', port=7911) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(iprot=protocol, oprot=protocol) transport.open() i = 0 print "Number of messages sent: ", while i < NB_MSGS: result = client.Log(messages=[logEntry]) print i, "..", i += 1
netflowPayload[31], netflowPayload[32], netflowPayload[33], netflowPayload[34], netflowPayload[35], netflowPayload[36], netflowPayload[37], netflowPayload[38], netflowPayload[39], netflowPayload[40], netflowPayload[41]) netflow.write(protocolOut) netflowInBytes = transportOut.getvalue() log.debug('Created: %s', str(netflow)) return scribe.LogEntry("Netflow", str(timeInMilliSinceEpoch) + ":" + base64.b64encode(netflowInBytes)) def __send_events(self): """Send all events """ socket = TSocket.TSocket(host=self.scribeHost, port=self.scribePort) transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) client = scribe.Client(iprot=protocol, oprot=protocol) transport.open() log.info('Sending %d messages from file %s to %s:%d', len(self.events), self.infile, self.scribeHost, self.scribePort) result = client.Log(messages=self.events) transport.close() return result
def log(self, category, message): log_entry = scribe.LogEntry(category, message) result = self.client.Log(messages=[log_entry]) return result # 0 for success
class ScribeClient(SendLog): """Python Client For Scribe_Client.""" global options global file_info def __init__(self): print "init" self.transport = None self.client = None self.connected = 0 def connect(self, host, port): socket = TSocket.TSocket(host=host, port=port) self.transport = TTransport.TFramedTransport(socket) protocol = TBinaryProtocol.TBinaryProtocol(trans=self.transport, strictRead=False, strictWrite=False) self.client = scribe.Client(iprot=protocol, oprot=protocol) self.connected = 1 self.thriftEx = TTransport.TTransportException() def disconnect(self): if (self.transport.isOpen()): self.transport.close() self.client.shutdown() self.connected = 0 def isConnected(self): return self.connected def file_send_log(self, fname): fullfname = fname fname = os.path.basename(fullfname) fsize = os.path.getsize(fullfname) if not file_info.has_key(fname): print("File %s add to monitor logs." % fname) file_info[fname] = FileInfo() return if fsize < file_info[fname]: print("File %s is been tracated?" % fname) file_info[fname] = FileInfo() return fobj = open(fullfname) fobj.seek(file_info[fname].getOffset()) if (not self.transport.isOpen()): try: self.transport.open() except TTransport.TTransportException, self.thriftEx: print self.thriftEx file_info[fname].setOffset(fobj.tell()) fobj.close() return -1 buf = fobj.read(BUF_MAX) while buf and len(buf) > 0: log_entry = scribe.LogEntry(category=options.type, message=buf) try: self.client.Log(messages=[log_entry]) except TTransport.TTransportException, self.thriftEx: print self.thriftEx fobj.seek(-len(buf), 1) self.transport.close() break except socket.error, socketex: print socketex fobj.seek(-len(buf), 1) self.transport.close() break
from thrift.protocol import TBinaryProtocol from thrift.transport import TTransport, TSocket _SCRIBE_PRESENT = True except ImportError: _SCRIBE_PRESENT = False _CATEGORY = "python_default" _HOST = "localhost" _PORT = 1463 _TEST_MSG = ( "For years, the war-crimes fugitive known as 'The Terminator' was so supremely " "confident that he played tennis at a luxury hotel near the Congo-Rwanda border, " "flaunting his freedom while United Nations peacekeepers drove past.") if _SCRIBE_PRESENT: _MESSAGES = [scribe.LogEntry(category=_CATEGORY, message=_TEST_MSG)] class TestHandler(unittest.TestCase): def setUp(self): self.mox = mox.Mox() self.handler = None self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) if _SCRIBE_PRESENT: self.handler_drop = ScribeHandler(buffer=False, category=_CATEGORY, host=_HOST, port=_PORT) self.handler_buffer = ScribeHandler(buffer=True, category=_CATEGORY,
def log(self, category, message): log_entry = scribe.LogEntry(category, message) result = self.client.Log(messages=[log_entry]) if (result != 0): print("scribe.client.log failed") return result # 0 for success