def log(self, category, message):
if self._add_newlines:
message = message + '\n'
if self._bucket is not None:
self._bucket.append(
scribe.LogEntry(category=category, message=message))
else:
self._client.Log(
messages=[scribe.LogEntry(category=category, message=message)])
def log(self, category, message):
if (not self.transport.isOpen()):
self.transport.open()
message = base64.b64encode(message).strip()
log_entry = scribe.LogEntry(category, message)
result = self.client.Log(messages=[log_entry])
return result # 0 for success
def sendMessages(self, category, nb):
msgs = []
for i in range(nb):
msgs.append(scribe.LogEntry(category, "This is message " + str(i)))
self.scribe_client.Log(messages=msgs)
# Throttle to make sure messages go all the way through and counters are updated
time.sleep(2)
def _send(self, span):
log_entry = scribe.LogEntry(CATEGORY, self._build_message(span))
try:
self.scribe_client.Log([log_entry])
except Exception as e:
msg = 'ZipkinClient send error %s' % str(e)
warnings.warn(msg)
self._connect()
def write(self, category, data):
"""Write data to scribe instance"""
if not self._is_scribe_ready():
return
if not isinstance(data, list):
data = [data]
category = category or self.default_category
messages = []
for msg in data:
try:
entry = scribe.LogEntry(category=category, message=msg)
except Exception, e:
entry = scribe.LogEntry(dict(category=category, message=msg))
messages.append(entry)
def emit(self, record):
"""Emit a record via Scribe."""
fmt_record = self.format(record)
self._log_buffer.append(
scribe.LogEntry(category=self._category, message=fmt_record))
try:
self.scribe_write(self._log_buffer)
except self.ScribeHandlerException:
if not self._buffer_enabled:
self._log_buffer = []
else:
self._log_buffer = []
def log(self, category, message):
'''specify a category and send the message'''
# message = time.strftime('%H:%M:%S') + '\t' + message # add timestamp before log
log_entry = scribe.LogEntry(category=category, message=message)
try:
self.client.Log([log_entry])
except Exception, e:
self._transObj.close()
self._transObj.connect()
if self._transObj.isOpen():
self.client.Log([log_entry])
else:
pass
def emit(self, record):
now = datetime.datetime.now()
record.servertime = "%s,%0.3d" % (now.strftime("%Y-%m-%d %H:%M:%S"),
now.microsecond / 1000)
message = "\\n".join(
[r for r in self.format(record).split("\n") if r != ""]) + "\n"
try:
self.transport.open()
entry = scribe.LogEntry(category=self.category, message=message)
self.client.Log([entry])
self.transport.close()
except Exception as e:
logging.exception(e)
class TailThread(threading.Thread):
def __init__(self, handleFile):
threading.Thread.__init__(self)
self.host = host
self.port = port
self.handleFile = handleFile
self.w = None
def run(self):
category_name = getLocalIp() + "%" + self.handleFile.split(
'/')[-2] + "%" + self.handleFile.split('/')[-1]
socket = TSocket.TSocket(host=self.host, port=self.port)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport,
strictRead=False,
strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
try:
transport.open()
except Exception, e:
syslog.syslog("%s\n" % e)
sys.exit(2)
if self.handleFile.endswith(logFileFormat):
try:
file = open(self.handleFile, 'r')
except Exception, e:
syslog.syslog("%s\n" % e)
sys.exit(2)
file.seek(0, 2)
while True:
where = file.tell()
line = file.readline().split('\n')[0]
if not line:
if not os.path.exists(self.handleFile):
sys.exit(2)
else:
time.sleep(1)
file.seek(where)
else:
try:
log_entry = scribe.LogEntry(category=category_name,
message=line)
print "f**k is : ", log_entry
result = client.Log(messages=[log_entry])
#transport.close()
except Exception, e:
syslog.syslog(
"disconnect from scribe server,error: %s\n" % e)
def emit(self, record):
"""
Emit a record.
If a formatter is specified, it is used to format the record.
The record is then logged to Scribe with a trailing newline.
"""
# apply formatting to the record.
msg = self.format(record)
# for backwards-compatibility, do not add in a line break if it
# is already being manually added into each record.
if not msg.startswith('\n') or msg.endswith('\n'):
msg += '\n'
if (self.client is None) or (self.transport is None):
raise ScribeTransportError('No transport defined')
# It looks like pypy Does not have logging.logRecord.processName
# This is a hackish workaround.
if hasattr(record, 'processName'):
pn = record.processName
else:
pn = 'Unknown'
category = self.category % {
'module': record.module,
'levelname': record.levelname,
'loggername': record.name,
'processName': pn,
'hostname': socket.gethostname(),
}
log_entry = scribe.LogEntry(category=category, message=msg)
try:
self.transport.open()
for le in self.get_entries(log_entry):
result = self.client.Log(messages=[le[1]])
if result != scribe.ResultCode.OK:
raise ScribeLogError(result)
self.pop_entry(le[0])
self.transport.close()
except TTransportException:
if self.file_buffer is not None:
self.add_entry(log_entry)
self._do_error(record)
except:
self._do_error(record)
class TailThread(threading.Thread):
def __init__(self, handleFile):
threading.Thread.__init__(self)
self.host = host
self.port = port
self.handleFile = handleFile
self.w = None
def run(self):
category_name = getLocalIp() + "%" + self.handleFile.split(
'/')[-2] + "%" + self.handleFile.split('/')[-1]
socket = TSocket.TSocket(host=self.host, port=self.port)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport,
strictRead=False,
strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
print self.handleFile
try:
transport.open()
except Exception, e:
syslog.syslog("%s\n" % e)
sys.exit(2)
while True:
try:
file = open(self.handleFile, 'r')
except Exception, e:
print e
sys.exit(2)
file.seek(0, 2)
if self.w:
file.seek(self.w)
#else:
# file.seek(0,2)
where = file.tell()
line = file.readline()
if not line:
time.sleep(1)
file.seek(where)
else:
print line, # already has newline
log_entry = scribe.LogEntry(category=category_name,
message=line)
print "f**k is: ", log_entry
result = client.Log(messages=[log_entry])
self.w = file.tell()
file.close()
def scribe_error(*args, **kw):
options = app.get_options()
socket = TSocket.TSocket(host=options.twitter_common_scribe_host,
port=options.twitter_common_scribe_port)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
value = BasicExceptionHandler.format(*args, **kw)
log_entry = scribe.LogEntry(category=options.twitter_common_scribe_category,
message=value)
try:
transport.open()
result = client.Log(messages=[log_entry])
transport.close()
if result != scribe.ResultCode.OK:
AppScribeExceptionHandler.log_error('Failed to scribe exception!')
except TTransport.TTransportException:
AppScribeExceptionHandler.log_error('Could not connect to scribe!')
def screenshot_scribe(msg):
try:
socket = TSocket.TSocket(host=SCRIBE_HOST, port=SCRIBE_PORT)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport,
strictRead=False,
strictWrite=False)
client = scribe.Client(protocol)
transport.open()
category = SCRIBE_CATEGORY
log_entry = scribe.LogEntry(category, msg)
result = client.Log(messages=[log_entry])
if result == 0:
return True
else:
return False
except Exception as e:
logging.error(e)
return False
def log(self, message):
"""
forward the already formatted message to the target destination
"""
message = message.replace('/|', '@@')
try:
type, _, _, level, _, _ = message.split('|')
except:
return True #invalid message, can't extract type and level
category = "%s-%s" % (type, level)
log_entry = scribe.LogEntry(dict(category=category, message=message))
if self._client == None:
return False
try:
self._client.Log(messages=[log_entry])
except:
self.enabled = False
self.close()
return False
return True
def emit(self, msg):
if not msg.startswith('\n') or msg.endswith('\n'):
msg += '\n'
if (self.client is None) or (self.transport is None):
raise ScribeTransportError('No transport defined')
log_entry = scribe.LogEntry(category=self.category, message=msg)
try:
self.transport.open()
result = self.client.Log(messages=[
log_entry,
])
if result != scribe.ResultCode.OK:
raise ScribeLogError(result)
self.transport.close()
except TTransportException:
raise
def send_bi(host, port, category, message):
assert message
assert category
if (message is None) or (category is None):
return "error args"
result = 0
socket = TSocket.TSocket(host=host, port=port)
socket.setTimeout(3000)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport,
strictRead=False,
strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
try:
transport.open()
log_entry = scribe.LogEntry(category=category, message=message)
result = client.Log(messages=[log_entry])
except Exception as e:
result = "failed to write bilog"
traceback.print_exc(file=sys.stdout)
finally:
try:
transport.close()
except Exception as e:
result = "failed to close socket"
traceback.print_exc(file=sys.stdout)
if result == scribe.ResultCode.OK:
result = "success for sending bi log"
elif result == scribe.ResultCode.TRY_LATER:
result = "try it later"
else:
result = "Unknown error code."
return result
def create_event(src, msgFrom, msgTo, msgSubject, msgDate):
"""Create Thrift event from a message
Keyword arguments:
src -- unique identifier for the source of the page (e.g. Nagios, Pages, ...)
msgFrom -- email FROM header
msgTo -- email TO header
msgSubject -- email subject
msgDate -- time when the message was received (seconds since epoch)
"""
transportOut = TTransport.TMemoryBuffer()
protocolOut = TBinaryProtocol.TBinaryProtocol(transportOut)
timeInMilliSinceEpoch = msgDate * 1000
opsAlert = OpsAlert(src, msgFrom, msgTo, msgSubject, timeInMilliSinceEpoch)
opsAlert.write(protocolOut)
opsAlertInBytes = transportOut.getvalue()
log.debug('Created: %s', str(opsAlert))
return scribe.LogEntry(
"OpsAlert",
str(timeInMilliSinceEpoch) + ":" + base64.b64encode(opsAlertInBytes))
# You should have received a copy of the GNU Affero General Public License
# along with SwiftGate. If not, see <http://www.gnu.org/licenses/>.
import sys
sys.path.append('/usr/lib/python2.7/site-packages/')
from flask import abort, Flask, make_response, request, abort
from httplib import HTTPConnection
from scribe import scribe
from thrift.transport import TTransport, TSocket
from thrift.protocol import TBinaryProtocol
import json, pika, memcache
application = Flask(__name__)
log_entry = scribe.LogEntry('swiftgate', 'Started Up')
socket = TSocket.TSocket(host='localhost', port=1463)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport,
strictRead=False,
strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
transport.open()
result = client.Log(messages=[log_entry])
transport.close()
pika_parameters = pika.ConnectionParameters('localhost')
pika_connection = pika.BlockingConnection(pika_parameters)
pika_channel = pika_connection.channel()
pika_channel.queue_declare(queue='swiftgate', durable=True)
except Exception, e:
syslog.syslog("file have to flushed, reopen it: %s"%e)
else:
try:
t = old_md5
except Exception,e:
file.seek(0,2)
else:
if new_md5 == old_md5:
file.seek(0,2)
max_wait = 0
else:
old_md5 = commands.getoutput("md5sum %s"%self.handleFile).split()[0]
max_wait = 0
try:
log_entry = scribe.LogEntry(category=category_name, message=line)
result = client.Log(messages=[log_entry])
#transport.close()
except Exception, e:
syslog.syslog("disconnect from scribe server,error: %s\n"%e)
while True:
try:
transport.open()
log_entry = scribe.LogEntry(category=category_name, message=line)
result = client.Log(messages=[log_entry])
except Exception, e:
syslog.syslog("can't conn scribe server.")
time.sleep(3)
else: break
file.seek(where)
else:
def send(self, msg):
log_entry = scribe.LogEntry(category=self.category, message=msg)
return self.client.Log(messages=[log_entry])
# Simple script to test the Thrift endpoint of the collector
#
import time
from scribe import scribe
from thrift.transport import TTransport, TSocket
from thrift.protocol import TBinaryProtocol
NB_MSGS = 20
transportOut = TTransport.TMemoryBuffer()
protocolOut = TBinaryProtocol.TBinaryProtocol(transportOut)
timeInMilliSinceEpoch = int(time.time()) * 1000
logEntry = scribe.LogEntry("hello", str(timeInMilliSinceEpoch) + ":world")
socket = TSocket.TSocket(host='127.0.0.1', port=7911)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport,
strictRead=False,
strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
transport.open()
i = 0
print "Number of messages sent: ",
while i < NB_MSGS:
result = client.Log(messages=[logEntry])
print i, "..",
i += 1
netflowPayload[31],
netflowPayload[32],
netflowPayload[33],
netflowPayload[34],
netflowPayload[35],
netflowPayload[36],
netflowPayload[37],
netflowPayload[38],
netflowPayload[39],
netflowPayload[40],
netflowPayload[41])
netflow.write(protocolOut)
netflowInBytes = transportOut.getvalue()
log.debug('Created: %s', str(netflow))
return scribe.LogEntry("Netflow", str(timeInMilliSinceEpoch) + ":" + base64.b64encode(netflowInBytes))
def __send_events(self):
"""Send all events
"""
socket = TSocket.TSocket(host=self.scribeHost, port=self.scribePort)
transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False)
client = scribe.Client(iprot=protocol, oprot=protocol)
transport.open()
log.info('Sending %d messages from file %s to %s:%d', len(self.events), self.infile, self.scribeHost, self.scribePort)
result = client.Log(messages=self.events)
transport.close()
return result
def log(self, category, message):
log_entry = scribe.LogEntry(category, message)
result = self.client.Log(messages=[log_entry])
return result # 0 for success
class ScribeClient(SendLog):
"""Python Client For Scribe_Client."""
global options
global file_info
def __init__(self):
print "init"
self.transport = None
self.client = None
self.connected = 0
def connect(self, host, port):
socket = TSocket.TSocket(host=host, port=port)
self.transport = TTransport.TFramedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(trans=self.transport,
strictRead=False,
strictWrite=False)
self.client = scribe.Client(iprot=protocol, oprot=protocol)
self.connected = 1
self.thriftEx = TTransport.TTransportException()
def disconnect(self):
if (self.transport.isOpen()):
self.transport.close()
self.client.shutdown()
self.connected = 0
def isConnected(self):
return self.connected
def file_send_log(self, fname):
fullfname = fname
fname = os.path.basename(fullfname)
fsize = os.path.getsize(fullfname)
if not file_info.has_key(fname):
print("File %s add to monitor logs." % fname)
file_info[fname] = FileInfo()
return
if fsize < file_info[fname]:
print("File %s is been tracated?" % fname)
file_info[fname] = FileInfo()
return
fobj = open(fullfname)
fobj.seek(file_info[fname].getOffset())
if (not self.transport.isOpen()):
try:
self.transport.open()
except TTransport.TTransportException, self.thriftEx:
print self.thriftEx
file_info[fname].setOffset(fobj.tell())
fobj.close()
return -1
buf = fobj.read(BUF_MAX)
while buf and len(buf) > 0:
log_entry = scribe.LogEntry(category=options.type, message=buf)
try:
self.client.Log(messages=[log_entry])
except TTransport.TTransportException, self.thriftEx:
print self.thriftEx
fobj.seek(-len(buf), 1)
self.transport.close()
break
except socket.error, socketex:
print socketex
fobj.seek(-len(buf), 1)
self.transport.close()
break
from thrift.protocol import TBinaryProtocol
from thrift.transport import TTransport, TSocket
_SCRIBE_PRESENT = True
except ImportError:
_SCRIBE_PRESENT = False
_CATEGORY = "python_default"
_HOST = "localhost"
_PORT = 1463
_TEST_MSG = (
"For years, the war-crimes fugitive known as 'The Terminator' was so supremely "
"confident that he played tennis at a luxury hotel near the Congo-Rwanda border, "
"flaunting his freedom while United Nations peacekeepers drove past.")
if _SCRIBE_PRESENT:
_MESSAGES = [scribe.LogEntry(category=_CATEGORY, message=_TEST_MSG)]
class TestHandler(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
self.handler = None
self.logger = logging.getLogger()
self.logger.setLevel(logging.DEBUG)
if _SCRIBE_PRESENT:
self.handler_drop = ScribeHandler(buffer=False,
category=_CATEGORY,
host=_HOST,
port=_PORT)
self.handler_buffer = ScribeHandler(buffer=True,
category=_CATEGORY,
def log(self, category, message):
log_entry = scribe.LogEntry(category, message)
result = self.client.Log(messages=[log_entry])
if (result != 0):
print("scribe.client.log failed")
return result # 0 for success
