class PyLogConf (PyLog):
def __init__ (self, conf):
self.conf = conf
self.client = Client (conf.RAVEN['dsn'])
self.formatters = {}
for k,v in self.conf.FILE_FORMATTERS.iteritems():
if isinstance(v,str):
raise ValueError ('Please use a list or a tuple for the file formatters values')
self.formatters[k] = [item_import(i)() for i in v]
dbname = os.path.join(os.path.dirname(conf.__file__),'pylogwatch.db')
return super(PyLogConf, self).__init__ (self.conf.FILE_FORMATTERS.keys(), dbname = dbname)
def get_file_signature(self, fname):
maxcount = 10
count = 0
result = []
with open(fname) as f:
while count < maxcount:
result.append(f.readline())
count+=1
return result
def process_lines (self, fname, lines):
for line in lines:
paramdict = {}
data = {'event_type':'Message', 'message': line.replace('%','%%'), 'data' :{'logger':fname}}
for fobj in self.formatters[fname]:
fobj.format_line(line, data, paramdict)
if not data.get('do_not_send', False):
if paramdict:
data['params'] = tuple([paramdict[i] for i in sorted(paramdict.keys())])
if self.conf.DEBUG:
print data
self.client.capture(**data)
def process(self, record):
if match_record(self.rules_tree, record):
dsn = self.dsns.get(record.application)
if dsn is None:
projects = requests.get("%s/api/0/projects/" % self.url,
auth=self.auth).json()
for project in projects:
if project["name"] == record.application:
break
else:
project = requests.post("%s/api/0/teams/%s/%s/projects/" % (self.url,
self.organization,
self.team),
auth=self.auth,
headers={"Content-type": "application/json"},
data=json.dumps({"name": record.application})).json()
for key in requests.get("%s/api/0/projects/%s/%s/keys/" % (self.url,
self.organization,
project["slug"]),
auth=self.auth).json():
dsn = key["dsn"]["secret"]
self.dsns[record.application] = dsn
client = Client(dsn, raise_send_errors=True)
client.capture("raven.events.Message",
message=record.msg,
formatted=record.explanation or record.msg,
data={"logger": record.logger},
date=record.datetime,
extra=record._asdict())
class PyLogConf (PyLog):
def __init__ (self, conf):
"""
Initialize object based on the provided configuration
"""
self.conf = conf
self.client = Client (conf.RAVEN['dsn'])
self.formatters = {}
for k,v in self.conf.FILE_FORMATTERS.iteritems():
if isinstance(v,str):
raise ValueError ('Please use a list or a tuple for the file formatters values')
self.formatters[k] = [item_import(i)() for i in v]
dbname = os.path.join(os.path.dirname(conf.__file__),'pylogwatch.db')
return super(PyLogConf, self).__init__ (self.conf.FILE_FORMATTERS.keys(), dbname = dbname)
def process_lines (self, fname, fileobject, lines):
"""Main workhorse. Called with the filename that is being logged and an iterable of lines"""
for line in lines:
paramdict = {}
data = {'event_type':'Message', 'message': line.replace('%','%%'), 'data' :{'logger':fname}}
for fobj in self.formatters[fname]:
fobj.format_line(line, data, paramdict)
if not data.pop('_do_not_send', False): # Skip lines that have the '_do_not_send' key
if paramdict:
data['params'] = tuple([paramdict[i] for i in sorted(paramdict.keys())])
if self.conf.DEBUG:
print data
self.client.capture(**data)
self.update_bytes(fname, fileobject.tell())
def save(self, *a, **k):
"""
Don't save to the database, send to sentry instead.
"""
log_level = self.log_level
filename = get_filename(self.js_url)
data={
filename: {
'url': self.js_url,
'data': {},
'query_string': '...',
'method': 'POST',
},
'logger': 'front_end',
'site': 'site.name',
}
if self.extra:
data.update({"extra": json.loads(self.extra)})
client = Client(settings.SENTRY_DSN)
client.capture(
"Message",
message=self.message,
data=data,
)
def send_to_sentry(self):
"""
Send this error to sentry where it will be stored and aggregated.
"""
print self.stack_trace
log_level = self.log_level
filename = get_filename(self.js_url)
data={
filename: {
'url': self.js_url,
'data': {},
'query_string': '...',
'method': 'POST',
},
'logger': 'front_end',
'site': 'site.name',
}
if self.extra:
data.update({"extra": json.loads(self.extra)})
client = Client(settings.SENTRY_DSN)
client.capture(
"Message",
message=self.message,
data=data,
)
def connect_sentry(message, result):
'''
Connect to the Sentry server
'''
pillar_data = __salt__['pillar.raw']()
grains = __salt__['grains.items']()
sentry_data = {
'result': result,
'pillar': pillar_data,
'grains': grains
}
data = {
'platform': 'python',
'culprit': ret['fun'],
'level': 'error'
}
tags = {}
if 'tags' in pillar_data['raven']:
for tag in pillar_data['raven']['tags']:
tags[tag] = grains[tag]
if ret['return']:
data['level'] = 'info'
servers = []
try:
for server in pillar_data['raven']['servers']:
servers.append(server + '/api/store/')
client = Client(
servers=servers,
public_key=pillar_data['raven']['public_key'],
secret_key=pillar_data['raven']['secret_key'],
project=pillar_data['raven']['project'],
)
except KeyError as missing_key:
logger.error(
'Sentry returner need config \'{0}\' in pillar'.format(
missing_key
)
)
else:
try:
client.capture(
'raven.events.Message',
message=message,
data=data,
extra=sentry_data,
tags=tags
)
except Exception as err:
logger.error(
'Can\'t send message to sentry: {0}'.format(err),
exc_info=True
)
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
project=settings.PROJECT,
servers=['http://localhost:8000%s' % reverse('sentry-api-store')],
key=settings.KEY,
)
client.capture('Message', message='foo')
self.assertEquals(Event.objects.count(), 1)
instance = Event.objects.get()
self.assertEquals(instance.message, 'foo')
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
dsn='http://%s:%s@localhost:8000/%s' %
(self.pk.public_key, self.pk.secret_key, self.pk.project_id))
client.capture('Message', message='foo')
send_remote.assert_called_once()
self.assertEquals(Group.objects.count(), 1)
group = Group.objects.get()
self.assertEquals(group.event_set.count(), 1)
instance = group.event_set.get()
self.assertEquals(instance.message, 'foo')
class SentryEmailProcessor(object):
NAME = "sentry"
EMAIL_SUBJECT_IDENTIFIER = "sentry"
def __init__(self, config):
self.client = Client(config['sentry_dsn'])
self.validation_key = config['email_body_validation_key']
def process(self, message):
"""Takes a mail message, parses the relevant part, and sends the parsed
data to Sentry using the Raven client.
The message must be a multipart message with a 'text/plain' part. This 'text/plain'
part must begin with the correct sentry validation key, followed by json data.
If the message is not in this format, it will be dropped."""
raised = False
# If message is multipart we only want the text version of the body,
# this walks the message and gets the body.
# multipart means dual html and text representations
if message.get_content_maintype() == 'multipart':
for part in message.walk():
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True)
# Check if this is the relevant part of the message
if body.startswith(self.validation_key):
json_ = self.parse_json_message(body[self.validation_key.__len__()+1:])
self.raise_to_sentry(json_)
raised = True
break
if not raised:
log.warning("Message is not in the correct format to be processed by SentryEmailProcessor, even though it's subject line indicates that it ought to be. Dropping mail.")
def raise_to_sentry(self, jsondata):
try:
event_date = dateutil.parser.parse(jsondata['date'])
except Exception:
event_date = datetime.utcnow()
event_data = jsondata['data']
event_data['server_name'] = jsondata['server_name']
self.client.capture('Exception', message = jsondata['message'], date = event_date, data = event_data)
def parse_json_message(self, text):
text = text.replace('\r\n ', '\n')
text = text.replace('\n', '')
parsed_data = json.loads(text)
return parsed_data
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
project=settings.PROJECT,
servers=['http://localhost:8000%s' % reverse('sentry-api-store')],
key=settings.KEY,
)
client.capture('Message', message='foo')
self.assertEquals(Group.objects.count(), 1)
group = Group.objects.get()
self.assertEquals(group.event_set.count(), 1)
instance = group.event_set.get()
self.assertEquals(instance.message, 'foo')
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
dsn='http://%s:%s@localhost:8000/%s' % (
self.pk.public_key, self.pk.secret_key, self.pk.project_id)
)
client.capture('Message', message='foo')
send_remote.assert_called_once()
self.assertEquals(Group.objects.count(), 1)
group = Group.objects.get()
self.assertEquals(group.event_set.count(), 1)
instance = group.event_set.get()
self.assertEquals(instance.message, 'foo')
def connect_sentry(message, result):
'''
Connect to the Sentry server
'''
pillar_data = __salt__['pillar.raw']()
grains = __salt__['grains.items']()
sentry_data = {
'result': result,
'pillar': pillar_data,
'grains': grains
}
data = {
'platform': 'python',
'culprit': ret['fun'],
'level': 'error'
}
tags = {}
if 'tags' in pillar_data['raven']:
for tag in pillar_data['raven']['tags']:
tags[tag] = grains[tag]
if ret['return']:
data['level'] = 'info'
try:
client = Client(dsn=pillar_data['raven']['dsn'],
raise_send_errors=True
)
except KeyError as missing_key:
logger.error(
'Sentry returner need config \'{0}\' in pillar'.format(
missing_key
)
)
else:
try:
client.capture(
'raven.events.Message',
message=message,
data=data,
extra=sentry_data,
tags=tags
)
time.sleep(3)
except Exception as err:
logger.error(
'Can\'t send message to sentry: {0}'.format(err),
exc_info=True
)
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
dsn="http://%s:%s@localhost:8000/%s" % (self.pk.public_key, self.pk.secret_key, self.pk.project_id)
)
with self.tasks():
client.capture("Message", message="foo")
send_remote.assert_called_once()
assert Group.objects.count() == 1
group = Group.objects.get()
assert group.event_set.count() == 1
instance = group.event_set.get()
assert instance.message == "foo"
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
dsn='http://%s:%s@localhost:8000/%s' %
(self.pk.public_key, self.pk.secret_key, self.pk.project_id))
with self.tasks():
client.capture('Message', message='foo')
assert send_remote.call_count is 1
assert Group.objects.count() == 1
group = Group.objects.get()
assert group.event_set.count() == 1
instance = group.event_set.get()
assert instance.message == 'foo'
def connect_sentry(message, result):
'''
Connect to the Sentry server
'''
pillar_data = __salt__['pillar.raw']()
grains = __salt__['grains.items']()
sentry_data = {
'result': result,
'pillar': pillar_data,
'grains': grains
}
data = {
'platform': 'python',
'culprit': ret['fun'],
'level': 'error'
}
tags = {}
if 'tags' in pillar_data['raven']:
for tag in pillar_data['raven']['tags']:
tags[tag] = grains[tag]
if ret['return']:
data['level'] = 'info'
servers = []
try:
for server in pillar_data['raven']['servers']:
servers.append(server + '/api/store/')
client = Client(
servers=servers,
public_key=pillar_data['raven']['public_key'],
secret_key=pillar_data['raven']['secret_key'],
project=pillar_data['raven']['project'],
)
except KeyError as missing_key:
logger.error(
'Sentry returner need config {0!r} in pillar'.format(
missing_key
)
)
else:
try:
client.capture('raven.events.Message', message=message, data=data, extra=sentry_data, tags=tags)
except Exception as err:
logger.error(
'Can\'t send message to sentry: {0}'.format(err),
exc_info=True
)
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
dsn='http://%s:%s@localhost:8000/%s' % (
self.pk.public_key, self.pk.secret_key, self.pk.project_id)
)
with self.tasks():
client.capture('Message', message='foo')
assert send_remote.call_count is 1
assert Group.objects.count() == 1
group = Group.objects.get()
assert group.event_set.count() == 1
instance = group.event_set.get()
assert instance.message == 'foo'
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
project=self.pk.project_id,
servers=['http://localhost:8000%s' % reverse('sentry-api-store', args=[self.pk.project_id])],
public_key=self.pk.public_key,
secret_key=self.pk.secret_key,
)
client.capture('Message', message='foo')
send_remote.assert_called_once()
self.assertEquals(Group.objects.count(), 1)
group = Group.objects.get()
self.assertEquals(group.event_set.count(), 1)
instance = group.event_set.get()
self.assertEquals(instance.message, 'foo')
def send_message(message, params, site, logger,
interface_type="sentry.interfaces.Message",
log_level=logging.ERROR, sentry_dsn=None):
data={
'site': site,
'logger': logger,
}
if sentry_dsn:
client = Client(dsn=sentry_dsn)
else:
raise Exception("No Sentry DSN")
if "request" in params.keys():
interface_type = "sentry.interfaces.Http"
data[interface_type] = params["request"]
else:
data[interface_type] = message
if params["QueryObject"] and params["QueryObject"] != "-":
tags = params["QueryObject"]
else:
tags = None
if params["urlpath"] and params["urlpath"] != "-":
if not tags:
tags = {}
tags["UrlPath"] = params["urlpath"]
if params["ip"] and params["ip"] != "-":
if not tags:
tags = {}
tags["UserIP"] = params["ip"]
date = datetime.datetime.strptime("%s %s" % (params["date"],params["time"]), "%Y/%b/%d %H:%M:%S")
subject = message.get("message", message.get("url", "Unknown Message"))
client.capture(
'Message',
message=subject,
data=data,
level=log_level,
extra=params,
tags=tags,
date=date
)
def connect_sentry(message, result):
'''
Connect to the Sentry server
'''
pillar_data = __salt__['pillar.raw']()
grains = __salt__['grains.items']()
raven_config = pillar_data['raven']
hide_pillar = raven_config.get('hide_pillar')
sentry_data = {
'result': result,
'pillar': 'HIDDEN' if hide_pillar else pillar_data,
'grains': grains
}
data = {
'platform': 'python',
'culprit': message,
'level': 'error'
}
tags = {}
if 'tags' in raven_config:
for tag in raven_config['tags']:
tags[tag] = grains[tag]
if ret_is_not_error(ret):
data['level'] = 'info'
if raven_config.get('report_errors_only') and data['level'] != 'error':
return
if raven_config.get('dsn'):
client = Client(raven_config.get('dsn'), transport=HTTPTransport)
else:
try:
servers = []
for server in raven_config['servers']:
servers.append(server + '/api/store/')
client = Client(
servers=servers,
public_key=raven_config['public_key'],
secret_key=raven_config['secret_key'],
project=raven_config['project'],
transport=HTTPTransport
)
except KeyError as missing_key:
logger.error(
'Sentry returner needs key \'%s\' in pillar',
missing_key
)
return
try:
msgid = client.capture('raven.events.Message', message=message, data=data, extra=sentry_data, tags=tags)
logger.info('Message id %s written to sentry', msgid)
except Exception as exc:
logger.error(
'Can\'t send message to sentry: {0}'.format(exc),
exc_info=True
)
def connect_sentry(message, result):
'''
Connect to the Sentry server
'''
pillar_data = __salt__['pillar.raw']()
grains = __salt__['grains.items']()
raven_config = pillar_data['raven']
hide_pillar = raven_config.get('hide_pillar')
sentry_data = {
'result': result,
'pillar': 'HIDDEN' if hide_pillar else pillar_data,
'grains': grains
}
data = {
'platform': 'python',
'culprit': message,
'level': 'error'
}
tags = {}
if 'tags' in raven_config:
for tag in raven_config['tags']:
tags[tag] = grains[tag]
if ret_is_not_error(ret):
data['level'] = 'info'
if raven_config.get('report_errors_only') and data['level'] != 'error':
return
if raven_config.get('dsn'):
client = Client(raven_config.get('dsn'), transport=HTTPTransport)
else:
try:
servers = []
for server in raven_config['servers']:
servers.append(server + '/api/store/')
client = Client(
servers=servers,
public_key=raven_config['public_key'],
secret_key=raven_config['secret_key'],
project=raven_config['project'],
transport=HTTPTransport
)
except KeyError as missing_key:
logger.error(
'Sentry returner needs key \'%s\' in pillar',
missing_key
)
return
try:
msgid = client.capture('raven.events.Message', message=message, data=data, extra=sentry_data, tags=tags)
logger.info('Message id %s written to sentry', msgid)
except Exception as exc:
logger.error(
'Can\'t send message to sentry: {0}'.format(exc),
exc_info=True
)
def test_basic(self, send_remote):
send_remote.side_effect = self.sendRemote
client = Client(
project=self.pk.project_id,
servers=[
'http://localhost:8000%s' %
reverse('sentry-api-store', args=[self.pk.project_id])
],
public_key=self.pk.public_key,
secret_key=self.pk.secret_key,
)
client.capture('Message', message='foo')
send_remote.assert_called_once()
self.assertEquals(Group.objects.count(), 1)
group = Group.objects.get()
self.assertEquals(group.event_set.count(), 1)
instance = group.event_set.get()
self.assertEquals(instance.message, 'foo')
def returner(ret):
"""
If an error occurs, log it to sentry
"""
if not ret['success']:
logger.debug("not a success, do something")
try:
pillar_data = __salt__['pillar.data']()
logger.error(pillar_data)
client = Client(
servers=pillar_data['raven']['servers'],
public_key=pillar_data['raven']['public_key'],
secret_key=pillar_data['raven']['secret_key'],
project=pillar_data['raven']['project'],
)
client.capture('Salt state failure', data=ret)
except Exception, err:
logger.error("Can't init client: %s", err, exc_info=True)
def _connect_sentry(message, result):
"""
Connect to the Sentry server
"""
pillar_data = __salt__["pillar.raw"]()
grains = __salt__["grains.items"]()
raven_config = pillar_data["raven"]
hide_pillar = raven_config.get("hide_pillar")
sentry_data = {
"result": result,
"pillar": "HIDDEN" if hide_pillar else pillar_data,
"grains": grains,
}
data = {"platform": "python", "culprit": message, "level": "error"}
tags = {}
if "tags" in raven_config:
for tag in raven_config["tags"]:
tags[tag] = grains[tag]
if _ret_is_not_error(result):
data["level"] = "info"
if raven_config.get("report_errors_only") and data["level"] != "error":
return
if raven_config.get("dsn"):
client = Client(raven_config.get("dsn"), transport=HTTPTransport)
else:
try:
servers = []
for server in raven_config["servers"]:
servers.append(server + "/api/store/")
client = Client(
servers=servers,
public_key=raven_config["public_key"],
secret_key=raven_config["secret_key"],
project=raven_config["project"],
transport=HTTPTransport,
)
except KeyError as missing_key:
log.error("Sentry returner needs key '%s' in pillar", missing_key)
return
try:
msgid = client.capture(
"raven.events.Message",
message=message,
data=data,
extra=sentry_data,
tags=tags,
)
log.info("Message id %s written to sentry", msgid)
except Exception as exc: # pylint: disable=broad-except
log.error("Can't send message to sentry: %s", exc, exc_info=True)
def save(self, *a, **k):
"""
Don't save to database, post to sentry instead.
"""
line_number = self.line_number
abbv_user_agent = get_pretty_useragent(self.user_agent)
filename = get_filename(self.url)
msg = "Front End: %s [%s] %s" % (filename, abbv_user_agent, self.message)
client = Client(settings.SENTRY_DSN)
client.capture(
"Message",
message=msg,
data={
"extra": {
"linenumber": self.line_number,
"user_agent": self.user_agent,
"url": self.url,
}
}
)
def send_to_sentry(self):
"""
Send this error to sentry where it will be stored and aggregated.
"""
line_number = self.line_number
abbv_user_agent = get_pretty_useragent(self.user_agent)
filename = get_filename(self.url)
msg = "Front End: %s [%s] %s" % (filename, abbv_user_agent, self.message)
client = Client(settings.SENTRY_DSN)
client.capture(
"Message",
message=msg,
data={
"extra": {
"linenumber": self.line_number,
"user_agent": self.user_agent,
"url": self.url,
}
}
)
class SentrySender(BaseSender):
name = "Sentry"
def __init__(self):
self.client = Client(getattr(settings, 'RAVEN_DSN_STACKTRACE', None),
name=getattr(settings, 'HOST_NAME', None),
release=getattr(settings, 'APP_VERSION', None))
def send(self, message, extra={}, tags={}, sentry_data={}, crash_obj=None):
event_id = self.client.capture('raven.events.Message',
message=message,
extra=extra,
tags=tags,
data=data)
signature("tasks.get_sentry_link",
args=(crash_obj.pk, event_id)).apply_async(queue='private',
countdown=1)
class SentrySender(BaseSender):
name="Sentry"
def __init__(self):
self.client = Client(
getattr(settings, 'RAVEN_DSN_STACKTRACE', None),
name=getattr(settings, 'HOST_NAME', None),
release=getattr(settings, 'APP_VERSION', None)
)
def send(self, message, extra={}, tags={}, sentry_data={}, crash_obj=None):
event_id = self.client.capture(
'raven.events.Message',
message=message,
extra=extra,
tags=tags,
data=sentry_data
)
signature("tasks.get_sentry_link", args=(crash_obj.pk, event_id)).apply_async(queue='private', countdown=1)
class LoggerClient:
def __init__(self, dsn):
self._client = Client(dsn=dsn, context={})
def _process_sourcefile(self, file_path, line_number, context_lines=10):
FileContext = namedtuple("FileContext", ["pre_context", "context", "post_context", "local_vars"])
start = max(line_number - context_lines, 0)
stop = line_number + context_lines
pre_context = []
post_context = []
local_vars = []
context = None
with open(file_path, "r") as f:
for i, line in enumerate(f):
current_line = i + 1
if current_line < start:
continue
elif current_line > stop:
break
# search for variable declaration
var = re.match(r"^(?P<name>[a-z]{1}\w*)=\S", line, re.IGNORECASE)
if var:
local_vars.append(var.group("name"))
if current_line < line_number:
pre_context.append(line.rstrip("\n"))
elif current_line == line_number:
context = line.rstrip("\n")
else:
post_context.append(line.rstrip("\n"))
return FileContext(pre_context, context, post_context, local_vars)
def _get_declares(self, declare_output, local_vars):
"""Parse `declare -p` output and get values from local variables"""
out = {}
for var in local_vars:
m = re.search(r"^(?P<name>" + re.escape(var) + r")=(?P<value>\S.*)", declare_output, re.MULTILINE)
if m:
out[m.group("name")] = m.group("value")
return out
def capture(self, shell_args):
frame = {
"filename": shell_args.script,
"function": shell_args.function or "main",
"lineno": shell_args.lineno,
"module": shell_args.command,
"vars": {},
}
if shell_args.pwd:
abspath = os.path.abspath(os.path.join(shell_args.pwd, shell_args.script))
filename = os.path.basename(abspath)
try:
srcfile = self._process_sourcefile(abspath, shell_args.lineno)
if shell_args.declares:
frame["vars"] = self._get_declares(shell_args.declares, srcfile.local_vars)
frame.update(
filename=filename,
abs_path=abspath,
pre_context=srcfile.pre_context,
context_line=srcfile.context,
post_context=srcfile.post_context,
)
except FileNotFoundError:
sys.stderr.write('Could not process file "{}"\n'.format(abspath))
data = {
"exception": {
"values": [
{
"module": "builtins",
"stacktrace": {"frames": [frame]},
"type": shell_args.script,
"value": "error on line {}".format(shell_args.lineno)
if not shell_args.function
else "error in '{}' on line {}".format(shell_args.function, shell_args.lineno),
}
]
}
}
# add ENV vars and stderr if provided
extra = {}
if shell_args.env:
extra["environment"] = dict([item.split("=", maxsplit=1) for item in shell_args.env.split("\n")])
if shell_args.stderr:
extra["stderr"] = shell_args.stderr
self._client.capture("raven.events.Exception", data=data, extra=extra)
def connect_sentry(result):
'''
Connect to the Sentry server
'''
raven_config = get_config_value('appliance:sentry', False)
if not raven_config:
return False, 'No \'appliance:sentry\' key was found in the configuration'
if 'dsn' not in raven_config:
return False, 'Appliance returner needs key appliance:sentry:dsn in the configuration'
sentry_data = {
'pillar': 'HIDDEN' if raven_config.get('hide_pillar', True) else __salt__['pillar.raw'](),
'grains': 'HIDDEN' if raven_config.get('hide_grains', True) else __salt__['grains.items'](),
}
data = {
'platform': 'python',
'culprit': 'salt-call',
}
tags = {}
for tag in raven_config.get('tags', ['os', 'master', 'saltversion', 'cpuarch']):
tags[tag] = __salt__['grains.get'](tag)
failed_states = {}
changed_states = {}
if result.get('return'):
if isinstance(result['return'], dict):
for state_id, state_result in six.iteritems(result['return']):
if not state_result['result']:
failed_states[state_id] = state_result
if (state_result['result'] and
len(state_result['changes']) > 0):
changed_states[state_id] = state_result
else:
if not result.get('success') or result.get('retcode', 0) != 0:
failed_states[result['fun']] = result['return']
client = Client(raven_config.get('dsn'), transport=RequestsHTTPTransport)
if has_failed(result):
data['level'] = raven_config.get('error_level', 'error')
message = "Salt error on " + result['id']
sentry_data['result'] = failed_states
try:
msgid = client.capture('raven.events.Message',
message=message, data=data, extra=sentry_data, tags=tags)
logger.info('Message id %s written to sentry', msgid)
except Exception as exc:
logger.error('Can\'t send message to sentry: {0}'.format(exc), exc_info=True)
if raven_config.get('report_errors_only', True):
return
if result['changed_states']:
data['level'] = raven_config.get('change_level', 'info')
message = "Salt change(s) on " + result['id']
sentry_data['result'] = changed_states
try:
msgid = client.capture('raven.events.Message',
message=message, data=data, extra=sentry_data, tags=tags)
logger.info('Message id %s written to sentry', msgid)
except Exception as exc:
logger.error('Can\'t send message to sentry: {0}'.format(exc), exc_info=True)
error = parseElasticsearchError(message['error'])
exceptions = set((error['name'],))
indices = set()
last = None
for i, s in error['exceptions'].items():
indices.add(i.split('][')[1])
last = s[-1].keys()[0]
for ex in s:
exceptions.add(ex.keys()[0])
pprint(error)
exception = {'values': [{'type': error['name'],
'value': error['description']
}]}
raven.http_context(rq)
raven.tags_context(dict( agent=agent,
lastException=last
))
raven.extra_context(dict(request=request,
stacktrace=error['exceptions'],
indices=list(indices),
source=source,
description=error['description']))
print raven.capture('raven.events.Exception', time_spent=ts,
message="%s:%s" % (error['description'], last),
culprit=error['name'],
data=dict(exception=exception)
)
try:
line = sys.stdin.readline().strip()
except KeyboardInterrupt:
print >> sys.stderr, "user abort"
sys.exit(0)
fields = re.split(' +', line.strip(), 12)
if len(fields) != 13:
continue
if should_skip(fields[12]):
continue
read_speed = float(fields[4])
write_speed = float(fields[6])
if read_speed > 1000 or write_speed > 1000:
date = time.strftime('%Y-%m-%d')
pid = fields[1]
client = Client(dsn)
message = '%s: PID(%s) IS USING TOO MUCH DISK IO' % (
socket.gethostname(), pid)
args = {
'time': date + ' ' + fields[0],
'iotop': line.strip(),
'proc': run_command('ls', '-lhR', '/proc/%s/fd' % pid),
}
print >> sys.stderr, message
print >> sys.stderr, json.dumps(args, indent=4)
client.capture('raven.events.Message', message=message, extra=args)
# https://www.felix021.com/blog/read.php?2170
class LoggerClient:
def __init__(self, dsn):
self._client = Client(dsn=dsn, context={})
def _process_sourcefile(self, file_path, line_number, context_lines=10):
FileContext = namedtuple('FileContext', ['pre_context', 'context', 'post_context', 'local_vars'])
start = max(line_number - context_lines, 0)
stop = line_number + context_lines
pre_context = []
post_context = []
local_vars = []
context = None
with open(file_path, 'r') as f:
for i, line in enumerate(f):
current_line = i + 1
if current_line < start:
continue
elif current_line > stop:
break
# search for variable declaration
var = re.match(r'^(?P<name>[a-z]{1}\w*)=\S', line, re.IGNORECASE)
if var:
local_vars.append(var.group('name'))
if current_line < line_number:
pre_context.append(line.rstrip('\n'))
elif current_line == line_number:
context = line.rstrip('\n')
else:
post_context.append(line.rstrip('\n'))
return FileContext(pre_context, context, post_context, local_vars)
def _get_declares(self, declare_output, local_vars):
"""Parse `declare -p` output and get values from local variables"""
out = {}
for var in local_vars:
m = re.search(r'^(?P<name>' + re.escape(var) + r')=(?P<value>\S.*)', declare_output, re.MULTILINE)
if m:
out[m.group('name')] = m.group('value')
return out
def capture(self, shell_args):
frame = {
'filename': shell_args.script,
'function': shell_args.function or 'main',
'lineno': shell_args.lineno,
'module': shell_args.command,
'vars': {},
}
if shell_args.pwd:
abspath = os.path.abspath(os.path.join(shell_args.pwd, shell_args.script))
filename = os.path.basename(abspath)
try:
srcfile = self._process_sourcefile(abspath, shell_args.lineno)
if shell_args.declares:
frame['vars'] = self._get_declares(shell_args.declares, srcfile.local_vars)
frame.update(
filename=filename,
abs_path=abspath,
pre_context=srcfile.pre_context,
context_line=srcfile.context,
post_context=srcfile.post_context
)
except FileNotFoundError:
sys.stderr.write('Could not process file "{}"\n'.format(abspath))
data = {
'exception': {
'values': [{
'module': 'builtins',
'stacktrace': {
'frames': [frame]
},
'type': shell_args.script,
'value': 'error on line {}'.format(shell_args.lineno) if not shell_args.function else "error in '{}' on line {}".format(shell_args.function, shell_args.lineno)
}]
},
}
# add ENV vars and stderr if provided
extra = {}
if shell_args.env:
extra['environment'] = dict([item.split('=', maxsplit=1) for item in shell_args.env.split('\n')])
if shell_args.stderr:
extra['stderr'] = shell_args.stderr
self._client.capture('raven.events.Exception', data=data, extra=extra)
def connect_sentry(result):
'''
Connect to the Sentry server
'''
raven_config = get_config_value('raven', False)
if not raven_config:
return False, 'No \'raven\' key was found in the configuration'
if 'dsn' not in raven_config:
return False, 'Raven returner needs key raven:dsn in the configuration'
sentry_data = {
'pillar': 'HIDDEN' if raven_config.get('hide_pillar', True) else __salt__['pillar.raw'](),
'grains': 'HIDDEN' if raven_config.get('hide_grains', True) else __salt__['grains.items'](),
}
data = {
'platform': 'python',
'culprit': 'salt-call',
}
tags = {}
for tag in raven_config.get('tags', ['os', 'master', 'saltversion', 'cpuarch']):
tags[tag] = __salt__['grains.get'](tag)
failed_states = {}
changed_states = {}
if result.get('return'):
if isinstance(result['return'], dict):
for state_id, state_result in six.iteritems(result['return']):
if not state_result['result']:
failed_states[state_id] = state_result
if (state_result['result'] and
len(state_result['changes']) > 0):
changed_states[state_id] = state_result
else:
if not result.get('success') or result.get('retcode', 0) != 0:
failed_states[result['fun']] = result['return']
client = Client(raven_config.get('dsn'), transport=RequestsHTTPTransport)
if has_failed(result):
data['level'] = raven_config.get('error_level', 'error')
message = "Salt error on " + result['id']
sentry_data['result'] = failed_states
try:
msgid = client.capture('raven.events.Message',
message=message, data=data, extra=sentry_data, tags=tags)
logger.info('Message id %s written to sentry', msgid)
except Exception as exc:
logger.error('Can\'t send message to sentry: {0}'.format(exc), exc_info=True)
if raven_config.get('report_errors_only', True):
return
if result['changed_states']:
data['level'] = raven_config.get('change_level', 'info')
message = "Salt change(s) on " + result['id']
sentry_data['result'] = changed_states
try:
msgid = client.capture('raven.events.Message',
message=message, data=data, extra=sentry_data, tags=tags)
logger.info('Message id %s written to sentry', msgid)
except Exception as exc:
logger.error('Can\'t send message to sentry: {0}'.format(exc), exc_info=True)
class Consumer(Worker):
def _init(self):
# Base initializations.
self._consuming = False
self._timeout = self._config.get('timeout', DEFAULT_TIMEOUT)
self._next_purge = 0
# Initialize Sentry client.
self._client = None
if 'dsn' in self._config:
self._client = Client(dsn=self._config['dsn'])
# Connect to libvarnishapi.
self._vap = api.VarnishAPI(
opt=self._get_vap_options(self._config.get('options', {})),
sopath=self._config.get('libvarnishapi', 'libvarnishapi.so.1'))
# Build list of known tx types.
self._types = TRANSACTIONS.keys()
# Normalize tx delimiting tags.
self._delimiters = {}
for type, item in TRANSACTIONS.iteritems():
self._delimiters[type] = {
'start': [self._vap.VSL_NameNormalize(tag) for tag in item['start']],
'end': [self._vap.VSL_NameNormalize(tag) for tag in item['end']],
}
# Initialize tx filters.
self._filters = {}
for tag, filters in self._config.get('filters', {}).iteritems():
tag = self._vap.VSL_NameNormalize(tag)
items = []
for filter in filters:
# Check regular expression.
assert \
'regexp' in filter, \
'All filters must contain a matching regexp.'
# Build filter.
item = {
'regexp': re.compile(filter['regexp']),
'name': filter.get('name', tag),
'level': filter.get('level', 'error'),
}
# Check level value.
assert \
item['level'] in FILTER_LEVELS, \
'"%s" is not a valid filter level.' % item['level']
# Append filter.
items.append(item)
self._filters[tag] = items
# Initialize tx buffers.
self._buffers = dict((type, {}) for type in self._types)
# Launch consumer thread.
self._thread = threading.Thread(target=self._loop)
self._thread.daemon = True
self._thread.start()
def _poll(self):
# Is the consumer thread still alive?
if not self._thread.is_alive():
raise Exception('Consumer thread has been unexpectedly stopped.')
def _shutdown(self):
# If running, stop the consuming thread.
if self._consuming:
self._consuming = False
def _loop(self):
self._consuming = True
while self._consuming:
try:
self._vap.VSL_Dispatch(self._vap_callBack, priv=False)
except Exception:
logging.getLogger('varnishsentry').error(
'Got unexpected exception while dispatching VSL item.',
exc_info=True)
def _get_vap_options(self, options):
result = []
if options.get('-c', True):
result.append('-c')
if options.get('-b', True):
result.append('-b')
if options.get('-d', False):
result.append('-d')
if options.get('-n'):
result.append('-n')
result.append(options['-n'])
return result
def _vap_callBack(self, priv, tag, fd, length, spec, ptr, bm):
# Log incoming tx item.
logging.getLogger('varnishsentry').debug(
'Handling new transaction item: '
'priv=%s, tag=%s, fd=%s, length=%s, spec=%s, ptr=%s, bm=%s.',
priv, tag, fd, length, spec, ptr, bm)
# Fetch current UNIX timestamp.
now = int(time.time())
# Normalize item fields.
item = self._vap.normalizeDic(priv, tag, fd, length, spec, ptr, bm)
# Does the item belong to any relevant tx type?
if item['type'] in self._types:
# Is this a brand new transaction?
if item['tag'] in self._delimiters[item['type']]['start']:
self._buffers[item['type']][fd] = Transaction(now, item['type'])
# Has we previously seen the tx?
tx = self._buffers[item['type']].get(fd)
if tx is not None:
# Append the new item to the tx.
tx.add(item['tag'], item['msg'])
# Try to match the item.
if item['tag'] in self._filters:
for filter in self._filters[item['tag']]:
if filter['regexp'].search(item['msg']):
tx.match(
item['tag'],
item['msg'],
filter['name'],
filter['level'])
# Is the tx ending?
if item['tag'] in self._delimiters[item['type']]['end']:
# Remove tx instance from the buffer.
del self._buffers[item['type']][fd]
# Commit tx
self._commit_tx(tx, timeout=False)
# Time to purge buffered txs?
if now > self._next_purge:
self._purge_buffers(now)
def _purge_buffers(self, now):
for txs in self._buffers.values():
for id in txs.keys():
tx = txs[id]
# Has the tx timed out?
if now - tx.timestamp >= self._timeout:
# Remove tx instance from the buffer.
del txs[id]
# Commit tx.
self._commit_tx(tx, timeout=True)
# Set next purgation timestamp.
self._next_purge = int(time.time()) + 1
def _commit_tx(self, tx, timeout=False):
if tx.is_matched and self._client is not None:
self._client.capture(
'raven.events.Message',
message=tx.matched_item,
data={
'timestamp': datetime.datetime.fromtimestamp(tx.timestamp),
'logger': 'varnishsentry',
'level': tx.matched_level,
'tags': {
'filter': tx.matched_name,
'type': TRANSACTIONS[tx.type]['name'],
'worker': self.name,
},
# 'sentry.interfaces.Message': {
# 'message': 'My raw message with interpreted strings like %s',
# 'params': ['this'],
# },
# 'sentry.interfaces.Http': {
# 'url': 'http://absolute.uri/foo',
# 'method': 'POST',
# 'data': {
# 'foo': 'bar',
# },
# 'query_string': 'hello=world',
# 'cookies': 'foo=bar',
# 'headers': {
# 'Content-Type': 'text/html',
# },
# 'env': {
# 'REMOTE_ADDR': '192.168.0.1',
# },
# },
# 'sentry.interfaces.User': {
# 'id': 'unique_id',
# 'username': '******',
# 'email': '*****@*****.**',
# 'ip_address': '127.0.0.1',
# },
},
extra={
'timeout': timeout,
'items': tx.items,
'matched': tx.matched_items,
})
dsn = 'https://*****:*****@sentry.fond.io/34' #default: test
client = Client(dsn)
# filename = sys.argv[1]
message = ' '.join(sys.argv[1:])
content = ''.join([line for line in sys.stdin.read()])
if content.strip() == '':
sys.exit(0)
elif len(sys.argv) == 4 and sys.argv[3] == 'diff':
f = open(sys.argv[1], "r")
temp_file = f.read()
f.close()
f = open(sys.argv[2], "w")
f.write(temp_file)
f.close()
# print content
client.capture('raven.events.Message',
message=message,
data={
'request': {
'url': '/sentry/report',
'data': content,
'query_string': '',
'method': 'PUSH',
}
})
