def handle(self,
message_id=None,
client_hostname=None,
client_ip=None,
orig_client_hostname=None,
orig_client_ip=None):
statsd.incr('postfix.messages.smtpd', 1)
pass
def _task_handler(self=None, message=None):
ts_message = TSMessage(message.pop('data'), message)
# TODO: @will-norris backwards compat - remove
if MARSHMALLOW_2:
deserialized_data, errors = schema.load(ts_message)
if errors:
statsd.incr(
'tasks.{}.ts_task.errors.schema'.format(task_name))
error_msg = 'inbound schema validation error for event {}'.format(
event_name)
logger.error(error_msg,
extra={
'errors': errors,
'data': ts_message
})
raise SchemaError(error_msg,
errors=errors,
data=ts_message)
else:
logger.info('received ts_task on {}'.format(event_name))
ts_message.data = deserialized_data
# passing task_func instead of passing self - @will-norris
return task_func(
self, ts_message) if bind else task_func(ts_message)
else:
try:
deserialized_data = schema.load(ts_message)
except ValidationError as vex:
statsd.incr(
'tasks.{}.ts_task.errors.schema'.format(task_name))
error_msg = 'inbound schema validation error for event {}'.format(
event_name)
logger.error(error_msg,
extra={
'errors': vex.messages,
'data': ts_message
})
raise SchemaError(error_msg,
errors=vex.messages,
data=ts_message)
else:
logger.info('received ts_task on {}'.format(event_name))
ts_message.data = deserialized_data
# passing task_func instead of passing self - @will-norris
return task_func(
self, ts_message) if bind else task_func(ts_message)
def handle(self,
message_id=None,
to_email=None,
relay=None,
conn_use=None,
delay=None,
delays=None,
dsn=None,
status=None,
response=None):
stat = 'recv' if '127.0.0.1' in relay else 'send'
statsd.incr('postfix.messages.{}.status.{}'.format(stat, status), 1)
statsd.incr(
'postfix.messages.{}.resp_codes.{}'.format(stat,
dsn.replace(".",
"_")), 1
) # we don't want to create tree from domain name so "." are forbidden
def annotation(self):
settings = self.request.registry.settings
try:
document = util.elasticsearch_client(settings).get(
index=settings["elasticsearch_index"],
doc_type="annotation",
id=self.request.matchdict["id"])
except exceptions.NotFoundError:
statsd.incr("views.annotation.404.annotation_not_found")
raise httpexceptions.HTTPNotFound(_("Annotation not found"))
try:
annotation_id, document_uri = util.parse_document(document)
except util.InvalidAnnotationError as exc:
statsd.incr("views.annotation.422.{}".format(exc.reason))
raise httpexceptions.HTTPUnprocessableEntity(str(exc))
# Remove any existing #fragment identifier from the URI before we
# append our own.
document_uri = parse.urldefrag(document_uri)[0]
if not (document_uri.startswith("http://")
or document_uri.startswith("https://")):
statsd.incr("views.annotation.422.not_an_http_or_https_document")
raise httpexceptions.HTTPUnprocessableEntity(
_("Sorry, but it looks like this annotation was made on a "
"document that is not publicly available."))
via_url = "{via_base_url}/{uri}#annotations:{id}".format(
via_base_url=settings["via_base_url"],
uri=document_uri,
id=annotation_id)
extension_url = "{uri}#annotations:{id}".format(uri=document_uri,
id=annotation_id)
parsed_url = parse.urlparse(document_uri)
pretty_url = parsed_url.netloc[:NETLOC_MAX_LENGTH]
if len(parsed_url.netloc) > NETLOC_MAX_LENGTH:
pretty_url = pretty_url + jinja2.Markup("…")
statsd.incr("views.annotation.200.annotation_found")
return {
"data":
json.dumps({
# Warning: variable names change from python_style to
# javaScriptStyle here!
"chromeExtensionId": settings["chrome_extension_id"],
"viaUrl": via_url,
"extensionUrl": extension_url,
}),
"pretty_url":
pretty_url
}
def reportStats(self, stats):
for row in stats:
base = 'haproxy.%s.%s.' % (row['pxname'], row['svname'])
for key in GAUGES:
value = row[key]
if value != '':
statsd.gauge(base + key, int(value))
for key in COUNTERS:
metric = base + key
newvalue = row[key]
if newvalue == '':
continue
newvalue = int(newvalue)
oldvalue = self.prevdata.get(metric)
if oldvalue is not None:
value = newvalue - oldvalue
statsd.incr(metric, value)
self.prevdata[metric] = newvalue
def annotation(self):
settings = self.request.registry.settings
try:
document = util.elasticsearch_client(settings).get(
index=settings["elasticsearch_index"],
doc_type="annotation",
id=self.request.matchdict["id"])
except exceptions.NotFoundError:
statsd.incr("views.annotation.404.annotation_not_found")
raise httpexceptions.HTTPNotFound(_("Annotation not found"))
try:
annotation_id, document_uri = util.parse_document(document)
except util.InvalidAnnotationError as exc:
statsd.incr("views.annotation.422.{}".format(exc.reason))
raise httpexceptions.HTTPUnprocessableEntity(str(exc))
# Remove any existing #fragment identifier from the URI before we
# append our own.
document_uri = parse.urldefrag(document_uri)[0]
if not (document_uri.startswith("http://") or
document_uri.startswith("https://")):
statsd.incr("views.annotation.422.not_an_http_or_https_document")
raise httpexceptions.HTTPUnprocessableEntity(
_("Sorry, but it looks like this annotation was made on a "
"document that is not publicly available."))
via_url = "{via_base_url}/{uri}#annotations:{id}".format(
via_base_url=settings["via_base_url"],
uri=document_uri,
id=annotation_id)
extension_url = "{uri}#annotations:{id}".format(
uri=document_uri, id=annotation_id)
parsed_url = parse.urlparse(document_uri)
pretty_url = parsed_url.netloc[:NETLOC_MAX_LENGTH]
if len(parsed_url.netloc) > NETLOC_MAX_LENGTH:
pretty_url = pretty_url + jinja2.Markup("…")
statsd.incr("views.annotation.200.annotation_found")
return {
"data": json.dumps({
# Warning: variable names change from python_style to
# javaScriptStyle here!
"chromeExtensionId": settings["chrome_extension_id"],
"viaUrl": via_url,
"extensionUrl": extension_url,
}),
"pretty_url": pretty_url
}
def handle(self,
message_id=None,
to_email=None,
orig_to_email=None,
relay=None,
delay=None,
delays=None,
dsn=None,
status=None,
response=None):
pemail = self.local_addresses_re.search(to_email)
if pemail:
search = pemail.group(1)
name, count = self.local_addresses[search]
logger.debug('Local address <%s> count (%s) as "%s"', search,
count, name)
statsd.incr('postfix.messages.local', 1)
statsd.incr('postfix.messages.in.status.{}'.format(status), 1)
statsd.incr(
'postfix.messages.in.resp_codes.{}'.format(
dsn.replace(".", "_")), 1
) # we don't want to create tree from domain name so "." are forbidden
def slurp(entry):
statsd.incr('workers')
for url in [entry.media_content[0]['url'], entry.ffffound_source['url']]:
try:
wget(url)
except :
statsd.incr('error')
statsd.incr('image')
statsd.decr('workers')
def handle(self, message_id=None, to_email=None, orig_to_email=None, relay=None, delay=None, delays=None, dsn=None, status=None, response=None):
pemail = self.local_addresses_re.search(to_email)
if pemail:
search = pemail.group(1)
name, count = self.local_addresses[search]
logger.debug('Local address <%s> count (%s) as "%s"', search, count, name)
statsd.incr('postfix.messages.local', 1)
statsd.incr('postfix.messages.in.status.{}'.format(status), 1)
statsd.incr('postfix.messages.in.resp_codes.{}'.format(dsn.replace(".", "_")), 1) # we don't want to create tree from domain name so "." are forbidden
def process_exception(self, request, exception):
self.timer.stop()
statsd.incr("view.exceptions.{0}".format(self.current_view))
def handle(self, message_id=None, bounce_message_id=None):
statsd.incr('postfix.messages.bounce', 1)
def handle(self, message_id=None, client_hostname=None, client_ip=None, orig_client_hostname=None, orig_client_ip=None):
statsd.incr('postfix.messages.smtpd', 1)
pass
def handle(self, message_id=None, to_email=None, relay=None, conn_use=None, delay=None, delays=None, dsn=None, status=None, response=None):
stat = 'recv' if '127.0.0.1' in relay else 'send'
statsd.incr('postfix.messages.{}.status.{}'.format(stat, status), 1)
statsd.incr('postfix.messages.{}.resp_codes.{}'.format(stat, dsn.replace(".", "_")), 1) # we don't want to create tree from domain name so "." are forbidden
def annotation(self):
settings = self.request.registry.settings
try:
document = self.request.es.get(
index=settings["elasticsearch_index"],
doc_type="annotation",
id=self.request.matchdict["id"])
except exceptions.NotFoundError:
statsd.incr("views.annotation.404.annotation_not_found")
raise httpexceptions.HTTPNotFound(_("Annotation not found"))
try:
parsed_document = util.parse_document(document)
authority = parsed_document["authority"]
annotation_id = parsed_document["annotation_id"]
document_uri = parsed_document["document_uri"]
show_metadata = parsed_document["show_metadata"]
quote = parsed_document["quote"]
text = parsed_document["text"]
except util.DeletedAnnotationError:
statsd.incr("views.annotation.404.annotation_not_found")
raise httpexceptions.HTTPNotFound(_("Annotation not found"))
except util.InvalidAnnotationError as exc:
statsd.incr("views.annotation.422.{}".format(exc.reason))
raise httpexceptions.HTTPUnprocessableEntity(str(exc))
# Remove any existing #fragment identifier from the URI before we
# append our own.
document_uri = parse.urldefrag(document_uri)[0]
if not _is_valid_http_url(document_uri):
statsd.incr("views.annotation.422.not_an_http_or_https_document")
raise httpexceptions.HTTPUnprocessableEntity(
_("Sorry, but it looks like this annotation was made on a "
"document that is not publicly available."))
via_url = None
if _can_use_proxy(settings, authority=authority):
via_url = "{via_base_url}/{uri}#annotations:{id}".format(
via_base_url=settings["via_base_url"],
uri=document_uri,
id=annotation_id)
extension_url = "{uri}#annotations:{id}".format(
uri=document_uri, id=annotation_id)
pretty_url = util.get_pretty_url(document_uri)
title = util.get_boilerplate_quote(document_uri)
statsd.incr("views.annotation.200.annotation_found")
return {
"data": json.dumps({
# Warning: variable names change from python_style to
# javaScriptStyle here!
"chromeExtensionId": settings["chrome_extension_id"],
"viaUrl": via_url,
"extensionUrl": extension_url,
}),
"show_metadata": show_metadata,
"pretty_url": pretty_url,
"quote": quote,
"text": text,
"title": title
}
def handle(self, message_id=None, ext_message_id=None):
statsd.incr('postfix.messages.cleanup', 1)
def index(request):
statsd.incr("views.index.302.redirected_to_hypothesis")
raise httpexceptions.HTTPFound(
location=request.registry.settings["hypothesis_url"])
def process_request(self, request):
self.current_view = resolve(request.path).url_name
statsd.incr("view.exceptions.{0}".format(self.current_view))
self.timer = statsd.timer('view.{0}'.format(self.current_view))
self.timer.start()
def process_request(self, request):
self.current_view = resolve(request.path).url_name
statsd.incr("view.exceptions.{0}".format(self.current_view))
self.timer = statsd.timer('view.{0}'.format(self.current_view))
self.timer.start()
def send_ts_task(event_name, schema, data, **kwargs):
"""Send a Thunderstorm messaging event
The correct task name is derived from the event name.
Example:
send_ts_task(
'domain.action.request', DomainActionRequestSchema(),
payload
)
Args:
event_name (str): The event name (this is also the routing key)
schema (marshmallow.Schema): The schema instance the payload must
comply to
data (dict or list): The event data to be emitted (does not need to be serialized yet)
Raises:
SchemaError If schema validation fails.
Returns:
The result of the send_task call.
"""
if {'name', 'args', 'exchange', 'routing_key'} & set(kwargs.keys()):
raise ValueError('Cannot override name, args, exchange or routing_key')
task_name = ts_task_name(event_name)
# TODO: @will-norris backwards compat - remove
if MARSHMALLOW_2:
data = schema.dump(data).data
data, errors = schema.load(data)
if errors:
statsd.incr(
'tasks.{}.send_ts_task.errors.schema'.format(task_name))
error_msg = 'Outbound schema validation error for event {}'.format(
event_name) # noqa
logger.error(error_msg, extra={'errors': errors, 'data': data})
raise SchemaError(error_msg, errors=errors, data=data)
else:
logger.info('send_ts_task on {}'.format(event_name))
event = {'data': data}
return current_app.send_task(task_name, (event, ),
exchange='ts.messaging',
routing_key=event_name,
**kwargs)
else:
try:
data = schema.dump(data)
except ValidationError as vex:
error_msg = 'Error serializing queue message data'
raise SchemaError(error_msg, errors=vex.messages, data=data)
try:
schema.load(data)
except ValidationError as vex:
statsd.incr(
'tasks.{}.send_ts_task.errors.schema'.format(task_name))
error_msg = 'Outbound schema validation error for event {}'.format(
event_name) # noqa
logger.error(error_msg,
extra={
'errors': vex.messages,
'data': data
})
raise SchemaError(error_msg, errors=vex.messages, data=data)
else:
logger.info('send_ts_task on {}'.format(event_name))
event = {'data': data}
return current_app.send_task(task_name, (event, ),
exchange='ts.messaging',
routing_key=event_name,
**kwargs)
def handle(self, message_id=None, ext_message_id=None):
statsd.incr('postfix.messages.cleanup', 1)
def process_exception(self, request, exception):
self.timer.stop()
statsd.incr("view.exceptions.{0}".format(self.current_view))
def handle(self, message_id=None, bounce_message_id=None):
statsd.incr('postfix.messages.bounce', 1)
def on_success(self, rv, task_id, args, kwargs):
statsd.incr("celery.task." + self.name + ".success")
def on_failure(self, exc, task_id, args, kwargs, einfo):
statsd.incr("celery.task." + self.name + ".error")
def index(request):
statsd.incr("views.index.302.redirected_to_hypothesis")
raise httpexceptions.HTTPFound(
location=request.registry.settings["hypothesis_url"])
#!/usr/bin/env python
from contextlib import closing
import requests
from statsd.defaults.env import statsd
url = 'http://cdimage.debian.org/debian-cd/7.7.0/amd64/iso-cd/debian-7.7.0-amd64-netinst.iso'
with closing(requests.get(url, stream=True)) as r:
with open('debian.iso', 'w') as f:
for bloc in r.iter_content(chunk_size=2048):
f.write(bloc)
statsd.incr('debian', len(bloc))