def incrstat(stat, count=1):
if not _is_setup():
log.info('STATSD_HOST or STATSD_PORT not set up')
return
client = Client(settings.STATSD_HOST, settings.STATSD_PORT)
client.increment(_statname(stat), count)
log.info('incrstat: %s %s' % (_statname(stat), count))
def expand_url(orig_url):
statz = Client(*settings.STATSD_SERVER) if settings.STATSD_SERVER else None
try:
url_obj = Url.objects.get(original=orig_url)
except Url.DoesNotExist:
pass
else:
if statz:
statz.increment('fstv.checked_url.database')
return url_obj.target
if urlparse(orig_url).scheme not in ('http', 'https'):
return None
log.debug('Scanning url %s', orig_url)
h = httplib2.Http(timeout=10)
# We don't get a content-location header when httplib2 follows redirects
# for a HEAD, so let's track it ourselves I guess.
h.follow_redirects = False
redirects = 5
url = orig_url
while True:
try:
resp, cont = h.request(url, method='HEAD', headers={'User-Agent': 'friendstream/1.0'})
except httplib.InvalidURL:
log.debug("Oops, %s isn't really an URL at all, skipping", url)
return None
# TODO: what does httplib2 raise when there's no Location header? (does it raise anything when follow_redirects=False?)
except (httplib2.ServerNotFoundError, httplib.BadStatusLine, socket.timeout, socket.error), exc:
log.debug("Oops, %s for URL %s (use it for now): %s", type(exc).__name__, url, str(exc))
return url
if resp.status in (301, 302, 303, 307):
location = resp['location']
try:
# Either the location is 7 bits, in which case this works fine...
location.decode('ascii')
except UnicodeDecodeError:
# ...or it's erroneously UTF-8 and we need to %-escape the high bytes.
location = quote(location, safe=';/?:@&=+$,%')
url = urljoin(url, location)
redirects -= 1
if redirects <= 0:
log.debug("Oops, hit redirect limit for %s (use it for now)", url)
return url
continue
break
def poll_account(account_pk, limited=False):
try:
account = Account.objects.get(pk=account_pk)
except Account.DoesNotExist:
return
if not account.authinfo:
return
if account.error:
# Don't try to scan accounts that have errors until they're saved again.
return
if account.user is None:
log.debug("Oops, account %r has no user?", account)
return
if limited:
now = datetime.now()
if account.user.last_login < now - timedelta(days=7):
log.debug("Marking %s's %s stream stale (hasn't logged in for a week)", account.user.username, account.service)
account.stale = True
account.save()
return
if account.last_updated > now - timedelta(minutes=14):
log.debug("Skipping %s's %s stream (updated too soon)", account.user.username, account.service)
return
if account.service == 'twitter.com':
poll_twitter(account)
elif account.service == 'facebook.com':
poll_facebook(account)
# TODO: vimeo, youtube accounts
account.last_updated = datetime.now()
account.save()
if settings.STATSD_SERVER:
statz = Client(*settings.STATSD_SERVER)
statz.increment('fstv.account_updated')
def main():
'''Main entry point for the meter_read CLI.'''
args = rcfile('meter_read', docopt(__doc__, version=__version__))
logging.basicConfig()
log = logging.getLogger(__name__)
if args['verbose']:
log.setLevel(logging.DEBUG)
sc = Client(args['addr'], args['port'])
re_valid = re.compile(args['valid'])
re_timer = re.compile(args['timer'])
aliasses = json.loads(args.get('aliasses', '{}'))
log.debug('found aliasses: %s' % aliasses)
log.debug('start reading %s' % args['dev'])
s = serial.Serial(port=args['dev'], baudrate=args['baud'])
while True:
line = s.readline()
match = re_valid.match(line)
if match:
check, name, value = match.groups()
name = aliasses.get(name, name)
log.debug("{0} {1}".format(name, value))
# send to statsd
if re_timer.match(line):
sc.timing(name, float(value))
else:
sc.gauge(name, float(value))
else:
log.debug('error %s' % unicode(line, errors='ignore'))
sc.increment('error')
})
created_at = datetime.utcfromtimestamp(status.created_at_in_seconds)
log.debug("Saving UserStream for user %r video %r poster %r posted %r", account.user, video, poster, created_at)
us = UserStream(
user=account.user,
video=video,
poster=poster,
posted=created_at,
message=status.text,
message_service='twitter.com',
message_ident=status.id,
)
us.save()
if statz:
statz.increment('fstv.video.new')
else:
log.debug("Video for %s already in %s's stream", url, account.user.username)
if statz:
statz.increment('fstv.video.old')
log.debug('Done scanning statuses for %s', account.user.username)
def poll_facebook(account):
facepi = facebook.GraphAPI(account.authinfo)
try:
home = facepi.get_object('me/home', limit=100)
except facebook.GraphAPIError, exc:
if exc.type in ('OAuthException', 'IDInvalidException'):
#!/usr/bin/env python
import time
from multiprocessing import Process
from pystatsd import Client, Server
def worker():
srvr = Server(debug=True, flush_interval=500)
srvr.serve()
p = Process(target=worker, daemon=False)
p.start()
time.sleep(1)
sc = Client('localhost', 8125)
sc.timing('python_test.time', 500)
sc.increment('python_test.inc_int')
sc.decrement('python_test.decr_int')
sc.gauge('python_test.gauge', 42)
time.sleep(2)
p.terminate()
class GraphiteProcessor(Plugin):
title = 'Graphite'
conf_key = 'graphite'
project_conf_form = GraphiteConfigurationForm
def __init__(self, min_level=NOTSET, include_loggers=NOTSET, exclude_loggers=NOTSET,
host=NOTSET, port=NOTSET, prefix=NOTSET, *args, **kwargs):
super(GraphiteProcessor, self).__init__(*args, **kwargs)
if min_level is NOTSET:
min_level = settings.GRAPIHTE_LEVEL
if include_loggers is NOTSET:
include_loggers = settings.GRAPHITE_INCLUDE_LOGGERS
if exclude_loggers is NOTSET:
exclude_loggers = settings.GRAPHITE_EXCLUDE_LOGGERS
if host is NOTSET:
host = settings.GRAPHITE_HOST
if port is NOTSET:
port = settings.GRAPHITE_PORT
if prefix is NOTSET:
prefix = settings.GRAPHITE_PREFIX
self.min_level = min_level
self.include_loggers = include_loggers
self.exclude_loggers = exclude_loggers
self.host = host
self.port = port
self.prefix = prefix
self.client = Client(host=self.host, port=self.port)
def record_event(self, group, event, fail_silently=True):
project = group.project
host = self.get_option('host', project) or self.host
port = self.get_option('port', project) or self.port
prefix = self.get_option('prefix', project) or self.prefix
key = '.'.join([prefix, event.message_top])
self.client.increment(key)
def should_record(self, group, event):
project = group.project
host = self.get_option('host', project) or self.host
if not host:
return False
port = self.get_option('port', project) or self.port
if not port:
return False
prefix = self.get_option('prefix', project) or self.prefix
if not prefix:
return False
min_level = self.get_option('min_level', project) or self.min_level
if min_level is not None and int(group.level) < min_level:
return False
include_loggers = self.get_option('include_loggers', project) or self.include_loggers
if include_loggers is not None and group.logger not in include_loggers:
return False
exclude_loggers = self.get_option('exclude_loggers', project) or self.exclude_loggers
if exclude_loggers and group.logger in exclude_loggers:
return False
return True
def post_process(self, group, event, is_new, is_sample, **kwargs):
if not self.should_record(group, event):
return
self.record_event(group, event)
#!/usr/bin/env python
from pystatsd import Client, Server
sc = Client('localhost', 8125)
sc.timing('python_test.time', 500)
sc.increment('python_test.inc_int')
sc.decrement('python_test.decr_int')
srvr = Server(debug=True)
srvr.serve()
#!/usr/bin/env python
from pystatsd import Client, Server
sc = Client("localhost", 8125)
sc.timing("python_test.time", 500)
sc.increment("python_test.inc_int")
sc.decrement("python_test.decr_int")
sc.gauge("python_test.gauge", 42)
srvr = Server(debug=True)
srvr.serve()
class ClientBasicsTestCase(unittest.TestCase):
"""
Tests the basic operations of the client
"""
def setUp(self):
self.patchers = []
socket_patcher = mock.patch('pystatsd.statsd.socket.socket')
self.mock_socket = socket_patcher.start()
self.patchers.append(socket_patcher)
self.client = Client()
self.addr = (socket.gethostbyname(self.client.host), self.client.port)
def test_client_create(self):
host, port = ('example.com', 8888)
client = Client(host=host, port=port, prefix='pystatsd.tests')
self.assertEqual(client.host, host)
self.assertEqual(client.port, port)
self.assertEqual(client.prefix, 'pystatsd.tests')
self.assertEqual(client.addr, (socket.gethostbyname(host), port))
def test_basic_client_incr(self):
stat = 'pystatsd.unittests.test_basic_client_incr'
stat_str = stat + ':1|c'
self.client.increment(stat)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_decr(self):
stat = 'pystatsd.unittests.test_basic_client_decr'
stat_str = stat + ':-1|c'
self.client.decrement(stat)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_update_stats(self):
stat = 'pystatsd.unittests.test_basic_client_update_stats'
stat_str = stat + ':5|c'
self.client.update_stats(stat, 5)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_update_stats_multi(self):
stats = [
'pystatsd.unittests.test_basic_client_update_stats',
'pystatsd.unittests.test_basic_client_update_stats_multi'
]
data = dict((stat, "%s|c" % '5') for stat in stats)
self.client.update_stats(stats, 5)
for stat, value in data.items():
stat_str = stat + value
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_call_any(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_timing(self):
stat = 'pystatsd.unittests.test_basic_client_timing.time'
stat_str = stat + ':5.000000|ms'
self.client.timing(stat, 5)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_timing_since(self):
ts = (1971, 6, 29, 4, 13, 0, 0, 0, -1)
now = time.mktime(ts)
# add 5 seconds
ts = (1971, 6, 29, 4, 13, 5, 0, 0, -1)
then = time.mktime(ts)
mock_time_patcher = mock.patch('time.time', return_value=now)
mock_time_patcher.start()
stat = 'pystatsd.unittests.test_basic_client_timing_since.time'
stat_str = stat + ':-5000000.000000|ms'
self.client.timing_since(stat, then)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
mock_time_patcher.stop()
def tearDown(self):
for patcher in self.patchers:
patcher.stop()
def record(millis):
c = Client('hammer.wallrazer.com', 8125)
print "recording...\n"
c.increment('wallrazer.deploys')
c.timing('wallrazer.deploys.time', millis)
class ClientBasicsTestCase(unittest.TestCase):
"""
Tests the basic operations of the client
"""
def setUp(self):
self.patchers = []
socket_patcher = mock.patch('pystatsd.statsd.socket.socket')
self.mock_socket = socket_patcher.start()
self.patchers.append(socket_patcher)
self.client = Client()
self.addr = (socket.gethostbyname(self.client.host), self.client.port)
def test_client_create(self):
host, port = ('example.com', 8888)
client = Client(
host=host,
port=port,
prefix='pystatsd.tests')
self.assertEqual(client.host, host)
self.assertEqual(client.port, port)
self.assertEqual(client.prefix, 'pystatsd.tests')
self.assertEqual(client.addr, (socket.gethostbyname(host), port))
def test_basic_client_incr(self):
stat = 'pystatsd.unittests.test_basic_client_incr'
stat_str = stat + ':1|c'
self.client.increment(stat)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_decr(self):
stat = 'pystatsd.unittests.test_basic_client_decr'
stat_str = stat + ':-1|c'
self.client.decrement(stat)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_update_stats(self):
stat = 'pystatsd.unittests.test_basic_client_update_stats'
stat_str = stat + ':5|c'
self.client.update_stats(stat, 5)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_update_stats_multi(self):
stats = [
'pystatsd.unittests.test_basic_client_update_stats',
'pystatsd.unittests.test_basic_client_update_stats_multi'
]
data = dict((stat, "%s|c" % '5') for stat in stats)
self.client.update_stats(stats, 5)
for stat, value in data.items():
stat_str = stat + value
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_call_any(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_timing(self):
stat = 'pystatsd.unittests.test_basic_client_timing.time'
stat_str = stat + ':5.000000|ms'
self.client.timing(stat, 5)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
def test_basic_client_timing_since(self):
ts = (1971, 6, 29, 4, 13, 0, 0, 0, -1)
now = time.mktime(ts)
# add 5 seconds
ts = (1971, 6, 29, 4, 13, 5, 0, 0, -1)
then = time.mktime(ts)
mock_time_patcher = mock.patch('time.time', return_value=now)
mock_time_patcher.start()
stat = 'pystatsd.unittests.test_basic_client_timing_since.time'
stat_str = stat + ':-5000000.000000|ms'
self.client.timing_since(stat, then)
# thanks tos9 in #python for 'splaining the return_value bit.
self.mock_socket.return_value.sendto.assert_called_with(
bytes(stat_str, 'utf-8'), self.addr)
mock_time_patcher.stop()
def tearDown(self):
for patcher in self.patchers:
patcher.stop()