def incrstat(stat, count=1): if not _is_setup(): log.info('STATSD_HOST or STATSD_PORT not set up') return client = Client(settings.STATSD_HOST, settings.STATSD_PORT) client.increment(_statname(stat), count) log.info('incrstat: %s %s' % (_statname(stat), count))
def expand_url(orig_url): statz = Client(*settings.STATSD_SERVER) if settings.STATSD_SERVER else None try: url_obj = Url.objects.get(original=orig_url) except Url.DoesNotExist: pass else: if statz: statz.increment('fstv.checked_url.database') return url_obj.target if urlparse(orig_url).scheme not in ('http', 'https'): return None log.debug('Scanning url %s', orig_url) h = httplib2.Http(timeout=10) # We don't get a content-location header when httplib2 follows redirects # for a HEAD, so let's track it ourselves I guess. h.follow_redirects = False redirects = 5 url = orig_url while True: try: resp, cont = h.request(url, method='HEAD', headers={'User-Agent': 'friendstream/1.0'}) except httplib.InvalidURL: log.debug("Oops, %s isn't really an URL at all, skipping", url) return None # TODO: what does httplib2 raise when there's no Location header? (does it raise anything when follow_redirects=False?) except (httplib2.ServerNotFoundError, httplib.BadStatusLine, socket.timeout, socket.error), exc: log.debug("Oops, %s for URL %s (use it for now): %s", type(exc).__name__, url, str(exc)) return url if resp.status in (301, 302, 303, 307): location = resp['location'] try: # Either the location is 7 bits, in which case this works fine... location.decode('ascii') except UnicodeDecodeError: # ...or it's erroneously UTF-8 and we need to %-escape the high bytes. location = quote(location, safe=';/?:@&=+$,%') url = urljoin(url, location) redirects -= 1 if redirects <= 0: log.debug("Oops, hit redirect limit for %s (use it for now)", url) return url continue break
def poll_account(account_pk, limited=False): try: account = Account.objects.get(pk=account_pk) except Account.DoesNotExist: return if not account.authinfo: return if account.error: # Don't try to scan accounts that have errors until they're saved again. return if account.user is None: log.debug("Oops, account %r has no user?", account) return if limited: now = datetime.now() if account.user.last_login < now - timedelta(days=7): log.debug("Marking %s's %s stream stale (hasn't logged in for a week)", account.user.username, account.service) account.stale = True account.save() return if account.last_updated > now - timedelta(minutes=14): log.debug("Skipping %s's %s stream (updated too soon)", account.user.username, account.service) return if account.service == 'twitter.com': poll_twitter(account) elif account.service == 'facebook.com': poll_facebook(account) # TODO: vimeo, youtube accounts account.last_updated = datetime.now() account.save() if settings.STATSD_SERVER: statz = Client(*settings.STATSD_SERVER) statz.increment('fstv.account_updated')
def main(): '''Main entry point for the meter_read CLI.''' args = rcfile('meter_read', docopt(__doc__, version=__version__)) logging.basicConfig() log = logging.getLogger(__name__) if args['verbose']: log.setLevel(logging.DEBUG) sc = Client(args['addr'], args['port']) re_valid = re.compile(args['valid']) re_timer = re.compile(args['timer']) aliasses = json.loads(args.get('aliasses', '{}')) log.debug('found aliasses: %s' % aliasses) log.debug('start reading %s' % args['dev']) s = serial.Serial(port=args['dev'], baudrate=args['baud']) while True: line = s.readline() match = re_valid.match(line) if match: check, name, value = match.groups() name = aliasses.get(name, name) log.debug("{0} {1}".format(name, value)) # send to statsd if re_timer.match(line): sc.timing(name, float(value)) else: sc.gauge(name, float(value)) else: log.debug('error %s' % unicode(line, errors='ignore')) sc.increment('error')
}) created_at = datetime.utcfromtimestamp(status.created_at_in_seconds) log.debug("Saving UserStream for user %r video %r poster %r posted %r", account.user, video, poster, created_at) us = UserStream( user=account.user, video=video, poster=poster, posted=created_at, message=status.text, message_service='twitter.com', message_ident=status.id, ) us.save() if statz: statz.increment('fstv.video.new') else: log.debug("Video for %s already in %s's stream", url, account.user.username) if statz: statz.increment('fstv.video.old') log.debug('Done scanning statuses for %s', account.user.username) def poll_facebook(account): facepi = facebook.GraphAPI(account.authinfo) try: home = facepi.get_object('me/home', limit=100) except facebook.GraphAPIError, exc: if exc.type in ('OAuthException', 'IDInvalidException'):
#!/usr/bin/env python import time from multiprocessing import Process from pystatsd import Client, Server def worker(): srvr = Server(debug=True, flush_interval=500) srvr.serve() p = Process(target=worker, daemon=False) p.start() time.sleep(1) sc = Client('localhost', 8125) sc.timing('python_test.time', 500) sc.increment('python_test.inc_int') sc.decrement('python_test.decr_int') sc.gauge('python_test.gauge', 42) time.sleep(2) p.terminate()
class GraphiteProcessor(Plugin): title = 'Graphite' conf_key = 'graphite' project_conf_form = GraphiteConfigurationForm def __init__(self, min_level=NOTSET, include_loggers=NOTSET, exclude_loggers=NOTSET, host=NOTSET, port=NOTSET, prefix=NOTSET, *args, **kwargs): super(GraphiteProcessor, self).__init__(*args, **kwargs) if min_level is NOTSET: min_level = settings.GRAPIHTE_LEVEL if include_loggers is NOTSET: include_loggers = settings.GRAPHITE_INCLUDE_LOGGERS if exclude_loggers is NOTSET: exclude_loggers = settings.GRAPHITE_EXCLUDE_LOGGERS if host is NOTSET: host = settings.GRAPHITE_HOST if port is NOTSET: port = settings.GRAPHITE_PORT if prefix is NOTSET: prefix = settings.GRAPHITE_PREFIX self.min_level = min_level self.include_loggers = include_loggers self.exclude_loggers = exclude_loggers self.host = host self.port = port self.prefix = prefix self.client = Client(host=self.host, port=self.port) def record_event(self, group, event, fail_silently=True): project = group.project host = self.get_option('host', project) or self.host port = self.get_option('port', project) or self.port prefix = self.get_option('prefix', project) or self.prefix key = '.'.join([prefix, event.message_top]) self.client.increment(key) def should_record(self, group, event): project = group.project host = self.get_option('host', project) or self.host if not host: return False port = self.get_option('port', project) or self.port if not port: return False prefix = self.get_option('prefix', project) or self.prefix if not prefix: return False min_level = self.get_option('min_level', project) or self.min_level if min_level is not None and int(group.level) < min_level: return False include_loggers = self.get_option('include_loggers', project) or self.include_loggers if include_loggers is not None and group.logger not in include_loggers: return False exclude_loggers = self.get_option('exclude_loggers', project) or self.exclude_loggers if exclude_loggers and group.logger in exclude_loggers: return False return True def post_process(self, group, event, is_new, is_sample, **kwargs): if not self.should_record(group, event): return self.record_event(group, event)
#!/usr/bin/env python from pystatsd import Client, Server sc = Client('localhost', 8125) sc.timing('python_test.time', 500) sc.increment('python_test.inc_int') sc.decrement('python_test.decr_int') srvr = Server(debug=True) srvr.serve()
#!/usr/bin/env python from pystatsd import Client, Server sc = Client("localhost", 8125) sc.timing("python_test.time", 500) sc.increment("python_test.inc_int") sc.decrement("python_test.decr_int") sc.gauge("python_test.gauge", 42) srvr = Server(debug=True) srvr.serve()
class ClientBasicsTestCase(unittest.TestCase): """ Tests the basic operations of the client """ def setUp(self): self.patchers = [] socket_patcher = mock.patch('pystatsd.statsd.socket.socket') self.mock_socket = socket_patcher.start() self.patchers.append(socket_patcher) self.client = Client() self.addr = (socket.gethostbyname(self.client.host), self.client.port) def test_client_create(self): host, port = ('example.com', 8888) client = Client(host=host, port=port, prefix='pystatsd.tests') self.assertEqual(client.host, host) self.assertEqual(client.port, port) self.assertEqual(client.prefix, 'pystatsd.tests') self.assertEqual(client.addr, (socket.gethostbyname(host), port)) def test_basic_client_incr(self): stat = 'pystatsd.unittests.test_basic_client_incr' stat_str = stat + ':1|c' self.client.increment(stat) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_decr(self): stat = 'pystatsd.unittests.test_basic_client_decr' stat_str = stat + ':-1|c' self.client.decrement(stat) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_update_stats(self): stat = 'pystatsd.unittests.test_basic_client_update_stats' stat_str = stat + ':5|c' self.client.update_stats(stat, 5) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_update_stats_multi(self): stats = [ 'pystatsd.unittests.test_basic_client_update_stats', 'pystatsd.unittests.test_basic_client_update_stats_multi' ] data = dict((stat, "%s|c" % '5') for stat in stats) self.client.update_stats(stats, 5) for stat, value in data.items(): stat_str = stat + value # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_call_any( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_timing(self): stat = 'pystatsd.unittests.test_basic_client_timing.time' stat_str = stat + ':5.000000|ms' self.client.timing(stat, 5) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_timing_since(self): ts = (1971, 6, 29, 4, 13, 0, 0, 0, -1) now = time.mktime(ts) # add 5 seconds ts = (1971, 6, 29, 4, 13, 5, 0, 0, -1) then = time.mktime(ts) mock_time_patcher = mock.patch('time.time', return_value=now) mock_time_patcher.start() stat = 'pystatsd.unittests.test_basic_client_timing_since.time' stat_str = stat + ':-5000000.000000|ms' self.client.timing_since(stat, then) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) mock_time_patcher.stop() def tearDown(self): for patcher in self.patchers: patcher.stop()
def record(millis): c = Client('hammer.wallrazer.com', 8125) print "recording...\n" c.increment('wallrazer.deploys') c.timing('wallrazer.deploys.time', millis)
class ClientBasicsTestCase(unittest.TestCase): """ Tests the basic operations of the client """ def setUp(self): self.patchers = [] socket_patcher = mock.patch('pystatsd.statsd.socket.socket') self.mock_socket = socket_patcher.start() self.patchers.append(socket_patcher) self.client = Client() self.addr = (socket.gethostbyname(self.client.host), self.client.port) def test_client_create(self): host, port = ('example.com', 8888) client = Client( host=host, port=port, prefix='pystatsd.tests') self.assertEqual(client.host, host) self.assertEqual(client.port, port) self.assertEqual(client.prefix, 'pystatsd.tests') self.assertEqual(client.addr, (socket.gethostbyname(host), port)) def test_basic_client_incr(self): stat = 'pystatsd.unittests.test_basic_client_incr' stat_str = stat + ':1|c' self.client.increment(stat) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_decr(self): stat = 'pystatsd.unittests.test_basic_client_decr' stat_str = stat + ':-1|c' self.client.decrement(stat) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_update_stats(self): stat = 'pystatsd.unittests.test_basic_client_update_stats' stat_str = stat + ':5|c' self.client.update_stats(stat, 5) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_update_stats_multi(self): stats = [ 'pystatsd.unittests.test_basic_client_update_stats', 'pystatsd.unittests.test_basic_client_update_stats_multi' ] data = dict((stat, "%s|c" % '5') for stat in stats) self.client.update_stats(stats, 5) for stat, value in data.items(): stat_str = stat + value # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_call_any( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_timing(self): stat = 'pystatsd.unittests.test_basic_client_timing.time' stat_str = stat + ':5.000000|ms' self.client.timing(stat, 5) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) def test_basic_client_timing_since(self): ts = (1971, 6, 29, 4, 13, 0, 0, 0, -1) now = time.mktime(ts) # add 5 seconds ts = (1971, 6, 29, 4, 13, 5, 0, 0, -1) then = time.mktime(ts) mock_time_patcher = mock.patch('time.time', return_value=now) mock_time_patcher.start() stat = 'pystatsd.unittests.test_basic_client_timing_since.time' stat_str = stat + ':-5000000.000000|ms' self.client.timing_since(stat, then) # thanks tos9 in #python for 'splaining the return_value bit. self.mock_socket.return_value.sendto.assert_called_with( bytes(stat_str, 'utf-8'), self.addr) mock_time_patcher.stop() def tearDown(self): for patcher in self.patchers: patcher.stop()