Ejemplo n.º 1
0
def incrstat(stat, count=1):
    if not _is_setup():
        log.info('STATSD_HOST or STATSD_PORT not set up')
        return
    client = Client(settings.STATSD_HOST, settings.STATSD_PORT)
    client.increment(_statname(stat), count)
    log.info('incrstat: %s %s' % (_statname(stat), count))
Ejemplo n.º 2
0
def expand_url(orig_url):
    statz = Client(*settings.STATSD_SERVER) if settings.STATSD_SERVER else None
    try:
        url_obj = Url.objects.get(original=orig_url)
    except Url.DoesNotExist:
        pass
    else:
        if statz:
            statz.increment('fstv.checked_url.database')
        return url_obj.target

    if urlparse(orig_url).scheme not in ('http', 'https'):
        return None

    log.debug('Scanning url %s', orig_url)
    h = httplib2.Http(timeout=10)
    # We don't get a content-location header when httplib2 follows redirects
    # for a HEAD, so let's track it ourselves I guess.
    h.follow_redirects = False

    redirects = 5
    url = orig_url
    while True:
        try:
            resp, cont = h.request(url, method='HEAD', headers={'User-Agent': 'friendstream/1.0'})
        except httplib.InvalidURL:
            log.debug("Oops, %s isn't really an URL at all, skipping", url)
            return None
        # TODO: what does httplib2 raise when there's no Location header? (does it raise anything when follow_redirects=False?)
        except (httplib2.ServerNotFoundError, httplib.BadStatusLine, socket.timeout, socket.error), exc:
            log.debug("Oops, %s for URL %s (use it for now): %s", type(exc).__name__, url, str(exc))
            return url

        if resp.status in (301, 302, 303, 307):
            location = resp['location']
            try:
                # Either the location is 7 bits, in which case this works fine...
                location.decode('ascii')
            except UnicodeDecodeError:
                # ...or it's erroneously UTF-8 and we need to %-escape the high bytes.
                location = quote(location, safe=';/?:@&=+$,%')
            url = urljoin(url, location)

            redirects -= 1
            if redirects <= 0:
                log.debug("Oops, hit redirect limit for %s (use it for now)", url)
                return url
            continue
        break
Ejemplo n.º 3
0
def poll_account(account_pk, limited=False):
    try:
        account = Account.objects.get(pk=account_pk)
    except Account.DoesNotExist:
        return
    if not account.authinfo:
        return
    if account.error:
        # Don't try to scan accounts that have errors until they're saved again.
        return
    if account.user is None:
        log.debug("Oops, account %r has no user?", account)
        return

    if limited:
        now = datetime.now()
        if account.user.last_login < now - timedelta(days=7):
            log.debug("Marking %s's %s stream stale (hasn't logged in for a week)", account.user.username, account.service)
            account.stale = True
            account.save()
            return
        if account.last_updated > now - timedelta(minutes=14):
            log.debug("Skipping %s's %s stream (updated too soon)", account.user.username, account.service)
            return

    if account.service == 'twitter.com':
        poll_twitter(account)
    elif account.service == 'facebook.com':
        poll_facebook(account)
    # TODO: vimeo, youtube accounts

    account.last_updated = datetime.now()
    account.save()

    if settings.STATSD_SERVER:
        statz = Client(*settings.STATSD_SERVER)
        statz.increment('fstv.account_updated')
Ejemplo n.º 4
0
def main():
    '''Main entry point for the meter_read CLI.'''
    args = rcfile('meter_read', docopt(__doc__, version=__version__))

    logging.basicConfig()
    log = logging.getLogger(__name__)

    if args['verbose']:
        log.setLevel(logging.DEBUG)

    sc = Client(args['addr'], args['port'])

    re_valid = re.compile(args['valid'])
    re_timer = re.compile(args['timer'])

    aliasses = json.loads(args.get('aliasses', '{}'))

    log.debug('found aliasses: %s' % aliasses)

    log.debug('start reading %s' % args['dev'])
    s = serial.Serial(port=args['dev'], baudrate=args['baud'])
    while True:
        line = s.readline()
        match = re_valid.match(line)
        if match:
            check, name, value = match.groups()
            name = aliasses.get(name, name)
            log.debug("{0} {1}".format(name, value))
            # send to statsd
            if re_timer.match(line):
                sc.timing(name, float(value))
            else:
                sc.gauge(name, float(value))
        else:
            log.debug('error %s' % unicode(line, errors='ignore'))
            sc.increment('error')
Ejemplo n.º 5
0
                    })
                created_at = datetime.utcfromtimestamp(status.created_at_in_seconds)
                log.debug("Saving UserStream for user %r video %r poster %r posted %r", account.user, video, poster, created_at)
                us = UserStream(
                    user=account.user,
                    video=video,
                    poster=poster,
                    posted=created_at,
                    message=status.text,
                    message_service='twitter.com',
                    message_ident=status.id,
                )
                us.save()

                if statz:
                    statz.increment('fstv.video.new')
            else:
                log.debug("Video for %s already in %s's stream", url, account.user.username)
                if statz:
                    statz.increment('fstv.video.old')

    log.debug('Done scanning statuses for %s', account.user.username)


def poll_facebook(account):
    facepi = facebook.GraphAPI(account.authinfo)

    try:
        home = facepi.get_object('me/home', limit=100)
    except facebook.GraphAPIError, exc:
        if exc.type in ('OAuthException', 'IDInvalidException'):
Ejemplo n.º 6
0
#!/usr/bin/env python

import time
from multiprocessing import Process

from pystatsd import Client, Server


def worker():
    srvr = Server(debug=True, flush_interval=500)
    srvr.serve()


p = Process(target=worker, daemon=False)
p.start()
time.sleep(1)


sc = Client('localhost', 8125)
sc.timing('python_test.time', 500)
sc.increment('python_test.inc_int')
sc.decrement('python_test.decr_int')
sc.gauge('python_test.gauge', 42)


time.sleep(2)
p.terminate()
Ejemplo n.º 7
0
class GraphiteProcessor(Plugin):
    title = 'Graphite'
    conf_key = 'graphite'
    project_conf_form = GraphiteConfigurationForm

    def __init__(self, min_level=NOTSET, include_loggers=NOTSET, exclude_loggers=NOTSET,
                host=NOTSET, port=NOTSET, prefix=NOTSET, *args, **kwargs):

        super(GraphiteProcessor, self).__init__(*args, **kwargs)

        if min_level is NOTSET:
            min_level = settings.GRAPIHTE_LEVEL
        if include_loggers is NOTSET:
            include_loggers = settings.GRAPHITE_INCLUDE_LOGGERS
        if exclude_loggers is NOTSET:
            exclude_loggers = settings.GRAPHITE_EXCLUDE_LOGGERS
        if host is NOTSET:
            host = settings.GRAPHITE_HOST
        if port is NOTSET:
            port = settings.GRAPHITE_PORT
        if prefix is NOTSET:
            prefix = settings.GRAPHITE_PREFIX

        self.min_level = min_level
        self.include_loggers = include_loggers
        self.exclude_loggers = exclude_loggers
        self.host = host
        self.port = port
        self.prefix = prefix

        self.client = Client(host=self.host, port=self.port)

    def record_event(self, group, event, fail_silently=True):
        project = group.project

        host = self.get_option('host', project) or self.host
        port = self.get_option('port', project) or self.port
        prefix = self.get_option('prefix', project) or self.prefix

        key = '.'.join([prefix, event.message_top])

        self.client.increment(key)

    def should_record(self, group, event):
        project = group.project
        host = self.get_option('host', project) or self.host
        if not host:
            return False
        port = self.get_option('port', project) or self.port
        if not port:
            return False
        prefix = self.get_option('prefix', project) or self.prefix
        if not prefix:
            return False
        min_level = self.get_option('min_level', project) or self.min_level
        if min_level is not None and int(group.level) < min_level:
            return False
        include_loggers = self.get_option('include_loggers', project) or self.include_loggers
        if include_loggers is not None and group.logger not in include_loggers:
            return False
        exclude_loggers = self.get_option('exclude_loggers', project) or self.exclude_loggers
        if exclude_loggers and group.logger in exclude_loggers:
            return False

        return True

    def post_process(self, group, event, is_new, is_sample, **kwargs):
        if not self.should_record(group, event):
            return

        self.record_event(group, event)
Ejemplo n.º 8
0
#!/usr/bin/env python

from pystatsd import Client, Server

sc = Client('localhost', 8125)

sc.timing('python_test.time', 500)
sc.increment('python_test.inc_int')
sc.decrement('python_test.decr_int')

srvr = Server(debug=True)
srvr.serve()
Ejemplo n.º 9
0
#!/usr/bin/env python

from pystatsd import Client, Server

sc = Client("localhost", 8125)

sc.timing("python_test.time", 500)
sc.increment("python_test.inc_int")
sc.decrement("python_test.decr_int")
sc.gauge("python_test.gauge", 42)

srvr = Server(debug=True)
srvr.serve()
Ejemplo n.º 10
0
class ClientBasicsTestCase(unittest.TestCase):
    """
    Tests the basic operations of the client
    """
    def setUp(self):
        self.patchers = []

        socket_patcher = mock.patch('pystatsd.statsd.socket.socket')
        self.mock_socket = socket_patcher.start()
        self.patchers.append(socket_patcher)

        self.client = Client()
        self.addr = (socket.gethostbyname(self.client.host), self.client.port)

    def test_client_create(self):
        host, port = ('example.com', 8888)

        client = Client(host=host, port=port, prefix='pystatsd.tests')
        self.assertEqual(client.host, host)
        self.assertEqual(client.port, port)
        self.assertEqual(client.prefix, 'pystatsd.tests')
        self.assertEqual(client.addr, (socket.gethostbyname(host), port))

    def test_basic_client_incr(self):
        stat = 'pystatsd.unittests.test_basic_client_incr'
        stat_str = stat + ':1|c'

        self.client.increment(stat)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_decr(self):
        stat = 'pystatsd.unittests.test_basic_client_decr'
        stat_str = stat + ':-1|c'

        self.client.decrement(stat)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_update_stats(self):
        stat = 'pystatsd.unittests.test_basic_client_update_stats'
        stat_str = stat + ':5|c'

        self.client.update_stats(stat, 5)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_update_stats_multi(self):
        stats = [
            'pystatsd.unittests.test_basic_client_update_stats',
            'pystatsd.unittests.test_basic_client_update_stats_multi'
        ]

        data = dict((stat, "%s|c" % '5') for stat in stats)

        self.client.update_stats(stats, 5)

        for stat, value in data.items():
            stat_str = stat + value
            # thanks tos9 in #python for 'splaining the return_value bit.
            self.mock_socket.return_value.sendto.assert_call_any(
                bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_timing(self):
        stat = 'pystatsd.unittests.test_basic_client_timing.time'
        stat_str = stat + ':5.000000|ms'

        self.client.timing(stat, 5)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_timing_since(self):
        ts = (1971, 6, 29, 4, 13, 0, 0, 0, -1)
        now = time.mktime(ts)
        # add 5 seconds
        ts = (1971, 6, 29, 4, 13, 5, 0, 0, -1)
        then = time.mktime(ts)
        mock_time_patcher = mock.patch('time.time', return_value=now)
        mock_time_patcher.start()

        stat = 'pystatsd.unittests.test_basic_client_timing_since.time'
        stat_str = stat + ':-5000000.000000|ms'

        self.client.timing_since(stat, then)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

        mock_time_patcher.stop()

    def tearDown(self):
        for patcher in self.patchers:
            patcher.stop()
Ejemplo n.º 11
0
def record(millis):
    c = Client('hammer.wallrazer.com', 8125)
    print "recording...\n"
    c.increment('wallrazer.deploys')
    c.timing('wallrazer.deploys.time', millis)
Ejemplo n.º 12
0
class ClientBasicsTestCase(unittest.TestCase):
    """
    Tests the basic operations of the client
    """
    def setUp(self):
        self.patchers = []

        socket_patcher = mock.patch('pystatsd.statsd.socket.socket')
        self.mock_socket = socket_patcher.start()
        self.patchers.append(socket_patcher)

        self.client = Client()
        self.addr = (socket.gethostbyname(self.client.host), self.client.port)

    def test_client_create(self):
        host, port = ('example.com', 8888)

        client = Client(
            host=host,
            port=port,
            prefix='pystatsd.tests')
        self.assertEqual(client.host, host)
        self.assertEqual(client.port, port)
        self.assertEqual(client.prefix, 'pystatsd.tests')
        self.assertEqual(client.addr, (socket.gethostbyname(host), port))

    def test_basic_client_incr(self):
        stat = 'pystatsd.unittests.test_basic_client_incr'
        stat_str = stat + ':1|c'

        self.client.increment(stat)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_decr(self):
        stat = 'pystatsd.unittests.test_basic_client_decr'
        stat_str = stat + ':-1|c'

        self.client.decrement(stat)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_update_stats(self):
        stat = 'pystatsd.unittests.test_basic_client_update_stats'
        stat_str = stat + ':5|c'

        self.client.update_stats(stat, 5)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_update_stats_multi(self):
        stats = [
            'pystatsd.unittests.test_basic_client_update_stats',
            'pystatsd.unittests.test_basic_client_update_stats_multi'
        ]

        data = dict((stat, "%s|c" % '5') for stat in stats)

        self.client.update_stats(stats, 5)

        for stat, value in data.items():
            stat_str = stat + value
            # thanks tos9 in #python for 'splaining the return_value bit.
            self.mock_socket.return_value.sendto.assert_call_any(
                bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_timing(self):
        stat = 'pystatsd.unittests.test_basic_client_timing.time'
        stat_str = stat + ':5.000000|ms'

        self.client.timing(stat, 5)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

    def test_basic_client_timing_since(self):
        ts = (1971, 6, 29, 4, 13, 0, 0, 0, -1)
        now = time.mktime(ts)
        # add 5 seconds
        ts = (1971, 6, 29, 4, 13, 5, 0, 0, -1)
        then = time.mktime(ts)
        mock_time_patcher = mock.patch('time.time', return_value=now)
        mock_time_patcher.start()

        stat = 'pystatsd.unittests.test_basic_client_timing_since.time'
        stat_str = stat + ':-5000000.000000|ms'

        self.client.timing_since(stat, then)

        # thanks tos9 in #python for 'splaining the return_value bit.
        self.mock_socket.return_value.sendto.assert_called_with(
            bytes(stat_str, 'utf-8'), self.addr)

        mock_time_patcher.stop()

    def tearDown(self):
        for patcher in self.patchers:
            patcher.stop()