def fetch_weather(): try: with statsd.StatsdTimer('fetch_wind'): fetch_wind() except: statsd.increment('fetch_wind_error') logger.exception('fetch_wind()') try: with statsd.StatsdTimer('fetch_solar'): fetch_solar() except: statsd.increment('fetch_solar_error') logger.exception('fetch_solar()')
def test_startstop(self): timer = statsd.StatsdTimer('timeit', statsd.StatsdClient('localhost', 8125, prefix='', sample_rate=None)) timer.start() time.sleep(0.25) timer.stop() self.assertTrue(timer._client._socket.data.startswith(b'timeit.total:2')) self.assertTrue(timer._client._socket.data.endswith(b'|ms'))
def fetch_weather(): try: with statsd.StatsdTimer('fetch_weather'): weather.fetch_weather() except: statsd.increment('fetch_weather_error') logger.exception('fetch_weather()')
def fetch_exchanges(): for k, parser in EXCHANGE_PARSERS.iteritems(): try: with statsd.StatsdTimer('fetch_one_exchange'): country_code1, country_code2 = k.split('->') if sorted([country_code1, country_code2])[0] != country_code1: raise Exception( 'Exchange key pair %s is not ordered alphabetically' % k) obj = parser(country_code1, country_code2, session) if not obj: continue if obj.get('sortedCountryCodes', None) != k: raise Exception( "Sorted country codes %s and %s don't match" % (obj.get('sortedCountryCodes', None), k)) if not 'datetime' in obj: raise Exception('datetime was not returned for %s' % k) if arrow.get(obj['datetime']) > arrow.now(): raise Exception("Data from %s can't be in the future" % k) # Database insert result = db_upsert(col_exchange, obj, 'sortedCountryCodes') if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_KEY) except: statsd.increment('fetch_one_exchange_error') logger.exception('Exception while fetching exchange of %s' % k)
def fetch_countries(): for parser in parsers: try: with statsd.StatsdTimer('fetch_one_country'): obj = parser() logging.info('INSERT %s' % obj) col.insert_one(obj) except: statsd.increment('fetch_one_country_error') logger.exception('fetch_one_country()')
def fetch_productions(): for country_code, parser in PRODUCTION_PARSERS.iteritems(): try: with statsd.StatsdTimer('fetch_one_production'): obj = parser(country_code, session) if not obj: continue validate_production(obj, country_code) # Database insert result = db_upsert(col_production, obj, 'countryCode') if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_STATE_KEY) except: statsd.increment('fetch_one_production_error') logger.exception('Exception while fetching production of %s' % country_code)
def test_with(self): mock_gevent_pool = mock(gevent_pool) when(mock_gevent_pool).full().thenReturn(False) when(mock_gevent_pool).spawn(any(), any(), any()).thenReturn(None) client = gevent_statsd.GEventStatsdClient('localhost', 8125, prefix='', sample_rate=None) client._send_pool = mock_gevent_pool timer = statsd.StatsdTimer('timeit', client) with timer: time.sleep(0.25) verify(mock_gevent_pool).spawn(any(), contains(b'timeit.total:'), any())
def fetch_productions(): for country_code, parser in PRODUCTION_PARSERS.iteritems(): try: with statsd.StatsdTimer('fetch_one_production'): obj = parser(country_code, session) if not obj: continue validate_production(obj, country_code) # Data quality check for k, v in obj['production'].iteritems(): if v is None: continue if v < 0: raise ValueError('%s: key %s has negative value %s' % (country_code, k, v)) # Database insert result = db_upsert(col_production, obj, 'countryCode') if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_KEY) except: statsd.increment('fetch_one_production_error') logger.exception('Exception while fetching production of %s' % country_code)
def fetch_weather(): try: with statsd.StatsdTimer('fetch_weather'): objs = fetch_next_forecasts(cached=True) for obj in objs: wind = { 'refTime': obj['refTime'], 'targetTime': obj['targetTime'], 'data': Binary(snappy.compress(json.dumps(obj['wind']))), 'key': 'wind' } solar = { 'refTime': obj['refTime'], 'targetTime': obj['targetTime'], 'data': Binary(snappy.compress(json.dumps(obj['solar']))), 'key': 'solar' } db_upsert_forecast(col_gfs, wind, 'key') db_upsert_forecast(col_gfs, solar, 'key') except: statsd.increment('fetch_weather_error') logger.exception('fetch_weather()')