def fetch_weather():
try:
with statsd.StatsdTimer('fetch_wind'): fetch_wind()
except:
statsd.increment('fetch_wind_error')
logger.exception('fetch_wind()')
try:
with statsd.StatsdTimer('fetch_solar'): fetch_solar()
except:
statsd.increment('fetch_solar_error')
logger.exception('fetch_solar()')
def test_startstop(self):
timer = statsd.StatsdTimer('timeit', statsd.StatsdClient('localhost', 8125, prefix='', sample_rate=None))
timer.start()
time.sleep(0.25)
timer.stop()
self.assertTrue(timer._client._socket.data.startswith(b'timeit.total:2'))
self.assertTrue(timer._client._socket.data.endswith(b'|ms'))
def fetch_weather():
try:
with statsd.StatsdTimer('fetch_weather'):
weather.fetch_weather()
except:
statsd.increment('fetch_weather_error')
logger.exception('fetch_weather()')
def fetch_exchanges():
for k, parser in EXCHANGE_PARSERS.iteritems():
try:
with statsd.StatsdTimer('fetch_one_exchange'):
country_code1, country_code2 = k.split('->')
if sorted([country_code1, country_code2])[0] != country_code1:
raise Exception(
'Exchange key pair %s is not ordered alphabetically' %
k)
obj = parser(country_code1, country_code2, session)
if not obj: continue
if obj.get('sortedCountryCodes', None) != k:
raise Exception(
"Sorted country codes %s and %s don't match" %
(obj.get('sortedCountryCodes', None), k))
if not 'datetime' in obj:
raise Exception('datetime was not returned for %s' % k)
if arrow.get(obj['datetime']) > arrow.now():
raise Exception("Data from %s can't be in the future" % k)
# Database insert
result = db_upsert(col_exchange, obj, 'sortedCountryCodes')
if (result.modified_count or result.upserted_id) and cache:
cache.delete(MEMCACHED_KEY)
except:
statsd.increment('fetch_one_exchange_error')
logger.exception('Exception while fetching exchange of %s' % k)
def fetch_countries():
for parser in parsers:
try:
with statsd.StatsdTimer('fetch_one_country'):
obj = parser()
logging.info('INSERT %s' % obj)
col.insert_one(obj)
except:
statsd.increment('fetch_one_country_error')
logger.exception('fetch_one_country()')
def fetch_productions():
for country_code, parser in PRODUCTION_PARSERS.iteritems():
try:
with statsd.StatsdTimer('fetch_one_production'):
obj = parser(country_code, session)
if not obj: continue
validate_production(obj, country_code)
# Database insert
result = db_upsert(col_production, obj, 'countryCode')
if (result.modified_count or result.upserted_id) and cache:
cache.delete(MEMCACHED_STATE_KEY)
except:
statsd.increment('fetch_one_production_error')
logger.exception('Exception while fetching production of %s' %
country_code)
def test_with(self):
mock_gevent_pool = mock(gevent_pool)
when(mock_gevent_pool).full().thenReturn(False)
when(mock_gevent_pool).spawn(any(), any(), any()).thenReturn(None)
client = gevent_statsd.GEventStatsdClient('localhost',
8125,
prefix='',
sample_rate=None)
client._send_pool = mock_gevent_pool
timer = statsd.StatsdTimer('timeit', client)
with timer:
time.sleep(0.25)
verify(mock_gevent_pool).spawn(any(), contains(b'timeit.total:'),
any())
def fetch_productions():
for country_code, parser in PRODUCTION_PARSERS.iteritems():
try:
with statsd.StatsdTimer('fetch_one_production'):
obj = parser(country_code, session)
if not obj: continue
validate_production(obj, country_code)
# Data quality check
for k, v in obj['production'].iteritems():
if v is None: continue
if v < 0:
raise ValueError('%s: key %s has negative value %s' %
(country_code, k, v))
# Database insert
result = db_upsert(col_production, obj, 'countryCode')
if (result.modified_count or result.upserted_id) and cache:
cache.delete(MEMCACHED_KEY)
except:
statsd.increment('fetch_one_production_error')
logger.exception('Exception while fetching production of %s' %
country_code)
def fetch_weather():
try:
with statsd.StatsdTimer('fetch_weather'):
objs = fetch_next_forecasts(cached=True)
for obj in objs:
wind = {
'refTime': obj['refTime'],
'targetTime': obj['targetTime'],
'data': Binary(snappy.compress(json.dumps(obj['wind']))),
'key': 'wind'
}
solar = {
'refTime': obj['refTime'],
'targetTime': obj['targetTime'],
'data': Binary(snappy.compress(json.dumps(obj['solar']))),
'key': 'solar'
}
db_upsert_forecast(col_gfs, wind, 'key')
db_upsert_forecast(col_gfs, solar, 'key')
except:
statsd.increment('fetch_weather_error')
logger.exception('fetch_weather()')