def _get_data(self, raw_data, callsign, config, module):
"""Attempt to parse data from what we know so far."""
sentences = config["payload_configuration"]["sentences"]
for sentence_index, sentence in enumerate(sentences):
if sentence["callsign"] != callsign:
continue
if sentence["protocol"] != module["name"]:
continue
data = self.filtering.intermediate_filter(raw_data, sentence)
try:
data = module["module"].parse(data, sentence)
except (ValueError, KeyError) as e:
logger.debug("Exception in {module} main parse: {e}"
.format(module=module['name'],
e=quick_traceback.oneline(e)))
statsd.increment("parser.parse_exception")
continue
data = self.filtering.post_filter(data, sentence)
data["_protocol"] = module["name"]
data["_parsed"] = {
"time_parsed": strict_rfc3339.now_to_rfc3339_utcoffset(),
"payload_configuration": config["id"],
"configuration_sentence_index": sentence_index
}
if "flight_id" in config:
data["_parsed"]["flight"] = config["flight_id"]
return data
raise CantGetData()
def fetch_weather():
try:
with statsd.StatsdTimer('fetch_weather'):
weather.fetch_weather()
except:
statsd.increment('fetch_weather_error')
logger.exception('fetch_weather()')
def _get_debug(self, raw_data):
if self.ascii_exp.search(raw_data):
statsd.increment("parser.ascii_doc")
return 'ascii', raw_data
else:
statsd.increment("parser.binary_doc")
return 'b64', base64.b64encode(raw_data)
def _get_config(self, callsign, config=None):
"""
Attempt to get a config doc given the callsign and maybe a provided
config doc.
"""
if config and not self._callsign_in_config(callsign, config):
logger.debug("Callsign {c!r} not found in configuration doc"
.format(c=callsign))
raise CantGetConfig()
elif config:
if "_id" not in config:
config["_id"] = None
logger.debug("payload_configuration provided (id: {0})"
.format(config["_id"]))
return {"id": config["_id"], "payload_configuration": config}
config = self._find_config_doc(callsign)
if not config:
logger.debug("No configuration doc for {callsign!r} found"
.format(callsign=callsign))
statsd.increment("parser.no_config_doc")
raise CantGetConfig()
if "flight_id" in config:
logger.debug("Selected payload_configuration {0} from flight {1} "
"for {2!r}"
.format(config["id"], config["flight_id"], callsign))
else:
logger.debug("Selected payload_configuration {0} for {1!r}"
.format(config["id"], callsign))
return config
def api_call_raw(method, uri, token=None, headers=None, get_vars=None, data=None, public=False):
"""
Takes the uri of an application URL to call
"""
statsd.increment('api.call')
if not headers:
headers = {}
if data is not None:
data = json.dumps(data, default=date_handler)
headers['Content-Type'] = 'application/json'
if not public:
if token is not None:
headers['Authorization'] = 'Bearer %s' % token
else:
encoded_consumer = b64encode('%s:%s' % (settings.MASTER_OAUTH_KEY, settings.MASTER_OAUTH_SECRET))
headers['Authorization'] = 'Basic %s' % encoded_consumer
api_url = build_url(uri, get_vars)
auth = build_auth()
req = requests.request(method, api_url, headers=headers, data=data, auth=auth, timeout=90)
return req
def _get_debug(self, raw_data):
if self.ascii_exp.search(raw_data):
statsd.increment("parser.ascii_doc")
return 'ascii', raw_data
else:
statsd.increment("parser.binary_doc")
return 'b64', base64.b64encode(raw_data)
def _get_config(self, callsign, config=None):
"""
Attempt to get a config doc given the callsign and maybe a provided
config doc.
"""
if config and not self._callsign_in_config(callsign, config):
logger.debug(
"Callsign {c!r} not found in configuration doc".format(
c=callsign))
raise CantGetConfig()
elif config:
if "_id" not in config:
config["_id"] = None
logger.debug("payload_configuration provided (id: {0})".format(
config["_id"]))
return {"id": config["_id"], "payload_configuration": config}
config = self._find_config_doc(callsign)
if not config:
logger.debug("No configuration doc for {callsign!r} found".format(
callsign=callsign))
statsd.increment("parser.no_config_doc")
raise CantGetConfig()
if "flight_id" in config:
logger.debug("Selected payload_configuration {0} from flight {1} "
"for {2!r}".format(config["id"], config["flight_id"],
callsign))
else:
logger.debug("Selected payload_configuration {0} for {1!r}".format(
config["id"], callsign))
return config
def fetch_exchanges():
for k, parser in EXCHANGE_PARSERS.iteritems():
try:
with statsd.StatsdTimer('fetch_one_exchange'):
country_code1, country_code2 = k.split('->')
if sorted([country_code1, country_code2])[0] != country_code1:
raise Exception(
'Exchange key pair %s is not ordered alphabetically' %
k)
obj = parser(country_code1, country_code2, session)
if not obj: continue
if obj.get('sortedCountryCodes', None) != k:
raise Exception(
"Sorted country codes %s and %s don't match" %
(obj.get('sortedCountryCodes', None), k))
if not 'datetime' in obj:
raise Exception('datetime was not returned for %s' % k)
if arrow.get(obj['datetime']) > arrow.now():
raise Exception("Data from %s can't be in the future" % k)
# Database insert
result = db_upsert(col_exchange, obj, 'sortedCountryCodes')
if (result.modified_count or result.upserted_id) and cache:
cache.delete(MEMCACHED_KEY)
except:
statsd.increment('fetch_one_exchange_error')
logger.exception('Exception while fetching exchange of %s' % k)
def _get_data(self, raw_data, callsign, config, module):
"""Attempt to parse data from what we know so far."""
sentences = config["payload_configuration"]["sentences"]
for sentence_index, sentence in enumerate(sentences):
if sentence["callsign"] != callsign:
continue
if sentence["protocol"] != module["name"]:
continue
data = self.filtering.intermediate_filter(raw_data, sentence)
try:
data = module["module"].parse(data, sentence)
except (ValueError, KeyError) as e:
logger.debug("Exception in {module} main parse: {e}".format(
module=module['name'], e=quick_traceback.oneline(e)))
statsd.increment("parser.parse_exception")
continue
data = self.filtering.post_filter(data, sentence)
data["_protocol"] = module["name"]
data["_parsed"] = {
"time_parsed": strict_rfc3339.now_to_rfc3339_utcoffset(),
"payload_configuration": config["id"],
"configuration_sentence_index": sentence_index
}
if "flight_id" in config:
data["_parsed"]["flight"] = config["flight_id"]
return data
raise CantGetData()
def _apply_filters(self, data, sentence, filter_type, result_type):
if "filters" in sentence:
if filter_type in sentence["filters"]:
for index, f in enumerate(sentence["filters"][filter_type]):
whence = (filter_type, index)
data = self._filter(data, f, result_type, whence)
statsd.increment("parser.filters.{0}".format(filter_type))
return data
def _apply_filters(self, data, sentence, filter_type, result_type):
if "filters" in sentence:
if filter_type in sentence["filters"]:
for index, f in enumerate(sentence["filters"][filter_type]):
whence = (filter_type, index)
data = self._filter(data, f, result_type, whence)
statsd.increment("parser.filters.{0}".format(filter_type))
return data
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
for i in range(51):
statsd.increment('mycounter')
t.assert_equal('\n'.join(['mycounter:1|c' for i in range(50)]), fake_socket.recv())
t.assert_equal('mycounter:1|c', fake_socket.recv())
def test_batched_buffer_autoflush(self):
fake_socket = FakeSocket()
with DogStatsd() as statsd:
statsd.socket = fake_socket
for i in range(51):
statsd.increment('mycounter')
t.assert_equal('\n'.join(['mycounter:1|c' for i in range(50)]), fake_socket.recv())
t.assert_equal('mycounter:1|c', fake_socket.recv())
def list(key):
librato.count('blueprint-acmeaws-com-server.requests.list')
statsd.increment('blueprint-acmeaws-com-server.requests.list')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
return b.list(key)
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def fetch_countries():
for parser in parsers:
try:
with statsd.StatsdTimer('fetch_one_country'):
obj = parser()
logging.info('INSERT %s' % obj)
col.insert_one(obj)
except:
statsd.increment('fetch_one_country_error')
logger.exception('fetch_one_country()')
def list(key):
"""
List objects in MongoDB whose key begins with the given prefix
"""
librato.count('blueprint-io-server.requests.list')
statsd.increment('blueprint-io-server.requests.list')
try:
result = collection.find({"key" : '^%s' % (key)})
except:
return False
return result
def login(request):
"""
Login page, for which all anonymous users attempting to visit a page without authorization will be redirected to
"""
if is_authenticated(request):
return redirect('dash')
if request.method == 'POST':
statsd.increment('login.attempt')
username = request.POST.get('username', None)
password = request.POST.get('password', None)
remember = True if request.POST.get('remember', False) else False
next = request.POST.get('next', None)
try:
token, user_data = _login_request(
request,
username=username,
password=password,
remember=remember,
)
except LoginFailure:
# If data posted but is invalid for any reason
statsd.increment('login.failure')
messages.error(request, "Sorry, those login details aren't valid")
next = 'login'
else:
try:
resolve(next)
except:
next = 'dash'
return redirect(next)
else:
next = request.GET.get('next', 'login')
login_form = AuthenticationForm()
return render_to_response(
'login.html',
{
'next': next,
'login_form': login_form,
# 'messages': messages,
},
context_instance=RequestContext(request)
)
def fetch_weather():
try:
with statsd.StatsdTimer('fetch_wind'): fetch_wind()
except:
statsd.increment('fetch_wind_error')
logger.exception('fetch_wind()')
try:
with statsd.StatsdTimer('fetch_solar'): fetch_solar()
except:
statsd.increment('fetch_solar_error')
logger.exception('fetch_solar()')
def health_GET():
statsd.increment('health_GET')
# Check last data point obtained
EXPIRATION_SECONDS = 30 * 60.0
result = list(col.find(sort=[('datetime', pymongo.DESCENDING)], limit=1))
if not len(result) or (arrow.now() - arrow.get(
result[0]['datetime'])).total_seconds() > EXPIRATION_SECONDS:
return flask.jsonify(
{'error':
'Database is empty or last measurement is too old.'}), 400
return flask.jsonify({'status': 'ok'})
def new_response(event):
request = event.request
if request.statsd:
if request.exception is None:
# Successful request
statsd.increment('request.pubsuccess')
statsd.timing('request.duration', time.time() * 1000 - request._pub_start)
else:
# Error response
statsd.increment('request.pubfailure')
statsd.timing('request.duration', time.time() * 1000 - request._pub_start)
def _hotfix_filter(self, data, f):
"""Load a filter specified by some code in the database. Check its
authenticity by verifying its certificate, then run if OK."""
self._sanity_check_hotfix(f)
cert = self._get_certificate(f["certificate"])
self._verify_certificate(f, cert)
env = self._compile_hotfix(f)
logger.debug("Executing a hotfix")
statsd.increment("parser.filters.hotfix.executed")
return env["f"](data)
def _hotfix_filter(self, data, f):
"""Load a filter specified by some code in the database. Check its
authenticity by verifying its certificate, then run if OK."""
self._sanity_check_hotfix(f)
cert = self._get_certificate(f["certificate"])
self._verify_certificate(f, cert)
env = self._compile_hotfix(f)
logger.debug("Executing a hotfix")
statsd.increment("parser.filters.hotfix.executed")
return env["f"](data)
def login(request):
"""
Login page, for which all anonymous users attempting to visit a page without authorization will be redirected to
"""
if is_authenticated(request):
return redirect('dash')
if request.method == 'POST':
statsd.increment('login.attempt')
username = request.POST.get('username', None)
password = request.POST.get('password', None)
remember = True if request.POST.get('remember', False) else False
next = request.POST.get('next', None)
try:
token, user_data = _login_request(
request,
username=username,
password=password,
remember=remember,
)
except LoginFailure:
# If data posted but is invalid for any reason
statsd.increment('login.failure')
messages.error(request, "Sorry, those login details aren't valid")
next = 'login'
else:
try:
resolve(next)
except:
next = 'dash'
return redirect(next)
else:
next = request.GET.get('next', 'login')
login_form = AuthenticationForm()
return render_to_response(
'login.html',
{
'next': next,
'login_form': login_form,
# 'messages': messages,
},
context_instance=RequestContext(request))
def list(key):
librato.count('blueprint-acmeaws-com-server.requests.list')
statsd.increment('blueprint-acmeaws-com-server.requests.list')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
return b.list(key)
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def _compile_hotfix(self, f):
"""Compile a hotfix into a function **f** in an empty namespace."""
logger.debug("Compiling a hotfix")
body = "def f(data):\n"
env = {}
try:
body += "\n".join(" " + l + "\n" for l in f["code"].split("\n"))
code = compile(body, "<filter>", "exec")
exec code in env
except (SyntaxError, AttributeError, TypeError):
statsd.increment("parser.filters.hotfix.compile_error")
raise ValueError("Hotfix code didn't compile: " + repr(f))
return env
def get(key):
"""
Fetch an object from MongoDB.
"""
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
try:
k = collection.find_one({"key" : key})
if k is None:
return None
except:
return False
return k['tarball']
def delete(key):
"""
Remove an object from MongoDB.
"""
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-io-server.requests.delete')
statsd.increment('blueprint-io-server.requests.delete')
try:
collection.delete({"key" : key})
except:
return False
def head(key):
librato.count('blueprint-acmeaws-com-server.requests.head')
statsd.increment('blueprint-acmeaws-com-server.requests.head')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
k = b.get_key(key)
if k is None:
return None
return k.size
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def _compile_hotfix(self, f):
"""Compile a hotfix into a function **f** in an empty namespace."""
logger.debug("Compiling a hotfix")
body = "def f(data):\n"
env = {}
try:
body += "\n".join(" " + l + "\n" for l in f["code"].split("\n"))
code = compile(body, "<filter>", "exec")
exec code in env
except (SyntaxError, AttributeError, TypeError):
statsd.increment("parser.filters.hotfix.compile_error")
raise ValueError("Hotfix code didn't compile: " + repr(f))
return env
def register_user(request):
if is_authenticated(request):
return redirect('dash')
if request.method == 'POST':
statsd.increment('registration.attempt')
first_name = request.POST.get('first_name', None)
last_name = request.POST.get('last_name', None)
email = request.POST.get('email', None)
username = request.POST.get('username', None)
password = request.POST.get('password', None)
password_confirm = request.POST.get('password_confirm', None)
if password != password_confirm:
messages.error(request, "Your passwords did not match")
return redirect('registration-user')
next = request.POST.get('next', None)
try:
token, user_data = _create_user(
request,
first_name=first_name,
last_name=last_name,
email=email,
username=username,
password=password,
)
except RegistrationFailure:
# If data posted but is invalid for any reason
statsd.increment('registration.failure')
messages.error(request,
"Sorry, those registration details aren't valid")
next = 'registration-user'
else:
try:
resolve(next)
except:
next = 'dash'
return redirect(next)
return render_to_response(
'registration/user.html',
{},
context_instance=RequestContext(request),
)
def head(key):
"""
Make a HEAD request for an object in MongoDB.
This returns the size of the tarball.
"""
librato.count('blueprint-io-server.requests.head')
statsd.increment('blueprint-io-server.requests.head')
try:
k = collection.find_one({"key" : key})
if k is None:
return None
except:
return False
return len(k['tarball'])
def put(key, data):
"""
Store an object in MongoDB.
"""
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
# TODO librato.something('blueprint-io-server.storage', len(data))
statsd.update('blueprint-io-server.storage', len(data))
element = StoredObject(key, data)
try:
collection.insert(element.__dict__)
except:
return False
return True
def register_user(request):
if is_authenticated(request):
return redirect('dash')
if request.method == 'POST':
statsd.increment('registration.attempt')
first_name = request.POST.get('first_name', None)
last_name = request.POST.get('last_name', None)
email = request.POST.get('email', None)
username = request.POST.get('username', None)
password = request.POST.get('password', None)
password_confirm = request.POST.get('password_confirm', None)
if password != password_confirm:
messages.error(request, "Your passwords did not match")
return redirect('registration-user')
next = request.POST.get('next', None)
try:
token, user_data = _create_user(
request,
first_name=first_name,
last_name=last_name,
email=email,
username=username,
password=password,
)
except RegistrationFailure:
# If data posted but is invalid for any reason
statsd.increment('registration.failure')
messages.error(request, "Sorry, those registration details aren't valid")
next = 'registration-user'
else:
try:
resolve(next)
except:
next = 'dash'
return redirect(next)
return render_to_response(
'registration/user.html',
{},
context_instance=RequestContext(request),
)
def production_GET():
statsd.increment('production_GET')
objs = col.aggregate([{
'$group': {
'_id': '$countryCode',
'lastDocument': {
'$last': '$$CURRENT'
}
}
}])
objs = map(bson_measurement_to_json,
map(lambda o: o['lastDocument'], list(objs)))
data = {obj['countryCode']: obj for obj in objs if 'countryCode' in obj}
return flask.jsonify({'status': 'ok', 'data': data})
def fetch_productions():
for country_code, parser in PRODUCTION_PARSERS.iteritems():
try:
with statsd.StatsdTimer('fetch_one_production'):
obj = parser(country_code, session)
if not obj: continue
validate_production(obj, country_code)
# Database insert
result = db_upsert(col_production, obj, 'countryCode')
if (result.modified_count or result.upserted_id) and cache:
cache.delete(MEMCACHED_STATE_KEY)
except:
statsd.increment('fetch_one_production_error')
logger.exception('Exception while fetching production of %s' %
country_code)
def delete(key):
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-acmeaws-com-server.requests.delete')
statsd.increment('blueprint-acmeaws-com-server.requests.delete')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
b.delete_key(key)
statsd.update('blueprint-acmeaws-com-server.storage', -content_length)
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def put(key, data):
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
statsd.update('blueprint-io-server.storage', len(data))
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
k.set_contents_from_string(data,
policy='public-read',
reduced_redundancy=True)
return True
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def list(key):
"""
List objects in S3 whose keys begin with the given prefix. This
function makes at least one billable request.
"""
librato.count('blueprint-io-server.requests.list')
statsd.increment('blueprint-io-server.requests.list')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
return b.list(key)
try:
return True
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def get_tarball(secret, name, sha):
validate_secret(secret)
validate_name(name)
sha = sha.lower()
validate_sha(sha)
content_length = backend.head_tarball(secret, name, sha)
if content_length is None:
abort(404)
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
librato.count('blueprint-io-server.bandwidth.out', content_length)
statsd.update('blueprint-io-server.bandwidth.out', content_length)
return redirect(backend.url_for_tarball(secret, name, sha), code=301)
def test_increment(self):
with mock.patch('statsd.Client') as mock_client:
self.counter.increment('')
mock_client._send.assert_called_with(mock.ANY, {'testing': '1|c'})
self.counter.increment('', 2)
mock_client._send.assert_called_with(mock.ANY, {'testing': '2|c'})
self.counter += 3
mock_client._send.assert_called_with(mock.ANY, {'testing': '3|c'})
statsd.increment('testing', 4)
mock_client._send.assert_called_with(mock.ANY, {'testing': '4|c'})
statsd.increment('testing')
mock_client._send.assert_called_with(mock.ANY, {'testing': '1|c'})
def get_tarball(secret, name, sha):
validate_secret(secret)
validate_name(name)
sha = sha.lower()
validate_sha(sha)
content_length = backend.head_tarball(secret, name, sha)
if content_length is None:
abort(404)
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
librato.count('blueprint-io-server.bandwidth.out', content_length)
statsd.update('blueprint-io-server.bandwidth.out', content_length)
return redirect(backend.url_for_tarball(secret, name, sha), code=301)
def head(key):
librato.count('blueprint-acmeaws-com-server.requests.head')
statsd.increment('blueprint-acmeaws-com-server.requests.head')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
k = b.get_key(key)
if k is None:
return None
return k.size
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def get(key):
"""
Fetch an object from S3. This function makes one billable request.
"""
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
return k.get_contents_as_string()
except boto.exception.S3ResponseError:
return None
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def _create_user(request, **kwargs):
res, status = api_call('post', '/api/v0/registration/user/', data=kwargs)
if status != 200 or 'token' not in res:
raise RegistrationFailure
token = res['token']
data, status = api_call('get', '/api/v0/account/authed/', token=token)
if status != 200 or data is None:
raise AuthFailure()
statsd.increment('registration.success')
for key, val in data.items():
request.session[key] = val
request.session['token'] = token
return token, data
def _create_user(request, **kwargs):
res, status = api_call('post', '/api/v0/registration/user/', data=kwargs)
if status != 200 or 'token' not in res:
raise RegistrationFailure
token = res['token']
data, status = api_call('get', '/api/v0/account/authed/', token=token)
if status != 200 or data is None:
raise AuthFailure()
statsd.increment('registration.success')
for key, val in data.items():
request.session[key] = val
request.session['token'] = token
return token, data
def head(key):
"""
Make a HEAD request for an object in S3. This is needed to find the
object's length so it can be accounted. This function makes one
billable request and anticipates another.
"""
librato.count('blueprint-io-server.requests.head')
statsd.increment('blueprint-io-server.requests.head')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
try:
k = b.get_key(key)
if k is None:
return None
return k.size
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def list(key):
"""
List objects in S3 whose keys begin with the given prefix. This
function makes at least one billable request.
"""
librato.count('blueprint-io-server.requests.list')
statsd.increment('blueprint-io-server.requests.list')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
return b.list(key)
try:
return True
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def delete(key):
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-acmeaws-com-server.requests.delete')
statsd.increment('blueprint-acmeaws-com-server.requests.delete')
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
try:
b.delete_key(key)
statsd.update('blueprint-acmeaws-com-server.storage', -content_length)
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def put(key, data):
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
statsd.update('blueprint-io-server.storage', len(data))
c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
k.set_contents_from_string(data,
policy='public-read',
reduced_redundancy=True)
return True
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def _login_request(request, remember=True, *args, **kwargs):
res, status = api_call('post', '/api/v0/account/login/', data=kwargs)
if status != 200 or 'token' not in res:
raise LoginFailure()
token = res['token']
data, status = api_call('get', '/api/v0/account/authed/', token=token)
if status != 200 or data is None:
raise AuthFailure()
statsd.increment('login.success')
for key, val in data.items():
request.session[key] = val
request.session['token'] = token
if remember:
request.session.set_expiry(0)
return token, data
def _login_request(request, remember=True, *args, **kwargs):
res, status = api_call('post', '/api/v0/account/login/', data=kwargs)
if status != 200 or 'token' not in res:
raise LoginFailure()
token = res['token']
data, status = api_call('get', '/api/v0/account/authed/', token=token)
if status != 200 or data is None:
raise AuthFailure()
statsd.increment('login.success')
for key, val in data.items():
request.session[key] = val
request.session['token'] = token
if remember:
request.session.set_expiry(0)
return token, data
def put(key, data):
"""
Store an object in S3. This function makes one billable request.
"""
librato.count('blueprint-io-server.requests.put')
statsd.increment('blueprint-io-server.requests.put')
# TODO librato.something('blueprint-io-server.storage', len(data))
statsd.update('blueprint-io-server.storage', len(data))
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
k.set_contents_from_string(data,
policy='public-read',
reduced_redundancy=True)
return True
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
httplib.HTTPException, socket.error, socket.gaierror):
return False
def get(key):
"""
Fetch an object from S3. This function makes one billable request.
"""
librato.count('blueprint-io-server.requests.get')
statsd.increment('blueprint-io-server.requests.get')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
k = b.new_key(key)
try:
return k.get_contents_as_string()
except boto.exception.S3ResponseError:
return None
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def set_language(request):
"""Sets the current language."""
if request.matchdict.get("lang", "fi") == "sv":
request.session["locale"] = "sv"
if request.statsd:
statsd.increment("lang.sv")
LOG.info("Changed language to swedish")
else:
if request.statsd:
statsd.increment("lang.fi")
# Fall back to whatever default
request.session.pop("locale")
# Make an attempt to redirect back to the page we came from
if request.referer is not None and request.referer.startswith(request.application_url):
location = request.referer
else:
location = request.application_url
return HTTPFound(location=location)
def _verify_certificate(self, f, cert):
"""Check that the certificate is cryptographically signed by a key
which is signed by a known CA."""
# Check the certificate is valid
for ca_cert in self.certificate_authorities:
if cert.verify(ca_cert.get_pubkey()):
break
raise ValueError("Certificate is not signed by a recognised CA.")
# Check the signature is valid
try:
digest = hashlib.sha256(f["code"]).hexdigest()
sig = base64.b64decode(f["signature"])
ok = cert.get_pubkey().get_rsa().verify(digest, sig, 'sha256')
except (TypeError, M2Crypto.RSA.RSAError):
statsd.increment("parser.filters.hotfix.invalid_signature")
raise ValueError("Hotfix signature is not valid")
if not ok:
statsd.increment("parser.filters.hotfix.invalid_signature")
raise ValueError("Hotfix signature is not valid")
def delete(key):
"""
Remove an object from S3. DELETE requests are free but this function
still makes one billable request to account for freed storage.
"""
content_length = head(key)
if content_length is None:
return None
librato.count('blueprint-io-server.requests.delete')
statsd.increment('blueprint-io-server.requests.delete')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
try:
b.delete_key(key)
# TODO librato.something('blueprint-io-server.storage', -content_length)
statsd.update('blueprint-io-server.storage', -content_length)
except (boto.exception.BotoClientError, boto.exception.BotoServerError,
boto.exception.S3ResponseError, httplib.HTTPException,
socket.error, socket.gaierror):
return False
def _verify_certificate(self, f, cert):
"""Check that the certificate is cryptographically signed by a key
which is signed by a known CA."""
# Check the certificate is valid
for ca_cert in self.certificate_authorities:
if cert.verify(ca_cert.get_pubkey()):
break
raise ValueError("Certificate is not signed by a recognised CA.")
# Check the signature is valid
try:
digest = hashlib.sha256(f["code"]).hexdigest()
sig = base64.b64decode(f["signature"])
ok = cert.get_pubkey().get_rsa().verify(digest, sig, 'sha256')
except (TypeError, M2Crypto.RSA.RSAError):
statsd.increment("parser.filters.hotfix.invalid_signature")
raise ValueError("Hotfix signature is not valid")
if not ok:
statsd.increment("parser.filters.hotfix.invalid_signature")
raise ValueError("Hotfix signature is not valid")
def head(key):
"""
Make a HEAD request for an object in S3. This is needed to find the
object's length so it can be accounted. This function makes one
billable request and anticipates another.
"""
librato.count('blueprint-io-server.requests.head')
statsd.increment('blueprint-io-server.requests.head')
c = boto.connect_s3(access_key, secret_key)
b = c.get_bucket(bucket, validate=False)
try:
k = b.get_key(key)
if k is None:
return None
return k.size
except (boto.exception.BotoClientError,
boto.exception.BotoServerError,
httplib.HTTPException,
socket.error,
socket.gaierror):
return False
def _save_updated_doc(self, doc, attempts=0):
"""
Save doc to the database, retrying with a merge in the event of
resource conflicts. This should definitely be a method of some Telem
class thing.
"""
latest = self.db[doc['_id']]
latest['data'].update(doc['data'])
try:
self.db.save_doc(latest)
logger.debug("Saved doc {0} successfully".format(doc["_id"]))
statsd.increment("parser_daemon.saved")
except couchdbkit.exceptions.ResourceConflict:
attempts += 1
if attempts >= 30:
err = "Could not save doc {0} after {1} conflicts." \
.format(doc["_id"], attempts)
logger.error(err)
statsd.increment("parser_daemon.save_error")
raise RuntimeError(err)
else:
logger.debug("Save conflict, trying again (#{0})" \
.format(attempts))
statsd.increment("parser_daemon.save_conflict")
self._save_updated_doc(doc, attempts)
except restkit.errors.Unauthorized as e:
logger.warn("Could not save doc {0}, unauthorized: {1}" \
.format(doc["_id"], e))
return
