def _get_data(self, raw_data, callsign, config, module): """Attempt to parse data from what we know so far.""" sentences = config["payload_configuration"]["sentences"] for sentence_index, sentence in enumerate(sentences): if sentence["callsign"] != callsign: continue if sentence["protocol"] != module["name"]: continue data = self.filtering.intermediate_filter(raw_data, sentence) try: data = module["module"].parse(data, sentence) except (ValueError, KeyError) as e: logger.debug("Exception in {module} main parse: {e}" .format(module=module['name'], e=quick_traceback.oneline(e))) statsd.increment("parser.parse_exception") continue data = self.filtering.post_filter(data, sentence) data["_protocol"] = module["name"] data["_parsed"] = { "time_parsed": strict_rfc3339.now_to_rfc3339_utcoffset(), "payload_configuration": config["id"], "configuration_sentence_index": sentence_index } if "flight_id" in config: data["_parsed"]["flight"] = config["flight_id"] return data raise CantGetData()
def fetch_weather(): try: with statsd.StatsdTimer('fetch_weather'): weather.fetch_weather() except: statsd.increment('fetch_weather_error') logger.exception('fetch_weather()')
def _get_debug(self, raw_data): if self.ascii_exp.search(raw_data): statsd.increment("parser.ascii_doc") return 'ascii', raw_data else: statsd.increment("parser.binary_doc") return 'b64', base64.b64encode(raw_data)
def _get_config(self, callsign, config=None): """ Attempt to get a config doc given the callsign and maybe a provided config doc. """ if config and not self._callsign_in_config(callsign, config): logger.debug("Callsign {c!r} not found in configuration doc" .format(c=callsign)) raise CantGetConfig() elif config: if "_id" not in config: config["_id"] = None logger.debug("payload_configuration provided (id: {0})" .format(config["_id"])) return {"id": config["_id"], "payload_configuration": config} config = self._find_config_doc(callsign) if not config: logger.debug("No configuration doc for {callsign!r} found" .format(callsign=callsign)) statsd.increment("parser.no_config_doc") raise CantGetConfig() if "flight_id" in config: logger.debug("Selected payload_configuration {0} from flight {1} " "for {2!r}" .format(config["id"], config["flight_id"], callsign)) else: logger.debug("Selected payload_configuration {0} for {1!r}" .format(config["id"], callsign)) return config
def api_call_raw(method, uri, token=None, headers=None, get_vars=None, data=None, public=False): """ Takes the uri of an application URL to call """ statsd.increment('api.call') if not headers: headers = {} if data is not None: data = json.dumps(data, default=date_handler) headers['Content-Type'] = 'application/json' if not public: if token is not None: headers['Authorization'] = 'Bearer %s' % token else: encoded_consumer = b64encode('%s:%s' % (settings.MASTER_OAUTH_KEY, settings.MASTER_OAUTH_SECRET)) headers['Authorization'] = 'Basic %s' % encoded_consumer api_url = build_url(uri, get_vars) auth = build_auth() req = requests.request(method, api_url, headers=headers, data=data, auth=auth, timeout=90) return req
def _get_config(self, callsign, config=None): """ Attempt to get a config doc given the callsign and maybe a provided config doc. """ if config and not self._callsign_in_config(callsign, config): logger.debug( "Callsign {c!r} not found in configuration doc".format( c=callsign)) raise CantGetConfig() elif config: if "_id" not in config: config["_id"] = None logger.debug("payload_configuration provided (id: {0})".format( config["_id"])) return {"id": config["_id"], "payload_configuration": config} config = self._find_config_doc(callsign) if not config: logger.debug("No configuration doc for {callsign!r} found".format( callsign=callsign)) statsd.increment("parser.no_config_doc") raise CantGetConfig() if "flight_id" in config: logger.debug("Selected payload_configuration {0} from flight {1} " "for {2!r}".format(config["id"], config["flight_id"], callsign)) else: logger.debug("Selected payload_configuration {0} for {1!r}".format( config["id"], callsign)) return config
def fetch_exchanges(): for k, parser in EXCHANGE_PARSERS.iteritems(): try: with statsd.StatsdTimer('fetch_one_exchange'): country_code1, country_code2 = k.split('->') if sorted([country_code1, country_code2])[0] != country_code1: raise Exception( 'Exchange key pair %s is not ordered alphabetically' % k) obj = parser(country_code1, country_code2, session) if not obj: continue if obj.get('sortedCountryCodes', None) != k: raise Exception( "Sorted country codes %s and %s don't match" % (obj.get('sortedCountryCodes', None), k)) if not 'datetime' in obj: raise Exception('datetime was not returned for %s' % k) if arrow.get(obj['datetime']) > arrow.now(): raise Exception("Data from %s can't be in the future" % k) # Database insert result = db_upsert(col_exchange, obj, 'sortedCountryCodes') if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_KEY) except: statsd.increment('fetch_one_exchange_error') logger.exception('Exception while fetching exchange of %s' % k)
def _get_data(self, raw_data, callsign, config, module): """Attempt to parse data from what we know so far.""" sentences = config["payload_configuration"]["sentences"] for sentence_index, sentence in enumerate(sentences): if sentence["callsign"] != callsign: continue if sentence["protocol"] != module["name"]: continue data = self.filtering.intermediate_filter(raw_data, sentence) try: data = module["module"].parse(data, sentence) except (ValueError, KeyError) as e: logger.debug("Exception in {module} main parse: {e}".format( module=module['name'], e=quick_traceback.oneline(e))) statsd.increment("parser.parse_exception") continue data = self.filtering.post_filter(data, sentence) data["_protocol"] = module["name"] data["_parsed"] = { "time_parsed": strict_rfc3339.now_to_rfc3339_utcoffset(), "payload_configuration": config["id"], "configuration_sentence_index": sentence_index } if "flight_id" in config: data["_parsed"]["flight"] = config["flight_id"] return data raise CantGetData()
def _apply_filters(self, data, sentence, filter_type, result_type): if "filters" in sentence: if filter_type in sentence["filters"]: for index, f in enumerate(sentence["filters"][filter_type]): whence = (filter_type, index) data = self._filter(data, f, result_type, whence) statsd.increment("parser.filters.{0}".format(filter_type)) return data
def test_batched_buffer_autoflush(self): fake_socket = FakeSocket() with DogStatsd() as statsd: statsd.socket = fake_socket for i in range(51): statsd.increment('mycounter') t.assert_equal('\n'.join(['mycounter:1|c' for i in range(50)]), fake_socket.recv()) t.assert_equal('mycounter:1|c', fake_socket.recv())
def list(key): librato.count('blueprint-acmeaws-com-server.requests.list') statsd.increment('blueprint-acmeaws-com-server.requests.list') c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) b = c.get_bucket(bucket, validate=False) try: return b.list(key) except (boto.exception.BotoClientError, boto.exception.BotoServerError, httplib.HTTPException, socket.error, socket.gaierror): return False
def fetch_countries(): for parser in parsers: try: with statsd.StatsdTimer('fetch_one_country'): obj = parser() logging.info('INSERT %s' % obj) col.insert_one(obj) except: statsd.increment('fetch_one_country_error') logger.exception('fetch_one_country()')
def list(key): """ List objects in MongoDB whose key begins with the given prefix """ librato.count('blueprint-io-server.requests.list') statsd.increment('blueprint-io-server.requests.list') try: result = collection.find({"key" : '^%s' % (key)}) except: return False return result
def login(request): """ Login page, for which all anonymous users attempting to visit a page without authorization will be redirected to """ if is_authenticated(request): return redirect('dash') if request.method == 'POST': statsd.increment('login.attempt') username = request.POST.get('username', None) password = request.POST.get('password', None) remember = True if request.POST.get('remember', False) else False next = request.POST.get('next', None) try: token, user_data = _login_request( request, username=username, password=password, remember=remember, ) except LoginFailure: # If data posted but is invalid for any reason statsd.increment('login.failure') messages.error(request, "Sorry, those login details aren't valid") next = 'login' else: try: resolve(next) except: next = 'dash' return redirect(next) else: next = request.GET.get('next', 'login') login_form = AuthenticationForm() return render_to_response( 'login.html', { 'next': next, 'login_form': login_form, # 'messages': messages, }, context_instance=RequestContext(request) )
def fetch_weather(): try: with statsd.StatsdTimer('fetch_wind'): fetch_wind() except: statsd.increment('fetch_wind_error') logger.exception('fetch_wind()') try: with statsd.StatsdTimer('fetch_solar'): fetch_solar() except: statsd.increment('fetch_solar_error') logger.exception('fetch_solar()')
def health_GET(): statsd.increment('health_GET') # Check last data point obtained EXPIRATION_SECONDS = 30 * 60.0 result = list(col.find(sort=[('datetime', pymongo.DESCENDING)], limit=1)) if not len(result) or (arrow.now() - arrow.get( result[0]['datetime'])).total_seconds() > EXPIRATION_SECONDS: return flask.jsonify( {'error': 'Database is empty or last measurement is too old.'}), 400 return flask.jsonify({'status': 'ok'})
def new_response(event): request = event.request if request.statsd: if request.exception is None: # Successful request statsd.increment('request.pubsuccess') statsd.timing('request.duration', time.time() * 1000 - request._pub_start) else: # Error response statsd.increment('request.pubfailure') statsd.timing('request.duration', time.time() * 1000 - request._pub_start)
def _hotfix_filter(self, data, f): """Load a filter specified by some code in the database. Check its authenticity by verifying its certificate, then run if OK.""" self._sanity_check_hotfix(f) cert = self._get_certificate(f["certificate"]) self._verify_certificate(f, cert) env = self._compile_hotfix(f) logger.debug("Executing a hotfix") statsd.increment("parser.filters.hotfix.executed") return env["f"](data)
def login(request): """ Login page, for which all anonymous users attempting to visit a page without authorization will be redirected to """ if is_authenticated(request): return redirect('dash') if request.method == 'POST': statsd.increment('login.attempt') username = request.POST.get('username', None) password = request.POST.get('password', None) remember = True if request.POST.get('remember', False) else False next = request.POST.get('next', None) try: token, user_data = _login_request( request, username=username, password=password, remember=remember, ) except LoginFailure: # If data posted but is invalid for any reason statsd.increment('login.failure') messages.error(request, "Sorry, those login details aren't valid") next = 'login' else: try: resolve(next) except: next = 'dash' return redirect(next) else: next = request.GET.get('next', 'login') login_form = AuthenticationForm() return render_to_response( 'login.html', { 'next': next, 'login_form': login_form, # 'messages': messages, }, context_instance=RequestContext(request))
def _compile_hotfix(self, f): """Compile a hotfix into a function **f** in an empty namespace.""" logger.debug("Compiling a hotfix") body = "def f(data):\n" env = {} try: body += "\n".join(" " + l + "\n" for l in f["code"].split("\n")) code = compile(body, "<filter>", "exec") exec code in env except (SyntaxError, AttributeError, TypeError): statsd.increment("parser.filters.hotfix.compile_error") raise ValueError("Hotfix code didn't compile: " + repr(f)) return env
def get(key): """ Fetch an object from MongoDB. """ librato.count('blueprint-io-server.requests.get') statsd.increment('blueprint-io-server.requests.get') try: k = collection.find_one({"key" : key}) if k is None: return None except: return False return k['tarball']
def delete(key): """ Remove an object from MongoDB. """ content_length = head(key) if content_length is None: return None librato.count('blueprint-io-server.requests.delete') statsd.increment('blueprint-io-server.requests.delete') try: collection.delete({"key" : key}) except: return False
def head(key): librato.count('blueprint-acmeaws-com-server.requests.head') statsd.increment('blueprint-acmeaws-com-server.requests.head') c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) b = c.get_bucket(bucket, validate=False) try: k = b.get_key(key) if k is None: return None return k.size except (boto.exception.BotoClientError, boto.exception.BotoServerError, httplib.HTTPException, socket.error, socket.gaierror): return False
def register_user(request): if is_authenticated(request): return redirect('dash') if request.method == 'POST': statsd.increment('registration.attempt') first_name = request.POST.get('first_name', None) last_name = request.POST.get('last_name', None) email = request.POST.get('email', None) username = request.POST.get('username', None) password = request.POST.get('password', None) password_confirm = request.POST.get('password_confirm', None) if password != password_confirm: messages.error(request, "Your passwords did not match") return redirect('registration-user') next = request.POST.get('next', None) try: token, user_data = _create_user( request, first_name=first_name, last_name=last_name, email=email, username=username, password=password, ) except RegistrationFailure: # If data posted but is invalid for any reason statsd.increment('registration.failure') messages.error(request, "Sorry, those registration details aren't valid") next = 'registration-user' else: try: resolve(next) except: next = 'dash' return redirect(next) return render_to_response( 'registration/user.html', {}, context_instance=RequestContext(request), )
def head(key): """ Make a HEAD request for an object in MongoDB. This returns the size of the tarball. """ librato.count('blueprint-io-server.requests.head') statsd.increment('blueprint-io-server.requests.head') try: k = collection.find_one({"key" : key}) if k is None: return None except: return False return len(k['tarball'])
def put(key, data): """ Store an object in MongoDB. """ librato.count('blueprint-io-server.requests.put') statsd.increment('blueprint-io-server.requests.put') # TODO librato.something('blueprint-io-server.storage', len(data)) statsd.update('blueprint-io-server.storage', len(data)) element = StoredObject(key, data) try: collection.insert(element.__dict__) except: return False return True
def production_GET(): statsd.increment('production_GET') objs = col.aggregate([{ '$group': { '_id': '$countryCode', 'lastDocument': { '$last': '$$CURRENT' } } }]) objs = map(bson_measurement_to_json, map(lambda o: o['lastDocument'], list(objs))) data = {obj['countryCode']: obj for obj in objs if 'countryCode' in obj} return flask.jsonify({'status': 'ok', 'data': data})
def fetch_productions(): for country_code, parser in PRODUCTION_PARSERS.iteritems(): try: with statsd.StatsdTimer('fetch_one_production'): obj = parser(country_code, session) if not obj: continue validate_production(obj, country_code) # Database insert result = db_upsert(col_production, obj, 'countryCode') if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_STATE_KEY) except: statsd.increment('fetch_one_production_error') logger.exception('Exception while fetching production of %s' % country_code)
def delete(key): content_length = head(key) if content_length is None: return None librato.count('blueprint-acmeaws-com-server.requests.delete') statsd.increment('blueprint-acmeaws-com-server.requests.delete') c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) b = c.get_bucket(bucket, validate=False) try: b.delete_key(key) statsd.update('blueprint-acmeaws-com-server.storage', -content_length) except (boto.exception.BotoClientError, boto.exception.BotoServerError, httplib.HTTPException, socket.error, socket.gaierror): return False
def put(key, data): librato.count('blueprint-io-server.requests.put') statsd.increment('blueprint-io-server.requests.put') statsd.update('blueprint-io-server.storage', len(data)) c = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) b = c.get_bucket(bucket, validate=False) k = b.new_key(key) try: k.set_contents_from_string(data, policy='public-read', reduced_redundancy=True) return True except (boto.exception.BotoClientError, boto.exception.BotoServerError, httplib.HTTPException, socket.error, socket.gaierror): return False
def list(key): """ List objects in S3 whose keys begin with the given prefix. This function makes at least one billable request. """ librato.count('blueprint-io-server.requests.list') statsd.increment('blueprint-io-server.requests.list') c = boto.connect_s3(access_key, secret_key) b = c.get_bucket(bucket, validate=False) return b.list(key) try: return True except (boto.exception.BotoClientError, boto.exception.BotoServerError, httplib.HTTPException, socket.error, socket.gaierror): return False
def get_tarball(secret, name, sha): validate_secret(secret) validate_name(name) sha = sha.lower() validate_sha(sha) content_length = backend.head_tarball(secret, name, sha) if content_length is None: abort(404) librato.count('blueprint-io-server.requests.get') statsd.increment('blueprint-io-server.requests.get') librato.count('blueprint-io-server.bandwidth.out', content_length) statsd.update('blueprint-io-server.bandwidth.out', content_length) return redirect(backend.url_for_tarball(secret, name, sha), code=301)
def test_increment(self): with mock.patch('statsd.Client') as mock_client: self.counter.increment('') mock_client._send.assert_called_with(mock.ANY, {'testing': '1|c'}) self.counter.increment('', 2) mock_client._send.assert_called_with(mock.ANY, {'testing': '2|c'}) self.counter += 3 mock_client._send.assert_called_with(mock.ANY, {'testing': '3|c'}) statsd.increment('testing', 4) mock_client._send.assert_called_with(mock.ANY, {'testing': '4|c'}) statsd.increment('testing') mock_client._send.assert_called_with(mock.ANY, {'testing': '1|c'})
def get(key): """ Fetch an object from S3. This function makes one billable request. """ librato.count('blueprint-io-server.requests.get') statsd.increment('blueprint-io-server.requests.get') c = boto.connect_s3(access_key, secret_key) b = c.get_bucket(bucket, validate=False) k = b.new_key(key) try: return k.get_contents_as_string() except boto.exception.S3ResponseError: return None except (boto.exception.BotoClientError, boto.exception.BotoServerError, httplib.HTTPException, socket.error, socket.gaierror): return False
def _create_user(request, **kwargs): res, status = api_call('post', '/api/v0/registration/user/', data=kwargs) if status != 200 or 'token' not in res: raise RegistrationFailure token = res['token'] data, status = api_call('get', '/api/v0/account/authed/', token=token) if status != 200 or data is None: raise AuthFailure() statsd.increment('registration.success') for key, val in data.items(): request.session[key] = val request.session['token'] = token return token, data
def head(key): """ Make a HEAD request for an object in S3. This is needed to find the object's length so it can be accounted. This function makes one billable request and anticipates another. """ librato.count('blueprint-io-server.requests.head') statsd.increment('blueprint-io-server.requests.head') c = boto.connect_s3(access_key, secret_key) b = c.get_bucket(bucket, validate=False) try: k = b.get_key(key) if k is None: return None return k.size except (boto.exception.BotoClientError, boto.exception.BotoServerError, httplib.HTTPException, socket.error, socket.gaierror): return False
def _login_request(request, remember=True, *args, **kwargs): res, status = api_call('post', '/api/v0/account/login/', data=kwargs) if status != 200 or 'token' not in res: raise LoginFailure() token = res['token'] data, status = api_call('get', '/api/v0/account/authed/', token=token) if status != 200 or data is None: raise AuthFailure() statsd.increment('login.success') for key, val in data.items(): request.session[key] = val request.session['token'] = token if remember: request.session.set_expiry(0) return token, data
def put(key, data): """ Store an object in S3. This function makes one billable request. """ librato.count('blueprint-io-server.requests.put') statsd.increment('blueprint-io-server.requests.put') # TODO librato.something('blueprint-io-server.storage', len(data)) statsd.update('blueprint-io-server.storage', len(data)) c = boto.connect_s3(access_key, secret_key) b = c.get_bucket(bucket, validate=False) k = b.new_key(key) try: k.set_contents_from_string(data, policy='public-read', reduced_redundancy=True) return True except (boto.exception.BotoClientError, boto.exception.BotoServerError, httplib.HTTPException, socket.error, socket.gaierror): return False
def set_language(request): """Sets the current language.""" if request.matchdict.get("lang", "fi") == "sv": request.session["locale"] = "sv" if request.statsd: statsd.increment("lang.sv") LOG.info("Changed language to swedish") else: if request.statsd: statsd.increment("lang.fi") # Fall back to whatever default request.session.pop("locale") # Make an attempt to redirect back to the page we came from if request.referer is not None and request.referer.startswith(request.application_url): location = request.referer else: location = request.application_url return HTTPFound(location=location)
def _verify_certificate(self, f, cert): """Check that the certificate is cryptographically signed by a key which is signed by a known CA.""" # Check the certificate is valid for ca_cert in self.certificate_authorities: if cert.verify(ca_cert.get_pubkey()): break raise ValueError("Certificate is not signed by a recognised CA.") # Check the signature is valid try: digest = hashlib.sha256(f["code"]).hexdigest() sig = base64.b64decode(f["signature"]) ok = cert.get_pubkey().get_rsa().verify(digest, sig, 'sha256') except (TypeError, M2Crypto.RSA.RSAError): statsd.increment("parser.filters.hotfix.invalid_signature") raise ValueError("Hotfix signature is not valid") if not ok: statsd.increment("parser.filters.hotfix.invalid_signature") raise ValueError("Hotfix signature is not valid")
def delete(key): """ Remove an object from S3. DELETE requests are free but this function still makes one billable request to account for freed storage. """ content_length = head(key) if content_length is None: return None librato.count('blueprint-io-server.requests.delete') statsd.increment('blueprint-io-server.requests.delete') c = boto.connect_s3(access_key, secret_key) b = c.get_bucket(bucket, validate=False) try: b.delete_key(key) # TODO librato.something('blueprint-io-server.storage', -content_length) statsd.update('blueprint-io-server.storage', -content_length) except (boto.exception.BotoClientError, boto.exception.BotoServerError, boto.exception.S3ResponseError, httplib.HTTPException, socket.error, socket.gaierror): return False
def _save_updated_doc(self, doc, attempts=0): """ Save doc to the database, retrying with a merge in the event of resource conflicts. This should definitely be a method of some Telem class thing. """ latest = self.db[doc['_id']] latest['data'].update(doc['data']) try: self.db.save_doc(latest) logger.debug("Saved doc {0} successfully".format(doc["_id"])) statsd.increment("parser_daemon.saved") except couchdbkit.exceptions.ResourceConflict: attempts += 1 if attempts >= 30: err = "Could not save doc {0} after {1} conflicts." \ .format(doc["_id"], attempts) logger.error(err) statsd.increment("parser_daemon.save_error") raise RuntimeError(err) else: logger.debug("Save conflict, trying again (#{0})" \ .format(attempts)) statsd.increment("parser_daemon.save_conflict") self._save_updated_doc(doc, attempts) except restkit.errors.Unauthorized as e: logger.warn("Could not save doc {0}, unauthorized: {1}" \ .format(doc["_id"], e)) return