def get(self, metric):
"""
Return data for a given metric as JSON.
Parameters
----------
metric : str or int
The metric name or the metric internal id (int) to get data for.
"""
logger.debug("GET /api/v1/data/%s" % metric)
# Support both metric_id and metric_name
try:
metric_id = int(metric)
metric_name = orm.Metric.get(
orm.Metric.metric_id == metric_id).name
except ValueError:
# We couldn't parse as an int, so it's a metric name instead.
metric_name = metric
try:
raw_data = db.get_data(metric_name)
units = db.get_units(metric_name)
except DoesNotExist:
return ErrorResponse.metric_not_found(metric_name)
if len(raw_data) == 0:
return ErrorResponse.metric_has_no_data(metric_name)
data = utils.format_data(raw_data, units)
return jsonify(data)
def post(self):
"""
Add a new value and possibly a metric if needed.
Expected JSON payload has the following key/value pairs::
metric: string
value: numeric
time: integer or missing
"""
data = request.get_json()
logger.debug("Received POST /api/v1/data: {}".format(data))
try:
metric = data['metric']
value = data['value']
except KeyError:
logger.warning("Missing JSON keys 'metric' or 'value'.")
return "Missing required key. Required keys are:", 400
time = data.get('time', None)
db.add_metric(metric)
new = db.insert_datapoint(metric, value, time)
msg = "Added DataPoint to Metric {}\n".format(new.metric)
logger.info("Added value %s to metric '%s'" % (value, metric))
return msg, 201
def insert_datapoint(metric, value, timestamp=None):
"""
Add a new datapoint for a given metric.
Parameters
----------
metric : str
The full metric name.
value : numeric
The value for this data point.
timestamp : int, optional
The POSIX timestamp for the data point. If ``None``, use
the current timestamp.
Returns
-------
new : :class:`orm.DataPoint` object
An instance of the newly-created model object.
"""
logger.debug("Adding data point %s to metric '%s'" % (value, metric))
metric = Metric.get(Metric.name == metric)
if timestamp is None:
logger.debug("Timestamp not given, using current time.")
timestamp = datetime.now(timezone.utc).timestamp()
new = DataPoint.create(
metric=metric,
value=value,
timestamp=timestamp,
)
return new
def delete(self, metric_id):
logger.debug("'api: DELETE '%s'" % metric_id)
try:
found = db.Metric.get(db.Metric.metric_id == metric_id)
found.delete_instance()
except DoesNotExist:
return ErrorResponse.metric_not_found(metric_id)
def add_metric(name, units=None, lower_limit=None, upper_limit=None):
"""
Add a new metric to the database.
If both ``lower_limit`` and ``upper_limit`` are given, then
``upper_limit`` must be greater than ``lower_limit``.
Parameters
----------
name : str
The full metric name. Eg. `tphweb.master.coverage`
units : str, optional
List the units for this metric.
lower_limit: float, optional
The lower limit for data. Data values below this limit will trigger
email alerts.
upper_limit: float, optional
The upper limit for data. Data values above this limit will trigger
email alerts.
Returns
-------
metric : :class:`orm.Metric` object
The metric that was added.
Raises
------
ValueError
The provide limits do not satisfy ``upper_limit <= lower_limit``.
TypeError
The provided limits are not numeric or ``None``.
"""
logger.debug("Querying metric '%s'" % name)
_t = (int, float, type(None))
if not isinstance(lower_limit, _t) or not isinstance(upper_limit, _t):
msg = "Invalid type for limits. lower_limit: {}. upper_limit: {}"
logger.error(msg.format(type(lower_limit), type(upper_limit)))
raise TypeError(
"upper_limit and lower_limit must be numerics or None.")
if lower_limit is not None and upper_limit is not None:
if upper_limit <= lower_limit:
logger.error("upper_limit not greater than lower_limit.")
raise ValueError("upper_limit must be greater than lower_limit")
metric, created = Metric.get_or_create(
name=name,
units=units,
lower_limit=lower_limit,
upper_limit=upper_limit,
)
if created:
logger.info("Metric '%s' created." % name)
else:
logger.debug("Found existing metric '%s'." % name)
return metric
def get_metrics():
"""
Return a list of all metrics.
Returns
-------
metrics : iterable of :class:`orm.Metric` objects
"""
logger.debug("Querying list of metrics.")
return Metric.select()
def get_datapoints():
"""
Return a list of all datapoints.
Returns
-------
datapoints : iterable of :class:`orm.DataPoint` objects
If no data exists, an empty iterable is returned.
"""
logger.debug("Querying list of datapoints.")
# TODO: Should I raise DoesNotExist if there's no data?
return DataPoint.select()
def get(self, datapoint_id):
"""
Return the data for a single datapoint.
"""
logger.debug("api: GET datapoint by ID")
try:
raw_data = db.get_datapoint(datapoint_id)
except DoesNotExist:
return ErrorResponse.datapoint_not_found(datapoint_id)
data = model_to_dict(raw_data)
return jsonify(data)
def delete(self, datapoint_id):
"""
Delete a datapoint.
"""
logger.debug("'api: DELETE datapoint '%s'" % datapoint_id)
try:
found = db.DataPoint.get(db.DataPoint.datapoint_id == datapoint_id)
found.delete_instance()
except DoesNotExist:
return ErrorResponse.datapoint_not_found(datapoint_id)
else:
return "", 204
def post(self):
"""
Create a new metric.
Accepts JSON data with the following format:
.. code-block::json
{
"name": "your.metric_name.here",
"units": string, optional,
"upper_limit": {float, optional},
"lower_limit": {float, optional},
}
Returns ``201`` on success, ``400`` on malformed JSON data (such as when
``name`` is missing), or ``409`` if the metric already exists.
See Also
--------
:func:`routes.get_metric_as_json`
:func:`routes.delete_metric`
"""
data = request.get_json()
try:
metric = data['name']
except KeyError:
return ErrorResponse.missing_required_key('name')
try:
exists = db.Metric.get(db.Metric.name == metric) is not None
if exists:
return ErrorResponse.metric_already_exists(metric)
except DoesNotExist:
logger.debug("Metric does not exist. Able to create.")
units = data.get('units', None)
lower_limit = data.get('lower_limit', None)
upper_limit = data.get('upper_limit', None)
new = db.add_metric(metric,
units=units,
lower_limit=lower_limit,
upper_limit=upper_limit)
# Our `db.add_metric` fuction doesn't pull the new metric_id, so we
# grab that separately.
new.metric_id = db.Metric.get(db.Metric.name == new.name).metric_id
return jsonify(model_to_dict(new)), 201
def get(self, metric_id):
"""
Return metric information as JSON
"""
logger.debug("API: get metric '%s'" % metric_id)
try:
raw_data = db.Metric.get(db.Metric.metric_id == metric_id)
except DoesNotExist:
return ErrorResponse.metric_not_found(metric_id)
data = model_to_dict(raw_data)
return jsonify(data)
def handle(self):
data = self.request.recv(1024).strip()
try:
parsed = utils.parse_socket_data(data)
logger.debug("TCP: {}".format(parsed))
except ValueError:
logger.warn("TCP: Failed to parse `%s`." % data)
return
try:
r = requests.post(URL, json=parsed)
logger.info(r.status_code)
except Exception:
raise
self.request.sendall(b"accepted")
def get_datapoint(datapoint_id):
"""
Return a single datapoint.
Parameters
----------
datapoint_id : int
The internal ID of the datapoint to query
Returns
-------
datapoint : :class:`orm.DataPoint` or None
``None`` if the item isn't found.
"""
logger.debug("Querying datapoint: %s" % datapoint_id)
return DataPoint.get_by_id(datapoint_id)
def get(self):
"""
Return a list of all metrics in the database.
"""
logger.debug("api: GET all metrics")
raw_data = db.get_metrics()
if len(raw_data) == 0:
# do a thing.
return ErrorResponse.no_data()
data = [model_to_dict(m) for m in raw_data]
# For now, fill in dummy values.
return jsonify({
"count": len(data),
"prev": None,
"next": None,
"results": data
})
def get_data(metric):
"""
Return all of the data for a given metric.
Parameters
----------
metric : str
The full metric name.
Returns
-------
data : :class:`peewee.ModelSelect`
The returned data. Acts like an iterable of
:class:`orm.DataPoint` objects
"""
logger.debug("Querying data for '%s'" % metric)
metric = Metric.get(Metric.name == metric)
data = DataPoint.select().where(DataPoint.metric == metric.metric_id)
return data
def delete_datapoint(datapoint):
"""
Delete a datapoint.
Parameters
----------
datapoint : int or :class:`orm.DataPoint`
The datapoint to delete
Raises
------
DataPointDoesNotExist : :class:`peewee.DoesNotExist`
if the ``datapoint`` or ``datapoint_id`` is not found.
"""
logger.debug("Deleting datapoint: %s" % datapoint)
if isinstance(datapoint, int):
datapoint = get_datapoint(datapoint)
datapoint.delete_instance()
def config_from_pyfile(app, filename, silent=False):
"""
Mimics Flask's config.from_pyfile()
Allows loading a separate, perhaps non `.py`, file into Celery.
Example:
>>> config_from_pyfile(celery, './some_dir/config_file.cfg')
Arguments:
app (Celery app instance): The celery app to update
filename (str): The file to load.
silent (bool): If true then import errors will be ignored.
Also shamelessly taken from Flask:
https://github.com/pallets/flask/blob/74691fbe0192de1134c93e9821d5f8ef65405670/flask/config.py#L111
"""
filename = str(Path(filename).resolve())
d = types.ModuleType('config')
d.__file__ = filename
try:
with open(filename, 'rb') as config_file:
exec(compile(config_file.read(), filename, 'exec'), d.__dict__)
except IOError as e:
if silent and e.errno in (errno.ENOENT, errno.EISDIR, errno.ENOTDIR):
return False
e.strerror = "Unable to load config file (%s)" % e.strerror
raise
# Remove any hidden attributes: __ and _
for k in list(d.__dict__.keys()):
if k.startswith("_"):
del d.__dict__[k]
logger.debug("Config values: %s" % d.__dict__)
app.conf.update(d.__dict__)
return True
def get_recent_data(metric, age):
"""
Return all data that is less than `age` seconds old.
Parameters
----------
metric : str
The full metric name.
age : int
Only return data that is less than `age` seconds old.
Returns
-------
data : iterable of :class:`orm.DataPoint` objects
"""
logger.debug("Querying last %s seconds of data for '%s'." % (age, metric))
metric = Metric.get(Metric.name == metric)
now = datetime.now(timezone.utc).timestamp()
data = DataPoint.select().where((DataPoint.metric == metric.metric_id)
& (DataPoint.timestamp > (now - age)))
return data
def create_celery():
celery = Celery(__name__, autofinalize=False)
# Pull config from file. This is basically the same as what is
# done in app_factory.create_app()
celery.config_from_object('trendlines.default_config')
try:
config_from_envvar(celery, CFG_VAR)
logger.info("Loaded config file '%s'" % os.environ[CFG_VAR])
except FileNotFoundError:
msg = "Failed to load config file. The file %s='%s' was not found."
logger.warning(msg % (CFG_VAR, os.environ[CFG_VAR]))
except (RuntimeError, ImproperlyConfigured) as err:
# Celery's error for missing env var is sufficient.
logger.warning(str(err))
except Exception as err:
logger.warning("An unknown error occured while reading from the"
" config file. See debug stack trace for details.")
logger.debug(format_exc())
UDP_PORT = celery.conf['UDP_PORT']
TCP_PORT = celery.conf['TCP_PORT']
HOST = celery.conf['TARGET_HOST']
URL = celery.conf['TRENDLINES_API_URL']
celery.finalize()
logger.debug("Celery has been finalized.")
class TCPHandler(socketserver.BaseRequestHandler):
def handle(self):
data = self.request.recv(1024).strip()
try:
parsed = utils.parse_socket_data(data)
logger.debug("TCP: {}".format(parsed))
except ValueError:
logger.warn("TCP: Failed to parse `%s`." % data)
return
try:
r = requests.post(URL, json=parsed)
logger.info(r.status_code)
except Exception:
raise
self.request.sendall(b"accepted")
@celery.task
def listen_to_tcp():
hp = (HOST, TCP_PORT)
logger.info("listening for TCP on %s:%s" % hp)
with socketserver.TCPServer(hp, TCPHandler) as server:
server.serve_forever()
# Start our tasks
logger.debug("Starting tasks")
# listen_to_udp.delay()
listen_to_tcp.delay()
return celery
def create_app():
"""
Primary application factory.
"""
_logging.setup_logging(logger)
logger.debug("Creating app.")
app = Flask(__name__)
app.config.from_object('trendlines.default_config')
try:
app.config.from_envvar(CFG_VAR)
logger.info("Loaded config file '%s'" % os.environ[CFG_VAR])
except FileNotFoundError:
msg = "Failed to load config file. The file %s='%s' was not found."
logger.warning(msg % (CFG_VAR, os.environ[CFG_VAR]))
except RuntimeError as err:
# Flask's error for missing env var is sufficient.
logger.warning(str(err))
except Exception as err:
logger.warning("An unknown error occured while reading from the"
" config file. See debug stack trace for details.")
logger.debug(format_exc())
logger.debug("Registering blueprints.")
app.register_blueprint(routes.pages)
# Initialize flask-rest-api for OpenAPI (Swagger) documentation
routes.api_class.init_app(app)
routes.api_class.register_blueprint(routes.api)
routes.api_class.register_blueprint(routes.api_datapoint)
routes.api_class.register_blueprint(routes.api_metric)
# Create the database file and populate initial tables if needed.
orm.create_db(app.config['DATABASE'])
# If I redesign the architecture a bit, then these could be moved so
# that they only act on the `api` blueprint instead of the entire app.
#
# Also, note that this is safe for multiple simultaneous requests, since
# the database object is thread local:
# "Peewee uses thread local storage to manage connection state, so
# this pattern can be used with multi-threaded WSGI servers."
# http://docs.peewee-orm.com/en/latest/peewee/example.html#establishing-a-database-connection
@app.before_request
def before_request():
"""
Attach the ORM to the flask ``g`` object before every request.
Why not just use ``from orm import db`` most of the time? Well, it's
because:
(a) that's how I'm used to from SQLAlchemy
(b) I may need ``db`` somewhere where I *can't* import ``orm``
(c) because that's how the example PeeWee project is set up. /shrug.
https://github.com/coleifer/peewee/blob/master/examples/twitter/app.py#L152
"""
g.db = orm.db
try:
g.db.connect()
except OperationalError:
pass
@app.after_request
def after_request(response):
g.db.close()
return response
return app
def update_datapoint(datapoint, metric=None, value=None, timestamp=None):
"""
Update the value or timestamp (or both) of a datapoint.
If ``metric``, ``value``, or ``timestamp`` is None, that item will not be
updated.
Parameters
----------
datapoint : int or :class:`orm.DataPoint`
The datapoint to update. Can be provided as an ``int`` for the
``datapoint_id`` or as a :class:`~orm.DataPoint` object directly.
metric : int, optional
The new metric_id that this datapoint should belong to.
value : float, optional
The new value for the datapoint.
timestamp : int or "now", optional
The new timestamp of the datapoint. If ``"now"``, then use the
current datetime. This should be the POSIX timestamp integer (UTC).
Returns
-------
datapoint : :class:`orm.DataPoint`
The updated datapoint.
Raises
------
DataPoint.DoesNotExist : :class:`peewee.DoesNotExist`
if the ``datapoint`` or ``datapoint_id`` is not found.
Metric.DoesNotExist : :class:`peewee.DoesNotExist`
if the ``metric`` is not found.
"""
logger.debug("Updating datapoint: %s" % datapoint)
# Shortcut the uncommon case where both things are None
if all(x is None for x in (metric, value, timestamp)):
logger.debug("No new values given. Nothing to do.")
return
# Make sure we're going to act on an existing object.
try:
if isinstance(datapoint, DataPoint):
DataPoint.get_by_id(datapoint.datapoint_id)
else:
datapoint = get_datapoint(datapoint)
except DataPoint.DoesNotExist:
msg = "Unable to find datapoint %s. Can't update values."
logger.warning(msg % datapoint)
raise
# We should only get down here if the row exists.
if metric is not None:
datapoint.metric = metric
if value is not None:
datapoint.value = value
if timestamp is not None:
if timestamp == "now":
timestamp = datetime.now(timezone.utc).timestamp()
# Convert our POSIX timestamp int to a python datetime object.
# We need to (1) specify that the timestamp is in UTC and then
# (2) make the timezone object naive because the DataPoint object
# uses naive datetimes.
new = datetime.fromtimestamp(timestamp,
tz=timezone.utc).replace(tzinfo=None)
datapoint.timestamp = new
# Note that `save()` will create the row if it doesn't exist. Even though
# we don't want that, we can still use it because we check for existance,
# and raise an error on DoesNotExist, in the above code. So in theory,
# we never get down here on a object that doesn't exist. Thus `save()`
# will not end up creating a row.
# We want people to either (a) use the PK or (b) query the DataPoint
# object before running this function.
datapoint.save()
return datapoint
def create_db(name):
"""
Create the database and the tables.
Applies any missing migrations. Does nothing if all migrations
have been applied.
Parameters
----------
name : str
The name/path of the database, as given by ``app.config['DATABASE']``.
"""
# import pdb; pdb.set_trace()
# Convert to a Path object because I like working with those better.
full_path = Path(name).resolve()
file_exists = full_path.exists()
if file_exists:
logger.debug("Connecting to existing database: '%s'." % full_path)
else:
logger.debug("Creating new database: '%s'" % full_path)
db.init(str(full_path), pragmas=DB_OPTS)
try:
# This will create the file if it doesn't exist.
db.connect()
except OperationalError:
# Try to figure out why OperationalError happened.
if file_exists:
msg = ("Database file %s exists, but we're unable to connect."
" Perhaps the permissions are incorrect?")
logger.error(msg % full_path)
else:
msg = ("Unable to create %s. Perhaps the parent folder is missing"
" or permissions are incorrect?")
logger.error(msg % full_path)
logger.error("Unable to create/open database file '%s'" % full_path)
raise
# Either way, we want to run migrations. However, we only need to make a
# backup if the file already exists.
if file_exists:
# Create a backup before doing anything.
backup_file = utils.backup_file(full_path)
logger.debug("Created database backup file: {}".format(backup_file))
# This will edit the database file, creating the `migration_history`
# table if needed. Hence why we do it *after* the backup.
try:
manager = DatabaseManager(db)
except PermissionError:
# When runnng in the docker container, this will attempt to create
# a `/migrations` directory. It should be `/trendlines/migrations`.
# If this still fails, let the error propogate but make sure to
# close the db connection
try:
msg = "Failed to open default migration directory, trying '%s'"
alt_dir = "/trendlines/migrations"
logger.debug(msg % alt_dir)
manager = DatabaseManager(db, directory=alt_dir)
logger.debug("Success")
except PermissionError:
raise
finally:
db.close()
# Check the status. Creating a new file means we'll need migrations.
# However, we don't need to check for that because it's guaranteed
# that a new file will have len(manager.diff) > 0
needs_migrations = len(manager.diff) > 0
if needs_migrations:
logger.info("Missing migrations: {}".format(manager.diff))
# Apply the migrations
success = manager.upgrade()
if success:
logger.info("Successfully applied database migrations.")
elif file_exists:
# revert our changes by restoring the backup
msg = ("Failed to apply database migrations. Reverting to backup"
" file. Please submit an issue at {} with details.")
logger.critical(msg.format(__project_url__))
shutil.copy(str(backup_file), str(full_path))
else:
# It's a new file, so no backup was made.
msg = ("Failed to apply database migrations to the new file."
" Please see the logs for more info.")
logger.critical(msg)
else:
logger.info("Database is up to date. No migrations to apply.")
# Since we didn't make any changes, we can remove the backup file.
# Is it possible to ever have backup_file not exist if we didn't
# apply migrations?
# No, because not applying migrations implies that the file already
# existed, and if the file already existed then a backup was made.
# Thus we don't need to check for FileNotFoundError.
backup_file.unlink()
logger.debug("Removed superfluous backup file: %s" % backup_file)
# Make sure to close the database if things went well.
db.close()