def get(self, metric): """ Return data for a given metric as JSON. Parameters ---------- metric : str or int The metric name or the metric internal id (int) to get data for. """ logger.debug("GET /api/v1/data/%s" % metric) # Support both metric_id and metric_name try: metric_id = int(metric) metric_name = orm.Metric.get( orm.Metric.metric_id == metric_id).name except ValueError: # We couldn't parse as an int, so it's a metric name instead. metric_name = metric try: raw_data = db.get_data(metric_name) units = db.get_units(metric_name) except DoesNotExist: return ErrorResponse.metric_not_found(metric_name) if len(raw_data) == 0: return ErrorResponse.metric_has_no_data(metric_name) data = utils.format_data(raw_data, units) return jsonify(data)
def post(self): """ Add a new value and possibly a metric if needed. Expected JSON payload has the following key/value pairs:: metric: string value: numeric time: integer or missing """ data = request.get_json() logger.debug("Received POST /api/v1/data: {}".format(data)) try: metric = data['metric'] value = data['value'] except KeyError: logger.warning("Missing JSON keys 'metric' or 'value'.") return "Missing required key. Required keys are:", 400 time = data.get('time', None) db.add_metric(metric) new = db.insert_datapoint(metric, value, time) msg = "Added DataPoint to Metric {}\n".format(new.metric) logger.info("Added value %s to metric '%s'" % (value, metric)) return msg, 201
def insert_datapoint(metric, value, timestamp=None): """ Add a new datapoint for a given metric. Parameters ---------- metric : str The full metric name. value : numeric The value for this data point. timestamp : int, optional The POSIX timestamp for the data point. If ``None``, use the current timestamp. Returns ------- new : :class:`orm.DataPoint` object An instance of the newly-created model object. """ logger.debug("Adding data point %s to metric '%s'" % (value, metric)) metric = Metric.get(Metric.name == metric) if timestamp is None: logger.debug("Timestamp not given, using current time.") timestamp = datetime.now(timezone.utc).timestamp() new = DataPoint.create( metric=metric, value=value, timestamp=timestamp, ) return new
def delete(self, metric_id): logger.debug("'api: DELETE '%s'" % metric_id) try: found = db.Metric.get(db.Metric.metric_id == metric_id) found.delete_instance() except DoesNotExist: return ErrorResponse.metric_not_found(metric_id)
def add_metric(name, units=None, lower_limit=None, upper_limit=None): """ Add a new metric to the database. If both ``lower_limit`` and ``upper_limit`` are given, then ``upper_limit`` must be greater than ``lower_limit``. Parameters ---------- name : str The full metric name. Eg. `tphweb.master.coverage` units : str, optional List the units for this metric. lower_limit: float, optional The lower limit for data. Data values below this limit will trigger email alerts. upper_limit: float, optional The upper limit for data. Data values above this limit will trigger email alerts. Returns ------- metric : :class:`orm.Metric` object The metric that was added. Raises ------ ValueError The provide limits do not satisfy ``upper_limit <= lower_limit``. TypeError The provided limits are not numeric or ``None``. """ logger.debug("Querying metric '%s'" % name) _t = (int, float, type(None)) if not isinstance(lower_limit, _t) or not isinstance(upper_limit, _t): msg = "Invalid type for limits. lower_limit: {}. upper_limit: {}" logger.error(msg.format(type(lower_limit), type(upper_limit))) raise TypeError( "upper_limit and lower_limit must be numerics or None.") if lower_limit is not None and upper_limit is not None: if upper_limit <= lower_limit: logger.error("upper_limit not greater than lower_limit.") raise ValueError("upper_limit must be greater than lower_limit") metric, created = Metric.get_or_create( name=name, units=units, lower_limit=lower_limit, upper_limit=upper_limit, ) if created: logger.info("Metric '%s' created." % name) else: logger.debug("Found existing metric '%s'." % name) return metric
def get_metrics(): """ Return a list of all metrics. Returns ------- metrics : iterable of :class:`orm.Metric` objects """ logger.debug("Querying list of metrics.") return Metric.select()
def get_datapoints(): """ Return a list of all datapoints. Returns ------- datapoints : iterable of :class:`orm.DataPoint` objects If no data exists, an empty iterable is returned. """ logger.debug("Querying list of datapoints.") # TODO: Should I raise DoesNotExist if there's no data? return DataPoint.select()
def get(self, datapoint_id): """ Return the data for a single datapoint. """ logger.debug("api: GET datapoint by ID") try: raw_data = db.get_datapoint(datapoint_id) except DoesNotExist: return ErrorResponse.datapoint_not_found(datapoint_id) data = model_to_dict(raw_data) return jsonify(data)
def delete(self, datapoint_id): """ Delete a datapoint. """ logger.debug("'api: DELETE datapoint '%s'" % datapoint_id) try: found = db.DataPoint.get(db.DataPoint.datapoint_id == datapoint_id) found.delete_instance() except DoesNotExist: return ErrorResponse.datapoint_not_found(datapoint_id) else: return "", 204
def post(self): """ Create a new metric. Accepts JSON data with the following format: .. code-block::json { "name": "your.metric_name.here", "units": string, optional, "upper_limit": {float, optional}, "lower_limit": {float, optional}, } Returns ``201`` on success, ``400`` on malformed JSON data (such as when ``name`` is missing), or ``409`` if the metric already exists. See Also -------- :func:`routes.get_metric_as_json` :func:`routes.delete_metric` """ data = request.get_json() try: metric = data['name'] except KeyError: return ErrorResponse.missing_required_key('name') try: exists = db.Metric.get(db.Metric.name == metric) is not None if exists: return ErrorResponse.metric_already_exists(metric) except DoesNotExist: logger.debug("Metric does not exist. Able to create.") units = data.get('units', None) lower_limit = data.get('lower_limit', None) upper_limit = data.get('upper_limit', None) new = db.add_metric(metric, units=units, lower_limit=lower_limit, upper_limit=upper_limit) # Our `db.add_metric` fuction doesn't pull the new metric_id, so we # grab that separately. new.metric_id = db.Metric.get(db.Metric.name == new.name).metric_id return jsonify(model_to_dict(new)), 201
def get(self, metric_id): """ Return metric information as JSON """ logger.debug("API: get metric '%s'" % metric_id) try: raw_data = db.Metric.get(db.Metric.metric_id == metric_id) except DoesNotExist: return ErrorResponse.metric_not_found(metric_id) data = model_to_dict(raw_data) return jsonify(data)
def handle(self): data = self.request.recv(1024).strip() try: parsed = utils.parse_socket_data(data) logger.debug("TCP: {}".format(parsed)) except ValueError: logger.warn("TCP: Failed to parse `%s`." % data) return try: r = requests.post(URL, json=parsed) logger.info(r.status_code) except Exception: raise self.request.sendall(b"accepted")
def get_datapoint(datapoint_id): """ Return a single datapoint. Parameters ---------- datapoint_id : int The internal ID of the datapoint to query Returns ------- datapoint : :class:`orm.DataPoint` or None ``None`` if the item isn't found. """ logger.debug("Querying datapoint: %s" % datapoint_id) return DataPoint.get_by_id(datapoint_id)
def get(self): """ Return a list of all metrics in the database. """ logger.debug("api: GET all metrics") raw_data = db.get_metrics() if len(raw_data) == 0: # do a thing. return ErrorResponse.no_data() data = [model_to_dict(m) for m in raw_data] # For now, fill in dummy values. return jsonify({ "count": len(data), "prev": None, "next": None, "results": data })
def get_data(metric): """ Return all of the data for a given metric. Parameters ---------- metric : str The full metric name. Returns ------- data : :class:`peewee.ModelSelect` The returned data. Acts like an iterable of :class:`orm.DataPoint` objects """ logger.debug("Querying data for '%s'" % metric) metric = Metric.get(Metric.name == metric) data = DataPoint.select().where(DataPoint.metric == metric.metric_id) return data
def delete_datapoint(datapoint): """ Delete a datapoint. Parameters ---------- datapoint : int or :class:`orm.DataPoint` The datapoint to delete Raises ------ DataPointDoesNotExist : :class:`peewee.DoesNotExist` if the ``datapoint`` or ``datapoint_id`` is not found. """ logger.debug("Deleting datapoint: %s" % datapoint) if isinstance(datapoint, int): datapoint = get_datapoint(datapoint) datapoint.delete_instance()
def config_from_pyfile(app, filename, silent=False): """ Mimics Flask's config.from_pyfile() Allows loading a separate, perhaps non `.py`, file into Celery. Example: >>> config_from_pyfile(celery, './some_dir/config_file.cfg') Arguments: app (Celery app instance): The celery app to update filename (str): The file to load. silent (bool): If true then import errors will be ignored. Also shamelessly taken from Flask: https://github.com/pallets/flask/blob/74691fbe0192de1134c93e9821d5f8ef65405670/flask/config.py#L111 """ filename = str(Path(filename).resolve()) d = types.ModuleType('config') d.__file__ = filename try: with open(filename, 'rb') as config_file: exec(compile(config_file.read(), filename, 'exec'), d.__dict__) except IOError as e: if silent and e.errno in (errno.ENOENT, errno.EISDIR, errno.ENOTDIR): return False e.strerror = "Unable to load config file (%s)" % e.strerror raise # Remove any hidden attributes: __ and _ for k in list(d.__dict__.keys()): if k.startswith("_"): del d.__dict__[k] logger.debug("Config values: %s" % d.__dict__) app.conf.update(d.__dict__) return True
def get_recent_data(metric, age): """ Return all data that is less than `age` seconds old. Parameters ---------- metric : str The full metric name. age : int Only return data that is less than `age` seconds old. Returns ------- data : iterable of :class:`orm.DataPoint` objects """ logger.debug("Querying last %s seconds of data for '%s'." % (age, metric)) metric = Metric.get(Metric.name == metric) now = datetime.now(timezone.utc).timestamp() data = DataPoint.select().where((DataPoint.metric == metric.metric_id) & (DataPoint.timestamp > (now - age))) return data
def create_celery(): celery = Celery(__name__, autofinalize=False) # Pull config from file. This is basically the same as what is # done in app_factory.create_app() celery.config_from_object('trendlines.default_config') try: config_from_envvar(celery, CFG_VAR) logger.info("Loaded config file '%s'" % os.environ[CFG_VAR]) except FileNotFoundError: msg = "Failed to load config file. The file %s='%s' was not found." logger.warning(msg % (CFG_VAR, os.environ[CFG_VAR])) except (RuntimeError, ImproperlyConfigured) as err: # Celery's error for missing env var is sufficient. logger.warning(str(err)) except Exception as err: logger.warning("An unknown error occured while reading from the" " config file. See debug stack trace for details.") logger.debug(format_exc()) UDP_PORT = celery.conf['UDP_PORT'] TCP_PORT = celery.conf['TCP_PORT'] HOST = celery.conf['TARGET_HOST'] URL = celery.conf['TRENDLINES_API_URL'] celery.finalize() logger.debug("Celery has been finalized.") class TCPHandler(socketserver.BaseRequestHandler): def handle(self): data = self.request.recv(1024).strip() try: parsed = utils.parse_socket_data(data) logger.debug("TCP: {}".format(parsed)) except ValueError: logger.warn("TCP: Failed to parse `%s`." % data) return try: r = requests.post(URL, json=parsed) logger.info(r.status_code) except Exception: raise self.request.sendall(b"accepted") @celery.task def listen_to_tcp(): hp = (HOST, TCP_PORT) logger.info("listening for TCP on %s:%s" % hp) with socketserver.TCPServer(hp, TCPHandler) as server: server.serve_forever() # Start our tasks logger.debug("Starting tasks") # listen_to_udp.delay() listen_to_tcp.delay() return celery
def create_app(): """ Primary application factory. """ _logging.setup_logging(logger) logger.debug("Creating app.") app = Flask(__name__) app.config.from_object('trendlines.default_config') try: app.config.from_envvar(CFG_VAR) logger.info("Loaded config file '%s'" % os.environ[CFG_VAR]) except FileNotFoundError: msg = "Failed to load config file. The file %s='%s' was not found." logger.warning(msg % (CFG_VAR, os.environ[CFG_VAR])) except RuntimeError as err: # Flask's error for missing env var is sufficient. logger.warning(str(err)) except Exception as err: logger.warning("An unknown error occured while reading from the" " config file. See debug stack trace for details.") logger.debug(format_exc()) logger.debug("Registering blueprints.") app.register_blueprint(routes.pages) # Initialize flask-rest-api for OpenAPI (Swagger) documentation routes.api_class.init_app(app) routes.api_class.register_blueprint(routes.api) routes.api_class.register_blueprint(routes.api_datapoint) routes.api_class.register_blueprint(routes.api_metric) # Create the database file and populate initial tables if needed. orm.create_db(app.config['DATABASE']) # If I redesign the architecture a bit, then these could be moved so # that they only act on the `api` blueprint instead of the entire app. # # Also, note that this is safe for multiple simultaneous requests, since # the database object is thread local: # "Peewee uses thread local storage to manage connection state, so # this pattern can be used with multi-threaded WSGI servers." # http://docs.peewee-orm.com/en/latest/peewee/example.html#establishing-a-database-connection @app.before_request def before_request(): """ Attach the ORM to the flask ``g`` object before every request. Why not just use ``from orm import db`` most of the time? Well, it's because: (a) that's how I'm used to from SQLAlchemy (b) I may need ``db`` somewhere where I *can't* import ``orm`` (c) because that's how the example PeeWee project is set up. /shrug. https://github.com/coleifer/peewee/blob/master/examples/twitter/app.py#L152 """ g.db = orm.db try: g.db.connect() except OperationalError: pass @app.after_request def after_request(response): g.db.close() return response return app
def update_datapoint(datapoint, metric=None, value=None, timestamp=None): """ Update the value or timestamp (or both) of a datapoint. If ``metric``, ``value``, or ``timestamp`` is None, that item will not be updated. Parameters ---------- datapoint : int or :class:`orm.DataPoint` The datapoint to update. Can be provided as an ``int`` for the ``datapoint_id`` or as a :class:`~orm.DataPoint` object directly. metric : int, optional The new metric_id that this datapoint should belong to. value : float, optional The new value for the datapoint. timestamp : int or "now", optional The new timestamp of the datapoint. If ``"now"``, then use the current datetime. This should be the POSIX timestamp integer (UTC). Returns ------- datapoint : :class:`orm.DataPoint` The updated datapoint. Raises ------ DataPoint.DoesNotExist : :class:`peewee.DoesNotExist` if the ``datapoint`` or ``datapoint_id`` is not found. Metric.DoesNotExist : :class:`peewee.DoesNotExist` if the ``metric`` is not found. """ logger.debug("Updating datapoint: %s" % datapoint) # Shortcut the uncommon case where both things are None if all(x is None for x in (metric, value, timestamp)): logger.debug("No new values given. Nothing to do.") return # Make sure we're going to act on an existing object. try: if isinstance(datapoint, DataPoint): DataPoint.get_by_id(datapoint.datapoint_id) else: datapoint = get_datapoint(datapoint) except DataPoint.DoesNotExist: msg = "Unable to find datapoint %s. Can't update values." logger.warning(msg % datapoint) raise # We should only get down here if the row exists. if metric is not None: datapoint.metric = metric if value is not None: datapoint.value = value if timestamp is not None: if timestamp == "now": timestamp = datetime.now(timezone.utc).timestamp() # Convert our POSIX timestamp int to a python datetime object. # We need to (1) specify that the timestamp is in UTC and then # (2) make the timezone object naive because the DataPoint object # uses naive datetimes. new = datetime.fromtimestamp(timestamp, tz=timezone.utc).replace(tzinfo=None) datapoint.timestamp = new # Note that `save()` will create the row if it doesn't exist. Even though # we don't want that, we can still use it because we check for existance, # and raise an error on DoesNotExist, in the above code. So in theory, # we never get down here on a object that doesn't exist. Thus `save()` # will not end up creating a row. # We want people to either (a) use the PK or (b) query the DataPoint # object before running this function. datapoint.save() return datapoint
def create_db(name): """ Create the database and the tables. Applies any missing migrations. Does nothing if all migrations have been applied. Parameters ---------- name : str The name/path of the database, as given by ``app.config['DATABASE']``. """ # import pdb; pdb.set_trace() # Convert to a Path object because I like working with those better. full_path = Path(name).resolve() file_exists = full_path.exists() if file_exists: logger.debug("Connecting to existing database: '%s'." % full_path) else: logger.debug("Creating new database: '%s'" % full_path) db.init(str(full_path), pragmas=DB_OPTS) try: # This will create the file if it doesn't exist. db.connect() except OperationalError: # Try to figure out why OperationalError happened. if file_exists: msg = ("Database file %s exists, but we're unable to connect." " Perhaps the permissions are incorrect?") logger.error(msg % full_path) else: msg = ("Unable to create %s. Perhaps the parent folder is missing" " or permissions are incorrect?") logger.error(msg % full_path) logger.error("Unable to create/open database file '%s'" % full_path) raise # Either way, we want to run migrations. However, we only need to make a # backup if the file already exists. if file_exists: # Create a backup before doing anything. backup_file = utils.backup_file(full_path) logger.debug("Created database backup file: {}".format(backup_file)) # This will edit the database file, creating the `migration_history` # table if needed. Hence why we do it *after* the backup. try: manager = DatabaseManager(db) except PermissionError: # When runnng in the docker container, this will attempt to create # a `/migrations` directory. It should be `/trendlines/migrations`. # If this still fails, let the error propogate but make sure to # close the db connection try: msg = "Failed to open default migration directory, trying '%s'" alt_dir = "/trendlines/migrations" logger.debug(msg % alt_dir) manager = DatabaseManager(db, directory=alt_dir) logger.debug("Success") except PermissionError: raise finally: db.close() # Check the status. Creating a new file means we'll need migrations. # However, we don't need to check for that because it's guaranteed # that a new file will have len(manager.diff) > 0 needs_migrations = len(manager.diff) > 0 if needs_migrations: logger.info("Missing migrations: {}".format(manager.diff)) # Apply the migrations success = manager.upgrade() if success: logger.info("Successfully applied database migrations.") elif file_exists: # revert our changes by restoring the backup msg = ("Failed to apply database migrations. Reverting to backup" " file. Please submit an issue at {} with details.") logger.critical(msg.format(__project_url__)) shutil.copy(str(backup_file), str(full_path)) else: # It's a new file, so no backup was made. msg = ("Failed to apply database migrations to the new file." " Please see the logs for more info.") logger.critical(msg) else: logger.info("Database is up to date. No migrations to apply.") # Since we didn't make any changes, we can remove the backup file. # Is it possible to ever have backup_file not exist if we didn't # apply migrations? # No, because not applying migrations implies that the file already # existed, and if the file already existed then a backup was made. # Thus we don't need to check for FileNotFoundError. backup_file.unlink() logger.debug("Removed superfluous backup file: %s" % backup_file) # Make sure to close the database if things went well. db.close()