def __init__(self, graph, uri, pi, pinNumber):
self.graph, self.uri, self.pi = graph, uri, pi
self.pinNumber = pinNumber
scales.init(self, self.__class__.__name__)
self.stats = scales.collection(self.__class__.__name__,
scales.PmfStat('poll'),
scales.PmfStat('output'),
)
self.hostStateInit()
def init_app(self, app):
self.config = app.config
enabled = self.config.setdefault("GRAPHITE_ENABLED", False)
self.app = app
if not enabled:
return
from greplin import scales
from greplin.scales.graphite import GraphitePeriodicPusher
from greplin.scales.meter import MeterStat
host = self.config.setdefault("GRAPHITE_HOST", "localhost")
port = self.config.setdefault("GRAPHITE_PORT", 2003)
prefix = self.config.setdefault("GRAPHITE_PREFIX", "doorman")
period = self.config.setdefault("GRAPHITE_REPORTING_INTERVAL", 60)
app.metrics = {}
for rule in app.url_map.iter_rules():
app.metrics[rule.endpoint] = scales.collection(
rule.endpoint,
MeterStat('count'),
scales.PmfStat('latency'),
)
app.graphite = GraphitePeriodicPusher(
host, port, period=period, prefix=prefix,
)
for rule in self.config.setdefault("GRAPHITE_ALLOW", ['*']):
app.graphite.allow(rule)
app.graphite.start()
return
def __init__(self, cluster_proxy):
log.debug("Starting metric capture")
self.stats = scales.collection('/cassandra',
scales.PmfStat('request_timer'),
scales.IntStat('connection_errors'),
scales.IntStat('write_timeouts'),
scales.IntStat('read_timeouts'),
scales.IntStat('unavailables'),
scales.IntStat('other_errors'),
scales.IntStat('retries'),
scales.IntStat('ignores'),
# gauges
scales.Stat('known_hosts',
lambda: len(cluster_proxy.metadata.all_hosts())),
scales.Stat('connected_to',
lambda: len(set(chain.from_iterable(s._pools.keys() for s in cluster_proxy.sessions)))),
scales.Stat('open_connections',
lambda: sum(sum(p.open_count for p in s._pools.values()) for s in cluster_proxy.sessions)))
self.request_timer = self.stats.request_timer
self.connection_errors = self.stats.connection_errors
self.write_timeouts = self.stats.write_timeouts
self.read_timeouts = self.stats.read_timeouts
self.unavailables = self.stats.unavailables
self.other_errors = self.stats.other_errors
self.retries = self.stats.retries
self.ignores = self.stats.ignores
self.known_hosts = self.stats.known_hosts
self.connected_to = self.stats.connected_to
self.open_connections = self.stats.open_connections
class RequestAnalyzer(object):
"""
Class used to track request and error counts for a Session.
Also computes statistics on encoded request size.
"""
requests = scales.PmfStat('request size')
errors = scales.IntStat('errors')
def __init__(self, session):
scales.init(self, '/cassandra')
# each instance will be registered with a session, and receive a callback for each request generated
session.add_request_init_listener(self.on_request)
def on_request(self, rf):
# This callback is invoked each time a request is created, on the thread creating the request.
# We can use this to count events, or add callbacks
rf.add_callbacks(self.on_success, self.on_error, callback_args=(rf,), errback_args=(rf,))
def on_success(self, _, response_future):
# future callback on a successful request; just record the size
self.requests.addValue(response_future.request_encoded_size)
def on_error(self, _, response_future):
# future callback for failed; record size and increment errors
self.requests.addValue(response_future.request_encoded_size)
self.errors += 1
def __str__(self):
# just extracting request count from the size stats (which are recorded on all requests)
request_sizes = dict(self.requests)
count = request_sizes.pop('count')
return "%d requests (%d errors)\nRequest size statistics:\n%s" % (count, self.errors, pp.pformat(request_sizes))
class Stats(ApplicationStats):
status_ok = meter.MeterStat('status_ok')
status_failed = meter.MeterStat('status_failed')
status_forbidden = meter.MeterStat('status_forbidden')
status_not_found = meter.MeterStat('status_not_found')
status_not_modified = meter.MeterStat('status_not_modified')
status_redirect = meter.MeterStat('status_redirect')
requests = meter.MeterStat('requests')
latency = scales.PmfStat('latency')
def __init__(self, parent, name, regexp=None):
scales._Stats.initChild(self, name, '', parent=parent)
if regexp is not None:
self.pattern = re.compile(regexp)
class RegisterView(FlaskView):
"""
Handle signup
"""
client_errors = scales.IntStat('4xx')
server_errors = scales.IntStat('5xx')
latency = scales.PmfStat('latency')
def __init__(self):
super(RegisterView, self).__init__()
scales.init(self, '/api/register')
def index(self):
return "Register", 200
class ProfileView(FlaskView):
"""
Shows personal profile
"""
client_errors = scales.IntStat('4xx')
server_errors = scales.IntStat('5xx')
latency = scales.PmfStat('latency')
def __init__(self):
super(ProfileView, self).__init__()
scales.init(self, '/api/profile')
self._users_backend = UsersBackend(config)
def index(self):
return "MyProfile", 200
def __init__(self, cluster_proxy):
log.debug("Starting metric capture")
self.stats_name = 'cassandra-{0}'.format(str(self._stats_counter))
Metrics._stats_counter += 1
self.stats = scales.collection(
self.stats_name,
scales.PmfStat('request_timer'),
scales.IntStat('connection_errors'),
scales.IntStat('write_timeouts'),
scales.IntStat('read_timeouts'),
scales.IntStat('unavailables'),
scales.IntStat('other_errors'),
scales.IntStat('retries'),
scales.IntStat('ignores'),
# gauges
scales.Stat('known_hosts',
lambda: len(cluster_proxy.metadata.all_hosts())),
scales.Stat(
'connected_to', lambda: len(
set(
chain.from_iterable(s._pools.keys() for s in
cluster_proxy.sessions)))),
scales.Stat(
'open_connections', lambda: sum(
sum(p.open_count for p in s._pools.values())
for s in cluster_proxy.sessions)))
# TODO, to be removed in 4.0
# /cassandra contains the metrics of the first cluster registered
if 'cassandra' not in scales._Stats.stats:
scales._Stats.stats['cassandra'] = scales._Stats.stats[
self.stats_name]
self.request_timer = self.stats.request_timer
self.connection_errors = self.stats.connection_errors
self.write_timeouts = self.stats.write_timeouts
self.read_timeouts = self.stats.read_timeouts
self.unavailables = self.stats.unavailables
self.other_errors = self.stats.other_errors
self.retries = self.stats.retries
self.ignores = self.stats.ignores
self.known_hosts = self.stats.known_hosts
self.connected_to = self.stats.connected_to
self.open_connections = self.stats.open_connections
class UsersView(FlaskView):
"""
Users list view
"""
client_errors = scales.IntStat('4xx')
server_errors = scales.IntStat('5xx')
latency = scales.PmfStat('latency')
def __init__(self):
super(UsersView, self).__init__()
scales.init(self, '/api/home')
def index(self):
return "UsersList", 200
def get(self, user_id):
return "UserID: %s" % user_id, 200
class MediasBackend(object):
"""
Controls media model and media data fetching from database
"""
requests = scales.IntStat('requests')
latency = scales.PmfStat('latency')
def __init__(self, config):
"""
:param config:
:type config: application.common.tools.Map
:return:
"""
scales.init(self, '/backend/users')
self._connection = None
self._db_name = config.mongo.db_name
replica_set = config.mongo.replicaset_name if 'replicaset_name' in config.mongo else None
self._connection = pymongo.MongoClient(
config.mongo.uri,
replicaSet=replica_set,
maxPoolSize=config.mongo.max_pool_size,
waitQueueMultiple=config.mongo.wait_queue_multiple,
waitQueueTimeoutMS=config.mongo.wait_queue_timeout_ms,
tz_aware=True)
self._db = self._connection[self._db_name]
self._medias = self._db[config.mongo.media_collection]
def fetch_medias(self, user_id, start=0, max_size=20):
"""
Fetch all medias of user matching user_id
:param user_id:
:type user_id: int
:param start: the document index from which we should fetch
:type start: int
:param max_size: the number of documents to fetch at a time
:return: list of medias, else None
:rtype: dict[str, T] | None
"""
raise NotImplementedError()
class MediasView(FlaskView):
"""
Medias list view
"""
client_errors = scales.IntStat('4xx')
server_errors = scales.IntStat('5xx')
latency = scales.PmfStat('latency')
def __init__(self):
super(MediasView, self).__init__()
scales.init(self, '/api/medias')
self._media_backend = MediasBackend(config)
self._image_processor = ImageProcessorService(config)
self._video_processor = VideoProcessorService(config)
def index(self):
return "MediaList", 200
def get(self, media_id):
"""
Return the media passed in parameter
:param media_id:
:type media_id: str
:return:
"""
return "MediaID: %s" % media_id, 200
def post(self, data):
"""
Handle new media
:param data: the media to save/handle
:return:
"""
raise NotImplementedError()
def delete(self, media_id):
"""
from cycloneerr import PrettyErrorHandler
from docopt import docopt
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from mqtt_client import MqttClient
from rdflib import Namespace
from standardservice.logsetup import log, verboseLogging
from twisted.internet import reactor
import rdf_over_http
ROOM = Namespace('http://projects.bigasterisk.com/room/')
STATS = scales.collection(
'/root',
scales.PmfStat('putRequests'),
scales.PmfStat('statement'),
scales.PmfStat('mqttPublish'),
)
devs = {
ROOM['kitchenLight']: {
'root': 'h801_skylight',
'hasWhite': True,
},
ROOM['kitchenCounterLight']: {
'root': 'h801_counter',
'hasWhite': True,
},
ROOM['livingLampShelf']: {
'root': 'sonoff_0/switch/sonoff_basic_relay/command',
from greplin import scales
from tornado import ioloop, web, httpserver, gen
STATS = scales.collection('/web', scales.PmfStat('latency'))
class TimedHandler(web.RequestHandler):
@gen.coroutine
def get(self):
with STATS.latency.time():
self.write('finished')
if __name__ == "__main__":
application = web.Application([
(r"/", TimedHandler),
])
server = httpserver.HTTPServer(application)
server.bind(8888)
server.start(0)
ioloop.IOLoop.instance().start()
from dateutil.relativedelta import relativedelta, FR
from rdflib import Namespace, Literal
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler
from twilight import isWithinTwilight
from standardservice.logsetup import log, verboseLogging
from rdfdoc import Doc
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
STATS = scales.collection(
'/root',
scales.PmfStat('update'),
)
class CycloneGraphEventsHandlerWithCors(CycloneGraphEventsHandler):
def flush(self):
self.set_header("Access-Control-Allow-Origin", "*")
return CycloneGraphEventsHandler.flush(self)
@STATS.update.time()
def update(masterGraph):
stmt = lambda s, p, o: masterGraph.patchObject(ROOM.environment, s, p, o)
now = datetime.datetime.now(tzlocal())
from rx.subjects import BehaviorSubject
from twisted.python.filepath import FilePath
from twisted.internet import reactor
from patchablegraph.patchsource import ReconnectingPatchSource
from rdfdb.rdflibpatch import patchQuads
from rdfdb.patch import Patch
log = logging.getLogger('fetch')
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
STATS = scales.collection(
'/web',
scales.PmfStat('combineGraph'),
)
def parseRdf(text: str, contentType: str):
g = Graph()
g.parse(StringInputSource(text), format={
'text/n3': 'n3',
}[contentType])
return g
class RemoteData(object):
def __init__(self, onChange: Callable[[], None]):
"""we won't fire onChange during init"""
self.onChange = onChange
import cyclone.web, cyclone.websocket
from rdflib import ConjunctiveGraph, URIRef, Graph
import twisted.internet.error
from rdfdb.file_vs_uri import correctToTopdirPrefix, fileForUri, uriFromFile, DirUriMap
from rdfdb.graphfile import GraphFile, PatchCb, GetSubgraph
from rdfdb.patch import Patch, ALLSTMTS
from rdfdb.rdflibpatch import patchQuads
from standardservice.scalessetup import gatherProcessStats
gatherProcessStats()
stats = scales.collection(
'/webServer',
scales.IntStat('clients'),
scales.IntStat('liveClients'),
scales.PmfStat('setAttr'),
)
graphStats = scales.collection(
'/graph',
scales.IntStat('statements'),
scales.RecentFpsStat('patchFps'),
)
fileStats = scales.collection(
'/file',
scales.IntStat('mappedGraphFiles'),
)
log = logging.getLogger('rdfdb')
class WebsocketDisconnect(ValueError):
import contextlib
import datetime
from greplin import scales
import logger
STATS = scales.collection('/goaldecider',
scales.PmfStat('duration')
)
def write(filename=None):
try:
if filename is None:
filename=logger.filepath.replace(".py",".json")
logger.dbg("Writing metrics to {}".format(filename))
scales.dumpStatsTo(filename)
except Exception as e:
logger.dbg("Exception in stats writing")
logger.log_exception(e)
@contextlib.contextmanager
def simple_timer(name):
start_time = datetime.datetime.now()
yield
delta = datetime.datetime.now() - start_time
logger.log("{} duration: {} s".format(name, delta.total_seconds()))
return False
from cyclone.httpclient import fetch
import cyclone
import logging, time, json, random, string, traceback
from logsetup import log, enableTwistedLog
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from export_to_influxdb import InfluxExporter
from tags import NfcDevice, FakeNfc, NfcError, AuthFailedError
ROOM = Namespace('http://projects.bigasterisk.com/room/')
ctx = ROOM['frontDoorWindowRfidCtx']
STATS = scales.collection(
'/root',
scales.PmfStat('cardReadPoll'),
scales.IntStat('newCardReads'),
)
class OutputPage(cyclone.web.RequestHandler):
def put(self):
arg = self.request.arguments
if arg.get('s') and arg.get('p'):
self._onQueryStringStatement(arg['s'][-1], arg['p'][-1],
self.request.body)
else:
self._onGraphBodyStatements(self.request.body,
self.request.headers)
post = put
from inference import infer, readRules
from actions import Actions, PutOutputsTable
from inputgraph import InputGraph
from escapeoutputstatements import unquoteOutputStatements
from standardservice.logsetup import log, verboseLogging
from patchablegraph import PatchableGraph, CycloneGraphHandler, CycloneGraphEventsHandler
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
NS = {'': ROOM, 'dev': DEV}
STATS = scales.collection(
'/web',
scales.PmfStat('graphChanged'),
scales.PmfStat('updateRules'),
)
def ntStatement(stmt: Tuple[Node, Node, Node]):
def compact(u):
if isinstance(u, URIRef) and u.startswith(ROOM):
return 'room:' + u[len(ROOM):]
return u.n3()
return '%s %s %s .' % (compact(stmt[0]), compact(stmt[1]), compact(
stmt[2]))
class Reasoning(object):
from patchablegraph.patchsource import ReconnectingPatchSource
from collector_config import config
#SourceUri = NewType('SourceUri', URIRef) # doesn't work
class SourceUri(URIRef):
pass
ROOM = Namespace("http://projects.bigasterisk.com/room/")
COLLECTOR = SourceUri(URIRef('http://bigasterisk.com/sse_collector/'))
STATS = scales.collection(
'/root',
scales.PmfStat('getState'),
scales.PmfStat('localStatementsPatch'),
scales.PmfStat('makeSyncPatch'),
scales.PmfStat('onPatch'),
scales.PmfStat('sendUpdatePatch'),
scales.PmfStat('replaceSourceStatements'),
)
class LocalStatements(object):
"""
functions that make statements originating from sse_collector itself
"""
def __init__(self, applyPatch: Callable[[URIRef, Patch], None]):
self.applyPatch = applyPatch
self._sourceState: Dict[SourceUri, URIRef] = {} # source: state URIRef
def poll(self):
milliC = open('/sys/class/thermal/thermal_zone0/temp').read().strip()
c = float(milliC) / 1000.
f = c * 1.8 + 32
return [
(self.uri, ROOM['temperatureF'], Literal(round(f, 3), datatype=XSD['decimal'])),
]
def watchPrefixes(self):
# these uris will become dynamic! see note on watchPrefixes
# about eliminating it.
return [(self.uri, ROOM['temperatureF']),
]
pixelStats = scales.collection('/rgbPixels',
scales.PmfStat('updateOutput'),
scales.PmfStat('currentColors'),
scales.PmfStat('poll'),
)
@register
class RgbPixels(DeviceType):
"""chain of ws2812 rgb pixels on pin GPIO18"""
deviceType = ROOM['RgbPixels']
def hostStateInit(self):
self.anim = RgbPixelsAnimation(self.graph, self.uri, self.updateOutput)
log.debug('%s maxIndex = %s', self.uri, self.anim.maxIndex())
self.neo = rpi_ws281x.Adafruit_NeoPixel(self.anim.maxIndex() + 1, pin=18)
self.neo.begin()
import os, contextlib
try:
from rdflib.Graph import Graph
except ImportError:
from rdflib import Graph
from rdflib import Namespace
from rdflib.parser import StringInputSource
from FuXi.Rete.Util import generateTokenSet
from FuXi.Rete import ReteNetwork
from FuXi.Rete.RuleStore import N3RuleStore
from greplin import scales
STATS = scales.collection('/web', scales.PmfStat('readRules'))
from escapeoutputstatements import escapeOutputStatements
ROOM = Namespace("http://projects.bigasterisk.com/room/")
def _loadAndEscape(ruleStore, n3, outputPatterns):
ruleGraph = Graph(ruleStore)
# Can't escapeOutputStatements in the ruleStore since it
# doesn't support removals. Can't copy plainGraph into
# ruleGraph since something went wrong with traversing the
# triples inside quoted graphs, and I lose all the bodies
# of my rules. This serialize/parse version is very slow (400ms),
# but it only runs when the file changes.
plainGraph = Graph()
class RequestMachine:
"""
Don't use IDLE as the primary state identifier when
trying to find out if the first request has been sent.
Use stats instead. One may put the machine in idle
after stopping, and it would still use the cached
limits.
"""
# state averages from 75, 95, 98, 99, 999,
# min, max, median, mean, and stddev
latency = scales.PmfStat('latency')
# timing for 1/5/15 minute averages
latency_window = meter.MeterStat('latency_window')
console = logging.StreamHandler()
format_str = '%(asctime)s\t%(levelname)s -- %(processName)s %(filename)s:%(lineno)s -- %(message)s'
console.setFormatter(logging.Formatter(format_str))
def __init__(self, session, request_spec):
self.log = logging.getLogger()
self.log.setLevel(logging.DEBUG)
self.log.addHandler(self.console)
self.uuid = uuid4() # unit identifier
scales.init(self, '/request-machine/%s' % self.uuid)
# self.register_child_stat('/request-machine/%s' % self.uuid)
self.session = session # request http session
self.request_spec = request_spec # request spec with spec of call
self.timings = RequestTimings(request_spec)
self.state = RequestMachineStates.Idle # current state
self._seed_session()
def _seed_temporal_data(self):
# add temporal based headers (time limits, etag, etc...)
if self.timings.etag is not None:
self.session.headers.update({
self.request_spec.headers.ifnonmatch:
RequestSpec.wrap(self.timings.etag)
})
def _seed_session(self):
# add all headers (token, auth, etc...)
self.session.headers.update(self.request_spec.send_headers)
self._seed_temporal_data()
def limit_has_reset(self):
"""
Call this if the remaining limit is 0.
:return:
"""
time_stamp = self.timings.last_request_timestamp
ttr = float(self.timings.time_to_reset)
now = timeutils.milliseconds()
# last time called was before or at the reset and
# time now is at or beyond the reset allow
if time_stamp <= ttr and now >= ttr:
print "ts less than ttr and now greater than ttr"
return True
else:
return False
def limit_reached(self):
return int(self.timings.rate_limit_remaining) <= 0
def past_interval(self):
now = timeutils.milliseconds()
last = self.timings.last_request_timestamp
estimated_interval_ts = last + float(self.timings.interval)
result = now > estimated_interval_ts
if not result:
self.log.debug("now (%d) not yet past the interval (%d)" %
(now, estimated_interval_ts))
return result
def past_reset_window(self):
now = timeutils.milliseconds()
reset_window = float(self.timings.time_to_reset)
result = now >= reset_window
if not result:
self.log.debug("request not yet past reset window")
return result
def request_made_since_reset_window(self):
timestamp = float(self.timings.last_request_timestamp)
reset_window = float(self.timings.time_to_reset)
result = timestamp >= reset_window
if result:
self.log.warn("request was made since reset window")
return result
def edge_case(self):
result = self.limit_reached \
and self.past_reset_window() \
and self.request_made_since_reset_window()
if result:
self.log.error("RequestMachine edge case detected")
return result
def can_request(self):
if self.edge_case():
return False, RequestMachineStates.EdgeCaseError
if self.past_interval():
# if the limit has been reached
if not self.limit_reached():
return True
else: # limit reached
if self.past_reset_window(): # limit should be reset
return True
return False, RequestMachineStates.WaitingForReset
return False, RequestMachineStates.WaitingForIntervalToPass
def _update(self, resp):
self.timings.update(resp)
self._seed_temporal_data()
def _idle_state(self):
self.state = RequestMachineStates.Idle
def _error_state(self):
self.state = RequestMachineStates.Error
def _processing_state(self):
self.state = RequestMachineStates.Processing
def reset_state(self):
self._idle_state()
def has_error_state(self):
return self.state == RequestMachineStates.Error \
or self.state == RequestMachineStates.EdgeCaseError
def get(self):
if self.has_error_state():
print "Cannot proceed, machine is in ERROR state. Please see the logs."
return None
can_request, proposed_error_state = self.can_request()
if can_request:
self._processing_state()
with self.latency.time():
self.latency_window.mark()
resp = self.session.get(self.request_spec.uri)
if resp.status_code == 304:
self.state = RequestMachineStates.WaitingForModifiedContent
# even with 304s, the server will still attempt
# to send limits back.
self._update(resp)
return resp
elif resp.status_code == 200:
self._update(resp)
self._idle_state()
return resp
else:
print "Request failed!"
print resp.status_code
print resp.content
self._error_state()
# even with 304s, the server will still attempt
# to send limits back.
self._update(resp)
self._idle_state()
return resp
else:
self.state = proposed_error_state
return None
class BaseService(ErrorHandlerMixin):
"""
A base service. Note this is a class which composites a greenlet. This is in
contrast with the Actor class which subclasses greenlet and may be more
performant and lightweight. This class is still aimed for performance but
has many management routines and actions.
This base class has many of the same attributes as the underlying gevent
greenlet but note they are not the same. Effort was made to make semantic
usage easier. Do not assume anything similar with gevent.
"""
# state averages from 75, 95, 98, 99, 999,
# min, max, median, mean, and stddev
latency = scales.PmfStat('latency')
# timing for 1/5/15 minute averages
latency_window = meter.MeterStat('latency_window')
# console = logging.StreamHandler()
# format_str = '%(asctime)s\t%(levelname)s -- %(processName)s %(filename)s:%(lineno)s -- %(message)s'
# console.setFormatter(logging.Formatter(format_str))
def __init__(self,
name="base-service",
directory_proxy=None,
parent_logger=None,
enable_service_recovery=False):
"""
uuid - a uuid4 value for the service
alias - a colloquial alias
unique_name - a name which includes an easier to remember alias with the uuid
:param name:
:param directory_proxy:
:param parent_logger:
:return:
"""
ErrorHandlerMixin.__init__(self)
# time indexes
self.time_starting_index = None # time index when service was 'starting'
self.time_started_index = None # time index when service was started
self.uuid = uuid4() # unique uuid
self.alias = name # name, may collide
self.unique_name = '%s/%s' % (
self.alias, self.uuid
) # a unique name for this service, will always be unique
scales.init(self, self.unique_name)
if parent_logger is None: # no parent, use fq name
self.lineage = "%s" % self.unique_name
else:
parent_name = parent_logger._context["name"]
self.lineage = "%s/%s" % (parent_name, self.unique_name)
self.log = Logger.get_logger(self.lineage)
self.greenlet = None
self._service_state = None
self.set_state(BaseStates.Idle)
# directory service proxy
self.directory_proxy = directory_proxy
# service recovery option
self.enable_service_recovery = enable_service_recovery
self.log.debug("Initialized.")
def event_loop(self):
"""
Override this
"""
# while True:
# with self.latency.time():
# self.latency_window.mark()
# # do some work here
# # sleep or idle
while self.should_loop():
gevent.idle()
def should_loop(self):
# return not self.ready() or not self.has_stopped()
# chose to signal on if the service has started rather than started or idle
# which would be a confusing state which a loop would be allowed execution.
# In an effort to narrow down to one state I choose `started`.
return self.has_started()
def register_child_stat(self, name):
scales.initChild(self, name)
def register(self):
"""
Once a service has been actively managed, it is populated
by the service manager with addtional information or services.
This method registers that data.
Typically you will find a database and queue proxy to be set
here.
:return:
"""
pass
def did_service_timeout(self):
"""
A timeout occurs when a service remains in the starting phase.
:return:
"""
timeout = self.get_directory_service_proxy().get_service_meta(
self.alias).start_timeout
if timeout > 0 and self.is_starting():
delay = self.get_directory_service_proxy().get_service_meta(
self.alias).delay
timeout = self.get_directory_service_proxy().get_service_meta(
self.alias).start_timeout
# calculate by adding the delay that will be introduced
# with the timeout value, and this time index will be the maximum time
# which with to wait for the service to start
expected_timeout = delay + timeout
# now determine if the time that has passed has met or exceeded the
# calculated timeout index from above
return self.start_time_delta() >= expected_timeout
return False
def start_time_delta(self):
"""
Returns the time difference between now and when the service entered into the
`Starting` state. This is how long since the service first indexed as being
in been in the `Starting` position. Note that this will always calculate,
and should be used as a utility method.
:return:
"""
now = time.time()
return now - self.time_starting_index
def pre_handle_error(self, exception):
"""
Triggered before error handlers have run.
This may be overriden for additional functionality.
:param exception: An optional exception.
:return:
"""
pass
def post_handle_error(self, exception=None):
"""
Triggered after error handlers have run.
This may be overriden for additional functionality.
:param exception: An optional exception.
:return:
"""
pass
def start_event_loop(self):
self.log.debug("service starting event loop...")
self.set_state(BaseStates.Started)
self.time_started_index = time.time()
# self.event_loop()
try:
self.event_loop()
except Exception as ex:
self.handle_error(ex)
self.post_handle_error(
ex
) # a built-in method which signals the post event of handling errors
def start(self, meta=None):
if self.get_state(
) is not BaseStates.Idle: # or not self.enable_service_recovery:
self.log.error(
"could not start service as it is not in an idle state, current state: [%s]"
% self.get_state(),
state=self.get_state())
raise ServiceNotIdleException()
if meta is not None:
# delays are only allowed on first start, after this a function must be supplied
delay = 0
msg = "service starting with delay..."
if meta.starts == 0 and meta.delay > 0:
self.log.debug("service starting with delay...", delay=delay)
delay = meta.delay
else:
self.log.debug("service starting with delay function...",
delay=delay,
delay_func=meta.retry_delay_fx.__name__)
delay = meta.next_delay()
self.time_starting_index = time.time()
self.greenlet = gevent.spawn_later(delay, self.start_event_loop)
self.set_state(
BaseStates.Starting
) # TODO: time how long services take to actually start
else: # no meta, assume base service
self.time_starting_index = time.time()
self.set_state(
BaseStates.Starting
) # TODO: time how long services take to actually start
self.log.debug("service starting...")
self.greenlet = gevent.spawn(self.start_event_loop)
#
# --- NO CODE BEYOND THE IF/ELSE BLOCK ABOVE ---
#
# this method works asynchronously and thus code here will execute immediately
return self.greenlet
def stop(self):
self.log.info("Stopping...")
self.set_state(BaseStates.Stopping)
if self.greenlet is not None:
gevent.kill(self.greenlet)
else:
self.log.warn("service [%s] was found already stopped." %
self.lineage)
self.set_state(BaseStates.Stopped)
return self.greenlet
def get_greenlet(self):
return self.greenlet
def ready(self):
"""
Only return true when state is Idle which means
it is ready to accept specs to start. Start
should assume the service is already acting on
a user's request for 'work'. This is an boolean
alias to check if in Idle state.
:return:
"""
if self.get_state() is BaseStates.Idle:
return True
return False
def has_started(self):
return self.get_state() is BaseStates.Started
def has_stopped(self):
return self.get_state() is BaseStates.Stopped
def is_starting(self):
return self.get_state() is BaseStates.Starting
def is_stopping(self):
return self.get_state() is BaseStates.Stopping
def has_state(self):
return self.get_state() is not None
def is_zombie(self):
"""
If there is no state such as Idle, Start, or Stop then this service
is a zombie.
:return:
"""
return not self.has_state() or not self.greenlet.started
def is_truly_dead(self):
"""
Not a zombie. Stopped is not dead. Zombie is not dead (It's alive stupid!).
This method checks if the greenlet was not successful and has logged an
exception.
Note: below I don't check for service.idle() because that is a sign that
a service is alive and about to be started. Instead I focus on the greenlet
which if was created will still have values such as exception and successful
registered.
:return:
"""
return self.greenlet is not None \
and not self.greenlet.successful() \
and self.greenlet.exception is not None
def idle(self):
"""
Resets a service which you expect to restart.
Prior to started a service it must be set to idle.
:return:
"""
if self.get_state() is BaseStates.Stopped or self.is_zombie():
self.set_state(BaseStates.Idle)
else:
raise IdleActionException()
def get_state(self):
return self._service_state
def set_state(self, state):
"""
Should be used only by base class and inheritors
"""
self.log.debug("Service state is being set to: [%s]" % state)
self._service_state = state
def set_directory_service_proxy(self, directory_proxy):
self.directory_proxy = directory_proxy
def get_directory_service_proxy(self):
return self.directory_proxy
from configparser import ConfigParser
from io import StringIO
try:
import _thread as thread
except ImportError:
import thread
log = logging.getLogger(__name__)
stats = scales.collection(
'/celery',
scales.IntStat('started'),
scales.IntStat('succeeded'),
scales.IntStat('failed'),
scales.IntStat('retried'),
scales.PmfStat('queuetime'),
scales.PmfStat('runtime'),
)
stats_queue = type('Stats:queues', (object, ), {})()
scales._Stats.initChild(stats_queue, 'queues', '', stats)
def task_handler(fn):
@wraps(fn)
def wrapper(self, event):
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
return fn(self, event, task)
return wrapper
import time, logging, math, os, sys, socket
from influxdb import InfluxDBClient
from rdflib import Namespace
from twisted.internet import task
from greplin import scales
log = logging.getLogger()
ROOM = Namespace('http://projects.bigasterisk.com/room/')
stats = scales.collection(
'/export_to_influxdb',
scales.PmfStat('exportToInflux'),
)
class RetentionPolicies(object):
def __init__(self, influx):
self.influx = influx
self.createdPolicies = set() # days
def getCreatedPolicy(self, days):
name = 'ret_%d_day' % days
if days not in self.createdPolicies:
self.influx.create_retention_policy(name,
duration='%dd' % days,
replication='1')
self.createdPolicies.add(days)
return name
class InfluxExporter(object):
def __init__(self, configGraph, influxHost='bang5'):
class BaseService:
"""
A base service. Note this is a class which
composites a greenlet. This is in contrast
with the Actor class which subclasses greenlet
for performance and being lightweight. This
class is still aimed for performance but has
many management routines and actions which are
necessary.
"""
# state averages from 75, 95, 98, 99, 999,
# min, max, median, mean, and stddev
latency = scales.PmfStat('latency')
# timing for 1/5/15 minute averages
latency_window = meter.MeterStat('latency_window')
# console = logging.StreamHandler()
# format_str = '%(asctime)s\t%(levelname)s -- %(processName)s %(filename)s:%(lineno)s -- %(message)s'
# console.setFormatter(logging.Formatter(format_str))
def event_loop(self):
"""
Override
"""
# while True:
# with self.latency.time():
# self.latency_window.mark()
# # do some work here
# # sleep or keep going
pass
def __init__(self, name="base-service", directory_proxy=None):
self.uuid = uuid4()
self.unique_name = '/%s/%s' % (name, self.uuid)
scales.init(self, self.unique_name)
# self.log = logging.getLogger(self.unique_name)
# self.log.setLevel(logging.DEBUG)
# self.log.addHandler(self.console)
self.log = Logger.get_logger(self.unique_name)
print "%s - Init" % name
self.name = name
self.greenlet = None
self._service_state = BaseStates.Idle
# directory service proxy
self._directory_proxy = directory_proxy
def register_child_stat(self, name):
scales.initChild(self, name)
def start(self):
# print "%s - Starting..." % self.name
self.greenlet = gevent.spawn(self.event_loop)
self._service_state = BaseStates.Started
return self.greenlet
def stop(self):
# print "%s - Stopping..." % self.name
gevent.kill(self.greenlet)
self._service_state = BaseStates.Stopped
return self.greenlet
def get_greenlet(self):
return self.greenlet
def ready(self):
"""
Only return true when state is Idle which means
it is ready to accept specs to start. Start
should assume the service is already acting on
a user's request for 'work'. This is an boolean
alias to check if in Idle state.
:return:
"""
if self.get_state() is BaseStates.Idle:
return True
return False
def idle(self):
if self.get_state() is BaseStates.Stopped:
self._service_state = BaseStates.Idle
else:
raise IdleActionException()
def get_state(self):
return self._service_state
def set_state(self, state):
"""
Should be used only by base class and inheritors
"""
self._service_state = state
def set_directory_service_proxy(self, directory_proxy):
self._directory_proxy = directory_proxy
def get_directory_service_proxy(self):
return self._directory_proxy
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from influxdb import InfluxDBClient
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks
import cyclone.web
from rdflib import Namespace, Literal
from standardservice.logsetup import log, verboseLogging
from patchablegraph import PatchableGraph, CycloneGraphHandler, CycloneGraphEventsHandler
from private_config import deviceIp, cloudId, installId, macId, periodSec
ROOM = Namespace("http://projects.bigasterisk.com/room/")
STATS = scales.collection('/root',
scales.PmfStat('poll'),
)
authPlain = cloudId + ':' + installId
auth = binascii.b2a_base64(authPlain.encode('ascii')).strip(b'=\n')
class Poller(object):
def __init__(self, influx, graph):
self.influx = influx
self.graph = graph
reactor.callLater(0, self.poll)
@STATS.poll.time()
@inlineCallbacks
def poll(self):
ret = None
class UsersBackend(object):
"""
Controls user model and user data fetching from database
"""
requests = scales.IntStat('requests')
latency = scales.PmfStat('latency')
def __init__(self, config):
"""
:param config:
:type config: application.common.tools.Map
:return:
"""
scales.init(self, '/backend/users')
self._connection = None
self._db_name = config.mongo.db_name
replica_set = config.mongo.replicaset_name if 'replicaset_name' in config.mongo else None
self._connection = pymongo.MongoClient(
config.mongo.uri,
replicaSet=replica_set,
maxPoolSize=config.mongo.max_pool_size,
waitQueueMultiple=config.mongo.wait_queue_multiple,
waitQueueTimeoutMS=config.mongo.wait_queue_timeout_ms,
tz_aware=True)
self._db = self._connection[self._db_name]
self._users = self._db[config.mongo.user_collection]
def register(self, username, email, password):
"""
Register the new user
:param username:
:type username: str
:param email:
:type email: str
:param password:
:type password: str
:return:
:rtype: None
:raise DuplicateUser: the username or email already exists
"""
raise NotImplementedError()
def authenticate(self, username, password):
"""
Validate login/password of a user
:param username:
:type username: str
:param password:
:type password: str
:return: the user id if authenticated, else None
:rtype: bool | None
"""
raise NotImplementedError()
def fetch_user(self, user_id):
"""
Fetch user_id informations
:param user_id:
:type user_id: int
:return: informations about the user if exists, else None
:rtype: dict[str, T] | None
"""
raise NotImplementedError()
from rdflib import URIRef, Namespace, Literal, RDF, RDFS, XSD, ConjunctiveGraph
from twisted.internet import reactor, task
import cyclone.web
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler
from standardservice.logsetup import log, verboseLogging
DEV = Namespace("http://projects.bigasterisk.com/device/")
ROOM = Namespace("http://projects.bigasterisk.com/room/")
ctx = DEV['dhcp']
STATS = scales.collection(
'/root',
scales.PmfStat('readLeases'),
scales.IntStat('filesDidntChange'),
)
def timeLiteral(dt):
return Literal(dt.replace(tzinfo=tzlocal()).isoformat(),
datatype=XSD.dateTime)
def macUri(macAddress: str) -> URIRef:
return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
class Poller:
def __init__(self, graph):
