def setupService(self):
self.log(log.info, u'Bootstrapping')
self.settings = self.parent.settings
# Optionally register subsystem component as child service
for subsystem in self.subsystems:
if hasattr(self, subsystem):
subsystem_service = getattr(self, subsystem)
if isinstance(subsystem_service, Service):
log.info('Registering subsystem component "{subsystem}" as service', subsystem=subsystem)
self.registerService(subsystem_service)
# Configure metrics to be collected each X seconds
metrics_interval = int(self.channel.get('metrics_logger_interval', 60))
self.metrics = Bunch(tx_count=0, starttime=time.time(), interval=metrics_interval)
subscriptions = read_list(self.channel.mqtt_topics)
self.mqtt_service = MqttAdapter(
name = u'mqtt-' + self.channel.realm,
broker_host = self.settings.mqtt.host,
broker_port = int(self.settings.mqtt.port),
broker_username = self.settings.mqtt.username,
broker_password = self.settings.mqtt.password,
callback = self.mqtt_receive,
subscriptions = subscriptions)
self.registerService(self.mqtt_service)
self.influx = InfluxDBAdapter(settings = self.settings.influxdb)
# Perform MQTT message processing using a different thread pool
self.threadpool = ThreadPool()
self.thimble = Thimble(reactor, self.threadpool, self, ["process_message"])
def forward(self, bucket):
"""
Receive data bucket from source, run through
transformation machinery and emit to target.
"""
# 1. Map/transform topology address information
if 'transform' in self.channel:
for entrypoint in read_list(self.channel.transform):
try:
transformer = KotoriBootloader.load_entrypoint(entrypoint)
bucket.tdata.update(transformer(bucket.tdata))
except ImportError as ex:
log.error('ImportError "{message}" when loading entrypoint "{entrypoint}"',
entrypoint=entrypoint, message=ex)
# MQTT doesn't prefer leading forward slashes with topic names, let's get rid of them
target_uri_tpl = self.target_uri.path.lstrip('/')
# Compute target bus topic from url matches
target_uri = target_uri_tpl.format(**bucket.tdata)
# Enrich bucket by putting source and target addresses into it
bucket.address = Bunch(source=self.source_address, target=self.target_address)
# 2. Reporting
bucket_logging = Bunch(bucket)
if 'body' in bucket_logging and len(bucket_logging.body) > 100:
bucket_logging.body = bucket_logging.body[:100] + ' [...]'
log.debug('Forwarding bucket to {target} with bucket={bucket}. Effective target uri is {target_uri}',
target=self.channel.target, target_uri=target_uri, bucket=dict(bucket_logging))
# 3. Adapt, serialize and emit appropriately
return self.target_service.emit(target_uri, bucket)
def lst(cls, settings):
from kotori.vendor.lst.application import lst_boot
debug = settings.options.debug
if settings.options.debug_vendor and 'lst' in read_list(
settings.options.debug_vendor):
changeLogLevel('kotori.vendor.lst', LogLevel.debug)
lst_boot(settings)
def lst_channels(config):
channel_labels = read_list(config['lst']['channels'])
channel_infos = []
for channel_label in channel_labels:
channel_settings = config[channel_label]
channel_info = get_channel_info(channel_label, channel_settings)
channel_infos.append(channel_info)
print tabulate(channel_infos, headers='keys')
def parse(self, address):
m = self.address_pattern.match(address)
match = m.groupdict()
if match:
self.raw_uri = match['uri']
self.raw_predicates = match['predicates']
self.parsed_uri = urlparse(self.raw_uri)
self.uri = bunchify(self.parsed_uri._asdict())
if self.raw_predicates:
self.predicates = read_list(self.raw_predicates)
def parse(self, address):
m = self.address_pattern.match(address)
match = m.groupdict()
if match:
self.raw_uri = match['uri']
self.raw_predicates = match['predicates']
self.parsed_uri = urlparse(self.raw_uri)
self.uri = bunchify(self.parsed_uri._asdict())
if self.raw_predicates:
self.predicates = read_list(self.raw_predicates)
def lst_boot(config, debug=False):
# TODO: refactor towards OO; e.g. BinaryMessageApplicationFactory
wamp_uri = unicode(config.wamp.uri)
# activate/mount multiple "lst" applications
for channel_name in read_list(config['lst']['channels']):
log.info('Starting "lst" channel "{}"'.format(channel_name))
channel = config[channel_name]
config = deepcopy(config)
config['_active_'] = channel
WampApplication(url=wamp_uri, realm=u'lst', session_class=UdpSession, config=config).make()
WampApplication(url=wamp_uri, realm=u'lst', session_class=StorageSession, config=config).make()
def lst_boot(config, debug=False):
# TODO: refactor towards OO; e.g. BinaryMessageApplicationFactory
wamp_uri = str(config.wamp.uri)
# activate/mount multiple "lst" applications
for channel_name in read_list(config['lst']['channels']):
log.info('Starting "lst" channel "{}"'.format(channel_name))
channel = config[channel_name]
config = deepcopy(config)
config['_active_'] = channel
WampApplication(url=wamp_uri, realm=u'lst', session_class=UdpSession, config=config).make()
WampApplication(url=wamp_uri, realm=u'lst', session_class=StorageSession, config=config).make()
def __init__(self,
name=None,
application_settings=None,
global_settings=None):
RootService.__init__(self, settings=global_settings)
# Compute name for Twisted service
self.name = u'app-{name}'.format(name=name)
# Make channel object from application settings configuration object
self.channel = Bunch(**application_settings)
# Main application components
for service_reference in read_list(application_settings.services):
# Load service component
try:
service_factory = KotoriBootloader.load_entrypoint(
service_reference)
except Exception as ex:
log.failure(
'Error loading composite service component "{service_name}" into "{app_name}":\n{log_failure}"',
service_name=service_reference,
app_name=name)
continue
# Load data processing strategy and graphing components
# TODO: Review whether this should be per-service or not
# TODO: Introduce strict/non-strict handling
strategy_factory = KotoriBootloader.load_entrypoint(
application_settings.strategy, onerror='ignore')
graphing_factory = KotoriBootloader.load_entrypoint(
application_settings.graphing, onerror='ignore')
# Create application service object composed of subsystem components
# TODO: Bundle all arguments into yet another wrapper object for an universal object factory
service = service_factory(
channel=self.channel,
strategy=strategy_factory(settings=global_settings),
graphing=graphing_factory(settings=global_settings,
channel=self.channel))
# Register service component with its container
self.registerService(service)
def __init__(self, name=None, application_settings=None, global_settings=None):
RootService.__init__(self, settings=global_settings)
# Compute name for Twisted service
self.name = u'app-{name}'.format(name=name)
# Make channel object from application settings configuration object
self.channel = Bunch(**application_settings)
# Main application components
for service_reference in read_list(application_settings.services):
# Load service component
try:
service_factory = KotoriBootloader.load_entrypoint(service_reference)
except Exception as ex:
log.failure('Error loading composite service component "{service_name}" into "{app_name}":\n{log_failure}"',
service_name=service_reference, app_name=name)
continue
# Load data processing strategy and graphing components
# TODO: Review whether this should be per-service or not
# TODO: Introduce strict/non-strict handling
strategy_factory = KotoriBootloader.load_entrypoint(application_settings.strategy, onerror='ignore')
graphing_factory = KotoriBootloader.load_entrypoint(application_settings.graphing, onerror='ignore')
# Create application service object composed of subsystem components
# TODO: Bundle all arguments into yet another wrapper object for an universal object factory
service = service_factory(
channel = self.channel,
strategy = strategy_factory(settings=global_settings),
graphing = graphing_factory(settings=global_settings, channel=self.channel)
)
# Register service component with its container
self.registerService(service)
def emit(self, uri, bucket):
"""
Adapt, serialize and emit data bucket to target service.
"""
log.debug('Emitting to target scheme {scheme}', scheme=self.scheme)
if self.scheme == 'mqtt':
# Publish JSON payload to MQTT bus
topic = uri
payload = bucket.json
# TODO: Use threads.deferToThread here?
return self.downstream.publish(topic, payload)
elif self.scheme == 'influxdb':
# InfluxDB query wrapper using expression derived from transformation data
dfq = DataFrameQuery(settings=self.settings, bucket=bucket)
# Perform query and obtain results as pandas DataFrame
df = dfq.query()
# Announce routing information via http response headers
bucket.request.setHeader('Target-Database', bucket.tdata.database)
bucket.request.setHeader('Target-Expression',
bucket.tdata.expression)
bucket.request.setHeader('Target-Address-Scheme', self.scheme)
bucket.request.setHeader('Target-Address-Uri', uri)
# Database result is empty, send appropriate response
if df is None or df.empty:
return self.response_no_results(bucket)
# DataFrame manipulation
# Drop some fields from DataFrame as requested
if 'exclude' in bucket.tdata and bucket.tdata.exclude:
drop_fields = read_list(bucket.tdata.exclude,
empty_elements=False)
try:
df.drop(drop_fields, axis=1, inplace=True)
except ValueError as ex:
log.error(last_error_and_traceback())
error_message = u'Error: {type} {message}'.format(
type=type(ex), message=ex)
return bucket.request.error_response(
bucket, error_message=error_message)
# Use only specified fields from DataFrame as requested
if 'include' in bucket.tdata and bucket.tdata.include:
use_fields = read_list(bucket.tdata.include,
empty_elements=False)
use_fields.insert(0, 'time')
try:
df = df.filter(use_fields, axis=1)
except ValueError as ex:
log.error(last_error_and_traceback())
error_message = u'Error: {type} {message}'.format(
type=type(ex), message=ex)
return bucket.request.error_response(
bucket, error_message=error_message)
# Propagate non-null values forward or backward.
# With time series data, using pad/ffill is extremely common so that the “last known value” is available at every time point.
# http://pandas.pydata.org/pandas-docs/stable/missing_data.html#filling-missing-values-fillna
if 'pad' in bucket.tdata and asbool(bucket.tdata.pad):
df.fillna(method='pad', inplace=True)
if 'backfill' in bucket.tdata and asbool(bucket.tdata.backfill):
df.fillna(method='backfill', inplace=True)
if 'interpolate' in bucket.tdata and asbool(
bucket.tdata.interpolate):
# Performs linear interpolation at missing datapoints,
# otherwise matplotlib would not plot the sparse data frame.
# http://pandas.pydata.org/pandas-docs/stable/missing_data.html#interpolation
df.interpolate(inplace=True)
if 'sorted' in bucket.tdata and asbool(bucket.tdata.sorted):
# http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort.html
df.sort(axis='columns', inplace=True)
# Compute http response from DataFrame, taking designated output format into account
response = HttpDataFrameResponse(bucket, dataframe=df)
# Synchronous, the worker-threading is already on the HTTP layer
return response.render()
# Asynchronous: Perform computation in separate thread
d = threads.deferToThread(response.render)
d.addErrback(handleFailure, bucket.request)
d.addBoth(bucket.request.write)
d.addBoth(lambda _: bucket.request.finish())
return server.NOT_DONE_YET
else:
message = 'No target/downstream dispatcher for scheme {scheme}'.format(
scheme=self.scheme)
log.error(message)
raise KeyError(message)
def build_firmware(self, bucket=None, options=None, data=None):
# FirmwareBuilder machinery
fwbuilder = FirmwareBuilder(
repo_url=options.url, repo_branch=options.ref, update_submodules=options.update_submodules,
workingdir=options.path,
architecture = options.get('architecture'), esp_root=options.esp_root)
# Run build process with capturing its output
with fwbuilder.capture() as result:
# Acquire source code from git repository
fwbuilder.acquire_source()
# Patch files with individual parameters
"""
data = {
'BOARD_TAG': 'mega2560',
'BOARD_SUB': 'atmega2560',
'HE_USER': '******',
'HE_SITE': 'Buxtehude',
'HE_HIVE': 'Raeuberhoehle',
'GPRSBEE_APN': 'internet.altes-land.de',
'GPRSBEE_VCC': 23,
}
"""
patch_files = read_list(self.channel.patch_files)
fwbuilder.patch_files(patch_files, data)
# Run build process
fwbuilder.run_build(makefile=options.makefile)
artefact = fwbuilder.make_artefact()
if result.success:
# Compute extended firmware name suffix from transformation dict
# TODO: Route this information to "Artefact" object
"""
fwparams = ''
if data:
data = data.copy()
del data['slot']
dlist = []
for key, value in data.iteritems():
dlist.append(u'{key}={value}'.format(**locals()))
fwparams = u'-' + u','.join(dlist)
"""
filename = '{realm}_{name}.{suffix}'.format(
realm=self.channel.realm, name=artefact.fullname, suffix=options.suffix)
log.info(u'Build succeeded, filename: {filename}, artefact: {artefact}', filename=filename, artefact=artefact)
bucket.request.setHeader('Content-Type', 'application/octet-stream')
bucket.request.setHeader('Content-Disposition', 'attachment; filename={filename}'.format(filename=filename))
# "options.suffix" may be "hex" or "elf"
payload = artefact.get_binary(options.suffix)
return payload
else:
error_message = fwbuilder.error_message()
log.error(u'Build failed\n{message}', message=error_message)
bucket.request.setResponseCode(http.BAD_REQUEST)
bucket.request.setHeader('Content-Type', 'text/plain; charset=utf-8')
return error_message.encode('utf-8')
def lst(cls, settings):
from kotori.vendor.lst.application import lst_boot
debug = settings.options.debug
if settings.options.debug_vendor and 'lst' in read_list(settings.options.debug_vendor):
changeLogLevel('kotori.vendor.lst', LogLevel.debug)
lst_boot(settings)
def emit(self, uri, bucket):
"""
Adapt, serialize and emit data bucket to target service.
"""
log.debug('Emitting to target scheme {scheme}', scheme=self.scheme)
if self.scheme == 'mqtt':
# Publish JSON payload to MQTT bus
topic = uri
payload = bucket.json
# TODO: Use threads.deferToThread here?
return self.downstream.publish(topic, payload)
elif self.scheme == 'influxdb':
# InfluxDB query wrapper using expression derived from transformation data
dfq = DataFrameQuery(settings=self.settings, bucket=bucket)
# Perform query and obtain results as pandas DataFrame
df = dfq.query()
# Announce routing information via http response headers
bucket.request.setHeader('Target-Database', bucket.tdata.database)
bucket.request.setHeader('Target-Expression', bucket.tdata.expression)
bucket.request.setHeader('Target-Address-Scheme', self.scheme)
bucket.request.setHeader('Target-Address-Uri', uri)
# Database result is empty, send appropriate response
if df is None or df.empty:
return self.response_no_results(bucket)
# DataFrame manipulation
# Drop some fields from DataFrame as requested
if 'exclude' in bucket.tdata and bucket.tdata.exclude:
drop_fields = read_list(bucket.tdata.exclude, empty_elements=False)
try:
df.drop(drop_fields, axis=1, inplace=True)
except ValueError as ex:
log.error(last_error_and_traceback())
error_message = u'Error: {type} {message}'.format(type=type(ex), message=ex)
return bucket.request.error_response(bucket, error_message=error_message)
# Use only specified fields from DataFrame as requested
if 'include' in bucket.tdata and bucket.tdata.include:
use_fields = read_list(bucket.tdata.include, empty_elements=False)
use_fields.insert(0, 'time')
try:
df = df.filter(use_fields, axis=1)
except ValueError as ex:
log.error(last_error_and_traceback())
error_message = u'Error: {type} {message}'.format(type=type(ex), message=ex)
return bucket.request.error_response(bucket, error_message=error_message)
# Propagate non-null values forward or backward.
# With time series data, using pad/ffill is extremely common so that the “last known value” is available at every time point.
# http://pandas.pydata.org/pandas-docs/stable/missing_data.html#filling-missing-values-fillna
if 'pad' in bucket.tdata and asbool(bucket.tdata.pad):
df.fillna(method='pad', inplace=True)
if 'backfill' in bucket.tdata and asbool(bucket.tdata.backfill):
df.fillna(method='backfill', inplace=True)
if 'interpolate' in bucket.tdata and asbool(bucket.tdata.interpolate):
# Performs linear interpolation at missing datapoints,
# otherwise matplotlib would not plot the sparse data frame.
# http://pandas.pydata.org/pandas-docs/stable/missing_data.html#interpolation
df.interpolate(inplace=True)
if 'sorted' in bucket.tdata and asbool(bucket.tdata.sorted):
# http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort.html
df.sort(axis='columns', inplace=True)
# Compute http response from DataFrame, taking designated output format into account
response = HttpDataFrameResponse(bucket, dataframe=df)
# Synchronous, the worker-threading is already on the HTTP layer
return response.render()
# Asynchronous: Perform computation in separate thread
d = threads.deferToThread(response.render)
d.addErrback(handleFailure, bucket.request)
d.addBoth(bucket.request.write)
d.addBoth(lambda _: bucket.request.finish())
return server.NOT_DONE_YET
else:
message = 'No target/downstream dispatcher for scheme {scheme}'.format(scheme=self.scheme)
log.error(message)
raise KeyError(message)
