def setupService(self): self.log(log.info, u'Bootstrapping') self.settings = self.parent.settings # Optionally register subsystem component as child service for subsystem in self.subsystems: if hasattr(self, subsystem): subsystem_service = getattr(self, subsystem) if isinstance(subsystem_service, Service): log.info('Registering subsystem component "{subsystem}" as service', subsystem=subsystem) self.registerService(subsystem_service) # Configure metrics to be collected each X seconds metrics_interval = int(self.channel.get('metrics_logger_interval', 60)) self.metrics = Bunch(tx_count=0, starttime=time.time(), interval=metrics_interval) subscriptions = read_list(self.channel.mqtt_topics) self.mqtt_service = MqttAdapter( name = u'mqtt-' + self.channel.realm, broker_host = self.settings.mqtt.host, broker_port = int(self.settings.mqtt.port), broker_username = self.settings.mqtt.username, broker_password = self.settings.mqtt.password, callback = self.mqtt_receive, subscriptions = subscriptions) self.registerService(self.mqtt_service) self.influx = InfluxDBAdapter(settings = self.settings.influxdb) # Perform MQTT message processing using a different thread pool self.threadpool = ThreadPool() self.thimble = Thimble(reactor, self.threadpool, self, ["process_message"])
def forward(self, bucket): """ Receive data bucket from source, run through transformation machinery and emit to target. """ # 1. Map/transform topology address information if 'transform' in self.channel: for entrypoint in read_list(self.channel.transform): try: transformer = KotoriBootloader.load_entrypoint(entrypoint) bucket.tdata.update(transformer(bucket.tdata)) except ImportError as ex: log.error('ImportError "{message}" when loading entrypoint "{entrypoint}"', entrypoint=entrypoint, message=ex) # MQTT doesn't prefer leading forward slashes with topic names, let's get rid of them target_uri_tpl = self.target_uri.path.lstrip('/') # Compute target bus topic from url matches target_uri = target_uri_tpl.format(**bucket.tdata) # Enrich bucket by putting source and target addresses into it bucket.address = Bunch(source=self.source_address, target=self.target_address) # 2. Reporting bucket_logging = Bunch(bucket) if 'body' in bucket_logging and len(bucket_logging.body) > 100: bucket_logging.body = bucket_logging.body[:100] + ' [...]' log.debug('Forwarding bucket to {target} with bucket={bucket}. Effective target uri is {target_uri}', target=self.channel.target, target_uri=target_uri, bucket=dict(bucket_logging)) # 3. Adapt, serialize and emit appropriately return self.target_service.emit(target_uri, bucket)
def lst(cls, settings): from kotori.vendor.lst.application import lst_boot debug = settings.options.debug if settings.options.debug_vendor and 'lst' in read_list( settings.options.debug_vendor): changeLogLevel('kotori.vendor.lst', LogLevel.debug) lst_boot(settings)
def lst_channels(config): channel_labels = read_list(config['lst']['channels']) channel_infos = [] for channel_label in channel_labels: channel_settings = config[channel_label] channel_info = get_channel_info(channel_label, channel_settings) channel_infos.append(channel_info) print tabulate(channel_infos, headers='keys')
def parse(self, address): m = self.address_pattern.match(address) match = m.groupdict() if match: self.raw_uri = match['uri'] self.raw_predicates = match['predicates'] self.parsed_uri = urlparse(self.raw_uri) self.uri = bunchify(self.parsed_uri._asdict()) if self.raw_predicates: self.predicates = read_list(self.raw_predicates)
def lst_boot(config, debug=False): # TODO: refactor towards OO; e.g. BinaryMessageApplicationFactory wamp_uri = unicode(config.wamp.uri) # activate/mount multiple "lst" applications for channel_name in read_list(config['lst']['channels']): log.info('Starting "lst" channel "{}"'.format(channel_name)) channel = config[channel_name] config = deepcopy(config) config['_active_'] = channel WampApplication(url=wamp_uri, realm=u'lst', session_class=UdpSession, config=config).make() WampApplication(url=wamp_uri, realm=u'lst', session_class=StorageSession, config=config).make()
def lst_boot(config, debug=False): # TODO: refactor towards OO; e.g. BinaryMessageApplicationFactory wamp_uri = str(config.wamp.uri) # activate/mount multiple "lst" applications for channel_name in read_list(config['lst']['channels']): log.info('Starting "lst" channel "{}"'.format(channel_name)) channel = config[channel_name] config = deepcopy(config) config['_active_'] = channel WampApplication(url=wamp_uri, realm=u'lst', session_class=UdpSession, config=config).make() WampApplication(url=wamp_uri, realm=u'lst', session_class=StorageSession, config=config).make()
def __init__(self, name=None, application_settings=None, global_settings=None): RootService.__init__(self, settings=global_settings) # Compute name for Twisted service self.name = u'app-{name}'.format(name=name) # Make channel object from application settings configuration object self.channel = Bunch(**application_settings) # Main application components for service_reference in read_list(application_settings.services): # Load service component try: service_factory = KotoriBootloader.load_entrypoint( service_reference) except Exception as ex: log.failure( 'Error loading composite service component "{service_name}" into "{app_name}":\n{log_failure}"', service_name=service_reference, app_name=name) continue # Load data processing strategy and graphing components # TODO: Review whether this should be per-service or not # TODO: Introduce strict/non-strict handling strategy_factory = KotoriBootloader.load_entrypoint( application_settings.strategy, onerror='ignore') graphing_factory = KotoriBootloader.load_entrypoint( application_settings.graphing, onerror='ignore') # Create application service object composed of subsystem components # TODO: Bundle all arguments into yet another wrapper object for an universal object factory service = service_factory( channel=self.channel, strategy=strategy_factory(settings=global_settings), graphing=graphing_factory(settings=global_settings, channel=self.channel)) # Register service component with its container self.registerService(service)
def __init__(self, name=None, application_settings=None, global_settings=None): RootService.__init__(self, settings=global_settings) # Compute name for Twisted service self.name = u'app-{name}'.format(name=name) # Make channel object from application settings configuration object self.channel = Bunch(**application_settings) # Main application components for service_reference in read_list(application_settings.services): # Load service component try: service_factory = KotoriBootloader.load_entrypoint(service_reference) except Exception as ex: log.failure('Error loading composite service component "{service_name}" into "{app_name}":\n{log_failure}"', service_name=service_reference, app_name=name) continue # Load data processing strategy and graphing components # TODO: Review whether this should be per-service or not # TODO: Introduce strict/non-strict handling strategy_factory = KotoriBootloader.load_entrypoint(application_settings.strategy, onerror='ignore') graphing_factory = KotoriBootloader.load_entrypoint(application_settings.graphing, onerror='ignore') # Create application service object composed of subsystem components # TODO: Bundle all arguments into yet another wrapper object for an universal object factory service = service_factory( channel = self.channel, strategy = strategy_factory(settings=global_settings), graphing = graphing_factory(settings=global_settings, channel=self.channel) ) # Register service component with its container self.registerService(service)
def emit(self, uri, bucket): """ Adapt, serialize and emit data bucket to target service. """ log.debug('Emitting to target scheme {scheme}', scheme=self.scheme) if self.scheme == 'mqtt': # Publish JSON payload to MQTT bus topic = uri payload = bucket.json # TODO: Use threads.deferToThread here? return self.downstream.publish(topic, payload) elif self.scheme == 'influxdb': # InfluxDB query wrapper using expression derived from transformation data dfq = DataFrameQuery(settings=self.settings, bucket=bucket) # Perform query and obtain results as pandas DataFrame df = dfq.query() # Announce routing information via http response headers bucket.request.setHeader('Target-Database', bucket.tdata.database) bucket.request.setHeader('Target-Expression', bucket.tdata.expression) bucket.request.setHeader('Target-Address-Scheme', self.scheme) bucket.request.setHeader('Target-Address-Uri', uri) # Database result is empty, send appropriate response if df is None or df.empty: return self.response_no_results(bucket) # DataFrame manipulation # Drop some fields from DataFrame as requested if 'exclude' in bucket.tdata and bucket.tdata.exclude: drop_fields = read_list(bucket.tdata.exclude, empty_elements=False) try: df.drop(drop_fields, axis=1, inplace=True) except ValueError as ex: log.error(last_error_and_traceback()) error_message = u'Error: {type} {message}'.format( type=type(ex), message=ex) return bucket.request.error_response( bucket, error_message=error_message) # Use only specified fields from DataFrame as requested if 'include' in bucket.tdata and bucket.tdata.include: use_fields = read_list(bucket.tdata.include, empty_elements=False) use_fields.insert(0, 'time') try: df = df.filter(use_fields, axis=1) except ValueError as ex: log.error(last_error_and_traceback()) error_message = u'Error: {type} {message}'.format( type=type(ex), message=ex) return bucket.request.error_response( bucket, error_message=error_message) # Propagate non-null values forward or backward. # With time series data, using pad/ffill is extremely common so that the “last known value” is available at every time point. # http://pandas.pydata.org/pandas-docs/stable/missing_data.html#filling-missing-values-fillna if 'pad' in bucket.tdata and asbool(bucket.tdata.pad): df.fillna(method='pad', inplace=True) if 'backfill' in bucket.tdata and asbool(bucket.tdata.backfill): df.fillna(method='backfill', inplace=True) if 'interpolate' in bucket.tdata and asbool( bucket.tdata.interpolate): # Performs linear interpolation at missing datapoints, # otherwise matplotlib would not plot the sparse data frame. # http://pandas.pydata.org/pandas-docs/stable/missing_data.html#interpolation df.interpolate(inplace=True) if 'sorted' in bucket.tdata and asbool(bucket.tdata.sorted): # http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort.html df.sort(axis='columns', inplace=True) # Compute http response from DataFrame, taking designated output format into account response = HttpDataFrameResponse(bucket, dataframe=df) # Synchronous, the worker-threading is already on the HTTP layer return response.render() # Asynchronous: Perform computation in separate thread d = threads.deferToThread(response.render) d.addErrback(handleFailure, bucket.request) d.addBoth(bucket.request.write) d.addBoth(lambda _: bucket.request.finish()) return server.NOT_DONE_YET else: message = 'No target/downstream dispatcher for scheme {scheme}'.format( scheme=self.scheme) log.error(message) raise KeyError(message)
def build_firmware(self, bucket=None, options=None, data=None): # FirmwareBuilder machinery fwbuilder = FirmwareBuilder( repo_url=options.url, repo_branch=options.ref, update_submodules=options.update_submodules, workingdir=options.path, architecture = options.get('architecture'), esp_root=options.esp_root) # Run build process with capturing its output with fwbuilder.capture() as result: # Acquire source code from git repository fwbuilder.acquire_source() # Patch files with individual parameters """ data = { 'BOARD_TAG': 'mega2560', 'BOARD_SUB': 'atmega2560', 'HE_USER': '******', 'HE_SITE': 'Buxtehude', 'HE_HIVE': 'Raeuberhoehle', 'GPRSBEE_APN': 'internet.altes-land.de', 'GPRSBEE_VCC': 23, } """ patch_files = read_list(self.channel.patch_files) fwbuilder.patch_files(patch_files, data) # Run build process fwbuilder.run_build(makefile=options.makefile) artefact = fwbuilder.make_artefact() if result.success: # Compute extended firmware name suffix from transformation dict # TODO: Route this information to "Artefact" object """ fwparams = '' if data: data = data.copy() del data['slot'] dlist = [] for key, value in data.iteritems(): dlist.append(u'{key}={value}'.format(**locals())) fwparams = u'-' + u','.join(dlist) """ filename = '{realm}_{name}.{suffix}'.format( realm=self.channel.realm, name=artefact.fullname, suffix=options.suffix) log.info(u'Build succeeded, filename: {filename}, artefact: {artefact}', filename=filename, artefact=artefact) bucket.request.setHeader('Content-Type', 'application/octet-stream') bucket.request.setHeader('Content-Disposition', 'attachment; filename={filename}'.format(filename=filename)) # "options.suffix" may be "hex" or "elf" payload = artefact.get_binary(options.suffix) return payload else: error_message = fwbuilder.error_message() log.error(u'Build failed\n{message}', message=error_message) bucket.request.setResponseCode(http.BAD_REQUEST) bucket.request.setHeader('Content-Type', 'text/plain; charset=utf-8') return error_message.encode('utf-8')
def lst(cls, settings): from kotori.vendor.lst.application import lst_boot debug = settings.options.debug if settings.options.debug_vendor and 'lst' in read_list(settings.options.debug_vendor): changeLogLevel('kotori.vendor.lst', LogLevel.debug) lst_boot(settings)
def emit(self, uri, bucket): """ Adapt, serialize and emit data bucket to target service. """ log.debug('Emitting to target scheme {scheme}', scheme=self.scheme) if self.scheme == 'mqtt': # Publish JSON payload to MQTT bus topic = uri payload = bucket.json # TODO: Use threads.deferToThread here? return self.downstream.publish(topic, payload) elif self.scheme == 'influxdb': # InfluxDB query wrapper using expression derived from transformation data dfq = DataFrameQuery(settings=self.settings, bucket=bucket) # Perform query and obtain results as pandas DataFrame df = dfq.query() # Announce routing information via http response headers bucket.request.setHeader('Target-Database', bucket.tdata.database) bucket.request.setHeader('Target-Expression', bucket.tdata.expression) bucket.request.setHeader('Target-Address-Scheme', self.scheme) bucket.request.setHeader('Target-Address-Uri', uri) # Database result is empty, send appropriate response if df is None or df.empty: return self.response_no_results(bucket) # DataFrame manipulation # Drop some fields from DataFrame as requested if 'exclude' in bucket.tdata and bucket.tdata.exclude: drop_fields = read_list(bucket.tdata.exclude, empty_elements=False) try: df.drop(drop_fields, axis=1, inplace=True) except ValueError as ex: log.error(last_error_and_traceback()) error_message = u'Error: {type} {message}'.format(type=type(ex), message=ex) return bucket.request.error_response(bucket, error_message=error_message) # Use only specified fields from DataFrame as requested if 'include' in bucket.tdata and bucket.tdata.include: use_fields = read_list(bucket.tdata.include, empty_elements=False) use_fields.insert(0, 'time') try: df = df.filter(use_fields, axis=1) except ValueError as ex: log.error(last_error_and_traceback()) error_message = u'Error: {type} {message}'.format(type=type(ex), message=ex) return bucket.request.error_response(bucket, error_message=error_message) # Propagate non-null values forward or backward. # With time series data, using pad/ffill is extremely common so that the “last known value” is available at every time point. # http://pandas.pydata.org/pandas-docs/stable/missing_data.html#filling-missing-values-fillna if 'pad' in bucket.tdata and asbool(bucket.tdata.pad): df.fillna(method='pad', inplace=True) if 'backfill' in bucket.tdata and asbool(bucket.tdata.backfill): df.fillna(method='backfill', inplace=True) if 'interpolate' in bucket.tdata and asbool(bucket.tdata.interpolate): # Performs linear interpolation at missing datapoints, # otherwise matplotlib would not plot the sparse data frame. # http://pandas.pydata.org/pandas-docs/stable/missing_data.html#interpolation df.interpolate(inplace=True) if 'sorted' in bucket.tdata and asbool(bucket.tdata.sorted): # http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort.html df.sort(axis='columns', inplace=True) # Compute http response from DataFrame, taking designated output format into account response = HttpDataFrameResponse(bucket, dataframe=df) # Synchronous, the worker-threading is already on the HTTP layer return response.render() # Asynchronous: Perform computation in separate thread d = threads.deferToThread(response.render) d.addErrback(handleFailure, bucket.request) d.addBoth(bucket.request.write) d.addBoth(lambda _: bucket.request.finish()) return server.NOT_DONE_YET else: message = 'No target/downstream dispatcher for scheme {scheme}'.format(scheme=self.scheme) log.error(message) raise KeyError(message)