def capture(self):
"""
Capture output of the whole build process.
Return stacktrace of Python driver and output of make command, if any.
"""
# The result object containing build success flag and error outputs.
# This gets bound to the context manager variable.
result = Bunch()
self.build_result['capture'] = result
# A temporary file to redirect make output to
self.stream = NamedTemporaryFile()
try:
# Pass execution flow to context manager body,
# effectively running the main build process
yield result
# Signal success if build engine completed
result.success = True
except Exception as ex:
# Signal failure
result.success = False
# Capture Python traceback
result.error = last_error_and_traceback()
# Capture make output
self.stream.seek(0)
result.output = self.stream.read()
def capture(self):
"""
Capture output of the whole build process.
Return stacktrace of Python driver and output of make command, if any.
"""
# The result object containing build success flag and error outputs.
# This gets bound to the context manager variable.
result = Bunch()
self.build_result['capture'] = result
# A temporary file to redirect make output to
self.stream = NamedTemporaryFile()
try:
# Pass execution flow to context manager body,
# effectively running the main build process
yield result
# Signal success if build engine completed
result.success = True
except Exception as ex:
# Signal failure
result.success = False
# Capture Python traceback
result.error = last_error_and_traceback()
# Capture make output
self.stream.seek(0)
result.output = self.stream.read()
def get_application_factory(self, name):
# Get application information from configuration object
try:
application_settings = self.settings[name]
app_factory = \
'app_factory' in application_settings and \
application_settings.app_factory or \
application_settings.application
except:
log.failure(
'Application configuration object for "{name}" not found, missing setting "app_factory"',
name=name)
return
try:
factory_callable = self.load_entrypoint(app_factory)
except:
log.failure(
'Error loading application entrypoint "{app_factory}" for "{name}":\n{ex}',
name=name,
app_factory=app_factory,
ex=last_error_and_traceback())
return
return factory_callable
def error_response(bucket, error_message='', code=http.BAD_REQUEST, with_traceback=False):
"""
Error handling method logging and returning appropriate stacktrace.
"""
# FIXME: Check for privacy. Do something more sane with the stacktrace
# or enable only when sending appropriate request arguments.
if with_traceback:
error_message += '\n' + last_error_and_traceback()
log.error(error_message)
bucket.request.setResponseCode(code)
#bucket.request.setHeader('Content-Type', 'text/plain; charset=utf-8')
return error_message.encode('utf-8')
def get_application_factory(self, name):
# Get application information from configuration object
try:
application_settings = self.settings[name]
app_factory = \
'app_factory' in application_settings and \
application_settings.app_factory or \
application_settings.application
except:
log.failure('Application configuration object for "{name}" not found, missing setting "app_factory"', name=name)
return
try:
factory_callable = self.load_entrypoint(app_factory)
except:
log.failure('Error loading application entrypoint "{app_factory}" for "{name}":\n{ex}',
name=name, app_factory=app_factory, ex=last_error_and_traceback())
return
return factory_callable
def emit(self, uri, bucket):
"""
Adapt, serialize and emit data bucket to target service.
"""
log.debug('Emitting to target scheme {scheme}', scheme=self.scheme)
if self.scheme == 'mqtt':
# Publish JSON payload to MQTT bus
topic = uri
payload = bucket.json
# TODO: Use threads.deferToThread here?
return self.downstream.publish(topic, payload)
elif self.scheme == 'influxdb':
# InfluxDB query wrapper using expression derived from transformation data
dfq = DataFrameQuery(settings=self.settings, bucket=bucket)
# Perform query and obtain results as pandas DataFrame
df = dfq.query()
# Announce routing information via http response headers
bucket.request.setHeader('Target-Database', bucket.tdata.database)
bucket.request.setHeader('Target-Expression',
bucket.tdata.expression)
bucket.request.setHeader('Target-Address-Scheme', self.scheme)
bucket.request.setHeader('Target-Address-Uri', uri)
# Database result is empty, send appropriate response
if df is None or df.empty:
return self.response_no_results(bucket)
# DataFrame manipulation
# Drop some fields from DataFrame as requested
if 'exclude' in bucket.tdata and bucket.tdata.exclude:
drop_fields = read_list(bucket.tdata.exclude,
empty_elements=False)
try:
df.drop(drop_fields, axis=1, inplace=True)
except ValueError as ex:
log.error(last_error_and_traceback())
error_message = u'Error: {type} {message}'.format(
type=type(ex), message=ex)
return bucket.request.error_response(
bucket, error_message=error_message)
# Use only specified fields from DataFrame as requested
if 'include' in bucket.tdata and bucket.tdata.include:
use_fields = read_list(bucket.tdata.include,
empty_elements=False)
use_fields.insert(0, 'time')
try:
df = df.filter(use_fields, axis=1)
except ValueError as ex:
log.error(last_error_and_traceback())
error_message = u'Error: {type} {message}'.format(
type=type(ex), message=ex)
return bucket.request.error_response(
bucket, error_message=error_message)
# Propagate non-null values forward or backward.
# With time series data, using pad/ffill is extremely common so that the “last known value” is available at every time point.
# http://pandas.pydata.org/pandas-docs/stable/missing_data.html#filling-missing-values-fillna
if 'pad' in bucket.tdata and asbool(bucket.tdata.pad):
df.fillna(method='pad', inplace=True)
if 'backfill' in bucket.tdata and asbool(bucket.tdata.backfill):
df.fillna(method='backfill', inplace=True)
if 'interpolate' in bucket.tdata and asbool(
bucket.tdata.interpolate):
# Performs linear interpolation at missing datapoints,
# otherwise matplotlib would not plot the sparse data frame.
# http://pandas.pydata.org/pandas-docs/stable/missing_data.html#interpolation
df.interpolate(inplace=True)
if 'sorted' in bucket.tdata and asbool(bucket.tdata.sorted):
# http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort.html
df.sort(axis='columns', inplace=True)
# Compute http response from DataFrame, taking designated output format into account
response = HttpDataFrameResponse(bucket, dataframe=df)
# Synchronous, the worker-threading is already on the HTTP layer
return response.render()
# Asynchronous: Perform computation in separate thread
d = threads.deferToThread(response.render)
d.addErrback(handleFailure, bucket.request)
d.addBoth(bucket.request.write)
d.addBoth(lambda _: bucket.request.finish())
return server.NOT_DONE_YET
else:
message = 'No target/downstream dispatcher for scheme {scheme}'.format(
scheme=self.scheme)
log.error(message)
raise KeyError(message)
def emit(self, uri, bucket):
"""
Adapt, serialize and emit data bucket to target service.
"""
log.debug('Emitting to target scheme {scheme}', scheme=self.scheme)
if self.scheme == 'mqtt':
# Publish JSON payload to MQTT bus
topic = uri
payload = bucket.json
# TODO: Use threads.deferToThread here?
return self.downstream.publish(topic, payload)
elif self.scheme == 'influxdb':
# InfluxDB query wrapper using expression derived from transformation data
dfq = DataFrameQuery(settings=self.settings, bucket=bucket)
# Perform query and obtain results as pandas DataFrame
df = dfq.query()
# Announce routing information via http response headers
bucket.request.setHeader('Target-Database', bucket.tdata.database)
bucket.request.setHeader('Target-Expression', bucket.tdata.expression)
bucket.request.setHeader('Target-Address-Scheme', self.scheme)
bucket.request.setHeader('Target-Address-Uri', uri)
# Database result is empty, send appropriate response
if df is None or df.empty:
return self.response_no_results(bucket)
# DataFrame manipulation
# Drop some fields from DataFrame as requested
if 'exclude' in bucket.tdata and bucket.tdata.exclude:
drop_fields = read_list(bucket.tdata.exclude, empty_elements=False)
try:
df.drop(drop_fields, axis=1, inplace=True)
except ValueError as ex:
log.error(last_error_and_traceback())
error_message = u'Error: {type} {message}'.format(type=type(ex), message=ex)
return bucket.request.error_response(bucket, error_message=error_message)
# Use only specified fields from DataFrame as requested
if 'include' in bucket.tdata and bucket.tdata.include:
use_fields = read_list(bucket.tdata.include, empty_elements=False)
use_fields.insert(0, 'time')
try:
df = df.filter(use_fields, axis=1)
except ValueError as ex:
log.error(last_error_and_traceback())
error_message = u'Error: {type} {message}'.format(type=type(ex), message=ex)
return bucket.request.error_response(bucket, error_message=error_message)
# Propagate non-null values forward or backward.
# With time series data, using pad/ffill is extremely common so that the “last known value” is available at every time point.
# http://pandas.pydata.org/pandas-docs/stable/missing_data.html#filling-missing-values-fillna
if 'pad' in bucket.tdata and asbool(bucket.tdata.pad):
df.fillna(method='pad', inplace=True)
if 'backfill' in bucket.tdata and asbool(bucket.tdata.backfill):
df.fillna(method='backfill', inplace=True)
if 'interpolate' in bucket.tdata and asbool(bucket.tdata.interpolate):
# Performs linear interpolation at missing datapoints,
# otherwise matplotlib would not plot the sparse data frame.
# http://pandas.pydata.org/pandas-docs/stable/missing_data.html#interpolation
df.interpolate(inplace=True)
if 'sorted' in bucket.tdata and asbool(bucket.tdata.sorted):
# http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort.html
df.sort(axis='columns', inplace=True)
# Compute http response from DataFrame, taking designated output format into account
response = HttpDataFrameResponse(bucket, dataframe=df)
# Synchronous, the worker-threading is already on the HTTP layer
return response.render()
# Asynchronous: Perform computation in separate thread
d = threads.deferToThread(response.render)
d.addErrback(handleFailure, bucket.request)
d.addBoth(bucket.request.write)
d.addBoth(lambda _: bucket.request.finish())
return server.NOT_DONE_YET
else:
message = 'No target/downstream dispatcher for scheme {scheme}'.format(scheme=self.scheme)
log.error(message)
raise KeyError(message)