def capture(self): """ Capture output of the whole build process. Return stacktrace of Python driver and output of make command, if any. """ # The result object containing build success flag and error outputs. # This gets bound to the context manager variable. result = Bunch() self.build_result['capture'] = result # A temporary file to redirect make output to self.stream = NamedTemporaryFile() try: # Pass execution flow to context manager body, # effectively running the main build process yield result # Signal success if build engine completed result.success = True except Exception as ex: # Signal failure result.success = False # Capture Python traceback result.error = last_error_and_traceback() # Capture make output self.stream.seek(0) result.output = self.stream.read()
def capture(self): """ Capture output of the whole build process. Return stacktrace of Python driver and output of make command, if any. """ # The result object containing build success flag and error outputs. # This gets bound to the context manager variable. result = Bunch() self.build_result['capture'] = result # A temporary file to redirect make output to self.stream = NamedTemporaryFile() try: # Pass execution flow to context manager body, # effectively running the main build process yield result # Signal success if build engine completed result.success = True except Exception as ex: # Signal failure result.success = False # Capture Python traceback result.error = last_error_and_traceback() # Capture make output self.stream.seek(0) result.output = self.stream.read()
def get_application_factory(self, name): # Get application information from configuration object try: application_settings = self.settings[name] app_factory = \ 'app_factory' in application_settings and \ application_settings.app_factory or \ application_settings.application except: log.failure( 'Application configuration object for "{name}" not found, missing setting "app_factory"', name=name) return try: factory_callable = self.load_entrypoint(app_factory) except: log.failure( 'Error loading application entrypoint "{app_factory}" for "{name}":\n{ex}', name=name, app_factory=app_factory, ex=last_error_and_traceback()) return return factory_callable
def error_response(bucket, error_message='', code=http.BAD_REQUEST, with_traceback=False): """ Error handling method logging and returning appropriate stacktrace. """ # FIXME: Check for privacy. Do something more sane with the stacktrace # or enable only when sending appropriate request arguments. if with_traceback: error_message += '\n' + last_error_and_traceback() log.error(error_message) bucket.request.setResponseCode(code) #bucket.request.setHeader('Content-Type', 'text/plain; charset=utf-8') return error_message.encode('utf-8')
def get_application_factory(self, name): # Get application information from configuration object try: application_settings = self.settings[name] app_factory = \ 'app_factory' in application_settings and \ application_settings.app_factory or \ application_settings.application except: log.failure('Application configuration object for "{name}" not found, missing setting "app_factory"', name=name) return try: factory_callable = self.load_entrypoint(app_factory) except: log.failure('Error loading application entrypoint "{app_factory}" for "{name}":\n{ex}', name=name, app_factory=app_factory, ex=last_error_and_traceback()) return return factory_callable
def emit(self, uri, bucket): """ Adapt, serialize and emit data bucket to target service. """ log.debug('Emitting to target scheme {scheme}', scheme=self.scheme) if self.scheme == 'mqtt': # Publish JSON payload to MQTT bus topic = uri payload = bucket.json # TODO: Use threads.deferToThread here? return self.downstream.publish(topic, payload) elif self.scheme == 'influxdb': # InfluxDB query wrapper using expression derived from transformation data dfq = DataFrameQuery(settings=self.settings, bucket=bucket) # Perform query and obtain results as pandas DataFrame df = dfq.query() # Announce routing information via http response headers bucket.request.setHeader('Target-Database', bucket.tdata.database) bucket.request.setHeader('Target-Expression', bucket.tdata.expression) bucket.request.setHeader('Target-Address-Scheme', self.scheme) bucket.request.setHeader('Target-Address-Uri', uri) # Database result is empty, send appropriate response if df is None or df.empty: return self.response_no_results(bucket) # DataFrame manipulation # Drop some fields from DataFrame as requested if 'exclude' in bucket.tdata and bucket.tdata.exclude: drop_fields = read_list(bucket.tdata.exclude, empty_elements=False) try: df.drop(drop_fields, axis=1, inplace=True) except ValueError as ex: log.error(last_error_and_traceback()) error_message = u'Error: {type} {message}'.format( type=type(ex), message=ex) return bucket.request.error_response( bucket, error_message=error_message) # Use only specified fields from DataFrame as requested if 'include' in bucket.tdata and bucket.tdata.include: use_fields = read_list(bucket.tdata.include, empty_elements=False) use_fields.insert(0, 'time') try: df = df.filter(use_fields, axis=1) except ValueError as ex: log.error(last_error_and_traceback()) error_message = u'Error: {type} {message}'.format( type=type(ex), message=ex) return bucket.request.error_response( bucket, error_message=error_message) # Propagate non-null values forward or backward. # With time series data, using pad/ffill is extremely common so that the “last known value” is available at every time point. # http://pandas.pydata.org/pandas-docs/stable/missing_data.html#filling-missing-values-fillna if 'pad' in bucket.tdata and asbool(bucket.tdata.pad): df.fillna(method='pad', inplace=True) if 'backfill' in bucket.tdata and asbool(bucket.tdata.backfill): df.fillna(method='backfill', inplace=True) if 'interpolate' in bucket.tdata and asbool( bucket.tdata.interpolate): # Performs linear interpolation at missing datapoints, # otherwise matplotlib would not plot the sparse data frame. # http://pandas.pydata.org/pandas-docs/stable/missing_data.html#interpolation df.interpolate(inplace=True) if 'sorted' in bucket.tdata and asbool(bucket.tdata.sorted): # http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort.html df.sort(axis='columns', inplace=True) # Compute http response from DataFrame, taking designated output format into account response = HttpDataFrameResponse(bucket, dataframe=df) # Synchronous, the worker-threading is already on the HTTP layer return response.render() # Asynchronous: Perform computation in separate thread d = threads.deferToThread(response.render) d.addErrback(handleFailure, bucket.request) d.addBoth(bucket.request.write) d.addBoth(lambda _: bucket.request.finish()) return server.NOT_DONE_YET else: message = 'No target/downstream dispatcher for scheme {scheme}'.format( scheme=self.scheme) log.error(message) raise KeyError(message)
def emit(self, uri, bucket): """ Adapt, serialize and emit data bucket to target service. """ log.debug('Emitting to target scheme {scheme}', scheme=self.scheme) if self.scheme == 'mqtt': # Publish JSON payload to MQTT bus topic = uri payload = bucket.json # TODO: Use threads.deferToThread here? return self.downstream.publish(topic, payload) elif self.scheme == 'influxdb': # InfluxDB query wrapper using expression derived from transformation data dfq = DataFrameQuery(settings=self.settings, bucket=bucket) # Perform query and obtain results as pandas DataFrame df = dfq.query() # Announce routing information via http response headers bucket.request.setHeader('Target-Database', bucket.tdata.database) bucket.request.setHeader('Target-Expression', bucket.tdata.expression) bucket.request.setHeader('Target-Address-Scheme', self.scheme) bucket.request.setHeader('Target-Address-Uri', uri) # Database result is empty, send appropriate response if df is None or df.empty: return self.response_no_results(bucket) # DataFrame manipulation # Drop some fields from DataFrame as requested if 'exclude' in bucket.tdata and bucket.tdata.exclude: drop_fields = read_list(bucket.tdata.exclude, empty_elements=False) try: df.drop(drop_fields, axis=1, inplace=True) except ValueError as ex: log.error(last_error_and_traceback()) error_message = u'Error: {type} {message}'.format(type=type(ex), message=ex) return bucket.request.error_response(bucket, error_message=error_message) # Use only specified fields from DataFrame as requested if 'include' in bucket.tdata and bucket.tdata.include: use_fields = read_list(bucket.tdata.include, empty_elements=False) use_fields.insert(0, 'time') try: df = df.filter(use_fields, axis=1) except ValueError as ex: log.error(last_error_and_traceback()) error_message = u'Error: {type} {message}'.format(type=type(ex), message=ex) return bucket.request.error_response(bucket, error_message=error_message) # Propagate non-null values forward or backward. # With time series data, using pad/ffill is extremely common so that the “last known value” is available at every time point. # http://pandas.pydata.org/pandas-docs/stable/missing_data.html#filling-missing-values-fillna if 'pad' in bucket.tdata and asbool(bucket.tdata.pad): df.fillna(method='pad', inplace=True) if 'backfill' in bucket.tdata and asbool(bucket.tdata.backfill): df.fillna(method='backfill', inplace=True) if 'interpolate' in bucket.tdata and asbool(bucket.tdata.interpolate): # Performs linear interpolation at missing datapoints, # otherwise matplotlib would not plot the sparse data frame. # http://pandas.pydata.org/pandas-docs/stable/missing_data.html#interpolation df.interpolate(inplace=True) if 'sorted' in bucket.tdata and asbool(bucket.tdata.sorted): # http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.sort.html df.sort(axis='columns', inplace=True) # Compute http response from DataFrame, taking designated output format into account response = HttpDataFrameResponse(bucket, dataframe=df) # Synchronous, the worker-threading is already on the HTTP layer return response.render() # Asynchronous: Perform computation in separate thread d = threads.deferToThread(response.render) d.addErrback(handleFailure, bucket.request) d.addBoth(bucket.request.write) d.addBoth(lambda _: bucket.request.finish()) return server.NOT_DONE_YET else: message = 'No target/downstream dispatcher for scheme {scheme}'.format(scheme=self.scheme) log.error(message) raise KeyError(message)