def test_make_lines(self): data = { "tags": { "empty_tag": "", "none_tag": None, "integer_tag": 2, "string_tag": "hello" }, "points": [ { "measurement": "test", "fields": { "string_val": "hello!", "int_val": 1, "float_val": 1.1, "none_field": None, "bool_val": True, } } ] } self.assertEqual( line_protocol.make_lines(data), 'test,integer_tag=2,string_tag=hello ' 'bool_val=True,float_val=1.1,int_val=1i,string_val="hello!"\n' )
def test_make_lines(self): """Test make new lines in TestLineProtocol object.""" data = { "tags": { "empty_tag": "", "none_tag": None, "backslash_tag": "C:\\", "integer_tag": 2, "string_tag": "hello" }, "points": [ { "measurement": "test", "fields": { "string_val": "hello!", "int_val": 1, "float_val": 1.1, "none_field": None, "bool_val": True, } } ] } self.assertEqual( line_protocol.make_lines(data), 'test,backslash_tag=C:\\\\ ,integer_tag=2,string_tag=hello ' 'bool_val=True,float_val=1.1,int_val=1i,string_val="hello!"\n' )
def write(self, data, params=None, expected_response_code=204): """Write data to InfluxDB. :param data: the data to be written :type data: dict :param params: additional parameters for the request, defaults to None :type params: dict :param expected_response_code: the expected response code of the write operation, defaults to 204 :type expected_response_code: int :returns: True, if the write operation is successful :rtype: bool """ headers = self._headers headers['Content-type'] = 'application/octet-stream' if params: precision = params.get('precision') else: precision = None self.request( url="write", method='POST', params=params, data=make_lines(data, precision).encode('utf-8'), expected_response_code=expected_response_code, headers=headers ) return True
def test_timezone(self): dt = datetime(2009, 11, 10, 23, 0, 0, 123456) utc = UTC.localize(dt) berlin = timezone('Europe/Berlin').localize(dt) eastern = berlin.astimezone(timezone('US/Eastern')) data = { "points": [ {"measurement": "A", "fields": {"val": 1}, "time": 0}, {"measurement": "A", "fields": {"val": 1}, "time": "2009-11-10T23:00:00.123456Z"}, {"measurement": "A", "fields": {"val": 1}, "time": dt}, {"measurement": "A", "fields": {"val": 1}, "time": utc}, {"measurement": "A", "fields": {"val": 1}, "time": berlin}, {"measurement": "A", "fields": {"val": 1}, "time": eastern}, ] } self.assertEqual( line_protocol.make_lines(data), '\n'.join([ 'A val=1i 0', 'A val=1i 1257894000123456000', 'A val=1i 1257894000123456000', 'A val=1i 1257894000123456000', 'A val=1i 1257890400123456000', 'A val=1i 1257890400123456000', ]) + '\n' )
def send_packet(self, packet): """Send an UDP packet. :param packet: the packet to be sent :type packet: dict """ data = make_lines(packet).encode('utf-8') self.udp_socket.sendto(data, (self._host, self.udp_port))
async def receive(request): """ Receive data and drop it in InfluxDB. """ data = await request.json() auth_header = request.headers.get('authorization') if auth_header: auth_parts = auth_header.split(' ') token = os.getenv('AUTH_TOKEN', None) if token: if token != auth_parts[1]: return web.Response(status=401) if 'data' in data.keys(): db = data.get('db', None) for item in data.get('data'): lines = line_protocol.make_lines(item) resp = await save(lines, db_name=db) resp.close() return web.Response(status=201) points = data.get('points') if points: # convert ints to floats for idx, item in enumerate(points): fields = item.get('fields') if fields: for k, v in fields.items(): if k != 'time': if isinstance(v, int): data['points'][idx]['fields'][k] = float(v) lines = line_protocol.make_lines(data) resp = await save(lines) resp.close() return web.Response(status=201)
def send_packet(self, packet, protocol='json'): """Send an UDP packet. :param packet: the packet to be sent :type packet: (if protocol is 'json') dict (if protocol is 'line') sequence of line protocol strings :param protocol: protocol of input data, either 'json' or 'line' :type protocol: str """ if protocol == 'json': data = make_lines(packet).encode('utf-8') elif protocol == 'line': data = ('\n'.join(data) + '\n').encode('utf-8') self.udp_socket.sendto(data, (self._host, self.udp_port))
def send_packet(self, packet, protocol='json', time_precision=None): """Send an UDP packet. :param packet: the packet to be sent :type packet: (if protocol is 'json') dict (if protocol is 'line') list of line protocol strings :param protocol: protocol of input data, either 'json' or 'line' :type protocol: str :param time_precision: Either 's', 'm', 'ms' or 'u', defaults to None :type time_precision: str """ if protocol == 'json': data = make_lines(packet, time_precision).encode('utf-8') elif protocol == 'line': data = ('\n'.join(packet) + '\n').encode('utf-8') self.udp_socket.sendto(data, (self._host, self._udp_port))
def test_string_val_newline(self): data = { "points": [ { "measurement": "m1", "fields": { "multi_line": "line1\nline1\nline3" } } ] } self.assertEqual( line_protocol.make_lines(data), 'm1 multi_line="line1\\nline1\\nline3"\n' )
def test_float_with_long_decimal_fraction(self): """Ensure precision is preserved when casting floats into strings.""" data = { "points": [ { "measurement": "test", "fields": { "float_val": 1.0000000000000009, } } ] } self.assertEqual( line_protocol.make_lines(data), 'test float_val=1.0000000000000009\n' )
def test_string_val_newline(self): """Test string value with newline in TestLineProtocol object.""" data = { "points": [ { "measurement": "m1", "fields": { "multi_line": "line1\nline1\nline3" } } ] } self.assertEqual( line_protocol.make_lines(data), 'm1 multi_line="line1\\nline1\\nline3"\n' )
def test_make_lines_unicode(self): data = { "tags": { "unicode_tag": "\'Привет!\'" # Hello! in Russian }, "points": [ { "measurement": "test", "fields": { "unicode_val": "Привет!", # Hello! in Russian } } ] } self.assertEqual( line_protocol.make_lines(data), 'test,unicode_tag=\'Привет!\' unicode_val="Привет!"\n' )
def test_empty_tag(self): data = { "tags": { "my_tag": "" }, "points": [ { "measurement": "test", "fields": { "value": "hello!" } } ] } self.assertEqual( line_protocol.make_lines(data), 'test value="hello!"\n' )
def write(self, data, params=None, expected_response_code=204, protocol='json'): """Write data to InfluxDB. :param data: the data to be written :type data: (if protocol is 'json') dict (if protocol is 'line') sequence of line protocol strings or single string :param params: additional parameters for the request, defaults to None :type params: dict :param expected_response_code: the expected response code of the write operation, defaults to 204 :type expected_response_code: int :param protocol: protocol of input data, either 'json' or 'line' :type protocol: str :returns: True, if the write operation is successful :rtype: bool """ headers = self._headers headers['Content-Type'] = 'application/octet-stream' if params: precision = params.get('precision') else: precision = None if protocol == 'json': data = make_lines(data, precision).encode('utf-8') elif protocol == 'line': if isinstance(data, str): data = [data] data = ('\n'.join(data) + '\n').encode('utf-8') self.request( url="write", method='POST', params=params, data=data, expected_response_code=expected_response_code, headers=headers ) return True
def write(self, data, params=None, expected_response_code=204, protocol='json'): """Write data to InfluxDB. :param data: the data to be written :type data: (if protocol is 'json') dict (if protocol is 'line') sequence of line protocol strings :param params: additional parameters for the request, defaults to None :type params: dict :param expected_response_code: the expected response code of the write operation, defaults to 204 :type expected_response_code: int :param protocol: protocol of input data, either 'json' or 'line' :type protocol: str :returns: True, if the write operation is successful :rtype: bool """ headers = self._headers headers['Content-type'] = 'application/octet-stream' if params: precision = params.get('precision') else: precision = None if protocol == 'json': data = make_lines(data, precision).encode('utf-8') elif protocol == 'line': data = ('\n'.join(data) + '\n').encode('utf-8') self.request( url="write", method='POST', params=params, data=data, expected_response_code=expected_response_code, headers=headers ) return True
def test_make_lines(self): data = { "tags": { "empty_tag": "", "none_tag": None, "integer_tag": 2, "string_tag": "hello" }, "points": [{ "measurement": "test", "fields": { "string_val": "hello!", "int_val": 1, "none_field": None, } }] } self.assertEqual( line_protocol.make_lines(data), 'test,integer_tag=2,string_tag=hello ' 'int_val=1,string_val="hello!"\n')
def datagram_received(self, raw_data, addr): logging.info('Received %s bytes: %r(...) from %s', len(raw_data), raw_data[0:3], addr) if len(raw_data) < 7: return identifier = raw_data[0:3] serializer = self._factory.get_serializer(identifier) mesg, fields = serializer.deserialize(raw_data, self._max_delta_t) # Verify nonce is not known for that timestamp if mesg.timestamp in self._known_nonces.keys() and \ mesg.nonce in self._known_nonces[mesg.timestamp]: raise Exception( 'Possible replay attack: Nonce {} already knwon for timestamp {}' .format(mesg.nonce, mesg.timestamp)) else: if not mesg.timestamp in self._known_nonces.keys(): self._known_nonces[mesg.timestamp] = set() self._known_nonces[mesg.timestamp].add(mesg.nonce) influxdb_points = [] for key, value in fields.items(): influxdb_points.append({ 'measurement': key, 'tags': {}, 'fields': dict(value) }) post_data = line_protocol.make_lines({ 'points': influxdb_points }).encode() asyncio.ensure_future( send(self._influx_url + '?db=' + serializer.database, post_data)) self.cleanup_known_nonces()
def export_to_line_protocol(self): """ Export object to InfluxDB Line Protocol syntax """ # Device self.add_tag(['device', 'mode']) self.add_tag(['device', 'state']) self.add_tags(['device', 'additionalData'], prefix=['device']) # Part self.add_tag(['part', 'id']) self.add_tag(['part', 'type']) self.add_tag(['part', 'typeId']) self.add_tag(['part', 'code']) self.add_tag(['part', 'result']) self.add_tags(['part', 'additionalData'], prefix=['part']) # TimeMeasurements for measurement in self.data['measurements']: # create new Measurement object to store local tags meas_obj = Measurement(json.dumps(measurement), self.hostname()) meas_obj.add_tag(['code'], ['measurement']) meas_obj.add_tag(['result'], ['measurement']) meas_obj.add_tags(['additionalData'], prefix=['measurement']) # TODO: context timestamp = parser.parse(measurement['ts']) keys = list( filter(lambda key: key != 'time', measurement['series'].keys())) fields = dict() self.points = [] for index in range(0, len(measurement['series'][keys[0]])): fields = { key.replace(' ', '_'): measurement['series'][key][index] for key in keys if key != '' and measurement['series'][key][index] != '' } # add offset to base timestamp ts_w_offset = timestamp + timedelta( milliseconds=measurement['series']['time'][index]) # round to InfluxDB compatible timestamp (nanoseconds) ts_w_offset = int(round(ts_w_offset.timestamp() * 1000000000)) self.add_point(fields, ts_w_offset) # merge global ProcessPayload tags with current ProcessMeasurement tags current_tags = self.tags.copy() current_tags.update(meas_obj.tags) # add measurement in line_protocol format if len(fields) > 0: tmp = line_protocol.make_lines({ 'tags': current_tags, 'points': self.points }) self.line_protocol_data += tmp # return sequence of line protocol strings return self.line_protocol_data if self.line_protocol_data else None
def read_nmea2k(): """Read the actisense-serial -r {device} | analyzer -json for given NMEA2000 NGT-1 device port""" # Actisense-Serial actisense_process = Popen(['actisense-serial', '-r', n2k_conf['port']], stdout=PIPE) # Analyzer Stream for output in JSON global analyzer_process analyzer_process = Popen(['analyzer', '-json'], stdin=actisense_process.stdout, stdout=PIPE, stderr=PIPE) PGNs = list(map(int, n2k_conf['pgnConfigs'].keys())) logger.debug('PGNs: {}'.format(PGNs)) while True: incoming_json = analyzer_process.stdout.readline().decode('utf-8') try: incoming_data = json.loads(incoming_json) if incoming_data['pgn'] in PGNs: # remove unnecessary keys del incoming_data['dst'] del incoming_data['prio'] # check if the configuration for the PGN has the `fromSource` Key if 'fromSource' in list(n2k_conf['pgnConfigs'][str( incoming_data['pgn'])].keys()): logger.info('PGN Source Filter Check') if incoming_data['src'] != n2k_conf['pgnConfigs'][str( incoming_data['pgn'])]['fromSource']: logger.info('PGN: {} with src: {}'.format( incoming_data['pgn'], incoming_data['src'])) logger.info('Skipping data for: {}'.format( incoming_data['description'])) continue measurement = { "tags": { "source": "nmea2k", "PGN": incoming_data['pgn'], "src": incoming_data['src'] }, "points": [] } # Create a set of all available fields from the incoming frame incoming_fields = set(incoming_data['fields'].keys()) fields_from_conf = set(n2k_conf['pgnConfigs'][str( incoming_data['pgn'])]['fieldLabels']) logger.debug('Fields To Log: {f}'.format( f=fields_from_conf.intersection(incoming_fields))) # Get all the Fields necessary to be stored into InfluxDB for selected_field in fields_from_conf.intersection( incoming_fields): # Measurement name is the profile type name e.g. control/environment/engine etc available # as the first level for the mqtt topics meas_name = n2k_conf['pgnConfigs'][str( incoming_data['pgn'])]['topics'][0].split('/')[0] point = { "measurement": meas_name, "time": int(time.time() * 1e9), "fields": {} } point['fields'][selected_field.replace( " ", "")] = incoming_data['fields'][selected_field] measurement['points'].append(point) # logger.debug(line_protocol.make_lines(measurement)) with concurrent.futures.ThreadPoolExecutor( max_workers=2) as executor: if executor.submit(save_to_db, measurement).result(): logger.info('saved to InfluxDB') if executor.submit( publish_data, incoming_data['pgn'], line_protocol.make_lines(measurement)).result(): logger.info('Published data successfully') time.sleep(0.05) except Exception as e: logger.exception(e)
def read_from_imu(i2c_port, updaterate): logger.info('Starting to Read BNO values on {} every {}s'.format( i2c_port, updaterate)) global sensor_bno sensor_bno = BNO055(i2c_bus_port=i2c_port) if sensor_bno.begin() is not True: raise ValueError('Initialization Failure for BNO055') sys.exit(1) time.sleep(1) sensor_bno.setExternalCrystalUse(True) time.sleep(2) measurement = { "tags": { "source": "imu" }, "points": [{ "measurement": "acceleration", "fields": { "liX": -10000, "liY": -10000, "liZ": -10000 } }, { "measurement": "acceleration", "fields": { "gX": -10000, "gY": -10000, "gZ": -10000 } }, { "measurement": "orientation", "fields": { "yaw": -10000 } }, { "measurement": "orientation", "fields": { "pitch": -10000 } }, { "measurement": "orientation", "fields": { "roll": -10000 } }] } logger.info('reading sensor information') while True: try: timestamp = int(time.time() * 1e9) lx, ly, lz = sensor_bno.getVector(BNO055.VECTOR_LINEARACCEL) measurement['points'][0]['fields']['liX'] = lx measurement['points'][0]['fields']['liY'] = ly measurement['points'][0]['fields']['liZ'] = lz logger.debug('linear acc.: x:{}, y:{}, z:{}'.format(lx, ly, lz)) gX, gY, gZ = sensor_bno.getVector(BNO055.VECTOR_GRAVITY) measurement['points'][1]['fields']['gX'] = gX measurement['points'][1]['fields']['gY'] = gY measurement['points'][1]['fields']['gZ'] = gZ logger.debug('gravity: x:{}, y:{}, z:{}'.format(gX, gY, gZ)) yaw, roll, pitch = sensor_bno.getVector(BNO055.VECTOR_EULER) measurement['points'][2]['fields']['yaw'] = yaw measurement['points'][3]['fields']['pitch'] = pitch measurement['points'][4]['fields']['roll'] = roll logger.debug('euler: yaw:{}, pitch:{}, roll:{}'.format( yaw, pitch, roll)) for point in measurement['points']: # insert timestamp to each point point['time'] = timestamp with concurrent.futures.ThreadPoolExecutor( max_workers=2) as executor: if executor.submit(save_to_db, measurement).result(): logger.info('saved data to InfluxDB') if executor.submit( publish_data, line_protocol.make_lines(measurement, precision='ns')).result(): logger.info('published data to MQTT broker') time.sleep(updaterate) except Exception as imu_e: logger.error('Error while reading IMU data: {}'.format(imu_e)) client.close() sys.exit(2)
def export_to_line_protocol(self): """ Export object to InfluxDB Line Protocol syntax """ """data = { "tags": { "empty_tag": "", "none_tag": None, "backslash_tag": "C:\\", "integer_tag": 2, "string_tag": "hello" }, "points": [ { "measurement": "test", "fields": { "string_val": "hello!", "int_val": 1, "float_val": 1.1, "none_field": None, "bool_val": True, }, "time": 0 } ] }""" # Device self.add_tag(['device', 'mode']) self.add_tag(['device', 'state']) self.add_tags(['device', 'additionalData'], prefix=['device']) # Part self.add_tag(['part', 'id']) self.add_tag(['part', 'type']) self.add_tag(['part', 'typeId']) self.add_tag(['part', 'code']) self.add_tag(['part', 'result']) self.add_tags(['part', 'additionalData'], prefix=['part']) # TODO: Process # ProcessMeasurements for measurement in self.data['measurements']: # create new Measurement object to store local tags meas_obj = Measurement(json.dumps(measurement), self.hostname()) meas_obj.add_tag(['code'], ['process']) meas_obj.add_tag(['name'], ['process']) meas_obj.add_tag(['phase'], ['process']) meas_obj.add_tag(['result'], ['process']) meas_obj.add_tags(['additionalData'], prefix=['process']) # TODO: context # TODO: specialValues timestamp = parser.parse(measurement['ts']) keys = list( filter(lambda key: key != 'time', measurement['series'].keys())) self.points = [] for index in range(0, len(measurement['series'][keys[0]])): fields = { key.replace(' ', '_'): measurement['series'][key][index] for key in keys if key != '' and measurement['series'][key][index] != '' } # add offset to base timestamp ts_w_offset = timestamp + timedelta( milliseconds=measurement['series']['time'][index]) # round to InfluxDB compatible timestamp (nanoseconds) ts_w_offset = int(round(ts_w_offset.timestamp() * 1000000000)) self.add_point(fields, ts_w_offset) # merge global ProcessPayload tags with current ProcessMeasurement tags current_tags = self.tags.copy() current_tags.update(meas_obj.tags) # add measurement in line_protocol format if len(fields) > 0: tmp = line_protocol.make_lines({ 'tags': current_tags, 'points': self.points }) self.line_protocol_data += tmp # return sequence of line protocol strings return self.line_protocol_data if self.line_protocol_data else None
def test_timezone(self): """Test timezone in TestLineProtocol object.""" dt = datetime(2009, 11, 10, 23, 0, 0, 123456) utc = UTC.localize(dt) berlin = timezone('Europe/Berlin').localize(dt) eastern = berlin.astimezone(timezone('US/Eastern')) data = { "points": [ { "measurement": "A", "fields": { "val": 1 }, "time": 0 }, { "measurement": "A", "fields": { "val": 1 }, "time": "2009-11-10T23:00:00.123456Z" }, { "measurement": "A", "fields": { "val": 1 }, "time": dt }, { "measurement": "A", "fields": { "val": 1 }, "time": utc }, { "measurement": "A", "fields": { "val": 1 }, "time": berlin }, { "measurement": "A", "fields": { "val": 1 }, "time": eastern }, ] } self.assertEqual( line_protocol.make_lines(data), '\n'.join([ 'A val=1i 0', 'A val=1i 1257894000123456000', 'A val=1i 1257894000123456000', 'A val=1i 1257894000123456000', 'A val=1i 1257890400123456000', 'A val=1i 1257890400123456000', ]) + '\n')
def test_request_log_useragent(self): lines = self.check_length('full-request-log-malicious-useragent.json', 2) make_lines({'points': lines})