def test_first(self):
tq = TransmitQueue(4)
self._fill(tq, 4)
data = tq.first_entries(2)
assert data[0]['data'] == 1
assert data[1]['data'] == 2
def test_remove_first(self):
tq = TransmitQueue(4)
self._fill(tq, 4)
tq.remove_first(2)
data = tq.all_entries()
assert data[0]['data'] == 3
assert data[1]['data'] == 4
def test_get_more_than_exist(self):
tq = TransmitQueue(2)
self._fill(tq, 2)
data = tq.first_entries(3)
print(data)
assert len(data) == 2
assert data[0]['data'] == 1
assert data[1]['data'] == 2
def test_get_all(self):
tq = TransmitQueue(4)
self._fill(tq, 4)
data = tq.all_entries()
assert tq.num_entries() == 4
assert data[0]['data'] == 1
assert data[1]['data'] == 2
assert data[2]['data'] == 3
assert data[3]['data'] == 4
def test_remove_more_than_present(self):
tq = TransmitQueue(4)
self._fill(tq, 2)
tq.remove_first(3)
assert tq.num_entries() == 0
tq.remove_first(99)
assert tq.num_entries() == 0
def test_limit(self):
tq = TransmitQueue(2)
self._fill(tq, 3)
assert tq.num_entries() == 2
data = tq.all_entries()
assert data[0]['data'] == 2
assert data[1]['data'] == 3
tq.add(4)
assert tq.num_entries() == 2
data = tq.all_entries()
assert data[0]['data'] == 3
assert data[1]['data'] == 4
def __init__(self, model):
super().__init__()
assert Things.instance is None
Things.instance = self
self.model = model
self.state = 'init'
self.active = False
self.config = configparser.ConfigParser()
try:
self.config.read('/etc/thingsboard.conf')
self.api_server = self.config.get('API', 'Server')
self.api_token = self.config.get('API', 'Token')
self.has_server = True
except configparser.Error as e:
logger.warning('ERROR: Cannot get Thingsboard config')
logger.info(e)
self.has_server = False
self._attributes_queue = TransmitQueue(1)
self._data_queue = TransmitQueue(self.MAX_QUEUE_SIZE)
self._data_collector = ThingsDataCollector(model, self._data_queue, self._attributes_queue)
def test_empty(self):
tq = TransmitQueue(3)
assert tq.num_entries() == 0
def test_add_num_entries(self):
tq = TransmitQueue(3)
tq.add(1)
assert tq.num_entries() == 1
tq.add(2)
assert tq.num_entries() == 2
tq.add(3)
assert tq.num_entries() == 3
class Things(threading.Thread):
# Singleton accessor
instance = None
# Upload every 30 seconds
TELEMETRY_UPLOAD_PERIOD = 30
# Upload at most 120 entries. Assuming around 100 bytes per entry,
# this makes 12 kBytes.
TELEMETRY_MAX_ITEMS_TO_UPLOAD = 120
# New entries are dropped when this size is reached
# Assumption is that this queue size is good for 10 minutes
MAX_QUEUE_SIZE = 600
def __init__(self, model):
super().__init__()
assert Things.instance is None
Things.instance = self
self.model = model
self.state = 'init'
self.active = False
self.config = configparser.ConfigParser()
try:
self.config.read('/etc/thingsboard.conf')
self.api_server = self.config.get('API', 'Server')
self.api_token = self.config.get('API', 'Token')
self.has_server = True
except configparser.Error as e:
logger.warning('ERROR: Cannot get Thingsboard config')
logger.info(e)
self.has_server = False
self._attributes_queue = TransmitQueue(1)
self._data_queue = TransmitQueue(self.MAX_QUEUE_SIZE)
self._data_collector = ThingsDataCollector(model, self._data_queue, self._attributes_queue)
def setup(self):
self.daemon = True
if self.has_server:
self.start()
def enable(self, enable):
if enable:
if self.has_server:
if not self.active:
logger.info("service starting")
self._data_collector.enable()
self.active = True
self.state = 'init'
res = 'Started cloud logger'
else:
res = 'Cloud logger already running'
else:
res = 'Cannot start. No configuration present'
else:
logger.info("service stopping")
if self.active:
self._data_collector.disable()
self.active = False
res = 'Stopped cloud logger'
else:
res = 'Cloud logger not running'
self.model.publish('cloud', self.active)
return res
def run(self):
cnt = 0
while True:
if self.active:
next_state = self.state
md = self.model.get_all()
if self.state != 'connected':
# Check if we are connected
if 'modem' in md:
m = md['modem']
if 'modem-id' in m:
if 'bearer-id' in m:
# logger.info('link ready, changing to connected state')
cnt = 0
next_state = 'connected'
elif self.state == 'connected':
# Upload any pending data
self._upload_attributes()
if cnt % Things.TELEMETRY_UPLOAD_PERIOD == 5:
self._upload_telemetry()
# TODO: check for error and switch to disconnected state in case of problem
# state change
if self.state != next_state:
logger.info(f'changed state from {self.state} to {next_state}')
self.state = next_state
cnt += 1
time.sleep(1.0)
def _upload_telemetry(self):
"""
Sends telemtry data
Checks for entries in _data_queue If entries are present, gets up to
TELEMETRY_MAX_ITEMS_TO_UPLOAD entries and tries to upload them. If upload is ok,
removes entries from queue. Otherwise leaves entries for next try.
"""
# Are there any entries at all?
queue_entries = self._data_queue.num_entries()
if queue_entries >= 1:
# On every upload report current queue size
data = {'tb-qsize': queue_entries}
self._data_queue.add(data)
# Build HTTP query string with queue data
entries = self._data_queue.first_entries(Things.TELEMETRY_MAX_ITEMS_TO_UPLOAD)
num_entries = len(entries)
assert len(entries) >= 0
post_data = list()
for entry in entries:
data = {'ts': entry['time'], 'values': entry['data']}
post_data.append(data)
# Upload the collected data
res = self._post_data('telemetry', post_data)
if res:
# Transmission was ok, remove data from queue
self._data_queue.remove_first(num_entries)
logger.debug(f'removing {num_entries} entries from queue')
else:
logger.warning('could not upload telemetry data, keeping in queue')
logger.warning(f'{queue_entries} entries in queue')
def _upload_attributes(self):
"""
Upload a single attribute entry.
Assumes all attributes are in one entry
TODO: Rework to allow more than one entry, combine code with _upload_telemetry
"""
if self._attributes_queue.num_entries() >= 1:
entry = self._attributes_queue.all_entries()[0]
post_data = entry['data']
res = self._post_data('attributes', post_data)
if res:
# Transmission was ok, remove data from queue
self._attributes_queue.remove_first(1)
else:
logger.warning('could not upload attribute data, keeping in queue')
def _post_data(self, msgtype, payload):
"""
Sends data with HTTP(S) POST request to Thingsboard server
Captures pycurl exceptions and checks for 200 (OK) response
from server.
TODO:
Check timeout behavior. While we are transmitting data is not captured and can get lost!
Ideally this method would run in it's own thread with transmit queue
"""
res = False
assert msgtype == 'attributes' or msgtype == 'telemetry'
c = pycurl.Curl()
c.setopt(pycurl.URL, f'{self.api_server}/api/v1/{self.api_token}/{msgtype}')
c.setopt(pycurl.HTTPHEADER, ['Content-Type:application/json'])
c.setopt(pycurl.POST, 1)
c.setopt(pycurl.CONNECTTIMEOUT_MS, 2000)
c.setopt(pycurl.TIMEOUT_MS, 3000)
# c.setopt(c.VERBOSE, True)
body_as_json_string = json.dumps(payload) # dict to json
body_as_json_bytes = body_as_json_string.encode()
body_as_file_object = BytesIO(body_as_json_bytes)
# prepare and send. See also: pycurl.READFUNCTION to pass function instead
c.setopt(pycurl.READDATA, body_as_file_object)
c.setopt(pycurl.POSTFIELDSIZE, len(body_as_json_string))
try:
info = dict()
info['state'] = 'sending'
self.model.publish('things', info)
c.perform()
bytes_sent = len(body_as_json_bytes)
logger.debug(f'sent {bytes_sent} to {self.api_server}')
info['state'] = 'sent'
info['bytes'] = bytes_sent
self.model.publish('things', info)
response = int(c.getinfo(pycurl.RESPONSE_CODE))
logger.debug(f'got response {response} from server')
if response == 200:
bytes_sent = int(c.getinfo(pycurl.CONTENT_LENGTH_UPLOAD))
logger.debug(f'{bytes_sent} bytes uploaded')
res = True
else:
logger.warning(f'bad HTTP response {response} received')
except pycurl.error as e:
logger.warning("failed uploading data to Thingsboard")
logger.warning(e)
finally:
c.close()
return res