def test_dao_sampler(self):
dao_sql_fname = '/tmp/rpiparticle/%d__.sql' % randint(2**42, 2**52)
dao = FriskbyDao(dao_sql_fname)
sampler = FriskbySampler(MockSDS011('/path/to/dev'), dao,
sample_time=0.1, sleep_time=0.1)
sampler.collect()
data = dao.get_non_uploaded(limit=30)
self.assertTrue(len(data) > 0)
def test_dao_sampler(self):
dao_sql_fname = '/tmp/rpiparticle/%d__.sql' % randint(2**42, 2**52)
dao = FriskbyDao(dao_sql_fname)
sampler = FriskbySampler(MockSDS011('/path/to/dev'),
dao,
sample_time=0.1,
sleep_time=0.1)
sampler.collect()
data = dao.get_non_uploaded(limit=30)
self.assertTrue(len(data) > 0)
def setUp(self):
self.dao = FriskbyDao(self._temp_fname('_db.sql'))
cfg_fname = self._temp_fname('_.cfg')
url_ = "https://friskby.herokuapp.com/sensor/api/device/FriskPITest/"
deviceconfig = DeviceConfig.download(url_, post_key="xxx")
deviceconfig.save(filename=cfg_fname)
self.cfg = deviceconfig
class SamplerTest(TestCase):
def setUp(self):
tmpf = temp(delete=False)
self.fname = tmpf.name + '_db.sql'
tmpf.close()
self.dao = FriskbyDao(self.fname)
def test_sampler(self):
sample_time = 2
sleep_time = 0.10
start = dt.now()
sampler = FriskbySampler(MockSDS011('/path/to/dev'),
self.dao,
sample_time=sample_time,
sleep_time=sleep_time)
sampler.collect()
stop = dt.now()
delta = stop - start
self.assertTrue((delta.total_seconds() - sample_time) > 0)
data = self.dao.get_non_uploaded(limit=30)
self.assertTrue(len(data) > 0)
def test_create_directory_structure(self):
_tmp_dir = '/tmp/friskby/%d/no/such/dir/here' % randint(2**32, 2**42)
_fpath = '%s/friskby.sql' % _tmp_dir
dao = FriskbyDao(_fpath)
sqlpath = dao.get_path()
self.assertEqual(_fpath, sqlpath)
t10 = gen_rand_ts(value=26.8)
t25 = gen_rand_ts(value=19.90)
data = {'PM10': t10, 'PM25': t25}
dao.persist_ts(data)
# data: 2 elts of (id, value, sensor, timestamp, upl)
data = dao.get_non_uploaded()
self.assertEqual(2, len(data))
db_10, db_25 = data
if db_10[2] != 'PM10':
db_25, db_10 = data
self.assertEqual(26.8, db_10[1])
self.assertEqual(19.9, db_25[1])
def setUp(self):
tmpf = temp(delete=False)
self.fname = tmpf.name + '_db.sql'
tmpf.close()
self.dao = FriskbyDao(self.fname)
class DaoTest(TestCase):
def setUp(self):
tmpf = temp(delete=False)
self.fname = tmpf.name + '_db.sql'
tmpf.close()
self.dao = FriskbyDao(self.fname)
def test_created(self):
q = "SELECT name FROM sqlite_master WHERE type='table';"
conn = sqlite3.connect(self.fname)
c = conn.execute(q)
result = c.fetchall()
self.assertEqual(1, len(result))
self.assertEqual('samples', result[0][0])
def _do_test_num_upl(self, dao, exp_num_all, exp_num_upl, exp_num_nup):
num_all, num_upl, num_nup = (dao.get_num_rows(),
dao.get_num_rows(uploaded_status=True),
dao.get_num_rows(uploaded_status=False))
self.assertEqual(exp_num_all, num_all)
self.assertEqual(exp_num_upl, num_upl)
self.assertEqual(exp_num_nup, num_nup)
def test_persist(self):
num_data = 13
for _ in range(num_data):
t10 = gen_rand_ts()
t25 = gen_rand_ts()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
self._do_test_num_upl(self.dao, 2 * num_data, 0, 2 * num_data)
data = self.dao.get_non_uploaded(limit=30)
self.assertEqual(2 * num_data, len(data))
def test_mark_uploaded(self):
num_data = 17
for _ in range(num_data):
t10 = gen_rand_ts()
t25 = gen_rand_ts()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
print(repr(self.dao))
self._do_test_num_upl(self.dao, 2 * num_data, 0, 2 * num_data)
data = self.dao.get_recent_samples(limit=30, uploaded=True)
self.assertEqual(0, len(data))
data = self.dao.get_recent_samples(limit=30, uploaded=None)
self.assertEqual(30, len(data))
data = self.dao.get_recent_samples(limit=30, uploaded=False)
self.assertEqual(30, len(data))
data = self.dao.get_non_uploaded(limit=30)
self.assertEqual(30, len(data))
self.dao.mark_uploaded(data)
data = self.dao.get_non_uploaded(limit=30)
self.assertEqual(4, len(data)) # total 34, marked 30
data = self.dao.get_recent_samples(limit=30, uploaded=False)
self.assertEqual(4, len(data))
data = self.dao.get_recent_samples(limit=30, uploaded=True)
self.assertEqual(30, len(data))
data = self.dao.get_recent_samples(limit=40, uploaded=None)
self.assertEqual(34, len(data))
print(repr(self.dao))
self.assertTrue(repr(self.dao).startswith('FriskbyDao'))
# test num / num upl / num non-upl
self._do_test_num_upl(self.dao, 2 * num_data, 30, 2 * num_data - 30)
def test_localtime(self):
t10 = gen_rand_ts()
t25 = gen_rand_ts()
now = dt.now()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
out = self.dao.get_non_uploaded(limit=1)[0]
delta = now - out[3]
# checking that we're in the same timezone
self.assertTrue(abs(delta.total_seconds()) < 1000)
def test_create_directory_structure(self):
_tmp_dir = '/tmp/friskby/%d/no/such/dir/here' % randint(2**32, 2**42)
_fpath = '%s/friskby.sql' % _tmp_dir
dao = FriskbyDao(_fpath)
sqlpath = dao.get_path()
self.assertEqual(_fpath, sqlpath)
t10 = gen_rand_ts(value=26.8)
t25 = gen_rand_ts(value=19.90)
data = {'PM10': t10, 'PM25': t25}
dao.persist_ts(data)
# data: 2 elts of (id, value, sensor, timestamp, upl)
data = dao.get_non_uploaded()
self.assertEqual(2, len(data))
db_10, db_25 = data
if db_10[2] != 'PM10':
db_25, db_10 = data
self.assertEqual(26.8, db_10[1])
self.assertEqual(19.9, db_25[1])
def test_last_entry(self):
self.assertIsNone(self.dao.last_entry(uploaded=True))
self.assertIsNone(self.dao.last_entry(uploaded=False))
self.assertIsNone(self.dao.last_entry())
t10 = gen_rand_ts()
t25 = gen_rand_ts()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
self.assertIsNone(self.dao.last_entry(uploaded=True))
self.assertIsNotNone(self.dao.last_entry(uploaded=False))
self.assertIsNotNone(self.dao.last_entry())
data = self.dao.get_non_uploaded(limit=30)
self.dao.mark_uploaded(data)
self.assertIsNotNone(self.dao.last_entry(uploaded=True))
self.assertIsNone(self.dao.last_entry(uploaded=False))
self.assertIsNotNone(self.dao.last_entry())
t10 = gen_rand_ts()
t25 = gen_rand_ts()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
self.assertIsNotNone(self.dao.last_entry(uploaded=True))
self.assertIsNotNone(self.dao.last_entry(uploaded=False))
self.assertIsNotNone(self.dao.last_entry())
def __init__(self):
self.systemd = SystemdDBus()
self.dao = FriskbyDao(fby_settings.get_setting("rpi_db"))
# Dict from e.g. http://friskby.herokuapp.com/sensor/api/device/foo
self._device_info = None
class FriskbyInterface():
known_units = {
'friskby': 'friskby.service',
'sampler': 'friskby-sampler.service',
'submitter': 'friskby-submitter.service',
'friskby_controlpanel': 'friskby-controlpanel.service',
'salt-minion': 'salt-minion.service',
}
def __init__(self):
self.systemd = SystemdDBus()
self.dao = FriskbyDao(fby_settings.get_setting("rpi_db"))
# Dict from e.g. http://friskby.herokuapp.com/sensor/api/device/foo
self._device_info = None
def _service_to_unit(self, service):
"""Returns a unit or None if the service wasn't pertinent to the
friskby system."""
return self.known_units.get(service, None)
def download_and_save_config(self, url, filename):
"""Downloads config from url and saves it to the given filename."""
config = DeviceConfig.download(url)
config.save(filename=filename)
def get_service_status(self, service):
"""Returns the unit status as defined by [1]. Only services pertinent
to the friskby system will be considered.
[1] https://www.freedesktop.org/wiki/Software/systemd/dbus/
"""
unit = self._service_to_unit(service)
if service is None:
raise KeyError('%s is not a pertinent service.' % service)
status = self.systemd.get_unit_status(unit)
try:
return status[3]
except TypeError:
return None
def get_service_journal(self, service, limit=50):
"""Returns a list of dicts containing output from journalctl where unit
is the given service. Only services pertinent to the friskby system
will be considered.
Limit is the maximum amount of log lines to be returned. None implies
no limit.
See https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html # noqa
for information on what the list of dicts contains.
"""
unit = self._service_to_unit(service)
if service is None:
raise KeyError('%s is not a pertinent service.' % service)
output = None
lines = []
args = [
"journalctl",
"--unit=%s" % unit, "--catalog", "--full", "-o", "json"
]
if limit is None:
args = args + ["--lines", "all"]
else:
args = args + ["--lines", str(limit)]
try:
output = subprocess.check_output(args)
except subprocess.CalledProcessError as e:
# This means we got a non-zero exit code from journalctl. We can
# do naught but log.
print("Failed to capture journalctl output for %s:"
" %s exited with %d.\nOutput:%s" %
(unit, e.cmd, int(e.returncode), e.output))
sys.stdout.flush()
else: # No process error, so let's take a look at the output
for line in output.split("\n"):
if line != "":
js = json.loads(line)
lines.append(js)
lines.reverse()
return lines
def get_device_id_and_api_key(self, config_file):
# Returns a tuple consisting of device ID and API key.
config = None
try:
config = DeviceConfig(config_file)
except IOError:
return (None, None)
device_id = config.getDeviceID()
post_key = config.getPostKey()
if device_id == "" or device_id is None:
device_id = None
if post_key == "" or post_key is None:
post_key = None
return (device_id, post_key)
def get_uploaded_samples_count(self):
fetch_uploaded = True
return self.dao.get_num_rows(fetch_uploaded)
def get_all_samples_count(self):
return self.dao.get_num_rows()
def get_most_recently_uploaded(self):
fetch_uploaded_entry = True
return self.dao.last_entry(fetch_uploaded_entry)
def get_most_recently_sampled(self):
return self.dao.last_entry()
def get_recent_samples(self):
data = self.dao.get_recent_samples(limit=10)
return data
@staticmethod
def get_socket_iface_name():
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("friskby.no", 80))
sockname = s.getsockname()[0]
s.close()
return sockname
except socket.error:
return None
def manage_service(self, service, action):
unit = self._service_to_unit(service)
if action == 'restart':
self.systemd.restart_unit(unit)
elif action == 'start':
self.systemd.start_unit(unit)
elif action == 'stop':
self.systemd.stop_unit(unit)
else:
raise KeyError('%s is not an action I know of...' % action)
def manage(self, action):
for service in self.known_units.keys():
self.manage_service(service, action)
def get_settings(self):
return fby_settings.get_settings()
def set_settings(self, settings):
fby_settings.set_setting('rpi_sample_time',
settings['rpi_sample_time'])
fby_settings.set_setting('rpi_control_panel_host',
settings['rpi_control_panel_host'])
fby_settings.set_setting('rpi_control_panel_port',
settings['rpi_control_panel_port'])
fby_settings.set_setting('rpi_sds011', settings['rpi_sds011'])
def set_location(self, lat, lon, altitude, name, uri, api_key):
r = None
try:
r = requests.get(uri,
params={
'key': api_key,
'latitude': lat,
'longitude': lon,
'altitude': altitude,
'name': name
})
except requests.exceptions.ConnectionError as e:
raise RuntimeError('Failed to set location: %s' % e)
if r is not None and r.status_code != 200:
print('Failed to set location: %s:' % r.text)
sys.stdout.flush()
raise RuntimeError(
'Failed to set location, friskby responded with %d.' %
r.status_code)
def get_device_info(self, uri):
r = None
try:
r = requests.get(uri)
except requests.exceptions.ConnectionError:
return None
if r.status_code != 200:
return None
return r.json()
class DaoTest(TestCase):
def setUp(self):
tmpf = temp(delete=False)
self.fname = tmpf.name + '_db.sql'
tmpf.close()
self.dao = FriskbyDao(self.fname)
def test_created(self):
q = "SELECT name FROM sqlite_master WHERE type='table';"
conn = sqlite3.connect(self.fname)
c = conn.execute(q)
result = c.fetchall()
self.assertEqual(1, len(result))
self.assertEqual('samples', result[0][0])
def _do_test_num_upl(self, dao, exp_num_all, exp_num_upl, exp_num_nup):
num_all, num_upl, num_nup = (dao.get_num_rows(),
dao.get_num_rows(uploaded_status=True),
dao.get_num_rows(uploaded_status=False))
self.assertEqual(exp_num_all, num_all)
self.assertEqual(exp_num_upl, num_upl)
self.assertEqual(exp_num_nup, num_nup)
def test_persist(self):
num_data = 13
for _ in range(num_data):
t10 = gen_rand_ts()
t25 = gen_rand_ts()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
self._do_test_num_upl(self.dao, 2*num_data, 0, 2*num_data)
data = self.dao.get_non_uploaded(limit=30)
self.assertEqual(2*num_data, len(data))
def _help_test_recent_samples(self, data, idx, upl):
"""Test that data has values idx+0.1 and idx+0.25 and correct upl stat"""
err_val = 'Sample should be %d.1 or %d.025, was %.3f'
d0, d1 = data[0], data[1]
err_upl = 'Wrong upload status, should have been %s'
self.assertTrue(idx < d0[1] < 1+idx, msg=err_val % (idx, idx+1, d0[1]))
self.assertTrue(idx < d1[1] < 1+idx, msg=err_val % (idx, idx+1, d1[1]))
self.assertEqual(d0[4], upl, msg=err_upl % upl)
self.assertEqual(d1[4], upl, msg=err_upl % upl)
def test_recent_samples(self):
num_data = 17
for i in range(num_data):
t10 = gen_rand_ts(value=i + 0.100)
t25 = gen_rand_ts(value=i + 0.025)
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
# dao has values [..., 14.1, 14.025, 15.1, 15.025, 16.1, 16.025]
idx = 16
data = self.dao.get_recent_samples(limit=2)
self._help_test_recent_samples(data, idx, False)
self.dao.mark_uploaded(data)
data = self.dao.get_recent_samples(limit=2)
self._help_test_recent_samples(data, idx, True)
data = self.dao.get_recent_samples(limit=2, uploaded=True)
self._help_test_recent_samples(data, idx, True)
idx = 15
data = self.dao.get_recent_samples(limit=2, uploaded=False)
self._help_test_recent_samples(data, idx, False)
self.dao.mark_uploaded(data)
idx = 14
data = self.dao.get_recent_samples(limit=2, uploaded=False)
self._help_test_recent_samples(data, idx, False)
def test_mark_uploaded(self):
num_data = 17
for _ in range(num_data):
t10 = gen_rand_ts()
t25 = gen_rand_ts()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
print(repr(self.dao))
self._do_test_num_upl(self.dao, 2*num_data, 0, 2*num_data)
data = self.dao.get_recent_samples(limit=30, uploaded=True)
self.assertEqual(0, len(data))
data = self.dao.get_recent_samples(limit=30, uploaded=None)
self.assertEqual(30, len(data))
data = self.dao.get_recent_samples(limit=30, uploaded=False)
self.assertEqual(30, len(data))
data = self.dao.get_non_uploaded(limit=30)
self.assertEqual(30, len(data))
self.dao.mark_uploaded(data)
data = self.dao.get_non_uploaded(limit=30)
self.assertEqual(4, len(data)) # total 34, marked 30
data = self.dao.get_recent_samples(limit=30, uploaded=False)
self.assertEqual(4, len(data))
data = self.dao.get_recent_samples(limit=30, uploaded=True)
self.assertEqual(30, len(data))
data = self.dao.get_recent_samples(limit=40, uploaded=None)
self.assertEqual(34, len(data))
print(repr(self.dao))
self.assertTrue(repr(self.dao).startswith('FriskbyDao'))
# test num / num upl / num non-upl
self._do_test_num_upl(self.dao, 2*num_data, 30, 2*num_data - 30)
def test_localtime(self):
t10 = gen_rand_ts()
t25 = gen_rand_ts()
now = dt.now()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
out = self.dao.get_non_uploaded(limit=1)[0]
delta = now - out[3]
# checking that we're in the same timezone
self.assertTrue(abs(delta.total_seconds()) < 1000)
def test_create_directory_structure(self):
_tmp_dir = '/tmp/friskby/%d/no/such/dir/here' % randint(2**32, 2**42)
_fpath = '%s/friskby.sql' % _tmp_dir
dao = FriskbyDao(_fpath)
sqlpath = dao.get_path()
self.assertEqual(_fpath, sqlpath)
t10 = gen_rand_ts(value=26.8)
t25 = gen_rand_ts(value=19.90)
data = {'PM10': t10, 'PM25': t25}
dao.persist_ts(data)
# data: 2 elts of (id, value, sensor, timestamp, upl)
data = dao.get_non_uploaded()
self.assertEqual(2, len(data))
db_10, db_25 = data
if db_10[2] != 'PM10':
db_25, db_10 = data
self.assertEqual(26.8, db_10[1])
self.assertEqual(19.9, db_25[1])
def test_last_entry(self):
self.assertIsNone(self.dao.last_entry(uploaded=True))
self.assertIsNone(self.dao.last_entry(uploaded=False))
self.assertIsNone(self.dao.last_entry())
t10 = gen_rand_ts()
t25 = gen_rand_ts()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
self.assertIsNone(self.dao.last_entry(uploaded=True))
self.assertIsNotNone(self.dao.last_entry(uploaded=False))
self.assertIsNotNone(self.dao.last_entry())
data = self.dao.get_non_uploaded(limit=30)
self.dao.mark_uploaded(data)
self.assertIsNotNone(self.dao.last_entry(uploaded=True))
self.assertIsNone(self.dao.last_entry(uploaded=False))
self.assertIsNotNone(self.dao.last_entry())
t10 = gen_rand_ts()
t25 = gen_rand_ts()
data = {'PM10': t10, 'PM25': t25}
self.dao.persist_ts(data)
self.assertIsNotNone(self.dao.last_entry(uploaded=True))
self.assertIsNotNone(self.dao.last_entry(uploaded=False))
self.assertIsNotNone(self.dao.last_entry())