class Application:
def __init__(self):
self.is_data_posted = False
self.log = Log() # log file to test and debug (writes debug messages)
self.log_data = LogData() # sensor data log (if no internet available)
self.api = SamsApi() # https://sams.science.itf.llu.lv/ Data Warehouse Plugin to send the data
self.config = Config() # Configurations (/config/config.ini)
self.config_data = self.config.get_config_data()
self.repost_seconds = int(self.config_data['INTERVAL']['repost_seconds'])
self.app_wait_time = int(self.config_data['INTERVAL']['app_wait_seconds'])
self.dataset_taken = False
self.dataset_taken_counter = 0
self.data = Dataset() # collect all the data from sensors
self.dataset = ""
def take_dataset(self):
self.dataset = "" # empty the dataset before take new data
self.dataset = self.data.get_dataset()
def start(self):
while True:
try:
self.log.write_log("take dataset")
self.take_dataset()
# if stored data (/log/*.json) available, then try to send this data to the data warehouse
if self.log_data.has_log_files():
self.log.write_log("has log files")
self.log_data.post_log_files(self.dataset)
# if not, take a new dataset to post
else:
response = self.api.call(self.dataset)
# try to post data. If api status is 200 then everything is right
if response == 200:
self.log.write_log("dataset posted")
# if no internet connection or the api do not allow to send, then store the data
# if the status code from api is 500 then the log function will delete the file
else:
self.log.write_log("dataset posting failed. Statuscode: {0}".format(response))
self.is_data_posted = False # data where not posted
self.log.write_log("log dataset")
self.log_data.insert(self.dataset) # write new log file with the dataset
# try to post every X seconds while the data is not posted (no internet connection)
while not self.is_data_posted:
self.is_data_posted = self.api.call(self.dataset)
time.sleep(self.repost_seconds)
self.log.write_log("wait: {}".format(self.app_wait_time))
self.dataset_taken = False
time.sleep(int(self.app_wait_time)) # sleep X seconds before collecting the new data
except Exception as e:
print(e)
self.log.write_log(e)
class SamsApi:
def __init__(self):
self.config = configparser.ConfigParser()
self.token_config = configparser.ConfigParser()
self.log = Log()
self.config_file = '/boot/credentials.ini'
self.token_config_file = '/home/pi/sams/data_logger/config/config.ini'
self.config.read(self.config_file)
self.secret_data = self.config['DEFAULT']
self.client_id = self.secret_data['client_id']
self.client_secret = self.secret_data['client_secret']
self.audience = self.secret_data['audience']
self.data_url = self.secret_data['data_url']
self.token_url = self.secret_data['token_url']
self.grant_type = self.secret_data['grant_type']
self.access_header = {'content-type': "application/json"}
self.header = {}
self.auth = {}
self.token_payload = {
"client_id": self.client_id,
"client_secret": self.client_secret,
"audience": self.audience,
"grant_type": self.grant_type
}
def get_access_token(self):
try:
self.auth = requests.post(self.token_url,
json=self.token_payload,
headers=self.access_header).json()
print(self.token_url)
print(self.token_payload)
token = self.auth['token_type'] + ' ' + self.auth['access_token']
print(token)
self.write_token(token)
return token
except Exception as e:
self.log.write_log("failed to take access token: {0}".format(e))
return False
def send_data(self, payload, has_token=True):
try:
if has_token:
token = self.read_token()
else:
token = self.get_access_token()
self.header = {
'content-type': 'application/json',
'Authorization': token
}
resp = requests.post(self.data_url,
json=payload,
headers=self.header)
print(resp.status_code)
return resp.status_code
except Exception as e:
self.log.write_log("failed to send data: {0}".format(e))
return False
def call(self, payload):
print(payload)
api_call = self.send_data(payload)
if api_call == 401:
second_call = self.send_data(payload, has_token=False)
if second_call == 200:
return second_call
else:
self.log.write_log(
"send to api failed! Status code: {0}".format(second_call))
return False
elif api_call == 200:
return api_call
else:
self.log.write_log(
"send to api failed! Status code: {0}".format(api_call))
return api_call
def read_token(self):
try:
self.token_config.read(self.token_config_file)
token_config_data = self.token_config['DEFAULT']
token = token_config_data['token']
return token
except Exception as e:
self.log.write_log("read token failed: {}".format(e))
def write_token(self, token):
try:
self.token_config.set("DEFAULT", "token", token)
with open(self.token_config_file, 'w+') as configfile:
self.token_config.write(configfile)
except Exception as e:
self.log.write_log("write token failed: {}".format(e))
class Dataset:
def __init__(self):
self.config = Config()
self.config_data = self.config.get_config_data()
try:
self.dht22 = DHT22(int(self.config_data['DHT22']['pin']))
except Exception as e:
self.log.write_log("Failed to initialize DHT22: {}".format(e))
try:
self.scale = Scale()
except Exception as e:
self.log.write_log("Failed to initialize scale: {}".format(e))
try:
self.DS18B20 = DS18B20()
except Exception as e:
self.log.write_log("Failed to initialize DS18B20: {}".format(e))
self.api = SamsApi()
self.log = Log()
self.median_interval = int(self.config_data['INTERVAL']['median'])
self.wait_time = int(self.config_data['INTERVAL']['wait_time_seconds'])
self.dataset = []
self.temp = []
self.hum = []
self.weight = []
self.ds_temp = []
self.median_temp = 0
self.median_hum = 0
self.median_weight = 0
self.median_ds_temp = 0
self.duration = int(self.config_data['AUDIO']['duration'])
@staticmethod
def get_time():
now = datetime.datetime.utcnow()
return now.strftime('%Y-%m-%dT%H:%M:%S') + now.strftime(
'.%f')[:0] + 'Z'
def error(self, device, error):
if not device == "audio":
self.dataset.append({
"sourceId":
"{0}-{1}".format(device, self.api.client_id),
"value": [
{
"ts": self.get_time(),
"value": 0
},
]
})
else:
self.dataset.append({
"sourceId":
"{0}-{1}".format("audio", self.api.client_id),
"value": [
{
"ts": self.get_time(),
"value": [0, 0]
},
]
})
self.log.write_log(
"something went wrong by collecting the {0} dataset! Error: {1}".
format(device, error))
def get_fft_data(self):
n_window = pow(2, 12)
n_overlap = n_window / 2
n_fft = n_window
fs = 16000
try:
audiodata = sd.rec(self.duration * fs,
samplerate=fs,
channels=1,
dtype='float64')
sd.wait()
data = audiodata.transpose()
[F, pxx] = scipy.signal.welch(data,
fs=fs,
window='hanning',
nperseg=n_window,
noverlap=n_overlap,
nfft=n_fft,
detrend=False,
return_onesided=True,
scaling='density')
temp_data = np.array(pxx).astype(float)
data = temp_data.tolist()
self.dataset.append({
"sourceId":
"audio-{0}".format(self.api.client_id),
"values": [
{
"ts": self.get_time(),
"values": data[0]
},
]
})
except Exception as e:
self.error("audio", e)
return True
def get_ds18b20_data(self):
sensor_counter = self.DS18B20.device_count()
try:
if sensor_counter != 0:
for x in range(sensor_counter):
self.median_ds_temp = []
for i in range(self.median_interval):
value = self.DS18B20.tempC(x)
if value == 998 or value == 85.0:
self.log.write_log(
"DS18B20 does not work properly...")
else:
self.ds_temp.append(self.DS18B20.tempC(x))
time.sleep(self.wait_time)
if len(self.ds_temp) != 0:
self.median_ds_temp = median(self.ds_temp)
del self.ds_temp[:]
self.dataset.append({
"sourceId":
"dsb18b20-{0}-{1}".format(x, self.api.client_id),
"values": [
{
"ts": self.get_time(),
"value": float(self.median_ds_temp)
},
]
})
self.median_ds_temp = ""
except Exception as e:
self.error("ds18b20", e)
def get_dht22_data(self):
try:
for i in range(self.median_interval):
dhtdata = self.dht22.get_data()
self.temp.append(dhtdata['temp'])
self.hum.append(dhtdata['hum'])
time.sleep(self.wait_time)
self.median_temp = median(self.temp)
self.median_hum = median(self.hum)
self.dataset.append({
"sourceId":
"dht22-temperature-{0}".format(self.api.client_id),
"values": [
{
"ts": self.get_time(),
"value": float(self.median_temp)
},
]
})
self.dataset.append({
"sourceId":
"dht22-humidity-{0}".format(self.api.client_id),
"values": [
{
"ts": self.get_time(),
"value": float(self.median_hum)
},
]
})
del self.temp[:]
del self.hum[:]
except Exception as e:
self.error("dht22", e)
def get_scale_data(self):
try:
for i in range(self.median_interval):
self.weight.append(self.scale.get_data())
print(self.weight)
time.sleep(self.wait_time)
self.median_weight = median(self.weight)
del self.weight[:]
self.dataset.append({
"sourceId":
"scale-{0}".format(self.api.client_id),
"values": [{
"ts": self.get_time(),
"value": float(self.median_weight)
}]
})
except Exception as e:
self.error("scale", e)
def get_dataset(self):
try:
self.dataset[:] = [] # empty the dataset before take new data
self.get_fft_data()
self.get_scale_data()
self.get_dht22_data()
self.get_ds18b20_data()
return self.dataset
except Exception as e:
self.log.write_log("Dataset error: {}".format(e))
return False
class LogData:
def __init__(self):
self.path = '/var/www/upload/data_logger/log/'
self.api = SamsApi()
self.status = []
self.files = os.listdir(self.path)
self.log = Log()
def insert(self, json_data):
files = os.listdir(self.path)
if len(files) != 0:
file = int(
len([
name for name in os.listdir(self.path)
if os.path.isfile(os.path.join(self.path, name))
])) + 1
else:
file = int(1)
try:
with open(self.path + str(file) + ".json", 'w') as f:
json.dump(json_data, f)
f.close()
except Exception as e:
print(e)
def list_dir(self):
self.files = os.listdir(self.path)
self.files.sort()
@staticmethod
def read_file(path):
with open(path) as json_file:
data = json.load(json_file)
return data
def has_log_files(self):
if not os.listdir(self.path):
return False
else:
return True
def post_log_files(self, dataset):
try:
self.log.write_log("log dataset...")
self.insert(dataset)
while self.has_log_files():
self.list_dir()
for x in self.files:
file = self.read_file(self.path + str(x))
self.log.write_log("try to post data")
if self.api.call(file) == 200:
self.log.write_log("status code ok! Delete file...")
os.remove(self.path + str(x))
if self.api.call(file) == 500:
self.log.write_log("File corrupted! Delete file...")
os.remove(self.path + str(x))
time.sleep(5)
return True
except Exception as e:
print(e)
self.log.write_log(e)