def main():
files = my.get_grouped_files(source_path=source_path,
delay=delay,
index_delay=index_delay)
path = delay_path + "delay_" + str(delay[index_delay]) + ".json"
data_delay = my.open_file_and_return_data(path=path)
i = 1
err = list()
dictionaries = dict()
data = dict()
for item in files:
data = my.open_file_and_return_data(path=item)
print("\x1b[1;30;43m " + item[28:] + " \x1b[0m ")
start = dt.datetime.strptime(data_delay[str(i)]['_info']['start'],
'%Y-%m-%d %H:%M:%S.%f')
end = dt.datetime.strptime(data_delay[str(i)]['_info']['end_sent'],
'%Y-%m-%d %H:%M:%S.%f')
# TODO shift focus about data --- duration 4 minute
new_start = start + dt.timedelta(0, 30) # right shift --> default 40 s
new_end = new_start + dt.timedelta(0, 240) # 4m <- (60s*4)
# end = end - dt.timedelta(0, 15)
new_diff = new_end - new_start
h, m, s = my.convert_timedelta(new_diff)
fault, outcome = analysis(data, new_start, new_end, i)
print("start: {} -- end: {} --- {}:{}.{} -- min: {} -- max: {}".format(
new_start, new_end, h, m, s, outcome['smaller'],
outcome['bigger']))
time = str(h) + ":" + str(m) + "." + str(s) + " [h:m.s]"
outcome['time'] = {
'old_start': start,
'old_end': end,
'new_start': new_start,
'new_end': new_end,
'new_time': time
}
err.append(fault)
dictionaries[i] = outcome
i += 1
mex_removed = (60 * 1000) / delay[index_delay]
total = data['_command']['n_mex'] - mex_removed
print("\x1b[1;34;40m [{}] Mex_removed: {} --> total: {} \x1b[0m ".format(
delay[index_delay], mex_removed, total))
print("err: --> ", err)
dictionaries['error'] = err
dictionaries['info'] = {'mex_removed': mex_removed, 'n_mex': total}
save_data(dictionaries)
def preprocessing(data):
print("- {}".format(preprocessing.__name__))
wrong_mex = dict()
time_change = list()
mex_correct = list()
end_sent = ""
for i, mex in enumerate(data['messages']):
string = mex['message_id']
string = string.split(',')
if re.match(regular_expression_mex, mex['message_id']):
# MEX string
mex_correct.append({
'mex_id': int(string[1]),
'type_mex': string[0],
'ttl': string[2],
'time': mex['time']
})
elif re.match(regular_expression_time, mex['message_id']):
# DELAY check WIFI string
time_change.append({
'type_mex': string[0],
'delay': int(string[1]),
'time': mex['time']
})
elif re.match(regular_expression_final, mex['message_id']):
end_sent = {'type_mex': string[0], 'time': mex['time']}
else:
# wrong saving
match = re.findall(find_all_matches, mex['message_id'])
wrong_mex[i] = {
'datetime': mex['time'],
'valid_mex_found': match,
'string': mex['message_id']
}
new_wrong_mex = dict()
for index in wrong_mex:
try:
string = wrong_mex[index]['string']
datetime = wrong_mex[index]['datetime']
if len(wrong_mex[index]['valid_mex_found']) > 0:
for m in wrong_mex[index]['valid_mex_found']:
string = string.replace(m, '')
m1 = m.split(',')
if re.match(regular_expression_mex, m):
mex_correct.append({
'mex_id': int(m1[1]),
'type_mex': m1[0],
'ttl': m1[2],
'time': datetime
})
elif re.match(regular_expression_time, m):
time_change.append({
'type_mex': m1[0],
'delay': int(m1[1]),
'time': datetime
})
elif re.match(regular_expression_final, m):
end_sent = {'type_mex': m1[0], 'time': datetime}
if len(string) > 0:
new_wrong_mex[index] = {
'datetime': datetime,
'string': string
}
else:
new_wrong_mex[index] = {'datetime': datetime, 'string': string}
except Exception:
print("Erore index: ", index)
# 1. creo una lista contenente tutte le strings con k propedeutici (23,24,25)
# 2. appena trovo un numero che non è immediatamente consecutivo agli elementi in key_list, analizzo la string creata
# 3. cerco tutti i possibili match con (mex, time, etc) e procedo a sistemare il tutto
string = ""
time_list = list()
key_list = list()
for k, v in new_wrong_mex.items():
if key_list:
if abs(k - key_list[-1]) != 1:
match = re.findall(find_all_matches, string)
i = 0
for m in match:
string = string.replace(m, '')
m1 = m.split(',')
if re.match(regular_expression_mex, m):
mex_correct.append({
'mex_id': int(m1[1]),
'type_mex': m1[0],
'ttl': m1[2],
'time': time_list[i]
})
elif re.match(regular_expression_time, m):
time_change.append({
'type_mex': m1[0],
'delay': int(m1[1]),
'time': time_list[i]
})
elif re.match(regular_expression_final, m):
end_sent = {'type_mex': m1[0], 'time': time_list[i]}
i += 1
key_list.clear()
time_list.clear()
if string == "":
string = str(v['string'])
if v['datetime'] == "2020-02-01 09:16:59.943322":
print(string)
else:
string += str(v['string'])
if v['datetime'] == "2020-02-01 09:16:59.943322":
print(string)
key_list.append(k)
time_list.append(v['datetime'])
# TODO PER GESTIRE L'ULTIMO ELEMENTO NELLA LISTA DEGLI ERRORI
if string != "":
match = re.findall(find_all_matches, string)
i = 0
for m in match:
string = string.replace(m, '')
m1 = m.split(',')
if re.match(regular_expression_mex, m):
mex_correct.append({
'mex_id': int(m1[1]),
'type_mex': m1[0],
'ttl': m1[2],
'time': time_list[i]
})
elif re.match(regular_expression_time, m):
time_change.append({
'type_mex': m1[0],
'delay': int(m1[1]),
'time': time_list[i]
})
elif re.match(regular_expression_final, m):
end_sent = {'type_mex': m1[0], 'time': time_list[i]}
i += 1
# TODO Sort given list of dictionaries by date
mex_correct.sort(
key=lambda x: dt.datetime.strptime(x['time'], '%Y-%m-%d %H:%M:%S.%f'))
time_change.sort(
key=lambda x: dt.datetime.strptime(x['time'], '%Y-%m-%d %H:%M:%S.%f'))
# PRINT MEX
start = dt.datetime.strptime(mex_correct[0]['time'],
'%Y-%m-%d %H:%M:%S.%f')
end_sent_ = dt.datetime.strptime(end_sent['time'], '%Y-%m-%d %H:%M:%S.%f')
end_test = dt.datetime.strptime(mex_correct[-1]['time'],
'%Y-%m-%d %H:%M:%S.%f')
diff_ = end_sent_ - start
h2, m2, s2 = my.convert_timedelta(diff_)
diff = end_test - start
h1, m1, s1 = my.convert_timedelta(diff)
print("Start: {}".format(start))
print("End_sent: {}".format(end_sent_))
print("End_test: {}".format(end_test))
print("Sent: {}:{}.{} [mm:s.us]".format(m2, s2, diff_.total_seconds()))
print("Test: {}:{}.{} [mm:s.us]".format(m1, s1, diff.total_seconds()))
# Salvare Preprocessing
t1 = str(h2) + ":" + str(m2) + "." + str(s2) + " [h:m.s]"
t2 = str(h1) + ":" + str(m1) + "." + str(s1) + " [h:m.s]"
my_data = dict()
my_data['_command'] = data['_command']
my_data['_info'] = {
'start': start,
'end_sent': end_sent_,
'end_test': end_test,
'time_send': t1,
'time_test': t2
}
my_data['_mex'] = mex_correct
my_data['_time'] = time_change
# TODO saving
print("Saving Preprocessing File")
my.save_json_data_elegant(path=preprocessing_path, data=my_data)
hash_table = dict()
length = len(mex_correct)
for i in range(length):
k = mex_correct[i]['mex_id']
if k in hash_table:
hash_table[k].append(i)
else:
hash_table[k] = list()
hash_table[k].append(i)
word_info = dict()
info_statistic = {
'send_ble': 0,
'receive_ble': 0,
'error_ble': 0,
'lost_ble': 0,
'double_sent': 0,
'send_wifi': 0,
'receive_wifi': 0,
'error_wifi': 0,
'lost_wifi': 0,
'sent_received': 0
}
ttl_receive = 3 - int(data['_command']['relay'])
list_wait = list()
error_1 = list()
error_2 = list()
diff_ble = list()
latency_ble = list()
diff_wifi = list()
latency_wifi = list()
hash_json_data = dict()
double = list()
set_1 = set()
set_2 = set()
for k, v in hash_table.items():
for i in v:
word_info, info_statistic, double = my.look_into_it(
word_info, info_statistic, mex_correct[i], double)
if mex_correct[i]['type_mex'] == "O":
set_1.add(mex_correct[i]['mex_id'])
if not k in hash_json_data:
hash_json_data[k] = []
hash_json_data[k].append(mex_correct[i])
# BLE
if 'receive_ble' in word_info:
if not int(word_info['send_ble']['ttl']) == 3:
print('\x1b[1;31;40m' + ' Error analysis_ttl --> send_: ' +
word_info['send_ble']['time'] + '\x1b[0m')
error_1.append({'k': k, 'time': word_info['send_ble']['time']})
if not int(word_info['receive_ble']['ttl']) == ttl_receive:
print('\x1b[1;31;40m' + ' Error analysis_ttl --> receive_: ' +
word_info['receive_ble']['time'] + '\x1b[0m')
error_2.append({'k': k, 'time': word_info['send_ble']['time']})
sent = dt.datetime.strptime(word_info['send_ble']['time'],
'%Y-%m-%d %H:%M:%S.%f')
receive_wifi = dt.datetime.strptime(
word_info['receive_ble']['time'], '%Y-%m-%d %H:%M:%S.%f')
difference = receive_wifi - sent
diff_ble.append(abs(difference.total_seconds()))
latency_ble.append(abs(difference.total_seconds()) / 2)
info_statistic['sent_received'] += 1
hash_json_data[k].append({
'ble': {
'send_time': sent,
'status_time': receive_wifi,
'difference': abs(difference.total_seconds()),
'latency': (abs(difference.total_seconds()) / 2)
}
})
elif 'error_ble' in word_info:
info_statistic['send_ble'] -= 1
else:
info_statistic['lost_ble'] += 1
# WIFI
if 'send_wifi' in word_info:
if 'receive_wifi' in word_info:
sent = dt.datetime.strptime(word_info['send_wifi']['time'],
'%Y-%m-%d %H:%M:%S.%f')
receive_wifi = dt.datetime.strptime(
word_info['receive_wifi']['time'], '%Y-%m-%d %H:%M:%S.%f')
difference = receive_wifi - sent
diff_wifi.append(abs(difference.total_seconds()))
latency_wifi.append(abs(difference.total_seconds()) / 2)
info_statistic['sent_received'] += 1
hash_json_data[k].append({
'wifi': {
'send_time': sent,
'status_time': receive_wifi,
'difference': abs(difference.total_seconds()),
'latency': (abs(difference.total_seconds()) / 2)
}
})
set_2.add(word_info['receive_wifi']['mex_id'])
elif 'error_wifi' in word_info:
info_statistic['send_wifi'] -= 1
else:
info_statistic['lost_wifi'] += 1
# waiting
send_ble = dt.datetime.strptime(word_info['send_ble']['time'],
'%Y-%m-%d %H:%M:%S.%f')
send_wifi = dt.datetime.strptime(word_info['send_wifi']['time'],
'%Y-%m-%d %H:%M:%S.%f')
wait = abs(send_wifi - send_ble)
list_wait.append(wait.total_seconds())
if 'receive_wifi' in word_info:
receive_wifi = dt.datetime.strptime(
word_info['receive_wifi']['time'], '%Y-%m-%d %H:%M:%S.%f')
diff_1 = (receive_wifi - send_ble).total_seconds() / 2
diff_2 = wait.total_seconds() + (
(receive_wifi - send_wifi).total_seconds() / 2)
hash_json_data[k].append({
'ble_wifi': {
'send_ble': send_ble,
'send_wifi': send_wifi,
'receive_wifi': receive_wifi,
'wait': wait.total_seconds(),
'latency_1': diff_1,
'latency_2': diff_2
}
})
else:
hash_json_data[k].append({
'ble_wifi': {
'send_ble': send_ble,
'send_wifi': send_wifi,
'wait': wait.total_seconds()
}
})
if 'receive_ble' in word_info and 'send_wifi' in word_info:
info_statistic['double_sent'] += 1
word_info.clear()
if len(double) != 0:
for k in double:
print(hash_json_data[k])
raise Exception(
'Indici Duplicati, esempio R,34,2 - R,34,1 --> Rimuovere R,34,2')
print("---------------")
print("[relay: {} -> ttl: {}]".format(data['_command']['relay'],
ttl_receive))
print("Changing Delay: {}".format(len(time_change)))
print("---------------")
if len(error_1) > 0 or len(error_2) > 0:
raise Exception('\x1b[1;31;40m' + ' Error TTL: sent: ' +
str(len(error_1)) + ' --> received: ' +
str(len(error_2)) + ' \x1b[0m')
print("Media DIff BLE: {}s".format(statistics.mean(diff_ble)))
print("Media Latency BLE: {}s".format(statistics.mean(latency_ble)))
print("Media Diff WIFI: {}s".format(statistics.mean(diff_wifi)))
print("Media Latency WIFI: {}s".format(statistics.mean(latency_wifi)))
print("Media Waiting BLE -> WIFI: {}s".format(statistics.mean(list_wait)))
info_statistic['valid_mex'] = info_statistic[
'sent_received'] - info_statistic['double_sent']
print("Total Send -> [{}]".format(data['_command']['n_mex']))
print("S (ble) -> {}".format(info_statistic['send_ble']))
print("R (ble) -> {}".format(info_statistic['receive_ble']))
print("E (ble) -> {}".format(info_statistic['error_ble'])) # not_sent_ble
print("Lost BLE: {}".format(info_statistic['lost_ble']))
print("I (wifi) -> {}".format(info_statistic['send_wifi']))
print("O (wifi) -> {}".format(info_statistic['receive_wifi']))
print("W (wifi) -> {}".format(
info_statistic['error_wifi'])) # not_sent_wifi
print("Lost WiFi: {}".format(info_statistic['lost_wifi']))
print("Double Sent -> {}".format(info_statistic['double_sent']))
print("Sent & Received -> {}".format(info_statistic['sent_received']))
print("Valid Mex -> {}".format(info_statistic['valid_mex']))
# Check statistics
if data['_command']['n_mex'] - info_statistic['send_ble'] - info_statistic[
'error_ble'] != 0:
raise Exception('\x1b[1;31;40m Error BLE: mex number [T-S-E] \x1b[0m')
elif info_statistic['send_ble'] - info_statistic[
'receive_ble'] != info_statistic['lost_ble']:
raise Exception('\x1b[1;31;40m Error BLE: mex number [S-R] \x1b[0m')
elif info_statistic['send_wifi'] - info_statistic[
'lost_wifi'] != info_statistic['receive_wifi']:
print(set_2.difference(set_1))
print(set_1.difference(set_2))
raise Exception('\x1b[1;31;40m Error WIFI: mex number [I-O] \x1b[0m')
elif info_statistic['valid_mex'] > data['_command']['n_mex']:
raise Exception('\x1b[1;31;40m Error MEX: Valid Mex \x1b[0m')
else:
print('\x1b[1;31;42m Check OK \x1b[0m')
my_data_2 = dict()
my_data_2['_command'] = my_data['_command']
my_data_2['_info'] = my_data['_info']
my_data_2['_info_2'] = {
'mean': {
'ble_mean_diff': statistics.mean(diff_ble),
'ble_mean_latency': statistics.mean(latency_ble),
'wifi_mean_diff': statistics.mean(diff_wifi),
'wifi_mean_latency': statistics.mean(latency_wifi),
'wait_mean': statistics.mean(list_wait)
},
'mex_': {
'ble': {
'send_ble': info_statistic['send_ble'],
'receive_ble': info_statistic['receive_ble'],
'not_send_ble': info_statistic['error_ble'],
'lost_ble': info_statistic['lost_ble']
},
'wifi': {
'send_wifi': info_statistic['send_wifi'],
'receive_wifi': info_statistic['receive_wifi'],
'not_send_wifi': info_statistic['error_wifi'],
'lost_wifi': info_statistic['lost_wifi']
},
'double_sent': info_statistic['double_sent'],
'valid_mex': info_statistic['valid_mex'],
'sent_received_total': info_statistic['sent_received']
}
}
my_data_2['_time'] = my_data['_time']
my_data_2['_mex'] = hash_json_data
print("Saving Analysis File")
my.save_json_data_elegant(path=analysis_path, data=my_data_2)
def statistics(data, run, x):
mex = data['_mex']
min_index = x[str(run)]['smaller']
max_index = x[str(run)]['bigger']
latency_ble = dict()
outcomes = {'S': 0, 'R': 0, 'L': 0, 'L1': 0, 'E': 0}
end_test = dt.datetime.strptime(x[str(run)]['time']['new_end'], '%Y-%m-%d %H:%M:%S.%f')
end_send = dt.datetime.strptime(x[str(run)]['time']['new_end'], '%Y-%m-%d %H:%M:%S.%f')
for index, list_of_value in mex.items():
for v in list_of_value:
if min_index <= int(index) <= max_index:
if 'type_mex' in v and v['type_mex'] == "S":
outcomes['S'] += 1
if 'type_mex' in v and v['type_mex'] == "E":
outcomes['E'] += 1
if 'ble' in v:
outcomes['R'] += 1
latency_ble[int(index)] = v['ble']['latency']
rcv = dt.datetime.strptime(v['ble']['status_time'], '%Y-%m-%d %H:%M:%S.%f')
if rcv > end_test:
end_test = rcv
outcomes['L1'] = outcomes['S'] - outcomes['R'] - outcomes['E']
outcomes['L'] = outcomes['S'] - outcomes['R']
if outcomes['S'] - outcomes['R'] - outcomes['L'] != 0:
raise Exception("Error counting: {} -- {}".format('ble', outcomes))
if outcomes['S'] != x[str(run)]['S'] or outcomes['R'] != x[str(run)]['R'] or outcomes['L'] != x[str(run)]['L']:
raise Exception('Error counting: {} -- {}\n{}'.format('total', outcomes, x[str(run)]))
if len(latency_ble) != outcomes['R']:
raise Exception(
'Error counting: size_total: {} -- total_R: {}'.format(len(latency_ble), outcomes['R']))
l_ble = list(latency_ble.values())
ble_ = my.intervalli_di_confidenza(dataset=l_ble)
start_test = dt.datetime.strptime(x[str(run)]['time']['new_start'], '%Y-%m-%d %H:%M:%S.%f')
time_test = abs(end_test - start_test)
time_ = abs(end_send - start_test)
# TODO BLE
pdr_ble = outcomes['R'] / outcomes['S']
goodput_ble = (outcomes['R'] * 2) / time_test.total_seconds() # 2 byte di dati utili
goodput_ble_1 = (outcomes['R'] * 2) / time_.total_seconds() # 2 byte di dati utili
h, m, s = my.convert_timedelta(time_)
time_send_ = str(h) + ":" + str(m) + "." + str(s) + " [h:m.s]"
h, m, s = my.convert_timedelta(time_test)
time_test_ = str(h) + ":" + str(m) + "." + str(s) + " [h:m.s]"
my_dictionary[str(run)] = {'_info_0': data['_info'],
'_info_1': {'end_sent': x[str(run)]['time']['new_end'], 'end_test': end_test,
'start': x[str(run)]['time']['new_start'], 'time_send': time_send_,
'time_test': time_test_},
'mex_': {'S': outcomes['S'], 'R': outcomes['R'], 'L': outcomes['L'], 'E': outcomes['E']},
'statistic_': {'sample_size': len(l_ble),
'pdr': pdr_ble,
'goodput': goodput_ble,
'goodput_1': goodput_ble_1,
'latency': {'mean': ble_['mean'], 'std': ble_['std'],
'm_e': ble_['m_e'],
'low': ble_['low'],
'up': ble_['up']}}}
return latency_ble
def statistics(data, run, x):
mex = data['_mex']
min_index = x[str(run)]['smaller']
max_index = x[str(run)]['bigger']
latency_ble = dict()
latency_wifi = dict()
latency_total = dict()
outcomes = {'ble': {'S': 0, 'R': 0, 'L': 0, 'E': 0},
'wifi': {'S': 0, 'R': 0, 'L': 0, 'E': 0},
'total': {'S': 0, 'R': 0, 'L': 0}}
end_test = dt.datetime.strptime(x[str(run)]['time']['new_end'], '%Y-%m-%d %H:%M:%S.%f')
for index, list_of_value in mex.items():
ble = False
wifi = False
for v in list_of_value:
if min_index <= int(index) <= max_index:
if 'type_mex' in v and v['type_mex'] == "S":
outcomes['ble']['S'] += 1
outcomes['total']['S'] += 1
if 'type_mex' in v and v['type_mex'] == "E":
outcomes['ble']['E'] += 1
if 'type_mex' in v and v['type_mex'] == "I":
wifi = True
outcomes['wifi']['S'] += 1
if 'type_mex' in v and v['type_mex'] == "W":
outcomes['wifi']['W'] += 1
if 'ble' in v:
ble = True
outcomes['ble']['R'] += 1
outcomes['total']['R'] += 1
latency_ble[int(index)] = v['ble']['latency']
latency_total[int(index)] = v['ble']['latency']
rcv = dt.datetime.strptime(v['ble']['status_time'], '%Y-%m-%d %H:%M:%S.%f')
if rcv > end_test:
end_test = rcv
if 'wifi' in v:
outcomes['wifi']['R'] += 1
outcomes['total']['R'] += 1
latency_wifi[int(index)] = v['wifi']['latency']
rcv = dt.datetime.strptime(v['wifi']['status_time'], '%Y-%m-%d %H:%M:%S.%f')
if rcv > end_test:
end_test = rcv
if 'ble_wifi' in v and 'latency_1' in v['ble_wifi']:
latency_total[int(index)] = v['ble_wifi']['latency_1']
if ble and wifi:
raise Exception('Double Sent: {}'.format(index))
outcomes['ble']['L'] = outcomes['ble']['S'] - outcomes['ble']['R'] - outcomes['ble']['E']
outcomes['wifi']['L'] = outcomes['wifi']['S'] - outcomes['wifi']['R'] - outcomes['wifi']['L']
outcomes['total']['L'] = outcomes['total']['S'] - outcomes['total']['R']
if outcomes['ble']['S'] - outcomes['ble']['R'] - outcomes['ble']['L'] - outcomes['ble']['E'] != 0:
raise Exception("Error counting: {} -- {}".format('ble', outcomes['ble']))
if outcomes['wifi']['S'] - outcomes['wifi']['R'] - outcomes['wifi']['L'] - outcomes['wifi']['E'] != 0:
raise Exception("Error counting: {} -- {}".format('wifi', outcomes['wifi']))
if outcomes['total']['S'] != x[str(run)]['S'] or outcomes['total']['S'] != x[str(run)]['S'] or outcomes['total'][
'S'] != \
x[str(run)]['S']:
raise Exception('Error counting: {} -- {}\n{}'.format('total', outcomes['total'], x[str(run)]))
if len(latency_total) != outcomes['total']['R']:
raise Exception(
'Error counting: size_total: {} -- total_R: {}'.format(len(latency_total), outcomes['total']['R']))
l_ble = list(latency_ble.values())
l_wifi = list(latency_wifi.values())
ble_ = my.intervalli_di_confidenza(dataset=l_ble)
wifi_ = my.intervalli_di_confidenza(dataset=l_wifi)
l_ = list(latency_total.values())
latency_ = my.intervalli_di_confidenza(dataset=l_)
start_test = dt.datetime.strptime(x[str(run)]['time']['new_start'], '%Y-%m-%d %H:%M:%S.%f')
end_send = dt.datetime.strptime(x[str(run)]['time']['new_end'], '%Y-%m-%d %H:%M:%S.%f')
time_test = abs(end_test - start_test)
time_ = abs(end_send - start_test)
# TODO BLE
pdr_ble = outcomes['ble']['R'] / outcomes['ble']['S']
goodput_ble = (outcomes['ble']['R'] * 2) / time_test.total_seconds() # 2 byte di dati utili
# TODO WIFI
s_t = start_test + dt.timedelta(0, 40) # aggiungo 40 secondi
t_t = end_test - s_t
pdr_wifi = outcomes['wifi']['R'] / outcomes['wifi']['S']
goodput_wifi = (outcomes['wifi']['R'] * 2) / t_t.total_seconds() # 2 byte di dati utili
# TODO General
pdr = outcomes['total']['R'] / outcomes['total']['S']
goodput = outcomes['total']['R'] * 2 / time_test.total_seconds() # 2 byte di dati utili
goodput_1 = outcomes['total']['R'] * 2 / time_.total_seconds() # 2 byte di dati utili
h, m, s = my.convert_timedelta(time_)
time_send_ = str(h) + ":" + str(m) + "." + str(s) + " [h:m.s]"
h, m, s = my.convert_timedelta(time_test)
time_test_ = str(h) + ":" + str(m) + "." + str(s) + " [h:m.s]"
my_dictionary[str(run)] = {'_info_0': data['_info'],
'_info_1': {'end_sent': x[str(run)]['time']['new_end'], 'end_test': end_test,
'start': x[str(run)]['time']['new_start'], 'time_send': time_send_,
'time_test': time_test_},
'mex_': {'ble': {'S': outcomes['ble']['S'], 'R': outcomes['ble']['R'],
'L': outcomes['ble']['L'], 'E': outcomes['ble']['E']},
'wifi': {'S': outcomes['wifi']['S'], 'R': outcomes['wifi']['R'],
'L': outcomes['wifi']['L'], 'E': outcomes['wifi']['E']},
'total': {'S': outcomes['total']['S'], 'R': outcomes['total']['R'],
'L': outcomes['total']['L']}},
'statistic_': {'ble': {'sample_size': len(l_ble),
'pdr': pdr_ble,
'goodput': goodput_ble,
'latency': {'mean': ble_['mean'], 'std': ble_['std'],
'm_e': ble_['m_e'],
'low': ble_['low'],
'up': ble_['up']}},
'wifi': {'sample_size': len(l_wifi),
'pdr': pdr_wifi,
'goodput': goodput_wifi,
'latency': {'mean': wifi_['mean'], 'std': wifi_['std'],
'm_e': wifi_['m_e'],
'low': wifi_['low'],
'up': wifi_['up']
}},
'total': {'sample_size': len(l_),
'pdr': pdr,
'goodput': goodput,
'goodput_1': goodput_1,
'latency': {'mean': latency_['mean'], 'std': latency_['std'],
'm_e': latency_['m_e'],
'low': latency_['low'],
'up': latency_['up']
}}}}
return latency_ble, latency_wifi, latency_total
def detect_time(data, new_start, new_end):
start_ok = False
end_ok = False
d = {int(k): v for k, v in data['second_analysis'].items()}
index = 1
first_send = 0
last_send = 0
s1_datetime = ""
while not start_ok:
if int(index) in d:
s1_datetime = dt.datetime.strptime(d[index]['send_time'], '%Y-%m-%d %H:%M:%S.%f')
if (new_start - s1_datetime).total_seconds() < 5:
print("new_start is {} --> {}".format(s1_datetime, index))
first_send = index
start_ok = True
index += 1
index = max(d, key=int)
s2_datetime = ""
s3_datetime = ""
while not end_ok:
if int(index) in d:
s2_datetime = dt.datetime.strptime(d[index]['send_time'], '%Y-%m-%d %H:%M:%S.%f')
if (s2_datetime - new_end).total_seconds() < 5:
print("new_end is {} --> {}".format(s2_datetime, index))
last_send = index
s3_datetime = dt.datetime.strptime(d[index]['status_time'], '%Y-%m-%d %H:%M:%S.%f')
print("new_end_correct [receive]: ", s3_datetime)
diff_new = s3_datetime - s1_datetime
h1, m1, s1 = my.convert_timedelta(diff_new)
print("{}:{}.{} [mm:s.us]".format(m1, s1, diff_new.total_seconds()))
end_ok = True
index -= 1
diff = s2_datetime - s1_datetime
hours, minutes, seconds = my.convert_timedelta(diff)
print("-----------------------------------")
print("new_start: {}".format(s1_datetime))
print("new_end {}".format(s2_datetime))
print("{m}:{s}.{f} [mm:s.us]".format(m=minutes, s=seconds, f=diff.microseconds))
print("-----------------------------------")
start = s1_datetime
end = s2_datetime
min_diff = 100
id_min = 0
max_diff = 0
id_max = 0
new_data = dict()
l_1 = set()
for k, v in data['second_analysis'].items():
s1 = dt.datetime.strptime(v['send_time'], '%Y-%m-%d %H:%M:%S.%f')
if start <= s1 <= end:
new_data[int(k)] = v
l_1.add(int(k))
if v['difference'] < min_diff:
min_diff = v['difference']
id_min = int(k)
if v['difference'] > min_diff:
max_diff = v['difference']
id_max = int(k)
print("Differeza min:{} [{}] --- max:{} [{}]".format(min_diff, id_min, max_diff, id_max))
sent, received, error, l_2 = analysis_preprocessing_2(start=start, end=s3_datetime, l_min=first_send,
l_max=last_send)
lost = sent - received - error
lost_and_error = lost + error
if (lost_and_error + received) != sent:
raise Exception("Errore conteggio")
if received != len(new_data):
raise Exception("ERORORE dimensioni---- received: {} couple: {}".format(received, len(new_data)))
print(l_1.difference(l_2))
print("[{}]s Delay: {}".format(diff.total_seconds(), ((s2_datetime - s1_datetime).total_seconds() / sent)))
return new_data, start, end, s3_datetime, sent, received, error, lost, lost_and_error
def second_analysis(data):
print("- {}".format(second_analysis.__name__))
# data = my.open_file_and_return_data(path=path)
messages = data['messages']
command = data['_command']
if len(data['error_first_analysis']) > 0:
raise Exception('\x1b[1;31;40m' + ' Error: List \'error_first_analysis\' not empty ' + '\x1b[0m')
if data['analysis_status'] != 1:
raise Exception('\x1b[1;31;40m' + ' Phase 2 is already executed ' + '\x1b[0m')
list_of_m_id = list()
for item in messages:
list_of_m_id.append(item['message_id'])
list_of_m_id = list(dict.fromkeys(list_of_m_id)) # definisco una lista contenente i message_id
if '0' in list_of_m_id:
list_of_m_id.remove('0')
packet_received = 0
packet_not_sent = 0
differences = list()
latencies = list()
dict_analysis = dict()
new_dataset = {'error_second_analysis': [], 'error_network_buffer': []}
size_messages = len(messages)
user_check = False
error = False
first_send = messages[0]['time']
last_received = messages[0]['time']
min_diff_string = ""
max_diff_string = ""
min_diff_time = ""
max_diff_time = ""
match_error = "^[E]$"
list_error = list()
for m_id in list_of_m_id:
couple = my.get_mex_couple(messages, m_id) # individuo le coppie di messaggi
if len(couple) == 2:
match_1, e_1, type_1 = my.look_into_element(messages[couple[0]])
match_2, e_2, type_2 = my.look_into_element(messages[couple[1]])
if not match_1 or not match_2:
raise Exception('\x1b[1;31;40m' + ' Error: value unexpected about message_id: ' + e_1 + ' \x1b[0m')
if analysis_string_check(match_error, messages[couple[0]]['type_mex'], messages[couple[1]]['type_mex']):
if analysis_string_check("^[P]$", messages[couple[0]]['type_mex'], messages[couple[1]]['type_mex']):
print("erororre {}".format(messages[couple[1]]['type_mex']))
if not user_check:
user_check = True
# add to list 'error_network_buffer' an index about mex not sent
new_dataset['error_network_buffer'].append(int(messages[couple[0]]['message_id']))
packet_not_sent += 1
# raise Exception('\x1b[1;31;40m' + ' Errors -->1 [' + messages[couple[0]]['type_mex'] + ']\x1b[0m')
else:
if type_1 == type_2:
raise Exception(
'\x1b[1;31;40m' + ' Error: couple with same type: ' + messages[couple[0]]['type_mex'] + ' - ' +
messages[couple[1]]['type_mex'] + ' \x1b[0m')
if type_1:
send_time = messages[couple[0]]['time']
receive_time = messages[couple[1]]['time']
analysis_ttl(send_mex=messages[couple[0]]['ttl'], receive_mex=messages[couple[1]]['ttl'],
time=receive_time)
else:
receive_time = messages[couple[0]]['time']
send_time = messages[couple[1]]['time']
analysis_ttl(send_mex=messages[couple[1]]['ttl'], receive_mex=messages[couple[0]]['ttl'],
time=receive_time)
if first_send > send_time:
first_send = send_time
if last_received < receive_time:
last_received = receive_time
packet_received += 1
send_datetime = dt.datetime.strptime(send_time, '%Y-%m-%d %H:%M:%S.%f')
receive_datetime = dt.datetime.strptime(receive_time, '%Y-%m-%d %H:%M:%S.%f')
# timedelta output (days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)
# 1 millisecond --> 1000 microseconds
difference = receive_datetime - send_datetime
differences.append(difference.total_seconds())
latencies.append(difference.total_seconds() / 2)
dict_analysis[int(m_id)] = {
'send_time': send_time,
'status_time': receive_time,
'difference': difference.total_seconds(),
'latency': (difference.total_seconds() / 2)
}
if min_diff_time == "":
min_diff_time = difference.total_seconds()
min_diff_string = m_id
elif difference.total_seconds() < min_diff_time:
min_diff_time = difference.total_seconds()
min_diff_string = m_id
if max_diff_time == "":
max_diff_time = difference.total_seconds()
max_diff_string = m_id
elif difference.total_seconds() > max_diff_time:
max_diff_time = difference.total_seconds()
max_diff_string = m_id
# print("m_id: {} --> {}".format(m_id, dict_analysis[m_id]['difference']))
elif len(couple) == 1:
if not user_check:
user_check = True
new_dataset = error_second_analysis(m_id=m_id, index=couple[0], size_messages=size_messages,
new_dataset=new_dataset, messages=messages)
else:
print("EROROR: {c} couple with same message_id [{id}]".format(id=m_id, c=len(couple)))
for i in range(len(couple)):
print("type_mex: {t_m}, mex_id: {m_id}, ttl: {ttl}, time: [{t}]".format(
m_id=messages[couple[i]]['message_id'], t=messages[couple[i]]['time'],
t_m=messages[couple[i]]['type_mex'], ttl=messages[couple[i]]['ttl']))
error = True
# print("Size: Duplicate message Received: {}".format(len(error_ttl)))
if error:
raise Exception("\x1b[1;31;40m Error second_analysis --> user_check \x1b[0m")
first_datetime = dt.datetime.strptime(first_send, '%Y-%m-%d %H:%M:%S.%f')
last_datetime = dt.datetime.strptime(last_received, '%Y-%m-%d %H:%M:%S.%f')
test_time = last_datetime - first_datetime
hours, minutes, seconds = my.convert_timedelta(test_time)
mex_sent = int(data['_command']['n_mex']) - packet_not_sent
print("--- Mex sent: {}".format(mex_sent))
print("--- Mex sent and received: {}".format(packet_received))
print("--- Mex not sent: {}".format(packet_not_sent))
print("--- Tempo test: {}:{}.{} [mm:s.us]".format(minutes, seconds, test_time.microseconds))
test_time_m = str(minutes) + " M, " + str(seconds) + " s"
new_dataset['second_analysis'] = dict_analysis
new_dataset['_command'] = data['_command']
new_dataset['analysis'] = {
'packet_received': int(packet_received),
'packet_sent': int(mex_sent),
'packet_not_sent': int(packet_not_sent),
'average_diff_time': statistics.mean(differences), # seconds
'min_diff_time': min(differences),
# 'min_diff_time_2': min_diff_time,
'min_diff_key': min_diff_string,
'max_diff_time': max(differences),
# 'max_diff_time_2': max_diff_time,
'max_diff_key': max_diff_string,
'average_latency_time': statistics.mean(latencies), # seconds
'min_latency_time': min(latencies),
'max_latency_time': max(latencies),
'test_time_first_time': first_send,
'test_time_last_time': last_received,
'test_time': test_time.total_seconds(),
'test_time_m': test_time_m,
'_comment': 'The times are expressed in seconds'
}
new_dataset['analysis_status'] = 2
return new_dataset, user_check
def clean_data(data, approach):
print("- {}".format(clean_data.__name__))
start_datetime = dt.datetime.strptime(data['analysis']['test_time_first_time'], '%Y-%m-%d %H:%M:%S.%f')
end_datetime = dt.datetime.strptime(data['analysis']['test_time_last_time'], '%Y-%m-%d %H:%M:%S.%f')
difference_1 = (end_datetime - start_datetime) - dt.timedelta(minutes=5)
sec_1 = int(math.ceil(difference_1.total_seconds() * 50 / 100)) # rimuovo il 65% all'inizio e 35% alla fine
new_start = start_datetime + dt.timedelta(seconds=sec_1)
new_end = new_start + dt.timedelta(minutes=5)
diff_old = end_datetime - start_datetime
h1, m1, s1 = my.convert_timedelta(diff_old)
print("-----------------------------------")
print("Start: {} --> end: {}".format(start_datetime, end_datetime))
print("{m1}:{s1}.{f1} [mm:s.us]".format(m1=m1, s1=s1, f1=diff_old.total_seconds()))
print("-----------------------------------")
sent = 0
received = 0
error = 0
lost = 0
lost_and_error = 0
time_start = ""
time_end = ""
time_end_sent = ""
new_data = dict()
if approach == "time":
new_data, time_start, time_end, time_end_sent, sent, received, error, lost, lost_and_error = detect_time(
data=data,
new_start=new_start,
new_end=new_end)
elif approach == "packets":
new_data, time_start, time_end, time_end_sent, sent, received, error, lost, lost_and_error = detect_packets(
data=data, duration="5m",
percentage=65)
else:
print("Errore approach value")
hours, minutes, seconds = my.convert_timedelta((time_end - time_start))
test_time_m = str(minutes) + " M, " + str(seconds) + " s"
hours, minutes, seconds = my.convert_timedelta((time_end_sent - time_start))
test_time_sent_m = str(minutes) + " M, " + str(seconds) + " s"
new_dataset = dict()
new_dataset['_command'] = data['_command']
new_dataset['last_analysis'] = data['analysis']
new_dataset['analysis_value'] = {
'inviati': sent, # comprende tutti i mex inviati anche quello con errore
'ricevuti': received,
'errore': error,
'persi': lost,
'persi_e_error': lost_and_error,
'time_start': time_start,
'time_end': time_end,
'time_end_sent': time_end_sent,
'test_time_sent_m': test_time_sent_m,
'test_time_m': test_time_m,
'test_time_sent': (time_end - time_start).total_seconds(),
'test_time': (time_end_sent - time_start).total_seconds()
}
new_dataset['couples'] = new_data
print("-------")
return new_dataset
def detect_packets(data, duration, percentage):
delay = int(data['_command']['delay'])
print("DELAY: {}ms".format(delay))
formule = 0
diff = 0
if duration == "5m":
formule = (5 * 60 * 1000) / delay
formule_1 = (6 * 60 * 1000) / delay
diff = formule_1 - formule
elif duration == "6m":
formule = (6 * 60 * 1000) / delay
else:
print("errore data_packets")
return
index_start = int(diff * int(percentage) / 100)
index_end = int(index_start + formule)
index_start += 1
print("start_index: {} end_index {}".format(index_start, index_end))
data_prepreocessing = my.open_file_and_return_data(code=0, path=path_1)
sent = 0
received = 0
error = 0
last_received_time = dt.datetime.strptime(data_prepreocessing['messages'][0]['time'], '%Y-%m-%d %H:%M:%S.%f')
last_received_index = 0
removed = 0
time_start = ""
time_end_sent = ""
l_2 = set()
for value in data_prepreocessing['messages']:
if index_start <= int(value['message_id']) <= index_end:
if value['type_mex'] == 'S':
if int(value['message_id']) == index_start:
time_start = dt.datetime.strptime(value['time'], '%Y-%m-%d %H:%M:%S.%f')
print("start_send: {} [{}]".format(time_start, value['message_id']))
elif int(value['message_id']) == index_end:
time_end_sent = dt.datetime.strptime(value['time'], '%Y-%m-%d %H:%M:%S.%f')
print("end_send: {} [{}]".format(time_end_sent, value['message_id']))
sent = sent + 1
elif value['type_mex'] == 'P':
received = received + 1
l_2.add(int(value['message_id']))
if dt.datetime.strptime(value['time'], '%Y-%m-%d %H:%M:%S.%f') > last_received_time:
last_received_time = dt.datetime.strptime(value['time'], '%Y-%m-%d %H:%M:%S.%f')
last_received_index = value['message_id']
elif value['type_mex'] == 'E':
error = error + 1
else:
removed += 1
time_end = last_received_time
lost = sent - received - error
lost_and_error = lost + error
if (lost_and_error + received) != sent:
raise Exception("Errore conteggio!")
difference = time_end_sent - time_start
hours, minutes, seconds = my.convert_timedelta(difference)
print("-----------------------------------")
print(
"TIME experiment: start: {} end: {} --- {}:{}.{} [mm:s.us]".format(time_start, time_end_sent, minutes, seconds,
difference.microseconds))
print("Last received at: {}".format(time_end))
difference_total = time_end - time_start
hours, minutes, seconds = my.convert_timedelta(difference_total)
print("TIME total: start: {} end: {} --- {}:{}.{} [mm:s.us]".format(time_start, time_end, minutes, seconds,
difference_total.microseconds))
print("-----------------------------------")
print("end_received: {} - [{}]".format(last_received_time, last_received_index))
print("Sent: {}\nReceived: {}\nError: {} Persi: {} [{} - comprende anche error]".format(sent, received, error, lost,
lost_and_error))
print("Non considerati: {}".format(removed))
print("-----------------------------------")
min_diff = 100
id_min = 0
max_diff = 0
id_max = 0
new_data = dict()
l_1 = set()
for k, v in data['second_analysis'].items():
s1 = dt.datetime.strptime(v['send_time'], '%Y-%m-%d %H:%M:%S.%f')
if time_start <= s1 <= time_end_sent:
new_data[int(k)] = v
l_1.add(int(k))
if v['difference'] < min_diff:
min_diff = v['difference']
id_min = int(k)
if v['difference'] > min_diff:
max_diff = v['difference']
id_max = int(k)
if received != len(new_data):
print("ERORORE dimensioni---- received: {} couple: {}".format(received, len(new_data)))
print(l_1.difference(l_2))
print("[{}]s Delay: {}".format(difference.total_seconds(), ((time_end - time_start).total_seconds() / sent)))
print("min_diff: {} [{}] \nmax_diff: {} [{}]".format(min_diff, id_min, max_diff, id_max))
return new_data, time_start, time_end, time_end_sent, sent, received, error, lost, lost_and_error
def first_analysis(path):
print("- {}".format(first_analysis.__name__))
data = my.open_file_and_return_data(code=0, path=path)
mex_correct = data['_mex']
hash_table = dict()
length = len(mex_correct)
for i in range(length):
k = mex_correct[i]['mex_id']
if k in hash_table:
hash_table[k].append(i)
else:
hash_table[k] = list()
hash_table[k].append(i)
word_info = dict()
statistic = {
'send_ble': 0,
'receive_ble': 0,
'error_ble': 0,
'lost_ble': 0
}
ttl_receive = 3 - int(data['_command']['relay'])
double_mex = list()
error_1 = list()
error_2 = list()
diff_ble = list()
latency_ble = list()
hash_json_data = dict()
start = ""
end_send = ""
end_test = ""
for k, v in hash_table.items():
for i in v:
word_info, statistic, double_mex = my.look_into_it(
word_info, statistic, mex_correct[i], double_mex)
if not k in hash_json_data:
hash_json_data[k] = []
hash_json_data[k].append(mex_correct[i])
# BLE
if 'send_ble' in word_info:
time = dt.datetime.strptime(word_info['send_ble']['time'],
'%Y-%m-%d %H:%M:%S.%f')
if start == '' and end_send == '':
start = time
end_send = start
elif time > end_send:
end_send = time
if 'receive_ble' in word_info:
if not int(word_info['send_ble']['ttl']) == 3:
print('\x1b[1;31;40m Error analysis_ttl --> send_: ' +
word_info['send_ble']['time'] + '\x1b[0m')
error_1.append({'k': k, 'time': word_info['send_ble']['time']})
if not int(word_info['receive_ble']['ttl']) == ttl_receive:
print('\x1b[1;31;40m Error analysis_ttl --> receive_: ' +
word_info['receive_ble']['time'] + '\x1b[0m')
error_2.append({'k': k, 'time': word_info['send_ble']['time']})
sent = dt.datetime.strptime(word_info['send_ble']['time'],
'%Y-%m-%d %H:%M:%S.%f')
receive_ble = dt.datetime.strptime(
word_info['receive_ble']['time'], '%Y-%m-%d %H:%M:%S.%f')
difference = receive_ble - sent
diff_ble.append(abs(difference.total_seconds()))
latency_ble.append(abs(difference.total_seconds()) / 2)
hash_json_data[k].append({
'ble': {
'send_time': sent,
'status_time': receive_ble,
'difference': abs(difference.total_seconds()),
'latency': (abs(difference.total_seconds()) / 2)
}
})
if end_test == '' or receive_ble > end_test:
end_test = receive_ble
elif 'error_ble' in word_info:
statistic['send_ble'] -= 1
else:
statistic['lost_ble'] += 1
word_info.clear()
if len(double_mex) != 0:
for k in double_mex:
print(hash_json_data[k])
raise Exception(
'Indici Duplicati, esempio R,34,2 - R,34,1 --> Rimuovere R,34,2')
if len(error_1) > 0 or len(error_2) > 0:
raise Exception('\x1b[1;31;40m Error TTL: sent: ' + str(len(error_1)) +
' --> received: ' + str(len(error_2)) + ' \x1b[0m')
print("Media DIff BLE: {}s".format(statistics.mean(diff_ble)))
print("Media Latency BLE: {}s".format(statistics.mean(latency_ble)))
print("Total Send -> [{}]".format(data['_command']['n_mex']))
print("S (ble) -> {}".format(statistic['send_ble']))
print("R (ble) -> {}".format(statistic['receive_ble']))
print("E (ble) -> {}".format(statistic['error_ble'])) # not_sent_ble
print("Lost BLE: {}".format(statistic['lost_ble']))
# Check statistics
if data['_command']['n_mex'] - statistic['send_ble'] - statistic[
'error_ble'] != 0:
raise Exception('\x1b[1;31;40m Error BLE: mex number [T-S-E] \x1b[0m')
elif statistic['send_ble'] - statistic['receive_ble'] != statistic[
'lost_ble']:
raise Exception('\x1b[1;31;40m Error BLE: mex number [S-R] \x1b[0m')
else:
print('\x1b[1;31;42m Check OK \x1b[0m')
# Time definition
diff_ = end_send - start
h2, m2, s2 = my.convert_timedelta(diff_)
diff = end_test - start
h1, m1, s1 = my.convert_timedelta(diff)
t1 = str(h2) + ":" + str(m2) + "." + str(s2) + " [h:m.s]"
t2 = str(h1) + ":" + str(m1) + "." + str(s1) + " [h:m.s]"
print("Start: {}".format(start))
print("End_sent: {}".format(end_send))
print("End_test: {}".format(end_test))
print("Sent: {}:{}.{} [mm:s.us]".format(m2, s2, diff_.total_seconds()))
print("Test: {}:{}.{} [mm:s.us]".format(m1, s1, diff.total_seconds()))
my_data = dict()
my_data['_command'] = data['_command']
my_data['_info'] = {
'start': start,
'end_sent': end_send,
'end_test': end_test,
'time_send': t1,
'time_test': t2
}
my_data['_info_2'] = {
'mean': {
'ble_mean_diff': statistics.mean(diff_ble),
'ble_mean_latency': statistics.mean(latency_ble)
},
'mex_': {
'ble': {
'send_ble': statistic['send_ble'],
'receive_ble': statistic['receive_ble'],
'not_send_ble': statistic['error_ble'],
'lost_ble': statistic['lost_ble']
}
}
}
my_data['_mex'] = hash_json_data
print_info(data['_command'])
return my_data