def analyse():
logs = connectDB.connectToCollection(collection_logs)
analysis = connectDB.connectToCollection(collection_analysis)
# items table
categs = getData.getAllExistingCategories(logs)
items = {}
for categ in categs:
items[categ] = getData.getAllIdFromCategory(logs, categ)
nb_line = max(len(value) for key, value in items.items())
# selection with button
chartLastFreq = None
chartPrediction = None
chartFullPeriod = None
heatMap = None
render = None
if request.method == 'POST':
req = request.form['item']
if req == 'all':
resheatMap = getData.getHeatMap(analysis, 'correlation')
heatMap = resheatMap['jsondumps']
else:
item = req.split()
item[1] = int(item[1])
resLastFreq = getData.getChart(analysis, 'lastObjectFreq', item[0],
item[1])
chartLastFreq = resLastFreq['jsondumps']
resPrediction = getData.getChart(analysis, 'prediction', item[0],
item[1])
chartPrediction = resPrediction['jsondumps']
resFullPeriod = getData.getChart(analysis, 'fullPeriod', item[0],
item[1])
chartFullPeriod = resFullPeriod['jsondumps']
else:
resheatMap = getData.getHeatMap(analysis, 'correlation')
heatMap = resheatMap['jsondumps']
return render_template("analyses.html",
items=items,
nb_line=nb_line,
plotLastFreq=chartLastFreq,
plotPredition=chartPrediction,
plotFullPeriod=chartFullPeriod,
plotHeatMap=heatMap)
def action_to_list(category, object_id, maxTime):
"""
Create a time-based status list for a given object.
category: category of the object
object_id: id of the object
maxTime: used to limit the amount of data to be processed.
"""
logs = connectDB.connectToCollection('logs4')
action_list = getData.getItem(
logs, category, object_id,
['EDGE RUNNED']) # retrieves all the actions of an object
time = [0]
state = [0]
i = 0
for action in action_list:
if time[-1] > maxTime:
break
while len(time) < action['Begin Date']:
if time[-1] > maxTime:
break
time += [i]
i += 1
state.append(state[-1])
if time[-1] <= maxTime:
time += [i]
i += 1
state += [int(action['Ending State'][1])]
return [time,
state] # return a list containing, a time list and a status list.
def correlation(maxSize):
"""
return the correlation matrix of all the objects
maxSize: the maximum size of the dataframe, (if it is too high it may create collumns with differents sizes and create a bug)
"""
# get the logs
logs = connectDB.connectToCollection('logs4')
categories = getData.getAllExistingCategories(logs)
data = {}
# build a dataframe with each column being a unique category,id couple
df = pd.DataFrame(data)
for category in categories:
ids = getData.getAllIdFromCategory(logs, category)
for id_c in ids:
a = action_to_list(category, int(id_c), float('inf'))[1]
df[category + ' ' + str(id_c)] = a[0:maxSize]
# create the correlation matrix
corr = df.corr()
# plot it
fig = go.Figure(data=go.Heatmap(x=corr.index.tolist(),
y=corr.index.tolist(),
z=corr,
colorscale="RdBu"))
fig.update_layout(height=700)
fig.show()
return fig
def getAllItems():
logs = connectDB.connectToCollection('logs4')
categs = getData.getAllExistingCategories(logs)
items = {}
for categ in categs:
items[categ] = getData.getAllIdFromCategory(logs, categ)
return items
def lastObjectFreq(category, object_id, time):
"""
calculates the frequency of occurrence of all objects before the action of the given object in the time period.
category: category of the object
object_id: id of the object
time: time period in which we will check the occurence of other objects
"""
logs = connectDB.connectToCollection('logs4')
# retrieve all the actions of an object
action_lists = getData.getItem(logs, category, object_id, ['EDGE RUNNED'])
dico_frequency = {}
nb_actions = len(action_lists)
# go through the actions
for action in action_lists:
# we get back all the actions that preceded our action
previous_action = getData.getDate(logs, 'begin', action['Begin Date'],
action['Begin Date'] + time,
['EDGE RUNNED'])
past_action = []
# we go through all the previous actions
for p_action in previous_action:
action_category = p_action['Category']
action_id = p_action['Id']
action_command = p_action['Function']
# if we haven't already added the action to our dictionary, we'll add it.
if [action_category, action_id] not in past_action:
past_action.append([action_category, action_id])
# if the key already exists we increment the counter
if (dico_frequency.get(action_category,
{}).get(action_id,
{}).get(action_command)):
dico_frequency[action_category][action_id][
action_command] += 1
# otherwise we create the key
elif action_category not in dico_frequency.keys():
dico_frequency[action_category] = {
action_id: {
action_command: 1
}
}
elif action_id not in dico_frequency[action_category].keys():
dico_frequency[action_category][action_id] = {
action_command: 1
}
else:
dico_frequency[action_category][action_id][
action_command] = 1
return (dico_frequency, nb_actions)
import Transfer.getData as getData
import Transfer.connectDB as connectDB
import Analysis.generate as generate
import os
import env
# Parameters
logs_file = "Files/WOF3.log"
logs_collection = os.getenv('COLLECTION_LOGS')
analysis_collection = os.getenv('COLLECTION_ANALYSIS')
# Send logs to DB
logsDB = connectDB.connectToCollection(logs_collection)
filtered_data = filterData.LineReadingFromFile(logs_file)
sendData.send_items(filtered_data, logsDB)
# Send analysis to DB
analysisDB = connectDB.connectToCollection(analysis_collection)
items = generate.getAllItems()
lastObjectFreq = generate.createGraphLastObjectFreq(items)
sendData.send_items(lastObjectFreq, analysisDB)
fullPeriod = generate.createFullPeriodGraph(items)
sendData.send_items(fullPeriod, analysisDB)
predictions = generate.createPredictions(items)
def logs():
logs = connectDB.connectToCollection(collection_logs)
# items table
categs = getData.getAllExistingCategories(logs)
items = {}
for categ in categs:
items[categ] = getData.getAllIdFromCategory(logs, categ)
nb_line = max(len(value) for key, value in items.items())
# events table
actions = getData.getAllExistingActions(logs)
# selection with button
chart = None
if request.method == 'POST':
req = request.form['item']
# all
if req == 'all':
events = getData.getAll(logs, 100, True)
else:
item = req.split()
if len(item) == 1: #category
events = getData.getCategory(logs, item[0], actions, 100, True)
else: #item
item[1] = int(item[1])
events = getData.getItem(logs, item[0], item[1], actions, 100,
True)
# create plot
bar_name = ['']
data = []
fig = go.Figure()
for i in range(len(events)):
if events[i]['Action'] == 'EDGE RUNNED':
rollingShutterState = ""
heaterState = ""
if (events[i]['Category'] in ['ROLLINGSHUTTER']):
rollingShutterState = int(
events[i]['Begin State'][1])
elif (events[i]['Category'] in ['HEATER']):
heaterState = int(events[i]['Begin State'][1])
ShutterColors = [
'#1D2000', '#3B4000', '#768000', '#939F00',
'#B1BF00', '#ECFF00'
]
heaterColors = [
'#FFDFDF', '#FF9F9F', '#FF8080', '#FF6060',
'#FF4040', '#FF0000'
]
y = events[i]['Begin Date']
if i > 1:
y -= events[i - 1]['Begin Date']
if ((events[i]['Command'] in ['OFF'])
and (events[i]['Category'] not in ['HEATER'])):
fig.add_trace(
go.Bar(name='ON',
y=bar_name,
x=[y],
marker=dict(color=['green']),
orientation='h'))
elif ((events[i]['Command'] in ['ON'])
and (events[i]['Category'] not in ['HEATER'])):
fig.add_trace(
go.Bar(name='OFF',
y=bar_name,
x=[y],
marker=dict(color=['red']),
orientation='h'))
elif rollingShutterState in [0, 1, 2, 3, 4, 5]:
fig.add_trace(
go.Bar(name=rollingShutterState,
y=bar_name,
x=[y],
marker=dict(color=ShutterColors[
rollingShutterState]),
orientation='h'))
elif heaterState in [0, 1, 2, 3, 4, 5]:
fig.add_trace(
go.Bar(name=heaterState,
y=bar_name,
x=[y],
marker=dict(
color=heaterColors[heaterState]),
orientation='h'))
fig.update_layout(
width=800,
height=50,
margin=dict(
l=10,
r=10,
b=10,
t=10,
),
barmode='stack',
showlegend=False,
)
chart = json.dumps(fig, cls=plotly.utils.PlotlyJSONEncoder)
# default page
else:
events = getData.getAll(logs, 100, True)
return render_template("logs.html",
items=items,
nb_line=nb_line,
events=events,
plot=chart)