def index(self):
"""Home page with the latest test results"""
# Migration (harstorage v1.0)
migration_handler = MongoDB(collection="migration")
if hasattr(c, "message"): return render("/error.html")
status = migration_handler.collection.find_one({"status": "ok"})
if status is None: redirect("/migration/status")
# MongoDB handler
mdb_handler = MongoDB()
if hasattr(c, "message"): return render("/error.html")
# Read aggregated data from database
# Aggregation is based on unique labels, urls and latest timestamps
latest_results = mdb_handler.collection.group(
key=["label", "url"],
condition=None,
initial={"timestamp": "1970-01-01 01:00:00"},
reduce="\
function(doc, prev) { \
if (doc.timestamp > prev.timestamp) { \
prev.timestamp = doc.timestamp; \
} \
}")
key = lambda timestamp: timestamp["timestamp"]
latest_results = sorted(latest_results, key=key, reverse=True)
# Numner of records
c.rowcount = len(latest_results)
# Populate data table with the latest test results
c.metrics_table = [[], [], [], [], [], []]
fields = [
"timestamp", "label", "url", "total_size", "requests",
"full_load_time"
]
for group in latest_results:
condition = {
"label": group["label"],
"timestamp": group["timestamp"]
}
result = mdb_handler.collection.find_one(condition, fields=fields)
c.metrics_table[0].append(result["timestamp"])
c.metrics_table[1].append(result["label"])
c.metrics_table[2].append(result["url"])
c.metrics_table[3].append(result["total_size"])
c.metrics_table[4].append(result["requests"])
c.metrics_table[5].append(
round(result["full_load_time"] / 1000.0, 1))
return render("/home/core.html")
def migration(self):
# MongoDB handler
mdb_handler = MongoDB()
if hasattr(c, "message"): return render("/error.html")
for document in mdb_handler.collection.find(fields=["_id", "har"]):
id = document["_id"]
har = HAR(document["har"], True)
har.analyze()
domains_req_ratio = dict()
domains_weight_ratio = dict()
for key, value in har.domains.items():
domains_req_ratio[key] = value[0]
domains_weight_ratio[key] = value[1]
data = {
"full_load_time": har.full_load_time,
"onload_event": har.onload_event,
"start_render_time": har.start_render_time,
"time_to_first_byte": har.time_to_first_byte,
"total_dns_time": har.total_dns_time,
"total_transfer_time": har.total_transfer_time,
"total_server_time": har.total_server_time,
"avg_connecting_time": har.avg_connecting_time,
"avg_blocking_time": har.avg_blocking_time,
"total_size": har.total_size,
"text_size": har.text_size,
"media_size": har.media_size,
"cache_size": har.cache_size,
"requests": har.requests,
"redirects": har.redirects,
"bad_requests": har.bad_requests,
"domains": len(har.domains),
"weights_ratio": har.weight_ratio(),
"requests_ratio": har.req_ratio(),
"domains_ratio": har.domains
}
mdb_handler.collection.update({"_id": id}, {"$set": data})
migration_handler = MongoDB(collection="migration")
migration_handler.collection.insert({"status": "ok"})
redirect("/")
def create(self):
"""Render form with list of labels and timestamps"""
# MongoDB handler
md_handler = MongoDB()
if hasattr(c, "message"): return render("/error.html")
# List of labels
c.labels = list()
for label in md_handler.collection.distinct("label"):
c.labels.append(label)
return render("/create/core.html")
def _set_options_in_selector(self, mode, label):
"""
Create context data - a list of timestamps.
@parameter label - label of set with test results
"""
# Read data for selector box from database
results = MongoDB().collection.find({mode: label},
fields=["timestamp"],
sort=[("timestamp", -1)])
c.timestamp = list()
for result in results:
c.timestamp.append(result["timestamp"])
def dates(self):
"""Return a list of timestamps for selected label"""
# Read label from GET request
label = request.GET["label"]
# Read data from database
documents = MongoDB().collection.find({"label": label},
fields=["timestamp"],
sort=[("timestamp", 1)])
dates = str()
for document in documents:
dates += document["timestamp"] + ";"
return dates[:-1]
def deleterun(self):
"""Controller for deletion of tests"""
# MongoDB handler
mdb_handler = MongoDB()
# Parameters from GET request
label = request.GET["label"]
timestamp = request.GET["timestamp"]
mode = request.GET["mode"]
if request.GET["all"] == "true":
del_all = True
else:
del_all = False
# Remove document from collection
if mode == "label":
if del_all:
mdb_handler.collection.remove({"label": label})
else:
mdb_handler.collection.remove({
"label": label,
"timestamp": timestamp
})
count = mdb_handler.collection.find({"label": label}).count()
else:
if del_all:
mdb_handler.collection.remove({"url": label})
else:
mdb_handler.collection.remove({
"url": label,
"timestamp": timestamp
})
count = mdb_handler.collection.find({"url": label}).count()
if count:
return "details?" + mode + "=" + label
else:
return "/"
def upload(self):
"""Controller for uploads of new test results"""
# HAR initialization
try:
har = HAR(request.POST["file"].value)
except:
har = HAR(request.POST["file"])
# Analysis of uploaded data
if har.parsing_status == "Successful":
# Parsing imported HAR file
try:
har.analyze()
except Exception as error:
return False, ": ".join([type(error).__name__, error.message])
# Evaluate Page Speed scores
if config["app_conf"]["ps_enabled"] == "true":
scores = self._get_pagespeed_scores(har.har)
else:
scores = dict([("Total Score", 100)])
# Add document to collection
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
result = {
"label": har.label,
"url": har.url,
"timestamp": timestamp,
"full_load_time": har.full_load_time,
"onload_event": har.onload_event,
"start_render_time": har.start_render_time,
"time_to_first_byte": har.time_to_first_byte,
"total_dns_time": har.total_dns_time,
"total_transfer_time": har.total_transfer_time,
"total_server_time": har.total_server_time,
"avg_connecting_time": har.avg_connecting_time,
"avg_blocking_time": har.avg_blocking_time,
"total_size": har.total_size,
"text_size": har.text_size,
"media_size": har.media_size,
"cache_size": har.cache_size,
"requests": har.requests,
"redirects": har.redirects,
"bad_requests": har.bad_requests,
"domains": len(har.domains),
"ps_scores": scores,
"har": har.origin,
"weights_ratio": har.weight_ratio(),
"requests_ratio": har.req_ratio(),
"domains_ratio": har.domains
}
# MongoDB handler
mdb_handler = MongoDB()
if hasattr(c, "message"):
return False, c.message
else:
mdb_handler.collection.insert(result)
return True, har.label
else:
return False, har.parsing_status
def index(self):
"""Home page with the latest test results"""
# Migration (harstorage v1.0)
migration_handler = MongoDB(collection="migration")
if hasattr(c, "message"):
return render("/error.html")
status = migration_handler.collection.find_one({"status": "ok"})
if status is None:
redirect("/migration/status")
# MongoDB handler
mdb_handler = MongoDB()
if hasattr(c, "message"):
return render("/error.html")
# Read aggregated data from database
# Aggregation is based on unique labels, urls and latest timestamps
'''
Replaced the original grouping with an aggregate function. This function
actually returns all of the fields needed such that we also do not needed
to make any subsequent requests back to MongoDB to retrieve details on
the list
'''
latest_results = mdb_handler.collection.aggregate([{
"$group": {
"_id": {
"label": "$label",
"url": "$url"
},
"timestamp": {
"$last": "$timestamp"
},
"total_size": {
"$last": "$total_size"
},
"requests": {
"$last": "$requests"
},
"full_load_time": {
"$last": "$full_load_time"
}
}
}, {
"$sort": {
"timestamp": -1
}
}])
'''
Get the number of records
Since we changed the initial request, we need to deal with the json array
differently as well.
'''
c.rowcount = len(latest_results["result"])
# Populate data table with the latest test results
c.metrics_table = [[], [], [], [], [], []]
'''
for group in latest_results["result"]:
condition = {"label": group["_id"]["label"], "timestamp": group["timestamp"]}
result = mdb_handler.collection.find_one(condition, fields=fields)
c.metrics_table[0].append(result["timestamp"])
c.metrics_table[1].append(result["label"])
c.metrics_table[2].append(result["url"])
c.metrics_table[3].append(result["total_size"])
c.metrics_table[4].append(result["requests"])
c.metrics_table[5].append(round(result["full_load_time"] / 1000.0, 1))
'''
# loop through our results and return them
for result in latest_results["result"]:
c.metrics_table[0].append(result["timestamp"])
c.metrics_table[1].append(result["_id"]["label"])
c.metrics_table[2].append(result["_id"]["url"])
c.metrics_table[3].append(result["total_size"])
c.metrics_table[4].append(result["requests"])
c.metrics_table[5].append(
round(result["full_load_time"] / 1000.0, 1))
return render("/home/core.html")
def runinfo(self):
"""Generate detailed data for each test run"""
# Parameters from GET request
timestamp = request.GET["timestamp"]
# DB query
test_results = MongoDB().collection.find_one({"timestamp": timestamp})
# Domains breakdown
domains_req_ratio = dict()
domains_weight_ratio = dict()
for hostname, value in test_results["domains_ratio"].items():
hostname = re.sub("\|", ".", hostname)
domains_req_ratio[hostname] = value[0]
domains_weight_ratio[hostname] = value[1]
# Summary stats
summary = {
"full_load_time": test_results["full_load_time"],
"onload_event": test_results["onload_event"],
"start_render_time": test_results["start_render_time"],
"time_to_first_byte": test_results["time_to_first_byte"],
"total_dns_time": test_results["total_dns_time"],
"total_transfer_time": test_results["total_transfer_time"],
"total_server_time": test_results["total_server_time"],
"avg_connecting_time": test_results["avg_connecting_time"],
"avg_blocking_time": test_results["avg_blocking_time"],
"total_size": test_results["total_size"],
"text_size": test_results["text_size"],
"media_size": test_results["media_size"],
"cache_size": test_results["cache_size"],
"requests": test_results["requests"],
"redirects": test_results["redirects"],
"bad_requests": test_results["bad_requests"],
"domains": test_results["domains"]
}
# Page Speed Scores
scores = dict()
for rule, score in test_results["ps_scores"].items():
scores[rule] = score
# Data for HAR Viewer
har_id = str(test_results["_id"])
filename = os.path.join(config["app_conf"]["temp_store"], har_id)
with open(filename, "w") as fh:
fh.write(test_results["har"].encode("utf-8"))
# Final JSON
return json.dumps({
"summary": summary,
"pagespeed": scores,
"weights": test_results["weights_ratio"],
"requests": test_results["requests_ratio"],
"d_weights": domains_weight_ratio,
"d_requests": domains_req_ratio,
"har": har_id
})
def timeline(self):
"""Generate data for timeline chart"""
# Parameters from GET request
label = h.decode_uri(request.GET["label"])
mode = request.GET["mode"]
limit = int(config["app_conf"].get("limit", 0))
# Metrics
METRICS = ("timestamp", "full_load_time", "requests", "total_size",
"ps_scores", "onload_event", "start_render_time",
"time_to_first_byte", "total_dns_time",
"total_transfer_time", "total_server_time",
"avg_connecting_time", "avg_blocking_time", "text_size",
"media_size", "cache_size", "redirects", "bad_requests",
"domains")
TITLES = [
"Full Load Time", "Total Requests", "Total Size",
"Page Speed Score", "onLoad Event", "Start Render Time",
"Time to First Byte", "Total DNS Time", "Total Transfer Time",
"Total Server Time", "Avg. Connecting Time", "Avg. Blocking Time",
"Text Size", "Media Size", "Cache Size", "Redirects",
"Bad Rquests", "Domains"
]
# Set of metrics to exclude (due to missing data)
exclude = set()
data = list()
for index in range(len(METRICS)):
data.append(str())
# Read data for timeline from database in custom format (hash separated)
results = MongoDB().collection.find({mode: label},
fields=METRICS,
limit=limit,
sort=[("timestamp", 1)])
for result in results:
index = 0
for metric in METRICS:
if metric != "ps_scores":
point = str(result[metric])
else:
point = str(result[metric]["Total Score"])
if point == "n/a":
exclude.add(metric)
data[index] += point + "#"
index += 1
# Update list of titles
if "onload_event" in exclude:
TITLES.pop(TITLES.index("onLoad Event"))
if "start_render_time" in exclude:
TITLES.pop(TITLES.index("Start Render Time"))
header = str()
for title in TITLES:
header += title + "#"
output = header[:-1] + ";"
for dataset in data:
if not dataset.count("n/a"):
output += dataset[:-1] + ";"
return output[:-1]
def display(self):
"""Render page with column chart and data table"""
# MongoDB handler
md_handler = MongoDB()
if hasattr(c, "message"):
return render("/error.html")
# Checkbox options
c.chart_type = request.GET.get("chart", None)
c.table = request.GET.get("table", "false")
init = request.GET.get("metric", "true")
c.chart = "true" if c.chart_type else "false"
# Aggregation option
c.agg_type = request.GET.get("metric", "Average")
# Number of records
if c.chart == "true" and c.table == "true" and init != "true":
c.rowcount = len(request.GET) / 3 - 1
else:
c.rowcount = len(request.GET) / 3
# Data table
c.headers = ["Label", "Full Load Time (ms)", "Total Requests",
"Total Size (kB)", "Page Speed Score",
"onLoad Event (ms)", "Start Render Time (ms)",
"Time to First Byte (ms)", "Total DNS Time (ms)",
"Total Transfer Time (ms)", "Total Server Time (ms)",
"Avg. Connecting Time (ms)", "Avg. Blocking Time (ms)",
"Text Size (kB)", "Media Size (kB)", "Cache Size (kB)",
"Redirects", "Bad Rquests", "Domains"]
c.metrics_table = list()
c.metrics_table.append(list())
# Chart points
c.points = str()
# Aggregator
aggregator = Aggregator()
# Test results from database
for row_index in range(c.rowcount):
# Parameters from GET request
label = request.GET["step_" + str(row_index + 1) + "_label"]
start_ts = request.GET["step_" + str(row_index + 1) + "_start_ts"]
end_ts = request.GET["step_" + str(row_index + 1) + "_end_ts"]
# Add label
c.metrics_table[0].append(label)
c.points += label + "#"
# Fetch test results
condition = {
"label": label,
"timestamp": {"$gte": start_ts, "$lte": end_ts}
}
documents = md_handler.collection.find(condition,
fields=aggregator.METRICS)
# Add data row to aggregator
aggregator.add_row(label, row_index, documents)
# Aggregated data per column
column = 1
for metric in aggregator.METRICS:
c.metrics_table.append(list())
c.points = c.points[:-1] + ";"
for row_index in range(c.rowcount):
data_list = aggregator.data[metric][row_index]
value = aggregator.get_aggregated_value(data_list, c.agg_type,
metric)
c.points += str(value) + "#"
c.metrics_table[column].append(value)
column += 1
# Names of series
titles = str()
for title in aggregator.TITLES:
titles += title + "#"
# Final chart points
c.points = titles[:-1] + ";" + c.points[:-1]
c.points = aggregator.exclude_missing(c.points)
return render("/display/core.html")
def histogram(self):
"""Render chart with histograms"""
# MongoDB handler
md_handler = MongoDB()
if hasattr(c, "message"):
return render("/error.html")
# Options
c.label = request.GET["label"]
c.metric = request.GET["metric"]
# Metrics
METRICS = [("full_load_time", "Full Load Time"),
("onload_event", "onLoad Event"),
("start_render_time", "Start Render Time"),
("time_to_first_byte", "Time to First Byte"),
("total_dns_time", "Total DNS Time"),
("total_transfer_time", "Total Transfer Time"),
("total_server_time", "Total Server Time"),
("avg_connecting_time", "Avg. Connecting Time"),
("avg_blocking_time", "Avg. Blocking Time")]
time_metrics = ["full_load_time", "onload_event", "start_render_time",
"time_to_first_byte"]
c.metrics = list()
# Read data from database
condition = {"label": c.label}
fields = (metric for metric, title in METRICS)
documents = md_handler.collection.find(condition, fields=fields)
full_data = list(document for document in documents)
for metric, title in METRICS:
try:
data = (result[metric] for result in full_data)
histogram = Histogram(data)
if metric in time_metrics:
ranges = histogram.ranges(True)
else:
ranges = histogram.ranges()
frequencies = histogram.frequencies()
if metric == c.metric:
c.data = ""
for occ_range in ranges:
c.data += occ_range + "#"
c.data = c.data[:-1] + ";"
for frequency in frequencies:
c.data += str(frequency) + "#"
c.data = c.data[:-1] + ";"
c.title = title
c.metrics.append((metric, title))
except IndexError:
pass
except TypeError:
pass
except ValueError:
pass
if len(c.metrics):
return render("/histogram/core.html")
else:
c.message = "Sorry! You haven't enough data."
return render("/error.html")
