def validate(terms, uobj):
# Get current user's subreddits
rinfo = RedditData(uobj.redtoken, uobj.redrefresh)
user_subs = set(rinfo.get_subs())
all_subs = []
for term in terms:
# Search and parse
results = search_reddit(term)
names_results = [sub['name'] for sub in results]
[all_subs.append(sub) for sub in names_results]
sub_matches = user_subs.intersection(names_results)
# Make results object
res_ob = {
'term': term,
'num_results': len(names_results),
'num_matches': len(sub_matches)
}
# Save to database
Stats.add_data(res_ob, uobj)
# Find number of user subs that matched
all_matches = user_subs.intersection(all_subs)
Stats.add_data(
{
'term': 'allsubs',
'num_results': len(user_subs),
'num_matches': len(all_matches)
}, uobj)
def index():
""" Home page. Has a jumbotron explaining the site, with a search-box call to action. Lists latest addeed replays,
latest archived replays, some public stats about the state of the site. """
# Redirect to "goodbye-world" blug
return redirect(url_for('goodbye_world'))
last_added_replays = get_last_added_replays()
last_archived_replays = get_last_archived_replays()
most_favourited_replays = get_most_favourited_replays()
most_liked_replays = get_most_liked_replays()
most_downloaded = get_most_downloaded_replays()
most_downloaded_30days = get_most_downloaded_30days_replays()
stats = Stats()
search_form = SearchForm()
return render_template("dotabank.html",
last_added_replays=last_added_replays,
last_archived_replays=last_archived_replays,
most_favourited_replays=most_favourited_replays,
most_liked_replays=most_liked_replays,
most_downloaded=most_downloaded,
most_downloaded_30days=most_downloaded_30days,
stats=stats,
search_form=search_form)
def scrape_channel(channel_id):
for videoId in get_channel_videos(channel_id):
videoData = get_video_data(videoId)
video = db.session.query(Video).filter_by(videoId=videoId).first()
if video:
pass
else:
video = Video(videoId=videoId,
title=videoData['title'],
channelId=channel_id,
publishedAt=videoData['publishedAt'],
description=videoData['description'],
thumbnail=videoData['thumbnail'])
db.session.add(video)
db.session.commit()
if videoData['tags']:
for tag in videoData['tags']:
video.tags.append(get_or_create_tag(tag))
db.session.commit()
stats = Stats(videoId=video.id,
viewCount=videoData['viewCount'],
commentCount=videoData['commentCount'],
likeCount=videoData['likeCount'],
dislikeCount=videoData['dislikeCount'],
favoriteCount=videoData['favoriteCount'])
db.session.add(stats)
db.session.commit()
def before_request():
visits = Stats.query.filter_by(day_visits = date.today()).first()
if visits is None:
new_day_visits = Stats()
db.session.add(new_day_visits)
db.session.commit()
else:
visits.visits += 1
db.session.commit()
def addToStats(track, listeningPercentage, user):
if Stats.objects.filter(user=user, track=track).count() == 0:
stat = Stats()
stat.track = track
stat.user = user
stat.playCounter = 1
stat.listeningPercentage = listeningPercentage
else:
stat = Stats.objects.get(user=user, track=track)
stat.playCounter += 1
stat.listeningPercentage += listeningPercentage
stat.save()
def test_check_threshold(self):
clients = self.get_client_nodes()
data = Main().parse_client(clients[0])
stats = Stats.get_or_create(**data)
with patch("app.tasks.send_email.delay"):
data = Main().parse_client(clients[0])
stats.cpu_usage = 18
stats.mem_usage = 55
cpu_alert, mem_alert = check_threshold(stats)
self.assertEqual(cpu_alert, False)
self.assertEqual(mem_alert, True)
def CV_count():
db.create_all()
today = str(dt.datetime.now())[:10]
a = Stats.is_date_in_database(today)
if a:
object = Stats.get_object_by_date(today)
object.visits += 1
Stats.save_to_db(object)
else:
new = Stats(date=today, visits=1)
Stats.save_to_db(new)
return redirect(url_for('report'))
def register():
if current_user.is_authenticated:
return redirect(url_for('index'))
form = RegistrationForm()
if form.validate_on_submit():
user = User(username=form.username.data, email=form.email.data)
user.set_password(form.password.data)
stat = Stats(login_timestamp = datetime.now(), user_id=current_user.get_id())
db.session.add(user)
db.session.add(stat)
db.session.commit()
flash('congratulations, you are in the database')
# session['logged_in_user'] = True
return redirect(url_for('index'))
return render_template('register.html',title='register', form=form)
def index():
stat_obj = Stats.query.get(1)
if (not stat_obj):
tot_users = User.query.count()
tot_urls = Links.query.count()
tot_clicks_obj = db.session.query(Links, db.func.sum(Links.clicks))
tot_clicks = tot_clicks_obj[0][1]
stat_obj = Stats(total_clicks=tot_clicks,
total_urls=tot_urls,
total_users=tot_users)
db.session.add(stat_obj)
db.session.commit()
tot_users = stat_obj.total_users
tot_urls = stat_obj.total_urls
tot_clicks = stat_obj.total_clicks
return render_template('index.html',
tot_clicks=tot_clicks,
tot_urls=tot_urls,
tot_users=tot_users)
def initialize_db():
# initialize Kalman Filter
u = Kalman(Q=Q, R=R, X0=X0, P0=P0, z=0, placement=0)
db.session.add(u)
db.session.commit()
l = Stats(requests_admitted=0,
requests_rejected=0,
requests_locally=0,
predicted_workload=0,
placement=0,
total_requests=0,
vdu1_requests=0,
vdu2_requests=0,
vdu3_requests=0,
average_computation_time_vdu1=0,
average_computation_time_vdu2=0,
average_computation_time_vdu3=0)
db.session.add(l)
db.session.commit()
return ("OK")
def temp_record_deamon():
"""loops the hat"""
sense = SenseHat()
sense.low_light = True
c_hist = []
while True:
sense.clear()
draw_temp_scale_in_col(0, sense)
current_temp = round(sense.get_temperature(), 1)
stat_record = Stats(temp=current_temp)
db.session.add(stat_record)
db.session.commit()
print(current_temp)
c_hist.append(current_temp)
if len(c_hist) > 7:
c_hist.pop(0)
for i in range(min(len(c_hist), 7)):
current_c_hist = c_hist[-(i + 1)]
draw_temp_in_col(i + 1, current_c_hist, sense)
sleep(10)
def get(self, segment):
if not current_user.is_authenticated:
return {'err': 'auth'}, 401
# See the model for details
val = Stats(segment).val
if 'traffic' == segment:
return {segment: val}, 200
elif 'users' == segment:
return {segment: val}, 200
elif 'sales' == segment:
return {segment: val}, 200
elif 'perf' == segment:
return {segment: val}, 200
else:
return {'err': 'unknown'}, 404
def charts_input(request):
context = {'segment': 'charts_from_input'}
html_template = loader.get_template('charts-from-input.html')
# -----------------------------------------------
# Use data from STATS Table
# -----------------------------------------------
stats, labels = Stats.get_report()
data = [{
'y': year,
'a': '{:.2f}'.format(stats[year].get('prod1_sales')),
'b': '{:.2f}'.format(stats[year].get('prod2_sales')),
'c': '{:.2f}'.format(stats[year].get('prod3_sales'))
} for year in stats]
context['chart_data'] = json.dumps({
'element':
'morris-bar-chart',
'data':
data,
'xkey':
'y',
'barSizeRatio':
0.70,
'barGap':
3,
'resize':
True,
'responsive':
True,
'ykeys': ['a', 'b', 'c'], # it can be custom
'labels':
labels,
'barColors': ['0-#1de9b6-#1dc4e9', '0-#899FD4-#A389D4',
'#04a9f5'] # it can be custom
})
return HttpResponse(html_template.render(context, request))
def post(id): #View for full post, since the blog and index view only display title + description
post = Post.query.filter_by(id=id).first()
page = request.args.get('page', 1, type=int)
comments = Comment.query.filter_by(post_id=id).order_by(Comment.timestamp.desc()).paginate(
page, app.config['COMMENTS_PER_POST'], False)
next_page = url_for('post', id=id, page=comments.next_num) if comments.has_next else None
prev_page = url_for('post', id=id, page=comments.prev_num) if comments.has_prev else None
form = CommentForm()
if form.validate_on_submit(): #Post a new comment
comment = Comment(email = form.email.data, name = form.name.data, comment = form.comment.data, post_id=id)
db.session.add(comment)
db.session.commit()
stat_comment = Stats.query.filter_by(day_comments = date.today()).first()
if stat_comment is None: #Probably will never happen since need visit first
new_stat = Stats()
db.session.add(new_stat)
db.session.commit()
else:
stat_comment.comments += 1
db.session.commit()
flash('Your comment has been added.')
return redirect(url_for('post', id=id))
return render_template('post.html', title=post.title, form=form, post=post,
comments=comments.items, next_page=next_page, prev_page=prev_page)
def login():
if current_user.is_authenticated:
return redirect(url_for('index'))
form = LoginForm()
if form.validate_on_submit():
user = User.query.filter_by(username=form.username.data).first()
if user is None or not user.check_password(form.password.data):
flash('Invalid username or password')
return redirect(url_for('login'))
login_user(user,True)
print(user)
# session['logged_in_user'] = True
flash("User logged in.")
stat = Stats(login_timestamp = datetime.now(), user_id=current_user.get_id())
db.session.add(stat)
db.session.commit()
next_page = request.args.get('next')
# check if the value in the next parameter is null, next page or a whole domain
# the domain in checked using netloc
if not next_page or url_parse(next_page).netloc != '':
next_page = url_for('index')
return redirect(next_page)
return render_template('login.html',title='Sign In', form=form)
def handle(self,*args,**options):
""" Initializing variables """
daily_rate = []
daily_projects=[]
point_list=[]
start = date(2015,8,1)
end = datetime.today()
y = end.year
m = end.month
d = end.day
end = date(y,m,d)
dateList = date_range(start, end)
""" Recolect data for daily_rate & daily_projects """
for n in dateList:
#mydates.append(n.strftime("%d/%m")) #used for x axis in stats
files = File.objects.filter(time=n)
daily_rate.append(files.aggregate(Avg("score"))["score__avg"])
for k in daily_rate:
if k == None:
daily_rate[daily_rate.index(k)]= 0
for i in files:
#mastery_list.append(i.score)
point_list.append(i.score)
daily_projects.append(len(files))
""" Stats by CT level """
master = 0
development = 0
basic = 0
totalProjects = File.objects.count()
for i in point_list:
if i >= 15:
master = master + 1
elif i > 7:
development = development + 1
else:
basic = basic + 1
basic = basic*100/totalProjects
development=development*100/totalProjects
master=master*100/totalProjects
"""This section calculates the average score by programming skill """
parallelism = File.objects.all().aggregate(Avg("parallelization"))
parallelism = int(parallelism["parallelization__avg"])
abstraction = File.objects.all().aggregate(Avg("abstraction"))
abstraction = int(abstraction["abstraction__avg"])
logic = File.objects.all().aggregate(Avg("logic"))
logic = int(logic["logic__avg"])
synchronization = File.objects.all().aggregate(Avg("synchronization"))
synchronization = int(synchronization["synchronization__avg"])
flowControl = File.objects.all().aggregate(Avg("flowControl"))
flowControl = int(flowControl["flowControl__avg"])
userInteractivity = File.objects.all().aggregate(Avg("userInteractivity"))
userInteractivity = int(userInteractivity["userInteractivity__avg"])
dataRepresentation = File.objects.all().aggregate(Avg("dataRepresentation"))
dataRepresentation = int(dataRepresentation["dataRepresentation__avg"])
"""This section calculates the average score by code smell"""
deadCode = File.objects.all().aggregate(Avg("deadCode"))
deadCode = int(deadCode["deadCode__avg"])
duplicateScript = File.objects.all().aggregate(Avg("duplicateScript"))
duplicateScript = int(duplicateScript["duplicateScript__avg"])
spriteNaming = File.objects.all().aggregate(Avg("spriteNaming"))
spriteNaming = int(spriteNaming["spriteNaming__avg"])
initialization = File.objects.all().aggregate(Avg("initialization"))
initialization = int(initialization["initialization__avg"])
"""This section calculates the average score by code smell"""
self.stdout.write("Doing all the stats!")
stats_today = Stats(daily_score=daily_rate,
basic= basic,
development = development,
master = master,
daily_projects=daily_projects,
parallelism=parallelism,
abstraction = abstraction,
logic = logic,
synchronization = synchronization,
flowControl = flowControl,
userInteractivity = userInteractivity,
dataRepresentation = dataRepresentation,
deadCode = deadCode,
duplicateScript = duplicateScript,
spriteNaming = spriteNaming,
initialization = initialization
)
stats_today.save()
def update_stats(self):
# Should be called whenever there is a change in the stats - New games atm
games = game_data.loadUser(self.user.id)['games']
games_played = len(games)
games_won = total_points = total_points_against = 0
users_played = []
for game_id in games:
game = game_data.loadGame(game_id)
if int(game['winner']) == self.user.id:
games_won += 1
if int(game['player1']['id']) == self.user.id:
# If player is P1
total_points += game['player1']['score']
total_points_against += game['player2']['score']
users_played.append(int(game['player2']['id']))
elif int(game['player2']['id']) == self.user.id:
# If player is P2
total_points += game['player2']['score']
total_points_against += game['player1']['score']
users_played.append(int(game['player1']['id']))
counted = Counter(users_played).most_common(1)
if counted:
most_played, games_against_most_played = counted[0]
else:
most_played = None
games_against_most_played = 0
# compute averages
if games_played: # Don't compute if no games (ZeroDivisionError)
avg_points = round(total_points / games_played, 1)
avg_points_against = round(total_points_against / games_played, 1)
win_ratio = round(games_won / games_played, 2)
else:
avg_points = avg_points_against = win_ratio = 0
# Save to db
if self.user.stats:
self.user.stats.games_played = games_played
self.user.stats.games_won = games_won
self.user.stats.total_points = total_points
self.user.stats.total_points_against = total_points_against
self.user.stats.avg_points = avg_points
self.user.stats.avg_points_against = avg_points_against
self.user.stats.most_played = most_played
self.user.stats.games_against_most_played = games_against_most_played
self.user.stats.win_ratio = win_ratio
else:
user_stats = Stats(
user_id=self.user.id,
games_played=games_played,
games_won=games_won,
total_points=total_points,
total_points_against=total_points_against,
avg_points=avg_points,
avg_points_against=avg_points_against,
most_played=most_played,
games_against_most_played=games_against_most_played,
win_ratio=win_ratio
)
db.session.add(user_stats)
db.session.commit()
def api_latest(game_code):
game = Games.query.filter_by(code=game_code).first_or_404()
if game_code[0] == "H": # If home game...
stands = {
"east": {
"main": ["ES-EA", "ES-EB", "ES-EC", "ES-ED", "ES-EE"]
},
"south": {
"upper": ["SU-SUF", "SU-SUG", "SU-SUHB", "SU-SUHY", "SU-SUI", "SU-SUJ"],
"lower": ["SL-SLK", "SL-SLL", "SL-SLMB", "SL-SLMY", "SL-SLN", "SL-SLO"]
},
"west": {
"family": ["WS-WP", "WS-WQ", "WS-WR"],
"main": ["WS-WS", "WS-WT", "WS-WU", "WS-WV", "WS-WW"],
"x": ["WS-WX"]
},
"north": {
"yz": ["NS-NWW"]
}
}
output = {
"east": {"main": [0, 0]},
"south": {"upper": [0, 0], "lower": [0, 0]},
"west": {"family": [0, 0], "main": [0, 0], "x": [0, 0]},
"north": {"yz": [0, 0]},
"total": [0, 0]
}
if not game.east_stand: del stands["east"]; del output["east"]
if not game.south_stand: del stands["south"]; del output["south"]
if not game.x_block: del stands["west"]["x"]; del output["west"]["x"]
if not game.yz_block: del stands["north"]["yz"]; del output["north"]["yz"]
if not game.west_stand: del stands["west"]; del output["west"]
for stand in stands:
for area in stands[stand]:
for block in stands[stand][area]:
data = seating_req_json.format(block, game_code)
response = requests.post(
"https://ticketing.southend-united.co.uk/PagesPublic/ProductBrowse/VisualSeatSelection.aspx/GetSeating",
data=data,
headers={'content-type': "application/json; charset=UTF-8"}
)
response.encoding = "unicode-escape"
x = xmltodict.parse(json.loads(response.text)["d"])["seats"]["s"]
total_sold = sum(1 for d in x if d.get("@a") == "." and "Segregation" not in d.get("@rsDesc"))
total_available = sum(1 for d in x if d.get("@a") == "A") + sum(1 for d in x if d.get("@a") == "X")
output[stand][area][0] += total_sold # Total seats
output[stand][area][1] += total_available # Total available
output["total"][0] += total_sold
output["total"][1] += total_available
output["time"] = datetime.datetime.utcnow().strftime("%H:%M:%S")
new = Stats(
game_id=game.id,
time=datetime.datetime.utcnow(),
total_sold=output["total"][0],
total_available=output["total"][1]
)
db.session.add(new)
db.session.commit()
return jsonify(output)
elif game_code[0] == "A":
output = {"total": [0, 0], "away": True}
response = requests.get(
"https://ticketing.southend-united.co.uk/PagesPublic/ProductBrowse/productAway.aspx",
headers={'content-type': "application/json; charset=UTF-8"},
verify=False
)
soup = BeautifulSoup(response.text, "html5lib")
game_soup = soup.find_all("div", {"class": game_code})[1]
inputs = game_soup.find_all("input")
value = int(inputs[3]["value"])
output["total"][1] = value # Total available.
try:
first_stat = Stats.query.filter_by(game_id=game.id).order_by(Stats.id.asc()).first()
output["total"][0] = first_stat.total_available - value # Total sold. (very rough/fragile)
except AttributeError: # Usually if it's the first time running...
output["total"][0] = 0 # This is where we get the inaccuracy.
output["time"] = datetime.datetime.utcnow().strftime("%H:%M:%S")
new = Stats(
game_id=game.id,
time=datetime.datetime.utcnow(),
total_sold=output["total"][0],
total_available=output["total"][1]
)
db.session.add(new)
db.session.commit()
return jsonify(output)
return abort(500)
def test_get_or_create(self):
clients = self.get_client_nodes()
data = Main().parse_client(clients[0])
stats = Stats.get_or_create(**data)
self.assertEqual(stats.ip, "11.11.11.11")
def update_kalman_placement():
kalman = Kalman.query.first()
###TODO accumulate stats for this interval
p = db.session.query(func.max(Stats.id)).scalar()
s = Stats.query.filter_by(id=p).first()
s.placement = kalman.placement
s.requests = s.requests_admitted + s.requests_rejected
s.predicted_workload = kalman.X0
try:
s.average_computation_time_vdu1 = s.average_computation_time_vdu1 / s.vdu1_requests
except ZeroDivisionError:
s.average_computation_time_vdu1 = 0
try:
s.average_computation_time_vdu2 = s.average_computation_time_vdu2 / s.vdu2_requests
except ZeroDivisionError:
s.average_computation_time_vdu2 = 0
try:
s.average_computation_time_vdu3 = s.average_computation_time_vdu3 / s.vdu3_requests
except ZeroDivisionError:
s.average_computation_time_vdu3 = 0
# start of kalman update
Q = kalman.Q
p0 = kalman.P0
x0 = kalman.X0
R = kalman.R
z = kalman.z
# z is given by the actual requests processed in this interval
xkp = x0
pkp = p0 + Q
Kk = pkp / (pkp + R)
xke = xkp + Kk * (z - xkp)
pk = (1 - Kk) * pkp
x0 = xke # return please
p0 = pk # return please
print("new X0 for Kalman: " + str(x0))
print("z for this interval: " + str(z))
print("new P0 for Kalman: " + str(p0))
kalman.X0 = x0 # x0 are the requests predicted for the next interval!!
kalman.P0 = p0
kalman.z = 0
# choose proper placement for load balancing
for item in placement_requests:
if x0 <= item:
topology = placement_requests.index(item)
break
print("new topology index for the next interval:" + str(topology))
kalman.placement = topology
db.session.commit()
# new interval new stats
l = Stats(requests_admitted=0,
requests_rejected=0,
requests_locally=0,
predicted_workload=0,
placement=0,
total_requests=0,
vdu1_requests=0,
vdu2_requests=0,
vdu3_requests=0,
average_computation_time_vdu1=0,
average_computation_time_vdu2=0,
average_computation_time_vdu3=0)
db.session.add(l)
db.session.commit()
