def _initailize_progress(pigeon_key, card_key):
progress = Progress(pigeon_key=pigeon_key,
card_key=card_key,
familiar_factor=0,
learn_factor=1)
progress.put()
return
def post(self):
from models import Course, Progress
user = users.get_current_user()
course_numbers = (self.request.get_all('courses[]'))
current_user = User.all().filter('email =', user.email()).get()
myclasses = []
myclasses2 = []
for classy in course_numbers:
test = [x.strip() for x in classy.split(',')]
myclasses.append(test)
for classy in myclasses:
coursename = classy[0].replace(" ", "")
coursetime = classy[1]
myclasses2.append((coursename,coursetime))
# [u'Fall12,PHYS 5A,Fall12,PHYS 5L,Fall12,CMPS 12B,Fall12,CMPS 12L,Fall12,CMPE 16,',
# u'Winter13,HCI 131 ,Winter13,CMPS 101 ,Winter13,DANM 250,Winter13,Math 21 ,',
# u'Spring13,PHYSC 5C,Spring13,PHYSC 5L,Spring13,AMS 131,',
# u'Summer13,CMPS109,Summer13,Math 24,']
# myclasses = filter(None, myclasses)
# test = [x.strip() for x in myclasses[0].split(',')]
progCheck = Progress.all().filter('user ='******'number =',coursename).get()
progress = Progress(user=current_user, course=course, completed=True, quarter=quarter, year=year)
if (course != None):
progress.put()
# if (course != None):
# quarter = course_number[1][:-2]
# year = int(course_number[1][-2:])
# # need to figure out how to query for specific course
# # so that we can remove it if neccessary
# progress = Progress(user=current_user, course=course, completed=True, quarter=quarter, year=year)
# progCheck = Progress.all().filter('user ='******'course =', course).fetch(limit=20)
# # if (progCheck.quarter != quarter):
# # progress.put()
# # #then remove the old class here too....
log = ""
# if not progCheck:
# progress.put()
# log = "progCheck==None , " + str(course_number)
# else:
# if not any(quarter == chances.quarter for chances in progCheck):
# progress.put()
# # for chances in progCheck:
# # if (chances.quarter != quarter):
# # itsHere
# progress.put()
self.response.write(log + str(myclasses2));
def post(self):
page = self.get_int_param("page")
location = self.get_int_param("location")
book_key_name = self.request.get("book")
if page or location:
try:
entry = Entry.create(book = Book.get_by_key_name(book_key_name),
reader = users.get_current_user(),
page = page,
location = location)
Progress.create(entry)
except ValueError, error:
self.set_cookie("flash_message", error)
def startSession(request):
if request.method == 'POST':
body = json.loads(request.body)
route = Route.objects.filter(pk=body['routeID'])[0]
userID = UserProfile.objects.filter(pk=body['deviceID'])[0]
# stage = Stage.objects.filter(pk=request.GET.get('stageID'))[0]
print route
startStage = None
routeCompleted = RoutesCompleted.objects.get_or_create(routeID=route,
userID=userID)[0]
if routeCompleted.currentJourney is None:
progress = Progress(stageID=route.startStage,
userID=userID)
progress.save()
routeProgress = RouteProgress(progress=progress)
routeProgress.save()
routeProgress.allProgress.add(progress)
routeProgress.save()
routeCompleted.allJourneys.add(routeProgress)
routeCompleted.currentJourney = routeProgress
routeCompleted.save()
routeProgress = routeCompleted.currentJourney
progress = routeProgress.progress
session = Session(userID=userID, routesCompleted=routeCompleted,
lastLat=float(body['lat']), lastLon=float(body['lon']),
lastTime=body['timestamp'])
session.save()
distance = routeCompleted.currentJourney.progress.totalDistance
response = {}
response['id'] = session.pk
response['distance'] = distance
response['distanceRemain'] = progress.stageID.distance - progress.totalDistance
response['stageLength'] = progress.stageID.distance
response['routeDistanceRemain'] = routeCompleted.routeID.length - routeProgress.distance
response['routeLength'] = routeCompleted.routeID.length
response['currentStage'] = routeProgress.progress.stageID.id
response['totalTime'] = session.totalTime
response['stageImage'] = session.routesCompleted.currentJourney.progress.stageID.image
return HttpResponse(json.dumps(response), content_type="application/json")
return HttpResponse("Unauthorized method", status=405)
def _delete_progress(pigeon_key, card_key):
progress_list = Progress.query(Progress.pigeon_key == pigeon_key,
Progress.card_key == card_key).fetch()
if progress_list:
progress = progress_list[0]
progress.key.delete()
return
else:
return
def get_learn_factor(user_id, house_name, card_key_str):
pigeon_key = ndb.Key(Pigeon, user_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
card_key = ndb.Key(Card, card_key_str, parent=house_key)
progress_list = Progress.query(Progress.pigeon_key == pigeon_key,
Progress.card_key == card_key).fetch()
if progress_list:
return progress_list[0].learn_factor
else:
_initailize_progress(pigeon_key, card_key)
return 1
def post(self):
import logging
from models import Course, Progress
user = users.get_current_user()
args = self.request.arguments()
course_numbers = []
# logging.debug("test")
idx = 0
# courses = self.request.get_all(args[0])
# for course in courses:
# for c2 in course:
# course_numbers.append(course)
for arg in args:
course = self.request.get_all(arg)
course.insert(0, arg)
# course = str(course))
course_numbers.append(course)
# idx= idx+1
current_user = User.all().filter('email =', user.email()).get()
# for course in course_numbers:
# course_numbers = args
for coursevals in course_numbers:
number = coursevals[0]
quarteryear = coursevals[1]
quarteryear = [x.strip() for x in quarteryear.split(',')]
if len(quarteryear) == 2:
quarter=quarteryear[0]
year=int(quarteryear[1][2:])
if quarter == "Fall":
year = year + 1
course = Course.all().filter('number =',number).get()
progress = Progress(user=current_user, course=course, quarter=quarter, year=year, completed=True)
progress.put()
# self.response.write("<br>")
# print (str(course_numbers))
return redirect('/dashboard')
def set_learn_factor(user_id, house_name, card_key_str, learn_factor):
pigeon_key = ndb.Key(Pigeon, user_id)
house_list = House.query(House.name == house_name).fetch()
house_key = house_list[0].key
card_key = ndb.Key(Card, card_key_str, parent=house_key)
progress_list = Progress.query(Progress.pigeon_key == pigeon_key,
Progress.card_key == card_key).fetch()
if progress_list:
progress = progress_list[0]
progress.learn_factor = learn_factor
progress.put()
return
else:
return
#!/usr/bin/env python
from models import User, UserPool, Article, ArticlePool, Progress
import requests
import re
from bs4 import BeautifulSoup
#page = requests.get("https://www.ptt.cc/bbs/Gossiping/index20373.html", cookies={"over18":"1"})
#bs = BeautifulSoup(page.content, "html.parser")
#print(list(div.find("a") for div in list(bs.find_all(class_="title"))))
#ap = ArticlePool()
#ap.load()
prg = Progress()
prg.load()
#print(len(ap.articles))
print(prg.current_index)
contributors = list(contributors)
organization.contributors = contributors
try:
session.add(organization)
session.commit()
except:
session.rollback()
if __name__ == "__main__":
logging.basicConfig(filename="scraper.log", level=logging.INFO)
G = Github(ACCESS_TOKEN)
progress = session.query(Progress).first()
if not progress:
progress = Progress(id="progress", value=0)
with open('government.github.com/_data/governments.yml') as infile:
_data = yaml.load(infile)
data = reshape_data(_data)
organizations_government = set(
[organization['entity'].lower() for organization in data])
with open('government.github.com/_data/civic_hackers.yml') as infile:
_data_civic = yaml.load(infile)
data_civic = reshape_data(_data_civic)
organizations_civic = set(
[organization['entity'].lower() for organization in data_civic])
for i in xrange(progress.value, len(data)):
logging.info("{} {} {}".format(i, data[i]['entity'],
def updateSession(request):
if request.method == 'PATCH':
body = json.loads(request.body)
if all(k in body for k in ("sessionID", "lon", "lat", "timestamp")):
try:
session = Session.objects.get(id=body['sessionID'])
except ObjectDoesNotExist:
return HttpResponse("Session Not Found", status=404)
if session.lastTime >= body['timestamp']:
return HttpResponse("Timestamp of request too old", status=400)
currentCoord = (float(session.lastLat), float(session.lastLon))
nextCoord = (float(body['lat']), float(body['lon']))
distance = haversine(currentCoord, nextCoord) * 1000 # to get m not km
timeIncrement = body['timestamp'] - session.lastTime
# distance * 1000 since timeIncrement is in milliseconds
if (distance * 1000 / timeIncrement) > settings.MAX_SPEED:
distance = (settings.MAX_SPEED * timeIncrement) / 1000
speed = distance * 1000 / timeIncrement
# print distance
session.distance = session.distance + distance
session.lastLat = body['lat']
session.lastLon = body['lon']
session.totalTime = session.totalTime + timeIncrement
session.lastTime = body['timestamp']
session.userID.totalTime = session.userID.totalTime + timeIncrement
session.userID.totalDistance = session.userID.totalDistance + distance
session.userID.save()
rc = session.routesCompleted
rp = rc.currentJourney
rp.time = rp.time + timeIncrement
rp.save()
progress = rp.progress
payload = {}
multiple = False
while (progress.totalDistance + distance >= progress.stageID.distance):
session.userID.stagesCompleted = session.userID.stagesCompleted + 1
session.userID.save()
multiple = True
# print "---"
# print progress.totalDistance
# print progress.stageID.distance
# print "here"
difference = progress.stageID.distance - progress.totalDistance
# print difference
distance = distance - difference
# print distance
timeDifference = difference / (speed / 1000) # Time is in ms, speed is m/second
timeIncrement = timeIncrement - timeDifference
# print "Time Difference: {}".format(timeDifference)
# print "Time Increment: {}".format(timeIncrement)
if not (timeIncrement > 0):
timeIncrement = 0
timeDifference = 0
rp.distance = rp.distance + difference
rp.save()
# rp.time = rp.time + timeDifference
session.stagesCompleted = session.stagesCompleted + 1
progress.totalTime = progress.totalTime + timeDifference
progress.completionDate = datetime.datetime.now()
progress.totalDistance = progress.stageID.distance
progress.completed = True
progress.save()
if (progress.stageID.nextStage is None):
rp.completed = True
rp.save()
if rc.bestTime == 0 or rc.bestTime > rp.time:
rc.bestTime = rp.time
achievements = Achievement.objects.filter(route=rc.routeID)
payload['achievements'] = []
for a in achievements:
if a.metric >= rc.bestTime:
obj, created = UserAchievement.objects.get_or_create(userID=rc.userID,
achievementID=a)
print "Created: {}".format(created)
if created:
ach = {}
ach['name'] = a.name
ach['value'] = a.value
payload['achievements'].append(ach)
# Add Award
rc.completed = True
rc.save()
nextRoute = rc.routeID.nextRoute
rc = RoutesCompleted.objects.get_or_create(routeID=nextRoute,
userID=rc.userID)[0]
if rc.currentJourney is None:
progress = Progress(stageID=nextRoute.startStage,
userID=rc.userID)
progress.save()
rp = RouteProgress(progress=progress)
rp.save()
rp.allProgress.add(progress)
rp.save()
rc.allJourneys.add(rp)
rc.currentJourney = rp
rc.save()
else:
rp = rc.currentJourney
if rp.completed:
progress = Progress(stageID=rc.routeID.startStage,
userID=rc.userID)
progress.save()
rp = RouteProgress(progress=progress)
rp.save()
rc.allJourneys.add(rp)
rc.currentJourney = rp
rc.save()
progress = rp.progress
else:
progress = Progress(userID=progress.userID,
stageID=progress.stageID.nextStage)
progress.save()
rp.allProgress.add(progress)
rp.progress = progress
# print "==="
# print progress.totalDistance
# print progress.stageID.distance
# print distance
# print "==="
if not multiple:
rp.distance = rp.distance + distance
# print "Distance: {}".format(rp.distance)
rp.save()
rc.save()
progress.totalDistance = progress.totalDistance + distance
progress.totalTime = progress.totalTime + timeIncrement
progress.save()
session.save()
payload['distance'] = session.distance
payload['sessionTime'] = session.totalTime
payload['distanceRemain'] = progress.stageID.distance - progress.totalDistance
payload['stageLength'] = progress.stageID.distance
payload['routeDistanceRemain'] = rc.routeID.length - rp.distance
payload['routeLength'] = rc.routeID.length
payload['currentStage'] = rp.progress.stageID.id
payload['stageImage'] = rp.progress.stageID.image
payload['id'] = session.pk
payload['totalTime'] = rc.currentJourney.time
payload['stagesCompleted'] = session.stagesCompleted
return HttpResponse(json.dumps(payload), content_type="application/json",
status=202)
else:
return HttpResponse("Bad Request", status=400)
return HttpResponse('Unauthorized method', status=401)
contributors = list(contributors)
organization.contributors = contributors
try:
session.add(organization)
session.commit()
except:
session.rollback()
if __name__ == "__main__":
logging.basicConfig(filename="scraper.log", level=logging.INFO)
G = Github(ACCESS_TOKEN)
progress = session.query(Progress).first()
if not progress:
progress = Progress(id="progress", value=0)
with open('government.github.com/_data/governments.yml') as infile:
_data = yaml.load(infile)
data = reshape_data(_data)
organizations_government = set([organization['entity'].lower() for organization in data])
with open('government.github.com/_data/civic_hackers.yml') as infile:
_data_civic = yaml.load(infile)
data_civic = reshape_data(_data_civic)
organizations_civic = set([organization['entity'].lower() for organization in data_civic])
for i in xrange(progress.value, len(data)):
logging.info("{} {} {}".format(i, data[i]['entity'], data[i]['grouping']))
try:
o_data = upsert_organization(data[i]['entity'], data[i]['grouping'], "government")
def get(self):
from models import MajorRequirements, Progress, Course
from datetime import date
import itertools
import operator
user = users.get_current_user()
name = "none"
if user:
logURL = (users.create_logout_url("/"))
name = user.nickname()
is_logged_in = True
else:
logURL = ("/login")
is_logged_in = False
return redirect ('/login')
userQuery = User.gql("WHERE name='{}'".format(user.nickname())).get()
major1 = userQuery.major1
major2 = userQuery.major2
major3 = userQuery.major3
minor1 = userQuery.minor1
minor2 = userQuery.minor2
minor3 = userQuery.minor3
m1prog = "30";
m2prog = "20";
m3prog = "10";
mi1prog = "10";
mi2prog = "15";
mi3prog = "50";
requirements = MajorRequirements.all().filter('major =', 'CMPS').fetch(limit=100)
requirements = [course.course for course in requirements]
completed1 = Progress.all().filter('user ='******'year').order('quarter').fetch(limit=100)
completed = [course.course for course in completed1]
# any(quarter == chances.quarter for chances in progCheck):
available = ['{}-{}'.format(course.number, course.name[:40]) for course in requirements if course not in completed]
maxyear = date.today().year + 5
# years = tuple((str(n-1), str(n)) for n in range(date.today().year, date.today().year + 5))
years = list((str(n-1) + " - " + str(n)) for n in range(date.today().year, date.today().year + 5))
# shortyears = list(years)
shortyears = list(("Fall" + str(n-1)[2:], "Winter" + str(n)[2:], "Spring" + str(n)[2:], "Summer" + str(n)[2:]) for n in range(date.today().year, date.today().year + 5))
shortyears2 = list((str(n-1)[2:]) for n in range(date.today().year, date.today().year + 5))
quarterlist = [ "Fall", "Winter", "Spring", "Summer"]
# prog2 = Progress.all().filter('user ='******'user ='******'quarter =', quarter)
prog2 = prog2.filter('year =', int(year))
prog2 = prog2.fetch(limit=10)
prog2 = [course1.course.number for course1 in prog2]
classes = [quarter+str(year), prog2]
allClasse.append(classes)
# year = "20" + str(year)
allyears.append([year,allClasse])
# completed1 = Progress.all().filter('user ='******'{}{}-{}'.format(course1.quarter, course1.year, course1.course.number) for course1 in completed1)
# curr = list((course1.quarter, course1.year, course1.course.number) for course1 in completed1)
# curr2 = list()
# for key,group in itertools.groupby(curr,operator.itemgetter(0)):
# curr2.append((list(group)))
# curr =curr2
# curr3 = []
# for listy in curr2:
# list1 = []
# for tup in listy:
# list1.append(listy[0])
# curr3.append(list1)
# curr = curr3
Fall12Classes = tuple(("AMS20", "CMPS101","CMPE100"))
Winter13Classes = tuple(("AMS20", "CMPS101","CMPE100"))
# shortyears = zip(shortyears,curr)
# shortyears = zip(years,shortyears)
# year = curr
logging = str(allyears)
# maxyear = max(maxyear, maxyear2)
# logging.debug(maxyear)
output = {
'quarterlist': quarterlist,
'logging': logging,
'allyears': allyears,
'major1': major1,
'major2': major2,
'major3': major3,
'minor1': minor1,
'minor2': minor2,
'minor3': minor3,
'm1prog': m1prog,
'm2prog': m2prog,
'm3prog': m3prog,
'mi1prog': mi1prog,
'mi2prog': mi2prog,
'mi3prog': mi3prog,
'years': years,
'shortyears': shortyears,
'logURL': logURL,
'is_logged_in': is_logged_in,
'name': name,
'available': available,
}
path = os.path.join(os.path.dirname(__file__), 'templates/dashboard.html')
self.response.write(template.render(path, output))
print("[Info]: Retriving ...")
page = requests.get("{}{}".format(ROOT_URL, url), cookies=COOKIES)
if page.status_code != requests.codes.ok:
error_message = "[Error {}]: Unable Retrive URL: {}\n".format(
page.status_code, url)
log_error_message(error_message)
return
round_trip = page.elapsed.total_seconds()
request_time_total += round_trip
request_counter += 1
print("[Info]: Time: {}s \t Counter: {} \t Average: {}s".format(
round_trip, request_counter, request_time_total / request_counter))
return BeautifulSoup(page.content, "html.parser")
progress = Progress()
user_pool = UserPool()
article_pool = ArticlePool()
progress.load()
user_pool.load()
article_pool.load()
for index_url in url_generator(progress):
print("[Info]: ".ljust(60, "="))
print("[Index Page]: {}{}".format(ROOT_URL, index_url))
index_html = retrive_html_from_url(index_url)
for article_item_div in index_html.find_all(class_="title"):
article_item_a = article_item_div.find("a")
if not article_item_a:
