def get(self):
batchid = self.request.get('batchid')
batch = db.get(batchid)
uobjects = []
for user in batch.users:
if User.all().filter("nickname=", user).count() > 0:
continue
#TODO query FF for the users prescriptions
n = random.randint(1,10)
friends = [ ("user%02d" % (random.randint(0,19))) for i in range(n)]
u = User( nickname = user, friends = friends)
uobjects += [ u ]
db.put(uobjects)
batch.done = True
key = db.Key( str(batch.parent_key() ))
batch.delete()
#nbatches = Batch.all().ancestor( key ).filter("done=", True).count()
nbatches = Batch.all(keys_only=True)
nbatches = nbatches.ancestor( key )
nbatches = nbatches.count()
logging.info("nbatches=%d" % nbatches)
if nbatches == 0:
job = db.get( key )
job.done = True
job.put()
t = Task(method="GET", url="/tasks/FinishJob?jobid=%s" % job.key())
t.add()
def task_enqueue_categories(request):
"""
Enqueues animal tables. To be run by cron
"""
q = Queue('animal-indexes')
category = None #request.GET.get('category')
if category:
taskqueue.add(
Task(url='/adopt/_tasks_/fetch_category',
method='post',
payload=json.dumps({
'category': category,
'url': _getpetharbor_url(x)
})))
else:
for x in PETHARBOR_URLS.keys():
q.add(
Task(url='/adopt/_tasks_/fetch_category',
name="fetch-%s-%s" %
(x, datetime.datetime.now().strftime('%Y%m%d%H%M%S')),
method='post',
payload=str(
json.dumps({
'category': x,
'url': _get_petharbor_url(x)
}))))
return TextResponse('OK')
def get(self):
user = self.current_user
if user:
if user.fbProfile_url:
greeting = ('<span class="user-name">%s<br/><a href="javascript:void()" onclick="fbLogout()">sign out</a></span><img src=http://graph.facebook.com/%s/picture>' % (user.nickname,user.userID))
else:
greeting = ('%s (<a href="%s">sign out</a>)' % (user.nickname, users.create_logout_url("/")))
else:
self.redirect('/')
return
phone = self.request.get('phone')
location = self.request.get('location')
if phone.find('+') < 0:
if phone[0] == '1':
phone = '+' + phone
else:
phone = '+1' + phone
logging.debug('new user has phone number of %s' % phone)
user.phone = phone
user.home_state = location.split(',')[1].lstrip()
user.active = True
user.put()
task = Task(url='/sms/outbound/task',
params={'phone':user.phone,
'msg':'Welcome! You are now registered for MobiNurse. You will hear from us again soon.',},
)
task.add('sms')
self.redirect('/')
def get(self):
# create a new task with this link
crawlURL = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"
task = Task(url='/crawl/crawlingtask', params={'crawl':crawlURL,'routeID':'00'})
task.add('crawler')
logging.info("Added new task for %s" % crawlURL)
return
def apiTimeStat(stat_key, value):
task = Task(url='/stats/new/value',
params={
'stat_key': stat_key,
'value': value
})
task.add('stats')
def get(self):
# review the results for popular stops
reqs = getRequestedStops()
stops_stats = []
keyLookup = []
totalReqs = 0
for key, value in reqs.items():
stops_stats.append({
'stopID': key,
'count': value,
})
totalReqs += value
keyLookup.append(key + ':loc')
# do we have the stop locations?
stopLocations = memcache.get_multi(keyLookup) #reqs.keys())
if stopLocations is None or len(stopLocations) == 0:
logging.error("unable to find stop locations!?")
# create an event to go get this data
task = Task(url='/labs/maptask', params={
'clean': '1',
})
task.add('crawler')
msg = "no data"
else:
logging.debug('yes... found cached copies of the stop locations!')
msg = "your data"
locations = []
median = 2.5
logging.debug('found a total of %s requests with a median %s' %
(str(totalReqs), str(median)))
for key, value in stopLocations.items():
if value is None:
continue
stopID = key.split(':')[0]
# normalized value = count/median * %Total + (count-median)+ base
weight = (float(reqs[stopID]) / median) + float(
reqs[stopID]) - median + 75.0
logging.debug('%s / %s weight is %s' %
(stopID, reqs[stopID], str(weight)))
locations.append({
'stopID': stopID,
'location': value,
'count': reqs[stopID],
'weight': weight,
})
template_values = {
'stops': stops_stats,
'locations': locations,
'message': msg,
}
# create a page that provides a form for sending an SMS message
path = os.path.join(os.path.dirname(__file__), 'mapit.html')
self.response.out.write(template.render(path, template_values))
def post(self):
bookKey = self.request.get("key")
logging.info("checkIN call made it for book: %s" % bookKey)
k = Key(bookKey)
book = Book.get(k)
if book is None:
logging.info("Oops. I couldn't the find book with that key")
else:
logging.info("I found your book! %s %s" % (book.title, book.author))
activeUser = userhandlers.getUser(users.get_current_user().user_id())
borrower = book.borrower
logging.debug("CHECKIN: active user is %s" % activeUser.nickname)
# construction the email with the transaction details
template_values = {'owner':book.owner.nickname,
'ownerEmail':book.owner.preferredEmail,
'title':book.title,
'author':book.author,
'bookProfile':'http://www.frinook.com/book?book='+str(book.key()),
'borrower':activeUser.nickname,
'borrowerFirst':activeUser.first,
'borrowerEmail':activeUser.preferredEmail,
}
# change the book state depending on who is checking it in.
# if it's the borrower, the book goes into TRANSIT mode
# if it's the owner, the book goes into AVAILABLE mode
if book.status == BOOK_STATUS_TRANSIT:
book.status = BOOK_STATUS_AVAILABLE
path = os.path.join(os.path.dirname(__file__), 'completed-email.html')
book.checkedOut = False
book.borrower = None
elif activeUser.userID == book.borrower.userID:
book.status = BOOK_STATUS_TRANSIT
path = os.path.join(os.path.dirname(__file__), 'checkin-email.html')
# log event
event.createEvent(event.EVENT_BOOK_RETURN, activeUser, book, book.title)
else:
book.status = BOOK_STATUS_AVAILABLE
path = os.path.join(os.path.dirname(__file__), 'completed-email.html')
book.checkedOut = False
book.borrower = None
# update the datastore with the new book state
book.put()
# send out email notification
logging.debug("creating email task for checkout... send to owner %s and borrower %s" % (book.owner.preferredEmail,activeUser.preferredEmail))
body = template.render(path, template_values)
task = Task(url='/emailqueue', params={'ownerEmail':book.owner.preferredEmail,
'borrowerEmail':borrower.preferredEmail,
'body':body})
task.add('emailqueue')
self.response.headers['Content-Type'] = 'text/xml'
self.response.out.write("<results>good</results>");
def get(self):
# validate it is in fact coming from twilio
if config.ACCOUNT_SID == self.request.get('AccountSid'):
logging.debug(
"PHONE request was confirmed to have come from Twilio.")
else:
logging.error("was NOT VALID. It might have been spoofed!")
self.response.out.write(errorResponse("Illegal caller"))
return
# pull the route and stopID out of the request body and
# pad it with a zero on the front if the message forgot
# to include it (that's how they are stored in the DB)
routeID = memcache.get(self.request.get('AccountSid'))
memcache.delete(self.request.get('AccountSid'))
stopID = self.request.get('Digits')
if len(stopID) == 3:
stopID = "0" + stopID
# hack - creating a string out of the details to conform to other interfaces
requestArgs = "%s %s" % (routeID, stopID)
## magic ##
logging.info('starting the magic... %s' % requestArgs)
textBody = api_bridge.getarrivals(requestArgs, 1)
logging.debug('phone results are %s' % textBody)
# create an event to log the event
task = Task(url='/loggingtask',
params={
'phone': self.request.get('Caller'),
'inboundBody': requestArgs,
'sid': self.request.get('SmsSid'),
'outboundBody': textBody,
})
task.add('eventlogger')
# transform the text a smidge so it can be pronounced more easily...
# 1. strip the colons
textBody = textBody.replace(':', ' ')
# 2. add a space between p-and-m and a-and-m
textBody = textBody.replace('pm', 'p m').replace('am', 'a m')
logging.debug('transformed results into %s' % textBody)
# setup the response
r = twilio.Response()
r.append(
twilio.Say(textBody,
voice=twilio.Say.MAN,
language=twilio.Say.ENGLISH,
loop=1))
self.response.out.write(r)
def get(self):
# validate the request parameters
devStoreKey = validateRequest(self.request, api_utils.GETSTOPLOCATION)
if devStoreKey is None:
logging.debug("unable to validate the request parameters")
self.response.headers['Content-Type'] = 'application/javascript'
self.response.out.write(
json.dumps(
api_utils.buildErrorResponse(
'-1', 'Illegal request parameters')))
return
# snare the inputs
stopID = api_utils.conformStopID(self.request.get('stopID'))
logging.debug('getstoplocation request parameters... stopID %s' %
stopID)
if api_utils.afterHours() is True:
# don't run these jobs during "off" hours
json_response = api_utils.buildErrorResponse(
'-1', 'The Metro service is not currently running')
elif stopID is not '':
json_response = stopLocationRequest(stopID)
api_utils.recordDeveloperRequest(devStoreKey, api_utils.GETSTOPS,
self.request.query_string,
self.request.remote_addr)
else:
logging.error("API: invalid request")
json_response = api_utils.buildErrorResponse(
'-1',
'Invalid Request parameters. Did you forget to include a stpID?'
)
api_utils.recordDeveloperRequest(
devStoreKey, api_utils.GETSTOPS, self.request.query_string,
self.request.remote_addr, 'illegal query string combination')
#logging.debug('API: json response %s' % json_response);
# encapsulate response in json
callback = self.request.get('callback')
if callback is not '':
self.response.headers['Content-Type'] = 'application/javascript'
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Access-Control-Allow-Methods'] = 'GET'
response = callback + '(' + json.dumps(json_response) + ');'
else:
self.response.headers['Content-Type'] = 'application/json'
response = json.dumps(json_response)
self.response.out.write(response)
stathat.apiStatCount()
# push event out to anyone watching the live board
task = Task(url='/map/task', params={'stopID': stopID})
task.add('eventlogger')
def application ( environ, start_response ):
start_response('200 OK', [('Content-Type', 'text/plain')])
form = cgi.FieldStorage(fp=environ['wsgi.input'],
environ=environ)
users = db.GqlQuery('SELECT * FROM User')
for user in users:
task = Task(payload=None, countdown =60, method = 'GET', url = '/get_timeline?account=' + user.screen_name)
task.add()
print user.screen_name
return "OK"
def get(self):
# validate it is in fact coming from twilio
if config.ACCOUNT_SID == self.request.get("AccountSid"):
logging.debug("PHONE request was confirmed to have come from Twilio.")
else:
logging.error("was NOT VALID. It might have been spoofed!")
self.response.out.write(errorResponse("Illegal caller"))
return
# pull the route and stopID out of the request body and
# pad it with a zero on the front if the message forgot
# to include it (that's how they are stored in the DB)
routeID = memcache.get(self.request.get("AccountSid"))
memcache.delete(self.request.get("AccountSid"))
stopID = self.request.get("Digits")
if len(stopID) == 3:
stopID = "0" + stopID
# hack - creating a string out of the details to conform to other interfaces
requestArgs = "%s %s" % (routeID, stopID)
## magic ##
logging.info("starting the magic... %s" % requestArgs)
textBody = api_bridge.getarrivals(requestArgs, 1)
logging.debug("phone results are %s" % textBody)
# create an event to log the event
task = Task(
url="/loggingtask",
params={
"from": self.request.get("Caller"),
"to": self.request.get("Called"),
"inboundBody": requestArgs,
"sid": self.request.get("SmsSid"),
"outboundBody": textBody,
},
)
task.add("eventlogger")
# transform the text a smidge so it can be pronounced more easily...
# 1. strip the colons
textBody = textBody.replace(":", " ")
# 2. add a space between p-and-m and a-and-m
textBody = textBody.replace("pm", "p m").replace("am", "a m")
logging.debug("transformed results into %s" % textBody)
# setup the response
r = twilio.Response()
r.append(twilio.Say(textBody, voice=twilio.Say.MAN, language=twilio.Say.ENGLISH, loop=1))
self.response.out.write(r)
def get(self):
subject = "APOD is coming back!"
template_file = "templates/adhocemail.html"
# use the task infrastructure to send emails
template_values = { }
path = os.path.join(os.path.dirname(__file__), template_file)
body = template.render(path, template_values)
task = Task(url='/emailqueue', params={'email':'*****@*****.**','subject':'testing','body':body})
task.add('emailqueue')
self.response.out.write(template.render(path, template_values))
def dequeuerating(request):
url = TallstreetUrls.get_url(request.POST['url'])
if not url:
logging.debug("no url %s " % request.POST['url'])
return HttpResponse("No Url", mimetype="text/plain")
keyword = TallstreetTags.get_by_key_name("tag%s" %request.POST['keyword'].lower())
if not keyword:
logging.debug("no Keyword")
return HttpResponse("No Keyword", mimetype="text/plain")
universe = TallstreetUniverse.get_universe(keyword, url)
if not universe:
logging.debug("no Universe")
return HttpResponse("No Universe", mimetype="text/plain")
historychanges = TallstreetHistoryChanges.get_or_insert(key_name="history%s%s" % (keyword.key(), url.key()))
ips = historychanges.ips
change = 0
if request.path == '/queue/click':
change = 1
elif request.path == '/queue/rating' and request.POST['rating'] == '5':
change = 50
elif request.path == '/queue/rating' and request.POST['rating'] == '4':
change = 20
elif request.path == '/queue/rating' and request.POST['rating'] == '3':
change = 10
elif request.path == '/queue/rating' and request.POST['rating'] == '2':
change = 5
elif request.path == '/queue/rating' and request.POST['rating'] == '1':
change = -20
date = datetime.datetime(datetime.datetime.today().year, datetime.datetime.today().month, datetime.datetime.today().day)
if request.POST['ip'] + request.path in ips:
return HttpResponse("Already Rated", mimetype="text/plain")
ips.append(request.POST['ip'] + request.path)
historychanges.ips = ips
historychanges.change += change
historychanges.put()
t = Task(url='/queue/calchistory', params={'historychanges': historychanges.key().id_or_name(), 'universe': universe.key().id_or_name()}, name=historychanges.key().id_or_name() + date.strftime('%Y%m%d'), eta=date + datetime.timedelta(days=1))
logging.debug(t)
try:
t.add('historyqueue')
except TaskAlreadyExistsError:
pass
logging.debug("Success")
return HttpResponse("Success", mimetype="text/plain")
def get(self):
# review the results for popular stops
reqs = getRequestedStops();
stops_stats = []
keyLookup = []
totalReqs = 0
for key,value in reqs.items():
stops_stats.append({'stopID':key,
'count':value,
})
totalReqs += value
keyLookup.append(key+':loc')
# do we have the stop locations?
stopLocations = memcache.get_multi(keyLookup) #reqs.keys())
if stopLocations is None or len(stopLocations) == 0:
logging.error("unable to find stop locations!?")
# create an event to go get this data
task = Task(url='/labs/maptask', params={'clean':'1',})
task.add('crawler')
msg = "no data"
else:
logging.debug('yes... found cached copies of the stop locations!')
msg = "your data"
locations = []
median = 2.5
logging.debug('found a total of %s requests with a median %s' % (str(totalReqs),str(median)))
for key,value in stopLocations.items():
if value is None:
continue;
stopID = key.split(':')[0]
# normalized value = count/median * %Total + (count-median)+ base
weight = (float(reqs[stopID]) / median) + float(reqs[stopID]) - median + 75.0
logging.debug('%s / %s weight is %s' % (stopID,reqs[stopID],str(weight)))
locations.append({'stopID':stopID,
'location':value,
'count':reqs[stopID],
'weight':weight,
})
template_values = {'stops':stops_stats,
'locations':locations,
'message':msg,
}
# create a page that provides a form for sending an SMS message
path = os.path.join(os.path.dirname(__file__), 'mapit.html')
self.response.out.write(template.render(path,template_values))
def get(self):
users = []
q = db.GqlQuery("SELECT * FROM RegisteredUser")
results = q.fetch(500)
for r in results:
if r not in users:
users.append(r)
# create an event to log the event
logging.info('send a message to %s' % r.phone)
msg = 'Notifications are on run today. But the scoring table keeps changing the format of the results. So I quite literally have no idea what will happen. :) Greg'
task = Task(url='/sendsmstask', params={'phone':r.phone,
'athlete':'ignore',
'event':'ignore',
'text':msg,})
task.add('smssender')
def get(self):
# validate the request parameters
devStoreKey = validateRequest(self.request,api_utils.GETSTOPS)
if devStoreKey is None:
if( not (self.request.get('key') == 'kiosk' and self.request.get('stopID') == '') ):
logging.error("failed to validate the request parameters")
self.response.headers['Content-Type'] = 'application/javascript'
self.response.out.write(json.dumps(api_utils.buildErrorResponse('-1','Illegal request parameters')))
return
# snare the inputs
stopID = api_utils.conformStopID(self.request.get('stopID'))
routeID = self.request.get('routeID')
destination = self.request.get('destination').upper()
logging.debug('getstops request parameters... routeID %s destination %s' % (routeID,destination))
if api_utils.afterHours() is True:
# don't run these jobs during "off" hours
json_response = api_utils.buildErrorResponse('-1','The Metro service is not currently running')
elif routeID is not '' and destination is '':
json_response = routeRequest(routeID, None)
api_utils.recordDeveloperRequest(devStoreKey,api_utils.GETSTOPS,self.request.query_string,self.request.remote_addr);
elif routeID is not '' and destination is not '':
json_response = routeRequest(routeID, destination)
api_utils.recordDeveloperRequest(devStoreKey,api_utils.GETSTOPS,self.request.query_string,self.request.remote_addr);
else:
logging.error("API: invalid request")
json_response = api_utils.buildErrorResponse('-1','Invalid Request parameters. Did you forget to include a routeID?')
api_utils.recordDeveloperRequest(devStoreKey,api_utils.GETSTOPS,self.request.query_string,self.request.remote_addr,'illegal query string combination');
#logging.debug('API: json response %s' % json_response);
# encapsulate response in json
callback = self.request.get('callback')
if callback is not '':
self.response.headers['Content-Type'] = 'application/javascript'
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Access-Control-Allow-Methods'] = 'GET'
response = callback + '(' + json.dumps(json_response) + ');'
else:
self.response.headers['Content-Type'] = 'application/json'
response = json.dumps(json_response)
self.response.out.write(response)
stathat.apiStatCount()
# push event out to anyone watching the live board
task = Task(url='/map/task', params={'stopID':stopID})
task.add('eventlogger')
def get(self):
subject = "APOD is coming back!"
template_file = "templates/adhocemail.html"
# use the task infrastructure to send emails
template_values = { }
path = os.path.join(os.path.dirname(__file__), template_file)
body = template.render(path, template_values)
users = db.GqlQuery("SELECT email FROM UserSignup")
for u in users:
#logging.info('Sending email to %s ' % u.email)
task = Task(url='/emailqueue', params={'email':u.email,'subject':subject,'body':body})
task.add('emailqueue')
self.response.out.write(template.render(path, template_values))
def _tx():
pending_email = PendingEmail(subscription=sub, edition=edition)
db.put(pending_email)
SendNewsletter.add(
Task(params={'pending_email': pending_email.key()}),
transactional=True)
def get(self, event=""):
CRAWL_BASE_URL = "http://fixme/THev" # 15.htm
url = CRAWL_BASE_URL + event + ".htm"
logging.info("getting ready to look at %s" % url)
loop = 0
done = False
result = None
while not done and loop < 2:
try:
result = urlfetch.fetch(url)
done = True
except urlfetch.DownloadError:
logging.error("Error loading page (%s)... sleeping" % loop)
if result:
logging.debug("Error status: %s" % result.status_code)
logging.debug("Error header: %s" % result.headers)
logging.debug("Error content: %s" % result.content)
time.sleep(6)
loop = loop + 1
if result is None or result.status_code != 200:
logging.error("Exiting early: error fetching URL: " + str(result.status_code))
return
# continue only if the event has been scored...
if result.content.find("not yet available") > 0:
logging.info("looks like event %s hasn't been scored yet" % event)
return
# loop through all of the registered users and spawn a task to
# scrape the event URL to find the athlete
q = db.GqlQuery("SELECT * FROM RegisteredUser")
results = q.fetch(500)
for r in results:
# create an event to log the event
logging.debug("creating task for %s, event %s, calling %s" % (r.athlete, event, r.phone))
task = Task(
url="/athletefindertask", params={"phone": r.phone, "athlete": r.athlete, "url": url, "event": event}
)
task.add("athletefinder")
return
def get(self):
path = os.path.join(os.path.dirname(__file__), 'index.html')
room = self.request.get('room')
if room is None:
self.response.out.write(template.render(path,{"body":"Need a room name"}))
#TODO: query for users from friendfeed
users = [ ('user%02d' % (i)) for i in range(20)]
job = Job(roomname=room, done=False, users = users, score = [], ready = False)
job.put()
#TODO: Tune BATCHSIZE
BATCHSIZE = 4
for i in range(0,len(users),BATCHSIZE):
b = Batch(parent=job, users=users[i:i + BATCHSIZE], done = False)
b.put()
t = Task(method="GET", url="/tasks/RunBatch?batchid=%s" % b.key());
t.add()
self.redirect("/wait?jobid=%s" % job.key())
def post(self):
user = users.get_current_user()
if user and users.is_current_user_admin():
phone = self.request.get('phone')
text = self.request.get('text')
logging.debug("the admin console is sending and SMS to %s with the message, %s" % (phone,text))
# log the event...
task = Task(url='/loggingtask', params={'from':phone,
'to':phone,
'inboundBody':text,
'sid':'admin request',
'outboundBody':text,})
task.add('eventlogger')
# send the SMS out...
task = Task(url='/admin/sendsms', params={'phone':phone,
'sid':'admin console',
'text':text,})
task.add('smssender')
return('Sent!')
elif user:
logging.error("illegal access to the admin console for sending sms messages!?! %s" % user.email())
return('Not so fast!')
def post(self):
activeUser = userhandlers.getUser(users.get_current_user().user_id())
bookKey = self.request.get("key")
logging.debug("looking for book key %s" % bookKey)
book = db.get(bookKey)
logging.info("checkout call made it for book: %s" % bookKey)
if book is None:
logging.info("Oops. I couldn't find book with that key")
else:
logging.info("I found your book! %s %s" % (book.title, book.author))
book.checkedOut = True
book.borrower = activeUser
book.status = BOOK_STATUS_CHECKED_OUT
book.put()
# construction the email with the transaction details
template_values = {'owner':book.owner.nickname,
'title':book.title,
'author':book.author,
'borrower':activeUser.nickname,
'borrowerFirst':activeUser.first,
'borrowerEmail':activeUser.preferredEmail,
}
path = os.path.join(os.path.dirname(__file__), 'checkout-email.html')
body = template.render(path, template_values)
# log event
event.createEvent(event.EVENT_BOOK_CHECKOUT, activeUser, book, book.title)
# send out email notification
logging.debug("creating email task for checkout... send to owner %s and borrower %s" % (book.owner.preferredEmail,activeUser.preferredEmail))
task = Task(url='/emailqueue', params={'ownerEmail':book.owner.preferredEmail,
'borrowerEmail':activeUser.preferredEmail,
'body':body})
task.add('emailqueue')
self.response.headers['Content-Type'] = 'text/xml'
self.response.out.write("<results>good</results>");
def post(self):
intersection = self.request.get('intersection')
latitude = self.request.get('latitude')
longitude = self.request.get('longitude')
direction = self.request.get('direction')
routeID = self.request.get('routeID')
stopID = self.request.get('stopID')
logging.info("storing route %s intersection %s at lat/lon %s,%s toward %s" %
(routeID,intersection,latitude,longitude,direction))
if len(intersection) > 400:
intersection = intersection.ljust(400)
if stopID == '00' or latitude is None or longitude is None:
# create a task event to process the error
task = Task(url='/crawl/errortask', params={'intersection':intersection,
'location':(latitude+","+longitude),
'direction':direction,
'metaStringOne':self.request.get('crawlLine'),
'metaStringTwo':'from geotask crawler',
'routeID':routeID,
'stopID':stopID,
})
task.add('crawlerrors')
else:
# ignore this stop if we've already stored it
# stopID + routeID
stop = db.GqlQuery("SELECT * FROM StopLocation WHERE stopID = :1 and routeID = :2", stopID, routeID).get()
if stop is None:
stop = StopLocation()
stop.stopID = stopID
stop.routeID = routeID
stop.intersection = intersection.upper()
stop.direction = direction.upper()
stop.location = GeoPt(latitude,longitude)
stop.update_location()
stop.put()
# update the route table to include a reference to the new geo data
if stopID != '00':
route = db.GqlQuery("SELECT * FROM RouteListing WHERE stopID = :1 and route = :2", stopID,routeID).get()
if route is None:
logging.error("IMPOSSIBLE... no stop on record?!? stop %s, route %s" % (stopID,routeID))
# create a task event to process the error
task = Task(url='/crawl/errortask', params={'intersection':intersection,
'location':(latitude+","+longitude),
'direction':direction,
'metaStringOne':self.request.get('crawlLine'),
'metaStringTwo':'routelisting update',
'routeID':routeID,
'stopID':stopID,
})
task.add('crawlerrors')
else:
route.stopLocation = stop
route.put()
return
def _task_fetch_category(url, category):
"""
Fetch an animal table, parse out information, and enqueue a task to fetch the image and store the data in the database
"""
q = Queue('animal-profiles')
page = fetch(url)
animals = parse_petharbor_table(page.content, category)
for animal in animals:
animal['src_url'] = url
q.add(
Task(url='/adopt/_tasks_/fetch_animal',
name="fetch-%s-%s-%s" %
(category, animal['code'],
datetime.datetime.now().strftime('%Y%m%d%H%M')),
method="post",
payload=str(json.dumps(animal))))
def mail_queue_expander(request):
BATCH_SIZE = 5
edition = db.get(request.form['edition'])
if not edition: pass
page = int(request.form.get('page', 0))
subscriber_q = Query(subscriptions.models.Subscription,
keys_only=True).filter('site =', edition.site).filter(
'active =', True)
if request.form.has_key('cursor'):
subscriber_q = subscriber_q.with_cursor(request.form['cursor'])
subscribers = subscriber_q.fetch(BATCH_SIZE)
if not subscribers:
edition.status = 'complete'
edition.put()
return
task = Task(params={
'edition': edition.key(),
'cursor': subscriber_q.cursor(),
'page': page + 1
},
name="%s-%s-%s-%s" %
(edition.site.slug, edition.issue_num,
edition.publish_after.strftime("%Y%j%H%M-%S"), page + 1))
try:
MailQueueExpander.add(task)
except (TaskAlreadyExistsError, TombstonedTaskError):
raise
for sub in subscribers:
def _tx():
pending_email = PendingEmail(subscription=sub, edition=edition)
db.put(pending_email)
SendNewsletter.add(
Task(params={'pending_email': pending_email.key()}),
transactional=True)
db.run_in_transaction_custom_retries(10, _tx)
def post(self):
phone = self.request.get('phone')
zipcode = self.request.get('zipcode')
place = self.request.get('place')
user = db.GqlQuery("select * from User where phone = :1", phone).get()
if user is None or not user.active:
logging.debug('no user or inactive user %s' % phone)
###
# CHECK Air Quality
###
airquality = airnow.search(zipcode)
if airquality is not None:
msg = "You've changed locations to "+place+". Just note that the air quality forecast where you now are is "+airquality
logging.debug('creating sms task with %s' % msg)
task = Task(url='/sms/outbound/task',
params={'phone':user.phone,
'msg':msg,
},
)
task.add('sms')
else:
logging.debug('no air quality data found!')
###
# CHECK Flu status
###
flustate = None #cdcflu.search(zipcode)
if flustate is not None:
msg = ".... also keep in mind that the flu state in your current state is "+flustate
logging.debug('creating sms task with %s' % msg)
task = Task(url='/sms/outbound/task',
params={'phone':user.phone,
'msg':msg,
},
)
task.add('sms')
else:
logging.debug('no air quality data found!')
def get(self):
start = time.time()
# snare the inputs
dev_key = self.request.get('key')
stopID = api_utils.conformStopID(self.request.get('stopID'))
routeID = self.request.get('routeID')
vehicleID = self.request.get('vehicleID')
#logging.debug('getarrivals request parameters... stopID %s routeID %s vehicleID %s' % (stopID,routeID,vehicleID))
self.request.registry['aggregated_results'] = []
try:
if api_utils.afterHours() is False: # and dev_key != 'uwkiosk9':
# validate the request parameters
devStoreKey = validateRequest(self.request)
if devStoreKey is None:
# filter out the kiosk errors from the log
if (not (dev_key == 'kiosk'
and self.request.get('stopID') == '')):
logging.error(
"failed to validate the request parameters")
self.response.headers[
'Content-Type'] = 'application/javascript'
self.response.out.write(
json.dumps(
api_utils.buildErrorResponse(
'-1',
'Unable to validate the request. There may be an illegal developer key.'
)))
return
if stopID is not '' and routeID is '':
json_response = stopRequest(stopID, dev_key)
api_utils.recordDeveloperRequest(devStoreKey,
api_utils.GETARRIVALS,
self.request.query_string,
self.request.remote_addr)
elif stopID is not '' and routeID is not '':
json_response = stopRouteRequest(stopID, routeID,
devStoreKey)
api_utils.recordDeveloperRequest(devStoreKey,
api_utils.GETARRIVALS,
self.request.query_string,
self.request.remote_addr)
elif routeID is not '' and vehicleID is not '':
json_response = routeVehicleRequest(
routeID, vehicleID, devStoreKey)
api_utils.recordDeveloperRequest(devStoreKey,
api_utils.GETVEHICLE,
self.request.query_string,
self.request.remote_addr)
else:
logging.debug("API: invalid request")
api_utils.recordDeveloperRequest(
devStoreKey, api_utils.GETARRIVALS,
self.request.query_string, self.request.remote_addr,
'illegal query string combination')
json_response = api_utils.buildErrorResponse(
'-1', 'Invalid Request parameters')
# push event out to anyone watching the live board
channels = memcache.get('channels')
if channels is not None:
task = Task(url='/map/task', params={'stopID': stopID})
task.add('eventlogger')
# stop statistics - DISABLED
# if( "kiosk" not in dev_key ):
# task = Task(url='/stats/stop', params={'apikey':dev_key,'stop':stopID})
# task.add('stats')
else:
# don't run these jobs during "off" hours
#logging.debug('shunted... off hour request')
#if dev_key.find('kiosk') >= 0:
# json_response = api_utils.buildErrorResponse('-1','Kiosk requests no longer supported')
#else:
json_response = api_utils.buildErrorResponse(
'-1', 'The Metro service is not currently running')
# encapsulate response in json or jsonp
callback = self.request.get('callback')
if callback is not '':
self.response.headers[
'Content-Type'] = 'application/javascript'
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Access-Control-Allow-Methods'] = 'GET'
response = callback + '(' + json.dumps(json_response) + ');'
else:
self.response.headers['Content-Type'] = 'application/json'
response = json.dumps(json_response)
self.response.out.write(response)
except DeadlineExceededError:
self.response.clear()
self.response.set_status(500)
self.response.out.write(
"This operation could not be completed in time...")
# persist some statistics
# stathat:
if api_utils.afterHours() is False and dev_key != 'uwkiosk9':
stathat.apiTimeStat(config.STATHAT_API_GETARRIVALS_TIME_KEY,
((time.time() - start) * 1000))
stathat.apiStatCount()
def apiTimeStat(stat_key,value):
task = Task(url='/stats/new/value', params={'stat_key':stat_key,'value':value})
task.add('stats')
def get(self):
debug = self.request.get('debug')
q = db.GqlQuery("select * from User")
us = q.fetch(100)
results = []
for u in us:
if u.access_token is not None:
logging.debug('facebook location : checking out %s' % u.nickname)
graph = facebook.GraphAPI(u.access_token)
fbuser = graph.get_object('me')
checkins = graph.get_connections(fbuser["id"], "checkins")
if len(checkins) == 0 or len(checkins['data']) == 0:
logging.debug('empty checkins %s' % checkins)
continue
# chop up the results. we only want the first (latest) checkin
spot = checkins['data'][0]
date = spot['created_time'].split('T')[0]
place = spot['place']['name']
if 'zip' in spot['place']['location']:
zip = spot['place']['location']['zip']
if zip.find('-') > 0:
zip = zip.split('-')[0]
else:
zip = getZip(spot['place']['location']['latitude'],
spot['place']['location']['longitude'])
logging.debug('reverse geo lookup found zipcode %s' % zip)
if zip is None or zip == 'unknown':
# hack. i don't know what to do if i can't get this from google
if u.current_zipcode is None:
zip = '53711'
else:
zip = u.current_zipcode
logging.debug('address analysis produced a zipcode of %s' % zip)
# monitor to see if the user has moved since there last
# checkin. if so, we may want to notify them of some news
if u.current_zipcode is None or len(u.current_zipcode) == 0:
# first checkin - definitely notify!
logging.debug('no zipcode!?')
notify = True
elif u.last_fb_checkin_date != date or u.last_fb_checkin_place != place:
notify = True
else:
notify = False
logging.debug('current location : %s %s %s' % (u.current_zipcode,u.last_fb_checkin_date,u.last_fb_checkin_place))
logging.debug('new location : %s %s %s' % (zip,date,place))
# update the user model with their latest checkin data.
u.current_zipcode = zip
u.last_fb_checkin_date = date
u.last_fb_checkin_place = place
u.put()
if notify and u.phone is not None and len(u.phone) > 0:
logging.debug('checking feeds to see if we can notify user %s' % u.phone)
task = Task(url='/feeds/scanuser',
params={'phone':u.phone,
'zipcode':'53711',#zip,
'place':place,
},
)
task.add('feedscan')
if debug:
# web debug result output
results.append({'nickname':u.nickname,
'profile':u.fbProfile_url,
'time':date,
'place':spot['place']['name'],
'api':spot,
})
if debug:
# add the counter to the template values
template_values = {'results':results,
}
# generate the html
path = os.path.join(os.path.dirname(__file__), 'index.html')
self.response.out.write(template.render(path, template_values))
def get(self):
#logging.debug("The request from Twilio %s" % self.request)
# validate it is in fact coming from twilio
if ACCOUNT_SID == self.request.get('AccountSid'):
logging.info("was confirmed to have come from Twilio.")
else:
logging.info("was NOT VALID. It might have been spoofed!")
self.response.out.write(errorResponse("Illegal caller"))
return
# verify the phone number as a known user
call_duration = self.request.get('Duration')
caller = self.request.get('Caller')
q = db.GqlQuery("SELECT * FROM User WHERE phone = :1", caller)
if q.count(1) == 0:
self.response.out.write(errorResponse("I don't know you"))
return
else:
user = q.get()
logging.info('recording received. registered user... id: %s, posterous: %s, phone: %s, duration: %s' %
(user.user, user.posterous, user.phone, call_duration))
# special handline for my own blogs...
if user.user == MY_EMAIL:
logging.info("Sweet! Greg's calling... Looks like a post for the family site.")
digit = self.request.get('Digits')
if digit == "1":
logging.info("Nope. He's posting to his PERSONAL site")
user.posterousID = posterous.GREG_SITE_ID
user.posterous = "gregtracy"
elif digit == "7":
logging.info("Nope. He's posting to his API TESTING site")
user.posterousID = posterous.APITESTING_SITE_ID
user.posterous = "apitesting"
# decrypt the password
obj = AES.new(SALT_KEY, AES.MODE_CFB)
password_clear = obj.decrypt(base64.decodestring(user.password))
#password_clear = obj.decrypt(base64.urlsafe_b64decode(user.password))
# post to the blog
blogTitle = user.postTitle
blogBody = "%s.mp3" % self.request.get('RecordingUrl')
blog = posterous.Posterous(user.user, password_clear)
postly = blog.postBlog(user.posterousID, blogTitle, blogBody, user.private)
# log this event...
log = PhoneLog()
log.phone = user.phone
log.link = blogBody
log.user = user.user
log.posterous = user.posterous
log.posterousID = user.posterousID
log.duration = call_duration
log.caller_city = self.request.get('CallerCity')
log.caller_state = self.request.get('CallerState')
log.caller_zip = self.request.get('CallerZip')
log.put()
if postly == "Error":
postly = "http://%s.posterous.com" % user.posterous
emailBody = "<html><body><p>Your posterous has been updated with your recorded message!<p>The message details are as follows...<ul><li>Phone: %s</li><li>Posterous: %s</li><li>Call duration: %s sec.</li><li>Your new post: <a href=%s>%s</a></li></ul><p>Thanks for using Ringerous!<br><a href=http://www.ringerous.com>http://www.ringerous.com</a></body></html>" % (user.phone, user.posterous, call_duration, postly, postly)
task = Task(url='/emailqueue', params={'email':user.user,'body':emailBody})
task.add('emailqueue')
# setup the response to get the recording from the caller
r = twilio.Response()
v = twilio.Verb()
r.append(twilio.Hangup())
self.response.out.write(r)
def post(self):
html = None
error = False
username = self.request.get('username')
password_clear = self.request.get('password')
posterousName = self.request.get('posterous')
defaultTitle = self.request.get('postTitle')
private = self.request.get('private')
# the phone number comes in three parts. stitch them together
callerOne = self.request.get('caller1')
callerTwo = self.request.get('caller2')
callerThree = self.request.get('caller3')
if (len(callerOne) == 0 or
len(callerTwo) == 0 or
len(callerThree) == 0 or
len(username) == 0 or
len(password_clear) == 0 or
len(posterousName) == 0):
html = errorOutput('Oops. Configuration problem.',
"It looks as though you've left out some required fields")
self.response.out.write(html)
return
# verify the length of the phone number
caller = callerOne + callerTwo + callerThree
logging.info("phone number is %s" % caller)
if len(caller) != 10:
html = errorOutput('Oops. Configuration problem.',
'The phone number you entered does not appear to be valid (%s). Please type a ten digit number without any special characters'%caller)
error = True
else:
# verify that we don't already have this phone
# number registered
q = db.GqlQuery("SELECT * FROM User WHERE phone = :1", caller)
if q.count(1) > 0:
logging.debug('Checking for duplicates... found %s',q.count(1))
html = errorOutput('Oops. Configuration problem.',
'This phone number is already registered!')
error = True
# validate the posterous credentials
if not error:
p = posterous.Posterous(userID=username,userPassword=password_clear)
site = p.getSites(posterousName)
# validate the posterous credentials and posterous ID
if site is None:
html = errorOutput('Oops. Configuration problem.',
'Unable to verify your Posterous configuration!')
error = True
if not error:
# encrypt the password and shove the user
# information in the database
obj = AES.new(SALT_KEY, AES.MODE_CFB)
cipher = base64.encodestring(obj.encrypt(password_clear))
#cipher = base64.urlsafe_b64encode(obj.encrypt(password_clear))
u = User()
u.user = username
u.password = cipher
u.phone = caller
u.posterous = posterousName
u.posterousID = site['id']
u.posterousURL = site['url']
u.name = site['name']
u.pin = '123'
u.private = private
u.postTitle = defaultTitle
u.put()
html = errorOutput('Configuration complete!', "We're ready to take your call - 415.367.3142")
emailBody = "<html><body><p>This email confirms your Ringerous registration...<ul><li>Username: %s</li><li>Phone: %s</li><li>Posterous: %s</li></ul><p>Thanks for using Ringerous!<br><a href=http://www.ringerous.com>http://www.ringerous.com</a></body></html>" % (u.user, u.phone, u.posterous)
task = Task(url='/emailqueue', params={'email':u.user,'body':emailBody})
task.add('emailqueue')
self.response.out.write(html)
def post(self):
logging.info("New signup request...\n\tEmail: %s\n\tAdd/Remove: %s\n\tReference: %s\n\tNotes: %s",
self.request.get('string'), self.request.get('signup'), self.request.get('reference'), self.request.get('comments'))
# first validate the email address
if not email_re.search(self.request.get('string')):
error_msg = "This address - %s - is not a valid email address. Check the formatting." % self.request.get('string')
logging.error(error_msg)
self.response.out.write("Oops. The email address was malformed! Please try again.")
return
message = mail.EmailMessage(sender="APOD Email <*****@*****.**>")
blocked = False
bcc = False
# determine if this is a signup or remove request
if self.request.get('signup') == 'signup':
# first validate the captcha
challenge = self.request.get('recaptcha_challenge_field')
response = self.request.get('recaptcha_response_field')
remoteip = self.request.get('remote_addr')
logging.info("Captcha request... \n\tchallenge: %s\n\tresponse: %s\n\tIP: %s\n\t",
challenge,response,remoteip)
cResponse = captcha.submit(
challenge,
response,
config.RECAPTCHA_PRIVATE_KEY,
remoteip)
if cResponse.is_valid:
logging.debug('Captcha Success!')
email_addr = self.request.get('string').lower()
# first check to see if the user is already on the list
q = db.GqlQuery("SELECT * FROM UserSignup WHERE email = :1", email_addr)
remove_entry = q.fetch(1)
if len(remove_entry) > 0:
error_msg = "This address - %s - is already on the distribution list." % email_addr
logging.error(error_msg)
self.response.out.write("Oops. It looks like this email address is already on the list.")
return
# if signing up, create a new user and add it to the store
userSignup = data_model.UserSignup()
userSignup.email = email_addr
userSignup.referral = self.request.get('reference')
userSignup.notes = self.request.get('comments')
# filter out requests if there is a URL in the comments field
if re.search("http",userSignup.referral) or re.search("http",userSignup.notes):
error_msg = "Request to add - %s - has been BLOCKED because of illegal text in the comments field." % email_addr
logging.error(error_msg)
template_values = {'error':error_msg}
# setup the specific email parameters
message.subject="APOD Email Signup BLOCKED"
message.to = email_addr
path = os.path.join(os.path.dirname(__file__), 'templates/invalid-email.html')
message.html = template.render(path,template_values)
# setup the confirmation page on the web
#path = os.path.join(os.path.dirname(__file__), 'templates/invalid.html')
msg = "Oops. No can do. This request is getting blocked because it looks fishy."
else:
# add the user to the database!
userSignup.put()
template_values = {'email':userSignup.email,
'referral':userSignup.referral,
'notes':userSignup.notes,
}
# setup the specific email parameters
message.subject="APOD Email Signup Confirmation"
message.to = email_addr
path = os.path.join(os.path.dirname(__file__), 'templates/added-email.html')
message.html = template.render(path,template_values)
# setup the confirmation page on the web
#path = os.path.join(os.path.dirname(__file__), 'templates/added.html')
msg = "<h2>Excellent! You've been added to the list.</h2><p class='lead'>Please consider a donation to support this project.</p>"
else:
blocked = True
msg = cResponse.error_code
logging.error('Captcha Fail %s ' % msg)
else:
email_addr = self.request.get('string').lower()
# if removing a user, first check to see that the request is valid
q = db.GqlQuery("SELECT * FROM UserSignup WHERE email = :1", email_addr)
remove_entry = q.fetch(1)
if len(remove_entry) > 0:
# if the user was found, remove them
db.delete(remove_entry)
# setup the specific email parameters
message.subject="APOD Email Removal Request"
message.to = self.request.get('string')
template_values = { 'email':email_addr,
'referral':self.request.get('reference'),
'notes':self.request.get('comments'),
}
path = os.path.join(os.path.dirname(__file__), 'templates/removed-email.html')
message.html = template.render(path,template_values)
# ... and show the thank you confirmation page
msg = "<h2>You've been removed from the list... Thanks!</h2>"
# email me if they've left comments
if len(self.request.get('comments')) > 0:
bcc = True
else:
error_msg = "This address - %s - is not on the distribution list!?" % email_addr
logging.error(error_msg)
msg = "Oops. This address doesn't appear to be on the list."
blocked = True
# send the message off...
if not blocked:
logging.debug("Sending email!")
task = Task(url='/emailqueue', params={'email':message.to,'subject':message.subject,'body':message.html,'bcc':bcc})
task.add('emailqueue')
# report back on the status
logging.debug(msg)
self.response.out.write(msg)
def post(self):
if serviceOn() == False:
return
# which event are we on?
eventString = memcache.get("eventNumber")
if eventString is None:
eventResult = db.GqlQuery("SELECT * FROM EventTracker").get()
if eventResult is None:
logging.error("IMPOSSIBLE! We don't know what event we're on!?!")
return
else:
eventString = str(eventResult.event)
event = int(eventString)
url = CRAWL_BASE_URL + eventString + ".htm"
logging.debug("getting ready to look at %s" % url)
loop = 0
done = False
result = None
while not done and loop < 2:
try:
result = urlfetch.fetch(url, headers={"Cache-Control": "max-age=240"})
done = True
except urlfetch.DownloadError:
logging.error("Error loading page (%s)... sleeping" % loop)
if result:
logging.debug("Error status: %s" % result.status_code)
logging.debug("Error header: %s" % result.headers)
logging.debug("Error content: %s" % result.content)
time.sleep(6)
loop = loop + 1
if result is None or result.status_code != 200:
logging.error("Exiting early: error fetching URL: " + str(result.status_code))
return
# continue only if the event has been scored...
if result.content.find("Sorry") > 0:
logging.info("looks like event %s hasn't been scored yet" % eventString)
logging.info(result.content)
return
if event == 30:
nextEvent = 33
elif event == 62:
setService(False)
else:
nextEvent = event + 1
# persist the new event number
eventResult = db.GqlQuery("SELECT * FROM EventTracker").get()
eventResult.event = nextEvent
eventResult.put()
memcache.set("eventNumber", str(nextEvent))
logging.debug("bumping up the event number to %s" % str(nextEvent))
# loop through all of the registered users and spawn a task to
# scrape the event URL to find the athlete
q = db.GqlQuery("SELECT * FROM RegisteredUser")
results = q.fetch(500)
for r in results:
# create an event to log the event
logging.info("adding new finder task for %s" % r.athlete)
task = Task(
url="/athletefindertask", params={"phone": r.phone, "athlete": r.athlete, "url": url, "event": event}
)
task.add("athletefinder")
# send out the text with the results
textBody = "just completed event %s" % event
task = Task(
url="/sendsmstask", params={"phone": OWNER_NUMBER, "athlete": "admin", "event": "status", "text": textBody}
)
task.add("smssender")
def get(self):
start = time.time()
# snare the inputs
dev_key = self.request.get('key')
stopID = api_utils.conformStopID(self.request.get('stopID'))
routeID = self.request.get('routeID')
vehicleID = self.request.get('vehicleID')
#logging.debug('getarrivals request parameters... stopID %s routeID %s vehicleID %s' % (stopID,routeID,vehicleID))
self.request.registry['aggregated_results'] = []
try:
if api_utils.afterHours() is False: # and dev_key != 'uwkiosk9':
# validate the request parameters
devStoreKey = validateRequest(self.request)
if devStoreKey is None:
# filter out the kiosk errors from the log
if( not (dev_key == 'kiosk' and self.request.get('stopID') == '') ):
logging.error("failed to validate the request parameters")
self.response.headers['Content-Type'] = 'application/javascript'
self.response.out.write(json.dumps(api_utils.buildErrorResponse('-1','Unable to validate the request. There may be an illegal developer key.')))
return
if stopID is not '' and routeID is '':
json_response = stopRequest(stopID, dev_key)
api_utils.recordDeveloperRequest(devStoreKey,api_utils.GETARRIVALS,self.request.query_string,self.request.remote_addr);
elif stopID is not '' and routeID is not '':
json_response = stopRouteRequest(stopID, routeID, devStoreKey)
api_utils.recordDeveloperRequest(devStoreKey,api_utils.GETARRIVALS,self.request.query_string,self.request.remote_addr);
elif routeID is not '' and vehicleID is not '':
json_response = routeVehicleRequest(routeID, vehicleID, devStoreKey)
api_utils.recordDeveloperRequest(devStoreKey,api_utils.GETVEHICLE,self.request.query_string,self.request.remote_addr);
else:
logging.debug("API: invalid request")
api_utils.recordDeveloperRequest(devStoreKey,api_utils.GETARRIVALS,self.request.query_string,self.request.remote_addr,'illegal query string combination');
json_response = api_utils.buildErrorResponse('-1','Invalid Request parameters')
# push event out to anyone watching the live board
channels = memcache.get('channels')
if channels is not None:
task = Task(url='/map/task', params={'stopID':stopID})
task.add('eventlogger')
# stop statistics - DISABLED
# if( "kiosk" not in dev_key ):
# task = Task(url='/stats/stop', params={'apikey':dev_key,'stop':stopID})
# task.add('stats')
else:
# don't run these jobs during "off" hours
#logging.debug('shunted... off hour request')
#if dev_key.find('kiosk') >= 0:
# json_response = api_utils.buildErrorResponse('-1','Kiosk requests no longer supported')
#else:
json_response = api_utils.buildErrorResponse('-1','The Metro service is not currently running')
# encapsulate response in json or jsonp
callback = self.request.get('callback')
if callback is not '':
self.response.headers['Content-Type'] = 'application/javascript'
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Access-Control-Allow-Methods'] = 'GET'
response = callback + '(' + json.dumps(json_response) + ');'
else:
self.response.headers['Content-Type'] = 'application/json'
response = json.dumps(json_response)
self.response.out.write(response)
except DeadlineExceededError:
self.response.clear()
self.response.set_status(500)
self.response.out.write("This operation could not be completed in time...")
# persist some statistics
# stathat:
if api_utils.afterHours() is False and dev_key != 'uwkiosk9':
stathat.apiTimeStat(config.STATHAT_API_GETARRIVALS_TIME_KEY,((time.time()-start)*1000))
stathat.apiStatCount()
def get(self):
# create an event to go get this data
task = Task(url='/labs/maptask', params={'clean':'1',})
task.add('crawler')
def put_paste (self, slug):
"""
Puts the paste to the datastore.
"""
is_reply = self.form_parent_slug != ""
self.paste = app.model.Pasty()
paste_is_private = self.request.get("submit") == "privately"
self.paste.set_code(self.form_code)
self.paste.edited_at = datetime.datetime.now()
self.paste.edited_by_ip = self.request.remote_addr
self.paste.forks = 0
self.paste.indirect_forks = 0
self.paste.parent_paste = ""
self.paste.posted_at = datetime.datetime.now()
self.paste.posted_by_ip = self.request.remote_addr
self.paste.replies = 0
self.paste.slug = slug
if not is_reply and paste_is_private:
self.paste.status = app.model.kPASTE_STATUS_PRIVATE
self.paste.secret_key = app.model.Pasty.make_secret_key()
else:
self.paste.status = app.model.kPASTE_STATUS_PUBLIC
self.paste.secret_key = ""
self.paste.title = app.pasty.filter_title(self.form_title, slug)
self.paste.user = self.user.db_user
if self.user.is_logged_in:
self.paste.posted_by_user_name = self.user.id
else:
self.paste.posted_by_user_name = settings.DEFAULT_USER_NAME
if is_reply:
is_first_of_thread = False
self.paste.parent_paste = self.form_parent_slug
self.paste.thread_level = self.parent_paste.thread_level + 1
self.paste.thread_position = self.parent_paste.thread_position \
+ self.parent_paste.indirect_forks + 1
if self.parent_paste.thread == None:
self.paste.thread = slug
else:
self.paste.thread = self.parent_paste.thread
else:
self.paste.thread_level = 0
self.paste.thread_position = 0
pasty_key = self.paste.put()
result = pasty_key != None
if result == True:
dbPaste = app.model.Pasty.get(pasty_key)
# If the paste is not a reply, then it's starting its own thread.
if not is_reply:
if dbPaste != None:
dbPaste.thread = slug
dbPaste.put()
if result == True:
task = Task(name = self.paste.slug, method="GET", url = "/" + self.paste.slug + "/recount")
task.add(queue_name="paste-recount")
self.put_log(dbPaste)
return result
class HitList(RequestHandler):
def get(self):
user = users.get_current_user()
type = self.get_value('type', required=False)
type = self.map_type(type)
self.render('priv/hit_list.html', {
'user': user,
'type': type,
'time': datetime_format(datetime.utcnow())
})
def post(self):
type = self.get_value('type', required=False)
type = self.map_type(type)
try:
hit = HIT()
hit.owner = users.get_current_user()
hit.lifetime = self.get_value('lifetime', required=True, mapfn=int)
hit.duration = self.get_value('duration', required=True, mapfn=int)
hit.approval_delay = self.get_value('approval_delay',
required=True,
mapfn=int)
hit.hit_approval = self.get_value('hit_approval',
required=False,
mapfn=int)
hit.hit_approval_rate = self.get_value('hit_approval_rate',
required=True,
mapfn=int)
hit.accepted_hit_rate = self.get_value('accepted_hit_rate',
required=True,
mapfn=int)
hit.returned_hit_rate = self.get_value('returned_hit_rate',
required=True,
mapfn=int)
hit.abandoned_hit_rate = self.get_value('abandoned_hit_rate',
required=True,
mapfn=int)
hit.rejected_hit_rate = self.get_value('rejected_hit_rate',
required=True,
mapfn=int)
hit.locale_qualification = self.get_value('locale_qualification',
required=True)
hit.frame_height = self.get_value('frame_height',
required=True,
mapfn=int)
if type != 'MULTIPLE_URLS':
hit.max_workers = self.get_value('max_workers',
required=True,
mapfn=int)
if type != 'PASS_THROUGH':
hit.min_workers = self.get_value('min_workers',
required=True,
mapfn=int)
if type == 'MULTIPLE_URLS':
hit.location1 = self.get_value('location1', required=True)
hit.size1 = self.get_value('size1', required=True, mapfn=int)
hit.location2 = self.get_value('location2', required=False)
hit.size2 = self.get_value('size2', required=False, mapfn=int)
hit.location3 = self.get_value('location3', required=False)
hit.size3 = self.get_value('size3', required=False, mapfn=int)
hit.location4 = self.get_value('location4', required=False)
hit.size4 = self.get_value('size4', required=False, mapfn=int)
hit.location5 = self.get_value('location5', required=False)
hit.size5 = self.get_value('size5', required=False, mapfn=int)
hit.location6 = self.get_value('location6', required=False)
hit.size6 = self.get_value('size6', required=False, mapfn=int)
hit.location7 = self.get_value('location7', required=False)
hit.size7 = self.get_value('size7', required=False, mapfn=int)
hit.location8 = self.get_value('location8', required=False)
hit.size8 = self.get_value('size8', required=False, mapfn=int)
hit.location9 = self.get_value('location9', required=False)
hit.size9 = self.get_value('size9', required=False, mapfn=int)
hit.location10 = self.get_value('location10', required=False)
hit.size10 = self.get_value('size10',
required=False,
mapfn=int)
else:
hit.location = self.get_value('location', required=True)
hit.aws_access_key = self.get_value('aws_access_key',
required=True)
hit.aws_secret_key = self.get_value('aws_secret_key',
required=True)
hit.auth_secret = self.get_value('auth_secret', required=False)
hit.title = self.get_value('title', required=True)
hit.description = self.get_value('description', required=True)
hit.reward = self.get_value('reward', required=True)
hit.handle_submit = self.get_value('handle_submit',
required=False,
mapfn=bool)
hit.always_pay = self.get_value('always_pay',
required=False,
mapfn=bool)
hit.sandbox = self.get_value('sandbox', required=False, mapfn=bool)
hit.blacklist = re.split('[,\s]+',
self.request.get('blacklist').strip())
hit.info = self.get_value('info')
if type == 'FOCAL_TIME':
hit.focal_time = iso8601.parse_date(
self.get_value('focal_time',
required=True).replace('Z', ':00Z'))
hit.type = type
hit.next_worker_number = 1
hit.put()
except BadValueError, error:
self.response.set_status(400)
self.response.out.write(error.reason)
return
try:
response = create_hit(self.hit_countdown_url(hit), hit)
hit.mturkid = response.identifier
hit.groupid = response.HITTypeId
hit.time = iso8601.parse_date(response.expiration)
hit.put()
if type == 'FOCAL_TIME':
task = Task(url='/notification',
params={'key': str(hit.key())},
eta=hit.focal_time - timedelta(minutes=2))
task.add('default')
self.response.out.write(self.hit_url(hit))
except (BotoClientError, BotoServerError), aws:
error = 'Error: %s: %s' % (aws.errors[0][0], aws.errors[0][1])
self.response.set_status(500)
self.response.out.write(error)
def click(request):
t = Task(url='/queue/click', params={'keyword': request.POST["keywords"], 'url': request.POST["url_id"],'ip': request.META["REMOTE_ADDR"]})
t.add('processratings')
return HttpResponse("", mimetype="text/plain")
def rating(request):
t = Task(url='/queue/rating', params={'keyword': request.POST["keywords"], 'rating': int(request.POST["rating"]), 'url': request.POST["url_id"],'ip': request.META["REMOTE_ADDR"]})
t.add('processratings')
return HttpResponse("Thanks for rating", mimetype="text/plain")
def get(self):
# create an event to go get this data
task = Task(url='/labs/maptask', params={
'clean': '1',
})
task.add('crawler')
def post(self):
if serviceOn() == False:
return
url = self.request.get("url")
loop = 0
done = False
result = None
while not done and loop < 2:
try:
result = urlfetch.fetch(url)
done = True
except urlfetch.DownloadError:
logging.error("Error loading page (%s)... sleeping" % loop)
if result:
logging.debug("Error status: %s" % result.status_code)
logging.debug("Error header: %s" % result.headers)
logging.debug("Error content: %s" % result.content)
time.sleep(6)
loop = loop + 1
if result is None or result.status_code != 200:
logging.error("Exiting early: error fetching URL: " + result.status_code)
return
hit = False
event = self.request.get("event")
athlete = self.request.get("athlete")
soup = BeautifulSoup(result.content)
for td in soup.html.body.findAll("td"):
# logging.debug(td)
if td.font.__str__().find(athlete) > 0:
# athlete found!
logging.debug("found: %s" % td.contents)
# now climb up the tag chain to find all result details
row = td.parent
# logging.debug("parent: %s" % row.contents)
rank = row.contents[1].font.string
name = row.contents[5].font.string
time = row.contents[13].font.string
hit = True
break
if hit == False:
# parse line by line
lines = result.content.splitlines()
for l in lines:
# logging.debug(l)
if l.find(athlete) > 0:
logging.debug("found raw: %s" % l)
data = re.search("(\d+)\s+(\w+\s\w+)\s+\d+\s.*?\d.*?\s+(([0-9]+:|)[0-9][0-9]\.[0-9][0-9])", l)
if data is not None:
rank = data.group(1).strip()
name = data.group(2).strip()
time = data.group(3).strip()
hit = True
break
else:
data = re.search("(\d+)\s+(\w+,\s\w+)\s+\d+\s.*?\d.*?\s+(([0-9]+:|)[0-9][0-9]\.[0-9][0-9])", l)
if data is not None:
rank = data.group(1).strip()
name = data.group(2).strip()
time = data.group(3).strip()
hit = True
break
else:
logging.error("False Positive!!")
if hit == True:
textBody = name + " finished event " + event + " in " + time + ", ranked " + rank
logging.info(textBody)
# send out the text with the results
task = Task(
url="/sendsmstask",
params={"phone": self.request.get("phone"), "athlete": athlete, "event": event, "text": textBody},
)
task.add("smssender")
# create an event to log the event
task = Task(
url="/loggingtask",
params={
"phone": self.request.get("phone"),
"athlete": self.request.get("athlete"),
"event": event,
"text": textBody,
},
)
task.add("eventlogger")
# else:
# logging.info("unable to find athlete %s for event %s!" % (athlete,self.request.get('event')))
return