def task_enqueue_categories(request):
"""
Enqueues animal tables. To be run by cron
"""
q = Queue('animal-indexes')
category = None #request.GET.get('category')
if category:
taskqueue.add(
Task(url='/adopt/_tasks_/fetch_category',
method='post',
payload=json.dumps({
'category': category,
'url': _getpetharbor_url(x)
})))
else:
for x in PETHARBOR_URLS.keys():
q.add(
Task(url='/adopt/_tasks_/fetch_category',
name="fetch-%s-%s" %
(x, datetime.datetime.now().strftime('%Y%m%d%H%M%S')),
method='post',
payload=str(
json.dumps({
'category': x,
'url': _get_petharbor_url(x)
}))))
return TextResponse('OK')
def apiTimeStat(stat_key, value):
task = Task(url='/stats/new/value',
params={
'stat_key': stat_key,
'value': value
})
task.add('stats')
def _tx():
pending_email = PendingEmail(subscription=sub, edition=edition)
db.put(pending_email)
SendNewsletter.add(
Task(params={'pending_email': pending_email.key()}),
transactional=True)
def get(self):
# review the results for popular stops
reqs = getRequestedStops()
stops_stats = []
keyLookup = []
totalReqs = 0
for key, value in reqs.items():
stops_stats.append({
'stopID': key,
'count': value,
})
totalReqs += value
keyLookup.append(key + ':loc')
# do we have the stop locations?
stopLocations = memcache.get_multi(keyLookup) #reqs.keys())
if stopLocations is None or len(stopLocations) == 0:
logging.error("unable to find stop locations!?")
# create an event to go get this data
task = Task(url='/labs/maptask', params={
'clean': '1',
})
task.add('crawler')
msg = "no data"
else:
logging.debug('yes... found cached copies of the stop locations!')
msg = "your data"
locations = []
median = 2.5
logging.debug('found a total of %s requests with a median %s' %
(str(totalReqs), str(median)))
for key, value in stopLocations.items():
if value is None:
continue
stopID = key.split(':')[0]
# normalized value = count/median * %Total + (count-median)+ base
weight = (float(reqs[stopID]) / median) + float(
reqs[stopID]) - median + 75.0
logging.debug('%s / %s weight is %s' %
(stopID, reqs[stopID], str(weight)))
locations.append({
'stopID': stopID,
'location': value,
'count': reqs[stopID],
'weight': weight,
})
template_values = {
'stops': stops_stats,
'locations': locations,
'message': msg,
}
# create a page that provides a form for sending an SMS message
path = os.path.join(os.path.dirname(__file__), 'mapit.html')
self.response.out.write(template.render(path, template_values))
def get(self):
# validate it is in fact coming from twilio
if config.ACCOUNT_SID == self.request.get('AccountSid'):
logging.debug(
"PHONE request was confirmed to have come from Twilio.")
else:
logging.error("was NOT VALID. It might have been spoofed!")
self.response.out.write(errorResponse("Illegal caller"))
return
# pull the route and stopID out of the request body and
# pad it with a zero on the front if the message forgot
# to include it (that's how they are stored in the DB)
routeID = memcache.get(self.request.get('AccountSid'))
memcache.delete(self.request.get('AccountSid'))
stopID = self.request.get('Digits')
if len(stopID) == 3:
stopID = "0" + stopID
# hack - creating a string out of the details to conform to other interfaces
requestArgs = "%s %s" % (routeID, stopID)
## magic ##
logging.info('starting the magic... %s' % requestArgs)
textBody = api_bridge.getarrivals(requestArgs, 1)
logging.debug('phone results are %s' % textBody)
# create an event to log the event
task = Task(url='/loggingtask',
params={
'phone': self.request.get('Caller'),
'inboundBody': requestArgs,
'sid': self.request.get('SmsSid'),
'outboundBody': textBody,
})
task.add('eventlogger')
# transform the text a smidge so it can be pronounced more easily...
# 1. strip the colons
textBody = textBody.replace(':', ' ')
# 2. add a space between p-and-m and a-and-m
textBody = textBody.replace('pm', 'p m').replace('am', 'a m')
logging.debug('transformed results into %s' % textBody)
# setup the response
r = twilio.Response()
r.append(
twilio.Say(textBody,
voice=twilio.Say.MAN,
language=twilio.Say.ENGLISH,
loop=1))
self.response.out.write(r)
def get(self):
# validate the request parameters
devStoreKey = validateRequest(self.request, api_utils.GETSTOPLOCATION)
if devStoreKey is None:
logging.debug("unable to validate the request parameters")
self.response.headers['Content-Type'] = 'application/javascript'
self.response.out.write(
json.dumps(
api_utils.buildErrorResponse(
'-1', 'Illegal request parameters')))
return
# snare the inputs
stopID = api_utils.conformStopID(self.request.get('stopID'))
logging.debug('getstoplocation request parameters... stopID %s' %
stopID)
if api_utils.afterHours() is True:
# don't run these jobs during "off" hours
json_response = api_utils.buildErrorResponse(
'-1', 'The Metro service is not currently running')
elif stopID is not '':
json_response = stopLocationRequest(stopID)
api_utils.recordDeveloperRequest(devStoreKey, api_utils.GETSTOPS,
self.request.query_string,
self.request.remote_addr)
else:
logging.error("API: invalid request")
json_response = api_utils.buildErrorResponse(
'-1',
'Invalid Request parameters. Did you forget to include a stpID?'
)
api_utils.recordDeveloperRequest(
devStoreKey, api_utils.GETSTOPS, self.request.query_string,
self.request.remote_addr, 'illegal query string combination')
#logging.debug('API: json response %s' % json_response);
# encapsulate response in json
callback = self.request.get('callback')
if callback is not '':
self.response.headers['Content-Type'] = 'application/javascript'
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Access-Control-Allow-Methods'] = 'GET'
response = callback + '(' + json.dumps(json_response) + ');'
else:
self.response.headers['Content-Type'] = 'application/json'
response = json.dumps(json_response)
self.response.out.write(response)
stathat.apiStatCount()
# push event out to anyone watching the live board
task = Task(url='/map/task', params={'stopID': stopID})
task.add('eventlogger')
def application ( environ, start_response ):
start_response('200 OK', [('Content-Type', 'text/plain')])
form = cgi.FieldStorage(fp=environ['wsgi.input'],
environ=environ)
users = db.GqlQuery('SELECT * FROM User')
for user in users:
task = Task(payload=None, countdown =60, method = 'GET', url = '/get_timeline?account=' + user.screen_name)
task.add()
print user.screen_name
return "OK"
def _task_fetch_category(url, category):
"""
Fetch an animal table, parse out information, and enqueue a task to fetch the image and store the data in the database
"""
q = Queue('animal-profiles')
page = fetch(url)
animals = parse_petharbor_table(page.content, category)
for animal in animals:
animal['src_url'] = url
q.add(
Task(url='/adopt/_tasks_/fetch_animal',
name="fetch-%s-%s-%s" %
(category, animal['code'],
datetime.datetime.now().strftime('%Y%m%d%H%M')),
method="post",
payload=str(json.dumps(animal))))
def mail_queue_expander(request):
BATCH_SIZE = 5
edition = db.get(request.form['edition'])
if not edition: pass
page = int(request.form.get('page', 0))
subscriber_q = Query(subscriptions.models.Subscription,
keys_only=True).filter('site =', edition.site).filter(
'active =', True)
if request.form.has_key('cursor'):
subscriber_q = subscriber_q.with_cursor(request.form['cursor'])
subscribers = subscriber_q.fetch(BATCH_SIZE)
if not subscribers:
edition.status = 'complete'
edition.put()
return
task = Task(params={
'edition': edition.key(),
'cursor': subscriber_q.cursor(),
'page': page + 1
},
name="%s-%s-%s-%s" %
(edition.site.slug, edition.issue_num,
edition.publish_after.strftime("%Y%j%H%M-%S"), page + 1))
try:
MailQueueExpander.add(task)
except (TaskAlreadyExistsError, TombstonedTaskError):
raise
for sub in subscribers:
def _tx():
pending_email = PendingEmail(subscription=sub, edition=edition)
db.put(pending_email)
SendNewsletter.add(
Task(params={'pending_email': pending_email.key()}),
transactional=True)
db.run_in_transaction_custom_retries(10, _tx)
def get(self):
start = time.time()
# snare the inputs
dev_key = self.request.get('key')
stopID = api_utils.conformStopID(self.request.get('stopID'))
routeID = self.request.get('routeID')
vehicleID = self.request.get('vehicleID')
#logging.debug('getarrivals request parameters... stopID %s routeID %s vehicleID %s' % (stopID,routeID,vehicleID))
self.request.registry['aggregated_results'] = []
try:
if api_utils.afterHours() is False: # and dev_key != 'uwkiosk9':
# validate the request parameters
devStoreKey = validateRequest(self.request)
if devStoreKey is None:
# filter out the kiosk errors from the log
if (not (dev_key == 'kiosk'
and self.request.get('stopID') == '')):
logging.error(
"failed to validate the request parameters")
self.response.headers[
'Content-Type'] = 'application/javascript'
self.response.out.write(
json.dumps(
api_utils.buildErrorResponse(
'-1',
'Unable to validate the request. There may be an illegal developer key.'
)))
return
if stopID is not '' and routeID is '':
json_response = stopRequest(stopID, dev_key)
api_utils.recordDeveloperRequest(devStoreKey,
api_utils.GETARRIVALS,
self.request.query_string,
self.request.remote_addr)
elif stopID is not '' and routeID is not '':
json_response = stopRouteRequest(stopID, routeID,
devStoreKey)
api_utils.recordDeveloperRequest(devStoreKey,
api_utils.GETARRIVALS,
self.request.query_string,
self.request.remote_addr)
elif routeID is not '' and vehicleID is not '':
json_response = routeVehicleRequest(
routeID, vehicleID, devStoreKey)
api_utils.recordDeveloperRequest(devStoreKey,
api_utils.GETVEHICLE,
self.request.query_string,
self.request.remote_addr)
else:
logging.debug("API: invalid request")
api_utils.recordDeveloperRequest(
devStoreKey, api_utils.GETARRIVALS,
self.request.query_string, self.request.remote_addr,
'illegal query string combination')
json_response = api_utils.buildErrorResponse(
'-1', 'Invalid Request parameters')
# push event out to anyone watching the live board
channels = memcache.get('channels')
if channels is not None:
task = Task(url='/map/task', params={'stopID': stopID})
task.add('eventlogger')
# stop statistics - DISABLED
# if( "kiosk" not in dev_key ):
# task = Task(url='/stats/stop', params={'apikey':dev_key,'stop':stopID})
# task.add('stats')
else:
# don't run these jobs during "off" hours
#logging.debug('shunted... off hour request')
#if dev_key.find('kiosk') >= 0:
# json_response = api_utils.buildErrorResponse('-1','Kiosk requests no longer supported')
#else:
json_response = api_utils.buildErrorResponse(
'-1', 'The Metro service is not currently running')
# encapsulate response in json or jsonp
callback = self.request.get('callback')
if callback is not '':
self.response.headers[
'Content-Type'] = 'application/javascript'
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Access-Control-Allow-Methods'] = 'GET'
response = callback + '(' + json.dumps(json_response) + ');'
else:
self.response.headers['Content-Type'] = 'application/json'
response = json.dumps(json_response)
self.response.out.write(response)
except DeadlineExceededError:
self.response.clear()
self.response.set_status(500)
self.response.out.write(
"This operation could not be completed in time...")
# persist some statistics
# stathat:
if api_utils.afterHours() is False and dev_key != 'uwkiosk9':
stathat.apiTimeStat(config.STATHAT_API_GETARRIVALS_TIME_KEY,
((time.time() - start) * 1000))
stathat.apiStatCount()
def get(self):
# create an event to go get this data
task = Task(url='/labs/maptask', params={
'clean': '1',
})
task.add('crawler')
class HitList(RequestHandler):
def get(self):
user = users.get_current_user()
type = self.get_value('type', required=False)
type = self.map_type(type)
self.render('priv/hit_list.html', {
'user': user,
'type': type,
'time': datetime_format(datetime.utcnow())
})
def post(self):
type = self.get_value('type', required=False)
type = self.map_type(type)
try:
hit = HIT()
hit.owner = users.get_current_user()
hit.lifetime = self.get_value('lifetime', required=True, mapfn=int)
hit.duration = self.get_value('duration', required=True, mapfn=int)
hit.approval_delay = self.get_value('approval_delay',
required=True,
mapfn=int)
hit.hit_approval = self.get_value('hit_approval',
required=False,
mapfn=int)
hit.hit_approval_rate = self.get_value('hit_approval_rate',
required=True,
mapfn=int)
hit.accepted_hit_rate = self.get_value('accepted_hit_rate',
required=True,
mapfn=int)
hit.returned_hit_rate = self.get_value('returned_hit_rate',
required=True,
mapfn=int)
hit.abandoned_hit_rate = self.get_value('abandoned_hit_rate',
required=True,
mapfn=int)
hit.rejected_hit_rate = self.get_value('rejected_hit_rate',
required=True,
mapfn=int)
hit.locale_qualification = self.get_value('locale_qualification',
required=True)
hit.frame_height = self.get_value('frame_height',
required=True,
mapfn=int)
if type != 'MULTIPLE_URLS':
hit.max_workers = self.get_value('max_workers',
required=True,
mapfn=int)
if type != 'PASS_THROUGH':
hit.min_workers = self.get_value('min_workers',
required=True,
mapfn=int)
if type == 'MULTIPLE_URLS':
hit.location1 = self.get_value('location1', required=True)
hit.size1 = self.get_value('size1', required=True, mapfn=int)
hit.location2 = self.get_value('location2', required=False)
hit.size2 = self.get_value('size2', required=False, mapfn=int)
hit.location3 = self.get_value('location3', required=False)
hit.size3 = self.get_value('size3', required=False, mapfn=int)
hit.location4 = self.get_value('location4', required=False)
hit.size4 = self.get_value('size4', required=False, mapfn=int)
hit.location5 = self.get_value('location5', required=False)
hit.size5 = self.get_value('size5', required=False, mapfn=int)
hit.location6 = self.get_value('location6', required=False)
hit.size6 = self.get_value('size6', required=False, mapfn=int)
hit.location7 = self.get_value('location7', required=False)
hit.size7 = self.get_value('size7', required=False, mapfn=int)
hit.location8 = self.get_value('location8', required=False)
hit.size8 = self.get_value('size8', required=False, mapfn=int)
hit.location9 = self.get_value('location9', required=False)
hit.size9 = self.get_value('size9', required=False, mapfn=int)
hit.location10 = self.get_value('location10', required=False)
hit.size10 = self.get_value('size10',
required=False,
mapfn=int)
else:
hit.location = self.get_value('location', required=True)
hit.aws_access_key = self.get_value('aws_access_key',
required=True)
hit.aws_secret_key = self.get_value('aws_secret_key',
required=True)
hit.auth_secret = self.get_value('auth_secret', required=False)
hit.title = self.get_value('title', required=True)
hit.description = self.get_value('description', required=True)
hit.reward = self.get_value('reward', required=True)
hit.handle_submit = self.get_value('handle_submit',
required=False,
mapfn=bool)
hit.always_pay = self.get_value('always_pay',
required=False,
mapfn=bool)
hit.sandbox = self.get_value('sandbox', required=False, mapfn=bool)
hit.blacklist = re.split('[,\s]+',
self.request.get('blacklist').strip())
hit.info = self.get_value('info')
if type == 'FOCAL_TIME':
hit.focal_time = iso8601.parse_date(
self.get_value('focal_time',
required=True).replace('Z', ':00Z'))
hit.type = type
hit.next_worker_number = 1
hit.put()
except BadValueError, error:
self.response.set_status(400)
self.response.out.write(error.reason)
return
try:
response = create_hit(self.hit_countdown_url(hit), hit)
hit.mturkid = response.identifier
hit.groupid = response.HITTypeId
hit.time = iso8601.parse_date(response.expiration)
hit.put()
if type == 'FOCAL_TIME':
task = Task(url='/notification',
params={'key': str(hit.key())},
eta=hit.focal_time - timedelta(minutes=2))
task.add('default')
self.response.out.write(self.hit_url(hit))
except (BotoClientError, BotoServerError), aws:
error = 'Error: %s: %s' % (aws.errors[0][0], aws.errors[0][1])
self.response.set_status(500)
self.response.out.write(error)
