def store_all_division_standings(league, division, gender, age, agegroup):
q = opl_db.Division.all()
if league:
q.filter("league =", league)
if division:
q.filter("division = ", division)
if gender:
q.filter("gender =", gender)
if age:
q.filter("age =", age)
if agegroup:
q.filter("agegroup = ", agegroup)
for r in q.run():
parms = {
'l': r.league,
'u': r.url,
'a': r.age,
'ag': r.agegroup,
'g': r.gender,
'd': r.division
}
url = '/store-division-standings?' + urllib.urlencode(parms)
logging.debug('URL:' + url)
t = Task(payload=None, method='GET', url=url)
t.add()
return 'Done'
def get(self):
# query the StopLocationLoader for all stops
q = db.GqlQuery("select * from StopLocationLoader")
stops = q.fetch(500)
offset = 500
while stops is not None and len(stops) > 0:
for s in stops:
# create a new task for each stop
task = Task(url='/port/stop/task/',
params={'stopID':s.stopID,
'name':s.name,
'description':s.description,
'lat':str(s.lat),
'lon':str(s.lon),
'direction':s.direction,
})
task.add('crawler')
# get the next bunch of stops from the query
stops = q.fetch(500,offset)
offset += 500
logging.debug('Finished spawning %s StopLocationLoader tasks!' % str(offset-500))
self.response.out.write('done spawning porting tasks!')
def mail(self):
admin_proxy = AdminApi()
subject = "New phonoblaster station: %s" % (self.station.name)
if self.station.type == 'page':
body = """
A new station (%s/%s) has been created on Phonoblaster:
%s, %s
Global number of stations: %s
""" %(config.SITE_URL, self.station.shortname, self.station.name, self.station.link, admin_proxy.number_of_stations)
else:
body = """
A new station (%s/%s) has been created on Phonoblaster:
%s, %s
Global number of stations: %s
""" %(config.SITE_URL, self.station.shortname, self.station.name, "https://graph.facebook.com/"+self.station.key().name(), admin_proxy.number_of_stations)
logging.info(body)
task = Task(
url = "/taskqueue/mail",
params = {
"to": "*****@*****.**",
"subject": subject,
"body": body,
}
)
task.add(queue_name = "worker-queue")
def get(self) :
# shove a task in the queue because we need more time then
# we may be able to get in the browser
for route in range(1, 100):
task = Task(url="/gtfs/port/routes/task",params={'route':route})
task.add('crawler')
self.response.out.write('done. spawned a task to go do the route transformations')
def _EnqueueMasterRecallTask(self, owner_email, message_criteria,
task_key_id):
"""Add master recall task with error handling.
Args:
owner_email: String email address of user running this recall.
message_criteria: String criteria (message-id) to recall.
task_key_id: Int unique id of the parent task.
Raises:
re-raises any task queue errors.
"""
task_name = '%s_%s' % (
view_utils.CreateSafeUserEmailForTaskName(owner_email),
view_utils.GetCurrentDateTimeForTaskName())
master_task = Task(name=task_name,
params={'owner_email': owner_email,
'task_key_id': task_key_id,
'message_criteria': message_criteria},
target='0.recall-backend',
url='/backend/recall_messages')
try:
master_task.add(queue_name='recall-messages-queue')
except TaskQueueError:
view_utils.FailRecallTask(task_key_id=task_key_id,
reason_string='Failed to enqueue master task.')
raise
def save(self, image):
image_url = "/picture/" + str(image.key()) + "/view"
thumbnail_blob_key = self.save_thumbnail(image)
thumbnail_url = "/picture/" + str(thumbnail_blob_key) + "/view"
self.station_proxy.update_background(image_url, thumbnail_url)
data = {
"entity": "background",
"event": "new",
"content": image_url,
}
task = Task(
url = "/taskqueue/multicast",
params = {
"station": config.VERSION + "-" + self.station_proxy.station.shortname,
"data": json.dumps(data)
}
)
task.add(queue_name="worker-queue")
self.response.out.write(json.dumps({
"response": True,
"src_full": image_url,
"src_thumb": thumbnail_url,
"blobstore_url": self.blobstore_url,
}))
def get(self):
day_of_week = makeDayOfWeek()
query = Alert.query(Alert.days.IN([day_of_week]))
to_put = []
for q in query:
if q.confirmed:
alert_today = AlertLog(
alert_id=q.key.id(),
email=q.email,
phone=q.phone,
carrier=q.carrier,
start1=q.start1,
start2=q.start2,
start3=q.start3,
end1=q.end1,
end2=q.end2,
end3=q.end3,
time=q.time
)
to_put.append(alert_today)
else:
continue # used for debugging
ndb.put_multi(to_put)
logging.debug("Prepared today's list of alerts to send.")
time.sleep(5) # give it 5 seconds for the new data to take
first_task = Task(payload=None, url="/admin/sendalerts")
first_task.add(queue_name="alertsqueue")
logging.debug('ManageAlertsListQ successfully initiated.')
self.response.out.write('ManageAlertsListQ successfully initiated.')
def post(self):
channel_id = str(self.request.get('from'))
logging.info("%s is ready to receive messages" %(channel_id))
# Init station proxy
m = re.match(r"(\w+).(\w+)", channel_id)
shortname = m.group(1)
station_proxy = StationApi(shortname)
extended_session = station_proxy.add_to_sessions(channel_id)
if(extended_session):
# Add a taskqueue to warn everyone
new_session_data = {
"entity": "session",
"event": "new",
"content": extended_session,
}
task = Task(
url = "/taskqueue/multicast",
params = {
"station": config.VERSION + "-" + shortname,
"data": json.dumps(new_session_data)
}
)
task.add(queue_name="sessions-queue")
def send_alerts(self, wait=0):
todays_alerts = AlertLog.query()
todays_alerts_len = todays_alerts.filter(AlertLog.complete == False).count()
if todays_alerts_len == 0:
logging.debug("Done for the day. See you tomorrow.")
else:
while todays_alerts_len > 0:
current_alerts = todays_alerts.filter(AlertLog.complete == False).order(AlertLog.time)
a = current_alerts.get()
now = makeNowTime()
if a.time <= now:
self.requestFreshData(a.time)
generate_msg_info(a)
logging.debug("I sent an alert that was scheduled for %s.", a.time.strftime("%I:%M %p"))
a.complete = True
a.sent = datetime.datetime.now()
a.put()
todays_alerts_len = todays_alerts_len - 1
time.sleep(1) # give it a second for the new data to take
else:
wait = makeWait(a.time)
logging.debug("Going to count down for %d seconds.", wait)
break
the_only_other_task = Task(payload=None, url="/admin/sendalerts", countdown=wait)
the_only_other_task.add(queue_name="alertsqueue")
def get(self,time_slot=""):
logging.debug('running cron for timeslot %s' % time_slot)
if systemIsOn() is False:
logging.error('bailing... the system is turned off')
return
# grab the row of data out of the spreadsheet
results = getResults(time_slot)
messages = getMessages(results)
# cycle over all the users and send them a message
users = db.GqlQuery("select * from User").fetch(200)
if len(users) <= 0:
logging.error('No users in the system!')
for u in users:
# send the SMS out with a background task
logging.debug('sending notifications to %s' % u.phone_number)
task = Task(url='/sendsmstask',
params={'phone':u.phone_number,
'msg_one':messages[0],
'msg_two':messages[1],
'msg_three':messages[2],
})
task.add('smssender')
def _EnqueueMasterRecallTask(self, owner_email, message_criteria,
task_key_id):
"""Add master recall task with error handling.
Args:
owner_email: String email address of user running this recall.
message_criteria: String criteria (message-id) to recall.
task_key_id: Int unique id of the parent task.
Raises:
re-raises any task queue errors.
"""
task_name = '%s_%s' % (view_utils.CreateSafeUserEmailForTaskName(
owner_email), view_utils.GetCurrentDateTimeForTaskName())
master_task = Task(name=task_name,
params={
'owner_email': owner_email,
'task_key_id': task_key_id,
'message_criteria': message_criteria
},
target='0.recall-backend',
url='/backend/recall_messages')
try:
master_task.add(queue_name='recall-messages-queue')
except TaskQueueError:
view_utils.FailRecallTask(
task_key_id=task_key_id,
reason_string='Failed to enqueue master task.')
raise
def post(self):
message = xmpp.Message(self.request.POST)
logging.info("XMPP request! Sent form %s with message %s" % (message.sender,message.body))
# normalize the XMPP requests
if message.sender.find('@'):
caller = message.sender.split('/')[0]
else:
caller = message.sender.get('from')
if message.body.lower().find('parking') > -1:
logging.info('parking request via XMPP')
response = api_bridge.getparking()
elif message.body.lower().find('help') > -1:
response = "Bus arrivals: stopID -or- routeID stopID Parking: 'parking' Stats: 'stats' Help: 'help'"
elif message.body.lower().find('stats') > -1:
response = meta.getStats(caller)
else:
## magic ##
response = api_bridge.getarrivals(message.body,10)
# to make it a little easier to read, add newlines before each route report line
response = response.replace('Route','\nRoute')
# create an event to log the request
task = Task(url='/loggingtask', params={'from':caller,
'to':message.to,
'inboundBody':message.body,
'sid':'xmpp',
'outboundBody':response,})
task.add('eventlogger')
# reply to the chat request
message.reply(response)
def MakeFollows(self):
"""
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
ADD FOLLOWS FOR ADMIN USERS
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
"""
nextURL = None
firstURL = self.request.get('nexturl')
query = PointRoot.query().order(PointRoot.url)
if firstURL:
query = query.filter(PointRoot.url >= firstURL)
pointRoots = query.fetch(11)
if len(pointRoots) == 11:
nextURL = pointRoots[-1].url
pointRootsToReview = pointRoots[:10]
else:
pointRootsToReview = pointRoots
i = 0
for pointRoot in pointRootsToReview:
pointRootKey = pointRoot.key
followers = {}
versions = pointRoot.getAllVersions()
for point in versions:
if point.version == 1:
followers[point.authorURL] = 'created'
elif not point.authorURL in followers:
followers[point.authorURL] = 'edited'
for comment in pointRoot.getComments():
if not comment.userUrl in followers:
followers[comment.userUrl] = 'commented'
logging.info('ROOT: %s FOLLOWERS: %s' %
(pointRoot.url, str(followers)))
for url in followers.iterkeys():
followType = followers[url]
previousNamespace = namespace_manager.get_namespace()
if previousNamespace and previousNamespace != '':
namespace_manager.set_namespace('') # DEFAULT NAMESPACE
usr = WhysaurusUser.getByUrl(url)
namespace_manager.set_namespace(previousNamespace)
else:
usr = WhysaurusUser.getByUrl(url)
logging.info('Trying to follow for U:%s, R:%s, T:%s' %
(url, pointRoot.url, followType))
f = None
f = Follow.createFollow(usr.key, pointRootKey, followType)
if f:
i = i + 1
logging.info('ADDED follow for U:%s, R:%s, T:%s' %
(url, pointRoot.url, followType))
logging.info('Added %d follows' % i)
if nextURL:
t = Task(url="/MakeFollows", params={'nexturl': nextURL})
t.add(queue_name="notifications")
logging.info('Requeing MakeFollows task to start at url %s ' %
nextURL)
def queueNightlyTask(self):
now = PST.convert(datetime.datetime.now())
tomorrow = now + datetime.timedelta(days=1)
half_past_midnight = datetime.time(hour=7, minute=30) # PST
taskTime = datetime.datetime.combine(tomorrow, half_past_midnight)
t = Task(url='/job/DBIntegrityCheck', method="GET", eta=taskTime)
t.add(queue_name="dbchecker")
def queueNightlyTask(self):
now = PST.convert(datetime.datetime.now())
tomorrow = now + datetime.timedelta(days = 1)
half_past_midnight = datetime.time(hour=7, minute=30) # PST
taskTime = datetime.datetime.combine(tomorrow, half_past_midnight)
t = Task(url='/job/DBIntegrityCheck', method="GET", eta=taskTime)
t.add(queue_name="dbchecker")
def get(self,routeID=""):
# create a new task with this link
#crawlURL = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"
crawlURL = URLBASE + 'r=' + routeID
task = Task(url='/routelist/crawlingtask', params={'crawl':crawlURL,'routeID':'00'})
task.add('crawler')
logging.info("Added new task for %s" % crawlURL)
return
def queue_token_run(token):
cls = token.prosthetic.classname
logging.info("Queueing run task for token %s for %s on %s" %
(token.oauth_key, token.weavr_name, cls))
task = Task(url='/runner/run_task/',
method='POST',
params={'token': token.oauth_key})
task.add('default')
def send(self):
user = self.current_user
message = self.request.get('message')
t = Task(url='/broadcastChatroom',
params={'userName':user.name,
'message':message
})
t.add(queue_name="notifications")
def welcomeSolicitor(phone):
# welcome the new user with an SMS message
welcome_message = "Welcome to SMSMyBus! The first three requests are complimentary, but going forward you will need to signup. Please visit smsmybus.com to learn more."
task = Task(url='/admin/sendsms',
params={'phone':phone,
'sid':'complimentary results',
'text':welcome_message
})
task.add('smssender')
def welcomeNewUser(phone):
# welcome the new user with an SMS message
welcome_message = "Welcome to SMSMyBus. Your account is now active! Just send in a stop ID to find your bus."
task = Task(url='/admin/sendsms',
params={'phone':phone,
'sid':'new user',
'text':welcome_message
})
task.add('smssender')
def get(self, model=""):
url = '/droptable/%s' % model
logging.debug('getting ready to start task to drop the %s table' %
model)
task = Task(url=url, params={})
task.add('crawler')
self.response.out.write(
'got it. started the background task to delete all %s entities' %
model)
def get(self):
"""Queue run tasks for each registered Weavr"""
logging.info("Running cron job. Queueing run tasks.")
for prosthetic in ProstheticData.all(keys_only=True):
logging.info("Queueing run task for %s" % str(prosthetic))
task = Task(url='/runner/prosthetic_task/', method='GET',
params={'key': str(prosthetic)})
task.add('default')
logging.info("Finished running cron job.")
def QueueTask(self):
taskurl = self.request.get('task')
if taskurl:
fullurl = '/' + taskurl
t = Task(url=fullurl)
t.add(queue_name="notifications")
self.response.out.write('OK I wrote %s to the notifications queue' % fullurl)
else:
self.response.out.write('Need a task URL parameter')
def send(self):
user = self.current_user
message = self.request.get('message')
t = Task(url='/broadcastChatroom',
params={
'userName': user.name,
'message': message
})
t.add(queue_name="notifications")
def MakeFollows(self):
"""
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
ADD FOLLOWS FOR ADMIN USERS
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
"""
nextURL = None
firstURL = self.request.get('nexturl')
query = PointRoot.query().order(PointRoot.url)
if firstURL:
query = query.filter(PointRoot.url >= firstURL)
pointRoots = query.fetch(11)
if len(pointRoots) == 11:
nextURL = pointRoots[-1].url
pointRootsToReview = pointRoots[:10]
else:
pointRootsToReview = pointRoots
i = 0
for pointRoot in pointRootsToReview:
pointRootKey = pointRoot.key
followers = {}
versions = pointRoot.getAllVersions()
for point in versions:
if point.version == 1:
followers[point.authorURL] = 'created'
elif not point.authorURL in followers:
followers[point.authorURL] = 'edited'
for comment in pointRoot.getComments():
if not comment.userUrl in followers:
followers[comment.userUrl] = 'commented'
logging.info('ROOT: %s FOLLOWERS: %s' % (pointRoot.url, str(followers)))
for url in followers.iterkeys():
followType = followers[url]
previousNamespace = namespace_manager.get_namespace()
if previousNamespace and previousNamespace != '':
namespace_manager.set_namespace('') # DEFAULT NAMESPACE
usr = WhysaurusUser.getByUrl(url)
namespace_manager.set_namespace(previousNamespace)
else:
usr = WhysaurusUser.getByUrl(url)
logging.info('Trying to follow for U:%s, R:%s, T:%s' % (url, pointRoot.url, followType))
f = None
f = Follow.createFollow(usr.key, pointRootKey, followType)
if f:
i = i + 1
logging.info('ADDED follow for U:%s, R:%s, T:%s' % (url, pointRoot.url, followType))
logging.info('Added %d follows' % i)
if nextURL:
t = Task(url="/MakeFollows", params={'nexturl':nextURL})
t.add(queue_name="notifications")
logging.info('Requeing MakeFollows task to start at url %s ' % nextURL)
def task(name, method):
task = Task(
url = "/taskqueue/counter",
params = {
"shard_name": name,
"method": method
}
)
task.add(queue_name = "counters-queue")
def QueueTask(self):
taskurl = self.request.get('task')
if taskurl:
fullurl = '/' + taskurl
t = Task(url=fullurl)
t.add(queue_name="notifications")
self.response.out.write('OK I wrote %s to the notifications queue' % fullurl)
else:
self.response.out.write('Need a task URL parameter')
def createTask(phone, msg, sec):
logging.debug("Creating new task to fire in %s minutes" %
str(int(sec) / 60))
task = Task(url='/reminder',
params={
'phone': phone,
'msg': msg
},
countdown=sec)
task.add('reminders')
def queueEventRecord(cls, userKeyUrlsafe, entityKey1Urlsafe,
entityKey2Urlsafe, eventName):
taskParams = {'userKeyUrlsafe': userKeyUrlsafe, 'eventName': eventName}
if entityKey1Urlsafe:
taskParams['entityKey1Urlsafe'] = entityKey1Urlsafe
if entityKey2Urlsafe:
taskParams['entityKey2Urlsafe'] = entityKey2Urlsafe
t = Task(url='/recordEvent', params=taskParams)
t.add(queue_name="recordEvents")
def _start_post_activity_completion_process(self, activity):
if os.environ.get('ENV_TYPE') is None:
if os.environ.get('SERVER_SOFTWARE', '').startswith('Development'):
eta = 420
else:
eta = 2100
task = Task(url='/post_activity_completion/',
method='GET',
params={'activity_key': activity.key.urlsafe()},
countdown=eta)
task.add('postActivityCompletion')
def _on_event_formation(self):
#Queue it for life cycle management
if os.environ.get('ENV_TYPE') is None:
if self._event.start_time is not None and self._event.start_time != "":
task_execution_time = self._event.start_time - timedelta(minutes=5)
if self._event.expiration is not None and self._event.expiration != "":
task_execution_time = datetime.utcnow() + timedelta(minutes=5)
self._event.start_time = datetime.utcnow() + timedelta(minutes=10)
self._event.put()
goTask = Task(eta=task_execution_time, url='/activity_life_cycle/',method='GET',params={'activity': self._event.key.urlsafe()})
goTask.add('activityLifeCycle')
def _start_activity_closure_process(self, activity):
if os.environ.get('ENV_TYPE') is None:
if os.environ.get('SERVER_SOFTWARE', '').startswith('Development'):
eta = 1800
else:
eta = 259200
task = Task(url='/activity_closure/',
method='GET',
params={'activity_key': activity.key.urlsafe()},
countdown=eta)
task.add('activityClosure')
def get(self, routeID=""):
# create a new task with this link
#crawlURL = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"
crawlURL = URLBASE + 'r=' + routeID
task = Task(url='/crawl/routelist/crawlingtask',
params={
'crawl': crawlURL,
'routeID': '00'
})
task.add('crawler')
logging.info("Added new task for %s" % crawlURL)
return
def stats_init(self, request):
query = request.q
if query == "" or query == None:
return StatsInitResponse(task_id="")
task_id = str(md5.new(str(time.time())).hexdigest())
task = Task(url='/queue/stats', params={'task_id': task_id, 'q': query}, name=task_id)
task.add(queue_name='stats')
memcache.set('task|' + task_id, {'status': 'pending'}, 3600)
return StatsInitResponse(task_id=task_id)
def send(self):
self._data.update({
"site_url": config.SITE_URL,
})
task = Task(
url="/queue/event",
params={
"name": self._name,
"data": json.dumps(self._data)
}
)
task.add(queue_name="events-queue")
def queueEventRecord(cls, userKeyUrlsafe, entityKey1Urlsafe, entityKey2Urlsafe, eventName):
taskParams = {
'userKeyUrlsafe':userKeyUrlsafe,
'eventName': eventName
}
if entityKey1Urlsafe:
taskParams['entityKey1Urlsafe'] = entityKey1Urlsafe
if entityKey2Urlsafe:
taskParams['entityKey2Urlsafe'] = entityKey2Urlsafe
t = Task(url='/recordEvent', params=taskParams)
t.add(queue_name="recordEvents")
def store_all_schedules(league=None, division=None, gender=None, age=None):
q = opl_db.Division.all()
if league:
q.filter("league = ", league)
if division:
q.filter("division = ", division)
if gender:
q.filter("gender =",gender)
if age:
q.filter("age = ", age)
for r in q.run():
logging.debug('R:',r.league,' ',r.gender)
if len(r.sched_urls) > 0:
for u in r.sched_urls:
parms = { 'u' : u, 'l' : r.league, 'g' : r.gender, 'd' : r.division, 'a' : r.age }
t = Task(method='GET', url='/store-schedule?'+urllib.urlencode(parms));
t.add()
##fetch_schedule_results(u, r.league, r.division, r.gender, r.age)
else:
##fetch_schedule_results(r.url, r.league, r.division, r.gender, r.age)
parms = { 'u' : r.url, 'l' : r.league, 'g' : r.gender, 'd' : r.division, 'a' : r.age }
t = Task(method='GET', url='/store-schedule?'+urllib.urlencode(parms));
t.add()
return 'done'
def post(self):
# validate it is in fact coming from twilio
if config.ACCOUNT_SID != self.request.get('AccountSid'):
logging.error("Inbound request was NOT VALID. It might have been spoofed!")
self.response.out.write(errorResponse("Illegal caller"))
return
# who called? and what did they ask for?
phone = self.request.get("From")
msg = self.request.get("Body")
logging.info("New inbound request from %s with message, %s" % (self.request.get('From'),msg))
if paywall.isUserValid(phone) is False:
if paywall.isUserVirgin(phone) is True:
logging.info('Brand new caller - welcome them')
paywall.welcomeSolicitor(phone)
else:
# ignore caller
logging.info('We have seen this number before. Ignore this request')
return
# interrogate the message body to determine what to do
if msg.lower().find('parking') > -1:
response = api_bridge.getparking()
elif msg.lower().find('help') > -1:
response = "Bus arrival requests are either, stopID -or- routeID stopID Send 'parking' to find parking details"
elif msg.lower().find('stats') > -1:
response = meta.getStats(phone)
else:
## magic ##
response = api_bridge.getarrivals(msg,4)
if len(response) > 140:
response = response[0:140]
# create an event to log the request
task = Task(url='/loggingtask', params={'from':self.request.get('From'),
'to':self.request.get('To'),
'inboundBody':self.request.get('Body'),
'sid':self.request.get('SmsSid'),
'outboundBody':response,})
task.add('eventlogger')
# setup the response SMS
r = twilio.Response()
r.append(twilio.Sms(response))
self.response.out.write(r)
return
def post(self):
inbound_message = mail.InboundEmailMessage(self.request.body)
logging.info("Email request! Sent from %s with message subject %s" % (inbound_message.sender,inbound_message.subject))
body = inbound_message.subject
logging.debug("email body arguments %s" % body)
# ignore anything sent by ourselves to avoid an infinite loop
if inbound_message.sender == config.EMAIL_SENDER_ADDRESS:
self.response.set_status(200)
return
if body.lower().find('parking') > -1:
logging.info('parking request via email')
response = api_bridge.getparking()
else:
## magic ##
response = api_bridge.getarrivals(body,10)
# to make it a little easier to read, add newlines before each route report line
response = response.replace('Route','\nRoute')
# send back the reply with the results
header = "Thanks for your request! Here are your results...\n\n"
footer = "\n\nThank you for using SMSMyBus!\nhttps://www.smsmybus.com"
# setup the response email
message = mail.EmailMessage()
message.sender = config.EMAIL_SENDER_ADDRESS
#message.bcc = config.EMAIL_BCC_ADDRESS
message.to = inbound_message.sender
message.subject = 'Your Metro schedule estimates for stop %s' % getStopID(body)
message.body = header + response + footer
logging.debug('sending results to %s' % message.to)
message.send()
# create an event to log the event
task = Task(url='/loggingtask', params={'from':inbound_message.sender,
'to':inbound_message.to,
'inboundBody':body,
'sid':'email',
'outboundBody':response,})
task.add('eventlogger')
self.response.set_status(200)
return
def post_all_feeds():
"""Post all new items for feeds for a specific interval"""
if request.headers.get('X-Appengine-Cron') != 'true':
raise ndb.Return(jsonify_error(message='Not a cron call'))
logger.info('Starting a post job')
futures = []
for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
feeds = feed_class.query(feed_class.is_dirty == True)
logger.info("Got some feeds_count: %s feeds_type: %s", feeds.count(), feed_type)
success = 0
more = True
cursor = None
while more:
feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch])
if not keys:
continue
futures.append(Queue().add_async(Task(url=url_for('tq_feed_post-canonical'), method='POST', params={'keys': keys})))
success += len(feeds_to_fetch)
logger.info('queued post for %d feeds feed_type:%s', success, feed_type)
for future in futures:
yield future
logger.info('Finished Post Job')
yield write_epoch_to_stat(Stat, 'post_job')
raise ndb.Return(jsonify(status='ok'))
def update_all_feeds(interval_id):
"""Update all feeds for a specific interval"""
if request.headers.get('X-Appengine-Cron') != 'true':
raise ndb.Return(jsonify_error(message='Not a cron call'))
for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
feeds = Feed.for_interval(interval_id)
success = 0
more = True
cursor = None
futures = []
while more:
feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
feeds_to_fetch = filter(lambda x: getattr(x, 'external_polling_bucket', DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET, feeds_to_fetch)
keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch])
if not keys:
continue
futures.append(Queue('poll').add_async(Task(url=url_for('tq_feed_poll-canonical'), method='POST', params={'keys': keys})))
success += 1
for future in futures:
yield future
logger.info('queued poll for %d feeds at interval_id=%s', success, interval_id)
raise ndb.Return(jsonify(status='ok'))
def instagram_push_update():
data = request.stream.read()
instagram_client_secret = Configuration.value_for_name('instagram_client_secret')
server_signature = request.headers.get('X-Hub-Signature', None)
signature = hmac.new(str(instagram_client_secret), data, digestmod=hashlib.sha1).hexdigest()
if server_signature != signature:
logger.warn('Got PuSH subscribe POST from instagram w/o valid signature: sent=%s != expected=%s',
server_signature, signature)
raise ndb.Return('')
logger.info('Got PuSH body: %s', data)
logger.info('Got PuSH headers: %s', request.headers)
parsed_feed = json.loads(data)
user_ids = [int(x.get('object_id')) for x in parsed_feed]
feeds = InstagramFeed.query(InstagramFeed.user_id.IN(user_ids))
cursor = None
more = True
keys = []
while more:
feed_keys, cursor, more = feeds.fetch_page(BATCH_SIZE, keys_only=True, start_cursor=cursor)
keys += feed_keys
keys = ','.join([x.urlsafe() for x in keys])
if keys:
yield Queue('poll').add_async(Task(url=url_for('tq_feed_poll-canonical'), method='POST', params={'keys': keys}))
raise ndb.Return('ok')
def schedule_task_for_each_table(self, project_id, dataset_id,
page_token=None):
list_response = self.big_query.list_tables(project_id, dataset_id,
page_token=page_token)
if 'tables' in list_response:
table_id_list = [table['tableReference']['tableId']
for table in list_response['tables']]
tasks = self.create_table_tasks(project_id, dataset_id,
table_id_list)
Tasks.schedule(queue_name='bigquery-tables', tasks=tasks)
else:
logging.info("Dataset %s.%s is empty", project_id, dataset_id)
return
if 'nextPageToken' in list_response:
url = '/bigQuery/project/%s/dataset/%s?pageToken=%s' % (
project_id, dataset_id, list_response['nextPageToken'])
task_name = '%s-%s-%s-%s' % (project_id, dataset_id,
list_response['nextPageToken'],
datetime.utcnow().strftime("%Y%m%d"))
next_task = Task(
method='GET',
url=url,
name=task_name)
Tasks.schedule(queue_name='bigquery-list', tasks=[next_task])
else:
logging.info("There is no more tables in this dataset")
def _EnqueueUserRetrievalTasks(self, message_criteria, owner_email):
"""Efficiently add tasks to enumerate domain users as a list (bulk add).
Bulk add() saves roundtrips (rpc calls).
Args:
message_criteria: String criteria (message-id) to recall.
owner_email: String email address of user running this recall.
"""
user_retrieval_tasks = []
# Use countdown to space out these requests a little.
# There is a 15 request/s quota on the Admin SDK API.
limit_requests_s = 15
for prefix_counter, email_prefix in enumerate(PartitionEmailPrefixes()):
user_retrieval_tasks.append(
Task(countdown=(prefix_counter / limit_requests_s),
name='%s_%s_%s' % (
view_utils.CreateSafeUserEmailForTaskName(owner_email),
email_prefix,
view_utils.GetCurrentDateTimeForTaskName()),
params={'email_prefix': email_prefix,
'message_criteria': message_criteria,
'owner_email': owner_email,
'task_key_id': self._task_key_id},
target='recall-backend',
url='/backend/retrieve_domain_users'))
self._AddUserRetrievalTask(task=user_retrieval_tasks)
def _EnqueueUserRecallTasks(self, message_criteria, owner_email):
"""Efficiently add tasks for each user to recall messages (bulk add).
Bulk add() saves roundtrips (rpc calls).
Args:
message_criteria: String criteria (message-id) to recall.
owner_email: String email address of user running this recall.
"""
if recall_task.RecallTaskModel.IsTaskAborted(self._task_key_id):
return
cursor = None
while True:
user_recall_tasks = []
results, cursor, unused_more = (
domain_user.DomainUserToCheckModel.FetchOnePageOfActiveUsersForTask(
task_key_id=self._task_key_id,
cursor=cursor))
for user in results:
user_recall_tasks.append(Task(
name='%s_%s_%s' % (
view_utils.CreateSafeUserEmailForTaskName(owner_email),
view_utils.CreateSafeUserEmailForTaskName(user.user_email),
view_utils.GetCurrentDateTimeForTaskName()),
params={'message_criteria': message_criteria,
'task_key_id': self._task_key_id,
'user_email': user.user_email,
'user_key_id': user.key.id()},
target='recall-backend',
url='/backend/recall_user_messages'))
if not user_recall_tasks:
break
self._AddUserRecallTasks(user_recall_tasks=user_recall_tasks)
def reset_endpoints():
"""
Handler which creates reset endpoint tasks for selected endpoints/orgs.
"""
endpoint_indexes = request.form.getlist('endpoint_index')
org_uid = request.form.get('org_uid')
if not endpoint_indexes:
flash("At least one endpoint is required")
return render_template('select_endpoints.html',
endpoints=ENDPOINTS,
org_uid=org_uid), 200
if org_uid:
taskqueue.add(target='admin',
url='/admin/reset_endpoints_task/{}'.format(org_uid),
params={'endpoint_index': endpoint_indexes})
flash("Kicked off reset of {} endpoints for {}".format(
len(endpoint_indexes), org_uid))
return redirect(prefix('/'))
else:
count = query_to_tasks(
query=Org.query(Org.status == CONNECTED),
queue=Queue('admin'),
task_generator=lambda key: Task(
url='/admin/reset_endpoints_task/{}'.format(key.string_id()),
params={'endpoint_index': endpoint_indexes}))
flash("Kicked off reset of {} endpoints for {} orgs".format(
len(endpoint_indexes), count))
return redirect(prefix('/commands'))
def log(**kwargs):
# THIS IS WHERE ALL THE HEAVY LIFTING IS DONE
logging.info("I AM LOGGING: %s"%kwargs)
# Put the logging info into memecache
# The key should be based on a incrementing the index for this instance
# k<instance_id><index>
# the index value can either be stored as a global for the instance
# or as a entry in memecache with key k<instance_id>
# After N writes to memecache we should sent out a Task like this
global index
index += 1
logging.info("index: %s"%index)
if (index % 100) == 0:
t = Task(params={'start':index,'length':100, 'instance_id':instance_id},method='GET')
t.add('bulk-log-processor')
def __init__(self, queue_name, url, **kwargs):
'''
kwargs are the same used on Task class
(https://developers.google.com/appengine/docs/python/taskqueue/tasks#Task)
'''
super(TaskQueueCommand, self).__init__()
self._task = Task(url=url, **kwargs)
self._queue_name = queue_name
def filter_the_abusers(caller):
# filter the troublemakers
if caller in config.ABUSERS:
counter = memcache.get(caller)
if counter is None:
memcache.set(caller,1)
elif int(counter) <= 3:
memcache.incr(caller,1)
else:
# create an event to log the quota problem
task = Task(url='/loggingtask', params={'phone':self.request.get('From'),
'inboundBody':self.request.get('Body'),
'sid':self.request.get('SmsSid'),
'outboundBody':'exceeded quota',})
task.add('eventlogger')
return True
return False
def sendInvite(request):
textBody = "You've been invited to use SMSMyBus to find real-time arrivals for your buses. Text your bus stop to this number to get started.(invited by "
textBody += request.get('From') + ')'
# parse the message to extract and format the phone number
# of the invitee. then create a task to send the message
smsBody = request.get('Body')
requestArgs = smsBody.split()
for r in requestArgs:
phone = r.replace('(','').replace('}','').replace('-','')
if phone.isdigit() == True:
task = Task(url='/admin/sendsmstask', params={'phone':phone,
'sid':request.get('SmsSid'),
'text':textBody,})
task.add('smssender')
return textBody
def get(self):
# query the StopLocationLoader for all stops
stops = StopLocationLoader.all()
for s in stops:
# create a new task for each stop
task = Task(url='/gtfs/port/stop/task/',
params={'stopID':s.stopID,
'name':s.name,
'description':s.description,
'lat':str(s.lat),
'lon':str(s.lon),
'direction':s.direction,
})
task.add('crawler')
logging.debug('Finished spawning StopLocationLoader tasks!')
self.response.out.write('done spawning porting tasks!')
def test_schedule_can_work_with_array_of_tasks(self):
# given
task1 = Task(
url='/example/task1',
)
task2 = Task(
url='/example/task2',
)
# when
Tasks.schedule("default", [task1, task2])
# then
executed_tasks = self.taskqueue_stub.get_filtered_tasks(
queue_names="default"
)
self.assertEqual(len(executed_tasks), 2, "Should create two tasks in queue")
self.assertEqual(executed_tasks[0].url, '/example/task1')
self.assertEqual(executed_tasks[1].url, '/example/task2')
def post(self):
# validate it is in fact coming from twilio
if config.ACCOUNT_SID != self.request.get('AccountSid'):
logging.error(
"Inbound request was NOT VALID. It might have been spoofed!")
self.response.out.write(errorResponse("Illegal caller"))
return
# who called? and what did they ask for?
phone = self.request.get("From")
msg = self.request.get("Body")
logging.info("New inbound request from %s with message, %s" %
(self.request.get('From'), msg))
# look out for the abusers
if filter_the_abusers(phone):
# don't reply!
return
# interrogate the message body to determine what to do
if msg.lower().find('invite') > -1:
# ... an invitation request
response = sendInvite(self.request)
else:
## magic ##
response = api_bridge.getarrivals(msg, 4)
# create an event to log the request
task = Task(url='/loggingtask',
params={
'phone': self.request.get('From'),
'inboundBody': self.request.get('Body'),
'sid': self.request.get('SmsSid'),
'outboundBody': response,
})
task.add('eventlogger')
# setup the response SMS
#smsBody = "Route %s, Stop %s" % (routeID, stopID) + "\n" + response
r = twilio.Response()
r.append(twilio.Sms(response))
self.response.out.write(r)
return
def post(self):
message = xmpp.Message(self.request.POST)
logging.info("XMPP request! Sent form %s with message %s" % (message.sender,message.body))
## magic ##
response = api_bridge.getarrivals(message.body,10)
# to make it a little easier to read, add newlines before each route report line
response = response.replace('Route','\nRoute')
# create an event to log the request
task = Task(url='/loggingtask', params={'phone':message.sender,
'inboundBody':message.body,
'sid':'xmpp',
'outboundBody':response,})
task.add('eventlogger')
# reply to the chat request
message.reply(response)
def get(self):
# query the StopLocationLoader for all stops
stops = StopLocationLoader.all()
for s in stops:
# create a new task for each stop
task = Task(url='/gtfs/port/stop/task/',
params={
'stopID': s.stopID,
'name': s.name,
'description': s.description,
'lat': str(s.lat),
'lon': str(s.lon),
'direction': s.direction,
})
task.add('crawler')
logging.debug('Finished spawning StopLocationLoader tasks!')
self.response.out.write('done spawning porting tasks!')
def post(self):
inbound_message = mail.InboundEmailMessage(self.request.body)
logging.info("Email request! Sent from %s with message subject %s" %
(inbound_message.sender, inbound_message.subject))
body = inbound_message.subject
logging.debug("email body arguments %s" % body)
## magic ##
response = api_bridge.getarrivals(body, 10)
# to make it a little easier to read, add newlines before each route report line
response = response.replace('Route', '\nRoute')
# send back the reply with the results
header = "Thanks for your request! Here are your results...\n\n"
footer = "\n\nThank you for using SMSMyBus!\nhttp://www.smsmybus.com"
# setup the response email
message = mail.EmailMessage()
message.sender = config.EMAIL_SENDER_ADDRESS
message.bcc = config.EMAIL_BCC_ADDRESS
message.to = inbound_message.sender
message.subject = 'Your Metro schedule estimates for stop %s' % getStopID(
body)
message.body = header + response + footer
logging.debug('sending results to %s' % message.to)
message.send()
# create an event to log the event
task = Task(url='/loggingtask',
params={
'phone': inbound_message.sender,
'inboundBody': body,
'sid': 'email',
'outboundBody': response,
})
task.add('eventlogger')
self.response.set_status(200)
return
def admin_action(request, key):
token = get_object_or_404(AccessToken, id=key)
if "enable" in request.POST:
token.enabled = True
token.save()
messages.add_message(request, messages.SUCCESS, "token enabled")
if "disable" in request.POST:
token.enabled = False
token.save()
messages.add_message(request, messages.SUCCESS, "token disabled")
if "force" in request.POST:
task = Task(url="/runner/run_task/", method="POST", params={"token": token.oauth_key, "force": "true"})
task.add("default")
messages.add_message(request, messages.SUCCESS, "run queued")
return redirect("/admin/webapp/accesstoken/%s/" % key)
def create(cls, **kwargs):
corr_id = request_correlation_id.get()
if corr_id:
if 'headers' in kwargs:
kwargs['headers'][request_correlation_id.HEADER_NAME] = corr_id
else:
kwargs['headers'] = {
request_correlation_id.HEADER_NAME: corr_id
}
return Task(**kwargs)
def admin_action(request, key):
token = get_object_or_404(AccessToken, id=key)
if "enable" in request.POST:
token.enabled = True
token.save()
messages.add_message(request, messages.SUCCESS, 'token enabled')
if "disable" in request.POST:
token.enabled = False
token.save()
messages.add_message(request, messages.SUCCESS, 'token disabled')
if "force" in request.POST:
task = Task(url='/runner/run_task/', method='POST', params={'token': token.oauth_key, "force":"true"})
task.add('default')
messages.add_message(request, messages.SUCCESS, 'run queued')
return redirect("/admin/webapp/accesstoken/%s/"%key)
