def store_all_division_standings(league, division, gender, age, agegroup):
q = opl_db.Division.all()
if league:
q.filter("league =", league)
if division:
q.filter("division = ", division)
if gender:
q.filter("gender =", gender)
if age:
q.filter("age =", age)
if agegroup:
q.filter("agegroup = ", agegroup)
for r in q.run():
parms = {
'l': r.league,
'u': r.url,
'a': r.age,
'ag': r.agegroup,
'g': r.gender,
'd': r.division
}
url = '/store-division-standings?' + urllib.urlencode(parms)
logging.debug('URL:' + url)
t = Task(payload=None, method='GET', url=url)
t.add()
return 'Done'
def get(self):
day_of_week = makeDayOfWeek()
query = Alert.query(Alert.days.IN([day_of_week]))
to_put = []
for q in query:
if q.confirmed:
alert_today = AlertLog(
alert_id=q.key.id(),
email=q.email,
phone=q.phone,
carrier=q.carrier,
start1=q.start1,
start2=q.start2,
start3=q.start3,
end1=q.end1,
end2=q.end2,
end3=q.end3,
time=q.time
)
to_put.append(alert_today)
else:
continue # used for debugging
ndb.put_multi(to_put)
logging.debug("Prepared today's list of alerts to send.")
time.sleep(5) # give it 5 seconds for the new data to take
first_task = Task(payload=None, url="/admin/sendalerts")
first_task.add(queue_name="alertsqueue")
logging.debug('ManageAlertsListQ successfully initiated.')
self.response.out.write('ManageAlertsListQ successfully initiated.')
def get(self,time_slot=""):
logging.debug('running cron for timeslot %s' % time_slot)
if systemIsOn() is False:
logging.error('bailing... the system is turned off')
return
# grab the row of data out of the spreadsheet
results = getResults(time_slot)
messages = getMessages(results)
# cycle over all the users and send them a message
users = db.GqlQuery("select * from User").fetch(200)
if len(users) <= 0:
logging.error('No users in the system!')
for u in users:
# send the SMS out with a background task
logging.debug('sending notifications to %s' % u.phone_number)
task = Task(url='/sendsmstask',
params={'phone':u.phone_number,
'msg_one':messages[0],
'msg_two':messages[1],
'msg_three':messages[2],
})
task.add('smssender')
def _EnqueueMasterRecallTask(self, owner_email, message_criteria,
task_key_id):
"""Add master recall task with error handling.
Args:
owner_email: String email address of user running this recall.
message_criteria: String criteria (message-id) to recall.
task_key_id: Int unique id of the parent task.
Raises:
re-raises any task queue errors.
"""
task_name = '%s_%s' % (view_utils.CreateSafeUserEmailForTaskName(
owner_email), view_utils.GetCurrentDateTimeForTaskName())
master_task = Task(name=task_name,
params={
'owner_email': owner_email,
'task_key_id': task_key_id,
'message_criteria': message_criteria
},
target='0.recall-backend',
url='/backend/recall_messages')
try:
master_task.add(queue_name='recall-messages-queue')
except TaskQueueError:
view_utils.FailRecallTask(
task_key_id=task_key_id,
reason_string='Failed to enqueue master task.')
raise
def MakeFollows(self):
"""
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
ADD FOLLOWS FOR ADMIN USERS
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
"""
nextURL = None
firstURL = self.request.get('nexturl')
query = PointRoot.query().order(PointRoot.url)
if firstURL:
query = query.filter(PointRoot.url >= firstURL)
pointRoots = query.fetch(11)
if len(pointRoots) == 11:
nextURL = pointRoots[-1].url
pointRootsToReview = pointRoots[:10]
else:
pointRootsToReview = pointRoots
i = 0
for pointRoot in pointRootsToReview:
pointRootKey = pointRoot.key
followers = {}
versions = pointRoot.getAllVersions()
for point in versions:
if point.version == 1:
followers[point.authorURL] = 'created'
elif not point.authorURL in followers:
followers[point.authorURL] = 'edited'
for comment in pointRoot.getComments():
if not comment.userUrl in followers:
followers[comment.userUrl] = 'commented'
logging.info('ROOT: %s FOLLOWERS: %s' %
(pointRoot.url, str(followers)))
for url in followers.iterkeys():
followType = followers[url]
previousNamespace = namespace_manager.get_namespace()
if previousNamespace and previousNamespace != '':
namespace_manager.set_namespace('') # DEFAULT NAMESPACE
usr = WhysaurusUser.getByUrl(url)
namespace_manager.set_namespace(previousNamespace)
else:
usr = WhysaurusUser.getByUrl(url)
logging.info('Trying to follow for U:%s, R:%s, T:%s' %
(url, pointRoot.url, followType))
f = None
f = Follow.createFollow(usr.key, pointRootKey, followType)
if f:
i = i + 1
logging.info('ADDED follow for U:%s, R:%s, T:%s' %
(url, pointRoot.url, followType))
logging.info('Added %d follows' % i)
if nextURL:
t = Task(url="/MakeFollows", params={'nexturl': nextURL})
t.add(queue_name="notifications")
logging.info('Requeing MakeFollows task to start at url %s ' %
nextURL)
def get(self) :
# shove a task in the queue because we need more time then
# we may be able to get in the browser
for route in range(1, 100):
task = Task(url="/gtfs/port/routes/task",params={'route':route})
task.add('crawler')
self.response.out.write('done. spawned a task to go do the route transformations')
def save(self, image):
image_url = "/picture/" + str(image.key()) + "/view"
thumbnail_blob_key = self.save_thumbnail(image)
thumbnail_url = "/picture/" + str(thumbnail_blob_key) + "/view"
self.station_proxy.update_background(image_url, thumbnail_url)
data = {
"entity": "background",
"event": "new",
"content": image_url,
}
task = Task(
url = "/taskqueue/multicast",
params = {
"station": config.VERSION + "-" + self.station_proxy.station.shortname,
"data": json.dumps(data)
}
)
task.add(queue_name="worker-queue")
self.response.out.write(json.dumps({
"response": True,
"src_full": image_url,
"src_thumb": thumbnail_url,
"blobstore_url": self.blobstore_url,
}))
def get(self):
# query the StopLocationLoader for all stops
q = db.GqlQuery("select * from StopLocationLoader")
stops = q.fetch(500)
offset = 500
while stops is not None and len(stops) > 0:
for s in stops:
# create a new task for each stop
task = Task(url='/port/stop/task/',
params={'stopID':s.stopID,
'name':s.name,
'description':s.description,
'lat':str(s.lat),
'lon':str(s.lon),
'direction':s.direction,
})
task.add('crawler')
# get the next bunch of stops from the query
stops = q.fetch(500,offset)
offset += 500
logging.debug('Finished spawning %s StopLocationLoader tasks!' % str(offset-500))
self.response.out.write('done spawning porting tasks!')
def store_all_schedules(league=None, division=None, gender=None, age=None):
q = opl_db.Division.all()
if league:
q.filter("league = ", league)
if division:
q.filter("division = ", division)
if gender:
q.filter("gender =",gender)
if age:
q.filter("age = ", age)
for r in q.run():
logging.debug('R:',r.league,' ',r.gender)
if len(r.sched_urls) > 0:
for u in r.sched_urls:
parms = { 'u' : u, 'l' : r.league, 'g' : r.gender, 'd' : r.division, 'a' : r.age }
t = Task(method='GET', url='/store-schedule?'+urllib.urlencode(parms));
t.add()
##fetch_schedule_results(u, r.league, r.division, r.gender, r.age)
else:
##fetch_schedule_results(r.url, r.league, r.division, r.gender, r.age)
parms = { 'u' : r.url, 'l' : r.league, 'g' : r.gender, 'd' : r.division, 'a' : r.age }
t = Task(method='GET', url='/store-schedule?'+urllib.urlencode(parms));
t.add()
return 'done'
def mail(self):
admin_proxy = AdminApi()
subject = "New phonoblaster station: %s" % (self.station.name)
if self.station.type == 'page':
body = """
A new station (%s/%s) has been created on Phonoblaster:
%s, %s
Global number of stations: %s
""" %(config.SITE_URL, self.station.shortname, self.station.name, self.station.link, admin_proxy.number_of_stations)
else:
body = """
A new station (%s/%s) has been created on Phonoblaster:
%s, %s
Global number of stations: %s
""" %(config.SITE_URL, self.station.shortname, self.station.name, "https://graph.facebook.com/"+self.station.key().name(), admin_proxy.number_of_stations)
logging.info(body)
task = Task(
url = "/taskqueue/mail",
params = {
"to": "*****@*****.**",
"subject": subject,
"body": body,
}
)
task.add(queue_name = "worker-queue")
def send_alerts(self, wait=0):
todays_alerts = AlertLog.query()
todays_alerts_len = todays_alerts.filter(AlertLog.complete == False).count()
if todays_alerts_len == 0:
logging.debug("Done for the day. See you tomorrow.")
else:
while todays_alerts_len > 0:
current_alerts = todays_alerts.filter(AlertLog.complete == False).order(AlertLog.time)
a = current_alerts.get()
now = makeNowTime()
if a.time <= now:
self.requestFreshData(a.time)
generate_msg_info(a)
logging.debug("I sent an alert that was scheduled for %s.", a.time.strftime("%I:%M %p"))
a.complete = True
a.sent = datetime.datetime.now()
a.put()
todays_alerts_len = todays_alerts_len - 1
time.sleep(1) # give it a second for the new data to take
else:
wait = makeWait(a.time)
logging.debug("Going to count down for %d seconds.", wait)
break
the_only_other_task = Task(payload=None, url="/admin/sendalerts", countdown=wait)
the_only_other_task.add(queue_name="alertsqueue")
def store_all_schedules(league=None, division=None, gender=None, age=None):
q = opl_db.Division.all()
if league:
q.filter("league = ", league)
if division:
q.filter("division = ", division)
if gender:
q.filter("gender =",gender)
if age:
q.filter("age = ", age)
for r in q.run():
logging.debug('R:',r.league,' ',r.gender)
if len(r.sched_urls) > 0:
for u in r.sched_urls:
parms = { 'u' : u, 'l' : r.league, 'g' : r.gender, 'd' : r.division, 'a' : r.age }
t = Task(method='GET', url='/store-schedule?'+urllib.urlencode(parms));
t.add()
##fetch_schedule_results(u, r.league, r.division, r.gender, r.age)
else:
##fetch_schedule_results(r.url, r.league, r.division, r.gender, r.age)
parms = { 'u' : r.url, 'l' : r.league, 'g' : r.gender, 'd' : r.division, 'a' : r.age }
t = Task(method='GET', url='/store-schedule?'+urllib.urlencode(parms));
t.add()
return 'done'
def post(self):
message = xmpp.Message(self.request.POST)
logging.info("XMPP request! Sent form %s with message %s" % (message.sender,message.body))
# normalize the XMPP requests
if message.sender.find('@'):
caller = message.sender.split('/')[0]
else:
caller = message.sender.get('from')
if message.body.lower().find('parking') > -1:
logging.info('parking request via XMPP')
response = api_bridge.getparking()
elif message.body.lower().find('help') > -1:
response = "Bus arrivals: stopID -or- routeID stopID Parking: 'parking' Stats: 'stats' Help: 'help'"
elif message.body.lower().find('stats') > -1:
response = meta.getStats(caller)
else:
## magic ##
response = api_bridge.getarrivals(message.body,10)
# to make it a little easier to read, add newlines before each route report line
response = response.replace('Route','\nRoute')
# create an event to log the request
task = Task(url='/loggingtask', params={'from':caller,
'to':message.to,
'inboundBody':message.body,
'sid':'xmpp',
'outboundBody':response,})
task.add('eventlogger')
# reply to the chat request
message.reply(response)
def post(self):
channel_id = str(self.request.get('from'))
logging.info("%s is ready to receive messages" %(channel_id))
# Init station proxy
m = re.match(r"(\w+).(\w+)", channel_id)
shortname = m.group(1)
station_proxy = StationApi(shortname)
extended_session = station_proxy.add_to_sessions(channel_id)
if(extended_session):
# Add a taskqueue to warn everyone
new_session_data = {
"entity": "session",
"event": "new",
"content": extended_session,
}
task = Task(
url = "/taskqueue/multicast",
params = {
"station": config.VERSION + "-" + shortname,
"data": json.dumps(new_session_data)
}
)
task.add(queue_name="sessions-queue")
def _EnqueueMasterRecallTask(self, owner_email, message_criteria,
task_key_id):
"""Add master recall task with error handling.
Args:
owner_email: String email address of user running this recall.
message_criteria: String criteria (message-id) to recall.
task_key_id: Int unique id of the parent task.
Raises:
re-raises any task queue errors.
"""
task_name = '%s_%s' % (
view_utils.CreateSafeUserEmailForTaskName(owner_email),
view_utils.GetCurrentDateTimeForTaskName())
master_task = Task(name=task_name,
params={'owner_email': owner_email,
'task_key_id': task_key_id,
'message_criteria': message_criteria},
target='0.recall-backend',
url='/backend/recall_messages')
try:
master_task.add(queue_name='recall-messages-queue')
except TaskQueueError:
view_utils.FailRecallTask(task_key_id=task_key_id,
reason_string='Failed to enqueue master task.')
raise
def get(self,routeID=""):
# create a new task with this link
#crawlURL = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"
crawlURL = URLBASE + 'r=' + routeID
task = Task(url='/routelist/crawlingtask', params={'crawl':crawlURL,'routeID':'00'})
task.add('crawler')
logging.info("Added new task for %s" % crawlURL)
return
def queueNightlyTask(self):
now = PST.convert(datetime.datetime.now())
tomorrow = now + datetime.timedelta(days = 1)
half_past_midnight = datetime.time(hour=7, minute=30) # PST
taskTime = datetime.datetime.combine(tomorrow, half_past_midnight)
t = Task(url='/job/DBIntegrityCheck', method="GET", eta=taskTime)
t.add(queue_name="dbchecker")
def queue_token_run(token):
cls = token.prosthetic.classname
logging.info("Queueing run task for token %s for %s on %s" %
(token.oauth_key, token.weavr_name, cls))
task = Task(url='/runner/run_task/',
method='POST',
params={'token': token.oauth_key})
task.add('default')
def send(self):
user = self.current_user
message = self.request.get('message')
t = Task(url='/broadcastChatroom',
params={'userName':user.name,
'message':message
})
t.add(queue_name="notifications")
def queueNightlyTask(self):
now = PST.convert(datetime.datetime.now())
tomorrow = now + datetime.timedelta(days=1)
half_past_midnight = datetime.time(hour=7, minute=30) # PST
taskTime = datetime.datetime.combine(tomorrow, half_past_midnight)
t = Task(url='/job/DBIntegrityCheck', method="GET", eta=taskTime)
t.add(queue_name="dbchecker")
def welcomeNewUser(phone):
# welcome the new user with an SMS message
welcome_message = "Welcome to SMSMyBus. Your account is now active! Just send in a stop ID to find your bus."
task = Task(url='/admin/sendsms',
params={'phone':phone,
'sid':'new user',
'text':welcome_message
})
task.add('smssender')
def task(name, method):
task = Task(
url = "/taskqueue/counter",
params = {
"shard_name": name,
"method": method
}
)
task.add(queue_name = "counters-queue")
def send(self):
user = self.current_user
message = self.request.get('message')
t = Task(url='/broadcastChatroom',
params={
'userName': user.name,
'message': message
})
t.add(queue_name="notifications")
def QueueTask(self):
taskurl = self.request.get('task')
if taskurl:
fullurl = '/' + taskurl
t = Task(url=fullurl)
t.add(queue_name="notifications")
self.response.out.write('OK I wrote %s to the notifications queue' % fullurl)
else:
self.response.out.write('Need a task URL parameter')
def MakeFollows(self):
"""
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
ADD FOLLOWS FOR ADMIN USERS
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
"""
nextURL = None
firstURL = self.request.get('nexturl')
query = PointRoot.query().order(PointRoot.url)
if firstURL:
query = query.filter(PointRoot.url >= firstURL)
pointRoots = query.fetch(11)
if len(pointRoots) == 11:
nextURL = pointRoots[-1].url
pointRootsToReview = pointRoots[:10]
else:
pointRootsToReview = pointRoots
i = 0
for pointRoot in pointRootsToReview:
pointRootKey = pointRoot.key
followers = {}
versions = pointRoot.getAllVersions()
for point in versions:
if point.version == 1:
followers[point.authorURL] = 'created'
elif not point.authorURL in followers:
followers[point.authorURL] = 'edited'
for comment in pointRoot.getComments():
if not comment.userUrl in followers:
followers[comment.userUrl] = 'commented'
logging.info('ROOT: %s FOLLOWERS: %s' % (pointRoot.url, str(followers)))
for url in followers.iterkeys():
followType = followers[url]
previousNamespace = namespace_manager.get_namespace()
if previousNamespace and previousNamespace != '':
namespace_manager.set_namespace('') # DEFAULT NAMESPACE
usr = WhysaurusUser.getByUrl(url)
namespace_manager.set_namespace(previousNamespace)
else:
usr = WhysaurusUser.getByUrl(url)
logging.info('Trying to follow for U:%s, R:%s, T:%s' % (url, pointRoot.url, followType))
f = None
f = Follow.createFollow(usr.key, pointRootKey, followType)
if f:
i = i + 1
logging.info('ADDED follow for U:%s, R:%s, T:%s' % (url, pointRoot.url, followType))
logging.info('Added %d follows' % i)
if nextURL:
t = Task(url="/MakeFollows", params={'nexturl':nextURL})
t.add(queue_name="notifications")
logging.info('Requeing MakeFollows task to start at url %s ' % nextURL)
def welcomeSolicitor(phone):
# welcome the new user with an SMS message
welcome_message = "Welcome to SMSMyBus! The first three requests are complimentary, but going forward you will need to signup. Please visit smsmybus.com to learn more."
task = Task(url='/admin/sendsms',
params={'phone':phone,
'sid':'complimentary results',
'text':welcome_message
})
task.add('smssender')
def get(self):
"""Queue run tasks for each registered Weavr"""
logging.info("Running cron job. Queueing run tasks.")
for prosthetic in ProstheticData.all(keys_only=True):
logging.info("Queueing run task for %s" % str(prosthetic))
task = Task(url='/runner/prosthetic_task/', method='GET',
params={'key': str(prosthetic)})
task.add('default')
logging.info("Finished running cron job.")
def get(self, model=""):
url = '/droptable/%s' % model
logging.debug('getting ready to start task to drop the %s table' %
model)
task = Task(url=url, params={})
task.add('crawler')
self.response.out.write(
'got it. started the background task to delete all %s entities' %
model)
def QueueTask(self):
taskurl = self.request.get('task')
if taskurl:
fullurl = '/' + taskurl
t = Task(url=fullurl)
t.add(queue_name="notifications")
self.response.out.write('OK I wrote %s to the notifications queue' % fullurl)
else:
self.response.out.write('Need a task URL parameter')
def createTask(phone, msg, sec):
logging.debug("Creating new task to fire in %s minutes" %
str(int(sec) / 60))
task = Task(url='/reminder',
params={
'phone': phone,
'msg': msg
},
countdown=sec)
task.add('reminders')
def queueEventRecord(cls, userKeyUrlsafe, entityKey1Urlsafe,
entityKey2Urlsafe, eventName):
taskParams = {'userKeyUrlsafe': userKeyUrlsafe, 'eventName': eventName}
if entityKey1Urlsafe:
taskParams['entityKey1Urlsafe'] = entityKey1Urlsafe
if entityKey2Urlsafe:
taskParams['entityKey2Urlsafe'] = entityKey2Urlsafe
t = Task(url='/recordEvent', params=taskParams)
t.add(queue_name="recordEvents")
def _on_event_formation(self):
#Queue it for life cycle management
if os.environ.get('ENV_TYPE') is None:
if self._event.start_time is not None and self._event.start_time != "":
task_execution_time = self._event.start_time - timedelta(minutes=5)
if self._event.expiration is not None and self._event.expiration != "":
task_execution_time = datetime.utcnow() + timedelta(minutes=5)
self._event.start_time = datetime.utcnow() + timedelta(minutes=10)
self._event.put()
goTask = Task(eta=task_execution_time, url='/activity_life_cycle/',method='GET',params={'activity': self._event.key.urlsafe()})
goTask.add('activityLifeCycle')
def _start_post_activity_completion_process(self, activity):
if os.environ.get('ENV_TYPE') is None:
if os.environ.get('SERVER_SOFTWARE', '').startswith('Development'):
eta = 420
else:
eta = 2100
task = Task(url='/post_activity_completion/',
method='GET',
params={'activity_key': activity.key.urlsafe()},
countdown=eta)
task.add('postActivityCompletion')
def _start_activity_closure_process(self, activity):
if os.environ.get('ENV_TYPE') is None:
if os.environ.get('SERVER_SOFTWARE', '').startswith('Development'):
eta = 1800
else:
eta = 259200
task = Task(url='/activity_closure/',
method='GET',
params={'activity_key': activity.key.urlsafe()},
countdown=eta)
task.add('activityClosure')
def post(self):
cursor = self.request.get("cursor")
q = Station.all()
q.order("updated")
# Is there a cursor?
if(cursor):
logging.info("Cursor found")
q.with_cursor(start_cursor = cursor)
else:
logging.info("No cursor")
stations = q.fetch(50)
to_put = []
done = False
for s in stations:
if s.active is None:
s.active = s.updated
to_put.append(s)
else:
done = True
break
db.put(to_put)
logging.info("Station entities updated")
if not done:
logging.info("Starting another task")
new_cursor = q.cursor()
task = Task(
url = "/taskqueue/upgrade",
params = {
'cursor': new_cursor,
},
)
task.add(queue_name = "upgrade-queue")
else:
logging.info("No more station to update")
subject = "Upgrade of station entities done"
body = "Everything is OK"
task = Task(
url = "/taskqueue/mail",
params = {
"to": "*****@*****.**",
"subject": subject,
"body": body,
}
)
task.add(queue_name = "worker-queue")
def get(self, routeID=""):
# create a new task with this link
#crawlURL = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"
crawlURL = URLBASE + 'r=' + routeID
task = Task(url='/crawl/routelist/crawlingtask',
params={
'crawl': crawlURL,
'routeID': '00'
})
task.add('crawler')
logging.info("Added new task for %s" % crawlURL)
return
def stats_init(self, request):
query = request.q
if query == "" or query == None:
return StatsInitResponse(task_id="")
task_id = str(md5.new(str(time.time())).hexdigest())
task = Task(url='/queue/stats', params={'task_id': task_id, 'q': query}, name=task_id)
task.add(queue_name='stats')
memcache.set('task|' + task_id, {'status': 'pending'}, 3600)
return StatsInitResponse(task_id=task_id)
def send(self):
self._data.update({
"site_url": config.SITE_URL,
})
task = Task(
url="/queue/event",
params={
"name": self._name,
"data": json.dumps(self._data)
}
)
task.add(queue_name="events-queue")
def queueEventRecord(cls, userKeyUrlsafe, entityKey1Urlsafe, entityKey2Urlsafe, eventName):
taskParams = {
'userKeyUrlsafe':userKeyUrlsafe,
'eventName': eventName
}
if entityKey1Urlsafe:
taskParams['entityKey1Urlsafe'] = entityKey1Urlsafe
if entityKey2Urlsafe:
taskParams['entityKey2Urlsafe'] = entityKey2Urlsafe
t = Task(url='/recordEvent', params=taskParams)
t.add(queue_name="recordEvents")
def post(self):
inbound_message = mail.InboundEmailMessage(self.request.body)
logging.info("Email request! Sent from %s with message subject %s" % (inbound_message.sender,inbound_message.subject))
body = inbound_message.subject
logging.debug("email body arguments %s" % body)
# ignore anything sent by ourselves to avoid an infinite loop
if inbound_message.sender == config.EMAIL_SENDER_ADDRESS:
self.response.set_status(200)
return
if body.lower().find('parking') > -1:
logging.info('parking request via email')
response = api_bridge.getparking()
else:
## magic ##
response = api_bridge.getarrivals(body,10)
# to make it a little easier to read, add newlines before each route report line
response = response.replace('Route','\nRoute')
# send back the reply with the results
header = "Thanks for your request! Here are your results...\n\n"
footer = "\n\nThank you for using SMSMyBus!\nhttps://www.smsmybus.com"
# setup the response email
message = mail.EmailMessage()
message.sender = config.EMAIL_SENDER_ADDRESS
#message.bcc = config.EMAIL_BCC_ADDRESS
message.to = inbound_message.sender
message.subject = 'Your Metro schedule estimates for stop %s' % getStopID(body)
message.body = header + response + footer
logging.debug('sending results to %s' % message.to)
message.send()
# create an event to log the event
task = Task(url='/loggingtask', params={'from':inbound_message.sender,
'to':inbound_message.to,
'inboundBody':body,
'sid':'email',
'outboundBody':response,})
task.add('eventlogger')
self.response.set_status(200)
return
def post(self):
# validate it is in fact coming from twilio
if config.ACCOUNT_SID != self.request.get('AccountSid'):
logging.error("Inbound request was NOT VALID. It might have been spoofed!")
self.response.out.write(errorResponse("Illegal caller"))
return
# who called? and what did they ask for?
phone = self.request.get("From")
msg = self.request.get("Body")
logging.info("New inbound request from %s with message, %s" % (self.request.get('From'),msg))
if paywall.isUserValid(phone) is False:
if paywall.isUserVirgin(phone) is True:
logging.info('Brand new caller - welcome them')
paywall.welcomeSolicitor(phone)
else:
# ignore caller
logging.info('We have seen this number before. Ignore this request')
return
# interrogate the message body to determine what to do
if msg.lower().find('parking') > -1:
response = api_bridge.getparking()
elif msg.lower().find('help') > -1:
response = "Bus arrival requests are either, stopID -or- routeID stopID Send 'parking' to find parking details"
elif msg.lower().find('stats') > -1:
response = meta.getStats(phone)
else:
## magic ##
response = api_bridge.getarrivals(msg,4)
if len(response) > 140:
response = response[0:140]
# create an event to log the request
task = Task(url='/loggingtask', params={'from':self.request.get('From'),
'to':self.request.get('To'),
'inboundBody':self.request.get('Body'),
'sid':self.request.get('SmsSid'),
'outboundBody':response,})
task.add('eventlogger')
# setup the response SMS
r = twilio.Response()
r.append(twilio.Sms(response))
self.response.out.write(r)
return
def create(cls,**kwargs):
'''
Creates an Interest based on the arguments passed in kwargs
Once an interest is created is added to matchmaker queue
:param cls:
:param kwargs:
:return:
'''
eventThrottler = EventThrottler(kwargs['username'])
# set the # of events to 3 instead of 5, and at join_flex_interest (for consistency). Throttler at 10 minutes
if eventThrottler.number_of_cached_events() > 3:
return False, "You have too many concurrent events.", None
event = Event(category = kwargs['category'],
date_entered = datetime.utcnow(),
type = Event.EVENT_TYPE_FLEX_INTEREST,
username = kwargs['username'],
building_name = kwargs['building_name'],
)
if 'expiration' in kwargs and kwargs['expiration'] != "":
event.expiration = kwargs['expiration']
if 'start_time' in kwargs and kwargs['start_time'] != "":
event.start_time = kwargs['start_time']
if 'duration' in kwargs and kwargs['duration'] != "":
event.duration = kwargs['duration']
if 'min_number_of_people_to_join' in kwargs and kwargs['min_number_of_people_to_join'] != 'None' and kwargs['min_number_of_people_to_join'] != "":
event.min_number_of_people_to_join = kwargs['min_number_of_people_to_join']
event.type = Event.EVENT_TYPE_SPECIFIC_INTEREST
if 'max_number_of_people_to_join' in kwargs and kwargs['max_number_of_people_to_join'] != 'None' and kwargs['max_number_of_people_to_join'] != "":
event.max_number_of_people_to_join = kwargs['max_number_of_people_to_join']
if 'note' in kwargs and kwargs['note'] != "":
event.note = kwargs['note']
if 'meeting_place' in kwargs and kwargs['meeting_place'] != "":
event.meeting_place = kwargs['meeting_place']
if 'activity_location' in kwargs and kwargs['activity_location'] != "":
event.activity_location = kwargs['activity_location']
event.put()
if os.environ.get('ENV_TYPE') is None:
if event.type == Event.EVENT_TYPE_FLEX_INTEREST:
task = Task(url='/match_maker/',method='GET',params={'interest': event.key.urlsafe()})
else:
task = Task(url='/match_maker/',method='GET',params={'activity': event.key.urlsafe()})
task.add('matchmaker')
logging.info('event created')
logging.info('match maker task queued')
eventThrottler.increment_activity_count()
return True, 'success', event
def filter_the_abusers(caller):
# filter the troublemakers
if caller in config.ABUSERS:
counter = memcache.get(caller)
if counter is None:
memcache.set(caller,1)
elif int(counter) <= 3:
memcache.incr(caller,1)
else:
# create an event to log the quota problem
task = Task(url='/loggingtask', params={'phone':self.request.get('From'),
'inboundBody':self.request.get('Body'),
'sid':self.request.get('SmsSid'),
'outboundBody':'exceeded quota',})
task.add('eventlogger')
return True
return False
def log(**kwargs):
# THIS IS WHERE ALL THE HEAVY LIFTING IS DONE
logging.info("I AM LOGGING: %s"%kwargs)
# Put the logging info into memecache
# The key should be based on a incrementing the index for this instance
# k<instance_id><index>
# the index value can either be stored as a global for the instance
# or as a entry in memecache with key k<instance_id>
# After N writes to memecache we should sent out a Task like this
global index
index += 1
logging.info("index: %s"%index)
if (index % 100) == 0:
t = Task(params={'start':index,'length':100, 'instance_id':instance_id},method='GET')
t.add('bulk-log-processor')
def sendInvite(request):
textBody = "You've been invited to use SMSMyBus to find real-time arrivals for your buses. Text your bus stop to this number to get started.(invited by "
textBody += request.get('From') + ')'
# parse the message to extract and format the phone number
# of the invitee. then create a task to send the message
smsBody = request.get('Body')
requestArgs = smsBody.split()
for r in requestArgs:
phone = r.replace('(','').replace('}','').replace('-','')
if phone.isdigit() == True:
task = Task(url='/admin/sendsmstask', params={'phone':phone,
'sid':request.get('SmsSid'),
'text':textBody,})
task.add('smssender')
return textBody
def get(self):
# query the StopLocationLoader for all stops
stops = StopLocationLoader.all()
for s in stops:
# create a new task for each stop
task = Task(url='/gtfs/port/stop/task/',
params={'stopID':s.stopID,
'name':s.name,
'description':s.description,
'lat':str(s.lat),
'lon':str(s.lon),
'direction':s.direction,
})
task.add('crawler')
logging.debug('Finished spawning StopLocationLoader tasks!')
self.response.out.write('done spawning porting tasks!')
def post(self):
# validate it is in fact coming from twilio
if config.ACCOUNT_SID != self.request.get('AccountSid'):
logging.error(
"Inbound request was NOT VALID. It might have been spoofed!")
self.response.out.write(errorResponse("Illegal caller"))
return
# who called? and what did they ask for?
phone = self.request.get("From")
msg = self.request.get("Body")
logging.info("New inbound request from %s with message, %s" %
(self.request.get('From'), msg))
# look out for the abusers
if filter_the_abusers(phone):
# don't reply!
return
# interrogate the message body to determine what to do
if msg.lower().find('invite') > -1:
# ... an invitation request
response = sendInvite(self.request)
else:
## magic ##
response = api_bridge.getarrivals(msg, 4)
# create an event to log the request
task = Task(url='/loggingtask',
params={
'phone': self.request.get('From'),
'inboundBody': self.request.get('Body'),
'sid': self.request.get('SmsSid'),
'outboundBody': response,
})
task.add('eventlogger')
# setup the response SMS
#smsBody = "Route %s, Stop %s" % (routeID, stopID) + "\n" + response
r = twilio.Response()
r.append(twilio.Sms(response))
self.response.out.write(r)
return
def get(self):
# query the StopLocationLoader for all stops
stops = StopLocationLoader.all()
for s in stops:
# create a new task for each stop
task = Task(url='/gtfs/port/stop/task/',
params={
'stopID': s.stopID,
'name': s.name,
'description': s.description,
'lat': str(s.lat),
'lon': str(s.lon),
'direction': s.direction,
})
task.add('crawler')
logging.debug('Finished spawning StopLocationLoader tasks!')
self.response.out.write('done spawning porting tasks!')
def admin_action(request, key):
token = get_object_or_404(AccessToken, id=key)
if "enable" in request.POST:
token.enabled = True
token.save()
messages.add_message(request, messages.SUCCESS, 'token enabled')
if "disable" in request.POST:
token.enabled = False
token.save()
messages.add_message(request, messages.SUCCESS, 'token disabled')
if "force" in request.POST:
task = Task(url='/runner/run_task/', method='POST', params={'token': token.oauth_key, "force":"true"})
task.add('default')
messages.add_message(request, messages.SUCCESS, 'run queued')
return redirect("/admin/webapp/accesstoken/%s/"%key)
def post(self):
inbound_message = mail.InboundEmailMessage(self.request.body)
logging.info("Email request! Sent from %s with message subject %s" %
(inbound_message.sender, inbound_message.subject))
body = inbound_message.subject
logging.debug("email body arguments %s" % body)
## magic ##
response = api_bridge.getarrivals(body, 10)
# to make it a little easier to read, add newlines before each route report line
response = response.replace('Route', '\nRoute')
# send back the reply with the results
header = "Thanks for your request! Here are your results...\n\n"
footer = "\n\nThank you for using SMSMyBus!\nhttp://www.smsmybus.com"
# setup the response email
message = mail.EmailMessage()
message.sender = config.EMAIL_SENDER_ADDRESS
message.bcc = config.EMAIL_BCC_ADDRESS
message.to = inbound_message.sender
message.subject = 'Your Metro schedule estimates for stop %s' % getStopID(
body)
message.body = header + response + footer
logging.debug('sending results to %s' % message.to)
message.send()
# create an event to log the event
task = Task(url='/loggingtask',
params={
'phone': inbound_message.sender,
'inboundBody': body,
'sid': 'email',
'outboundBody': response,
})
task.add('eventlogger')
self.response.set_status(200)
return
def filter_the_abusers(caller):
# filter the troublemakers
if caller in config.ABUSERS:
counter = memcache.get(caller)
if counter is None:
memcache.set(caller, 1)
elif int(counter) <= 3:
memcache.incr(caller, 1)
else:
# create an event to log the quota problem
task = Task(url='/loggingtask',
params={
'phone': self.request.get('From'),
'inboundBody': self.request.get('Body'),
'sid': self.request.get('SmsSid'),
'outboundBody': 'exceeded quota',
})
task.add('eventlogger')
return True
return False
def sendInvite(request):
textBody = "You've been invited to use SMSMyBus to find real-time arrivals for your buses. Text your bus stop to this number to get started.(invited by "
textBody += request.get('From') + ')'
# parse the message to extract and format the phone number
# of the invitee. then create a task to send the message
smsBody = request.get('Body')
requestArgs = smsBody.split()
for r in requestArgs:
phone = r.replace('(', '').replace('}', '').replace('-', '')
if phone.isdigit() == True:
task = Task(url='/sendsmstask',
params={
'phone': phone,
'sid': request.get('SmsSid'),
'text': textBody,
})
task.add('smssender')
return textBody
def pointRootsMap(self, f, taskURL, firstURL=None):
nextURL = None
query = PointRoot.query().order(PointRoot.url)
if firstURL:
query = query.filter(PointRoot.url >= firstURL)
pointRoots = query.fetch(11)
if len(pointRoots) == 11:
nextURL = pointRoots[-1].url
pointRootsToReview = pointRoots[:10]
else:
pointRootsToReview = pointRoots
for pointRoot in pointRootsToReview:
f(pointRoot)
if nextURL:
t = Task(url=taskURL, params={'nexturl': nextURL})
t.add(queue_name="notifications")
logging.info('Requeing task to start at url %s ' % nextURL)
def post(self):
message = xmpp.Message(self.request.POST)
logging.info("XMPP request! Sent form %s with message %s" %
(message.sender, message.body))
## magic ##
response = api_bridge.getarrivals(message.body, 10)
# to make it a little easier to read, add newlines before each route report line
response = response.replace('Route', '\nRoute')
# create an event to log the request
task = Task(url='/loggingtask',
params={
'phone': message.sender,
'inboundBody': message.body,
'sid': 'xmpp',
'outboundBody': response,
})
task.add('eventlogger')
# reply to the chat request
message.reply(response)
def post(self):
try:
scrapeURL = self.request.get('crawl')
direction = self.request.get('direction')
routeID = self.request.get('routeID')
logging.debug("task scraping for %s, direction %s, route %s" %
(scrapeURL, direction, routeID))
loop = 0
done = False
result = None
#start = quota.get_request_cpu_usage()
while not done and loop < 3:
try:
# fetch the page
result = urlfetch.fetch(scrapeURL)
done = True
except urlfetch.DownloadError:
logging.info("Error loading page (%s)... sleeping" % loop)
if result:
logging.debug("Error status: %s" % result.status_code)
logging.debug("Error header: %s" % result.headers)
logging.debug("Error content: %s" % result.content)
time.sleep(4)
loop = loop + 1
#end = quota.get_request_cpu_usage()
#logging.info("scraping took %s cycles" % (end-start))
# start to interrogate the results
soup = BeautifulSoup(result.content)
for slot in soup.html.body.findAll("a", "ada"):
logging.info("pulling out data from page... %s" % slot)
if slot.has_key('href'):
href = slot['href']
title = slot['title']
logging.info("FOUND A TITLE ----> %s" % title)
# route crawler looks for titles with an ID# string
if title.find("#") > 0:
# we finally got down to the page we're looking for
# pull the stopID from the page content...
stopID = title.split("#")[1].split("]")[0]
# pull the intersection from the page content...
intersection = title.split("[")[0].strip()
logging.info("found stop %s, %s" %
(stopID, intersection))
# check for conflicts...
stop = db.GqlQuery(
"SELECT * FROM StopLocation WHERE stopID = :1",
stopID).get()
if stop is None:
logging.error(
"Missing stop %s which should be impossible" %
stopID)
# pull the route and direction data from the URL
routeData = scrapeURL.split('?')[1]
logging.info(
"FOUND THE PAGE ---> arguments: %s stopID: %s" %
(routeData, stopID))
routeArgs = routeData.split('&')
routeID = routeArgs[0].split('=')[1]
directionID = routeArgs[1].split('=')[1]
timeEstimatesURL = CRAWL_URLBASE + href
# check for conflicts...
r = db.GqlQuery(
"SELECT * FROM RouteListing WHERE route = :1 AND direction = :2 AND stopID = :3",
routeID, directionID, stopID).get()
if r is None:
# add the new route to the DB
route = RouteListing()
route.route = routeID
route.direction = directionID
route.stopID = stopID
route.scheduleURL = timeEstimatesURL
route.stopLocation = stop
route.put()
logging.info(
"added new route listing entry to the database!"
)
else:
logging.error("we found a duplicate entry!?! %s",
r.scheduleURL)
#else: # title.split(",")[0].isdigit():
else:
if href.find("?r=") > -1:
# create a new task with this link
crawlURL = CRAWL_URLBASE + href
if routeID == '00':
routeID = href.split('r=')[1]
elif href.find("&") > -1:
routeID = href.split('&')[0].split('r=')[1]
task = Task(url='/crawl/routelist/crawlingtask',
params={
'crawl': crawlURL,
'direction': title,
'routeID': routeID
})
task.add('crawler')
logging.info(
"Added new task for %s, direction %s, route %s"
% (title.split(",")[0], title, routeID))
# label crawler looks for titles with letters for extraction/persistence
if title.replace('-', '').replace(' ', '').isalpha():
logging.info(
"found the route LABEL page! href: %s" % href)
routeData = href.split('?')[1]
routeArgs = routeData.split('&')
directionID = routeArgs[1].split('=')[1]
l = DestinationListing.get_or_insert(
title, id=directionID, label=title)
except apiproxy_errors.DeadlineExceededError:
logging.error("DeadlineExceededError exception!?")
return
return
def run_token(modeladmin, request, queryset):
for token in queryset:
task = Task(url='/runner/run_task/', method='POST', params={'token': token.oauth_key, "force":"true"})
task.add('default')
def text_message(self, message=None):
global NOBACKOUT
global GREETING
global AHGRAND
global GOOD_IDEA
global HUH
global RUDE
global NO_TEA_TODAY
global JUST_MISSED
global ADDPERSON
global WANT_TEA
global TRIGGER_HELLO
global TRIGGER_YES
global TRIGGER_TEA
global TRIGGER_RUDE
global TRIGGER_GOAWAY
global TRIGGER_ADDPERSON
global TRIGGER_TEAPREFS
global teacountdown
global drinkers
global settingprefs
global lastround
fromaddr = self.request.get('from').split("/")[0]
talker = Roster.get_or_insert(key_name=fromaddr, jid=fromaddr)
# If they showed up in the middle of a round, ask them if they want tea in
# the normal way after we've responded to this message
if (talker.askme == False and teacountdown):
send_random(fromaddr, WANT_TEA)
informed.add(fromaddr)
talker.askme = True
talker.put()
# Mrs Doyle takes no crap
if re.search(base64.b64decode(TRIGGER_RUDE), message.body,
re.IGNORECASE):
send_random(fromaddr, RUDE)
return
# And sometimes people take no crap.
if re.search(TRIGGER_GOAWAY, message.body, re.IGNORECASE):
talker.askme = False
talker.put()
send_random(fromaddr, NO_TEA_TODAY)
xmpp.send_presence(fromaddr,
status=":( Leaving " + getSalutation(fromaddr) +
" alone. So alone...",
presence_type=xmpp.PRESENCE_TYPE_AVAILABLE)
return
xmpp.send_presence(fromaddr,
status="",
presence_type=xmpp.PRESENCE_TYPE_AVAILABLE)
# See if we're expecting an answer as regards tea preferences
if fromaddr in settingprefs:
talker.teaprefs = message.body
talker.put()
settingprefs.remove(fromaddr)
xmpp.send_message(fromaddr, "Okay!")
return
if teacountdown:
if fromaddr in drinkers:
if re.search(TRIGGER_TEAPREFS, message.body, re.IGNORECASE):
xmpp.send_message(
fromaddr,
"So you like your tea '" + message.body + "'?")
talker.teaprefs = message.body
talker.put()
elif re.search(TRIGGER_YES, message.body, re.IGNORECASE):
xmpp.send_message(fromaddr, "Okay!")
else:
send_random(fromaddr, NOBACKOUT)
return
if re.search(TRIGGER_YES, message.body, re.IGNORECASE):
drinkers.add(fromaddr)
send_random(fromaddr, AHGRAND)
howTheyLikeItClause(message.body, talker)
else:
send_random(fromaddr, AH_GO_ON)
elif re.search(TRIGGER_ADDPERSON, message.body, re.IGNORECASE):
emailtoinvite = re.search("(" + TRIGGER_ADDPERSON + ")",
message.body, re.IGNORECASE).group(0)
xmpp.send_invite(emailtoinvite)
send_random(fromaddr, ADDPERSON)
elif re.search(TRIGGER_TEA, message.body, re.IGNORECASE):
send_random(fromaddr, GOOD_IDEA)
howTheyLikeItClause(message.body, talker)
drinkers.add(fromaddr)
informed.add(fromaddr)
for person in get_roster():
if person.askme and not person.jid == fromaddr:
xmpp.send_presence(jid=person.jid,
presence_type=xmpp.PRESENCE_TYPE_PROBE)
doittask = Task(countdown="120", url="/maketea")
doittask.add()
teacountdown = True
elif re.search(TRIGGER_HELLO, message.body, re.IGNORECASE):
send_random(fromaddr, GREETING)
elif re.search(TRIGGER_YES, message.body, re.IGNORECASE) and (
datetime.now() - lastround) < timedelta(seconds=120):
send_random(fromaddr, JUST_MISSED)
else:
send_random(fromaddr, HUH)
def create_and_queue(domain, ids, page=1):
params = {'ids': ";".join(ids), 'page': page}
comments_task = Task(url='/comments/%s' % (domain, ), params=params)
comments_task.add(queue_name="comments")