class TaskMaster(object):
"""Creates and enqueues tasks"""
def __init__(self):
self.queue = Queue()
def add_feed_update_task(self):
"""Enqueue task updating feeds"""
self.queue.add(Task(url='/task/update_feeds')) # TODO make url a parameter
def add_torrent_task(self, torrent_entry):
""""Enqueue task for torrent entry represented by dict"""
task = Task(url='/task/torrent', params=torrent_entry) # TODO make url a parameter
self.queue.add(task)
def add_new_torrents(self, scraper):
"""Enqueues tasks for all new torrents"""
try:
new_entries = scraper.get_new_torrents()
except webclient.NotLoggedIn: # Session expired
pass
except webclient.RequestError: # Tracker is down, happens sometimes
pass
else:
for e in new_entries:
self.add_torrent_task(e)
def _queue_tasks(self):
scanners = QuestionsScanner.all()
tasks = []
for scanner in scanners:
domain = scanner.key().name()
task = Task(url='/tasks/scan_new_questions/%s' % (domain, ))
tasks.append(task)
if len(tasks) > 0:
queue = Queue(name="scannewquestions")
queue.add(tasks)
def addTasks(self):
q = Comic.all()
queue = Queue(name='update-queue')
tasks = []
for comic in q:
if len(tasks) >= 100:
queue.add(tasks)
tasks = []
else:
task = Task(url='/tasks/update/' + str(comic.id))
tasks.append(task)
self.response.out.write("OK")
def addTasks(self):
q = Comic.all()
queue = Queue(name='update-queue')
tasks = []
for comic in q:
if len(tasks) >= 100:
queue.add(tasks)
tasks = []
else:
task = Task(url='/tasks/update/' + str(comic.id))
tasks.append(task)
self.response.out.write("OK")
def check_users_sweeper(request):
logger.info('check users sweeper runnning...')
expired_users = [user for user in NIH_User.objects.all()
if is_very_expired(user.NIH_assertion_expiration) and user.active]
if expired_users:
fallback_queue = Queue(name=FALLBACK_QUEUE_NAME)
batch_size = 25
for i in xrange(0, len(expired_users), batch_size):
users = expired_users[i:i+batch_size]
logger.info('scheduling check_user_login tasks for the following users: ')
logger.info(str([str(user.NIH_username) for user in users]))
tasks = [Task(url='/tasks/check_user_login', params={'user_id': user.user_id}, countdown=0) for user in users]
fallback_queue.add(tasks)
return HttpResponse('')
def add(self, params):
event = Event(
name = params["name"],
file = params["file"],
file150 = params["file150"],
filetype = params["filetype"].split('.')[1],
start_date = params["start_date"],
# end_date = params["end_date"],
description = params["description"],
creator = params["creator"],
# people_invited = params["people_invited"],
# type = params["type"],
)
event.put()
list = params["people_invited"]
for l in list:
iu = InvitedUser(
email = l,
event = event
)
queue = Queue('mail-queue')
subject = "You have been invited to the event " + event.name + " in Rockolin'"
body = """ Hi!, You have been invited to the event """ + event.name + """
This event would be on: """ + str(event.start_date) + """
If you want to decide the music justo go to the following
link: http://rockolinapp.appspot.com/event/""" +str(event.key())
queue.add(Task(url='/task/mail', params = { 'to' : l, 'subject' : subject, 'body' : body }))
iu.put()
queue.purge()
song_list = params["setlist"]
for s in params["setlist"]:
song = Song.get(s)
slv = SetListVotes(
event = event,
song = song,
votes = 0
)
slv.put()
return event
def schedule_tasks(tasks, queue_name=deferred._DEFAULT_QUEUE):
# type: (list[Task], str) -> list[Task]
queue = Queue(queue_name)
results = []
for chunk in chunks(tasks, MAX_TASKS_PER_ADD):
results.extend(queue.add(chunk))
return results
def send_voting_email(voting_user):
""" Create voting user and add voting email to mail-queue
Input:
voting_user: VotingUser, the user to be sent.
"""
# Make email content with token-link
election = voting_user.key.parent().get()
if not isinstance(election, Election):
msg = 'voting_user should have election as ndb ancestor'
raise ValueError(msg)
voting_link = "http://ntuvb-allstar.appspot.com/vote/"\
+ voting_user.token
from_email = "*****@*****.**"
email_body = (
u"<h3>您好 {student_id}:</h3>"
u"<p>感謝您參與{election_title} <br>"
u"<h4><a href='{voting_link}'> 投票請由此進入 </a></h4> <br>"
u"<p><b style=\"color: red\">此為您個人的投票連結，請勿轉寄或外流</b><br>"
u"若您未參與本次投票，請直接刪除本封信件 <br>"
u"任何疑問請來信至: {help_mail} <br></p>"
).format(
student_id=voting_user.student_id,
election_title=election.title,
voting_link=voting_link,
help_mail=from_email)
text_body = u"投票請進: %s" % voting_link
email_subject = election.title+u"投票認證信"
to_email = voting_user.student_id+"@ntu.edu.tw"
queue = Queue('mail-queue')
queue.add(Task(
url='/queue/mail',
params={
'subject': email_subject,
'body': email_body,
'text_body': text_body,
'to': to_email,
'from': from_email,
}
))
voting_user.last_time_mail_queued = datetime.now()
key = voting_user.put()
key.get() # for strong consistency
def post(self):
# refuse to start the tasks if there are some already running
queue = Queue()
stats = queue.fetch_statistics()
if stats.tasks == 0:
# queue up all of the groups into individual tasks. the configuration
# in queue.yaml only allows 10 tasks to run at once. the builder only
# allows 10 URL requests at a time, which should hopefully keep the
# Blizzard API queries under control.
q = Group.query()
groups = q.fetch()
for g in groups:
taskqueue.add(url='/builder', params={'group': g.name})
checker = Task(url='/builder', params={'group': 'ctrp-taskcheck'})
taskcheck = Queue(name='taskcheck')
taskcheck.add(checker)
self.redirect('/rank')
class TestTaskQueue(TestGae):
"""
Test testbed task queue
"""
def setUp(self):
TestGae.setUp(self, PROJECT_DIR)
from google.appengine.api.taskqueue import Queue, Task
self.queue = Queue('default')
self.queue.add(Task('xxx', url='/'))
def test_queue(self):
"""
check for number of tasks in queue
"""
self.assert_tasks_num(1)
tasks = self.gae_tasks(queue_name='default', flush_queue=False)
assert len(tasks) == 1
self.assert_tasks_num(1)
tasks = self.gae_tasks(queue_name='default', flush_queue=True)
assert len(tasks) == 1
self.assert_tasks_num(0)
def test_flask_execute(self):
"""
execute queue in fask app context
"""
from google.appengine.api.taskqueue import Task
from flask import Flask
app = Flask(__name__)
app.config['TESTING'] = True
@app.route('/', methods=['POST'])
def root_page():
"""
flask view
"""
return 'OK'
client = app.test_client()
data = self.gae_tasks_dict()
assert len(data) == 1
task = data[data.keys()[0]]
self.gae_task_flask_execute(task,
client,
is_delete=False,
is_debug_print=True)
data = self.gae_tasks_dict()
assert len(data) == 1
self.gae_task_flask_execute(task, client, is_debug_print=True)
data = self.gae_tasks_dict()
assert not data
self.queue.add(Task('xxx', url='/'))
self.gae_queue_flask_execute(client)
data = self.gae_tasks_dict()
assert not data
def test_dict(self):
"""
get queue content as dict
"""
data = self.gae_tasks_dict()
assert len(data) == 1
assert 'task1' in data
def test_dump(self):
"""
dump queue content
"""
self.gae_queue_dump()
self.gae_queue_dump(fields=['name', 'url'])
def post(self):
twitter_id = self.request.get('twitter_id', None)
if twitter_id is None:
return
user = User.get_by_key_name(twitter_id)
username = user.username
twitter = Twython(
twitter_token = CONSUMER_KEY,
twitter_secret = CONSUMER_SECRET,
oauth_token = user.oauth_token,
oauth_token_secret = user.oauth_token_secret,
)
statistic = UserStatistic.get_by_key_name(twitter_id)
stat = dict()
total = 0
page = 0
max_id = None
start_time = None
end_time = None
while True:
if max_id is None:
tweets = twitter.getFriendsTimeline(
count=200,
include_entities=1,
)
else:
tweets = twitter.getFriendsTimeline(
count=200,
include_entities=1,
max_id=max_id,
#page=page,
)
if len(tweets) == 0:
break
else:
page = page + 1
total = total + len(tweets)
if end_time is None:
end_time = datetime \
.strptime(tweets[0]['created_at'],
"%a %b %d %H:%M:%S +0000 %Y")
last_tweet = tweets[len(tweets) - 1]
start_time = datetime \
.strptime(last_tweet['created_at'],
"%a %b %d %H:%M:%S +0000 %Y")
for tweet in tweets:
user = tweet['user']['screen_name']
if not stat.has_key(user):
stat[user] = 0
stat[user] = stat[user] + 1
max_id = tweets[len(tweets) - 1]['id']
if total > MAX_TWEETS:
break
sorted_stat = sorted(stat, key=stat.get)
sorted_stat.reverse()
sorted_dict = []
for item in sorted_stat:
if stat[item] > 4:
sorted_dict.append(dict(
user = item,
count = stat[item],
))
statistic.start_time = start_time
statistic.end_time = end_time
statistic.total = total
statistic.statistics = simplejson.dumps(sorted_dict)
statistic.count += 1
statistic.put()
# notify user
try:
task = Task(
url = '/notify',
params = {'username': username}
)
queue = Queue(name='notify-user')
queue.add(task)
except:
# need to send to error page if error is happened
logging.exception('something bad happened')
def get(self):
session = get_current_session()
authorized_tokens = session.get('authorized_tokens', None)
if authorized_tokens is None:
self.redirect('/connect')
twitter = Twython(
twitter_token = CONSUMER_KEY,
twitter_secret = CONSUMER_SECRET,
oauth_token = authorized_tokens['oauth_token'],
oauth_token_secret = authorized_tokens['oauth_token_secret']
)
twitter_id = authorized_tokens['user_id']
username = authorized_tokens['screen_name']
statistic = UserStatistic.get_by_key_name(twitter_id)
if statistic is None:
statistic = UserStatistic(
key_name=twitter_id,
twitter_id=long(twitter_id),
)
statistic.put()
if statistic.statistics == None \
or (statistic.updated + timedelta(hours=1)) < datetime.now():
try:
task = Task(
url = '/fetch',
params = {'twitter_id': twitter_id}
)
queue = Queue(name='fetch-tweets')
queue.add(task)
statistic.updated = datetime.now()
statistic.put()
template_values = {
'username': username,
'updated': True,
'error': False,
}
except:
# need to send to error page if error is happened
logging.exception('something bad happened')
template_values = {
'username': username,
'updated': False,
'error': True,
}
else:
template_values = {
'username': username,
'updated': False,
'error': False,
}
path = os.path.join(TEMPLATE, 'message.html')
self.response.out.write(template.render(path, template_values))
def get(self):
num_pages_to_fetch=50
q = Queue('scrape')
for i in range(0,num_pages_to_fetch):
q.add(Task(url='/tasks/scrape?page=%d' % i, method='GET'))
self.response.out.write("done")
def get(self):
num_pages_to_fetch = 50
q = Queue('scrape')
for i in range(0, num_pages_to_fetch):
q.add(Task(url='/tasks/scrape?page=%d' % i, method='GET'))
self.response.out.write("done")
