def post(self, *ar, **kw): time = self.request.get('time') today = datetime.date.today() if not time or datetime.date.fromtimestamp(float(time)) < today: logging.info( "Ignoring deprecated task UpdateRanksHandler:post(time = " + time + ")") return last_key = self.request.get('last_key') if not last_key: query = db.GqlQuery( 'SELECT __key__ FROM KeywordSearchEngine ORDER BY __key__') else: query = db.GqlQuery( 'SELECT __key__ FROM KeywordSearchEngine WHERE __key__ > :last_key ORDER BY __key__', last_key=db.Key(last_key)) entities = query.fetch(100) if entities: default_queue = taskqueue.Queue("default") se_calls_queue = taskqueue.Queue("search-engine-calls") for key in entities: task = taskqueue.Task(url='/tasks/update_keyword_se_rank', params={'key': key}) se_calls_queue.add(task) last_key = key task = taskqueue.Task(url='/tasks/update_ranks', params={ 'time': time, 'last_key': last_key }) default_queue.add(task)
def get(self): totalPart = memcache.get('sitemap.xml_total_part') if (totalPart is None): pQueue = taskqueue.Queue(name = 'CreateSitemap') taskurl = 'http://' + self.request.host_url pTask = taskqueue.Task(url='/sitemap.xml/Create', params=dict(url=taskurl)) pQueue.add(pTask) logging.info('Task queue started!') xml = '' else: partNo = 1 xml = self.xmlHeader() while(partNo <= totalPart): key = 'sitemap.xml_part' + str (partNo) partNo += 1 partBody = memcache.get(key) if (partBody is None): partBody = '' xml += partBody xml += "</urlset>\n" self.response.headers['Content-Type'] = 'application/xml' self.response.out.write(xml)
def post(self, API_VERSION, ACCOUNT_SID, *args): format = response.response_format(self.request.path.split('/')[-1]) if parameters.required(['From','To','Body'],self.request): Message = messages.Message.new( To = self.request.get('To'), From = self.request.get('From'), Body = self.request.get('Body'), AccountSid = ACCOUNT_SID, Direction = 'outbound-api', Status = 'queued' ) if self.request.get('StatusCallback',None) is not None: Message.StatusCallback = self.request.get('StatusCallback') response_data = Message.get_dict() self.response.out.write(response.format_response(response.add_nodes(self,response_data,format),format)) Message.put() #DO SOME THINGS DEPENDING ON ACCOUNT SETTINGS #DEFAULT WILL BE TO SEND MESSAGE, CHARGE FOR IT AND UPDATE WHEN SENT Message.send() #make sure put happens before callback happens if Message.StatusCallback is not None: taskqueue.Queue('StatusCallbacks').add(taskqueue.Task(url='/Callbacks/SMS', params = {'SmsSid':Message.Sid})) else: #This should either specify a twilio code either 21603 or 21604 self.response.out.write(response.format_response(errors.rest_error_response(400,"Missing Parameters",format),format))
def saveBookmark(self): bm = ''.join(["saveBookmarks_" , str(time.time())]) memcache.add(bm, self.tempList, 120) self.tempList = [] queue = taskqueue.Queue("bookmark") queue.add(taskqueue.Task(url = "/bookmark/q/put", params = {'key': bm})) pass
def post(self, *ar, **kw): key = self.request.get('keyword_se_key') batch_size = 10 query = models.KeywordRankLog.all(keys_only=True).filter( 'keyword_se = ', db.Key(key)) count = query.count() db.delete(query.fetch(batch_size)) if count > batch_size: queue = taskqueue.Queue("default") task = taskqueue.Task(url='/tasks/delete/keyword_rank_logs', params={'keyword_se_key': key}) queue.add(task)
def post(self): memcache.flush_all() logging.info('Memcache has all flushed! ') db.delete(DBCache().all()) if (DBCache().all().count() > 1): pQueue = taskqueue.Queue(name='DeleteDBCache') taskurl = 'http://' + self.request.host_url pTask = taskqueue.Task(url='/cacheflush', params=dict(url=taskurl)) pQueue.add(pTask) else: logging.info('DBcache has all flushed! ')
def buildXml(self, offSet): count = self.buildElement(offSet) partNo = memcache.get('sitemap.xml_total_part') if (partNo is None): partNo = 0 partNo += 1 memcache.set('sitemap.xml_total_part', partNo, self.sitemapCacheTime) logging.info('Start caching part ' + str(partNo)) finishedCount = offSet + count key = 'sitemap.xml_part' + str (partNo) memcache.set(key, self.XMLBody, self.sitemapCacheTime) memcache.set('tmp_sitemap_finished_count', finishedCount, 60*5) if (count == self.PreLoad): pQueue = taskqueue.Queue(name = 'CreateSitemap') taskurl = 'http://' + self.request.host_url pTask = taskqueue.Task(url='/sitemap.xml/Create', params=dict(url=taskurl)) pQueue.add(pTask) logging.info('Task queue added!') else: pQueue = taskqueue.Queue(name = 'CreateSitemap') pQueue.purge() memcache.delete('tmp_sitemap_maxload') memcache.delete('tmp_sitemap_finished_count') logging.info('Purged all temp values.') return count
def post(self, *ar, **kw): if not config.data_retention_days: return batch_size = 10 retention_delta = datetime.timedelta(days=config.data_retention_days) date = datetime.datetime.today() - retention_delta query = models.KeywordRankLog.all(keys_only=True).filter( 'date < ', date) count = query.count() db.delete(query.fetch(batch_size)) if count > batch_size: queue = taskqueue.Queue("default") task = taskqueue.Task(url='/tasks/delete/old_logs') queue.add(task)
def post(self, *ar, **kw): key = self.request.get('account_key') batch_size = 10 query = models.Site.all(keys_only=True).filter('account = ', db.Key(key)) count = query.count() site_keys = query.fetch(batch_size) for site_key in site_keys: models.Site.cascade_delete(site_key) db.delete(site_keys) if count > batch_size: queue = taskqueue.Queue("default") task = taskqueue.Task(url='/tasks/delete/sites', params={'account_key': key}) queue.add(task)
def post(self, *ar, **kw): key = self.request.get('keyword_key') batch_size = 10 query = models.KeywordSearchEngine.all(keys_only=True).filter( 'keyword = ', db.Key(key)) count = query.count() kw_se_keys = query.fetch(batch_size) for kw_se_key in kw_se_keys: models.KeywordSearchEngine.cascade_delete(kw_se_key) db.delete(kw_se_keys) if count > batch_size: queue = taskqueue.Queue("default") task = taskqueue.Task(url='/task/delete/keyword_search_engines', params={'keyword_key': key}) queue.add(task)
def disconnect(self,StatusCallback = None,StatusCallbackMethod = 'POST'): self.Status = 'complete' self.EndTime = datetime.datetime.now() self.Duration = (self.EndTime - self.StartTime).seconds if self.Direction == 'outgoing-api' or self.Direction == 'outbound-dial': #should be dependent on country code, but will need more work self.Price = self.Duration * (0.02) elif self.Direction == 'inbound': self.Price = self.Duration * (0.01) self.put() if StatusCallback is not None: try: taskqueue.Queue('StatusCallbacks').add(taskqueue.Task(url='/Callbacks/Call', params = {'CallSid':self.Sid,'StatusCallback':StatusCallback,'StatusCallbackMethod':StatusCallbackMethod})) except Exception, e: pass
def do_for_candidacy(candidacy, seat): global global_email_delta assert seat.frozen_local_issues if not candidacy.candidate.validated_email(): log("Not queueing, invalid email " + str(candidacy.candidate.email) + " for candidacy " + seat.name + ", " + candidacy.candidate.name) else: global_email_delta += 2 # two seconds between sending each mail, try to keep within GAE limits if options.real: log(str(global_email_delta) + " queued invite for candidacy " + seat.name + ", " + candidacy.candidate.name + " email: " + candidacy.candidate.email) eta = datetime.datetime.utcnow() + datetime.timedelta(seconds=global_email_delta) # AppEngine servers use UTC taskqueue.Queue('survey-email').add(taskqueue.Task(url='/task/invite_candidacy_survey/' + str(candidacy.key().name()), eta = eta)) candidacy.log("Queued task to send survey invite email") else: log(str(global_email_delta) + " would queue invite for candidacy " + seat.name + ", " + candidacy.candidate.name+ " email: " + candidacy.candidate.email)
def post(self, *ar, **kw): if not config.max_inactive_account_days: return batch_size = 10 inactivity_delta = datetime.timedelta( days=config.max_inactive_account_days) date = datetime.datetime.today() - inactivity_delta query = models.Account.all(keys_only=True).filter( 'last_login < ', date) count = query.count() account_keys = query.fetch(batch_size) for account_key in account_keys: models.Account.cascade_delete(account_key) db.delete(account_keys) if count > batch_size: queue = taskqueue.Queue("default") task = taskqueue.Task(url='/task/delete/inactive_accounts') queue.add(task)
def add_task(queue_name='default',payload=None,**kwargs): # A payload can also be sent if kwargs.get('params', None): for json_type in ['kwargs', 'entities']: # if kwargs['params'].get(json_type, None): from django.utils import simplejson kwargs['params'][json_type] = simplejson.dumps( kwargs['params'][json_type]) from google.appengine.api.labs import taskqueue queue = taskqueue.Queue(name=queue_name) try: task = taskqueue.Task(payload=payload, **kwargs) logging.info('adding task: %s' % kwargs) queue.add(task) # TODO: Why don't these exceptions work? except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError): logging.warning('unable to create task with name %s' % kwargs.get('name','(no name provided)'), exc_info=True)
def post(self, page): try: queue = taskqueue.Queue("import") wpfile = page.param('wpfile') #global imt imt = import_wordpress(wpfile) imt.parse() #OptionSet.setValue('wpimport_data',imt) memcache.set("imt", imt) queue.add(taskqueue.Task(url="/admin/wp_import")) return self.render_content("wpimport.html", {'postback': True}) except Exception, e: return self.error( "Import Error:<p style='color:red;font-size:11px;font-weight:normal'>%s</p>" % e.message)
def task_average_response_by_party(request, party_key_name, refined_issue_key_name): party = Party.get_by_key_name(party_key_name) refined_issue = RefinedIssue.get_by_key_name(refined_issue_key_name) arbp = db.Query(AverageResponseByParty).filter('party =', party).filter('refined_issue =', refined_issue).get() if not arbp: arbp = AverageResponseByParty(party = party, refined_issue = refined_issue, average_agreement = None, processing_running_total = 0, processing_running_count = 0, processing_next_key = None) chunk = db.Query(Candidacy).filter('deleted = ', False).filter('survey_filled_in =', True) # carry on calculation where we left off if arbp.processing_last_candidacy == None: assert arbp.processing_running_total == 0 assert arbp.processing_running_count == 0 else: chunk = chunk.filter('__key__ >', arbp.processing_last_candidacy.key()) # do 100 candidacies at a time, as too slow otherwise chunk.fetch(10) # XXX candidacy = None for candidacy in chunk: survey_response = db.Query(SurveyResponse).filter('candidacy =', candidacy).filter('refined_issue =', refined_issue).get() if survey_response: arbp.processing_running_total += survey_response.agreement arbp.processing_running_count += 1 arbp.processing_last_candidacy = candidacy # if we've finished, work out average if candidacy == None: arbp.average_agreement = float(arbp.processing_running_total) / float(arbp.processing_running_count) arbp.put() # calculate next chunk if candidacy == None: return HttpResponse("Calculation complete for " + party.name + " question: " + refined_issue.question) else: taskqueue.Queue('average-calc').add(taskqueue.Task(url='/task/average_response_by_party/' + party_key_name + "/" + refined_issue_key_name)) return HttpResponse("Done " + str(arbp.processing_running_count) + ", queued next chunk for " + party.name + " question: " + question)
def buildTask(self, offSet): count = self.buildElement(offSet) finishedCount = offSet + count CacheTempData().save('tmp_sitemap_finished_count', finishedCount) pQueue = taskqueue.Queue(name='CreateSitemap') if (count == self.PreLoad): taskurl = 'http://' + self.request.host_url pTask = taskqueue.Task(url='/sitemap.xml/Create', params=dict(url=taskurl)) pQueue.add(pTask) logging.info('Task queue added!') else: CacheXMLSiteMap().finish() pQueue.purge() logging.info('Purged all temp values.')
def get(self): totalPart = CacheXMLSiteMap().getCount() if (totalPart is None): pQueue = taskqueue.Queue(name='CreateSitemap') taskurl = 'http://' + self.request.host_url pTask = taskqueue.Task(url='/sitemap.xml/Create', params=dict(url=taskurl)) pQueue.add(pTask) logging.info('Task queue started!') xml = '' else: xml = self.xmlHeader() xml += CacheXMLSiteMap().load() print x xml += "</urlset>\n" self.response.headers['Content-Type'] = 'application/xml' self.response.out.write(xml)
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ import random from wsgiref.handlers import CGIHandler from google.appengine.api.labs import taskqueue from google.appengine.api import memcache from google.appengine.api.capabilities import CapabilitySet memcache_service = CapabilitySet('memcache', methods=['set','get']) hot_handler_queue = taskqueue.Queue(name='hothandler') HOT_HANDLER_PREFIX = '/_ah/queue/hothandler/' def wsgi_app(env, res): """ visit '/_ah/queue/hothandler/start' as admin to start a task """ token = env['PATH_INFO'].replace(HOT_HANDLER_PREFIX,'') cur_token = memcache.get(HOT_HANDLER_PREFIX) if cur_token is None: if not memcache_service.is_enabled(): cur_token = token if token in [cur_token, 'start']: next_token = str(random.random()) url = '%s%s'%(HOT_HANDLER_PREFIX, next_token) next_task = taskqueue.Task(countdown=10, url=url) hot_handler_queue.add(next_task) memcache.set(HOT_HANDLER_PREFIX, next_token) res('200 OK',[('Content-Type','text/plain')])
def fetch_old(self): logging.info("fetch old buzz for %s" % self.username) fetch_queue = taskqueue.Queue("fetch-old") task = taskqueue.Task(url='/fetchold',params=dict(topic=self.topic)) #self.buzztopic fetch_queue.add(task)
def startTask(url, queue_name='default', context=None, **kwargs): """Adds a new task to one of the queues. """ queue = taskqueue.Queue(name=queue_name) return queue.add(taskqueue.Task(url=url, params=context))
def get(self, *ar, **kw): default_queue = taskqueue.Queue("default") task = taskqueue.Task(url='/tasks/update_ranks', params={'time': time.time()}) default_queue.add(task)
def post(self): if not self.is_login: self.redirect(users.create_login_url(self.request.uri)) try: #global imt imt=memcache.get("imt") #imt=OptionSet.getValue('wpimport_data') import_data=imt.pop() #if tdata=='men': memcache.set('imt',imt) #else: # OptionSet.setValue('wpimport_data',imt) try: cmtimport=memcache.get("cmtimport") except: cmtimport=False if import_data: try: if import_data[0]=='cat': _cat=import_data[1] nicename=_cat['slug'] cat=Category.get_by_key_name(nicename) if not cat: cat=Category(key_name=nicename) cat.name=_cat['name'] cat.slug=nicename cat.put() elif import_data[0]=='entry': _entry=import_data[1] logging.debug('importing:'+_entry['title']) hashkey=str(hash(_entry['title'])) entry=Entry.get_by_key_name(hashkey) if not entry: entry=Entry(key_name=hashkey) entry.title=_entry['title'] entry.author=self.login_user entry.is_wp=True #entry.date=datetime.strptime( _entry['pubDate'],"%a, %d %b %Y %H:%M:%S +0000") try: entry.date=datetime.strptime( _entry['pubDate'][:-6],"%a, %d %b %Y %H:%M:%S") except: try: entry.date=datetime.strptime( _entry['pubDate'][0:19],"%Y-%m-%d %H:%M:%S") except: entry.date=datetime.now() entry.entrytype=_entry['post_type'] entry.content=_entry['content'] entry.excerpt=_entry['excerpt'] entry.post_id=_entry['post_id'] entry.slug=urldecode(_entry['post_name']) entry.entry_parent=_entry['post_parent'] entry.menu_order=_entry['menu_order'] for cat in _entry['categories']: c=Category.get_by_key_name(cat['slug']) if c: entry.categorie_keys.append(c.key()) entry.settags(','.join(_entry['tags'])) ## for tag in _entry['tags']: ## entry.tags.append(tag) if _entry['published']: entry.save(True) else: entry.save() if cmtimport: for com in _entry['comments']: try: date=datetime.strptime(com['date'][0:19],"%Y-%m-%d %H:%M:%S") except: date=datetime.now() comment=Comment(author=com['author'], content=com['content'], entry=entry, date=date ) try: comment.email=com['email'] comment.weburl=com['weburl'] except: pass try: if len(com['ip'])>4: comment.ip=com['ip'] except: pass comment.store() finally: queue=taskqueue.Queue("import") queue.add(taskqueue.Task( url="/admin/wp_import")) except Exception,e : logging.info("import error: %s"%e.message)
def get(self, *ar, **kw): if not config.max_inactive_account_days: return default_queue = taskqueue.Queue("default") task = taskqueue.Task(url='/tasks/delete/inactive_accounts') default_queue.add(task)
def get(self): outString="" postKey=self.request.get("key") postPdf=self.request.get("pdf") postClean=self.request.get("clean") postFront=self.request.get("front") outTitle='' if postKey!=userKey: outString='It works' self.response.out.write(outString) return else: postUrl=self.request.get("url") if sendMail==True: ddqueue = taskqueue.Queue('default') ddqueue.add(taskqueue.Task(url='/mainTask', params={'key': postKey,'url': postUrl,'pdf': postPdf,'clean': postClean}, retry_options=taskqueue.TaskRetryOptions(task_retry_limit=3))) if postFront=='0': outString='It works' self.response.out.write(outString) return try: result= urlfetch.fetch(postUrl) self.response.out.write(outString) except: result=0 if result!=0: if result.status_code== 200: ###this line solve the <!-- tag problem of some webpage(like sina blog) tmps=result.content.replace('\xe2\x80\x93','--') tdc=chardet.detect(tmps) if tdc.get('confidence')>=0.6: htmlCode=tmps.decode(tdc.get('encoding'),'ignore') else: try: htmlCode=tmps.decode('utf-8') except: try: htmlCode=tmps.decode('gbk') except: try: htmlCode=tmps.decode('gb2312') except: htmlCode=tmps.decode(chardet.detect(tmps).get('encoding'),'ignore') else: htmlCode="" if htmlCode!="": readData = Readability(htmlCode, postUrl) outString=readData.content outTitle=readData.title else: outString='' outTitle='' if postClean=='0': outString=htmlCode else: outString='' outTitle='' if postPdf and postPdf!='0': self.response.headers['Content-Type'] = 'application/pdf' if postClean!='0': outString=u'''<style type="text/css"> @font-face { font-family: "'''+fontName+'''"; src: url("fonts/'''+fontFile+'''") } html { font-family: '''+fontName+'''; } </style>'''+u'<html><head><title>'+outTitle+u'</title></head><body>'+u'<h1>'+outTitle+u'</h1>'+outString+u'</body></html>' else: outString=outString=u'''<style type="text/css"> @font-face { font-family: "'''+fontName+'''"; src: url("fonts/'''+fontFile+'''") } html { font-family: '''+fontName+'''; } </style>'''+outString rawData=StringIO(outString.encode('utf-8')) output=StringIO() pisa.log.setLevel('WARNING') #suppress debug log output pdf = pisa.CreatePDF( rawData, output, encoding='utf-8', ) pdfData=pdf.dest.getvalue() self.response.out.write(pdfData) else: self.response.headers['Content-Type'] = 'text/html' if postClean!='0': outString=u'<html><head><title>'+outTitle+u'</title></head><body>'+u'<h1>'+outTitle+u'</h1>'+outString+u'</body></html>' self.response.out.write(outString)
from google.appengine.ext.webapp import template from google.appengine.ext.webapp.util import run_wsgi_app import datetime import feedparser import fetchpage import hashlib import logging import model import random import sys import time import urllib import urlparse import zrqutil default_feed_queue = taskqueue.Queue("feed") add_feed_queue = taskqueue.Queue("add-feed") HUB_URL = 'http://pubsubhubbub.appspot.com/' PIPE_OUTPUT_FEED_XML_MEMCACHE_KEY = 'PIPE_OUTPUT_FEED_XML' INPUT_FEED_XML_MEMCACHE_KEY = 'INPUT_FEED_XML' PIPE_MEMCACHE_KEY = 'PIPE' class PublishHandler(BaseRequestHandler): def get(self, slug): if slug and slug[0] == '/': slug = slug[1:] is_remove = self.request.get('is_remove') if slug: if is_remove == 1: feed_xml_memcache_key = PIPE_OUTPUT_FEED_XML_MEMCACHE_KEY + '_' + slug
from google.appengine.api.labs import taskqueue from google.appengine.ext import webapp, db from google.appengine.ext.webapp import template from google.appengine.ext.webapp.util import run_wsgi_app from model import FeedEntry import datetime import feedparser import hashlib import logging import model import random import time import urllib import zzzutil feed_queue = taskqueue.Queue("feed") class PublishHandler(BaseRequestHandler): def get(self, slug): if slug and slug[0] == '/': slug = slug[1:] if slug: publish_url = self.request.host_url + "/pipes/" + slug post_params = { 'hub.mode': 'publish', 'hub.url': publish_url, } payload = urllib.urlencode(post_params) hub_url = 'http://pubsubhubbub.appspot.com/' try:
def get(self, *ar, **kw): if not config.data_retention_days: return default_queue = taskqueue.Queue("default") task = taskqueue.Task(url='/tasks/delete/old_logs') default_queue.add(task)
def dispatchBookEvent(book, reader, url, queue_name): task = taskqueue.Task(url=url, params={'book' : book.identifier(), 'reader' : reader.identifier()}) queue = taskqueue.Queue(queue_name) queue.add(task)
def add(url, params, queue='default'): q = taskqueue.Queue(queue) t = taskqueue.Task(url=url, method=POST, params=params) q.add(t)