def send_emails(request): num_emails = 0 if 'pid' not in request.GET: players = Player.objects.all() urls = [] for player in players: url = '/tournament/send_emails/?pid=%s' % player.pk taskqueue.add(url=url, method='GET') urls.append(url) return HttpResponse("Queued:\n%s" % urls) really = True player = Player.objects.get(pk=request.GET['pid']) to_email = '*****@*****.**' cc = ['*****@*****.**'] if really: to_email = player.user.email cc.extend(('*****@*****.**', '*****@*****.**')) message_body = render_to_string('tournament/begin_email.txt', {'player': player}) message = mail.EmailMessage( sender="Table Tennis Tournament <*****@*****.**>", to=to_email, cc=cc, subject='Table Tennis Tournament Begins!', body=message_body) message.send() num_emails += 1 return HttpResponse("sent %d emails" % num_emails)
def POST(self, user_id): user = users.get_current_user() if user: d = web.input() f = contact_form(message=d.message) if f.validate() and util.user_exists(user_id.lower()): taskqueue.add( url="/task/send_mail", queue_name="email-throttle", params={"sender_id": user.user_id(), "recipient_id": user_id, "message": f.message.data}, ) raise web.seeother("/" + user_id + "#message_sent") elif f.validate() and user_id.lower() == "us": taskqueue.add( url="/task/send_mail", queue_name="email-throttle", params={"sender_id": user.user_id(), "recipient_id": "us", "message": f.message.data}, ) raise web.seeother("/" + user_id + "#message_sent") else: return t.render( util.data( title="Get in touch!", instructions="""You will always reveal your email address when you send a message!""", form=f, subject=" ".join([user.nickname(), "wants to get in touch!"]), ) ) else: return t.render(util.data(title="Not allowed!", instructions="You must be signed in to send messages!"))
def get(self, topic_num): if 'Referer' in self.request.headers: go = self.request.headers['Referer'] else: go = '/' member = CheckAuth(self) t = self.request.get('t').strip() if member: if member.username_lower_md5 == t: topic = GetKindByNum('Topic', int(topic_num)) if topic is not False: q = db.GqlQuery("SELECT * FROM TopicBookmark WHERE topic = :1 AND member = :2", topic, member) if q.count() == 0: bookmark = TopicBookmark(parent=member) bookmark.topic = topic bookmark.member = member bookmark.put() member = db.get(member.key()) member.favorited_topics = member.favorited_topics + 1 member.put() memcache.set('Member_' + str(member.num), member, 86400) n = 'r/t' + str(topic.num) + '/m' + str(member.num) memcache.set(n, True, 86400 * 14) taskqueue.add(url='/add/star/topic/' + str(topic.key())) self.redirect(go)
def enque(self, stock_code, target_price): url = '/stock/monitor/worker/' params = { 'stock_code': stock_code, 'target_price': target_price, } taskqueue.add(url=url, params=params)
def send_emails(request): num_emails = 0 if 'pid' not in request.GET: players = Player.objects.all() urls = [] for player in players: url = '/tournament/send_emails/?pid=%s' % player.pk taskqueue.add(url=url, method='GET') urls.append(url) return HttpResponse("Queued:\n%s" % urls) really = True player = Player.objects.get(pk=request.GET['pid']) to_email = '*****@*****.**' cc = ['*****@*****.**'] if really: to_email = player.user.email cc.extend(('*****@*****.**', '*****@*****.**')) message_body = render_to_string('tournament/begin_email.txt', {'player': player}) message = mail.EmailMessage(sender="Table Tennis Tournament <*****@*****.**>", to=to_email, cc=cc, subject='Table Tennis Tournament Begins!', body=message_body) message.send() num_emails += 1 return HttpResponse("sent %d emails" % num_emails)
def delete(self, guid): """Delete the task with the given id""" if helpers.authorized(self.request.params['UUID'], self.request.params['ATO'], self.request.params['action']): # search for the Project and delete if found key = db.Key.from_path('Task', int(guid)) task = db.get(key) wantsNotifications = {"true": True, "false": False}.get(self.request.params['notify'].lower()) currentUserId = self.request.params['UUID'] cukey = db.Key.from_path('User', int(currentUserId)) user = db.get(cukey) if not task == None: # cache current values before updates taskName = task.name taskType = task.type taskPriority = task.priority taskStatus = task.developmentStatus taskValidation = task.validation taskSubmitterId = task.submitterId taskAssigneeId = task.assigneeId taskEffort = task.effort taskProjectId = task.projectId taskDescription = task.description # Push notification email on the queue if we need to notify if notification.should_notify(currentUserId,task,"deleteTask",wantsNotifications): taskqueue.add(url='/mailer', params={'taskId': int(guid), 'currentUUID': self.request.params['UUID'], 'action': "deleteTask", 'name': taskName, 'type': taskType, 'priority': taskPriority, 'status': taskStatus, 'validation': taskValidation, 'submitterId': taskSubmitterId, 'assigneeId': taskAssigneeId, 'effort': taskEffort, 'projectId': taskProjectId, 'description': taskDescription}) task.delete() self.response.set_status(204, "Deleted") else: self.response.set_status(404, "Not Found") else: self.response.set_status(401, "Not Authorized")
def get(self): logging.info("Starting to load tasks %s" % datetime.datetime.now()) ret = '' gp_pub = RsaKey.gql("WHERE name = :1", 'gp_pub').get() gae_priv = RsaKey.gql("WHERE name = :1", 'gae_priv').get() gp_pubkey = self.makePubKey(gp_pub.keystring) gae_privkey = self.makePrivKey(gae_priv.keystring) users = Account.gql('WHERE active = :1', True) count = 0 for user in users: #logging.info(user.user) try: ret = '%s!gp!%s!gp!%s' % (user.user, user.gPass, user.twitter) gae_one = rsa.encrypt(str(ret), gp_pubkey) #logging.debug("WTF %s" % ret) send_key = gae_one.replace('\n', '!gp!') taskqueue.add(url='/worker/', params={'key': send_key}) #logging.error("KEY !%s!" % gae_one.replace('\n', '!gp!')) #logging.error("KEY !%d!" % gae_one.find('\n')) logging.debug("USER: %s TWITTER: %s KEY: %s" % (user.user, user.twitter, send_key)) count += 1 except: logging.error("something is f****d with USER: %s TWITTER: %s" % (user.user, user.twitter)) logging.info("Ended load tasks (%d users) %s" % (count, datetime.datetime.now()))
def post(self): start = self.request.get('start') kind = self.request.get('kind') if kind == 'second': to_kind = SecondUserKind from_kind = FirstUserKind migrate = second_from_first else: to_kind = FirstUserKind from_kind = SecondUserKind migrate = first_from_second query = from_kind.all() if start: query.filter('__key__ >', db.Key(start)) old = query.fetch(3) if not old: logging.info('All done!') return last_key = old[-1].key() new = [migrate(x) for x in old] db.put(new) db.delete(old) taskqueue.add( url='/worker/migration', params=dict( start=last_key, kind=kind))
def refresh_movie_source(request): if 'source_key' not in request.REQUEST: return HttpResponseServerError( 'No source key specified in request params') source = models.MovieListingSource.get(request.REQUEST['source_key']) if not source: logging.error('Unable to find MovieListingSource: %s', request.REQUEST['source_key']) return HttpResponse('Error unable to find Source') elif not source.yql: logging.error('No yql for MovieListingSource: %s' % str(source)) return HttpResponse('No YQL for source') elif not source.settings: logging.error('No settings for MovieListingSource: %s' % str(source)) return HttpResponse('No settings for source') logging.info('Refreshing movie from source %s', str(source)) yql = source.yql if 'offset' in request.REQUEST: query_offset = int(request.REQUEST['offset']) + 1 yql = '%s limit %d offset %d' % ( yql, settings.MOVIE_REFRESH_QUERY_SIZE, query_offset) form_data = urllib.urlencode({ "q": yql, "format": "xml", "diagnostics": "false" }) result = urlfetch.fetch(url=settings.YQL_BASE_URL, payload=form_data, method=urlfetch.POST) dom = minidom.parseString(result.content) result_nodes = dom.getElementsByTagName('results')[0].childNodes name_nodes = xpath.find(source.settings.name_xpath, dom) leaches_nodes = xpath.find(source.settings.leaches_xpath, dom) logging.info('Found %d raw names', len(name_nodes)) strip_white_pattern = re.compile(r"\s+") source_results = [] for index, name_node in enumerate(name_nodes): logging.debug('Node: ' + result_nodes[index].toxml()) raw_name = strip_white_pattern.sub(' ', getText(name_node)) leaches = strip_white_pattern.sub(' ', getText(leaches_nodes[index])) logging.info('Raw Name: %s, Leaches: %s', raw_name, leaches) source_results.append( models.MovieListEntry(raw_movie_name=raw_name, leaches=int(leaches), active=False)) db.put(source_results) #Refresh done using map/reduce. First we map to find the movie details for source_result in source_results: taskqueue.add(url=reverse('topmovies.task_handler.find_movie'), params={'source_entry_key': source_result.key()}) return HttpResponse("Loaded results for source: %s" % str(source))
def post(self): key = self.request.get('key') # Add the task to the default queue. taskqueue.add(url='/worker', params={'key': key}) self.redirect('/')
def schedule_next_newsletter(request): try: site=db.get(db.Key(request.POST['site'])) now=datetime.now() schedule=site.tz.localize(now+relativedelta(days=1,weekday=MO, hour=6, minute=0, second=0)).astimezone(utc).replace(tzinfo=None) if os.environ['SERVER_SOFTWARE'].startswith('Dev'):schedule=datetime.now() #next_newsletter=SitesNextNewsletter.get_or_insert('%s-%s' %(site.slug, schedule.strftime("%Y%W")), #publish_at=site.tz.localize(schedule)) chimp=site.chimp if chimp: url="http://%s/week-of/%s/newsletter" %( site.host,date(schedule,"Y-n-j") ) url=url.replace('localhost:8083', 'dctechevents.com') params={'url':url, 'apikey':chimp.apikey, 'listid':chimp.listid, 'from_name': site.name, 'from_email': '*****@*****.**', 'subject': "%s Weekly" % site.name } logging.info("Scheduling campaign with params %s" % str(params) ) taskqueue.add(url='/subscriptions/create_and_send/', params=params, name="weekly-%s-%s"%(site.slug, schedule.strftime("%Y%W")), eta=schedule) else: logging.error("no mailchimp setup for %s" % site.slug) except TaskAlreadyExistsError: pass except Exception,e: logging.error("%s in \n%s"% (traceback.format_exc(),str(request.POST)))
def get(self): data = self.__get_page_content() data = json.loads(data) count = 0 for stock in data['searchresults']: if stock['ticker'].find('399') == 0 or stock['ticker'].find( '000') == 0 and stock['exchange'] == 'SHA' or stock[ 'ticker'].find('900') == 0 and stock[ 'exchange'] == 'SHA' or stock['ticker'].find( '200') == 0 and stock['exchange'] == 'SHE': continue queue_name = 'queue' + str(count % 10) taskqueue.add(url='/tasks/updatesinglemarketcapital', queue_name='updatesinglemarketcapital', params={ 'ticker': stock['ticker'], 'title': stock['title'], 'exchange': stock['exchange'], 'local_currency_symbol': stock['local_currency_symbol'], 'value': stock['columns'][0]['value'] }, method='GET') count += 1
def post(self): all_chapters_query = Database.Chapter.all() (broken_chapters, checked_chapter_number, last_chapter_cursor) = get_broken_chapters() if last_chapter_cursor: all_chapters_query.with_cursor(last_chapter_cursor) else: checked_chapter_number = 0 broken_chapters = [] chapters = all_chapters_query.fetch(20) for chapter in chapters: checked_chapter_number = checked_chapter_number + 1 try: chapter.catalog_ref except: broken_chapters.append(chapter.key()) if chapters: last_chapter_cursor = all_chapters_query.cursor() else: last_chapter_cursor = None memcache.set('chapter_cursor', last_chapter_cursor) memcache.set('broken_chapters', broken_chapters) memcache.set('checked_chapter_number', checked_chapter_number) if last_chapter_cursor: taskqueue.add(url='/admin/task/check_broken_chapters', params={})
def task_enqueue_categories(request): """ Enqueues animal tables. To be run by cron """ q = Queue('animal-indexes') category = None #request.GET.get('category') if category: taskqueue.add( Task(url='/adopt/_tasks_/fetch_category', method='post', payload=json.dumps({ 'category': category, 'url': _getpetharbor_url(x) }))) else: for x in PETHARBOR_URLS.keys(): q.add( Task(url='/adopt/_tasks_/fetch_category', name="fetch-%s-%s" % (x, datetime.datetime.now().strftime('%Y%m%d%H%M%S')), method='post', payload=str( json.dumps({ 'category': x, 'url': _get_petharbor_url(x) })))) return TextResponse('OK')
def post(self): n = int(self.request.get('n')) l = 50 import csv csvReader = csv.reader(open('attractions4.csv')) count = 1 for row in csvReader: if count >= n and count < n + l: attractions = Attraction.all() attractions.filter("id =", row[0]) attraction = attractions.get() if attraction: attraction.datetime = datetime.datetime( year=int(row[8][0:4]), month=int(row[8][5:7]), day=int(row[8][8:10]), hour=int(row[8][11:13]), minute=int(row[8][14:16]), second=int(row[8][17:19])) attraction.put() elif count >= n: taskqueue.add(url='/fixdate', params={ 'n': n + l, }) break count = count + 1
def append_call_task(self, f): taskqueue.add(url='/worker', params={ 'task': 'call_function', 'func': f, '_pass': botconfig.BotConfig["InternalPass"] })
def get(self): if not users.is_current_user_admin(): self.redirect("/") for i in xrange(100): taskqueue.add(url='/cleaner', params={}, method="GET") self.response.out.write ("Queued 100 jobs to clean 1000 images")
def probe_one_website(request): if request.method == 'GET': url = request.GET.get('url') q = Website.all() q.filter("url =", url) website_obj = q.get() try: result = urlfetch.fetch(url, method='HEAD', deadline=6, headers={'Cache-Control': 'max-age=30'}) result_code = result.status_code except: result_code = 0 if website_obj.status == 'Up' and result_code != 200: taskqueue.add(url='/verify_down/', params={'url': url}) elif website_obj.status == 'Down' and result_code == 200: taskqueue.add(url='/verify_up/', params={'url': url}) else: website_obj.status_code = result_code website_obj.probe_time = datetime.now() website_obj.put() return HttpResponse("OK")
def txn(): key_name = cls.get_key_name(user_name) user = cls.get_by_key_name(key_name) if user: from kay.auth import DuplicateKeyError raise DuplicateKeyError(_(u"This user name is already taken." " Please choose another user name.")) salt = crypto.gen_salt() activation_key = crypto.sha1(salt+user_name).hexdigest() profile_key = db.Key.from_path(cls.kind(), key_name, RegistrationProfile.kind(), activation_key) expiration_date = datetime.datetime.now() + \ datetime.timedelta(seconds=settings.ACCOUNT_ACTIVATION_DURATION) taskqueue.add(url=url_for('_internal/expire_registration', registration_key=str(profile_key)), eta=expiration_date, transactional=True) taskqueue.add(url=url_for('_internal/send_registration_confirm', registration_key=str(profile_key)), transactional=True) user = cls(key_name=key_name, activated=False, user_name=user_name, password=crypto.gen_pwhash(password), email=email) profile = RegistrationProfile(user=user, parent=user, key_name=activation_key) db.put([profile, user]) return user
def fetch_snow_tweets(): logging.info("Fetching tweets for tag #uksnow") key = 'uksnow' feeditemcount = 1 latest_cached = 0 since_id = models.get_from_keystore(key) tweets = twitter.get_twitter_search('uksnow&filter=links', since_id) for tweet in tweets['results']: if feeditemcount == 1: try: latest_cached = str(tweet['id']) except: latest_cached = 0 taskqueue.add(url='/services/store_tweet', params={ 'userid': tweet['from_user_id'], 'username': tweet['from_user'], 'imgurl': tweet['profile_image_url'], 'message': tweet['text'], 'messageid': tweet['id'], 'message_created': tweet['created_at'], 'geo': tweet['geo'], }) feeditemcount += 1 if latest_cached > 0: models.add_to_keystore(key, latest_cached)
def get(self): n = self.request.get('n') or 1 f = self.request.get('f') or 1 taskqueue.add(url='/fixregion', params={'n': n, 'f': f}) self.response.out.write("running")
def get(self, key_name): p = OSMPOI.get_by_key_name(key_name) p.update_importance(p.importance + 1, max_z=config.max_z_osm, gt_key_name="osm") taskqueue.add(url='/gt/admin/update_tiles', method='GET') self.response.out.write('increased importance')
def post(self): delete = self.request.get("delete") if delete: g = Guild.get( delete ) if users.is_current_user_admin() or g.owner == users.get_current_user(): # the character fetcher deletes the characters once it notices that their # guild is gone - this makes the delete step here faster. g.delete() return if not users.get_current_user(): return self.redirect("/") continent = self.request.get("continent") realm = self.request.get("realm") guildname = self.request.get("guild") guild = Guild.find_or_create( continent, realm, guildname ) guild.owner = users.get_current_user() guild.put() if not guild: return self.error(404) taskqueue.add(url='/fetcher/guild/', params={'key': guild.key()}) self.redirect( guild.url() )
def get(self): if is_download_on(): ll = self.request.get('ll').strip() taskqueue.add(url='/gt/do_update', params={'ll': ll}) self.response.out.write("added task") else: self.response.out.write("OSM download state is off")
def get(self): db.delete(DeviceVersions.all().fetch(400)) total = (Device.all().count() / 10) + 1 for x in xrange(total): offset = x * 10 taskqueue.add(url='/tasks/AggregateVersionsWorker', params={'offset': offset})
def main(): """ This method adds a task to the task queue intended to querie the SoundCloud API, fetch the latest tracks having been uploaded since the last backend update, and adds them to the task queue for further processing. It is intended to be called by a cronjob on a short basis, like every 3 minutes. """ try: logging.info("Backend update started") time_from = backend_utils.calculate_time_from() time_to = datetime.datetime.now().isoformat() taskqueue.add(url='/backend-update-task', params={ 'time_from': time_from, 'time_to': time_to }) logging.info("Added backend update task to task queue. time_from: %s time_to: %s" % \ (time_from, time_to)) except DeadlineExceededError: logging.warning( "Backend Update has been canceled due to Deadline Exceeded") for name in os.environ.keys(): logging.info("%s = %s" % (name, os.environ[name]))
def test_get_registers_appropriate_tasks(self): moxer = Mox() request, response = new_mock_request_response(moxer) _stub_taskqueue(moxer) moxer.StubOutWithMock(Feed, "find_active", use_mock_anything=True) def create_call(i): source_name = "source-%i" % i source = MockEntity(key_name=source_name, name=source_name) return MockEntity(key_name="feed-%i" % i, artifact_source=source, url="hi") q_range = xrange(0, 5) Feed.find_active().AndReturn( MockQuery(q_range, create_call=create_call)) # expects queued tasks for each feed for i in q_range: taskqueue.add(name=IgnoreArg(), url=IgnoreArg()) moxer.ReplayAll() handler = CronIngestDriverHandler() handler.initialize(request, response) handler.get() moxer.VerifyAll()
def split_gdata(request): try: if request.method == 'POST': key=db.Key(request.POST.get('ical_key')) source=ICalendarSource.get(key) gdata_source=memcache.get(request.POST.get('cache_key')) memcache.delete(request.POST.get('cache_key')) feed=gdata.calendar.CalendarEventFeedFromString(gdata_source) cal_count=0 for gevent in feed.entry: cal_count=cal_count +1 source_cache_key=request.POST.get('cache_key') cache_key=source_cache_key +"-"+ str(cal_count) memcache.set(cache_key, gevent.ToString(),1200) params=params={'cache_key': cache_key, 'ical_key': request.POST['ical_key']} taskqueue.add(url='/events/parse_one_gdata/', params=params, name=cache_key,countdown=30) except urlfetch.DownloadError: raise except Exception,e: logging.error("%s in \n%s"% (traceback.format_exc(),str(request.POST)))
def migrate_profiles(request): try: set_namespace('') old_site = db.get(db.Key(request.POST.get('old_site'))) q = Query(Profile, namespace='').filter('site = ', old_site) old_profiles = q.fetch(1000) set_namespace(request.POST.get('new_namespace')) new_site = db.get(db.Key(request.POST.get('new_site'))) for old_profile in old_profiles: new_profile = clone_entity(old_profile, key_name=old_profile.key().name()) new_profile.site = new_site new_profile.put() #old_profile.delete() taskqueue.add( url='/admin/migrate-sources/', params={ 'new_namespace': request.POST.get('new_namespace'), 'old_site': old_site.key(), 'new_site': new_site.key(), }, ) except Exception, e: logging.error("%s in \n%s" % (traceback.format_exc(), str(request.POST)))
def post(self): """Starts crawling.""" project_name = self.GetRequiredParameter("project_name") start_index = self.GetOptionalParameter("start_index", None) if not start_index: last = crawl_state.GetLastCrawlResults(bugs_util.Provider.ISSUETRACKER, project_name) start_index = last.end_index else: start_index = int(start_index) max_results = 25 query = gdata.projecthosting.client.Query(start_index=start_index, max_results=max_results) phclient = gdata.projecthosting.client.ProjectHostingClient() try: issues = phclient.get_issues(project_name, query=query) except gdata.client.Error, e: retries = int(self.request.headers.get("X-AppEngine-TaskRetryCount", 0)) if retries < 4: logging.warning( "Retry crawling, retries is less than 5, " "current retries: %s, start_index: %d", retries, start_index, ) raise # Re-raise, so that the task is re-tried. else: # Skip current, try at start_index + 1. logging.warning("Skipping current index, start_index: %d", start_index) taskqueue.add( url="/tasks/crawl/issuetracker/recrawl_project", params={"project_name": project_name, "start_index": start_index + 1}, ) return
def post(self): """Starts crawling.""" project_name = self.GetRequiredParameter('project_name') start_index = self.GetOptionalParameter('start_index', None) if not start_index: last = crawl_state.GetLastCrawlResults( bugs_util.Provider.ISSUETRACKER, project_name) start_index = last.end_index else: start_index = int(start_index) max_results = 25 query = gdata.projecthosting.client.Query(start_index=start_index, max_results=max_results) phclient = gdata.projecthosting.client.ProjectHostingClient() try: issues = phclient.get_issues(project_name, query=query) except gdata.client.Error, e: retries = int( self.request.headers.get('X-AppEngine-TaskRetryCount', 0)) if retries < 4: logging.warning( 'Retry crawling, retries is less than 5, ' 'current retries: %s, start_index: %d', retries, start_index) raise # Re-raise, so that the task is re-tried. else: # Skip current, try at start_index + 1. logging.warning('Skipping current index, start_index: %d', start_index) taskqueue.add(url='/tasks/crawl/issuetracker/recrawl_project', params={ 'project_name': project_name, 'start_index': start_index + 1 }) return
def get(self): key = self.request.get('key') if key: query = db.GqlQuery( "SELECT * FROM Inventory WHERE __key__ > :1 ORDER BY __key__", db.Key(key)) else: query = db.GqlQuery("SELECT * FROM Inventory ORDER BY __key__") newKey = None inventories = query.fetch(PAGE_SIZE + 1) if len(inventories) == PAGE_SIZE + 1: newKey = str(inventories[-1].key()) for inventory in inventories: span = datetime.datetime.utcnow() - inventory.dateCreated if inventory.tag.status != 'new' and span.days > jt.gamesettings.autoDropTagDays: inventory.tag.pickedUp = False inventory.tag.lastUpdated = datetime.datetime.utcnow() inventory.tag.put() db.delete(inventory) if newKey: nextUrl = '/cron/tag_timeout?key=%s' % newKey else: nextUrl = None if nextUrl: taskqueue.add(url=nextUrl, method='GET')
def enqueue_notify(email, admin=False): clean_email = email.encode('base64').rstrip().rstrip('=') taskqueue.add(url="/filters/email/one/", params={"email": email, "admin": str(int(admin))}, name="Notify-%s-%s" % (clean_email, randstr()), method="GET")
def post(self): if experiment_grouping_already_started( self.experiment): # be idempotent return cycle = Cycle(range(self.experiment.cohort_count)) workers, peer_workers = [], {} for worker in Worker.all().filter('experiment = ', self.experiment): worker.cohort_index = cycle.next() workers.append(worker) if peer_workers.has_key(worker.cohort_index): peer_workers[worker.cohort_index].append(worker) else: peer_workers[worker.cohort_index] = [worker] for worker in workers: previous_cohort_index = index_decr(worker.cohort_index, self.experiment.cohort_count) previous_cohort = peer_workers[previous_cohort_index] worker.peer_worker = previous_cohort.pop( random.randrange(len(previous_cohort))) worker.put() if worker.cohort_index == 0: taskqueue.add(queue_name='worker-notification', params={'key': worker.key()})
def queueTask(camkey, delay=0): taskqueue.add(queue_name="poll-source-queue", url="/tasks/poll_sources", method="POST", countdown=delay, params=dict(camera=camkey, epoch=time.mktime(datetime.now().timetuple()) ), transactional=False)
def get(self): feed_id = self.request.get('feed_id') if feed_id: feed_key = db.Key(feed_id) feed = bloggerproxy.BlogProxy.get(feed_key) now = datetime.datetime.utcnow() if now - feed.last_crawled > datetime.timedelta(seconds=30): # Update the time first. Even if the crawl fails, we want to # throttle it. feed.last_crawled = now feed.put() bloggerproxy.crawlProxiedFeed(feed_key) else: logging.info('Feed crawled recently enough. Skipping.') else: proxied = bloggerproxy.BlogProxy.all().fetch(500) for bp in proxied: taskqueue.add(url='/recrawl.do', method='GET', params=dict(feed_id=bp.key())) self.response.set_status(200)
def get(self, sitecode): site = models.Site.get_by_key_name(sitecode) feeds = models.Feed.all().filter('site =', site) for feed in feeds: taskqueue.add(url='/index/feed', params={'page': "1", "url":feed.url}, method='GET') self.response.out.write(feed.url) self.response.out.write("<br />")
def get(self): """ Query the DB and queue any feeds that haven't been processed since update_interval """ update_interval = timedelta(hours=1) current_datetime = timeutils.now_utc() query = ContentFeed.all() query.filter("last_update <", current_datetime - update_interval) if query.count() == 0: logging.debug("No entries to queue") else: for feed in query: # get the member's OAuth token and secret last_update = timeutils.add_utc_tzinfo(feed.last_update) feed_consumer_params = { "feed_key": feed.key(), "owner_id": feed.owner.user_id() } try: taskqueue.add(url="/blogs/feed/consumer", params=feed_consumer_params) logging.debug("Queued feed: \"%s\" %s" % (feed.url, last_update.ctime())) except taskqueue.Error: logging.error("Unable to queue feed: \"%s\"", feed.url) return
def post(self): wantsNotifications = {"true": True, "false": False}.get(self.request.params['notify'].lower()) # collect the data from the record task_json = simplejson.loads(self.request.body) logging.info(self.request.body) # if the user is a guest the project must be unallocated currentUserId = self.request.params['UUID'] cukey = db.Key.from_path('User', int(currentUserId)) user = db.get(cukey) if str(user.role) != '_Guest' or (task_json.has_key('projectId') == False or task_json['projectId'] == None): # create a new task with the passed in json task = helpers.apply_json_to_model_instance(Task(),task_json) # save task task.put() guid = task.key().id_or_name() # Push notification email on the queue if the task has some sort of status, etc.. if notification.should_notify(currentUserId,task,"createTask", wantsNotifications): taskqueue.add(url='/mailer', params={'taskId': int(guid), 'currentUUID': self.request.params['UUID'], 'action': "createTask", 'name': "New Task"}) new_url = "/tasks-server/task/%s" % guid task_json["id"] = guid self.response.set_status(201, "Task created") self.response.headers['Location'] = new_url self.response.headers['Content-Type'] = 'text/json' self.response.out.write(simplejson.dumps(task_json)) else: self.response.set_status(401, "Not Authorized")
def txn(): key_name = cls.get_key_name(user_name) user = cls.get_by_key_name(key_name) if user: from kay.auth import DuplicateKeyError raise DuplicateKeyError( _(u"This user name is already taken." " Please choose another user name.")) salt = crypto.gen_salt() activation_key = crypto.sha1(salt + user_name).hexdigest() profile_key = db.Key.from_path(cls.kind(), key_name, RegistrationProfile.kind(), activation_key) expiration_date = datetime.datetime.now() + \ datetime.timedelta(seconds=settings.ACCOUNT_ACTIVATION_DURATION) taskqueue.add(url=url_for('_internal/expire_registration', registration_key=str(profile_key)), eta=expiration_date, transactional=True) taskqueue.add(url=url_for('_internal/send_registration_confirm', registration_key=str(profile_key)), transactional=True) user = cls(key_name=key_name, activated=False, user_name=user_name, password=crypto.gen_pwhash(password), email=email) profile = RegistrationProfile(user=user, parent=user, key_name=activation_key) db.put([profile, user]) return user
def post(self, entryid): user = users.get_current_user() if user: logout_url = users.create_logout_url("/") microUser = getMicroUser(user) entry = MicroEntry.get_by_id(int(entryid)) if not entry: self.redirect('/') content = self.request.get('content').strip() if content: content = content.replace('\n','').replace('\r',' ').replace('\t',' ') comment = Comment(author=microUser, content=content) addCommentEntry(entry, comment) # send the comment to the entry owner (but not myself) if entry.author.validated and not entry.author.silent and microUser.nick != entry.author.nick: msg = "comment on entry #%d:\n" % int(entryid) msg += content msg += "\nhttp://%s/entry/%d\n" % (settings.SITE_URL, int(entryid)) taskqueue.add(url="/send", params={"from":microUser.nick, "to":entry.author.nick, "message":msg, "secret":microUser.secret}) self.redirect("/entry/%d" % int(entryid)) else: login_url = users.create_login_url('/') self.redirect(login_url)
def get(self): rivi=VoittoRivi.gql("ORDER BY vuosi DESC,kierros DESC").get() if rivi: tuorein_kierros=rivi.kierros else: tuorein_kierros=0 taskqueue.add(url=self.request.path,params={"kierros":tuorein_kierros})
def get(self): self.redirect(ICECAST_URL) try: taskqueue.add(url='/_tasks/counters', params={'key': 'icecast-directory'}) except taskqueue.Error, e: logging.warning('Failed to add task: %s', e)
def post(self): # New user has joined in. Follow him and post a welcome message try: sms_client = OAuthClient('twitter', self) sms_client.token = OAuthAccessToken.all().filter( 'specifier =', 'smstweetin').filter( 'service =', 'twitter').fetch(1)[0] user = self.request.get('screen_name') count = int(self.request.get('count')) info = sms_client.post('/friendships/create', 'POST', (200,401,403), screen_name=user) # TODO : this may fail, try three times # Stop sending the follow status #status = "@%s has started using SMSTweet. Welcome %s to the group and tell about us to your friends" % (user, user) #info = sms_client.post('/statuses/update', 'POST', (200,401), status=status) # TODO : this may fail, try three times sms_client.token = OAuthAccessToken.all().filter( 'specifier =', user).filter( 'service =', 'twitter').fetch(1)[0] info = sms_client.post('/friendships/create', 'POST', (200,401,403), screen_name='smstweetin') # TODO : this may fail, try three times except (urlfetch.DownloadError, ValueError, Timeout), e: logging.warning("SmsTweetin:Friendship/create failed (%d) %s" % (count,e)) if count > 10: logging.error("SmsTweetin:Friendship/create Finally giving up") else: # Try again taskqueue.add(url = '/tasks/follow_new_user', params = { 'screen_name' : user, 'count' : count + 1 })
def update_or_create_document(yaml_obj): """ Submit an object read from our YAML files and it will update it in the database, creating it if it doesn't already exist. Returns the database object, and a boolean that is true if a new object was created. """ # Check if the table already exists in the datastore obj = Document.get_by_key_name(yaml_obj.get('slug')) # Update the obj if it exists if obj: # Loop through the keys and update the object one by one. for key in yaml_obj.keys(): # With some special casing for projects... if key == 'project_slug': proj = Project.get_by_key_name(yaml_obj.get('project_slug')) obj.project = proj # ...and for tags. elif key == 'tags': obj.tags = get_tag_keys(yaml_obj.get("tags")) else: setattr(obj, key, yaml_obj.get(key)) # Save it out obj.put() created = False # Create it if it doesn't else: # If it has tags.... if yaml_obj.has_key('tags'): # Convert to database keys tags = get_tag_keys(yaml_obj.pop("tags")) # Load the data obj = Document(key_name=yaml_obj.get('slug'), **yaml_obj) # Set the tags obj.tags = tags # Otherwise.... else: # Update the basic values obj = Document(key_name=yaml_obj.get('slug'), **yaml_obj) # And clear out the tag data obj.tags = [] obj.similar_documents = [] # Connected it to a project, if it exists if yaml_obj.has_key('project_slug'): proj = Project.get_by_key_name(yaml_obj.get('project_slug')) obj.project = proj # Save it out obj.put() created = True # Update the similarity lists of documents with the same tags taskqueue.add( url='/_/document/update-similar/', params=dict(key=obj.key()), method='GET' ) # Pass it out return obj, created
def post(self): status = self.request.get('status') phone = self.request.get('phone') count = int(self.request.get('count')) tuser = TwitterUser.get_by_phonenumber(phone) if tuser == None: logging.warning("Could not fetch tuser based on phone number %s",phone) return client = OAuthClient('twitter', self) try: info = client.post('/statuses/update', 'POST', (200,401,403), tuser, status=status) if 'error' in info: logging.warning("Submiting failed as credentials were incorrect (user:%s) %s", tuser.user, info['error']) tuser.lastError = "Twitter returned '%s' for your last update. You may be over limit or may have to register with SMSTweet again" % info['error'] tuser.put() else: logging.debug("updated the status for user %s", tuser.user) Tweet.save_tweet(info) except (urlfetch.DownloadError, ValueError), e: logging.warning("Update:update (%d) could not be fetched. %s " % (count,e)) if count > 10: logging.error("Tried updating the message 10 times. Finally giving up.") else: # Try again taskqueue.add(url = '/tasks/post_message', params = { 'phone' : phone, 'count' : count + 1, 'status' : status })
def get(self): url = self.request.get("url") item = models.Item.get_by_key_name(url) query = "select src, height, width from html where url='"+ url +"' and xpath='//div[@id=\"main\"]//img' and height > 200 and width > 200" result = helpers.do_yql(query) self.response.out.write(url) self.response.out.write("<br />") imgurl = False try: element = result['query']['results']['img'][0] if "www.dn.se" in element['src']: imgurl = element['src'] else: imgurl = "http://www.dn.se%s" % (element['src']) self.response.out.write("<img src=\"%s\"/>" % imgurl) if imgurl: item.img_url = imgurl else: item.img_url = False item.put() self.response.out.write("<br />") self.response.out.write(imgurl) except: self.response.out.write(result) self.response.out.write("<br />") if item: self.response.out.write("is an item") taskqueue.add(url='/scrape/imagecache', params={"url":item.item_url, "imgurl":imgurl}, method='GET') else: self.response.out.write("scrape item") self.response.out.write("<br />")
def post(self): cursor = int(self.request.get("cursor", default_value=0)) tokens = OAuthAccessToken.all().fetch(10, offset = cursor) for token in tokens: if xmpp.get_presence(token.email): client = OAuthClient('twitter', self, token.email) user_cache = memcache.get("user_cache_" + token.email) if user_cache is not None and user_cache.since_id is not None: #response = client.get('/statuses/home_timeline', count = 5, # since_id = last_sent_id) logging.info('since_id:' + str(user_cache.since_id)) response = self.fetchTweets(client, token.email, '/statuses/home_timeline', count = 5, since_id = user_cache.since_id) else: response = self.fetchTweets(client, token.email, '/statuses/home_timeline', count = 5) if len(response) > 0: xmpp.send_message(token.email, 'Some recent tweets:' + getFormattedMessage(response)) if len(tokens) == 10: taskqueue.add(url='/sendrecentworker', params={'cursor': cursor + 10})
def get(self): clsName = self.request.get('cls') cls = loadClass(clsName) total = (cls.all().count() / 100) + 1 for x in xrange(total): taskqueue.add(url='/tasks/FlushCounterWorker', params={'cls': clsName})
def get(self): size = 10 obj = MyPixivRtagData(tag=u'ケモノ',pagenum = int(0), paretntag=u'ケモノ') obj.save() #Delete all tags in MyRtagData. query = MyPixivRtagData.all().order('time').fetch(1500) db.delete(query) #Find Relative tags of kemono x 5 pages kemono='ケモノ' kemonotag = [] for i in range(1, 6): print i kemonosubtag = [] tgnm = reltagparser(i,kemono) for j in range(0, len(tgnm)): tg = tgnm[j] obj = MyPixivRtagData(tag=tg,pagenum = int(i), parenttag=kemono) obj.save() kemonosubtag.append(tg) kemonotag.append(kemonosubtag) #Request relative tags in RtagDataVertex datas = MyPixivRtagData.all().fetch(100) for i in range(0, len(datas), size): params = {} for j in range(0, size): if i+ j >= len(datas): break params["tag"+str(j)] = datas[i+j].tag params["pagenum"+str(j)] = datas[i+j].pagenum taskqueue.add(url='/pixivrtagtask', params = params)
def post(self, slug): body = self.get_argument("body") title = self.get_argument("title") blog = Entry.all().filter("slug =", slug).get() was_public = blog.was_public blog.body = body blog.title = title if self.get_argument("save", None): blog.is_public = False else: if not blog.was_public: blog.published = datetime.now() blog.is_public = True blog.was_public = True blog.put() cat = Categories.all().ancestor(blog).get() cat.is_public = blog.is_public cat.put() if was_public: taskqueue.add(url="/admin/ping_hub", method="GET") self.recalc_archive() self.recalc_tags() self.redirect("/%s" % blog.slug)
def sendNotifyIoNotifications(update): profiles = Profile.all().filter("notifyIoNotification =", True) for profile in profiles: taskqueue.add( url="/notifications/notifyio/post", params={"email": profile.user, "text": update.user_fullname() + ":" + update.body, "title": "New HD-Log"}, )
def fetch_pages(request): user_email = request.POST.get('user_email', None) user = users.User(email=user_email) if user is None: logging.error('User not found: %s', user_email) raise TypeError('User not found') page_urls_p = request.POST.get('page_urls', None) if not page_urls_p: logging.error('Page urls not given') raise TypeError('Page urls not given') page_urls = pickle.loads(str(page_urls_p)) logging.debug('fetching pages: %s;;%s', user_email, page_urls) pages = [] for url in page_urls: html = urllib.urlopen(url).read(10240) doc = redability.Document(html) pages.append(doc.summary().encode('ascii','ignore')) rd = ReadyData(owner=user, data_type='page') rd.content = '<hr>'.join(p for p in pages) rd.merged = len(pages) rd.put() logging.debug('ReadyData for fetched pages created: %s', page_urls) # schedule task for fetched data send params = {'ready_data_key': rd.key()} taskqueue.add(url=reverse('fetcher-send'), params=params) logging.debug('task created') return True
def getAndPostNextTask(self, objCommonTaskMessage, taskStatusHistoryKey): nextSeqNum = objCommonTaskMessage.currentSeqNum + 1 logging.info("nextSeqNum=" + str(nextSeqNum) + " processCode=" + objCommonTaskMessage.processCode + "\n\n") query = Tasks.gql("WHERE sequence > :1 and processCode = :2", nextSeqNum, objCommonTaskMessage.processCode) LIMIT = 1 #just get the one next record with the next higher sequence number taskList = query.fetch(LIMIT,offset=0) logging.info("After DB Query: len(taskList)=" + str(len(taskList)) + "\n\n") if len(taskList) < 1: #then we are done, cuz no more tasks for this process code self.UpdateDatabaseForEndProcess(objCommonTaskMessage, taskStatusHistoryKey) return #if taskList[0].processCode != objCommonTaskMessage.processCode: #then we are done because we found # BUT query was changed to look for same process code above so we don't need this "IF" #UpdateDatabaseForEndProcess(objCommonTaskMessage) logging.info("nextTaskCode=" + taskList[0].taskCode + " seq=" + str(taskList[0].sequence) ) #set the next task code and seqNum, and write back to task/queue objCommonTaskMessage.taskCode = taskList[0].taskCode objCommonTaskMessage.currentSeqNum = taskList[0].sequence objCommonTaskMessage.isManual = taskList[0].isManual strPickledObject = jsonpickle.encode(objCommonTaskMessage) logging.info("strPickledObject to taskQueue = " + strPickledObject) taskqueue.add(url='/commonTaskHandler', method='Post', payload = str(strPickledObject) )