def post(self, location='southbank'):
memcache.set(ICS_CACHEKEY_TMPL % (location, None), generate_calendar(location, None), time=1800)
listing_urls = BFIParser.generate_listing_urls()
countdown = 1
cachekey = LATCH_CACHEKEY_TMPL % (location, time.time())
for url in listing_urls:
logging.debug("Queueing listing url:%s" % url)
taskqueue.add(url='/tasks/process_listings_url', params={'url': url, 'cachekey': cachekey}, queue_name='background-queue', countdown=countdown)
countdown = countdown + 1
self.response.out.write(listing_urls)
def post(self, location='southbank'):
if continue_processing_task(self.request):
eventurl = self.request.get('url')
eventyear = int(self.request.get('eventyear'))
cachekey = self.request.get('cachekey')
logging.debug("Processing event url %s for year %s" % (eventurl, eventyear))
# Parse page
bfievent = BFIParser.parse_event_page(eventurl, eventyear)
def persist_showings(dbevent, bfievent):
# Delete existing showings for this event
db.delete(db.Query(Showing).ancestor(dbevent))
# Save events
for showing in bfievent.showings:
Showing(parent=dbevent,
ident=showing.id,
location=showing.location,
master_location=location,
start=showing.start,
end=showing.end).put()
event = Event.get_or_insert(key_name=bfievent.url,
src_url=db.Link(bfievent.url),
name=bfievent.title,
precis=bfievent.precis,
year=bfievent.year,
directors=bfievent.directors,
cast=bfievent.cast,
description=bfievent.description)
event.src_url=db.Link(bfievent.url)
event.name=bfievent.title
event.precis=bfievent.precis
event.year=bfievent.year
event.directors=bfievent.directors
event.cast=bfievent.cast
event.description=bfievent.description
event.put()
db.run_in_transaction(persist_showings, event, bfievent)
logging.debug("Processed event url %s" % eventurl)
remaining = memcache.decr(cachekey, initial_value=0)
logging.debug("Decremented remaining count to %s" % remaining)
if remaining <= 0:
db.delete(db.Query(Showing).filter("updated <", date.today()))
db.delete(db.Query(Event).filter("updated <", date.today()))
memcache.delete(HOME_CACHEKEY_TMPL % location)
taskqueue.add(url='/tasks/generate_calendar')
def post(self):
if continue_processing_task(self.request):
(year, urls) = BFIParser.parse_listings_page(self.request.get('url'))
countdown = 1
cachekey = self.request.get('cachekey')
for url in urls:
if memcache.get(url) is None:
memcache.set(url, 1, time=1800)
logging.debug("Queueing event url:%s" % url)
remaining = memcache.incr(cachekey, initial_value=0)
logging.debug("Incremented remaining count to %s" % remaining)
taskqueue.add(url='/tasks/process_event_url', params={'url': url, 'cachekey': cachekey, 'eventyear': year}, queue_name='background-queue', countdown=countdown)
countdown = countdown + 1
self.response.out.write(url)
else:
logging.debug("Already process(ed|ing) url %s" % url)