def get(self):
id = self.request.get('id')
if id:
url = Url.get_by_id(int(id))
if url:
try:
prefix = self.request.get('prefix', '')
name = str(prefix) + str(id)
taskqueue.add(name=name,
queue_name='urlfetch',
url='/tasks/valid',
params={'id': id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s' % name)
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s' % name)
else:
# Routine check
url_keys = Url.query().order(Url.last_check,
Url.status).fetch(50, keys_only=True)
for key in url_keys:
id = key.id()
try:
prefix = self.request.get('prefix', '')
name = str(prefix) + str(id) + '_rc'
taskqueue.add(name=name,
queue_name='urlfetch',
url='/tasks/valid',
params={'id': id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s' % name)
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s' % name)
# Fix <missing> fields: document_date, last_check (valid)
url_keys = Url.query(Url.status == None).order(Url.idate).fetch(
50, keys_only=True)
for key in url_keys:
id = key.id()
try:
prefix = self.request.get('prefix', '')
name = str(prefix) + str(id) + '_fix'
taskqueue.add(name=name + '_dd',
queue_name='document',
url='/tasks/update_document',
params={'doc_id': id})
taskqueue.add(name=name + '_lc',
queue_name='urlfetch',
url='/tasks/valid',
params={'id': id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s' % name)
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s' % name)
redirect = self.request.get('redirect')
if redirect:
return self.redirect(redirect)
def get(self):
doc_id = self.request.get('doc_id', '')
if doc_id:
try:
prefix = self.request.get('prefix', '')
name = str(prefix) + str(doc_id)
taskqueue.add(name=name,
queue_name='document',
url='/tasks/update_document',
params={'doc_id': doc_id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s' % (name))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s' % (name))
else:
doc_ids = []
url_keys = Url.query().order(Url.document_date).fetch(
100, keys_only=True)
for key in url_keys:
doc_id = key.id()
try:
prefix = self.request.get('prefix', '')
name = str(prefix) + str(doc_id)
taskqueue.add(name=str(doc_id),
queue_name='document',
url='/tasks/update_document',
params={'doc_id': str(doc_id)})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s' % (str(name)))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s' % (str(name)))
def __unicode__(self):
if self.link:
if self.link_text:
return '%s: <a target="_blank" href="%s">%s</a>' % (
self.content, self.link, self.link_text)
else:
text = ''
comments = []
url = Url.query(Url.url == self.link).get(keys_only=True)
channel_url = ChannelUrl.query(ChannelUrl.url == url).get(
keys_only=True)
extra = Extra.query(Extra.channelurl == channel_url).get(
keys_only=False)
if extra and extra.comment:
text = extra.comment
if not text:
text = ''.join(self.link.split('/')[-1:])
return '%s: <a target="_blank" href="%s">%s</a>' % (
self.content, self.link, text)
return self.content
# Add http:// when needed
if not url.lower().startswith('http'):
url='http://'+url
#logging.info('Url/API/Post: Channel=%s User=%s Url=%s' % (channel,user,url))
# Fetch url (async):
# a) check statuscode LATER
# b) get title LATER
rpc = urlfetch.create_rpc()
urlfetch.make_fetch_call(rpc, url,allow_truncated=True)
# Get url from DB:
# a) already exists
# b) ChannelCheck
# 1. tarkista onko olemassa jo ko. Url, lisää jos ei, muuten päivitä (udate, valid?): valid-juttu joo ehkä jos tarpeen, ei muuten
urlquery=Url.query(Url.url==url)
urlinstance=urlquery.get()
if not urlinstance:
urlinstance=Url(url=url)
urlinstance.put()
#logging.debug('New url %s' % (url))
else:
logging.info('Old url %s' % (url))
# 2. tarkista onko olemassa jo ko. Channel, lisää jos ei
channelquery=Channel.query(Channel.name==channel)
channelinstance=channelquery.get()
if not channelinstance:
if channel.startswith('#'):
private=False
else: