def view(request, urlid):
data = []
logging.debug('View ChannelUrl %s' % (urlid))
try:
urlid = int(urlid)
except:
pass
channelurl = ChannelUrl.get_by_id(urlid)
if channelurl:
channel = channelurl.channel.get()
url = channelurl.url.get()
if channel.private == False:
extras = Extra.query(Extra.channelurl == channelurl.key)
rates = Rate.query(Rate.channelurl == channelurl.key)
rating = channelurl.rating()
#data.append({'channel':channel,'post':post,'url':url,'extras': extras})
data = {
'channel': channel,
'post': post,
'url': url,
'extras': extras,
'rates': rates,
'rating': rating
}
template_values = {
'data': data,
}
return render_to_response('url.html', template_values)
def view_master(request, urlid):
data = []
logging.debug('View (Master)Url %s' % (urlid))
try:
urlid = int(urlid)
except:
pass
url = Url.get_by_id(urlid)
if url:
channelurls = ChannelUrl.query(ChannelUrl.url == url.key)
for channelurl in channelurls:
channel = channelurl.channel.get()
if channel.private == False:
extras = Extra.query(Extra.channelurl == channelurl.key)
rates = Rate.query(Rate.channelurl == channelurl.key)
rating = channelurl.rating()
#data.append({'channel':channel,'post':post,'url':url,'extras': extras})
data.append({
'channel': channel,
'channelurl': channelurl,
'post': post,
'url': url,
'extras': extras,
'rates': rates,
'rating': rating
})
template_values = {
'data': data,
'user': users.get_current_user(),
}
return render_to_response('masterurl.html', template_values)
def post(self):
type = self.request.get('type', '')
if type == 'stats':
pass
elif type == 'cleanup':
last_year = datetime.datetime.now() - datetime.timedelta(days=365)
last_quarter = datetime.datetime.now() - datetime.timedelta(
days=92)
last_month = datetime.datetime.now() - datetime.timedelta(days=31)
# Old news
old_news = News.query(News.date < last_quarter).order(
News.date).fetch(500, keys_only=True)
#logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True))
ndb.delete_multi(old_news)
elif type == 'tag_cloud':
channel_urls = []
tags = {}
extras = Extra.query(Extra.tag != None)
for extra in extras:
if extra.channelurl not in channel_urls:
channel_urls.append(extra.channelurl)
tag = extra.tag
if tag in tags:
tags[tag] += 1
else:
tags[tag] = 1
tags_sorted = sorted(tags.iteritems(),
key=operator.itemgetter(1),
reverse=True)
memcache.set("tag_cloud", tags_sorted)
logging.debug('Tags: %s' % (tags_sorted))
elif type == 'fix':
test_channel = '#kanava'
channel = Channel.query(Channel.name == test_channel).get()
channelurls = ChannelUrl.query(
ChannelUrl.channel == channel.key).fetch(50)
for channelurl in channelurls:
url = channelurl.url.get()
logging.debug('Channel: %s, channelurl: %s (id %s)' %
(test_channel, url, channelurl))
posts = Post.query(Post.channelurl == channelurl.key)
for post in posts:
logging.debug(' * posted by %s' % (post.user))
post.key.delete()
rates = Rate.query(Rate.channelurl == channelurl.key)
for rate in rates:
logging.debug(' * rate %s' % (rate))
rate.key.delete()
extras = Extra.query(Extra.channelurl == channelurl.key)
for extra in extras:
logging.debug(' * extra %s, by %s' % (extra, extra.user))
extra.key.delete()
channelurl.key.delete()
def post(self):
post_channel = self.request.get('post_channel', '')
post_user = self.request.get('post_user', '')
post_url = self.request.get('post_url', '')
type = self.request.get('type', '')
url = Url.all().filter('url =', post_url).get()
channel = Channel.all().filter('name =', post_channel).get()
channelurl = ChannelUrl.all().filter('channel =',
channel).filter('url =',
url).get()
if channelurl:
rate = Rate()
rate.channelurl = channelurl
rate.user = post_user
rate.type = type
rate.put()
else:
logging.warning('ChannelUrl not found: %s %s' %
(post_channel, post_url))
def tag(request, tag):
extras = Extra.query(Extra.tag == tag)
#logging.debug('Tag %s' % (tag))
data = []
urls = []
for extra in extras:
channelurl = extra.channelurl.get()
channel = channelurl.channel.get()
posts = Post.query(Post.channelurl == channelurl.key)
for post in posts:
url = channelurl.url.get()
#logging.debug('Url %s' % (url.url))
#data.append({'url':url})
if channel.private == False:
extras = Extra.query(Extra.channelurl == post.channelurl,
Extra.tag != tag)
rates = Rate.query(Rate.channelurl == post.channelurl)
d_dict = {
'date': post.date,
'channelurl': channelurl,
'channel': channel,
'post': post,
'url': url,
'extras': extras,
'rates': rates
}
if url not in urls:
data.append(d_dict)
urls.append(url)
data.sort(key=lambda item: item['date'], reverse=True)
template_values = {
'data': data,
'selected_tag': tag,
'tag_cloud': memcache.get('tag_cloud'),
}
return render_to_response('tags.html', template_values)
def index(request, date=None, cursor=None, rss=None, channel_filter=None):
data = []
today = datetime.date.today()
#logging.debug('data %s' % (str(data)))
tag_cloud = memcache.get('tag_cloud')
if not tag_cloud:
taskqueue.add(queue_name='default',
url='/tasks/maintenance',
params={'type': 'tag_cloud'})
if not date:
date = today
else:
date = datetime.datetime.strptime(date, '%Y-%m-%d')
next_date = date - datetime.timedelta(days=-1)
prev_date = date - datetime.timedelta(days=1)
#postqry=Post.query(Post.date>=datetime.datetime.combine(date, datetime.time()),Post.date<datetime.datetime.combine(next_date, datetime.time())).order(-Post.date)
postqry = Post.query(
Post.date < datetime.datetime.combine(next_date, datetime.time())
).order(-Post.date)
if cursor:
cursor = Cursor(urlsafe=cursor)
posts, next_cursor, more = postqry.fetch_page(10, start_cursor=cursor)
else:
posts, next_cursor, more = postqry.fetch_page(20)
#urls=Url.query().order(-Url.idate).fetch(10)
for post in posts: # TODO pagination
#logging.debug('Post %s' % (post))
channelurl = post.channelurl.get()
#logging.debug('ChannelUrl %s' % (channelurl))
channel = channelurl.channel.get()
#logging.debug('Channel %s' % (channel))
url = channelurl.url.get()
#logging.debug('Url %s' % (url))
#logging.debug('URL/post: channel=%s, user=%s, url=%s' % (channel.name, post.user, url.url))
#logging.debug('Private channel? %s (%s)' % (channel,channel.private))
if not channel_filter or channel_filter.lower(
) == channel.name[1:].lower():
if channel.private == False:
extras = Extra.query(Extra.channelurl == post.channelurl)
rates = Rate.query(Rate.channelurl == post.channelurl)
data.append({
'channelurl': channelurl,
'channel': channel,
'post': post,
'url': url,
'extras': extras,
'rates': rates
})
'''
channelurls = post.channelurl
logging.debug('CU: %s: %s' % (post.key.id(),channelurls))
data[post.key.id()]={}
#for channelurl in channelurls:
data[post.key.id()]['channelurls']=channelurls
'''
template_values = {
'data': data,
'tag_cloud': tag_cloud,
'next': next_cursor,
'date': date,
'next_date': next_date,
'prev_date': prev_date,
'more': more,
'channel': channel_filter,
}
if cursor:
return render_to_response('urls2.html', template_values)
return render_to_response('urls.html', template_values)
value=int(data.get('value',0))
logging.debug('user: %s, id: %s, value: %s' % (user,id,value))
except Exception, e:
logging.warning('Error %s' % (e))
try:
id=int(id)
except:
pass
channelurl=ChannelUrl.get_by_id(id)
if not channelurl:
retval=simplejson.dumps({'id':0,'rate': ''})
return HttpResponse(retval, mimetype="application/json")
else:
rate=Rate(user=user,value=value,channelurl=channelurl.key)
rate.put()
# Update Document (FullTextSearch)
url=channelurl.url.get()
doc_id=str(url.key.id())
try:
doc = search.Index(name='url').get(doc_id)
if not doc:
logging.warning('Document not found.')
try:
taskqueue.add(name=str(doc_id)+'_update',queue_name='document',url='/tasks/update_document', params={'doc_id': doc_id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s_update' % (str(doc_id)))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s_update' % (str(doc_id)))
def post(self):
doc_id = self.request.get('doc_id', '')
if doc_id:
#logging.debug('doc_id: %s' % (doc_id))
urlinstance = Url.get_by_id(int(doc_id))
if urlinstance:
# If not valid url, delete from index
if urlinstance.valid < 0:
doc_index = search.Index(name='url')
logging.info(
'Delete invalid (%s) url (ID %s) from document index \'url\' (%s)'
% (str(urlinstance.valid), doc_id, doc_index))
doc_index.delete(doc_id)
else:
url = urlinstance.url
title = urlinstance.title
#logging.debug('url: %s, title: %s' % (url, title))
channels = []
channel = None
users = []
user = None
date = datetime.datetime.fromtimestamp(0)
comments = []
comment = None
tags = []
tag = None
rate = 0
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key)
for channelurlinstance in channelurlquery:
channelinstance = channelurlinstance.channel.get()
if channelinstance.name not in channels:
channels.append(channelinstance.name)
#logging.info('Adding channel %s' % (channelinstance.name))
postquery = Post.query(
Post.channelurl == channelurlinstance.key)
for postinstance in postquery:
if postinstance.user not in users:
users.append(postinstance.user)
if date:
if date < postinstance.date:
date = postinstance.date
else:
date = postinstance.date
extraquery = Extra.query(
Extra.channelurl == channelurlinstance.key)
for extrainstance in extraquery:
if extrainstance.tag:
if extrainstance.tag not in tags:
tags.append(extrainstance.tag)
#logging.info('Adding tag %s' % (extrainstance.tag))
if extrainstance.comment:
if extrainstance.comment not in comments:
comments.append(extrainstance.comment)
#logging.info('Adding comment %s' % (extrainstance.comment))
ratequery = Rate.query(
Rate.channelurl == channelurlinstance.key)
for rateinstance in ratequery:
rate += rateinstance.value
#logging.debug('rate %s' % (rate))
if not date:
date = datetime.datetime.fromtimestamp(0)
# lists to strings
channel = ' '.join(channels)
user = '******'.join(users)
tag = ' '.join(tags)
if not tag:
tag = None
comment = ' '.join(comments)
if not comment:
comment = None
logging.debug(
'doc; channel=%s, user=%s, url=%s, date=%s, title=%s, comment=%s, tag=%s, rate=%s'
%
(channel, user, url, date, title, comment, tag, rate))
try:
doc = search.Document(
doc_id=str(doc_id),
fields=[
search.TextField(name='channel',
value=channel),
search.TextField(name='user', value=user),
search.TextField(name='url', value=url),
search.DateField(name='date', value=date),
search.TextField(name='title', value=title),
search.TextField(name='comment',
value=comment,
language='fi'),
search.TextField(name='tag',
value=tag,
language='fi'),
search.NumberField(name='rate', value=rate)
],
language='en')
except Exception, e:
logging.error('doc_id: %s, error %s' %
(str(doc_id), e))
doc = None
try:
if doc:
search.Index(name='url').put(doc)
urlinstance.document_date = datetime.datetime.now()
urlinstance.put()
else:
logging.error('Doc missing.')
except search.Error:
logging.error('Create Document failed.')
else:
logging.debug('No urlinstance for doc_id: %s' % (doc_id))