def view_master(request, urlid):
data = []
logging.debug('View (Master)Url %s' % (urlid))
try:
urlid = int(urlid)
except:
pass
url = Url.get_by_id(urlid)
if url:
channelurls = ChannelUrl.query(ChannelUrl.url == url.key)
for channelurl in channelurls:
channel = channelurl.channel.get()
if channel.private == False:
extras = Extra.query(Extra.channelurl == channelurl.key)
rates = Rate.query(Rate.channelurl == channelurl.key)
rating = channelurl.rating()
#data.append({'channel':channel,'post':post,'url':url,'extras': extras})
data.append({
'channel': channel,
'channelurl': channelurl,
'post': post,
'url': url,
'extras': extras,
'rates': rates,
'rating': rating
})
template_values = {
'data': data,
'user': users.get_current_user(),
}
return render_to_response('masterurl.html', template_values)
def post(self):
type = self.request.get('type', '')
if type == 'stats':
pass
elif type == 'cleanup':
last_year = datetime.datetime.now() - datetime.timedelta(days=365)
last_quarter = datetime.datetime.now() - datetime.timedelta(
days=92)
last_month = datetime.datetime.now() - datetime.timedelta(days=31)
# Old news
old_news = News.query(News.date < last_quarter).order(
News.date).fetch(500, keys_only=True)
#logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True))
ndb.delete_multi(old_news)
elif type == 'tag_cloud':
channel_urls = []
tags = {}
extras = Extra.query(Extra.tag != None)
for extra in extras:
if extra.channelurl not in channel_urls:
channel_urls.append(extra.channelurl)
tag = extra.tag
if tag in tags:
tags[tag] += 1
else:
tags[tag] = 1
tags_sorted = sorted(tags.iteritems(),
key=operator.itemgetter(1),
reverse=True)
memcache.set("tag_cloud", tags_sorted)
logging.debug('Tags: %s' % (tags_sorted))
elif type == 'fix':
test_channel = '#kanava'
channel = Channel.query(Channel.name == test_channel).get()
channelurls = ChannelUrl.query(
ChannelUrl.channel == channel.key).fetch(50)
for channelurl in channelurls:
url = channelurl.url.get()
logging.debug('Channel: %s, channelurl: %s (id %s)' %
(test_channel, url, channelurl))
posts = Post.query(Post.channelurl == channelurl.key)
for post in posts:
logging.debug(' * posted by %s' % (post.user))
post.key.delete()
rates = Rate.query(Rate.channelurl == channelurl.key)
for rate in rates:
logging.debug(' * rate %s' % (rate))
rate.key.delete()
extras = Extra.query(Extra.channelurl == channelurl.key)
for extra in extras:
logging.debug(' * extra %s, by %s' % (extra, extra.user))
extra.key.delete()
channelurl.key.delete()
def __unicode__(self):
if self.link:
if self.link_text:
return '%s: <a target="_blank" href="%s">%s</a>' % (
self.content, self.link, self.link_text)
else:
text = ''
comments = []
url = Url.query(Url.url == self.link).get(keys_only=True)
channel_url = ChannelUrl.query(ChannelUrl.url == url).get(
keys_only=True)
extra = Extra.query(Extra.channelurl == channel_url).get(
keys_only=False)
if extra and extra.comment:
text = extra.comment
if not text:
text = ''.join(self.link.split('/')[-1:])
return '%s: <a target="_blank" href="%s">%s</a>' % (
self.content, self.link, text)
return self.content
logging.info('Old url %s' % (url))
# 2. tarkista onko olemassa jo ko. Channel, lisää jos ei
channelquery=Channel.query(Channel.name==channel)
channelinstance=channelquery.get()
if not channelinstance:
if channel.startswith('#'):
private=False
else:
private=True
channelinstance=Channel(name=channel,private=private)
channelinstance.put()
logging.info('New channel %s' % (channel))
# 3. tarkista onko url jo olemassa channel-tasolla
channelurlquery=ChannelUrl.query(ChannelUrl.url==urlinstance.key, ChannelUrl.channel==channelinstance.key)
channelurlinstance=channelurlquery.get()
if not channelurlinstance:
l=list(string.ascii_uppercase)
l.append('Z')
DEFAULT_COUNTER_NAME=chr(now.isocalendar()[0]-2010+65)+l[(now.isocalendar()[1]-1)/2]
#logging.debug('DEFAULT_COUNTER_NAME: %s' % (DEFAULT_COUNTER_NAME))
counter.increment(DEFAULT_COUNTER_NAME)
key_name=DEFAULT_COUNTER_NAME+str(counter.get_count(DEFAULT_COUNTER_NAME))
#logging.debug('key_name %s' % (key_name))
channelurlinstance=ChannelUrl(id=key_name,channel=channelinstance.key,url=urlinstance.key)
channelurlinstance.put()
#logging.debug('New channelurl %s/%s' % (channel,url))
def post(self):
doc_id = self.request.get('doc_id', '')
if doc_id:
#logging.debug('doc_id: %s' % (doc_id))
urlinstance = Url.get_by_id(int(doc_id))
if urlinstance:
# If not valid url, delete from index
if urlinstance.valid < 0:
doc_index = search.Index(name='url')
logging.info(
'Delete invalid (%s) url (ID %s) from document index \'url\' (%s)'
% (str(urlinstance.valid), doc_id, doc_index))
doc_index.delete(doc_id)
else:
url = urlinstance.url
title = urlinstance.title
#logging.debug('url: %s, title: %s' % (url, title))
channels = []
channel = None
users = []
user = None
date = datetime.datetime.fromtimestamp(0)
comments = []
comment = None
tags = []
tag = None
rate = 0
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key)
for channelurlinstance in channelurlquery:
channelinstance = channelurlinstance.channel.get()
if channelinstance.name not in channels:
channels.append(channelinstance.name)
#logging.info('Adding channel %s' % (channelinstance.name))
postquery = Post.query(
Post.channelurl == channelurlinstance.key)
for postinstance in postquery:
if postinstance.user not in users:
users.append(postinstance.user)
if date:
if date < postinstance.date:
date = postinstance.date
else:
date = postinstance.date
extraquery = Extra.query(
Extra.channelurl == channelurlinstance.key)
for extrainstance in extraquery:
if extrainstance.tag:
if extrainstance.tag not in tags:
tags.append(extrainstance.tag)
#logging.info('Adding tag %s' % (extrainstance.tag))
if extrainstance.comment:
if extrainstance.comment not in comments:
comments.append(extrainstance.comment)
#logging.info('Adding comment %s' % (extrainstance.comment))
ratequery = Rate.query(
Rate.channelurl == channelurlinstance.key)
for rateinstance in ratequery:
rate += rateinstance.value
#logging.debug('rate %s' % (rate))
if not date:
date = datetime.datetime.fromtimestamp(0)
# lists to strings
channel = ' '.join(channels)
user = '******'.join(users)
tag = ' '.join(tags)
if not tag:
tag = None
comment = ' '.join(comments)
if not comment:
comment = None
logging.debug(
'doc; channel=%s, user=%s, url=%s, date=%s, title=%s, comment=%s, tag=%s, rate=%s'
%
(channel, user, url, date, title, comment, tag, rate))
try:
doc = search.Document(
doc_id=str(doc_id),
fields=[
search.TextField(name='channel',
value=channel),
search.TextField(name='user', value=user),
search.TextField(name='url', value=url),
search.DateField(name='date', value=date),
search.TextField(name='title', value=title),
search.TextField(name='comment',
value=comment,
language='fi'),
search.TextField(name='tag',
value=tag,
language='fi'),
search.NumberField(name='rate', value=rate)
],
language='en')
except Exception, e:
logging.error('doc_id: %s, error %s' %
(str(doc_id), e))
doc = None
try:
if doc:
search.Index(name='url').put(doc)
urlinstance.document_date = datetime.datetime.now()
urlinstance.put()
else:
logging.error('Doc missing.')
except search.Error:
logging.error('Create Document failed.')
else:
logging.debug('No urlinstance for doc_id: %s' % (doc_id))