def post(self):
type = self.request.get('type', '')
if type == 'stats':
pass
elif type == 'cleanup':
last_year = datetime.datetime.now() - datetime.timedelta(days=365)
last_quarter = datetime.datetime.now() - datetime.timedelta(
days=92)
last_month = datetime.datetime.now() - datetime.timedelta(days=31)
# Old news
old_news = News.query(News.date < last_quarter).order(
News.date).fetch(500, keys_only=True)
#logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True))
ndb.delete_multi(old_news)
elif type == 'tag_cloud':
channel_urls = []
tags = {}
extras = Extra.query(Extra.tag != None)
for extra in extras:
if extra.channelurl not in channel_urls:
channel_urls.append(extra.channelurl)
tag = extra.tag
if tag in tags:
tags[tag] += 1
else:
tags[tag] = 1
tags_sorted = sorted(tags.iteritems(),
key=operator.itemgetter(1),
reverse=True)
memcache.set("tag_cloud", tags_sorted)
logging.debug('Tags: %s' % (tags_sorted))
elif type == 'fix':
test_channel = '#kanava'
channel = Channel.query(Channel.name == test_channel).get()
channelurls = ChannelUrl.query(
ChannelUrl.channel == channel.key).fetch(50)
for channelurl in channelurls:
url = channelurl.url.get()
logging.debug('Channel: %s, channelurl: %s (id %s)' %
(test_channel, url, channelurl))
posts = Post.query(Post.channelurl == channelurl.key)
for post in posts:
logging.debug(' * posted by %s' % (post.user))
post.key.delete()
rates = Rate.query(Rate.channelurl == channelurl.key)
for rate in rates:
logging.debug(' * rate %s' % (rate))
rate.key.delete()
extras = Extra.query(Extra.channelurl == channelurl.key)
for extra in extras:
logging.debug(' * extra %s, by %s' % (extra, extra.user))
extra.key.delete()
channelurl.key.delete()
def tag(request, tag):
extras = Extra.query(Extra.tag == tag)
#logging.debug('Tag %s' % (tag))
data = []
urls = []
for extra in extras:
channelurl = extra.channelurl.get()
channel = channelurl.channel.get()
posts = Post.query(Post.channelurl == channelurl.key)
for post in posts:
url = channelurl.url.get()
#logging.debug('Url %s' % (url.url))
#data.append({'url':url})
if channel.private == False:
extras = Extra.query(Extra.channelurl == post.channelurl,
Extra.tag != tag)
rates = Rate.query(Rate.channelurl == post.channelurl)
d_dict = {
'date': post.date,
'channelurl': channelurl,
'channel': channel,
'post': post,
'url': url,
'extras': extras,
'rates': rates
}
if url not in urls:
data.append(d_dict)
urls.append(url)
data.sort(key=lambda item: item['date'], reverse=True)
template_values = {
'data': data,
'selected_tag': tag,
'tag_cloud': memcache.get('tag_cloud'),
}
return render_to_response('tags.html', template_values)
def index(request, date=None, cursor=None, rss=None, channel_filter=None):
data = []
today = datetime.date.today()
#logging.debug('data %s' % (str(data)))
tag_cloud = memcache.get('tag_cloud')
if not tag_cloud:
taskqueue.add(queue_name='default',
url='/tasks/maintenance',
params={'type': 'tag_cloud'})
if not date:
date = today
else:
date = datetime.datetime.strptime(date, '%Y-%m-%d')
next_date = date - datetime.timedelta(days=-1)
prev_date = date - datetime.timedelta(days=1)
#postqry=Post.query(Post.date>=datetime.datetime.combine(date, datetime.time()),Post.date<datetime.datetime.combine(next_date, datetime.time())).order(-Post.date)
postqry = Post.query(
Post.date < datetime.datetime.combine(next_date, datetime.time())
).order(-Post.date)
if cursor:
cursor = Cursor(urlsafe=cursor)
posts, next_cursor, more = postqry.fetch_page(10, start_cursor=cursor)
else:
posts, next_cursor, more = postqry.fetch_page(20)
#urls=Url.query().order(-Url.idate).fetch(10)
for post in posts: # TODO pagination
#logging.debug('Post %s' % (post))
channelurl = post.channelurl.get()
#logging.debug('ChannelUrl %s' % (channelurl))
channel = channelurl.channel.get()
#logging.debug('Channel %s' % (channel))
url = channelurl.url.get()
#logging.debug('Url %s' % (url))
#logging.debug('URL/post: channel=%s, user=%s, url=%s' % (channel.name, post.user, url.url))
#logging.debug('Private channel? %s (%s)' % (channel,channel.private))
if not channel_filter or channel_filter.lower(
) == channel.name[1:].lower():
if channel.private == False:
extras = Extra.query(Extra.channelurl == post.channelurl)
rates = Rate.query(Rate.channelurl == post.channelurl)
data.append({
'channelurl': channelurl,
'channel': channel,
'post': post,
'url': url,
'extras': extras,
'rates': rates
})
'''
channelurls = post.channelurl
logging.debug('CU: %s: %s' % (post.key.id(),channelurls))
data[post.key.id()]={}
#for channelurl in channelurls:
data[post.key.id()]['channelurls']=channelurls
'''
template_values = {
'data': data,
'tag_cloud': tag_cloud,
'next': next_cursor,
'date': date,
'next_date': next_date,
'prev_date': prev_date,
'more': more,
'channel': channel_filter,
}
if cursor:
return render_to_response('urls2.html', template_values)
return render_to_response('urls.html', template_values)
DEFAULT_COUNTER_NAME=chr(now.isocalendar()[0]-2010+65)+l[(now.isocalendar()[1]-1)/2]
#logging.debug('DEFAULT_COUNTER_NAME: %s' % (DEFAULT_COUNTER_NAME))
counter.increment(DEFAULT_COUNTER_NAME)
key_name=DEFAULT_COUNTER_NAME+str(counter.get_count(DEFAULT_COUNTER_NAME))
#logging.debug('key_name %s' % (key_name))
channelurlinstance=ChannelUrl(id=key_name,channel=channelinstance.key,url=urlinstance.key)
channelurlinstance.put()
#logging.debug('New channelurl %s/%s' % (channel,url))
else:
#logging.info('OLDIE! %s %s' % (channelurlinstance.channel.name,channelurlinstance.url.url))
logging.info('Old channelurl %s %s' % (channel,url))
old_url=True
old_post=Post.query(Post.channelurl==channelurlinstance.key).order(Post.date).get()
try:
old_date=old_post.date.strftime("%d.%m.%y %H:%M")
except:
try:
old_date=old_post.idate.strftime("%d.%m.%y %H:%M")
except:
old_date=''
old_user=old_post.user
# 4. Lisätään postaus (tarkistetaan ettei ole jo)
postquery=Post.query(Post.channelurl==channelurlinstance.key,Post.user==user,Post.date==date)
postinstance=postquery.get()
if postinstance:
logging.info('Old post; channel: %s, url: %s, user: %s' % (channel,url,user))
else:
def post(self):
doc_id = self.request.get('doc_id', '')
if doc_id:
#logging.debug('doc_id: %s' % (doc_id))
urlinstance = Url.get_by_id(int(doc_id))
if urlinstance:
# If not valid url, delete from index
if urlinstance.valid < 0:
doc_index = search.Index(name='url')
logging.info(
'Delete invalid (%s) url (ID %s) from document index \'url\' (%s)'
% (str(urlinstance.valid), doc_id, doc_index))
doc_index.delete(doc_id)
else:
url = urlinstance.url
title = urlinstance.title
#logging.debug('url: %s, title: %s' % (url, title))
channels = []
channel = None
users = []
user = None
date = datetime.datetime.fromtimestamp(0)
comments = []
comment = None
tags = []
tag = None
rate = 0
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key)
for channelurlinstance in channelurlquery:
channelinstance = channelurlinstance.channel.get()
if channelinstance.name not in channels:
channels.append(channelinstance.name)
#logging.info('Adding channel %s' % (channelinstance.name))
postquery = Post.query(
Post.channelurl == channelurlinstance.key)
for postinstance in postquery:
if postinstance.user not in users:
users.append(postinstance.user)
if date:
if date < postinstance.date:
date = postinstance.date
else:
date = postinstance.date
extraquery = Extra.query(
Extra.channelurl == channelurlinstance.key)
for extrainstance in extraquery:
if extrainstance.tag:
if extrainstance.tag not in tags:
tags.append(extrainstance.tag)
#logging.info('Adding tag %s' % (extrainstance.tag))
if extrainstance.comment:
if extrainstance.comment not in comments:
comments.append(extrainstance.comment)
#logging.info('Adding comment %s' % (extrainstance.comment))
ratequery = Rate.query(
Rate.channelurl == channelurlinstance.key)
for rateinstance in ratequery:
rate += rateinstance.value
#logging.debug('rate %s' % (rate))
if not date:
date = datetime.datetime.fromtimestamp(0)
# lists to strings
channel = ' '.join(channels)
user = '******'.join(users)
tag = ' '.join(tags)
if not tag:
tag = None
comment = ' '.join(comments)
if not comment:
comment = None
logging.debug(
'doc; channel=%s, user=%s, url=%s, date=%s, title=%s, comment=%s, tag=%s, rate=%s'
%
(channel, user, url, date, title, comment, tag, rate))
try:
doc = search.Document(
doc_id=str(doc_id),
fields=[
search.TextField(name='channel',
value=channel),
search.TextField(name='user', value=user),
search.TextField(name='url', value=url),
search.DateField(name='date', value=date),
search.TextField(name='title', value=title),
search.TextField(name='comment',
value=comment,
language='fi'),
search.TextField(name='tag',
value=tag,
language='fi'),
search.NumberField(name='rate', value=rate)
],
language='en')
except Exception, e:
logging.error('doc_id: %s, error %s' %
(str(doc_id), e))
doc = None
try:
if doc:
search.Index(name='url').put(doc)
urlinstance.document_date = datetime.datetime.now()
urlinstance.put()
else:
logging.error('Doc missing.')
except search.Error:
logging.error('Create Document failed.')
else:
logging.debug('No urlinstance for doc_id: %s' % (doc_id))
def post(self):
if users.get_current_user():
node.author = users.get_current_user()
post_channel = self.request.get('post_channel', '')
post_user = self.request.get('post_user', '')
post_url = self.request.get('post_url', '')
# Add http:// when needed
if not post_url.startswith('http'):
post_url = 'http://' + post_url
logging.debug('Post: C=%s U=%s P=%s' %
(post_channel, post_user, post_url))
# 1. tarkista onko olemassa jo ko. Url, lisää jos ei, muuten päivitä (udate, valid?): valid-juttu joo ehkä jos tarpeen, ei muuten
url = Url.all().filter('url =', post_url).get()
if not url:
url = Url()
url.url = post_url
url.put()
# Title
name = ''.join(
re.findall('[a-zA-Z0-9_-]',
post_channel + '_' + post_url))[:500]
try:
taskqueue.add(name=name,
queue_name='urlfetch',
url='/tasks/title',
params={'post_url': post_url})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedError %s' % post_url)
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlredyExists: %s' % post_url)
# 2. tarkista onko olemassa jo ko. Channel, lisää jos ei
channel = Channel.all().filter('name =', post_channel).get()
if not channel:
channel = Channel()
channel.name = post_channel
if post_channel.startswith('!'):
channel.private = True
channel.put()
# 3. tarkista onko url jo olemassa channel-tasolla
channelurl = ChannelUrl.all().filter('url =',
url).filter('channel =',
channel).get()
if not channelurl:
channelurl = ChannelUrl()
channelurl.channel = channel
channelurl.url = url
#channelurl.user=post_user
channelurl.put()
else:
logging.info('OLDIE! %s %s' %
(channelurl.channel.name, channelurl.url.url))
# 4. Lisätään postaus
post = Post()
post.channelurl = channelurl
post.user = post_user
post.put()