def view_master(request, urlid):
data = []
logging.debug('View (Master)Url %s' % (urlid))
try:
urlid = int(urlid)
except:
pass
url = Url.get_by_id(urlid)
if url:
channelurls = ChannelUrl.query(ChannelUrl.url == url.key)
for channelurl in channelurls:
channel = channelurl.channel.get()
if channel.private == False:
extras = Extra.query(Extra.channelurl == channelurl.key)
rates = Rate.query(Rate.channelurl == channelurl.key)
rating = channelurl.rating()
#data.append({'channel':channel,'post':post,'url':url,'extras': extras})
data.append({
'channel': channel,
'channelurl': channelurl,
'post': post,
'url': url,
'extras': extras,
'rates': rates,
'rating': rating
})
template_values = {
'data': data,
'user': users.get_current_user(),
}
return render_to_response('masterurl.html', template_values)
def view(request, urlid):
data = []
logging.debug('View ChannelUrl %s' % (urlid))
try:
urlid = int(urlid)
except:
pass
channelurl = ChannelUrl.get_by_id(urlid)
if channelurl:
channel = channelurl.channel.get()
url = channelurl.url.get()
if channel.private == False:
extras = Extra.query(Extra.channelurl == channelurl.key)
rates = Rate.query(Rate.channelurl == channelurl.key)
rating = channelurl.rating()
#data.append({'channel':channel,'post':post,'url':url,'extras': extras})
data = {
'channel': channel,
'post': post,
'url': url,
'extras': extras,
'rates': rates,
'rating': rating
}
template_values = {
'data': data,
}
return render_to_response('url.html', template_values)
def post(self):
post_channel = self.request.get('post_channel', '')
post_user = self.request.get('post_user', '')
post_url = self.request.get('post_url', '')
post_extra = self.request.get('extra', '')
# Related/tag
if post_extra.startwith('#'):
# TODO: check
if post_extra[1:].isdigit():
type = 'related'
else:
type = 'tag'
else:
type = 'comment'
url = Url.all().filter('url =', post_url).get()
channel = Channel.all().filter('name =', post_channel).get()
channelurl = ChannelUrl.all().filter('channel =',
channel).filter('url =',
url).get()
if channelurl:
extra = Extra()
extra.channelurl = channelurl
extra.user = post_user
setattr(extra, type, post_extra)
extra.put()
else:
logging.warning('ChannelUrl not found: %s %s' %
(post_channel, post_url))
def post(self):
type = self.request.get('type', '')
if type == 'stats':
pass
elif type == 'cleanup':
last_year = datetime.datetime.now() - datetime.timedelta(days=365)
last_quarter = datetime.datetime.now() - datetime.timedelta(
days=92)
last_month = datetime.datetime.now() - datetime.timedelta(days=31)
# Old news
old_news = News.query(News.date < last_quarter).order(
News.date).fetch(500, keys_only=True)
#logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True))
ndb.delete_multi(old_news)
elif type == 'tag_cloud':
channel_urls = []
tags = {}
extras = Extra.query(Extra.tag != None)
for extra in extras:
if extra.channelurl not in channel_urls:
channel_urls.append(extra.channelurl)
tag = extra.tag
if tag in tags:
tags[tag] += 1
else:
tags[tag] = 1
tags_sorted = sorted(tags.iteritems(),
key=operator.itemgetter(1),
reverse=True)
memcache.set("tag_cloud", tags_sorted)
logging.debug('Tags: %s' % (tags_sorted))
elif type == 'fix':
test_channel = '#kanava'
channel = Channel.query(Channel.name == test_channel).get()
channelurls = ChannelUrl.query(
ChannelUrl.channel == channel.key).fetch(50)
for channelurl in channelurls:
url = channelurl.url.get()
logging.debug('Channel: %s, channelurl: %s (id %s)' %
(test_channel, url, channelurl))
posts = Post.query(Post.channelurl == channelurl.key)
for post in posts:
logging.debug(' * posted by %s' % (post.user))
post.key.delete()
rates = Rate.query(Rate.channelurl == channelurl.key)
for rate in rates:
logging.debug(' * rate %s' % (rate))
rate.key.delete()
extras = Extra.query(Extra.channelurl == channelurl.key)
for extra in extras:
logging.debug(' * extra %s, by %s' % (extra, extra.user))
extra.key.delete()
channelurl.key.delete()
def post(self):
post_channel = self.request.get('post_channel', '')
post_user = self.request.get('post_user', '')
post_url = self.request.get('post_url', '')
type = self.request.get('type', '')
url = Url.all().filter('url =', post_url).get()
channel = Channel.all().filter('name =', post_channel).get()
channelurl = ChannelUrl.all().filter('channel =',
channel).filter('url =',
url).get()
if channelurl:
rate = Rate()
rate.channelurl = channelurl
rate.user = post_user
rate.type = type
rate.put()
else:
logging.warning('ChannelUrl not found: %s %s' %
(post_channel, post_url))
def __unicode__(self):
if self.link:
if self.link_text:
return '%s: <a target="_blank" href="%s">%s</a>' % (
self.content, self.link, self.link_text)
else:
text = ''
comments = []
url = Url.query(Url.url == self.link).get(keys_only=True)
channel_url = ChannelUrl.query(ChannelUrl.url == url).get(
keys_only=True)
extra = Extra.query(Extra.channelurl == channel_url).get(
keys_only=False)
if extra and extra.comment:
text = extra.comment
if not text:
text = ''.join(self.link.split('/')[-1:])
return '%s: <a target="_blank" href="%s">%s</a>' % (
self.content, self.link, text)
return self.content
logging.info('Old url %s' % (url))
# 2. tarkista onko olemassa jo ko. Channel, lisää jos ei
channelquery=Channel.query(Channel.name==channel)
channelinstance=channelquery.get()
if not channelinstance:
if channel.startswith('#'):
private=False
else:
private=True
channelinstance=Channel(name=channel,private=private)
channelinstance.put()
logging.info('New channel %s' % (channel))
# 3. tarkista onko url jo olemassa channel-tasolla
channelurlquery=ChannelUrl.query(ChannelUrl.url==urlinstance.key, ChannelUrl.channel==channelinstance.key)
channelurlinstance=channelurlquery.get()
if not channelurlinstance:
l=list(string.ascii_uppercase)
l.append('Z')
DEFAULT_COUNTER_NAME=chr(now.isocalendar()[0]-2010+65)+l[(now.isocalendar()[1]-1)/2]
#logging.debug('DEFAULT_COUNTER_NAME: %s' % (DEFAULT_COUNTER_NAME))
counter.increment(DEFAULT_COUNTER_NAME)
key_name=DEFAULT_COUNTER_NAME+str(counter.get_count(DEFAULT_COUNTER_NAME))
#logging.debug('key_name %s' % (key_name))
channelurlinstance=ChannelUrl(id=key_name,channel=channelinstance.key,url=urlinstance.key)
channelurlinstance.put()
#logging.debug('New channelurl %s/%s' % (channel,url))
def post(self):
doc_id = self.request.get('doc_id', '')
if doc_id:
#logging.debug('doc_id: %s' % (doc_id))
urlinstance = Url.get_by_id(int(doc_id))
if urlinstance:
# If not valid url, delete from index
if urlinstance.valid < 0:
doc_index = search.Index(name='url')
logging.info(
'Delete invalid (%s) url (ID %s) from document index \'url\' (%s)'
% (str(urlinstance.valid), doc_id, doc_index))
doc_index.delete(doc_id)
else:
url = urlinstance.url
title = urlinstance.title
#logging.debug('url: %s, title: %s' % (url, title))
channels = []
channel = None
users = []
user = None
date = datetime.datetime.fromtimestamp(0)
comments = []
comment = None
tags = []
tag = None
rate = 0
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key)
for channelurlinstance in channelurlquery:
channelinstance = channelurlinstance.channel.get()
if channelinstance.name not in channels:
channels.append(channelinstance.name)
#logging.info('Adding channel %s' % (channelinstance.name))
postquery = Post.query(
Post.channelurl == channelurlinstance.key)
for postinstance in postquery:
if postinstance.user not in users:
users.append(postinstance.user)
if date:
if date < postinstance.date:
date = postinstance.date
else:
date = postinstance.date
extraquery = Extra.query(
Extra.channelurl == channelurlinstance.key)
for extrainstance in extraquery:
if extrainstance.tag:
if extrainstance.tag not in tags:
tags.append(extrainstance.tag)
#logging.info('Adding tag %s' % (extrainstance.tag))
if extrainstance.comment:
if extrainstance.comment not in comments:
comments.append(extrainstance.comment)
#logging.info('Adding comment %s' % (extrainstance.comment))
ratequery = Rate.query(
Rate.channelurl == channelurlinstance.key)
for rateinstance in ratequery:
rate += rateinstance.value
#logging.debug('rate %s' % (rate))
if not date:
date = datetime.datetime.fromtimestamp(0)
# lists to strings
channel = ' '.join(channels)
user = '******'.join(users)
tag = ' '.join(tags)
if not tag:
tag = None
comment = ' '.join(comments)
if not comment:
comment = None
logging.debug(
'doc; channel=%s, user=%s, url=%s, date=%s, title=%s, comment=%s, tag=%s, rate=%s'
%
(channel, user, url, date, title, comment, tag, rate))
try:
doc = search.Document(
doc_id=str(doc_id),
fields=[
search.TextField(name='channel',
value=channel),
search.TextField(name='user', value=user),
search.TextField(name='url', value=url),
search.DateField(name='date', value=date),
search.TextField(name='title', value=title),
search.TextField(name='comment',
value=comment,
language='fi'),
search.TextField(name='tag',
value=tag,
language='fi'),
search.NumberField(name='rate', value=rate)
],
language='en')
except Exception, e:
logging.error('doc_id: %s, error %s' %
(str(doc_id), e))
doc = None
try:
if doc:
search.Index(name='url').put(doc)
urlinstance.document_date = datetime.datetime.now()
urlinstance.put()
else:
logging.error('Doc missing.')
except search.Error:
logging.error('Create Document failed.')
else:
logging.debug('No urlinstance for doc_id: %s' % (doc_id))
def post(self):
if users.get_current_user():
node.author = users.get_current_user()
post_channel = self.request.get('post_channel', '')
post_user = self.request.get('post_user', '')
post_url = self.request.get('post_url', '')
# Add http:// when needed
if not post_url.startswith('http'):
post_url = 'http://' + post_url
logging.debug('Post: C=%s U=%s P=%s' %
(post_channel, post_user, post_url))
# 1. tarkista onko olemassa jo ko. Url, lisää jos ei, muuten päivitä (udate, valid?): valid-juttu joo ehkä jos tarpeen, ei muuten
url = Url.all().filter('url =', post_url).get()
if not url:
url = Url()
url.url = post_url
url.put()
# Title
name = ''.join(
re.findall('[a-zA-Z0-9_-]',
post_channel + '_' + post_url))[:500]
try:
taskqueue.add(name=name,
queue_name='urlfetch',
url='/tasks/title',
params={'post_url': post_url})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedError %s' % post_url)
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlredyExists: %s' % post_url)
# 2. tarkista onko olemassa jo ko. Channel, lisää jos ei
channel = Channel.all().filter('name =', post_channel).get()
if not channel:
channel = Channel()
channel.name = post_channel
if post_channel.startswith('!'):
channel.private = True
channel.put()
# 3. tarkista onko url jo olemassa channel-tasolla
channelurl = ChannelUrl.all().filter('url =',
url).filter('channel =',
channel).get()
if not channelurl:
channelurl = ChannelUrl()
channelurl.channel = channel
channelurl.url = url
#channelurl.user=post_user
channelurl.put()
else:
logging.info('OLDIE! %s %s' %
(channelurl.channel.name, channelurl.url.url))
# 4. Lisätään postaus
post = Post()
post.channelurl = channelurl
post.user = post_user
post.put()