def view_master(request, urlid): data = [] logging.debug('View (Master)Url %s' % (urlid)) try: urlid = int(urlid) except: pass url = Url.get_by_id(urlid) if url: channelurls = ChannelUrl.query(ChannelUrl.url == url.key) for channelurl in channelurls: channel = channelurl.channel.get() if channel.private == False: extras = Extra.query(Extra.channelurl == channelurl.key) rates = Rate.query(Rate.channelurl == channelurl.key) rating = channelurl.rating() #data.append({'channel':channel,'post':post,'url':url,'extras': extras}) data.append({ 'channel': channel, 'channelurl': channelurl, 'post': post, 'url': url, 'extras': extras, 'rates': rates, 'rating': rating }) template_values = { 'data': data, 'user': users.get_current_user(), } return render_to_response('masterurl.html', template_values)
def view(request, urlid): data = [] logging.debug('View ChannelUrl %s' % (urlid)) try: urlid = int(urlid) except: pass channelurl = ChannelUrl.get_by_id(urlid) if channelurl: channel = channelurl.channel.get() url = channelurl.url.get() if channel.private == False: extras = Extra.query(Extra.channelurl == channelurl.key) rates = Rate.query(Rate.channelurl == channelurl.key) rating = channelurl.rating() #data.append({'channel':channel,'post':post,'url':url,'extras': extras}) data = { 'channel': channel, 'post': post, 'url': url, 'extras': extras, 'rates': rates, 'rating': rating } template_values = { 'data': data, } return render_to_response('url.html', template_values)
def post(self): post_channel = self.request.get('post_channel', '') post_user = self.request.get('post_user', '') post_url = self.request.get('post_url', '') post_extra = self.request.get('extra', '') # Related/tag if post_extra.startwith('#'): # TODO: check if post_extra[1:].isdigit(): type = 'related' else: type = 'tag' else: type = 'comment' url = Url.all().filter('url =', post_url).get() channel = Channel.all().filter('name =', post_channel).get() channelurl = ChannelUrl.all().filter('channel =', channel).filter('url =', url).get() if channelurl: extra = Extra() extra.channelurl = channelurl extra.user = post_user setattr(extra, type, post_extra) extra.put() else: logging.warning('ChannelUrl not found: %s %s' % (post_channel, post_url))
def post(self): type = self.request.get('type', '') if type == 'stats': pass elif type == 'cleanup': last_year = datetime.datetime.now() - datetime.timedelta(days=365) last_quarter = datetime.datetime.now() - datetime.timedelta( days=92) last_month = datetime.datetime.now() - datetime.timedelta(days=31) # Old news old_news = News.query(News.date < last_quarter).order( News.date).fetch(500, keys_only=True) #logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True)) ndb.delete_multi(old_news) elif type == 'tag_cloud': channel_urls = [] tags = {} extras = Extra.query(Extra.tag != None) for extra in extras: if extra.channelurl not in channel_urls: channel_urls.append(extra.channelurl) tag = extra.tag if tag in tags: tags[tag] += 1 else: tags[tag] = 1 tags_sorted = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True) memcache.set("tag_cloud", tags_sorted) logging.debug('Tags: %s' % (tags_sorted)) elif type == 'fix': test_channel = '#kanava' channel = Channel.query(Channel.name == test_channel).get() channelurls = ChannelUrl.query( ChannelUrl.channel == channel.key).fetch(50) for channelurl in channelurls: url = channelurl.url.get() logging.debug('Channel: %s, channelurl: %s (id %s)' % (test_channel, url, channelurl)) posts = Post.query(Post.channelurl == channelurl.key) for post in posts: logging.debug(' * posted by %s' % (post.user)) post.key.delete() rates = Rate.query(Rate.channelurl == channelurl.key) for rate in rates: logging.debug(' * rate %s' % (rate)) rate.key.delete() extras = Extra.query(Extra.channelurl == channelurl.key) for extra in extras: logging.debug(' * extra %s, by %s' % (extra, extra.user)) extra.key.delete() channelurl.key.delete()
def post(self): post_channel = self.request.get('post_channel', '') post_user = self.request.get('post_user', '') post_url = self.request.get('post_url', '') type = self.request.get('type', '') url = Url.all().filter('url =', post_url).get() channel = Channel.all().filter('name =', post_channel).get() channelurl = ChannelUrl.all().filter('channel =', channel).filter('url =', url).get() if channelurl: rate = Rate() rate.channelurl = channelurl rate.user = post_user rate.type = type rate.put() else: logging.warning('ChannelUrl not found: %s %s' % (post_channel, post_url))
def __unicode__(self): if self.link: if self.link_text: return '%s: <a target="_blank" href="%s">%s</a>' % ( self.content, self.link, self.link_text) else: text = '' comments = [] url = Url.query(Url.url == self.link).get(keys_only=True) channel_url = ChannelUrl.query(ChannelUrl.url == url).get( keys_only=True) extra = Extra.query(Extra.channelurl == channel_url).get( keys_only=False) if extra and extra.comment: text = extra.comment if not text: text = ''.join(self.link.split('/')[-1:]) return '%s: <a target="_blank" href="%s">%s</a>' % ( self.content, self.link, text) return self.content
logging.info('Old url %s' % (url)) # 2. tarkista onko olemassa jo ko. Channel, lisää jos ei channelquery=Channel.query(Channel.name==channel) channelinstance=channelquery.get() if not channelinstance: if channel.startswith('#'): private=False else: private=True channelinstance=Channel(name=channel,private=private) channelinstance.put() logging.info('New channel %s' % (channel)) # 3. tarkista onko url jo olemassa channel-tasolla channelurlquery=ChannelUrl.query(ChannelUrl.url==urlinstance.key, ChannelUrl.channel==channelinstance.key) channelurlinstance=channelurlquery.get() if not channelurlinstance: l=list(string.ascii_uppercase) l.append('Z') DEFAULT_COUNTER_NAME=chr(now.isocalendar()[0]-2010+65)+l[(now.isocalendar()[1]-1)/2] #logging.debug('DEFAULT_COUNTER_NAME: %s' % (DEFAULT_COUNTER_NAME)) counter.increment(DEFAULT_COUNTER_NAME) key_name=DEFAULT_COUNTER_NAME+str(counter.get_count(DEFAULT_COUNTER_NAME)) #logging.debug('key_name %s' % (key_name)) channelurlinstance=ChannelUrl(id=key_name,channel=channelinstance.key,url=urlinstance.key) channelurlinstance.put() #logging.debug('New channelurl %s/%s' % (channel,url))
def post(self): doc_id = self.request.get('doc_id', '') if doc_id: #logging.debug('doc_id: %s' % (doc_id)) urlinstance = Url.get_by_id(int(doc_id)) if urlinstance: # If not valid url, delete from index if urlinstance.valid < 0: doc_index = search.Index(name='url') logging.info( 'Delete invalid (%s) url (ID %s) from document index \'url\' (%s)' % (str(urlinstance.valid), doc_id, doc_index)) doc_index.delete(doc_id) else: url = urlinstance.url title = urlinstance.title #logging.debug('url: %s, title: %s' % (url, title)) channels = [] channel = None users = [] user = None date = datetime.datetime.fromtimestamp(0) comments = [] comment = None tags = [] tag = None rate = 0 channelurlquery = ChannelUrl.query( ChannelUrl.url == urlinstance.key) for channelurlinstance in channelurlquery: channelinstance = channelurlinstance.channel.get() if channelinstance.name not in channels: channels.append(channelinstance.name) #logging.info('Adding channel %s' % (channelinstance.name)) postquery = Post.query( Post.channelurl == channelurlinstance.key) for postinstance in postquery: if postinstance.user not in users: users.append(postinstance.user) if date: if date < postinstance.date: date = postinstance.date else: date = postinstance.date extraquery = Extra.query( Extra.channelurl == channelurlinstance.key) for extrainstance in extraquery: if extrainstance.tag: if extrainstance.tag not in tags: tags.append(extrainstance.tag) #logging.info('Adding tag %s' % (extrainstance.tag)) if extrainstance.comment: if extrainstance.comment not in comments: comments.append(extrainstance.comment) #logging.info('Adding comment %s' % (extrainstance.comment)) ratequery = Rate.query( Rate.channelurl == channelurlinstance.key) for rateinstance in ratequery: rate += rateinstance.value #logging.debug('rate %s' % (rate)) if not date: date = datetime.datetime.fromtimestamp(0) # lists to strings channel = ' '.join(channels) user = '******'.join(users) tag = ' '.join(tags) if not tag: tag = None comment = ' '.join(comments) if not comment: comment = None logging.debug( 'doc; channel=%s, user=%s, url=%s, date=%s, title=%s, comment=%s, tag=%s, rate=%s' % (channel, user, url, date, title, comment, tag, rate)) try: doc = search.Document( doc_id=str(doc_id), fields=[ search.TextField(name='channel', value=channel), search.TextField(name='user', value=user), search.TextField(name='url', value=url), search.DateField(name='date', value=date), search.TextField(name='title', value=title), search.TextField(name='comment', value=comment, language='fi'), search.TextField(name='tag', value=tag, language='fi'), search.NumberField(name='rate', value=rate) ], language='en') except Exception, e: logging.error('doc_id: %s, error %s' % (str(doc_id), e)) doc = None try: if doc: search.Index(name='url').put(doc) urlinstance.document_date = datetime.datetime.now() urlinstance.put() else: logging.error('Doc missing.') except search.Error: logging.error('Create Document failed.') else: logging.debug('No urlinstance for doc_id: %s' % (doc_id))
def post(self): if users.get_current_user(): node.author = users.get_current_user() post_channel = self.request.get('post_channel', '') post_user = self.request.get('post_user', '') post_url = self.request.get('post_url', '') # Add http:// when needed if not post_url.startswith('http'): post_url = 'http://' + post_url logging.debug('Post: C=%s U=%s P=%s' % (post_channel, post_user, post_url)) # 1. tarkista onko olemassa jo ko. Url, lisää jos ei, muuten päivitä (udate, valid?): valid-juttu joo ehkä jos tarpeen, ei muuten url = Url.all().filter('url =', post_url).get() if not url: url = Url() url.url = post_url url.put() # Title name = ''.join( re.findall('[a-zA-Z0-9_-]', post_channel + '_' + post_url))[:500] try: taskqueue.add(name=name, queue_name='urlfetch', url='/tasks/title', params={'post_url': post_url}) except taskqueue.TombstonedTaskError: logging.warning('TombstonedError %s' % post_url) except taskqueue.TaskAlreadyExistsError: logging.warning('TaskAlredyExists: %s' % post_url) # 2. tarkista onko olemassa jo ko. Channel, lisää jos ei channel = Channel.all().filter('name =', post_channel).get() if not channel: channel = Channel() channel.name = post_channel if post_channel.startswith('!'): channel.private = True channel.put() # 3. tarkista onko url jo olemassa channel-tasolla channelurl = ChannelUrl.all().filter('url =', url).filter('channel =', channel).get() if not channelurl: channelurl = ChannelUrl() channelurl.channel = channel channelurl.url = url #channelurl.user=post_user channelurl.put() else: logging.info('OLDIE! %s %s' % (channelurl.channel.name, channelurl.url.url)) # 4. Lisätään postaus post = Post() post.channelurl = channelurl post.user = post_user post.put()