def add(board, thread, post, data): cache = Rss.load("thread", board, thread) if not cache: cache = Rss.create("thread", board, thread) rendered = render_template("rss_post.xml", board = board, thread = thread, data = escape(data), post = post, cache = cache, ) # FIXME: cant save long posts (>500) if len(rendered) > 500: rendered = render_template("rss_post.xml", board = board, thread = thread, data = "too long", post = post, cache = cache, ) cache.posts.append(rendered) cache.posts = cache.posts[-20:] xml = render_template("rss.xml", board = board, thread = thread, cache=cache) cache.xml = xml.encode("utf8") cache.put()
def save_rss(sort, rss_json): counter = 0 for rss in rss_json: if not Rss.objects.filter(hash_code=rss['hash']): new_rss = Rss(title=rss['title'], link=rss['url'], hash_code=rss['hash'], sort=sort, timestamp=datetime.now()) new_rss.save() #analysis_tags(new_rss) counter += 1 else: break if counter > 0: # send notification to analysis tags pass return counter
def parse_rss(self): # self.remove_old_history() rss_list = Rss.select() rss_card_dict = {} post_url_list = [ rss_history.url for rss_history in History.select().where( History.publish_at == datetime.today().strftime("%Y-%m-%d")) ] for rss in rss_list: rss_history_list = [] card_list = [ CardItem(title=rss.title, url=rss.url, pic_url=rss.cover) ] feed = feedparser.parse(rss.feed) for entry in feed.entries: if entry.link not in post_url_list and self.is_today(entry): card_list.append( CardItem( title=f'{entry.title}', url=entry.link, pic_url= 'https://ftp.bmp.ovh/imgs/2020/07/6cdb9f606677c9e3.jpg' )) rss_history_list.append(History(url=entry.link)) if len(card_list) > 1: rss_card_dict[rss.title] = card_list with db.atomic(): History.bulk_create(rss_history_list, batch_size=10) return rss_card_dict
def get(self, board, thread): cache = Rss.load("thread", board, thread) if not cache: raise NotFound() return Response(cache.xml, content_type="application/rss+xml; charset=UTF-8")
def new(request): if request.method != 'POST': return HttpResponse(status=405) else: form = SubscribeForm(request.POST) if form.is_valid(): email = form.cleaned_data['email'] url = form.cleaned_data['feed_link'] try: rss_url = Rss.objects.get(link=url) except Rss.DoesNotExist: rss_url = Rss(link=url) rss_url.save() token = random_string(24) subscribe = Subscription(rss=rss_url, email=email, active=False, token=token) try: subscribe.save() except IntegrityError: return response_json({'status': 1, 'message': 'subscribe already exists'}) #письмо активации подписчику params = {'email': email.encode('utf8'), 'rss': url.encode('utf8'), 'activate_link': '%s%s' % (re.sub(r'/$', '', settings.SERVICE_EXTERNAL_ROOT_URL), reverse(manage, args=['activate', subscribe.id, subscribe.token]))} m_task = MailTask(subscribe=subscribe, rss=rss_url, title=settings.MAIL_ACTIVATE_TITLE, message=settings.MAIL_ACTIVATE_TEMPLATE % params) m_task.save() return response_json({'status': 0, 'message': 'subscribe added'}) else: return response_json({'status': 1, 'message': 'form is invalid', 'form_errors': form.errors})
def main(url: str): """Точка входа""" uri: ParseResult = urlparse(url) log.debug('%s', uri) module = uri.netloc.replace('.', '_') # def parser(uri: ParseResult) -> List[Item]: result = getattr(__import__('parsers.{}'.format(module)), module).parser(uri) xml = Rss(link=url, items=result).to_xml() return '<?xml version="1.0" encoding="utf-8"?>' + etree.tostring( xml).decode()