def add(board, thread, post, data):
cache = Rss.load("thread", board, thread)
if not cache:
cache = Rss.create("thread", board, thread)
rendered = render_template("rss_post.xml",
board = board,
thread = thread,
data = escape(data),
post = post,
cache = cache,
)
# FIXME: cant save long posts (>500)
if len(rendered) > 500:
rendered = render_template("rss_post.xml",
board = board,
thread = thread,
data = "too long",
post = post,
cache = cache,
)
cache.posts.append(rendered)
cache.posts = cache.posts[-20:]
xml = render_template("rss.xml",
board = board, thread = thread, cache=cache)
cache.xml = xml.encode("utf8")
cache.put()
def save_rss(sort, rss_json):
counter = 0
for rss in rss_json:
if not Rss.objects.filter(hash_code=rss['hash']):
new_rss = Rss(title=rss['title'], link=rss['url'],
hash_code=rss['hash'], sort=sort, timestamp=datetime.now())
new_rss.save()
#analysis_tags(new_rss)
counter += 1
else:
break
if counter > 0:
# send notification to analysis tags
pass
return counter
def parse_rss(self):
# self.remove_old_history()
rss_list = Rss.select()
rss_card_dict = {}
post_url_list = [
rss_history.url for rss_history in History.select().where(
History.publish_at == datetime.today().strftime("%Y-%m-%d"))
]
for rss in rss_list:
rss_history_list = []
card_list = [
CardItem(title=rss.title, url=rss.url, pic_url=rss.cover)
]
feed = feedparser.parse(rss.feed)
for entry in feed.entries:
if entry.link not in post_url_list and self.is_today(entry):
card_list.append(
CardItem(
title=f'{entry.title}',
url=entry.link,
pic_url=
'https://ftp.bmp.ovh/imgs/2020/07/6cdb9f606677c9e3.jpg'
))
rss_history_list.append(History(url=entry.link))
if len(card_list) > 1:
rss_card_dict[rss.title] = card_list
with db.atomic():
History.bulk_create(rss_history_list, batch_size=10)
return rss_card_dict
def get(self, board, thread):
cache = Rss.load("thread", board, thread)
if not cache:
raise NotFound()
return Response(cache.xml,
content_type="application/rss+xml; charset=UTF-8")
def new(request):
if request.method != 'POST':
return HttpResponse(status=405)
else:
form = SubscribeForm(request.POST)
if form.is_valid():
email = form.cleaned_data['email']
url = form.cleaned_data['feed_link']
try:
rss_url = Rss.objects.get(link=url)
except Rss.DoesNotExist:
rss_url = Rss(link=url)
rss_url.save()
token = random_string(24)
subscribe = Subscription(rss=rss_url,
email=email,
active=False,
token=token)
try:
subscribe.save()
except IntegrityError:
return response_json({'status': 1, 'message': 'subscribe already exists'})
#письмо активации подписчику
params = {'email': email.encode('utf8'),
'rss': url.encode('utf8'),
'activate_link': '%s%s' % (re.sub(r'/$', '', settings.SERVICE_EXTERNAL_ROOT_URL),
reverse(manage, args=['activate', subscribe.id, subscribe.token]))}
m_task = MailTask(subscribe=subscribe,
rss=rss_url,
title=settings.MAIL_ACTIVATE_TITLE,
message=settings.MAIL_ACTIVATE_TEMPLATE % params)
m_task.save()
return response_json({'status': 0, 'message': 'subscribe added'})
else:
return response_json({'status': 1, 'message': 'form is invalid', 'form_errors': form.errors})
def main(url: str):
"""Точка входа"""
uri: ParseResult = urlparse(url)
log.debug('%s', uri)
module = uri.netloc.replace('.', '_')
# def parser(uri: ParseResult) -> List[Item]:
result = getattr(__import__('parsers.{}'.format(module)),
module).parser(uri)
xml = Rss(link=url, items=result).to_xml()
return '<?xml version="1.0" encoding="utf-8"?>' + etree.tostring(
xml).decode()
