def scrap_article(url: str, source: Source):
"""
This method extract article from web
:param url: URL of article
:param source: Source of Article
:return: Article
"""
logger.info('Scrapping %s', url)
response = requests.get(url)
if response.status_code == 200:
soup = bs4.BeautifulSoup(response.text, 'html.parser')
article = Article()
article.uuid = hashlib.md5(bytes(url.encode())).hexdigest()
article.url = url
article.source = source
try:
article.title = soup.select('.' +
source.article_title_class)[0].text
article.body = soup.select('.' + source.article_body_class)[0].text
article.image = soup.select('.' +
source.article_image_class)[0]['src']
article.save()
return article
except IndexError:
logger.error('Error while scrapping article %s', url)
return None
except ValidationError:
logger.error('Error while validating article %s', url)
return None
return None
def save_clone_page(request):
save_article = Article()
save_data = Cache().get(request.POST.get('save_key'))
save_article.group = ArticleGroup.objects.get(groupid=0)
save_article.title = save_data['title']
save_article.comment = '转发文章'
save_article.summary = '本文转自{0}，侵权删除'.format(save_data['url'])
save_article.context = save_data['article']
save_article.save()
if 'pic_source' in save_data.keys():
for item in save_data['pic_source']:
try:
Image.objects.get(name=str(item))
except Image.DoesNotExist:
save_img = Image()
save_img.name = item
save_img.path.save(
save_img.name,
File(open(os.path.join(PIC_TMP_PATH, item), 'rb')))
save_img.save()
save_article.image.add(save_img)
if 'code_source' in save_data.keys():
need_js = Script.objects.get(name='prettify.js')
need_css = Script.objects.get(name='prettify.css')
save_article.script.add(need_js, need_css)
Cache().remove('tops')
return save_article.articleid
def publish_article():
if request.method == 'POST':
title = request.form.get('title')
content = request.form.get('content')
uid = request.form.get('uid')
# 添加文章
article = Article()
article.title = title
article.content = content
article.user_id = uid
db.session.add(article)
db.session.commit()
return '添加成功！'
else:
users = User.query.filter(User.isdelete == False).all()
return render_template('article/add_article.html', users=users)
def publish_article():
if request.method == 'POST':
title = request.form.get('title')
type_id = request.form.get('type')
content = request.form.get('content')
article = Article()
article.title = title
article.type_id = type_id
article.content = content
article.user_id = g.user.id
db.session.add(article)
db.session.commit()
return redirect(url_for('user.index'))
def publish_article():
if request.method == 'POST':
title = request.form.get('title')
type_id = request.form.get('type')
content = request.form.get('content')
article = Article()
article.title = title
article.type_id = type_id
article.content = content.encode('utf-8')
article.userId = g.user.id
db.session.add(article)
db.session.commit()
return redirect(url_for('user.index'))
return render_template('user/user_center.html')
def publish_article():
if request.method == 'POST':
title = request.form.get('title')
type_id = request.form.get('type')
content = request.form.get('content')
# print(title)
# print(type_id)
# print(content)
# print(g.user.id)
# 添加文章
article = Article()
article.title = title
article.content = content
article.type_id = type_id
#
article.user_id = g.user.id
#
db.session.add(article)
db.session.commit()
# return '发表成功！'
return redirect(url_for('user.index'))
def create_article(self):
return Article(code='0001A',
description='I Have Hungry',
price=90.12,
cost=2.87).save()
def post_receiver(request):
response = {'status': 200, 'message': 'ok'}
content = "<!--Markdown-->\n" + request.POST.get('content', '')
if not request.POST.has_key('title'):
if not request.POST.has_key('commentType'):
return JsonResponse({
'status': 500,
'message': 'Article title required.'
})
else:
if not request.POST.has_key('commentId'):
return JsonResponse({
'status': 500,
'message': 'Comment id required.'
})
comment_type = request.POST.get('commentType', None)
if comment_type not in ['a', 'c', 'r']:
return JsonResponse({
'status': 500,
'message': 'Comment type unsupported.'
})
comment_id = request.POST.get('commentId', None)
if comment_type == 'a':
to_user = None
to_reply = None
topic = Article.objects.get(tid=comment_id)
else:
comment = Reply.objects.get(rid=comment_id)
to_user = comment.author
topic = comment.topic
if comment_type == 'c':
to_reply = comment
else:
to_reply = comment.to_reply
reply = Reply(author=request.user,
author_username=request.user.username,
to_user=to_user,
time=timezone.now(),
content=content,
topic=topic,
to_reply=to_reply,
status=0)
reply.save()
return JsonResponse({'status': 200})
else:
title = request.POST.get('title', '')
if title == '':
return JsonResponse({
'status': 500,
'message': 'Article title required.'
})
article_type = request.POST.get('type', 0)
pid = 0
if article_type == 0:
pass
else:
pid = request.POST.get('pid', None)
print(pid)
if not pid:
return JsonResponse({
'status': 500,
'message': 'problem id required.'
})
try:
article = Article(content=content,
pid=pid,
title=title,
author=request.user,
updated_at=timezone.now(),
status=0,
top_level=0)
article.save()
except:
response['status'] = 500
response['message'] = 'unknown error.'
return JsonResponse(response)
def post(self, request, *args, **kwargs):
article = Article(author=request.user,
text=request.POST.get('text'),
img=request.FILES.get('images') or None)
article.save()
return redirect('home:home_page')
def create_article(self, code, price):
return Article(code=code,
description='I Have Hungry',
price=price,
cost=2.87).save()
def handle(self, *args, **options):
path = options['path'][0]
list_multiple = []
list_doesnt_exist = []
with open(path, 'r') as csvfile:
spamreader = csv.reader(csvfile, delimiter=' ')
list_wrong_date = []
author = User.objects.get(username='******')
row_count = 0
for row in spamreader:
regex_result = url_pattern.match(row[2])
article_year = int(regex_result.group("year"))
article_month = int(regex_result.group("month"))
article_slug = regex_result.group("slug")
# date_result = date_pattern.match(row[0])
# date_year = int(date_result.group("year"))
# date_month = int(date_result.group("month"))
# date_day = date_result.group("day")
article = get_article(article_year, article_month,
article_slug)['article']
article_date = str(article_year) + "-" + (
str(article_month) if +article_month > 9 else "0" +
str(article_month)) + "-" + "01 21:00:00.000000"
if not article:
article = Article(author=author,
createdin=article_date,
updatein=article_date,
publishin=article_date,
first_slug=article_slug,
title=row[1],
slug=article_slug,
text=row[3],
status=4)
row_count += 1
article.save()
save_feed_item(article, {})
else:
article.createdin = article_date
article.updatein = article_date
article.publishin = article_date
article.slug = article_slug
article.first_slug = article_slug
article.author = author
article.save()
