def update_haksa():
try:
key = Post.objects.filter(department="학사공지").latest('upload_dt')
except:
key = None
if key:
latest = haksaCrawl.check_latest()
if key.title != latest:
data_dict = haksaCrawl.extract_latest_notices(key.title)
for data in data_dict:
if data['content'] != "":
print(f"{data['title']} is updated")
fb = Post(title=data['title'],
upload_dt=data['modify_dt'],
department=data['type'],
content=data['content'],
url=data['url'])
fb.save()
return latest
else:
return None
else:
data_dict = haksaCrawl.extract_indeed_notices(3)
# data_dict = haksaCrawl.extract_indeed_notices(haksaCrawl.extract_indeed_pages())
for data in data_dict:
if data['content'] != "":
fb = Post(title=data['title'],
upload_dt=data['modify_dt'],
department=data['type'],
content=data['content'],
url=data['url'])
fb.save()
return haksaCrawl.check_latest()
def post_create(request):
if request.method == "POST":
do = request.POST.get('do')
if do == 'del':
return redirect(reverse('index'))
# handle ingredient table
items = []
for i, q in zip(request.POST.getlist('item'),
request.POST.getlist('quantity')):
items.append({'item': i, 'quantity': q})
post = {
'title': request.POST.get('post-title'),
'intro': request.POST.get('post-intro'),
'items': items,
'steps': request.POST.getlist('post-step'),
'fyi': request.POST.get('post-fyi'),
}
if do == 'pub':
p = Post(creator=request.user,
post_type='R',
published_at=timezone.now,
content=json.dumps(post))
else: # sav
p = Post(creator=request.user,
post_type='R',
content=json.dumps(post))
p.save()
return redirect(reverse('post_view', args=[p.id]))
else:
return render(request, 'post/create.html')
def update_fasion():
try:
key = Post.objects.filter(department="의류학과").latest('upload_dt')
except:
key = None
if key:
latest = fashionCrawl.fashion_check_latest()
if key.title != latest:
data_dict = fashionCrawl.fashion_extract_latest_notices(key.title)
for data in data_dict:
if data['content'] != "":
fb = Post(title=data['title'],
upload_dt=data['modify_dt'],
department=data['type'],
content=data['content'],
url=data['url'])
fb.save()
return latest
else:
return None
else:
# data_dict = fashionCrawl.fashion_extract_indeed_notices(3)
data_dict = fashionCrawl.fashion_extract_indeed_notices(
fashionCrawl.fashion_extract_indeed_pages())
for data in data_dict:
if data['content'] != "":
fb = Post(title=data['title'],
upload_dt=data['modify_dt'],
department=data['type'],
content=data['content'],
url=data['url'])
fb.save()
return fashionCrawl.fashion_check_latest()
def update_korean():
try:
key = Post.objects.filter(department="국어국문학과").latest('upload_dt')
except:
key = None
if key:
latest = korean_crawl.check_latest()
if key.title != latest:
data_dict = korean_crawl.extract_latest_notices(key.title)
for data in data_dict:
if data['content'] != "":
fb = Post(title=data['title'],
upload_dt=data['modify_dt'],
department=data['type'],
content=data['content'],
url=data['url'])
fb.save()
return latest
else:
return None
else:
data_dict = korean_crawl.extract_korean_notices(3)
# data_dict = korean_crawl.extract_korean_notices(korean_crawl.extract_last_pages())
for data in data_dict:
if data['content'] != "":
fb = Post(title=data['title'],
upload_dt=data['modify_dt'],
department=data['type'],
content=data['content'],
url=data['url'])
fb.save()
return korean_crawl.check_latest()
def addPost(host):
try:
if session.get('username') != host:
return render_template('notlogin.html')
else:
try:
sql = 'SELECT * FROM users WHERE user_name = %s;'
parm = (host, )
rows = User().get_User(sql, parm)
hostid = rows[0]
print(hostid)
content = request.form['postbox']
except:
traceback.print_exc()
return render_template('error1.html')
if content.strip() == '':
error = 'You can not send nothing!'
# can not send nothing
try:
sql = 'SELECT * FROM message WHERE user_id = %s ORDER BY message_id DESC;'
parm = (hostid, )
posts = Post().get_AllPost(sql, parm)
except:
traceback.print_exc()
return render_template('error1.html')
return render_template('homeopage.html',
hosts=rows,
posts=posts,
error=error)
else:
try:
# insert a post
sql_add = 'INSERT INTO message (message_info,message_time,user_id) VALUES (%s,%s,%s);'
# get now time
import datetime
now = datetime.datetime.now()
otherStyleTime = now.strftime("%Y-%m-%d %H:%M:%S")
parm_add = (request.form['postbox'], otherStyleTime,
hostid)
Post().set_Post(sql_add, parm_add)
except:
conn = connect_db()
conn.rollback()
conn.close()
traceback.print_exc()
return render_template('error1.html')
return redirect(url_for('home', host=host))
except:
traceback.print_exc()
return render_template('error.html')
def setUp(self):
self.author = User.objects.create(username='******',
password=make_password('secret'),
email='*****@*****.**')
self.viewer = User.objects.create(username='******',
password=make_password('secret'),
email='*****@*****.**')
self.post = Post(creator=self.author,
published_at=timezone.now(),
content="{}")
self.draft = Post(creator=self.author, content="{}")
self.post.save()
self.draft.save()
def test_metadb_build(self):
test_cat = Category(name='Test')
test_cat2 = Category(name='Test Posts')
db.session.add(test_cat)
db.session.add(test_cat2)
db.session.commit()
test_post = Post(title='One', path='',
date=datetime.datetime.now(),
categories=[test_cat, test_cat2])
test_post2 = Post(title='Two', path='',
date=datetime.datetime.now(),
categories=[test_cat])
db.session.add(test_post)
db.session.add(test_post2)
db.session.commit()
def comment(postid, host):
try:
if session.get('username') != host:
return render_template('notlogin.html')
else:
try:
# 获取post内容，在评论页面显示
sql1 = 'SELECT * FROM message WHERE message_id = %s;'
parm1 = (postid, )
rows = Post().get_Post(sql1, parm1)
post = rows[1]
posttime = rows[2]
# 获取发post的用户信息，传递给页面
posthostid = rows[6]
sql2 = 'SELECT * FROM users WHERE user_id = %s;'
parm2 = (posthostid, )
row = User().get_User(sql2, parm2)
posthost = row[1]
posthostpic = row[6]
# 获取post的所有评论，把信息返回给评论页面
sql2 = 'SELECT comment.*,users.userpic FROM comment,users WHERE message_id = %s AND users.user_id = comment.user_id ORDER BY comment_id DESC;;'
parm2 = (postid, )
comms = Comment().get_AllComment(sql2, parm2)
# 查询post的评论数量
sql3 = 'SELECT COUNT(user_id) FROM comment WHERE message_id = %s;'
parm3 = (postid, )
commnum = Comment().get_Comment(sql3, parm3)
# 更新到数据库
sql4 = 'UPDATE message SET message_commentnum = %s WHERE message_id = %s;'
parm4 = (commnum[0], postid)
Post().set_Post(sql4, parm4)
except:
conn = connect_db()
conn.rollback()
conn.close()
traceback.print_exc()
return render_template('error1.html')
return render_template('comments.html',
postid=postid,
host=host,
post=post,
posthost=posthost,
posthostpic=posthostpic,
posttime=posttime,
comms=comms)
except:
traceback.print_exc()
return render_template('error.html')
def editPost(postid, host):
try:
if session.get('username') != host:
return render_template('notlogin.html')
else:
content = request.form['posteditbox']
if content.strip() == '':
error = 'you left nothing'
return render_template('post_edit.html',
host=host,
postid=postid,
error=error)
else:
try:
# update post to datebase
sql = 'UPDATE message SET message_info = %s WHERE message_id = %s;'
parm = (request.form['posteditbox'], postid)
Post().set_Post(sql, parm)
except:
conn = connect_db()
conn.rollback()
conn.close()
traceback.print_exc()
return render_template('error1.html')
return redirect(url_for('home', host=host))
except:
traceback.print_exc()
return render_template('error.html')
def analysis_view(request):
if request.method == "POST":
body_unicode = request.body.decode('utf-8')
body = json.loads(request.body)
print(request.body)
content = body['b64_image']
filename = str(dateformat.format(timezone.now(), 'Y-m-d_H-i-s')) + '.jpg'
post = Post()
post.image = ContentFile(base64.b64decode(content), name=filename)
post.name = filename
post.save()
object, result_data, total_percent = analysis.draw_line(post.pk)
user_image = base64.b64encode(post.image.read())
result_image = base64.b64encode(object.result.read())
print(test)
data = {
"message" : str(result_data[0]),
"image_name" : post.image.name.split('.')[0],
"image" : str(user_image),
"result" : str(result_image)
}
return JsonResponse(data)
else:
return HttpResponse("get is worng request")
def deletePostlist(postid, host):
try:
if session.get('username') != host:
return render_template('notlogin.html')
else:
try:
#delete post
sql_del = 'DELETE FROM message WHERE message_id = %s;'
parm_del = (postid, )
Post().set_Post(sql_del, parm_del)
#udate the number of post
sql_update = 'UPDATE users SET postnum = postnum - 1 WHERE user_name = %s;'
parm = (host, )
User().set_User(sql_update, parm)
#delete the like of post
sql_del1 = 'DELETE FROM likes WHERE message_id = %s;'
parm_del1 = (postid, )
Like().del_Like(sql_del1, parm_del1)
#delete comments of post
sql_del2 = 'DELETE FROM comment WHERE message_id = %s;'
parm_del2 = (postid, )
Comment().set_Comment(sql_del2, parm_del2)
except:
conn = connect_db()
conn.rollback()
conn.close()
traceback.print_exc()
return render_template('error1.html')
return redirect(url_for('postlist', host=host))
except:
traceback.print_exc()
return render_template('error.html')
def handle(self, *args, **options):
from bs4 import BeautifulSoup
import requests
url = "https://doroshenkoaa.ru/med/"
response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")
links_list = []
for link in soup.find_all("h2", "title", "a"):
for tmp in link.find_all("a"):
links_list.append(tmp.get("href"))
for link in links_list:
response = requests.get(link)
soup = BeautifulSoup(response.text, "html.parser")
article_title = soup.find("h1", {
"itemprop": "headline"
}).text.strip()
article_content = ""
for p in soup.find_all("div", {"itemprop": "articleBody"}, "p"):
article_content += p.text
Post(title=article_title, content=article_content).save()
def like(postid, host):
if session.get('username') != host:
return render_template('notlogin.html')
else:
# 查找当前用户ID
print(postid, host)
sql = 'SELECT * FROM users WHERE user_name = %s;'
parm = (host, )
rows = User().get_User(sql, parm)
hostid = rows[0]
# 如果用户没有点过赞，那么添加一条点赞
sql_search = 'SELECT * FROM likes WHERE message_id = %s AND user_id = %s;'
parm = (postid, hostid)
result = Like().get_Like(sql_search, parm)
if result is not None:
# 对应post点赞数-1
sql1 = 'SELECT * FROM message WHERE message_id = %s;'
parm1 = (postid, )
rows = Post().get_Post(sql1, parm1)
likenum = rows[4]
likenew = likenum - 1
# 更新点赞数的值
sql2 = 'UPDATE message SET message_likenum = %s WHERE message_id = %s;'
parm2 = (likenew, postid)
Post().set_Post(sql2, parm2)
# 删除点赞信息
sql_del = 'DELETE FROM likes WHERE message_id = %s and user_id = %s;'
parm_del = (postid, hostid)
Like().del_Like(sql_del, parm_del)
return redirect(url_for('home', host=host))
else:
sql_add = 'INSERT INTO likes (message_id,user_id) VALUES (%s,%s);'
parm_add = (postid, hostid)
Like().add_Like(sql_add, parm_add)
# 对应post点赞数+1
sql1 = 'SELECT * FROM message WHERE message_id = %s;'
parm1 = (postid, )
rows = Post().get_Post(sql1, parm1)
likenum = rows[4]
likenew = likenum + 1
# 更新点赞数的值
sql2 = 'UPDATE message SET message_likenum = %s WHERE message_id = %s;'
parm2 = (likenew, postid)
Post().set_Post(sql2, parm2)
return redirect(url_for('home', host=host))
def crwal_blog_data(self, blog_id, ismanual=False):
#初始化数据集
_set_task_progress(self=self, pr=1)
blog = Blog.query.filter_by(id=blog_id).first()
a_list_rule = {
"p_home": blog.crawl.get("p_home"),
"p_page": blog.crawl.get("p_page"),
"p_title": blog.crawl.get("p_title"),
"p_link": blog.crawl.get("p_link")
}
#加载urlset
url_set_json = blog.crawl.urlset
url_set = None
if url_set_json:
url_set = set(json.loads(url_set_json))
#爬虫执行
a_list = get_article_list(self=self, abs_url=blog.homepage, **a_list_rule)
#爬虫处理
new_a_list = a_list
if url_set:
new_a_list = [(t, l) for t, l in a_list
if hash_md5_text(l) not in url_set]
#进度操作
_set_task_progress(self=self, pr=50)
a_len = len(new_a_list)
#保存文章列表
if blog.big_categor_id == 1:
recommend = True
else:
recommend = False
save_post = [
Post(url=l, title=t, blog=blog, recommend=recommend)
for t, l in new_a_list
]
db.session.add_all(save_post)
#保存url_md5
blog.crawl.urlset = create_md5s_json([l for t, l in a_list])
db.session.commit()
#获取正文Body
i = 0
for p in save_post:
i += 1
a_body = get_article_body(p.url, blog.crawl.get("p_body"))
p.body = a_body
p.digest = pq(a_body).text()[:300]
_set_task_progress(self=self, pr=50 + 50 * i // a_len)
#完成结束
blog.crawl.taskdone = True
db.session.commit()
_set_task_progress(self=self, pr=100)
def mail(modeladmin, request, queryset):
post = Post()
post.save()
for user in queryset:
post.users.add(user)
return redirect(f'/main/post/{post.pk}/change/')
def make_post():
for i in range(50):
db.session.add(
Post(title=fake.text(max_nb_chars=16, ext_word_list=None),
body=fake.text(max_nb_chars=200, ext_word_list=None),
recommend=True,
blog_id=6))
db.session.commit()
def mutate(self, info, title, body, username):
user = User.query.filter_by(username=username).first()
post = Post(title=title, body=body)
if user is not None:
post.author = user
db.session.add(post)
db.session.commit()
return CreatePost(post=post)
def post_all():
"""Post All."""
key = Post().__class__.cache_key()
if key in cache:
objects_all = cache.get(key)
else:
objects_all = Post.objects.all()
cache.set(key, objects_all, 30)
return objects_all
def create(): # 새글 추가시 post_id가 필요없음
form = PostForm()
# breakpoint()
if request.method == 'POST' and form.validate_on_submit():
post = Post(subject=form.subject.data, content=form.content.data, create_date=datetime.now(), user=g.user)
db.session.add(post)
db.session.commit()
return redirect(url_for('main.index'))
return render_template('post/post_form.html', form=form)
def new_post():
form = PostForm()
if form.validate_on_submit():
post = Post(title=form.title.data, content=form.content.data, author=current_user)
db.session.add(post)
db.session.commit()
flash('Your post has been created!', 'success')
return redirect(url_for("main.images"))
return render_template("create_post.html", form=form, legend='New Post')
def dtf_main():
names = dtf_names()
links = dtf_links()
conts = dtf_content()
dts = []
date = dtf_date()
time = dtf_time()
site = "dtf"
#"https://leonardo.osnova.io/91e47474-c70d-55ad-af16-b3bc2335e282/"
chck = re.compile(r'\"https\:\/\/.*\/\"')
url = 'https://dtf.ru/gameindustry/entries/new'
img_urls = []
res = requests.get(url)
soup = bs4.BeautifulSoup(res.text, "html.parser")
out = soup.find_all('div', class_='content-image')
urls = []
for item in out:
mo = chck.search(str(item))
img_urls.append(mo.group()[1:-1])
del img_urls[10:]
i = 0
for item in date:
dts.append(str(item) + " " + time[i])
i = i + 1
i = 1
for item in img_urls:
res = requests.get(item)
img_file = open(
os.path.join(
'D:\\agregator\\gamers_gazette\\game_news_site\\media\\images',
'dtf{}.png'.format(i)), 'wb')
urls.append('images/dtf{}.png'.format(i))
i = i + 1
for chunk in res.iter_content(100000):
img_file.write(chunk)
img_file.close()
i = 1
for i in range(0, 10):
p = Post(site=site,
title=names[i],
img=urls[i],
pub_date=dts[i],
time=time[i],
date=date[i],
text=conts[i])
p.save()
def comment_list(request):
if request.method == 'GET':
comments = Comment.objects.all()
comments = [c.to_json() for c in comments]
return JsonResponse(comments, safe=False)
elif request.method == 'POST':
data = json.loads(request.body)
post = Post(content=data['content'], date=data['date'])
post.save()
return JsonResponse(post.to_json())
def add_post():
form = PostForm()
if form.validate_on_submit():
post = Post(title=form.title.data, content=form.content.data)
post.slug_(str(form.title.data))
db.session.add(post)
db.session.commit()
return redirect(url_for('index'))
return render_template('add_post.html', form=form)
def new_question():
form = QuestionForm()
if form.validate_on_submit():
question = Post(title = form.title.data, content=form.content.data, author=current_user)
db.session.commit()
flash('Your post has been created!', 'success')
question = Post.query.filter_by(title = form.title.data, author=current_user).all()[-1]
insert_data(question.title, question.id)
return redirect(url_for('home'))
return render_template('create_question.html', title='New Question', form=form, legend = 'Ask Question')
def home(host):
if session.get('username') == host:
# get infomation of host
sql = 'SELECT * FROM users WHERE user_name = %s;'
parm = (host, )
hosts = User().get_User(sql, parm)
# get infomation the posts of host and friend
sql1 = 'SELECT message.*,users.user_name,users.userpic FROM message,relation,users WHERE relation.user_id = %s AND message.user_id = relation.follow_id AND message.user_id = users.user_id;'
parm1 = (hosts[0], )
posts1 = Post().get_AllPost(sql1, parm1)
sql2 = 'SELECT message.*,users.user_name,users.userpic FROM message,users WHERE message.user_id = %s and message.user_id = users.user_id;'
parm2 = (hosts[0], )
posts2 = Post().get_AllPost(sql2, parm2)
posts = sorted(posts1 + posts2, reverse=True)
return render_template('homeopage.html', hosts=hosts, posts=posts)
else:
return render_template('notlogin.html')
def add_post(request):
if request.method == 'POST':
form = PostForm(request.POST)
if form.is_valid():
post_item = form.save(commit=False)
post_item.author = request.user
post_item.save()
return redirect('post_view', pk=post_item.pk)
else:
form = PostForm(instance=Post())
return render(request, 'post_edit.html', {'form': form, 'action': 'add'})
def postList(host):
#get infomation of host
sql = 'SELECT * FROM users WHERE user_name = %s;'
parm = (host, )
hosts = User().get_User(sql, parm)
#get infomation of posts
sql1 = 'SELECT * FROM message WHERE user_id = %s ORDER BY message_id DESC;'
parm = (hosts[0], )
posts = Post().get_AllPost(sql1, parm)
return render_template('postlist.html', hosts=hosts, posts=posts)
def index():
lform = login_form()
rform = register_form()
nform = newpost_form()
if lform.lsubmit.data and lform.validate_on_submit():
user = User.query.filter_by(email=lform.lemail.data).first()
if user and bcrypt.check_password_hash(user.password,
lform.lpassword.data):
login_user(user)
nextpage = request.args.get("next")
flash("شما وارد شدید", category="success")
if nextpage:
return redirect(nextpage)
else:
return redirect(url_for("index"))
else:
flash("ایمیل و پسورد را دوباره چک کنید", category="danger")
return redirect(url_for("index"))
if rform.rsubmit.data and rform.validate_on_submit():
hashed_password = bcrypt.generate_password_hash(rform.rpassword.data)
user = User(firstname=rform.firstname.data,
lastname=rform.lastname.data,
email=rform.remail.data,
password=hashed_password)
db.session.add(user)
db.session.commit()
# send_email(user)
flash("ثبت نام شما با موفقیت انجام شد.", category="success")
return redirect(url_for("index"))
if current_user.is_authenticated == True:
postslen = len(current_user.posts)
page = request.args.get("page", 1, int)
posts = Post.query.filter_by(author=current_user).order_by(
Post.id.desc()).paginate(page=page, per_page=5)
else:
postslen = 0
posts = None
if nform.nsubmit.data and nform.validate_on_submit():
post = Post(title=nform.title.data,
date=nform.date.data,
time=nform.time.data,
user_id=current_user.id)
db.session.add(post)
db.session.commit()
flash("پست شما افزوده شد", category="success")
return redirect(url_for("index"))
return render_template("index.html",
lform=lform,
rform=rform,
nform=nform,
title="صفحه اصلی",
posts=posts,
postslen=postslen)
def new_post():
form = PostForm()
if form.validate_on_submit():
post = Post(title=form.title.data,
content=form.content.data,
author=current_user)
db.session.add(post)
db.session.commit()
flash('Blog added to community', 'success')
return redirect(url_for('home'))
return render_template('new_post.html', form=form, titlt="NEW BLOG")
def test_follow_posts(self):
# create four users
u1 = User(username='******', email='*****@*****.**')
u2 = User(username='******', email='*****@*****.**')
u3 = User(username='******', email='*****@*****.**')
u4 = User(username='******', email='*****@*****.**')
db.session.add_all([u1, u2, u3, u4])
# create four posts
now = datetime.utcnow()
p1 = Post(body="post from john",
author=u1,
timestamp=now + timedelta(seconds=1))
p2 = Post(body="post from susan",
author=u2,
timestamp=now + timedelta(seconds=4))
p3 = Post(body="post from mary",
author=u3,
timestamp=now + timedelta(seconds=3))
p4 = Post(body="post from david",
author=u4,
timestamp=now + timedelta(seconds=2))
db.session.add_all([p1, p2, p3, p4])
db.session.commit()
# setup the followers
u1.follow(u2) # john follows susan
u1.follow(u4) # john follows david
u2.follow(u3) # susan follows mary
u3.follow(u4) # mary follows david
db.session.commit()
# check the followed posts of each user
f1 = u1.followed_posts().all()
f2 = u2.followed_posts().all()
f3 = u3.followed_posts().all()
f4 = u4.followed_posts().all()
self.assertEqual(f1, [p2, p4, p1])
self.assertEqual(f2, [p2, p3])
self.assertEqual(f3, [p3, p4])
self.assertEqual(f4, [p4])
