def handle_add_bookmark(request):
put = request.REQUEST
bm = models.Bookmark()
is_public = put.get('is_public')
if is_public == 'False':
bm.is_public = False
else:
bm.is_public = True
bm.description = put.get('description')
bm.title = put.get('title')
bm.user_id = int(put.get('user_id'))
bm.username = put.get('username')
try:
bm.save()
except Exception as e:
logerror(e.message)
return HttpResponseBadRequest
if not bm.is_public:
loginfo("Bookmark added")
return HttpResponse(json.dumps({'bookmark_id': bm.id}))
raw_tags = models.split_title_on_tags(bm.title)
tags = models.get_tags(raw_tags)
for tag in tags:
bm_tag = models.BookmarkTag()
bm_tag.tag = tag
bm_tag.bookmark = bm
bm_tag.save()
loginfo("Bookmark added")
return HttpResponse(json.dumps({'bookmark_id': bm.id}))
def subscribe(request):
get = request.REQUEST
subscriber_id = int(get.get('subscriber'))
owner_id = int(get.get('owner'))
ans = local_is_subscribed(owner_id, subscriber_id)
if ans:
loginfo("success")
return HttpResponse()
try:
sbs = models.Subscriber()
sbs.subscriber = subscriber_id
sbs.owner = owner_id
sbs.save()
except Exception as e:
logerror(e.message)
return HttpResponseServerError
records = models.Record.objects.filter(owner_id=owner_id)
for record in records:
news = models.SubscriberNews()
news.record = record
news.subscriber = sbs
news.save()
loginfo("success")
return HttpResponse()
def subscribe(request):
get = request.REQUEST
subscriber_id = int(get.get('subscriber'))
owner_id = int(get.get('owner'))
ans = local_is_subscribed(owner_id, subscriber_id)
if ans:
loginfo("success")
return HttpResponse()
try:
sbs = models.Subscriber()
sbs.subscriber = subscriber_id
sbs.owner = owner_id
sbs.save()
except Exception as e:
logerror(e.message)
return HttpResponseServerError
records = models.Record.objects.filter(owner_id=owner_id)
for record in records:
news = models.SubscriberNews()
news.record = record
news.subscriber = sbs
news.save()
loginfo("success")
return HttpResponse()
def handle_add_bookmark(request):
put = request.REQUEST
bm = models.Bookmark()
is_public = put.get('is_public')
if is_public == 'False':
bm.is_public = False
else:
bm.is_public = True
bm.description = put.get('description')
bm.title = put.get('title')
bm.user_id = int(put.get('user_id'))
bm.username = put.get('username')
try:
bm.save()
except Exception as e:
logerror(e.message)
return HttpResponseBadRequest
if not bm.is_public:
loginfo("Bookmark added")
return HttpResponse(json.dumps({'bookmark_id': bm.id}))
raw_tags = models.split_title_on_tags(bm.title)
tags = models.get_tags(raw_tags)
for tag in tags:
bm_tag = models.BookmarkTag()
bm_tag.tag = tag
bm_tag.bookmark = bm
bm_tag.save()
loginfo("Bookmark added")
return HttpResponse(json.dumps({'bookmark_id': bm.id}))
def add_bookmark(request):
par = request.REQUEST
user_id = int(par.get('user_id'))
bm_id = int(par.get('bookmark_id'))
title = par.get('title')
record = models.Record()
record.bookmark_id = bm_id
record.title = title
record.owner_id = user_id
record.save()
loginfo("record created")
subscribers = models.Subscriber.objects.filter(owner=user_id)
for user in subscribers:
link = models.SubscriberNews()
link.record = record
link.subscriber = user
link.save()
loginfo("news created")
return HttpResponse()
def get_news(request):
get = request.GET
page = int(get.get('page'))
per_page = int(get.get('per_page'))
subscriber = int(get.get('user_id'))
objects = models.SubscriberNews.objects.filter(subscriber__subscriber=subscriber).order_by('record__time')
paginator = Paginator(objects, per_page)
try:
result = paginator.page(page)
except PageNotAnInteger:
page = 1
result = paginator.page(page)
except EmptyPage:
page = paginator.num_pages
result = paginator.page(page)
objects = result.object_list
data = [x.record.json() for x in objects]
objects = data
data = {'objects': objects, 'page':page, 'count': paginator.num_pages}
loginfo("success")
return HttpResponse(json.dumps(data))
def get_newsowners_list(request):
get = request.GET
subscriber = get.get('user_id')
owners = local_get_newsowners_list(subscriber)
data = [str(x.owner) for x in owners]
loginfo("success")
return HttpResponse(json.dumps(data))
def get_news(request):
get = request.GET
page = int(get.get('page'))
per_page = int(get.get('per_page'))
subscriber = int(get.get('user_id'))
objects = models.SubscriberNews.objects.filter(
subscriber__subscriber=subscriber).order_by('record__time')
paginator = Paginator(objects, per_page)
try:
result = paginator.page(page)
except PageNotAnInteger:
page = 1
result = paginator.page(page)
except EmptyPage:
page = paginator.num_pages
result = paginator.page(page)
objects = result.object_list
data = [x.record.json() for x in objects]
objects = data
data = {'objects': objects, 'page': page, 'count': paginator.num_pages}
loginfo("success")
return HttpResponse(json.dumps(data))
def get_newsowners_list(request):
get = request.GET
subscriber = get.get('user_id')
owners = local_get_newsowners_list(subscriber)
data = [str(x.owner) for x in owners]
loginfo("success")
return HttpResponse(json.dumps(data))
def add_bookmark(request):
par = request.REQUEST
user_id = int(par.get('user_id'))
bm_id = int(par.get('bookmark_id'))
title = par.get('title')
record = models.Record()
record.bookmark_id = bm_id
record.title = title
record.owner_id = user_id
record.save()
loginfo("record created")
subscribers = models.Subscriber.objects.filter(owner=user_id)
for user in subscribers:
link = models.SubscriberNews()
link.record = record
link.subscriber = user
link.save()
loginfo("news created")
return HttpResponse()
def handle_user_request(request):
get = request.GET
userId = get.get('userId')
user = models.get_user_via_id(userId)
loginfo('user email' + str(user.email))
return HttpResponse(user.json())
def handleCheckCookieRequest(request):
get = request.GET
token = get.get('token')
sessionId = get.get('id')
cookie = models.get_cookie(sessionId, token)
if cookie:
loginfo('valid cookie')
return HttpResponse(json.dumps({'userId': cookie.userId}))
else:
loginfo('invalid cookie')
return HttpResponseBadRequest()
def handleCheckCookieRequest(request):
get = request.GET
token = get.get('token')
sessionId = get.get('id')
cookie = models.get_cookie(sessionId, token)
if cookie:
loginfo('valid cookie')
return HttpResponse(json.dumps({'userId': cookie.userId}))
else:
loginfo('invalid cookie')
return HttpResponseBadRequest()
def unsubscribe(request):
get = request.REQUEST
subscriber_id = int(get.get('subscriber'))
owner_id = int(get.get('owner'))
ans = local_is_subscribed(owner_id, subscriber_id)
if ans:
news = models.SubscriberNews.objects.filter(subscriber=ans)
for ns in news:
ns.delete()
ans.delete()
loginfo("success")
return HttpResponse()
def unsubscribe(request):
get = request.REQUEST
subscriber_id = int(get.get('subscriber'))
owner_id = int(get.get('owner'))
ans = local_is_subscribed(owner_id, subscriber_id)
if ans:
news = models.SubscriberNews.objects.filter(subscriber=ans)
for ns in news:
ns.delete()
ans.delete()
loginfo("success")
return HttpResponse()
def handle_get_user_bookmark(request):
get = request.GET
id = get.get('bookmark_id')
if id is None:
logerror("Id is none")
return HttpResponseBadRequest()
try:
bookmark = models.Bookmark.objects.get(pk=int(id))
except Exception as e:
logerror(e.message)
return HttpResponseBadRequest
loginfo("bookmark is found")
return HttpResponse(json.dumps(bookmark.full_json()))
def remove_bookmark(request):
par = request.REQUEST
bm_id = int(par.get('bookmark_id'))
try:
record = models.Record.objects.filter(bookmark_id=bm_id)[0]
except Exception as e:
loginfo(e.message)
return HttpResponse()
news = models.SubscriberNews.objects.filter(record=record)
for unit in news:
unit.delete()
record.delete()
loginfo("success")
return HttpResponse()
def remove_bookmark(request):
par = request.REQUEST
bm_id = int(par.get('bookmark_id'))
try:
record = models.Record.objects.filter(bookmark_id=bm_id)[0]
except Exception as e:
loginfo(e.message)
return HttpResponse()
news = models.SubscriberNews.objects.filter(record=record)
for unit in news:
unit.delete()
record.delete()
loginfo("success")
return HttpResponse()
def handle_get_user_bookmark(request):
get = request.GET
id = get.get('bookmark_id')
if id is None:
logerror("Id is none")
return HttpResponseBadRequest()
try:
bookmark = models.Bookmark.objects.get(pk=int(id))
except Exception as e:
logerror(e.message)
return HttpResponseBadRequest
loginfo("bookmark is found")
return HttpResponse(json.dumps(bookmark.full_json()))
def handleRegisterRequest(request):
put = request.REQUEST
user = models.User()
user.name = put.get('name')
user.email = put.get('email')
user.password = put.get('password')
user.last_name = put.get('last_name')
loginfo("register: " + "email " + user.email)
if models.get_user_via_email(user.email):
logerror("user with this email is existing")
return HttpResponseBadRequest()
user.save()
loginfo("user registered")
return HttpResponse()
def handleLogoutRequest(request):
delete = request.REQUEST
token = delete.get('token')
sessionId = delete.get('id')
loginfo("clear cookie: ")
loginfo("token: " + token)
loginfo('id: ' + sessionId)
cookie = models.get_cookie(sessionId, token)
if cookie:
cookie.delete()
loginfo('cookie deleted')
return HttpResponse()
}
# print(content_items)
content_all.update(content_items)
#print(content_all)
now = datetime.datetime.now().strftime("%Y-%m-%d")
if not os.path.exists("./newsfolder/" + now + "_Reuters_news.json"):
with open("./newsfolder/" + now + "_Reuters_news.json",
"w",
encoding="utf-8") as f:
json.dump(content_all, f)
else:
with open("./newsfolder/" + now + "_Reuters_news.json",
"r",
encoding="utf-8") as f:
old_content_all = json.load(f)
old_content_all.update(content_all)
with open("./newsfolder/" + now + "_Reuters_news.json",
"w",
encoding="utf-8") as f:
json.dump(old_content_all, f)
os.remove("reuters_news_url_tmp.txt.bak")
now_s = str(datetime.datetime.now())
loginfo("reuters_news_crawler_finished_" + now_s)
def handleLoginRequest(request):
# loginfo(request.get_full_path())
post = request.POST
email = post.get('email')
password = post.get('password')
loginfo("login request")
loginfo("email: " + email)
user = models.get_user_via_email(email)
if user is None:
loginfo("user is none")
result = {}
if user and user.password == password:
cookie = models.generate_cookie(user.id)
loginfo(str(cookie.id))
result['id'] = cookie.id
result['token'] = cookie.token
result['userId'] = cookie.userId
loginfo('login successful: ')
loginfo('id: ' + str(cookie.userId))
loginfo('token: ' + cookie.token)
return HttpResponse(json.dumps(result))
else:
logerror('login failed')
return HttpResponseBadRequest()
def wrapper(r, *args, **kwargs):
loginfo(r.method + " " + r.get_full_path())
return func(r, *args, **kwargs)
}
# print(content_items)
content_all.update(content_items)
#print(content_all)
now = datetime.datetime.now().strftime("%Y-%m-%d")
if not os.path.exists("./newsfolder/" + now + "_DW_news.json"):
with open("./newsfolder/" + now + "_DW_news.json",
"w",
encoding="utf-8") as f:
json.dump(content_all, f)
else:
with open("./newsfolder/" + now + "_DW_news.json",
"r",
encoding="utf-8") as f:
old_content_all = json.load(f)
old_content_all.update(content_all)
with open("./newsfolder/" + now + "_DW_news.json",
"w",
encoding="utf-8") as f:
json.dump(old_content_all, f)
os.remove("dw_news_url_tmp.txt.bak")
now_s = str(datetime.datetime.now())
loginfo("dw_news_crawler_finished_" + now_s)
def wrapper(r, *args, **kwargs): loginfo(r.method + " " + r.get_full_path()) return func(r, *args, **kwargs)
if_exists='append',
index=False)
# app.show()
from pyspark.sql.types import *
final_news_id_df = spark.createDataFrame(final_news_id_list, StringType())
for i in final_news_id_df:
put = final_news_id_df.select(i.alias('news_id'))
put.registerTempTable("put")
put_pd = put.toPandas()
# app_pd
put_pd.to_sql(name='hot_foreign_news_id',
con=con,
if_exists='append',
index=False)
# app.show()
try:
cursorInstance = dbConnection.cursor()
sqlDeleteRows = "DELETE FROM hot_foreign_news WHERE news_id NOT IN (SELECT news_id FROM hot_foreign_news_id)"
cursorInstance.execute(sqlDeleteRows)
except Exception as ex:
print("Exception occured: %s" % ex)
finally:
dbConnection.close()
now_s = str(datetime.datetime.now())
loginfo("hot_foreign_news_updated_" + now_s)
}
# print(content_items)
content_all.update(content_items)
#print(content_all)
now = datetime.datetime.now().strftime("%Y-%m-%d")
if not os.path.exists("./newsfolder/" + now + "_Yomiuri_news.json"):
with open("./newsfolder/" + now + "_Yomiuri_news.json",
"w",
encoding="utf-8") as f:
json.dump(content_all, f)
else:
with open("./newsfolder/" + now + "_Yomiuri_news.json",
"r",
encoding="utf-8") as f:
old_content_all = json.load(f)
old_content_all.update(content_all)
with open("./newsfolder/" + now + "_Yomiuri_news.json",
"w",
encoding="utf-8") as f:
json.dump(old_content_all, f)
os.remove("yomiuri_news_url_tmp.txt.bak")
now_s = str(datetime.datetime.now())
loginfo("yomiuri_news_crawler_finished_" + now_s)
