def inner_update(para_date):
if para_date is None:
url = 'http://news.at.zhihu.com/api/1.2/news/latest'
else:
time1 = datetime.strptime(para_date, '%Y%m%d')
para_date = get_tomorrow(time1).strftime('%Y%m%d')
url = 'http://news.at.zhihu.com/api/1.2/news/before/'+para_date
header = {'User-Agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4'}
req = urllib2.Request(url, headers=header)
webcode = urllib2.urlopen(req).read()
data_dict = json.loads(webcode) # 所有数据的dict
news_count = len(data_dict['news']) # 子dick news
if news_count < 1:
return False
the_date = data_dict['date']
data_dict['news'].reverse() # 反转List
for i in range(0, news_count):
new_obj = News()
a_new = data_dict['news'][i]
new_obj.id = a_new['id']
new_obj.title = a_new['title']
new_obj.share_url = a_new['share_url']
new_obj.api_url = a_new['url']
new_obj.image_url = a_new['image']
new_obj.image_source = a_new['image_source']
new_obj.seq = i
new_obj.date = the_date
new_obj.save()
return True
def add_news(req):
check = News.objects.filter(link=req.POST['link'])
mnews = ""
if check.count() != 0:
# record already exist
if check[0].time > int(req.POST['time']):
return HttpResponse(json.dumps({'status': 'OK'}, indent=4),
content_type='application/json')
mnews = check[0]
else:
mnews = News()
mnews.title = req.POST['title']
mnews.content = req.POST['content']
mnews.link = req.POST['link']
mnews.time = int(req.POST['time'])
mnews.save()
keywords = json.loads(req.POST['keywords'])
refers = json.loads(req.POST['refers'])
for word in keywords:
mkeywords = Keywords()
mkeywords.news = mnews
mkeywords.words = word
mkeywords.save()
for refer in refers:
mrefer = Refers()
mrefer.news = mnews
mrefer.refer = refer
mrefer.save()
return HttpResponse(json.dumps({'status': 'OK'}),
content_type='application/json')
def add_news(req):
check = News.objects.filter(link=req.POST['link'])
mnews = ""
if check.count() != 0:
# record already exist
if check[0].time > int(req.POST['time']):
return HttpResponse(json.dumps({'status':'OK'},indent=4),content_type='application/json')
mnews = check[0]
else:
mnews = News()
mnews.title = req.POST['title']
mnews.content = req.POST['content']
mnews.link = req.POST['link']
mnews.time = int(req.POST['time'])
mnews.save()
keywords = json.loads(req.POST['keywords'])
refers = json.loads(req.POST['refers'])
for word in keywords:
mkeywords = Keywords()
mkeywords.news = mnews
mkeywords.words = word
mkeywords.save()
for refer in refers:
mrefer = Refers()
mrefer.news = mnews
mrefer.refer = refer
mrefer.save()
return HttpResponse(json.dumps({'status':'OK'}),content_type='application/json')
def add_news(request):
if request.user.is_authenticated()==False:
return HttpResponse("<h2>Для виконання операції, авторизуйтесь</h2>")
a = News()
if request.method == 'POST':
form = NewsForm(request.POST, instance=a)
if form.is_valid():
title = form.cleaned_data['title']
text = form.cleaned_data['text']
# date = form.cleaned_data['date']
link = form.cleaned_data['link']
author = form.cleaned_data['author']
category = form.cleaned_data['category']
comm = form.cleaned_data['comments']
# user = form.cleaned_data['user']
n = News(title=title, text=text, date=datetime.datetime.now(), link=link, author=author, category=category, user=request.user)
#n.comments.add(comm)
n.save()
return HttpResponseRedirect('/')
else:
form = NewsForm(instance=a, initial={'author': request.user })
photo1 = Photo.objects.random()
photo2 = Photo.objects.random()
vars = {'weblink': 'news_add.html', 'sel_menu': 'main', 'photo1': photo1, 'photo2': photo2, 'entry': get_funn(), 'form': form}
calendar = embeded_calendar()
vars.update(calendar)
return render_to_response('index.html', vars, context_instance=RequestContext(request, processors=[custom_proc]))
def news_form():
form = NewsForm()
if request.method == 'POST' and form.validate:
# print(form, form.title.data)
title = request.form['title']
description = request.form['description']
image_path = save_file(form.news_image.data)
news = News(title, image_path, description)
news.save()
return render_template('card_form.html', form=form)
def add_news(request):
imgroot = '/var/pj/gk/st/news/'
imgroot54x54 = '/var/pj/gk/st/news/54x54/'
if request.method == "POST":
title = request.POST['news_name']
tags = request.POST['news_tags']
active = request.POST['news_status']
category = request.POST['news_category']
sweight = request.POST['news_priority']
date = request.POST['news_date']
text = request.POST['news_content']
n = News()
if 'news_autor' in request.POST:
n.autor = request.POST['news_autor']
if 'news_photo_autor' in request.POST:
n.photo_title = request.POST['news_photo_autor']
if 'news_origin' in request.POST:
n.origin = request.POST['news_origin']
if 'news_city' in request.POST:
n.city = request.POST['news_city']
if 'news_media' in request.POST:
n.media = request.POST['news_media']
user = User.objects.get(username=request.user)
n.owner = user
n.title = title
if active == 'on':
active = True
else:
active = False
n.active = active
n.text = text
n.sweight = sweight
n.date = date
n.save()
tagsl = tags.split(',')
tagsitems = []
for item in tagsl:
if len(item.strip()):
tagsitems.append(Tag.objects.get_or_create(name=item.strip())[0].id)
n.tags = tagsitems
categoryl = category.split(',')
categoryitems = []
for item in categoryl:
if len(item.strip()):
categoryitems.append(item)
n.category = categoryitems
tmpfile = handle_uploaded_file(request.FILES['news_logo'], str(n.id) + request.FILES['news_logo'].name[-4:])
img = Image.open('/tmp/' + tmpfile)
img.save(imgroot + str(n.id) + '.png', 'PNG')
img.thumbnail([54, 54], Image.ANTIALIAS)
img.save(imgroot54x54 + str(n.id) + '.png', 'PNG')
request.session['alert'] = 'Новость успешно добавлена'
return redirect('/news/admin/')
def post(self, request):
title = request.POST['title']
shortNews = request.POST['shortNews']
try:
current_user = request.user
user = AuthUser.objects.get(id=current_user.id)
datatime = timezone.localtime(timezone.now()).strftime('%d %b %Y')
news = News(title=title,datatime=datatime,shortnews=shortNews,auth_user =user)
news.save()
return JsonResponse({'response': 'OK'})
except DatabaseError:
return JsonResponse({'response': 'FAIL'})
def addNews(request):
url=request.POST['url']
news = News()
user = request.user # change to request.user
news.author = user
news.picture = open(request.FILES['picture'].temporary_file_path(), "rb")
news.voice = open(request.FILES['voice'].temporary_file_path(), "rb")#change to data from request
result=urlparse.urlparse(url)
params=urlparse.parse_qs(result.query,True)
gid=params['id']
news.good = Goods.objects(pk=gid[0]).first()
news.save()
return HttpResponse("success")
def populate_news(request):
data = get_from_hackernews()
for dx in data:
try:
n = News.objects.get(headline=dx['news'])
except:
n = News(headline=dx['news'],
source=dx['url'],
points=dx['pts'],
author=dx['user'])
n.save()
response = json.dumps({'data': data})
return HttpResponse(response)
def insert_new_news():
last_time = News.select(fn.MAX(
News.time).alias("time")).get().time or datetime(1970, 1, 1, 0, 0, 0)
news = extract_company(get_news())
news = list(filter(lambda n: n["time"] > last_time, news))
for n in news:
dbN = News()
dbN.title = n["title"]
dbN.body = n["body"]
dbN.time = n["time"]
dbN.link = n["link"]
dbN.company = n["company"]
info = sentence_info(dbN)
dbN.sent_score = info["sent_score"]
dbN.word_count = info["word_count"]
dbN.words = json.dumps(info["words"])
dbN.parsed_sentence = info["sentence"]
dbN.save()
def fetch(link, dt):
try:
news = News.objects.get(url=link)
news = None
print ">>", "exists", link
except DoesNotExist, e:
print ">>", "fetch", link
try:
res = requests.get(link, timeout=3)
print ">>", "get done", link
res.encoding = res.apparent_encoding
if not "text" in res.headers.get("content-type", "none"):
return
if res.apparent_encoding is not None:
html = unicode(res.content, res.apparent_encoding)
news = News(url=link, html=html, published_at=dt)
news.save()
news = None
except UnicodeDecodeError, e:
print ">>", "error", link, e
def post(self, request):
try:
news_detail_list = request.data
if news_detail_list:
for news_details in news_detail_list:
news = News(
news_id=long(news_details["news_id"]),
username=news_details["username"],
title=news_details["title"],
url=news_details["url"] if "url" in news_details else '',
score=int(news_details["score"]) if "score" in news_details else 0,
description=news_details["description"] if "description" in news_details else '',
descendants=int(news_details["descendants"]) if "descendants" in news_details else 0,
confidence=float(news_details["confidence"]) if "confidence" in news_details else 0,
sentiment=news_details["sentiment"] if "sentiment" in news_details else '',
)
news.save()
return Response({"statusCode" : 200})
else:
return Response({"statusCode" : 505, "message" : "wrong type of arguments : " + news_detail_list})
except Exception, e:
log.debug(str(e) + " IN SaveDetailsApi ")
return Response({"statusCode" : 404, "message" : "Exception occured while saving : " + str(e)})
def post(self, request):
try:
news_details = request.data
news_details = json.dumps(news_details)
news_details = ast.literal_eval(news_details)
news_details = news_details.keys()[0]
news_details = ast.literal_eval(news_details)
news = News(
news_id=long(news_details["news_id"]),
username=news_details["username"],
title=news_details["title"],
url=news_details["url"] if "url" in news_details else '',
score=int(news_details["score"]) if "score" in news_details else 0,
description=news_details["description"] if "description" in news_details else '',
descendants=int(news_details["descendants"]) if "descendants" in news_details else 0,
confidence=float(news_details["confidence"]) if "confidence" in news_details else 0,
sentiment=news_details["sentiment"] if "sentiment" in news_details else '',
)
news.save()
return Response({"status": "success"})
except Exception, e:
log.debug(str(e) + " IN SaveDetailsApi ")
return Response({"status": "failure"})
print dt, title
link = entry.link
hres = requests.head(link)
if hres.status_code == 302:
link = hres.headers["location"]
link = link.replace("?ref=rss", "")
print link
try:
news = News.objects.get(url=link)
except DoesNotExist, e:
res = requests.get(link)
res.encoding = res.apparent_encoding
html = unicode(res.content, res.apparent_encoding)
time.sleep(1)
news = News(url=link, html=html, title=title, published_at=dt)
news.save()
# fetch nhk
for cat in range(1, 7):
for day in range(7):
target = datetime.now() - timedelta(day)
date = target.strftime("%Y%m%d")
unixtime = str(int(time.time()))
cat = str(cat)
url = "http://www3.nhk.or.jp/news/html/{date}/xml/cat{cat}.xml?t={unixtime}".format(date=date, unixtime=unixtime, cat=cat)
print url
news = feedparser.parse(url)
for entry in news.entries:
print entry.title
print entry.link
link = entry.link
from django.test import TestCase
from models import News
# Create your tests here.
b = News(
news_title='ペドリも続くのか…？ ロナルド・クーマンにブレイクのきっかけを与えられた選手たち',
news_url='https://headlines.yahoo.co.jp/hl?a=20201104-01137988-soccerk-socc'
)
b.save()
