def add_news(req):
check = News.objects.filter(link=req.POST['link'])
mnews = ""
if check.count() != 0:
# record already exist
if check[0].time > int(req.POST['time']):
return HttpResponse(json.dumps({'status': 'OK'}, indent=4),
content_type='application/json')
mnews = check[0]
else:
mnews = News()
mnews.title = req.POST['title']
mnews.content = req.POST['content']
mnews.link = req.POST['link']
mnews.time = int(req.POST['time'])
mnews.save()
keywords = json.loads(req.POST['keywords'])
refers = json.loads(req.POST['refers'])
for word in keywords:
mkeywords = Keywords()
mkeywords.news = mnews
mkeywords.words = word
mkeywords.save()
for refer in refers:
mrefer = Refers()
mrefer.news = mnews
mrefer.refer = refer
mrefer.save()
return HttpResponse(json.dumps({'status': 'OK'}),
content_type='application/json')
def add_news(req):
check = News.objects.filter(link=req.POST['link'])
mnews = ""
if check.count() != 0:
# record already exist
if check[0].time > int(req.POST['time']):
return HttpResponse(json.dumps({'status':'OK'},indent=4),content_type='application/json')
mnews = check[0]
else:
mnews = News()
mnews.title = req.POST['title']
mnews.content = req.POST['content']
mnews.link = req.POST['link']
mnews.time = int(req.POST['time'])
mnews.save()
keywords = json.loads(req.POST['keywords'])
refers = json.loads(req.POST['refers'])
for word in keywords:
mkeywords = Keywords()
mkeywords.news = mnews
mkeywords.words = word
mkeywords.save()
for refer in refers:
mrefer = Refers()
mrefer.news = mnews
mrefer.refer = refer
mrefer.save()
return HttpResponse(json.dumps({'status':'OK'}),content_type='application/json')
def insert_new_news():
last_time = News.select(fn.MAX(
News.time).alias("time")).get().time or datetime(1970, 1, 1, 0, 0, 0)
news = extract_company(get_news())
news = list(filter(lambda n: n["time"] > last_time, news))
for n in news:
dbN = News()
dbN.title = n["title"]
dbN.body = n["body"]
dbN.time = n["time"]
dbN.link = n["link"]
dbN.company = n["company"]
info = sentence_info(dbN)
dbN.sent_score = info["sent_score"]
dbN.word_count = info["word_count"]
dbN.words = json.dumps(info["words"])
dbN.parsed_sentence = info["sentence"]
dbN.save()
def save(item):
urod_id = int(re.sub('\D', '', item.find('link').text) or None)
if not urod_id:
return
if session.query(News).get(urod_id):
return
soup = BeautifulSoup(item.find('description').text, 'html.parser')
if soup.img:
n = News(format='img', text=soup.img['src'])
elif soup.iframe:
text = soup.iframe['src']
if not text:
text = soup.find('div', {'class': 'spoilerContent'})['data']
n = News(format='video', text=text)
elif len(soup.get('greeting', {}).get('text', '').strip()) == 0:
n = News(format='none', text=None)
else:
n = News(format='text', text=soup.greeting.text)
n.urod_id = urod_id
n.link = item.find('link').text
n.title = item.find('title').text
session.add(n)
session.commit()
logging.info('Broken url: %s' % (url))
urlinstance.put()
#logging.debug('URL %s saved (status code %s).' % (url,str(result.status_code)))
except (urlfetch.DownloadError, urlfetch.DeadlineExceededError, urlfetch.InternalTransientError):
# Request timed out or failed.
# ...
urlinstance.valid=urlinstance.valid-1
urlinstance.put()
logging.warning('Urlfetch \'%s\' failed.' % (url))
# Update News
if channelinstance.private==False and date.date()>=today:
try:
news=News(content='Link')
news.link=url
news.link_text=url_title or url
news.put()
#logging.debug('News updated')
except:
logging.warning('News update failed')
else:
logging.info('News not updated, private channel/old url')
if not url_title:
url_title=''.join(url.split('/')[-1:])
#logging.debug('Title: %s' % (url_title))
# Create Document (FullTextSearch)
doc_id=str(urlinstance.key.id())
try: