def push_to_server(self, retry=False):
try:
resps = self.diffbot_caller.diffbot_article_api(1)
if not resps:
logger.warning(
'All text of articles replied from diffbot is empty')
else:
try:
for res in resps:
url = res['pageUrl']
title = res['title']
# body = (''.join(res['text']))
body = res['html']
if res['tags']:
words = res['tags']
key_words = [{
'count': w['count'],
'word': w['label']
} for w in words]
language = res['humanLanguage']
if language == 'zh':
language = 'cn'
# Get all corresponding index_urls of each article_url
# return type is selectQuery, which is iterable
records = (Subscription.select(
Subscription.index_url).join(
SubscriptionArticle).join(Article).where(
Article.article_url == url))
resp_codes = []
for rec in records:
dl_data = {
'entry_page_url': rec.index_url,
'news': {
'url': url,
'title': title,
'body': body,
'language': language,
'keywords': key_words,
}
}
# print(dl_data['news']['language'])
payload = json.dumps(
dl_data, ensure_ascii=False).encode('utf-8')
resp_codes.append(push_aricles(payload))
if 200 in resp_codes:
u = Article.update(
status=2,
modified_utc=datetime.utcnow(),
).where(Article.article_url == url)
u.execute()
#except (KeyError, NameError, AttributeError, TypeError) as e:
except Exception as e:
raise RuntimeError(
'Error happened when pushing. Reason: ' + str(e))
except RuntimeError:
raise
def push_aricles(payload):
if True == SETTINGS['PRODUCTION_FLAG']:
server_url = "https://newsapi.com/api/v1/ebot/news"
else:
server_url = SETTINGS[
'http_auth'] + "@staging.newsapi.net/api/v1/ebot/news"
r = requests.post(server_url,
headers={
'X-Api-Token': SETTINGS['JSON_API_TOKEN'],
'Content-Type': 'application/json'
},
data=payload,
timeout=10)
logger.warning('response code is %d', r.status_code)
return r.status_code
def _find_alinks(self, event_expr):
evNode = self.currSent[event_expr.locInSent]
if evNode is None:
logger.warning("No event node found at locInSent")
forwardFSAs = event_expr.alinkingContexts('forward')
if forwardFSAs:
logger.debug("PROCESS for FORWARD alinks")
evNode.createForwardAlink(forwardFSAs)
if evNode.createdAlink:
evNode.createdAlink = 0
return
backwardFSAs = event_expr.alinkingContexts('backwards')
if backwardFSAs:
logger.debug("PROCESS for BACKWARD alinks")
evNode.createBackwardAlink(backwardFSAs)
if evNode.createdAlink:
evNode.createdAlink = 0
def _find_alinks(self, event_expr):
evNode = self.currSent[event_expr.locInSent]
if evNode is None:
logger.warning("No event node found at locInSent")
forwardFSAs = event_expr.alinkingContexts('forward')
if forwardFSAs:
logger.debug("PROCESS for FORWARD alinks")
evNode.createForwardAlink(forwardFSAs)
if evNode.createdAlink:
evNode.createdAlink = 0
return
backwardFSAs = event_expr.alinkingContexts('backwards')
if backwardFSAs:
logger.debug("PROCESS for BACKWARD alinks")
evNode.createBackwardAlink(backwardFSAs)
if evNode.createdAlink:
evNode.createdAlink = 0
def _create_timex_link(self, timex1, timex2):
"""Try to create a TLINK between two timexes."""
creation_year = self.dct[0:4]
date1 = timex1.attrs.get(VALUE)
date2 = timex2.attrs.get(VALUE)
if date1 is None or date2 is None:
logger.warning("Missing value in %s or %s" % (date1, date2))
return
date1 = _fix_timex_val(date1)
date2 = _fix_timex_val(date2)
tid1 = timex1.attrs['tid']
tid2 = timex2.attrs['tid']
origin = "%s-TimexLinking" % BLINKER
if date1 == date2:
if date1 not in ('PAST_REF', 'FUTURE_REF'):
self._add_tlink('IDENTITY', tid1, tid2, origin)
else:
rel = compare_date(date1, date2, creation_year)
if rel != 'IDENTITY':
self._add_tlink(rel, tid1, tid2, origin)
def _create_timex_link(self, timex1, timex2):
"""Try to create a TLINK between two timexes."""
creation_year = self.dct[0:4]
date1 = timex1.attrs.get(VALUE)
date2 = timex2.attrs.get(VALUE)
if date1 is None or date2 is None:
logger.warning("Missing value in %s or %s" % (date1, date2))
return
date1 = _fix_timex_val(date1)
date2 = _fix_timex_val(date2)
tid1 = timex1.attrs['tid']
tid2 = timex2.attrs['tid']
origin = "%s-TimexLinking" % BLINKER
if date1 == date2:
if date1 not in ('PAST_REF', 'FUTURE_REF'):
self._add_tlink('IDENTITY', tid1, tid2, origin)
else:
rel = compare_date(date1, date2, creation_year)
if rel != 'IDENTITY':
self._add_tlink(rel, tid1, tid2, origin)