def setFeedRetrieved(self, feedid, retrieved):
log.debug("Updating feed %s with last retrieved time of %s." %
(feedid, retrieved))
filters = [dict(name='id', op='equals', val=feedid)]
data = dict(retrieved=retrieved)
print("doput(feed, %s, %s)" % (filters, data))
response = None
try:
response = rest.doPut("feed", filters, data)
except Exception as e:
log.error(
"Failed to updaste feed %s with last retrieved time of %s." %
(feedid, retrieved))
log.exception(e)
raise Exception()
return None
if response.json()['num_modified'] != 0:
log.info("Updating feed %s with last retrieved time of %s." %
(feedid, retrieved))
return True
else:
log.error(
"Failed to updaste feed %s with last retrieved time of %s." %
(feedid, retrieved))
return False
def requestToken(self):
headers = {
'Content-Type': 'application/json',
'user-agent': collector_config.USER_AGENT
}
url = "%s/auth" % (collector_config.API_BASE_URL)
data = {
'username': collector_config.API_USERNAME,
'password': collector_config.API_PASSWORD
}
response = ""
log.info("Requesting JWT token from %s for %s" %
(url, data['username']))
try:
response = requests.post(url,
headers=headers,
data=json.dumps(data))
response.raise_for_status()
newtoken = response.json()['access_token']
log.info("Retreived JWT token from %s for %s" %
(url, data['username']))
return newtoken
except requests.exceptions.HTTPError as e:
log.error(
"Error requesting token from %s: %s %s (%s)" %
(url, response.json()['status_code'], response.json()['error'],
response.json()['description']))
raise Exception()
return False
except Exception as e:
log.exception(e)
raise Exception()
return False
def addArticleKeyword(self, articleid, keywordid):
data = {"articleid": articleid, "keywordid": keywordid}
response = None
try:
response = rest.doPost("articlekeyword", data)
return (response.json()['id'])
except Exception as e:
log.exception(e)
return None
def addArticleKeyword(self, articleid, keywordid):
data = { "articleid": articleid, "keywordid": keywordid }
response = None
try:
response = rest.doPost("articlekeyword", data)
return(response.json()['id'])
except Exception as e:
log.exception(e)
return None
def keywordExists(this, word):
filters = [dict(name='word', op='equals', val=word)]
response = None
try:
response = rest.doGet("keyword", filters)
if response.json()['num_results'] != 0:
return response.json()['objects'].pop()['id']
else:
return False
except Exception as e:
log.exception(e)
return None
def articleExists(this, article):
filters = [dict(name='contenthash', op='equals', val=article.contenthash)]
response = None
try:
response = rest.doGet("article", filters)
if response.json()['num_results'] != 0:
return True;
else:
return False;
except Exception as e:
log.exception(e)
return None
def keywordExists(this, word):
filters = [dict(name='word', op='equals', val=word)]
response = None
try:
response = rest.doGet("keyword", filters)
if response.json()['num_results'] != 0:
return response.json()['objects'].pop()['id']
else:
return False
except Exception as e:
log.exception(e)
return None
def retrieveContent(self, link):
try:
a = newspaper.Article(link)
a.download()
a.parse()
text = a.text
a.nlp()
self.keywords = a.keywords
self.retrieved = str(datetime.datetime.utcnow())
return a.text
except Exception as e:
log.error("Exception retrieving %s" % (link))
log.exception(e)
def retrieveContent(self, link):
try:
a = newspaper.Article(link)
a.download()
a.parse()
text = a.text
a.nlp()
self.keywords = a.keywords
self.retrieved = str(datetime.datetime.utcnow())
return a.text
except Exception as e:
log.error("Exception retrieving %s" % (link))
log.exception(e)
def articleExists(this, article):
filters = [
dict(name='contenthash', op='equals', val=article.contenthash)
]
response = None
try:
response = rest.doGet("article", filters)
if response.json()['num_results'] != 0:
return True
else:
return False
except Exception as e:
log.exception(e)
return None
def addArticle(self, article):
if self.articleExists(article) == False:
log.info("adding article %s" % (article.title))
data = { "title": article.title, "link": article.link, "keywords": article.keywords, "description": article.description, "content": article.content, "contenthash": article.contenthash, "retrieved": article.retrieved, "published": article.published }
try:
response = rest.doPost("article", data)
for word in article.keywords:
keywordid = self.addKeyword(word)
self.addArticleKeyword(response.json()['id'], keywordid)
except Exception as e:
log.exception(e)
return None
else:
return False
def addKeyword(self, word):
ke = self.keywordExists(word)
if ke == False:
log.debug("adding keyword %s" % (word))
data = { "word": word }
response = None
try:
response = rest.doPost("keyword", data)
except Exception as e:
log.exception(e)
return None
return(response.json()['id'])
else:
return ke
def addKeyword(self, word):
ke = self.keywordExists(word)
if ke == False:
log.debug("adding keyword %s" % (word))
data = {"word": word}
response = None
try:
response = rest.doPost("keyword", data)
except Exception as e:
log.exception(e)
return None
return (response.json()['id'])
else:
return ke
def getFeedRetrieved(self, feedid):
log.debug("Checking when feed %s last retreived." % (feedid))
filters = [dict(name='id', op='equals', val=feedid)]
response = None
try:
response = rest.doGet(filters)
if response.json()['num_results'] != 0:
r = response.json()['objects'].pop()
log.info("Feed %s last retreived %s" % (feedid, r['retrieved']))
return r['retrieved']
else:
log.info("No retreived timestamp for %s" % (feedid))
return False
except Exception as e:
log.exception(e)
raise Exception()
return None
def getFeedRetrieved(self, feedid):
log.debug("Checking when feed %s last retreived." % (feedid))
filters = [dict(name='id', op='equals', val=feedid)]
response = None
try:
response = rest.doGet(filters)
if response.json()['num_results'] != 0:
r = response.json()['objects'].pop()
log.info("Feed %s last retreived %s" %
(feedid, r['retrieved']))
return r['retrieved']
else:
log.info("No retreived timestamp for %s" % (feedid))
return False
except Exception as e:
log.exception(e)
raise Exception()
return None
def setFeedRetrieved(self, feedid, retrieved):
log.debug("Updating feed %s with last retrieved time of %s." % (feedid, retrieved))
filters = [dict(name='id', op='equals', val=feedid)]
data = dict(retrieved=retrieved)
print("doput(feed, %s, %s)" % (filters, data))
response = None
try:
response = rest.doPut("feed", filters, data)
except Exception as e:
log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved))
log.exception(e)
raise Exception()
return None
if response.json()['num_modified'] != 0:
log.info("Updating feed %s with last retrieved time of %s." % (feedid, retrieved))
return True
else:
log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved))
return False
def requestToken(self):
headers = {"Content-Type": "application/json", "user-agent": collector_config.USER_AGENT}
url = "%s/auth" % (collector_config.API_BASE_URL)
data = {"username": collector_config.API_USERNAME, "password": collector_config.API_PASSWORD}
response = ""
log.info("Requesting JWT token from %s for %s" % (url, data["username"]))
try:
response = requests.post(url, headers=headers, data=json.dumps(data))
response.raise_for_status()
newtoken = response.json()["access_token"]
log.info("Retreived JWT token from %s for %s" % (url, data["username"]))
return newtoken
except requests.exceptions.HTTPError as e:
log.error(
"Error requesting token from %s: %s %s (%s)"
% (url, response.json()["status_code"], response.json()["error"], response.json()["description"])
)
raise Exception()
return False
except Exception as e:
log.exception(e)
raise Exception()
return False
def addArticle(self, article):
if self.articleExists(article) == False:
log.info("adding article %s" % (article.title))
data = {
"title": article.title,
"link": article.link,
"keywords": article.keywords,
"description": article.description,
"content": article.content,
"contenthash": article.contenthash,
"retrieved": article.retrieved,
"published": article.published
}
try:
response = rest.doPost("article", data)
for word in article.keywords:
keywordid = self.addKeyword(word)
self.addArticleKeyword(response.json()['id'], keywordid)
except Exception as e:
log.exception(e)
return None
else:
return False
