def setFeedRetrieved(self, feedid, retrieved):
log.debug("Updating feed %s with last retrieved time of %s." %
(feedid, retrieved))
filters = [dict(name='id', op='equals', val=feedid)]
data = dict(retrieved=retrieved)
print("doput(feed, %s, %s)" % (filters, data))
response = None
try:
response = rest.doPut("feed", filters, data)
except Exception as e:
log.error(
"Failed to updaste feed %s with last retrieved time of %s." %
(feedid, retrieved))
log.exception(e)
raise Exception()
return None
if response.json()['num_modified'] != 0:
log.info("Updating feed %s with last retrieved time of %s." %
(feedid, retrieved))
return True
else:
log.error(
"Failed to updaste feed %s with last retrieved time of %s." %
(feedid, retrieved))
return False
def requestToken(self):
headers = {
'Content-Type': 'application/json',
'user-agent': collector_config.USER_AGENT
}
url = "%s/auth" % (collector_config.API_BASE_URL)
data = {
'username': collector_config.API_USERNAME,
'password': collector_config.API_PASSWORD
}
response = ""
log.info("Requesting JWT token from %s for %s" %
(url, data['username']))
try:
response = requests.post(url,
headers=headers,
data=json.dumps(data))
response.raise_for_status()
newtoken = response.json()['access_token']
log.info("Retreived JWT token from %s for %s" %
(url, data['username']))
return newtoken
except requests.exceptions.HTTPError as e:
log.error(
"Error requesting token from %s: %s %s (%s)" %
(url, response.json()['status_code'], response.json()['error'],
response.json()['description']))
raise Exception()
return False
except Exception as e:
log.exception(e)
raise Exception()
return False
def getFeeds(this):
log.info("Retreiving list of feeds from API server.")
response = ""
try:
response = rest.doGet("feed")
return response.json()['objects']
except:
log.error("Error retreiving list of feeds from API server.")
raise Exception()
return False
def addFeed(this, title, source, description="", type=2, interval=300, active=0):
if this.feedExists(title) == False:
log.warn("Adding type %s feed %s to be retrieved from %s to API server." % (type, title, source))
data = { 'title': title, 'source': source, 'type': type, 'interval': interval, 'active': active }
response = None
try:
response = rest.doPost("feed", data)
return response
except:
log.error("Error adding feed %s to API server." % (feed))
raise Exception()
return False
def retrieveContent(self, link):
try:
a = newspaper.Article(link)
a.download()
a.parse()
text = a.text
a.nlp()
self.keywords = a.keywords
self.retrieved = str(datetime.datetime.utcnow())
return a.text
except Exception as e:
log.error("Exception retrieving %s" % (link))
log.exception(e)
def retrieveContent(self, link):
try:
a = newspaper.Article(link)
a.download()
a.parse()
text = a.text
a.nlp()
self.keywords = a.keywords
self.retrieved = str(datetime.datetime.utcnow())
return a.text
except Exception as e:
log.error("Exception retrieving %s" % (link))
log.exception(e)
def feedExists(this, title):
log.debug("Checking to see if feed %s exists on API server." % (title))
filters = [dict(name='title', op='equals', val=title)]
try:
response = rest.doGet("feed", filters)
if response.json()['num_results'] != 0:
return True
else:
return False
except:
log.error("Error checking to see if feed %s exists on API server." % (title))
raise Exception()
return True
def setFeedRetrieved(self, feedid, retrieved):
log.debug("Updating feed %s with last retrieved time of %s." % (feedid, retrieved))
filters = [dict(name='id', op='equals', val=feedid)]
data = dict(retrieved=retrieved)
print("doput(feed, %s, %s)" % (filters, data))
response = None
try:
response = rest.doPut("feed", filters, data)
except Exception as e:
log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved))
log.exception(e)
raise Exception()
return None
if response.json()['num_modified'] != 0:
log.info("Updating feed %s with last retrieved time of %s." % (feedid, retrieved))
return True
else:
log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved))
return False
def requestToken(self):
headers = {"Content-Type": "application/json", "user-agent": collector_config.USER_AGENT}
url = "%s/auth" % (collector_config.API_BASE_URL)
data = {"username": collector_config.API_USERNAME, "password": collector_config.API_PASSWORD}
response = ""
log.info("Requesting JWT token from %s for %s" % (url, data["username"]))
try:
response = requests.post(url, headers=headers, data=json.dumps(data))
response.raise_for_status()
newtoken = response.json()["access_token"]
log.info("Retreived JWT token from %s for %s" % (url, data["username"]))
return newtoken
except requests.exceptions.HTTPError as e:
log.error(
"Error requesting token from %s: %s %s (%s)"
% (url, response.json()["status_code"], response.json()["error"], response.json()["description"])
)
raise Exception()
return False
except Exception as e:
log.exception(e)
raise Exception()
return False