def doPut(self, endpoint, filters="", data=dict()):
putdata = dict()
if filters != "":
putdata.update(dict(q=dict(filters=filters)))
putdata.update(data)
headers = {
"Content-Type": "application/json",
"user-agent": collector_config.USER_AGENT,
"Authorization": "JWT %s" % self.token,
}
url = "%s/api/%s" % (collector_config.API_BASE_URL, endpoint)
params = ""
response = ""
log.debug("PUT request to %s" % (url))
try:
if putdata != "":
response = requests.put(url, headers=headers, data=json.dumps(putdata))
response.raise_for_status()
else:
response = requests.post(url, headers=headers)
response.raise_for_status()
log.debug("Retrieved put data from %s" % (url))
return response
except requests.exceptions.HTTPError as e:
log.warning(
"Error PUTting data to %s: %s %s (%s)"
% (url, response.json()["status_code"], response.json()["error"], response.json()["description"])
)
return False
def setFeedRetrieved(self, feedid, retrieved):
log.debug("Updating feed %s with last retrieved time of %s." %
(feedid, retrieved))
filters = [dict(name='id', op='equals', val=feedid)]
data = dict(retrieved=retrieved)
print("doput(feed, %s, %s)" % (filters, data))
response = None
try:
response = rest.doPut("feed", filters, data)
except Exception as e:
log.error(
"Failed to updaste feed %s with last retrieved time of %s." %
(feedid, retrieved))
log.exception(e)
raise Exception()
return None
if response.json()['num_modified'] != 0:
log.info("Updating feed %s with last retrieved time of %s." %
(feedid, retrieved))
return True
else:
log.error(
"Failed to updaste feed %s with last retrieved time of %s." %
(feedid, retrieved))
return False
def doPost(self, endpoint, data=""):
headers = {
'Content-Type': 'application/json',
'user-agent': collector_config.USER_AGENT,
'Authorization': "JWT %s" % self.token
}
url = "%s/api/%s" % (collector_config.API_BASE_URL, endpoint)
params = ""
response = ""
log.debug("POST request to %s" % (url))
try:
if data != "":
response = requests.post(url,
headers=headers,
data=json.dumps(data))
response.raise_for_status()
else:
response = requests.post(url, headers=headers)
response.raise_for_status()
log.debug("Retrieved POST data from %s" % (url))
return response
except requests.exceptions.HTTPError as e:
log.warning(
"Error POSTing data to %s: %s %s (%s)" %
(url, response.json()['status_code'], response.json()['error'],
response.json()['description']))
return False
def feedExists(this, title):
log.debug("Checking to see if feed %s exists on API server." % (title))
filters = [dict(name='title', op='equals', val=title)]
try:
response = rest.doGet("feed", filters)
if response.json()['num_results'] != 0:
return True
else:
return False
except:
log.error("Error checking to see if feed %s exists on API server." % (title))
raise Exception()
return True
def addKeyword(self, word):
ke = self.keywordExists(word)
if ke == False:
log.debug("adding keyword %s" % (word))
data = { "word": word }
response = None
try:
response = rest.doPost("keyword", data)
except Exception as e:
log.exception(e)
return None
return(response.json()['id'])
else:
return ke
def addKeyword(self, word):
ke = self.keywordExists(word)
if ke == False:
log.debug("adding keyword %s" % (word))
data = {"word": word}
response = None
try:
response = rest.doPost("keyword", data)
except Exception as e:
log.exception(e)
return None
return (response.json()['id'])
else:
return ke
def getFeedRetrieved(self, feedid):
log.debug("Checking when feed %s last retreived." % (feedid))
filters = [dict(name='id', op='equals', val=feedid)]
response = None
try:
response = rest.doGet(filters)
if response.json()['num_results'] != 0:
r = response.json()['objects'].pop()
log.info("Feed %s last retreived %s" % (feedid, r['retrieved']))
return r['retrieved']
else:
log.info("No retreived timestamp for %s" % (feedid))
return False
except Exception as e:
log.exception(e)
raise Exception()
return None
def getFeedRetrieved(self, feedid):
log.debug("Checking when feed %s last retreived." % (feedid))
filters = [dict(name='id', op='equals', val=feedid)]
response = None
try:
response = rest.doGet(filters)
if response.json()['num_results'] != 0:
r = response.json()['objects'].pop()
log.info("Feed %s last retreived %s" %
(feedid, r['retrieved']))
return r['retrieved']
else:
log.info("No retreived timestamp for %s" % (feedid))
return False
except Exception as e:
log.exception(e)
raise Exception()
return None
def setFeedRetrieved(self, feedid, retrieved):
log.debug("Updating feed %s with last retrieved time of %s." % (feedid, retrieved))
filters = [dict(name='id', op='equals', val=feedid)]
data = dict(retrieved=retrieved)
print("doput(feed, %s, %s)" % (filters, data))
response = None
try:
response = rest.doPut("feed", filters, data)
except Exception as e:
log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved))
log.exception(e)
raise Exception()
return None
if response.json()['num_modified'] != 0:
log.info("Updating feed %s with last retrieved time of %s." % (feedid, retrieved))
return True
else:
log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved))
return False