def doPut(self, endpoint, filters="", data=dict()): putdata = dict() if filters != "": putdata.update(dict(q=dict(filters=filters))) putdata.update(data) headers = { "Content-Type": "application/json", "user-agent": collector_config.USER_AGENT, "Authorization": "JWT %s" % self.token, } url = "%s/api/%s" % (collector_config.API_BASE_URL, endpoint) params = "" response = "" log.debug("PUT request to %s" % (url)) try: if putdata != "": response = requests.put(url, headers=headers, data=json.dumps(putdata)) response.raise_for_status() else: response = requests.post(url, headers=headers) response.raise_for_status() log.debug("Retrieved put data from %s" % (url)) return response except requests.exceptions.HTTPError as e: log.warning( "Error PUTting data to %s: %s %s (%s)" % (url, response.json()["status_code"], response.json()["error"], response.json()["description"]) ) return False
def setFeedRetrieved(self, feedid, retrieved): log.debug("Updating feed %s with last retrieved time of %s." % (feedid, retrieved)) filters = [dict(name='id', op='equals', val=feedid)] data = dict(retrieved=retrieved) print("doput(feed, %s, %s)" % (filters, data)) response = None try: response = rest.doPut("feed", filters, data) except Exception as e: log.error( "Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved)) log.exception(e) raise Exception() return None if response.json()['num_modified'] != 0: log.info("Updating feed %s with last retrieved time of %s." % (feedid, retrieved)) return True else: log.error( "Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved)) return False
def doPost(self, endpoint, data=""): headers = { 'Content-Type': 'application/json', 'user-agent': collector_config.USER_AGENT, 'Authorization': "JWT %s" % self.token } url = "%s/api/%s" % (collector_config.API_BASE_URL, endpoint) params = "" response = "" log.debug("POST request to %s" % (url)) try: if data != "": response = requests.post(url, headers=headers, data=json.dumps(data)) response.raise_for_status() else: response = requests.post(url, headers=headers) response.raise_for_status() log.debug("Retrieved POST data from %s" % (url)) return response except requests.exceptions.HTTPError as e: log.warning( "Error POSTing data to %s: %s %s (%s)" % (url, response.json()['status_code'], response.json()['error'], response.json()['description'])) return False
def feedExists(this, title): log.debug("Checking to see if feed %s exists on API server." % (title)) filters = [dict(name='title', op='equals', val=title)] try: response = rest.doGet("feed", filters) if response.json()['num_results'] != 0: return True else: return False except: log.error("Error checking to see if feed %s exists on API server." % (title)) raise Exception() return True
def addKeyword(self, word): ke = self.keywordExists(word) if ke == False: log.debug("adding keyword %s" % (word)) data = { "word": word } response = None try: response = rest.doPost("keyword", data) except Exception as e: log.exception(e) return None return(response.json()['id']) else: return ke
def addKeyword(self, word): ke = self.keywordExists(word) if ke == False: log.debug("adding keyword %s" % (word)) data = {"word": word} response = None try: response = rest.doPost("keyword", data) except Exception as e: log.exception(e) return None return (response.json()['id']) else: return ke
def getFeedRetrieved(self, feedid): log.debug("Checking when feed %s last retreived." % (feedid)) filters = [dict(name='id', op='equals', val=feedid)] response = None try: response = rest.doGet(filters) if response.json()['num_results'] != 0: r = response.json()['objects'].pop() log.info("Feed %s last retreived %s" % (feedid, r['retrieved'])) return r['retrieved'] else: log.info("No retreived timestamp for %s" % (feedid)) return False except Exception as e: log.exception(e) raise Exception() return None
def setFeedRetrieved(self, feedid, retrieved): log.debug("Updating feed %s with last retrieved time of %s." % (feedid, retrieved)) filters = [dict(name='id', op='equals', val=feedid)] data = dict(retrieved=retrieved) print("doput(feed, %s, %s)" % (filters, data)) response = None try: response = rest.doPut("feed", filters, data) except Exception as e: log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved)) log.exception(e) raise Exception() return None if response.json()['num_modified'] != 0: log.info("Updating feed %s with last retrieved time of %s." % (feedid, retrieved)) return True else: log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved)) return False