Example #1
0
 def setFeedRetrieved(self, feedid, retrieved):
     log.debug("Updating feed %s with last retrieved time of %s." %
               (feedid, retrieved))
     filters = [dict(name='id', op='equals', val=feedid)]
     data = dict(retrieved=retrieved)
     print("doput(feed, %s, %s)" % (filters, data))
     response = None
     try:
         response = rest.doPut("feed", filters, data)
     except Exception as e:
         log.error(
             "Failed to updaste feed %s with last retrieved time of %s." %
             (feedid, retrieved))
         log.exception(e)
         raise Exception()
         return None
     if response.json()['num_modified'] != 0:
         log.info("Updating feed %s with last retrieved time of %s." %
                  (feedid, retrieved))
         return True
     else:
         log.error(
             "Failed to updaste feed %s with last retrieved time of %s." %
             (feedid, retrieved))
         return False
Example #2
0
 def requestToken(self):
     headers = {
         'Content-Type': 'application/json',
         'user-agent': collector_config.USER_AGENT
     }
     url = "%s/auth" % (collector_config.API_BASE_URL)
     data = {
         'username': collector_config.API_USERNAME,
         'password': collector_config.API_PASSWORD
     }
     response = ""
     log.info("Requesting JWT token from %s for %s" %
              (url, data['username']))
     try:
         response = requests.post(url,
                                  headers=headers,
                                  data=json.dumps(data))
         response.raise_for_status()
         newtoken = response.json()['access_token']
         log.info("Retreived JWT token from %s for %s" %
                  (url, data['username']))
         return newtoken
     except requests.exceptions.HTTPError as e:
         log.error(
             "Error requesting token from %s: %s %s (%s)" %
             (url, response.json()['status_code'], response.json()['error'],
              response.json()['description']))
         raise Exception()
         return False
     except Exception as e:
         log.exception(e)
         raise Exception()
         return False
Example #3
0
 def getFeeds(this):
     log.info("Retreiving list of feeds from API server.")
     response = ""
     try:
         response = rest.doGet("feed")
         return response.json()['objects']
     except:
         log.error("Error retreiving list of feeds from API server.")
         raise Exception() 
         return False
Example #4
0
 def addFeed(this, title, source, description="", type=2, interval=300, active=0):
     if this.feedExists(title) == False:
         log.warn("Adding type %s feed %s to be retrieved from %s to API server." % (type, title, source))
         data = { 'title': title, 'source': source, 'type': type, 'interval': interval, 'active': active }
         response = None
         try:
             response = rest.doPost("feed", data)
             return response
         except:
             log.error("Error adding feed %s to API server." % (feed))
             raise Exception()
             return False
Example #5
0
 def retrieveContent(self, link):
     try:
         a = newspaper.Article(link)
         a.download()
         a.parse()
         text = a.text
         a.nlp()
         self.keywords = a.keywords
         self.retrieved = str(datetime.datetime.utcnow())
         return a.text
     except Exception as e:
         log.error("Exception retrieving %s" % (link))
         log.exception(e)
Example #6
0
 def retrieveContent(self, link):
     try:
         a = newspaper.Article(link)
         a.download()
         a.parse()
         text = a.text
         a.nlp()
         self.keywords = a.keywords
         self.retrieved = str(datetime.datetime.utcnow())
         return a.text
     except Exception as e:
         log.error("Exception retrieving %s" % (link))
         log.exception(e)
Example #7
0
 def feedExists(this, title):
     log.debug("Checking to see if feed %s exists on API server." % (title)) 
     filters = [dict(name='title', op='equals', val=title)]
     try: 
         response = rest.doGet("feed", filters)
         if response.json()['num_results'] != 0:
             return True
         else:
             return False
     except:
         log.error("Error checking to see if feed %s exists on API server." % (title))
         raise Exception()
         return True
Example #8
0
 def setFeedRetrieved(self, feedid, retrieved):
     log.debug("Updating feed %s with last retrieved time of %s." % (feedid, retrieved))
     filters = [dict(name='id', op='equals', val=feedid)]
     data = dict(retrieved=retrieved)
     print("doput(feed, %s, %s)" % (filters, data))
     response = None
     try:
         response = rest.doPut("feed", filters, data)
     except Exception as e:
         log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved))
         log.exception(e)
         raise Exception()
         return None
     if response.json()['num_modified'] != 0:
         log.info("Updating feed %s with last retrieved time of %s." % (feedid, retrieved))
         return True
     else:
         log.error("Failed to updaste feed %s with last retrieved time of %s." % (feedid, retrieved))
         return False
Example #9
0
 def requestToken(self):
     headers = {"Content-Type": "application/json", "user-agent": collector_config.USER_AGENT}
     url = "%s/auth" % (collector_config.API_BASE_URL)
     data = {"username": collector_config.API_USERNAME, "password": collector_config.API_PASSWORD}
     response = ""
     log.info("Requesting JWT token from %s for %s" % (url, data["username"]))
     try:
         response = requests.post(url, headers=headers, data=json.dumps(data))
         response.raise_for_status()
         newtoken = response.json()["access_token"]
         log.info("Retreived JWT token from %s for %s" % (url, data["username"]))
         return newtoken
     except requests.exceptions.HTTPError as e:
         log.error(
             "Error requesting token from %s: %s %s (%s)"
             % (url, response.json()["status_code"], response.json()["error"], response.json()["description"])
         )
         raise Exception()
         return False
     except Exception as e:
         log.exception(e)
         raise Exception()
         return False