def doPut(self, endpoint, filters="", data=dict()):
putdata = dict()
if filters != "":
putdata.update(dict(q=dict(filters=filters)))
putdata.update(data)
headers = {
"Content-Type": "application/json",
"user-agent": collector_config.USER_AGENT,
"Authorization": "JWT %s" % self.token,
}
url = "%s/api/%s" % (collector_config.API_BASE_URL, endpoint)
params = ""
response = ""
log.debug("PUT request to %s" % (url))
try:
if putdata != "":
response = requests.put(url, headers=headers, data=json.dumps(putdata))
response.raise_for_status()
else:
response = requests.post(url, headers=headers)
response.raise_for_status()
log.debug("Retrieved put data from %s" % (url))
return response
except requests.exceptions.HTTPError as e:
log.warning(
"Error PUTting data to %s: %s %s (%s)"
% (url, response.json()["status_code"], response.json()["error"], response.json()["description"])
)
return False
def doPost(self, endpoint, data=""):
headers = {
'Content-Type': 'application/json',
'user-agent': collector_config.USER_AGENT,
'Authorization': "JWT %s" % self.token
}
url = "%s/api/%s" % (collector_config.API_BASE_URL, endpoint)
params = ""
response = ""
log.debug("POST request to %s" % (url))
try:
if data != "":
response = requests.post(url,
headers=headers,
data=json.dumps(data))
response.raise_for_status()
else:
response = requests.post(url, headers=headers)
response.raise_for_status()
log.debug("Retrieved POST data from %s" % (url))
return response
except requests.exceptions.HTTPError as e:
log.warning(
"Error POSTing data to %s: %s %s (%s)" %
(url, response.json()['status_code'], response.json()['error'],
response.json()['description']))
return False
#!flask/bin/python3
##
## This is the data collector for noozjunkie. It interacts with
## the REST API in the webapp to get the feeds and store the
## articles.
##
from noozjunkie_collector import log, stats
log.warning("NoozJunkie Collector trying to score...")
from noozjunkie_collector import FeedMgr, Collector, ArticleMgr
if __name__ == '__main__':
f = FeedMgr()
c = Collector()
a = ArticleMgr()
feeds = f.getFeeds()
c.processFeeds(feeds)
articles = c.getArticles()
a.processArticles(articles)
log.warning(
"%s feeds available, %s feeds processed, %s articles retrieved, %s articles added"
% (stats.feeds_total, stats.feeds_processed, stats.articles_retreived,
stats.articles_added))
log.warning("NoozJunkie Collector is holding. Shutting down. Bye bye.")
#!flask/bin/python3
##
## This is the data collector for noozjunkie. It interacts with
## the REST API in the webapp to get the feeds and store the
## articles.
##
from noozjunkie_collector import log, stats
log.warning("NoozJunkie Collector trying to score...")
from noozjunkie_collector import FeedMgr, Collector, ArticleMgr
if __name__ == "__main__":
f = FeedMgr()
c = Collector()
a = ArticleMgr()
feeds = f.getFeeds()
c.processFeeds(feeds)
articles = c.getArticles()
a.processArticles(articles)
log.warning(
"%s feeds available, %s feeds processed, %s articles retrieved, %s articles added"
% (stats.feeds_total, stats.feeds_processed, stats.articles_retreived, stats.articles_added)
)
log.warning("NoozJunkie Collector is holding. Shutting down. Bye bye.")
