def fetch(self, url): if r.hget(generate_hash(url), 'url') is None: logbook.debug('URL unknown, checking {url}'.format(url=url)) self.dispatch("url_to_check", url) else: logbook.debug('URL known, refreshing {url}'.format(url=url)) self.dispatch("url_to_refresh", url)
def __init__(self, data): self.author = data["a"] self.title = data["d"].encode('ascii','ignore') self.content = data["n"] self.url = data["u"] self.dt = datetime.strptime(data["dt"], '%Y-%m-%dT%H:%M:%SZ') self.hash = generate_hash(data["dt"], data["d"])
def pull_feed(): req = urllib2.Request(app.config["FEED_URL"]) opener = urllib2.build_opener() f = opener.open(req).read() results = simplejson.loads(f) for post in results: if not Post.query.filter_by(hash=generate_hash(post["dt"],post["d"])).first(): db.session.add(Post(post)) print Post db.session.commit() return redirect(url_for('manage'))
def check_one(self, request): data = json.loads(request.get_data()) url = data.get('url', None) if url is None: logbook.debug(('"url" parameter not found in {data}' .format(data=data))) return 400, 'Please specify a "url" parameter.' url_hash = generate_hash(url) logbook.debug(('Checking "{url}" ({hash})' .format(url=url, hash=url_hash))) self.fetch(url) return json.dumps({'url-hash': url_hash}, indent=2)
def check_many(self, request): data = json.loads(request.get_data()) urls = data.get('urls', None) if urls is None: logbook.debug(('"urls" parameter not found in {data}' .format(data=data))) return 400, 'Please specify a "urls" parameter.' group = data.get('group', None) if group is None: logbook.debug(('"group" parameter not found in {data}' .format(data=data))) return 400, 'Please specify a "group" parameter.' group_hash = generate_hash(group) for url in urls: url_hash = generate_hash(url) logbook.debug(('Checking "{url}" ({hash}) for group {group}' .format(url=url, hash=url_hash, group=group))) r.hset(url_hash, 'group', group_hash) r.hset(group_hash, 'name', group) r.hset(group_hash, url_hash, url) self.fetch(url) return json.dumps({'group-hash': group_hash}, indent=2)