def test_iteration_stops(): q.clear(q.TEST) items = ["one", "two", "three"] for item in items: q.add(q.TEST, item) gen = q.iterate(q.TEST) for i in next(gen): pass
def test_iteration_stops(): q.clear(q.TEST) items = ["one", "two", "three"] for item in items: q.add(q.TEST, item) gen = q.iterate(q.TEST) for i in next(gen): pass
def test_pop(): q.clear(q.TEST) b = bookmark.Bookmark(bookmark={"bookmark": "bookmark"}, info={"info": "info"}, errors={"errors": "errors"}, status={"status": "status"}) q.add(q.TEST, b) stack = q.pop_bookmark(q.TEST) assert stack.bookmark == b.bookmark assert stack.info == b.info assert stack.errors == b.errors assert stack.status == None #status isn't serialized
def test_iteration(): q.clear(q.TEST) items = ["one", "two", "three"] for item in items: q.add(q.TEST, item) gen = q.iterate(q.TEST) results = [next(gen), next(gen), next(gen)] assert results == items with pytest.raises(StopIteration): results = next(gen)
def test_iteration(): q.clear(q.TEST) items = ["one", "two", "three"] for item in items: q.add(q.TEST, item) gen = q.iterate(q.TEST) results = [next(gen), next(gen), next(gen)] assert results == items with pytest.raises(StopIteration): results = next(gen)
def test_pop(): q.clear(q.TEST) b = bookmark.Bookmark(bookmark={"bookmark": "bookmark"}, info={"info": "info"}, errors={"errors": "errors"}, status={"status": "status"}) q.add(q.TEST, b) stack = q.pop_bookmark(q.TEST) assert stack.bookmark == b.bookmark assert stack.info == b.info assert stack.errors == b.errors assert stack.status == None #status isn't serialized
def delete_bookmark(): """ Grab a bookmark from the delete queue and tell pinboard to delete it """ item = q.pop_bookmark(q.DELETE) if not item: logging.info("Queue is empty, returning") return url = item.bookmark['href'] if not pinboard_is_available(): q.add(q.DELETE, item) return params = __base_request_params(url) result = requests.get(config.PINBOARD['api_base_url'] + "posts/delete", params=params) if result.status_code == requests.codes.too_many_requests: # 429 - Too many requests - give it a rest logging.warning("Pinboard returned error 429 - adding backoff flag") q.add(q.DELETE, item) elif result.status_code != requests.codes.ok: q.add(q.RETRY, item) return result.status_code
def update_bookmark_location(): """ Update a pinboard bookmark. Create a new bookmark with the new location. If successful, add the old bookmark to the delete queue. #TODO? call the delete task in celery? will it run amok? """ if not pinboard_is_available(): logging.info("Pinboard is unavailable.") return item = q.pop_bookmark(q.INSPECT) if not item: logging.info("Queue is empty, returning") return logging.info("Starting an update") bookmark = item.bookmark if (not bookmark['tags'] or not INSPECTION_TAG in bookmark['tags']): bookmark['tags'] = INSPECTION_TAG + " " + bookmark['tags'] params = __full_request_params(item.info['url'], bookmark) params['replace'] = 'yes' try: response = requests.get(config.PINBOARD['api_base_url'] + 'posts/add', params) if response.status_code == requests.codes.ok: logging.info("Successfully updated tag") else: logging.info("booo, got a code of {}".format(response.status_code)) q.add(q.INSPECT, item) except Exception as e: logging.error("Failed on PB tag for inspection: url: {} error: {}".format(item.bookmark, e)) q.add(q.INSPECT, item) # inspect
def update_bookmark_location(): """ Update a pinboard bookmark. Create a new bookmark with the new location. If successful, add the old bookmark to the delete queue. #TODO? call the delete task in celery? will it run amok? """ if not pinboard_is_available(): logging.info("Pinboard is unavailable.") return item = q.pop_bookmark(q.UPDATE) if not item: logging.info("Queue is empty, returning") return logging.info("Starting an update") bookmark = item.bookmark params = __full_request_params(item.info['url'], bookmark) params['replace'] = 'no', # throw the error so we can check it out """ params = {'url': item.info['url'], 'auth_token': API_TOKEN, 'format': 'json', 'description': bookmark['description'], 'tags': bookmark['tags'], 'dt': bookmark['time'], 'replace': 'no', # throw the error so we can check it out 'shared': bookmark['shared'], 'toread': bookmark['toread'] } """ try: response = requests.get(config.PINBOARD['api_base_url'] + 'posts/add', params ) if response.status_code == requests.codes.ok: logging.info("Got a success code, queueing for deletion") #old_url = bookmark['href']# item.info['old_location'] old_bookmark = Bookmark(bookmark={'href': bookmark['href']}, info=item.info) q.add(q.DELETE, old_bookmark) else: logging.info("booo, got a code of {}".format(response.status_code)) q.add(q.UPDATE, item) except Exception as e: logging.error("Failed on PB add: url: {} error: {}".format(item.bookmark, e)) q.add(q.RETRY, item) #inspect
import neat_as_a_pin.src.pinboard from neat_as_a_pin.conf import config from neat_as_a_pin.src import url_checker from neat_as_a_pin.src import url_store """ Given a pinboard export file, do an HTTP HEAD request on each link. Save results to a redis queue based on HTTP status and next action to take. """ if __name__ == "__main__": pinboard_bookmarks_file = os.environ.get('FILE') or ( config.DATA_DIR + '/pinboard_export.json') bookmarks = neat_as_a_pin.src.pinboard.read_pinboard_exports( pinboard_bookmarks_file) counter = 0 for bookmark in bookmarks: result = url_checker.check_status_and_make_bookmark(bookmark) queue = url_store.status_to_queue(result.status) if queue is not url_store.DONE: url_store.add(queue, result, autosave=False) counter += 1 if counter % 100 == 0: url_store.save() if counter % 400 == 0: print("At #{} {}".format(counter, datetime.datetime.now()))
import os import datetime import neat_as_a_pin.src.pinboard from neat_as_a_pin.conf import config from neat_as_a_pin.src import url_store """ Given a pinboard export file, do an HTTP HEAD request on each link. Save results to a redis queue based on HTTP status and next action to take. """ if __name__ == "__main__": pinboard_bookmarks_file = os.environ.get('FILE') or (config.DATA_DIR + '/pinboard_export.json') bookmarks = neat_as_a_pin.src.pinboard.read_pinboard_exports(pinboard_bookmarks_file) counter = 0 for bookmark in bookmarks: url_store.add(url_store.INBOX, bookmark, autosave=False) counter += 1 if counter % 100 == 0: url_store.save() break if counter % 400 == 0: print("At #{} {}".format(counter, datetime.datetime.now())) url_store.save() print("Finished at #{}, total {}".format(datetime.datetime.now(), counter))
import datetime import neat_as_a_pin.src.pinboard from neat_as_a_pin.conf import config from neat_as_a_pin.src import url_checker from neat_as_a_pin.src import url_store """ Given a pinboard export file, do an HTTP HEAD request on each link. Save results to a redis queue based on HTTP status and next action to take. """ if __name__ == "__main__": pinboard_bookmarks_file = os.environ.get('FILE') or (config.DATA_DIR + '/pinboard_export.json') bookmarks = neat_as_a_pin.src.pinboard.read_pinboard_exports(pinboard_bookmarks_file) counter = 0 for bookmark in bookmarks: result = url_checker.check_status_and_make_bookmark(bookmark) queue = url_store.status_to_queue(result.status) if queue is not url_store.DONE: url_store.add(queue, result, autosave=False) counter += 1 if counter % 100 == 0: url_store.save() if counter % 400 == 0: print("At #{} {}".format(counter, datetime.datetime.now()))