def test_iteration_stops():
q.clear(q.TEST)
items = ["one", "two", "three"]
for item in items:
q.add(q.TEST, item)
gen = q.iterate(q.TEST)
for i in next(gen):
pass
def test_iteration_stops():
q.clear(q.TEST)
items = ["one", "two", "three"]
for item in items:
q.add(q.TEST, item)
gen = q.iterate(q.TEST)
for i in next(gen):
pass
def test_pop():
q.clear(q.TEST)
b = bookmark.Bookmark(bookmark={"bookmark": "bookmark"}, info={"info": "info"}, errors={"errors": "errors"},
status={"status": "status"})
q.add(q.TEST, b)
stack = q.pop_bookmark(q.TEST)
assert stack.bookmark == b.bookmark
assert stack.info == b.info
assert stack.errors == b.errors
assert stack.status == None #status isn't serialized
def test_iteration():
q.clear(q.TEST)
items = ["one", "two", "three"]
for item in items:
q.add(q.TEST, item)
gen = q.iterate(q.TEST)
results = [next(gen), next(gen), next(gen)]
assert results == items
with pytest.raises(StopIteration):
results = next(gen)
def test_iteration():
q.clear(q.TEST)
items = ["one", "two", "three"]
for item in items:
q.add(q.TEST, item)
gen = q.iterate(q.TEST)
results = [next(gen), next(gen), next(gen)]
assert results == items
with pytest.raises(StopIteration):
results = next(gen)
def test_pop():
q.clear(q.TEST)
b = bookmark.Bookmark(bookmark={"bookmark": "bookmark"},
info={"info": "info"},
errors={"errors": "errors"},
status={"status": "status"})
q.add(q.TEST, b)
stack = q.pop_bookmark(q.TEST)
assert stack.bookmark == b.bookmark
assert stack.info == b.info
assert stack.errors == b.errors
assert stack.status == None #status isn't serialized
def delete_bookmark():
"""
Grab a bookmark from the delete queue
and tell pinboard to delete it
"""
item = q.pop_bookmark(q.DELETE)
if not item:
logging.info("Queue is empty, returning")
return
url = item.bookmark['href']
if not pinboard_is_available():
q.add(q.DELETE, item)
return
params = __base_request_params(url)
result = requests.get(config.PINBOARD['api_base_url'] + "posts/delete", params=params)
if result.status_code == requests.codes.too_many_requests: # 429 - Too many requests - give it a rest
logging.warning("Pinboard returned error 429 - adding backoff flag")
q.add(q.DELETE, item)
elif result.status_code != requests.codes.ok:
q.add(q.RETRY, item)
return result.status_code
def update_bookmark_location():
"""
Update a pinboard bookmark. Create a new bookmark with the new location.
If successful, add the old bookmark to the delete queue.
#TODO? call the delete task in celery? will it run amok?
"""
if not pinboard_is_available():
logging.info("Pinboard is unavailable.")
return
item = q.pop_bookmark(q.INSPECT)
if not item:
logging.info("Queue is empty, returning")
return
logging.info("Starting an update")
bookmark = item.bookmark
if (not bookmark['tags'] or
not INSPECTION_TAG in bookmark['tags']):
bookmark['tags'] = INSPECTION_TAG + " " + bookmark['tags']
params = __full_request_params(item.info['url'], bookmark)
params['replace'] = 'yes'
try:
response = requests.get(config.PINBOARD['api_base_url'] + 'posts/add', params)
if response.status_code == requests.codes.ok:
logging.info("Successfully updated tag")
else:
logging.info("booo, got a code of {}".format(response.status_code))
q.add(q.INSPECT, item)
except Exception as e:
logging.error("Failed on PB tag for inspection: url: {} error: {}".format(item.bookmark, e))
q.add(q.INSPECT, item) # inspect
def update_bookmark_location():
"""
Update a pinboard bookmark. Create a new bookmark with the new location.
If successful, add the old bookmark to the delete queue.
#TODO? call the delete task in celery? will it run amok?
"""
if not pinboard_is_available():
logging.info("Pinboard is unavailable.")
return
item = q.pop_bookmark(q.UPDATE)
if not item:
logging.info("Queue is empty, returning")
return
logging.info("Starting an update")
bookmark = item.bookmark
params = __full_request_params(item.info['url'], bookmark)
params['replace'] = 'no', # throw the error so we can check it out
"""
params = {'url': item.info['url'],
'auth_token': API_TOKEN,
'format': 'json',
'description': bookmark['description'],
'tags': bookmark['tags'],
'dt': bookmark['time'],
'replace': 'no', # throw the error so we can check it out
'shared': bookmark['shared'],
'toread': bookmark['toread']
}
"""
try:
response = requests.get(config.PINBOARD['api_base_url'] + 'posts/add', params )
if response.status_code == requests.codes.ok:
logging.info("Got a success code, queueing for deletion")
#old_url = bookmark['href']# item.info['old_location']
old_bookmark = Bookmark(bookmark={'href': bookmark['href']}, info=item.info)
q.add(q.DELETE, old_bookmark)
else:
logging.info("booo, got a code of {}".format(response.status_code))
q.add(q.UPDATE, item)
except Exception as e:
logging.error("Failed on PB add: url: {} error: {}".format(item.bookmark, e))
q.add(q.RETRY, item) #inspect
import neat_as_a_pin.src.pinboard
from neat_as_a_pin.conf import config
from neat_as_a_pin.src import url_checker
from neat_as_a_pin.src import url_store
"""
Given a pinboard export file, do an HTTP HEAD request on each link.
Save results to a redis queue based on HTTP status and next action to take.
"""
if __name__ == "__main__":
pinboard_bookmarks_file = os.environ.get('FILE') or (
config.DATA_DIR + '/pinboard_export.json')
bookmarks = neat_as_a_pin.src.pinboard.read_pinboard_exports(
pinboard_bookmarks_file)
counter = 0
for bookmark in bookmarks:
result = url_checker.check_status_and_make_bookmark(bookmark)
queue = url_store.status_to_queue(result.status)
if queue is not url_store.DONE:
url_store.add(queue, result, autosave=False)
counter += 1
if counter % 100 == 0:
url_store.save()
if counter % 400 == 0:
print("At #{} {}".format(counter, datetime.datetime.now()))
import os
import datetime
import neat_as_a_pin.src.pinboard
from neat_as_a_pin.conf import config
from neat_as_a_pin.src import url_store
"""
Given a pinboard export file, do an HTTP HEAD request on each link.
Save results to a redis queue based on HTTP status and next action to take.
"""
if __name__ == "__main__":
pinboard_bookmarks_file = os.environ.get('FILE') or (config.DATA_DIR + '/pinboard_export.json')
bookmarks = neat_as_a_pin.src.pinboard.read_pinboard_exports(pinboard_bookmarks_file)
counter = 0
for bookmark in bookmarks:
url_store.add(url_store.INBOX, bookmark, autosave=False)
counter += 1
if counter % 100 == 0:
url_store.save()
break
if counter % 400 == 0:
print("At #{} {}".format(counter, datetime.datetime.now()))
url_store.save()
print("Finished at #{}, total {}".format(datetime.datetime.now(), counter))
import datetime
import neat_as_a_pin.src.pinboard
from neat_as_a_pin.conf import config
from neat_as_a_pin.src import url_checker
from neat_as_a_pin.src import url_store
"""
Given a pinboard export file, do an HTTP HEAD request on each link.
Save results to a redis queue based on HTTP status and next action to take.
"""
if __name__ == "__main__":
pinboard_bookmarks_file = os.environ.get('FILE') or (config.DATA_DIR + '/pinboard_export.json')
bookmarks = neat_as_a_pin.src.pinboard.read_pinboard_exports(pinboard_bookmarks_file)
counter = 0
for bookmark in bookmarks:
result = url_checker.check_status_and_make_bookmark(bookmark)
queue = url_store.status_to_queue(result.status)
if queue is not url_store.DONE:
url_store.add(queue, result, autosave=False)
counter += 1
if counter % 100 == 0:
url_store.save()
if counter % 400 == 0:
print("At #{} {}".format(counter, datetime.datetime.now()))
