def save_data(self, token, trigger_id, **data):
"""
let's save the data
"""
if token and 'link' in data and data['link'] is not None and len(data['link']) > 0:
# get the data of this trigger
trigger = Readability.objects.get(trigger_id=trigger_id)
token_key, token_secret = token.split('#TH#')
readability_instance = ReaderClient(self.consumer_key,
self.consumer_secret,
token_key,
token_secret)
bookmark_id = readability_instance.add_bookmark(url=data['link'])
if trigger.tag is not None and len(trigger.tag) > 0:
try:
readability_instance.add_tags_to_bookmark(
bookmark_id, tags=(trigger.tag.lower()))
sentence = str('readability {} created item id {}').format(
data['link'], bookmark_id)
logger.debug(sentence)
except Exception as e:
logger.critical(e)
else:
sentence = "no token or link provided for trigger ID {} "
logger.critical(sentence.format(trigger_id))
def create_client(self):
"""
Connect to Readability API
(http://readability-python-library.readthedocs.org/en/latest/clients.html#readability.ReaderClient)
"""
rdb_client = ReaderClient(CONSUMER_KEY, CONSUMER_SECRET, self.token[0],
self.token[1])
return rdb_client
def setUp(self):
"""Need to get a token for each test.
"""
token_pair = xauth(CONSUMER_KEY, CONSUMER_SECRET, USERNAME, PASSWORD)
self.token_key = token_pair[0]
self.token_secret = token_pair[1]
self.base_client = ReaderClient(CONSUMER_KEY, CONSUMER_SECRET,
self.token_key, self.token_secret)
def __init__(self, token=None):
base = 'https://www.readability.com'
self.AUTH_URL = '{}/api/rest/v1/oauth/authorize/'.format(base)
self.REQ_TOKEN = '{}/api/rest/v1/oauth/request_token/'.format(base)
self.ACC_TOKEN = '{}/api/rest/v1/oauth/access_token/'.format(base)
self.consumer_key = settings.TH_READABILITY['consumer_key']
self.consumer_secret = settings.TH_READABILITY['consumer_secret']
if token:
token_key, token_secret = token.split('#TH#')
self.client = ReaderClient(token_key, token_secret,
self.consumer_key, self.consumer_secret)
def setUp(self):
"""Get a client and add a bookmark
"""
token_pair = xauth(CONSUMER_KEY, CONSUMER_SECRET, USERNAME, PASSWORD)
self.token_key = token_pair[0]
self.token_secret = token_pair[1]
self.base_client = ReaderClient(CONSUMER_KEY, CONSUMER_SECRET,
self.token_key, self.token_secret)
self.url = 'http://www.theatlantic.com/technology/archive/2013/01/the-never-before-told-story-of-the-worlds-first-computer-art-its-a-sexy-dame/267439/'
add_response = self.base_client.add_bookmark(self.url)
self.assertEqual(add_response.status, 202)
def process_data(self, token, trigger_id, date_triggered):
"""
get the data from the service
:param trigger_id: trigger ID to process
:param date_triggered: the date of the last trigger
:type trigger_id: int
:type date_triggered: datetime
:return: list of data found from the date_triggered filter
:rtype: list
"""
data = []
if token is not None:
token_key, token_secret = token.split('#TH#')
client = ReaderClient(self.consumer_key, self.consumer_secret,
token_key, token_secret)
bookmarks = client.get_bookmarks(
added_since=date_triggered).content
for bookmark in bookmarks.values():
for b in bookmark:
if 'article' in b:
title = ''
if 'title' in b['article']:
title = b['article']['title']
link = ''
if 'url' in b['article']:
link = b['article']['url']
content = ''
if 'excerpt' in b['article']:
content = b['article']['excerpt']
data.append({
'title': title,
'link': link,
'content': content
})
return data
def __init__(self, token=None, **kwargs):
super(ServiceReadability, self).__init__(token, **kwargs)
base = 'https://www.readability.com'
self.AUTH_URL = '{}/api/rest/v1/oauth/authorize/'.format(base)
self.REQ_TOKEN = '{}/api/rest/v1/oauth/request_token/'.format(base)
self.ACC_TOKEN = '{}/api/rest/v1/oauth/access_token/'.format(base)
self.consumer_key = settings.TH_READABILITY['consumer_key']
self.consumer_secret = settings.TH_READABILITY['consumer_secret']
self.token = token
self.service = 'ServiceReadability'
self.oauth = 'oauth1'
kwargs = {
'consumer_key': self.consumer_key,
'consumer_secret': self.consumer_secret
}
if token:
token_key, token_secret = self.token.split('#TH#')
self.client = ReaderClient(token_key, token_secret, **kwargs)
def save_data(self, token, trigger_id, **data):
"""
let's save the data
:param trigger_id: trigger ID from which to save data
:param **data: the data to check to be used and save
:type trigger_id: int
:type **data: dict
:return: the status of the save statement
:rtype: boolean
"""
status = False
if token and 'link' in data and data['link'] is not None and len(
data['link']) > 0:
# get the data of this trigger
trigger = Readability.objects.get(trigger_id=trigger_id)
token_key, token_secret = token.split('#TH#')
readability_instance = ReaderClient(self.consumer_key,
self.consumer_secret,
token_key, token_secret)
bookmark_id = readability_instance.add_bookmark(url=data['link'])
if trigger.tag is not None and len(trigger.tag) > 0:
try:
readability_instance.add_tags_to_bookmark(
bookmark_id, tags=(trigger.tag.lower()))
sentence = str('readability {} created item id {}').format(
data['link'], bookmark_id)
logger.debug(sentence)
status = True
except Exception as e:
logger.critical(e)
status = False
else:
sentence = "no token or link provided for trigger ID {} "
logger.critical(sentence.format(trigger_id))
status = False
return status
def setUp(self):
"""Add a few bookmarks.
"""
token_pair = xauth(CONSUMER_KEY, CONSUMER_SECRET, USERNAME, PASSWORD)
self.token_key = token_pair[0]
self.token_secret = token_pair[1]
self.base_client = ReaderClient(CONSUMER_KEY, CONSUMER_SECRET,
self.token_key, self.token_secret)
self.urls = [
'http://www.theatlantic.com/technology/archive/2013/01/the-never-before-told-story-of-the-worlds-first-computer-art-its-a-sexy-dame/267439/',
'http://www.theatlantic.com/business/archive/2013/01/why-smart-poor-students-dont-apply-to-selective-colleges-and-how-to-fix-it/272490/',
]
self.favorite_urls = [
'http://www.theatlantic.com/sexes/archive/2013/01/the-lonely-existence-of-mel-feit-mens-rights-advocate/267413/',
'http://www.theatlantic.com/technology/archive/2013/01/women-in-combat-an-idea-whose-time-has-come-aided-by-technology/272483/'
]
self.archive_urls = [
'http://www.theatlantic.com/business/archive/2013/01/what-economics-can-and-cant-tell-us-about-the-legacy-of-legal-abortion/267459/',
'http://www.theatlantic.com/business/archive/2013/01/5-ways-to-understand-just-how-absurd-spains-26-unemployment-rate-is/272502/'
]
self.all_urls = self.urls + self.favorite_urls + self.archive_urls
for url in self.urls:
add_response = self.base_client.add_bookmark(url)
self.assertEqual(add_response.status, 202)
for url in self.favorite_urls:
add_response = self.base_client.add_bookmark(url, favorite=True)
self.assertEqual(add_response.status, 202)
for url in self.archive_urls:
add_response = self.base_client.add_bookmark(url, archive=True)
self.assertEqual(add_response.status, 202)
def process_data(self, token, trigger_id, date_triggered):
"""
get the data from the service
"""
data = []
if token is not None:
token_key, token_secret = token.split('#TH#')
client = ReaderClient(
self.consumer_key, self.consumer_secret,
token_key, token_secret)
bookmarks = client.get_bookmarks(
added_since=date_triggered).content
for bookmark in bookmarks.values():
for b in bookmark:
if 'article' in b:
title = ''
if 'title' in b['article']:
title = b['article']['title']
link = ''
if 'url' in b['article']:
link = b['article']['url']
content = ''
if 'excerpt' in b['article']:
content = b['article']['excerpt']
data.append(
{'title': title,
'link': link,
'content': content})
return data
def readability_exporter(api_key, api_secret, login_user, login_pw, format,
bookmarks, export_directory, export_filename,
not_show_file, http_error_threshold):
"""Readability Exporter
This little script takes credentials for a Readability account and the API key and secret to create the standard
Readability export file in JSON format that can be used to import your links into Instapaper.
We think about adding support for the del.icio.us flavour of bookmarks.html to allow direct import into Pocket.
You can find more information about this script on GitHub at https://github.com/goetzb/readability-exporter
"""
click.echo(click.style('Readability Exporter', bold=True))
# Create string for export filename
if '{timestamp}' not in export_filename:
export_filename += '_{timestamp}'
if '{format}' not in export_filename:
export_filename += '_{format}'
if '{filetype}' not in export_filename:
export_filename += '.{filetype}'
# Prepare json export dictionary
export_dict = OrderedDict()
export_dict['bookmarks'] = []
# Add empty recommendations list - in my example this was just empty.
export_dict["recommendations"] = []
# Prepare jsonraw export dictionary
export_dict_raw = OrderedDict()
# Prepare html export string
export_html = """<!DOCTYPE NETSCAPE-Bookmark-file-1>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
<!-- This is an automatically generated file.
It will be read and overwritten.
Do Not Edit! -->
<TITLE>Bookmarks</TITLE>
<H1>Bookmarks</H1>
<DL><p>
"""
auth_tokens = get_auth_tokens(api_key=api_key,
api_secret=api_secret,
login_user=login_user,
login_pw=login_pw)
if len(auth_tokens) != 2:
click.ClickException(
"""An error occurred and you could not be authenticated successfully.
Please check your Readability API keys and login details.""")
client = ReaderClient(token_key=auth_tokens[0],
token_secret=auth_tokens[1],
consumer_key=api_key,
consumer_secret=api_secret)
meta_infos = get_readability_meta_infos(readability_reader_client=client)
click.echo("* You have saved " + click.style("{link_count}".format(
link_count=meta_infos['bookmarks_total']),
bold=True) +
" links on Readability")
if bookmarks == 0:
bookmarks = meta_infos['bookmarks_total']
click.echo("* We are now going to export the latest " +
click.style("{export_count}".format(export_count=bookmarks),
bold=True) + " of your links")
if bookmarks < 50:
bookmarks_per_page = bookmarks + 1
else:
bookmarks_per_page = 50
bookmarks_get_pages = int(ceil(bookmarks / bookmarks_per_page))
data_export = export_bookmarks_via_api(
readability_reader_client=client,
export_formats=format,
bookmarks_number=bookmarks,
bookmarks_per_page=bookmarks_per_page,
bookmarks_get_pages=bookmarks_get_pages,
export_json=export_dict,
export_html=export_html,
export_jsonraw=export_dict_raw,
error_threshold=http_error_threshold)
click.echo(
"Bear with us, we are almost there. In the next step we bring the data into the right formats."
)
exported_files = []
files_export_count = 0
timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S")
with click.progressbar(label='Writing export files...',
length=len(format),
show_percent=True) as files_bar:
for export_format in format:
output_data = data_export[export_format]
if export_format == 'json' or export_format == 'jsonraw':
filetype = 'json'
elif export_format == 'html':
filetype = 'html'
output_data += "</DL><p>"
format_export_filename = export_filename.format(
timestamp=timestamp, format=export_format, filetype=filetype)
export_file = write_export_data(filetype=filetype,
directory=export_directory,
filename=format_export_filename,
data=output_data)
exported_files.append(export_file)
files_export_count += 1
files_bar.update(files_export_count)
# TODO: Improve bookmark count to verify if actually the correct number of bookmarks has been exported
# Get length of 'bookmarks' in json export dictionary
click.echo(
click.style(
"Good news! We have successfully exported {number_of_bookmarks} bookmarks for you."
.format(number_of_bookmarks=bookmarks),
fg='green'))
for exported_file in exported_files:
if exported_file['file_size'] / 1024 > 1024:
file_size_string = '{size:d} MiB'.format(
size=int(round(exported_file['file_size'] / 1024 / 1024)))
elif exported_file['file_size'] > 1024:
file_size_string = '{size:d} KiB'.format(
size=int(round(exported_file['file_size'] / 1024)))
else:
file_size_string = '{size:d} B'.format(
size=exported_file['file_size'])
click.echo("You can find your exported data in ")
click.echo(
click.style(" {path}".format(path=exported_file['file_path']),
bold=True))
click.echo(" (~{size})".format(size=file_size_string))
click.echo(
click.style("Thank you for using this little tool!", reverse=True))
if not not_show_file:
click.echo(
click.style(
"We will now try to open your file manager with the exported file selected.",
reverse=True))
click.launch(export_directory, locate=True)