def get_auth_url(wf):
request_token = Pocket.get_request_token(consumer_key=config.CONSUMER_KEY, redirect_uri=config.REDIRECT_URI)
wf.cache_data("pocket_request_token", request_token)
auth_url = Pocket.get_auth_url(code=request_token, redirect_uri=config.REDIRECT_URI)
return auth_url
def get_list(wf, access_token):
pocket_instance = Pocket(config.CONSUMER_KEY, access_token)
try:
get = pocket_instance.get(sort='newest')
get_list = get[0]['list']
if get_list == []:
return None
# Unpack and sort items
item_list = []
for item in sorted(get_list.values(), key=lambda x: int(x['time_added']), reverse=True):
item_list.append(item)
return item_list
except AuthException:
return 'error1'
wf.delete_password('pocket_access_token')
wf.logger.error(
'There was a problem receiving your Pocket list. The workflow has been deauthenticated automatically. Please try again!')
except Exception:
return 'error2'
wf.logger.error(
'Could not contact getpocket.com. Please check your Internet connection and try again!')
return None
def dispatch(self, request, *args, **kwargs):
if not request.session.get('access_token'):
request_token = Pocket.get_request_token(consumer_key=settings.CONSUMER_KEY)
request.session['request_token'] = request_token
auth_url = Pocket.get_auth_url(code=request_token, redirect_uri='http://localhost:8000/token')
return HttpResponseRedirect(auth_url)
return super(HomeView, self).dispatch(request, *args, **kwargs)
def get_client(self):
"""
Connects to Pocket API using OAuth
"""
request_token = pckt.get_request_token(
consumer_key=self.consumer_key,
redirect_uri=self.REDIRECT_URI
)
log.debug(request_token)
# URL to redirect user to, to authorize your app
auth_url = pckt.get_auth_url(
code=request_token,
redirect_uri=self.REDIRECT_URI
)
log.debug(auth_url)
# only needed to authorize your app the first time
#request_token2 = input('request token: ')
if not self.access_token:
self.access_token = pckt.get_access_token(
consumer_key=self.consumer_key, code=request_token)
log.debug(self.access_token)
return pckt(
consumer_key=self.consumer_key,
access_token=self.access_token)
def archive_item(item_id):
access_token = WF.get_password('pocket_access_token')
pocket_instance = Pocket(config.CONSUMER_KEY, access_token)
try:
pocket_instance.archive(item_id, wait=False)
return 'Link archived'
except PocketException:
return 'Connection error'
def get_request_token():
consumer_key = get_consumer_key()
request_token = Pocket.get_request_token(consumer_key = consumer_key)
auth_url = Pocket.get_auth_url(code=request_token)
os.system('open "{0}"'.format(auth_url))
time.sleep(5)
return request_token
def form_valid(self, form):
pocket_instance = Pocket(settings.CONSUMER_KEY, self.request.session['access_token'])
file = self.request.FILES['file']
for url in file.readlines():
pocket_instance.add(url.decode().rstrip())
# pocket_instance.bulk_add(item_id='', url=url.decode().rstrip())
# pocket_instance.commit()
return super(HomeView, self).form_valid(form)
def delete_item(item_id):
access_token = wf.get_password('pocket_access_token')
pocket_instance = Pocket(CONSUMER_KEY, access_token)
try:
pocket_instance.delete(item_id, wait=False)
return 'Link deleted'
except ConnectionError:
return 'Connection error'
def connect_to_pocket(request):
redirect_uri = request.build_absolute_uri(reverse('feed_pocket_success'))
request_token = Pocket.get_request_token(
consumer_key=settings.POCKET_CONSUMER_KEY,
redirect_uri=redirect_uri)
auth_url = Pocket.get_auth_url(code=request_token,
redirect_uri=redirect_uri)
request.user.profile.pocket_request_token = request_token
request.user.profile.save()
return HttpResponseRedirect(auth_url)
def push_to_pocket(link_id):
link = Document.find_one({'user': g.user._id, '_id': ObjectId(link_id)})
if link:
user = User.find_one({'_id': link.user})
pocket_instance = Pocket(POCKET_APP_TOKEN,
user.credentials['pocket']['token'])
pocket_instance.add(link.url, wait=False)
link.read = True
link.save()
return "OK"
def add_item(item):
if item is not None:
access_token = wf.get_password('pocket_access_token')
pocket_instance = Pocket(config.CONSUMER_KEY, access_token)
try:
pocket_instance.add(
url=item['url'], title=item['title'], tags="alfred")
return True
except InvalidQueryException:
pass
return False
def auth():
pocket = Pocket(config.POCKET_CONSUMER_KEY, config.BASE_URL +
url_for('auth_callback'))
try:
code = pocket.get_request_token()
url = pocket.get_authorize_url(code)
except APIError as apie:
return str(apie)
session.pop('code', None)
session['code'] = code
return redirect(url)
def pocket_login():
"""Calling into authorize will cause the OpenID auth machinery to kick
in. When all worked out as expected, the remote application will
redirect back to the callback URL provided.
"""
redirect_uri = url_for('pocket_authorized', _external=True)
request_token = Pocket.get_request_token(consumer_key=POCKET_APP_TOKEN,
redirect_uri=redirect_uri)
session['request_token'] = request_token
return redirect(Pocket.get_auth_url(code=request_token,
redirect_uri=redirect_uri))
def send_to_pocket(url):
if mconfig:
if 'share_settings' in mconfig:
if 'pocket' in mconfig['share_settings']:
pocket_config = mconfig['share_settings']['pocket']
pc = Pocket(
consumer_key=pocket_config['consumer_key'],
access_token=pocket_config['access_token']
)
return pc.add( url=url )
else:
return {"status":0,"message":"Pocket not configured."}
else:
return {"status":0,"message":"No configuration file found."}
def auth(self, request):
"""
let's auth the user to the Service
"""
callback_url = "http://%s%s" % (request.get_host(), reverse("pocket_callback"))
request_token = Pocket.get_request_token(consumer_key=self.consumer_key, redirect_uri=callback_url)
# Save the request token information for later
request.session["request_token"] = request_token
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(code=request_token, redirect_uri=callback_url)
return auth_url
def callback(self, request):
"""
Called from the Service when the user accept to activate it
"""
try:
# finally we save the user auth token
# As we already stored the object ServicesActivated
# from the UserServiceCreateView now we update the same
# object to the database so :
# 1) we get the previous object
us = UserService.objects.get(
user=request.user,
name=ServicesActivated.objects.get(name='ServicePocket'))
# 2) then get the token
access_token = Pocket.get_access_token(
consumer_key=self.consumer_key,
code=request.session['request_token'])
us.token = access_token
# 3) and save everything
us.save()
except KeyError:
return '/'
return 'pocket/callback.html'
def auth(self, request):
"""
let's auth the user to the Service
:param request: request object
:return: callback url
:rtype: string that contains the url to redirect after auth
"""
callback_url = self.callback_url(request)
request_token = Pocket.get_request_token(consumer_key=self.consumer_key, redirect_uri=callback_url)
# Save the request token information for later
request.session['request_token'] = request_token
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(code=request_token, redirect_uri=callback_url)
return auth_url
def delete_item(item_id):
access_token = WF.get_password('pocket_access_token')
pocket_instance = Pocket(config.CONSUMER_KEY, access_token)
try:
pocket_instance.delete(item_id, wait=False)
# remove entry in cache
item_list = WF.cached_data('pocket_list', max_age=0)
if type(item_list) is list and len(item_list) > 0:
item_list[:] = [
d for d in item_list if d.get('item_id') != item_id]
WF.cache_data('pocket_list', item_list)
return 'Link deleted'
except PocketException:
return 'Connection error'
def pocket_authorized():
access_token = Pocket.get_access_token(consumer_key=POCKET_APP_TOKEN,
code=session['request_token'])
resp = {'oauth_token': access_token, 'oauth_secret': ''}
creation_or_link, redirect = process_authorized('pocket', resp,
lambda x: True, 'oauth_token', 'oauth_token_secret')
return redirect
def __init__(self):
self.redis = redis.StrictRedis(host='localhost', port=6379, db=0)
self.key, self.token = get_credentials()
self.pocket = Pocket(self.key, self.token)
loader = FileSystemLoader(os.path.join(os.path.dirname(os.path.realpath(__file__)), "reports"))
print os.path.join(os.path.dirname(os.path.realpath(__file__)), "reports")
self.env = Environment(loader=loader)
def __init__(self, token=None, **kwargs):
super(ServicePocket, self).__init__(token, **kwargs)
self.consumer_key = settings.TH_POCKET['consumer_key']
self.token = token
self.oauth = 'oauth1'
self.service = 'ServicePocket'
if token:
self.pocket = Pocket(self.consumer_key, token)
def _initPocket(self):
access_token = u'3d50b538-8bfc-d571-1488-fabf77'
consumer_key = self.config.get("AuthPocket", "consumer_key", None)
if access_token is None:
redirect_uri = self.config.get("AuthPocket", "redirect_uri", None)
access_token = Pocket.auth(consumer_key=consumer_key, redirect_uri=redirect_uri)
self._pocket_instance = pocket.Pocket(consumer_key, access_token)
def __init__(self):
self._configs = Configs()
self._storage = Storage()
self._pocket = Pocket(
self._configs.get('consumer_key'),
self._configs.get('access_token')
)
def auth_callback():
pocket = Pocket(config.POCKET_CONSUMER_KEY)
code = session['code']
try:
resp = pocket.get_access_token(code)
session.pop('access_token', None)
username = resp['username']
token = resp['access_token']
session['access_token'] = token
session['username'] = username
model.User.insert(name = username, token = token).upsert().execute()
# add async task
sync.pool.add_task(sync.sync_all_for_user, username, token)
except APIError as apie:
return str(apie)
return session['username'] + " " + session['access_token']
def auth(self, request):
"""
let's auth the user to the Service
"""
callback_url = self.callback_url(request, 'pocket')
request_token = Pocket.get_request_token(
consumer_key=self.consumer_key,
redirect_uri=callback_url)
# Save the request token information for later
request.session['request_token'] = request_token
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(
code=request_token, redirect_uri=callback_url)
return auth_url
def callback(self, request, **kwargs):
"""
Called from the Service when the user accept to activate it
:param request: request object
:return: callback url
:rtype: string , path to the template
"""
access_token = Pocket.get_access_token(consumer_key=self.consumer_key, code=request.session['request_token'])
kwargs = {'access_token': access_token}
return super(ServicePocket, self).callback(request, **kwargs)
def post_linkPocket():
uid = request.headers.get('Authorization')
content = request.get_json()
request_token = content['code']
if uid is None:
print "WTF uid is none on linkPocket"
user_credentials = Pocket.get_credentials(consumer_key=CONSUMER_KEY, code=request_token)
secret = user_credentials['access_token']
accounts.set_secret(uid, secret)
return 'OK'
def sync_all_for_user(username, resync_all = False):
try:
u = model.User.get(model.User.name == username)
except model.UserDoesNotExist:
logger.error("no such user %s", username)
return None
logger.info("start fetching items for user: %s", username)
pocket = Pocket(config.POCKET_CONSUMER_KEY)
pocket.set_access_token(u.token)
try:
if resync_all == True:
resp = pocket.get(sort = 'newest', detailType = 'simple')
else:
resp = pocket.get(sort = 'newest', detailType = 'simple', since =
time.mktime(u.last_sync.timetuple()))
except Exception, e:
logger.error(e)
return None
def login():
# If user is already logged in, simply redirect to index
if g.user is not None and g.user.is_authenticated():
return redirect(url_for('home'))
# Load login if not logged in
else:
if os.environ.get('HEROKU') is None:
url_prefix = 'http://localhost:5000/'
else:
url_prefix = 'http://bulletin.herokuapp.com/'
form = LoginForm()
if form.validate_on_submit():
# Grabs request token from Pocket and stores in session for later authentication
request_token = Pocket.get_request_token(consumer_key=CONSUMER_KEY, redirect_uri=url_prefix + 'auth')
session['request_token'] = request_token
# Grabs auth url from Pocket to redirect user to
auth_url = Pocket.get_auth_url(code=request_token, redirect_uri=url_prefix + 'auth')
return redirect(auth_url)
return render_template('index.html',
form = form)
def callback(self, request, **kwargs):
"""
Called from the Service when the user accept to activate it
"""
access_token = Pocket.get_access_token(
consumer_key=self.consumer_key,
code=request.session['request_token'])
kwargs = {'access_token': access_token, 'service': 'ServicePocket',
'return': 'pocket'}
return super(ServicePocket, self).callback(request, **kwargs)
def callback(self, request, **kwargs):
"""
Called from the Service when the user accept to activate it
:param request: request object
:return: callback url
:rtype: string , path to the template
"""
access_token = Pocket.get_access_token(
consumer_key=self.consumer_key,
code=request.session['request_token'])
kwargs = {'access_token': access_token}
return super(ServicePocket, self).callback(request, **kwargs)
def get_access_token():
print("Fetching request token from pocket...")
request_token = Pocket.get_request_token(consumer_key=CONSUMER_KEY,
redirect_uri=REDIRECT_URI)
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(code=request_token,
redirect_uri=REDIRECT_URI)
user_credentials = None
while user_credentials == None:
try:
print("Trying to get access token from pocket...")
user_credentials = Pocket.get_credentials(
consumer_key=CONSUMER_KEY, code=request_token)
except Exception as e:
print(e)
print(
"In order to get an access token, you must visit this url to authenticate:",
auth_url)
print("Press enter after giving permission in your browser")
print("Got an access token")
return user_credentials['access_token']
class PocketAPI:
def __init__(self, consumer_key, access_token):
self.pocket = Pocket(consumer_key=consumer_key, access_token=access_token)
self.retrieve()
def retrieve(self):
self.json = self.pocket.retrieve()
def __len__(self):
return len(self.json["list"])
def __getitem__(self, select):
for i, elt in enumerate(self.json["list"].values()):
if i == select:
return elt
def __iter__(self):
yield from self.json["list"].values()
def authorize():
request_token = wf.cached_data('pocket_request_token')
if request_token:
try:
user_credentials = Pocket.get_credentials(
consumer_key=config.CONSUMER_KEY, code=request_token)
wf.save_password('pocket_access_token',
user_credentials['access_token'])
# We don't need the cache anymore. clear it for security reasons
wf.clear_cache()
def wrapper():
return get_list(wf, user_credentials['access_token'])
wf.cached_data('pocket_list', data_func=wrapper, max_age=1)
except RateLimitException:
pass
def _sync_pocket_for_all_users():
logger = logging.getLogger("main")
app = FirebaseApp.admin(project_id)
db = FirestoreDatabase.admin(app)
users = db.get_users_private()
for user in users:
if user["pocket_sync_enabled"] and user["pocket_access_token"]:
logger.debug(f"syncing {user['uid']}")
pocket = Pocket(consumer_key, user["pocket_access_token"])
bookmarks = PocketBookmarks(user["uid"],
pocket,
db,
requests,
since=datetime(2020, 9, 1))
logger.debug(f"initialized, syncing")
count = bookmarks.sync_latest()
logger.debug(f"done; count={count}")
else:
logger.debug(f"syncing disabled for {user['uid']}, skipping")
def main(argv):
parsed_args = parse_args()
try:
from pocket import Pocket, PocketException
except ImportError:
print_error('You need to `pip3 install pocket-api`')
print_refresh()
sys.exit(1)
pocket = Pocket(consumer_key=CONSUMER_KEY, access_token=ACCESS_TOKEN)
if parsed_args.delete:
pocket.delete(parsed_args.delete).commit()
sys.exit()
elif parsed_args.add:
new_url = get_url()
if new_url:
pocket.add(url=new_url)
sys.exit()
try:
raw_answer = pocket.retrieve(sort='newest')
except (Exception, PocketException) as error:
print_error(error)
print_refresh()
sys.exit(1)
adapted_articles = [
Article(i.get('item_id'), i.get('resolved_url', i.get('given_url')),
i.get('resolved_title', i.get('given_title')), argv[0])
for i in raw_answer['list'].values()
]
print(
f'{len(adapted_articles)}|font=Verdana templateImage={pocket_icon()}')
print('---')
print(*adapted_articles, sep='\n')
print('---')
print(
f'➕ Save a URL|bash={argv[0]} param1=--add terminal=false refresh=true'
)
print_refresh()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--tag', default='python')
parser.add_argument('--deleted', action='store_true')
args = parser.parse_args()
tag = args.tag
deleted = args.deleted
if 'POCKET_CONSUMER_KEY' not in os.environ or 'POCKET_ACCESS_TOKEN' not in os.environ:
print(
"[ERR] no set POCKET_CONSUMER_KEY or POCKET_ACCESS_TOKEN environment variables."
)
exit()
p = Pocket(
consumer_key=os.environ['POCKET_CONSUMER_KEY'],
access_token=os.environ['POCKET_ACCESS_TOKEN'],
)
res = p.get(
offset=0,
count=500,
tag=tag,
)
li = res[0]['list']
for k in li:
v = li[k]
item_id, title, url = get_item_info(v)
if deleted: # 削除
p.delete(item_id)
print("deleted [%s](%s)" % (title, url))
else:
print("- [%s](%s)" % (title, url))
if deleted: # 削除コミット
p.commit()
def sync_all_for_user(username, resync_all=False):
try:
u = model.User.get(model.User.name == username)
except model.UserDoesNotExist:
logger.error("no such user %s", username)
return None
logger.info("start fetching items for user: %s", username)
pocket = Pocket(config.POCKET_CONSUMER_KEY)
pocket.set_access_token(u.token)
try:
if resync_all == True:
resp = pocket.get(sort='newest', detailType='simple')
else:
resp = pocket.get(sort='newest',
detailType='simple',
since=time.mktime(u.last_sync.timetuple()))
except Exception, e:
logger.error(e)
return None
class ServicePocket(ServicesMgr):
"""
Service Pocket
"""
def __init__(self, token=None, **kwargs):
super(ServicePocket, self).__init__(token, **kwargs)
self.consumer_key = settings.TH_POCKET['consumer_key']
self.token = token
self.oauth = 'oauth1'
self.service = 'ServicePocket'
if token:
try:
self.pocket = Pocket(self.consumer_key, token)
except (AuthException, RateLimitException) as e:
us = UserService.objects.get(token=token)
logger.error(e.msg, e.error_code)
update_result(us.trigger_id, msg=e.msg, status=False)
def _create_entry(self, url, title, tags):
"""
Create an entry
:param url: url to save
:param title: title to set
:param tags: tags to set
:return: status
"""
try:
self.pocket.add(url=url, title=title, tags=tags)
sentence = str('pocket {} created').format(url)
logger.debug(sentence)
status = True
except Exception as e:
logger.critical(e)
update_result(self.trigger_id, msg=e, status=False)
status = False
return status
def read_data(self, **kwargs):
"""
get the data from the service
as the pocket service does not have any date
in its API linked to the note,
add the triggered date to the dict data
thus the service will be triggered when data will be found
:param kwargs: contain keyword args : trigger_id at least
:type kwargs: dict
:rtype: list
"""
trigger_id = kwargs.get('trigger_id')
date_triggered = kwargs.get('date_triggered')
data = list()
# pocket uses a timestamp date format
since = arrow.get(date_triggered).timestamp
if self.token is not None:
# get the data from the last time the trigger have been started
# timestamp form
pockets = self.pocket.get(since=since, state="unread")
content = ''
if pockets is not None and len(pockets[0]['list']) > 0:
for my_pocket in pockets[0]['list'].values():
if my_pocket.get('excerpt'):
content = my_pocket['excerpt']
elif my_pocket.get('given_title'):
content = my_pocket['given_title']
my_date = arrow.get(str(date_triggered),
'YYYY-MM-DD HH:mm:ss')\
.to(settings.TIME_ZONE)
data.append({
'my_date': str(my_date),
'tag': '',
'link': my_pocket['given_url'],
'title': my_pocket['given_title'],
'content': content,
'tweet_id': 0
})
cache.set('th_pocket_' + str(trigger_id), data)
return data
def save_data(self, trigger_id, **data):
"""
let's save the data
:param trigger_id: trigger ID from which to save data
:param data: the data to check to be used and save
:type trigger_id: int
:type data: dict
:return: the status of the save statement
:rtype: boolean
"""
if data.get('link'):
if len(data.get('link')) > 0:
# get the pocket data of this trigger
from th_pocket.models import Pocket as PocketModel
trigger = PocketModel.objects.get(trigger_id=trigger_id)
title = self.set_title(data)
# convert htmlentities
title = HtmlEntities(title).html_entity_decode
status = self._create_entry(url=data.get('link'),
title=title,
tags=(trigger.tag.lower()))
else:
msg = "no link provided for trigger ID {}," \
" so we ignore it".format(trigger_id)
logger.warning(msg)
update_result(trigger_id, msg=msg, status=True)
status = True
else:
msg = "no token provided for trigger ID {}".format(trigger_id)
logger.critical(msg)
update_result(trigger_id, msg=msg, status=False)
status = False
return status
def auth(self, request):
"""
let's auth the user to the Service
:param request: request object
:return: callback url
:rtype: string that contains the url to redirect after auth
"""
callback_url = self.callback_url(request)
request_token = Pocket.get_request_token(
consumer_key=self.consumer_key, redirect_uri=callback_url)
# Save the request token information for later
request.session['request_token'] = request_token
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(code=request_token,
redirect_uri=callback_url)
return auth_url
def callback(self, request, **kwargs):
"""
Called from the Service when the user accept to activate it
:param request: request object
:return: callback url
:rtype: string , path to the template
"""
access_token = Pocket.get_access_token(
consumer_key=self.consumer_key,
code=request.session['request_token'])
kwargs = {'access_token': access_token}
return super(ServicePocket, self).callback(request, **kwargs)
def __init__(self, token=None):
super(ServicePocket, self).__init__(token)
self.consumer_key = settings.TH_POCKET['consumer_key']
self.token = token
if token:
self.pocket = Pocket(self.consumer_key, token)
import requests_cache
#monkey patching... FIXME use CacheControl
requests_cache.install_cache('tmp/cache2',
backend='sqlite',
expire_after=12 * 3600)
import datetime
import PyRSS2Gen
import concurrent.futures
import urllib.request
#get config
config = lib.config
#pocket library
pocketreader = Pocket(consumer_key=config['pocket']['consumer_key'],
access_token=config['pocket']['access_token'])
class Article(object):
def __init__(self):
super().__init__()
self.canonical = ""
self.urls = set()
self.title = ""
self.cleaned_title = ""
self.feeds = set()
self.description = ""
self.published = ""
self.source = ""
current_script_file = os.path.realpath(__file__)
current_script_dir = os.path.abspath(os.path.join(current_script_file, os.pardir))
fh = logging.handlers.RotatingFileHandler(
"%s/logs/garyreporter.log" % (current_script_dir),
maxBytes=200000, backupCount=5)
#logging.FileHandler("%s/eepm_video_processor.log" % (current_script_dir))
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)
j = json.loads(open("config.json").read())
p = Pocket(
consumer_key=j['consumer_key'],
access_token=j['access_token']
)
# Fetch a list of articles
mycount = 100
myoffset = 0
tagfilter = "garyreporter"
articles_json_data_file = 'articles_json_data.json'
try:
#articles_json = json.loads(p.retrieve(offset=myoffset, count=mycount))
articles_json = None
data_origin = None
if not os.path.isfile(articles_json_data_file):
class ServicePocket(ServicesMgr):
def __init__(self, token=None):
super(ServicePocket, self).__init__(token)
self.consumer_key = settings.TH_POCKET['consumer_key']
self.token = token
if token:
self.pocket = Pocket(self.consumer_key, token)
def read_data(self, **kwargs):
"""
get the data from the service
as the pocket service does not have any date
in its API linked to the note,
add the triggered date to the dict data
thus the service will be triggered when data will be found
:param kwargs: contain keyword args : trigger_id at least
:type kwargs: dict
:rtype: list
"""
trigger_id = kwargs['trigger_id']
date_triggered = kwargs['date_triggered']
data = list()
# pocket uses a timestamp date format
since = int(
time.mktime(datetime.datetime.timetuple(date_triggered)))
if self.token is not None:
# get the data from the last time the trigger have been started
# timestamp form
pockets = self.pocket.get(since=since, state="unread")
content = ''
if pockets is not None and len(pockets[0]['list']) > 0:
for my_pocket in pockets[0]['list'].values():
if my_pocket['excerpt']:
content = my_pocket['excerpt']
elif my_pocket['given_title']:
content = my_pocket['given_title']
my_date = arrow.get(str(date_triggered),
'YYYY-MM-DD HH:mm:ss')\
.to(settings.TIME_ZONE)
data.append({'my_date': str(my_date),
'tag': '',
'link': my_pocket['given_url'],
'title': my_pocket['given_title'],
'content': content,
'tweet_id': 0})
cache.set('th_pocket_' + str(trigger_id), data)
return data
def process_data(self, **kwargs):
"""
get the data from the cache
:param kwargs: contain keyword args : trigger_id at least
:type kwargs: dict
"""
kw = {'cache_stack': 'th_pocket',
'trigger_id': str(kwargs['trigger_id'])}
return super(ServicePocket, self).process_data(**kw)
def save_data(self, trigger_id, **data):
"""
let's save the data
:param trigger_id: trigger ID from which to save data
:param data: the data to check to be used and save
:type trigger_id: int
:type data: dict
:return: the status of the save statement
:rtype: boolean
"""
from th_pocket.models import Pocket as PocketModel
status = False
if self.token and 'link' in data and data['link'] is not None\
and len(data['link']) > 0:
# get the pocket data of this trigger
trigger = PocketModel.objects.get(trigger_id=trigger_id)
title = self.set_title(data)
# convert htmlentities
title = HtmlEntities(title).html_entity_decode
try:
self.pocket.add(
url=data['link'], title=title, tags=(trigger.tag.lower()))
sentence = str('pocket {} created').format(data['link'])
logger.debug(sentence)
status = True
except Exception as e:
logger.critical(e)
status = False
else:
logger.critical("no token provided for trigger ID %s ", trigger_id)
return status
def auth(self, request):
"""
let's auth the user to the Service
:param request: request object
:return: callback url
:rtype: string that contains the url to redirect after auth
"""
callback_url = self.callback_url(request, 'pocket')
request_token = Pocket.get_request_token(
consumer_key=self.consumer_key,
redirect_uri=callback_url)
# Save the request token information for later
request.session['request_token'] = request_token
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(
code=request_token, redirect_uri=callback_url)
return auth_url
def callback(self, request, **kwargs):
"""
Called from the Service when the user accept to activate it
:param request: request object
:return: callback url
:rtype: string , path to the template
"""
access_token = Pocket.get_access_token(
consumer_key=self.consumer_key,
code=request.session['request_token'])
kwargs = {'access_token': access_token, 'service': 'ServicePocket',
'return': 'pocket'}
return super(ServicePocket, self).callback(request, **kwargs)
'quiet': ''
}
pdfkit.from_url(url, pname, options=options)
except:
print("PDFkit error occurred\n")
if urlmissing:
bad_links.append(match)
else:
good_links.append(match)
urldict[tag] = urllist
dump_links(tag + ".good", good_links)
dump_links(tag + ".bad", bad_links)
p = Pocket(
consumer_key='92012-eed59ea9253b3c580d01447a',
access_token='16751f51-3306-eb9c-66d0-ca57fb'
)
urld = {}
if os.path.exists('tagdict.json'):
with open('tagdict.json') as json_file:
urldict = json.load(json_file)
dump_tag(p, "practice", urld)
dump_tag(p, "lisp", urld)
with open('tagdict.json', 'w') as json_file:
json.dump(urld, json_file)
def __init__(self, token=None):
self.consumer_key = settings.TH_POCKET['consumer_key']
if token:
self.pocket = Pocket(self.consumer_key, token)
from pocket import Pocket
from . import config
pocket = Pocket(config.POCKET_CONSUMER_KEY, config.POCKET_ACCESS_TOKEN)
def pocket_add(urls, tag='boring'):
if not urls: return
new_urls = set(urls) - set(pocket_list())
if not new_urls: return
for url in new_urls:
print('Adding "%s".' % url)
pocket.bulk_add(None, url=url, tags=tag)
pocket.commit()
def pocket_list(tag='boring', state='all'):
response = pocket.get(tag=tag, state=state)
data_list = response[0]['list']
if not data_list:
return []
links = [article['given_url'] for article in data_list.values()]
return links
def __init__(self, consumer_key, access_token):
self.pocket_client = Pocket(consumer_key, access_token)
for attr in attrs:
if attr[0] == linkTag:
link = attr[1]
print "link: ", link
newPair = [link, currentTag]
urlList.append(newPair)
def handle_data(self, data):
if self.readData:
print "Tag: ", data
currentTag = data
self.readData = False
request_token = Pocket.get_request_token(consumer_key, redirect_uri)
auth_url = Pocket.get_auth_url(request_token, redirect_uri)
webbrowser.open_new_tab(auth_url)
print 'Please authorize the app using the following url and press ENTER here', auth_url
raw_input()
access_token = Pocket.get_access_token(consumer_key=consumer_key,
code=request_token)
print 'Got request token', request_token
print 'Got access token', access_token
def add_url(url, tag):
print 'adding', url
def add_url(url, tag):
print 'adding', url
pocket_instance = Pocket(consumer_key, access_token)
print pocket_instance.add(url=url, tags=[tag])
j = k + 1
while is_date(' '.join(string[i:j])) and j < l + 1:
j += 1
return is_valid_date(' '.join(string[i:j - 1]))
return False
pocket_tokens = os.environ.get("JON_POCKET")
c_key = os.environ.get("POCKET_TOKEN")
SMMRY_API = os.environ.get('SMMRY_API')
links = {}
tt =[]
since = None
wanted_fields = set(['resolved_url', 'given_url', 'given_title', 'resolved_url', 'tags', 'resolved_title'])
p = Pocket(consumer_key = c_key, access_token = pocket_tokens)
poks = p.get(sort = 'newest', detailType = 'complete', since = since)[0] #tag, since parameters too
#print(poks)
for key in poks['list']:
if key == '1502819':
print('skip')
continue
# print(poks['list'][key].keys())
item = poks['list'][key].keys()
links[key] = {k:poks['list'][key][k] for k in item if k in wanted_fields}
#links[key] = {'url' : poks['list'][key]['given_url'], 'title' : poks['list'][key]['given_title'], \
#'tags': poks['list'][key]['tags']}
link_keys = [k for k in links]
e = requests.Session()
def main():
parsed_args = parse_args()
try:
global keyring, Pocket, PocketException
import keyring
from pocket import Pocket, PocketException
except ImportError:
print_import_error()
print_refresh()
return
consumer_key, access_token = get_secrets()
pocket = Pocket(consumer_key=consumer_key, access_token=access_token)
if parsed_args.add:
new_url = get_input('\"Save an item to Pocket:\"')
if new_url:
pocket.add(url=new_url)
return
elif parsed_args.delete:
pocket.delete(parsed_args.delete).commit()
return
elif parsed_args.secrets:
update_secrets()
return
raw_articles = {} if parsed_args.full else get_cache(CACHE_PATH)
try:
raw_answer = pocket.retrieve(detailType='simple',
since=raw_articles.get('since'))
except PocketException as e:
if e.http_code in (400, 401):
print_secrets_error()
else:
print_error(e)
print_refresh()
return
except Exception as e:
print_error(e)
print_refresh()
return
raw_articles = update_from_cache(raw_articles, raw_answer)
set_cache(CACHE_PATH, raw_articles)
adapted_articles = [
Article(id=i.get('item_id'),
link=i.get('resolved_url', i.get('given_url')),
title=i.get('resolved_title', i.get('given_title')),
cmd=CMD) for i in sorted(raw_articles['list'].values(),
key=lambda x: x.get('time_added', ''),
reverse=True) if i['status'] == '0'
]
print(
f'{len(adapted_articles)}|font=Verdana size=14 templateImage={pocket_icon()}'
)
print('---')
print(*adapted_articles, sep='\n')
print('---')
print(f'➕ Save a URL|bash={CMD} param1=--add terminal=false refresh=true')
print_refresh()
class PocketStats(object):
def main(self):
self.parse_arguments()
if self.args.access_token:
self.print_stats()
else:
self.get_access_token()
def parse_arguments(self):
parser = argparse.ArgumentParser()
parser.add_argument("--consumer-key", required=True)
parser.add_argument("--access-token")
self.args = parser.parse_args()
def get_access_token(self):
request_token = Pocket.get_request_token(
consumer_key=self.args.consumer_key)
auth_url = Pocket.get_auth_url(code=request_token)
print(auth_url)
input("Press enter after visiting the URL: ")
user_credentials = Pocket.get_credentials(
consumer_key=self.args.consumer_key, code=request_token)
access_token = user_credentials['access_token']
print("Access token: {}".format(access_token))
def print_stats(self):
stats = {
'total': 0,
'total_words': 0,
'favorite': 0,
'has_image': 0,
'has_video': 0,
'is_article': 0,
'add_dates': [],
'tags': {},
'tagged': 0,
'untagged': 0,
}
for item in self.get_all_items():
stats['total'] += 1
stats['total_words'] += int(item['word_count'])
if item['favorite'] == '1':
stats['favorite'] += 1
if item['has_image'] == '1':
stats['has_image'] += 1
if item['has_video'] == '1':
stats['has_video'] += 1
if item['is_article'] == '1':
stats['is_article'] += 1
if 'tags' in item:
stats['tagged'] += 1
for tag in item['tags']:
stats['tags'][tag] = stats['tags'].get(tag, 0) + 1
else:
stats['untagged'] += 1
stats['add_dates'].append(
datetime.fromtimestamp(int(item['time_added'])))
print("Total: {}".format(stats['total']))
print("Total words: {}".format(stats['total_words']))
print("Average words: {}".format(stats['total_words'] /
stats['total']))
print("Favorite: {}".format(stats['favorite']))
print("Has image: {}".format(stats['has_image']))
print("Has video: {}".format(stats['has_video']))
print("Article: {}".format(stats['is_article']))
print("Tagged: {}".format(stats['tagged']))
print("Untagged: {}".format(stats['untagged']))
print("Tags:")
for (tag, count) in sorted(stats['tags'].items(),
key=operator.itemgetter(1),
reverse=True):
print("\t{}: {}".format(tag, count))
matplot_dates = matplotlib.dates.date2num(stats['add_dates'])
_, ax = matplotlib.pyplot.subplots(1, 1)
ax.hist(matplot_dates, bins=12)
ax.xaxis.set_major_formatter(
matplotlib.dates.DateFormatter('%y-%m-%d'))
matplotlib.pyplot.xticks(rotation=90)
matplotlib.pyplot.show()
matplotlib.pyplot.bar(range(len(stats['tags'])),
stats['tags'].values(),
align='center')
matplotlib.pyplot.xticks(range(len(stats['tags'])),
stats['tags'].keys())
matplotlib.pyplot.xticks(rotation=90)
matplotlib.pyplot.show()
def get_all_items(self):
self.pocket = Pocket(self.args.consumer_key, self.args.access_token)
response, headers = self.pocket.get(detailType='complete')
# Surprisingly, this isn't paginated and returns at least 1,000 items.
for key, item in response['list'].items():
yield item
def setup_function(function):
global _pocket
_pocket = Pocket(_consumer_key, _access_token)
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO,
handlers=[file_handler, console_handler])
logger = logging.getLogger(__name__)
# Reading configuration and authentication data
with open(CONFIG_FILE_NAME, 'r') as conf_file:
conf_data = json.load(conf_file)
auth_data = conf_data[AUTH_DATA_KEY]
# Logging in to Pocket
pocket_consumer_key = conf_data[AUTH_DATA_KEY]['pocket_consumer_key']
pocket_client = Pocket(
conf_data[AUTH_DATA_KEY]['pocket_consumer_key'],
conf_data[AUTH_DATA_KEY]['pocket_user_credentials']['access_token'])
logger.info('Logged in to Pocket')
# Logging in to Trello
trello_client = TrelloClient(
api_key=conf_data[AUTH_DATA_KEY]['trello_api_key'],
token=conf_data[AUTH_DATA_KEY]['trello_token'])
trello_list = trello_client.get_list(conf_data['trello_list_id'])
logger.info('Logged in to Trello')
now_timestamp = int(datetime.now().timestamp())
since_timestamp = conf_data[
'pocket_last_checked'] if 'pocket_last_checked' in conf_data else now_timestamp
new_pocket_items, _ = pocket_client.get(since=since_timestamp)
def init_consumer_key(self, consumer_key):
self._pocket = Pocket(consumer_key)
from pprint import pprint
from pocket import Pocket, PocketException
p = Pocket(consumer_key='73969-97b3dedc26da58ec74e67608',
access_token='facbe785-b308-4f4b-fbd5-fdf1b3')
def archived(tag):
last = p.retrieve(state="archive", tag=tag, count=1000, offset=0)
if last['list']:
return [value['resolved_url'] for value in last['list'].values()]
else:
return []
# # Fetch a list of articles
# try:
# print(p.retrieve(offset=0, count=10))
# except PocketException as e:
# print(e.message)
# Add an article
# p.add('https://pymotw.com/3/asyncio/')
class ServicePocket(ServicesMgr):
def __init__(self, token=None):
self.consumer_key = settings.TH_POCKET['consumer_key']
if token:
self.pocket = Pocket(self.consumer_key, token)
def read_data(self, token, trigger_id, date_triggered):
"""
get the data from the service
as the pocket service does not have any date
in its API linked to the note,
add the triggered date to the dict data
thus the service will be triggered when data will be found
:param trigger_id: trigger ID to process
:param date_triggered: the date of the last trigger
:type trigger_id: int
:type date_triggered: datetime
:return: list of data found from the date_triggered filter
:rtype: list
"""
data = list()
# pocket uses a timestamp date format
since = int(
time.mktime(datetime.datetime.timetuple(date_triggered)))
if token is not None:
# get the data from the last time the trigger have been started
# timestamp form
pockets = self.pocket.get(since=since, state="unread")
if pockets is not None and len(pockets[0]['list']) > 0:
for my_pocket in pockets[0]['list'].values():
if my_pocket['excerpt']:
content = my_pocket['excerpt']
elif my_pocket['given_title']:
content = my_pocket['given_title']
my_date = arrow.get(str(date_triggered),
'YYYY-MM-DD HH:mm:ss')\
.to(settings.TIME_ZONE)
data.append({'my_date': str(my_date),
'tag': '',
'link': pocket['given_url'],
'title': pocket['given_title'],
'content': content,
'tweet_id': 0})
cache.set('th_pocket_' + str(trigger_id), data)
return data
def process_data(self, trigger_id):
"""
get the data from the cache
:param trigger_id: trigger ID from which to save data
:type trigger_id: int
"""
cache_data = cache.get('th_pocket_' + str(trigger_id))
return PublishingLimit.get_data('th_pocket', cache_data, trigger_id)
def save_data(self, token, trigger_id, **data):
"""
let's save the data
:param trigger_id: trigger ID from which to save data
:param **data: the data to check to be used and save
:type trigger_id: int
:type **data: dict
:return: the status of the save statement
:rtype: boolean
"""
from th_pocket.models import Pocket as PocketModel
status = False
if token and 'link' in data and data['link'] is not None\
and len(data['link']) > 0:
# get the pocket data of this trigger
trigger = PocketModel.objects.get(trigger_id=trigger_id)
title = ''
title = (data['title'] if 'title' in data else '')
# convert htmlentities
title = HtmlEntities(title).html_entity_decode
try:
self.pocket.add(
url=data['link'], title=title, tags=(trigger.tag.lower()))
sentence = str('pocket {} created').format(data['link'])
logger.debug(sentence)
status = True
except Exception as e:
logger.critical(e)
status = False
else:
logger.critical("no token provided for trigger ID %s ", trigger_id)
return status
def auth(self, request):
"""
let's auth the user to the Service
"""
callback_url = 'http://%s%s' % (
request.get_host(), reverse('pocket_callback'))
request_token = Pocket.get_request_token(
consumer_key=self.consumer_key,
redirect_uri=callback_url)
# Save the request token information for later
request.session['request_token'] = request_token
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(
code=request_token, redirect_uri=callback_url)
return auth_url
def callback(self, request):
"""
Called from the Service when the user accept to activate it
"""
try:
# finally we save the user auth token
# As we already stored the object ServicesActivated
# from the UserServiceCreateView now we update the same
# object to the database so :
# 1) we get the previous object
us = UserService.objects.get(
user=request.user,
name=ServicesActivated.objects.get(name='ServicePocket'))
# 2) then get the token
access_token = Pocket.get_access_token(
consumer_key=self.consumer_key,
code=request.session['request_token'])
us.token = access_token
# 3) and save everything
us.save()
except KeyError:
return '/'
return 'pocket/callback.html'
from pocket import Pocket
ckey = '47167-9324a229a155827bc214aa2a'
redir = 'http://google.com'
request_token = Pocket.get_request_token(consumer_key=ckey, redirect_uri=redir)
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(code=request_token, redirect_uri=redir)
print(auth_url)
input()
user_credentials = Pocket.get_credentials(consumer_key=ckey, code=request_token)
access_token = user_credentials['access_token']
pocketi = Pocket(ckey, access_token)
for link in links[500:]:
pocketi.add(link)
import numpy as np
import matplotlib.pyplot as plt
xaxis = range(1,50)
plt.plot(xaxis, [np.sqrt(6*sum([1./n**2 for n in range(1,x)])) for x in xaxis])
plt.axhline(np.pi, color='r')
plt.show()z
'''
redirect_uri = "http://example.com/"
request_token = Pocket.get_request_token(consumer_key=consumer_key, redirect_uri=redirect_uri)
# URL to redirect user to, to authorize your app
auth_url = Pocket.get_auth_url(code=request_token, redirect_uri=redirect_uri)
input(auth_url)
user_credentials = Pocket.get_credentials(consumer_key=consumer_key, code=request_token)
access_token = user_credentials['access_token']
print(access_token);
'''
access_token = "23432-4324321432-4-2134-324"
pocket = Pocket(consumer_key, access_token)
lasturl = "http://www.economist.com/news/asia/21678115-display-amity-points-tougher-times-ahead-leaders-taiwan-and-china-hold-historic"
while True:
stack = []
import itertools
for i in itertools.count(0):
from urllib.request import urlopen, Request
from bs4 import BeautifulSoup
request = Request(
"http://www.economist.com/latest-updates?page=" + str(i),
headers={
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0}"
})
bs = BeautifulSoup(urlopen(request))
import os
import feedparser
from pocket import Pocket
import boto3
pocket_instance = Pocket(os.environ['POCKET_CONSUMER_KEY'],
os.environ['POCKET_ACCESS_TOKEN'])
important_keys = ['title', 'summary', 'id', 'link', 'tags']
dynamodb = boto3.resource('dynamodb')
class Feed:
def __init__(self, url, whitelist, filters, items):
self.url = url
self.whitelist = whitelist
self.filters = filters
self.items = items
def get_feed(url):
return feedparser.parse(url)
def pocket_add(url):
pocket_instance.add(url)
print('Added url: ' + url)
def filter_entry_keys(entries):
items = list()