def __init__(self, token=None):
super(ServiceWallabag, self).__init__(token)
self.token = token
if token:
self.wall = Wallabag(
host=settings.TH_WALLABAG['host'],
client_secret=settings.TH_WALLABAG['client_secret'],
client_id=settings.TH_WALLABAG['client_id'],
token=token)
def wall(self):
"""
refresh the token from the API
then call a Wallabag instance
then store the token
:return: wall instance
"""
us = UserService.objects.get(user=self.user, name='ServiceWallabag')
params = {
'client_id': us.client_id,
'client_secret': us.client_secret,
'username': us.username,
'password': us.password,
}
try:
token = Wall.get_token(host=us.host, **params)
except Exception as e:
update_result(self.trigger_id, msg=e, status=False)
logger.critical('{} {}'.format(self.user, e))
return False
wall = Wall(host=us.host, client_secret=us.client_secret,
client_id=us.client_id, token=token)
UserService.objects.filter(user=self.user,
name='ServiceWallabag').update(token=token)
return wall
async def main(config: Config, loop):
token = await Wallabag.get_token(host=config.hostname,
client_id=config.client_id,
client_secret=config.client_secret,
username=config.username,
password=config.password)
async with aiohttp.ClientSession(loop=loop) as session:
wallabag = Wallabag(host=config.hostname,
client_secret=config.client_secret,
client_id=config.client_id,
token=token,
extension="json",
aio_sess=session)
all_article = await get_articles(wallabag)
for article in all_article:
is_processed = has_processed_tag(article)
if not is_processed:
print(f"processing {article['id']}")
d = {'tags': get_tags_with_processed(article)}
for enhancer in enhancers:
if enhancer.should(article):
result = await enhancer.patch(article, session=session)
d = {**d, **result}
await patch_article(wallabag, article, **d)
else:
print(f"skiping {article['id']}")
def wall(self):
"""
refresh the token from the API
then call a Wallabag instance
then store the token
:return: wall instance
"""
us = UserService.objects.get(user=self.user, name='ServiceWallabag')
params = {
'client_id': us.client_id,
'client_secret': us.client_secret,
'username': us.username,
'password': us.password,
}
try:
token = Wall.get_token(host=us.host, **params)
except Exception as e:
update_result(self.trigger_id, msg=e, status=False)
logger.critical('{} {}'.format(self.user, e))
return False
wall = Wall(host=us.host, client_secret=us.client_secret,
client_id=us.client_id, token=token)
UserService.objects.filter(user=self.user,
name='ServiceWallabag').update(token=token)
return wall
def check(self, request, user):
"""
check if the service is well configured
:return: Boolean
"""
us = UserService.objects.get(user=user, name='ServiceWallabag')
params = {'username': us.username,
'password': us.password,
'client_id': us.client_id,
'client_secret': us.client_secret}
try:
Wall.get_token(host=us.host, **params)
return True
except requests.exceptions.HTTPError as e:
return e
async def main(loop, sites):
token = await Wallabag.get_token(**config["wallabag"])
async with aiohttp.ClientSession(loop=loop) as session:
wall = Wallabag(host=config["wallabag"]["host"], client_secret=config["wallabag"]["client_secret"],
client_id=config["wallabag"]["client_id"], token=token, aio_sess=session)
await asyncio.gather(*[handle_feed(session, wall, sitetitle, site) for sitetitle, site in sites.items()])
def check(self, request, user):
"""
check if the service is well configured
:return: Boolean
"""
us = UserService.objects.get(user=user, name='ServiceWallabag')
params = {
'username': us.username,
'password': us.password,
'client_id': us.client_id,
'client_secret': us.client_secret
}
try:
Wall.get_token(host=us.host, **params)
return True
except requests.exceptions.HTTPError as e:
return e
def _refresh_token(self):
"""
refresh the expired token
get the token of the service Wallabag
for the user that uses Wallabag
:return: boolean
"""
us = UserService.objects.get(user=self.user, name='ServiceWallabag')
params = {'username': us.username,
'password': us.password,
'client_id': us.client_id,
'client_secret': us.client_secret}
return Wall.get_token(host=us.host, **params)
def auth(self, request):
"""
let's auth the user to the Service
"""
callback_url = 'http://%s%s' % (request.get_host(),
reverse('wallabag_callback'))
params = {
'username': settings.TH_WALLABAG['username'],
'password': settings.TH_WALLABAG['password'],
'client_id': settings.TH_WALLABAG['client_id'],
'client_secret': settings.TH_WALLABAG['client_secret']
}
acces_token = Wallabag.get_token(host=settings.TH_WALLABAG['host'],
**params)
request.session['oauth_token'] = acces_token
return callback_url
def auth(self, request):
"""
let's auth the user to the Service
:param request: request object
:return: callback url
:rtype: string that contains the url to redirect after auth
"""
service = UserService.objects.get(user=request.user, name='ServiceWallabag')
callback_url = '%s://%s%s' % (request.scheme, request.get_host(), reverse('wallabag_callback'))
params = {'username': service.username,
'password': service.password,
'client_id': service.client_id,
'client_secret': service.client_secret}
access_token = Wall.get_token(host=service.host, **params)
request.session['oauth_token'] = access_token
return callback_url
def auth(self, request):
"""
let's auth the user to the Service
:param request: request object
:return: callback url
:rtype: string that contains the url to redirect after auth
"""
service = UserService.objects.get(
user=request.user, name='ServiceWallabag')
callback_url = 'http://%s%s' % (
request.get_host(), reverse('wallabag_callback'))
params = {'username': service.username,
'password': service.password,
'client_id': service.client_id,
'client_secret': service.client_secret}
access_token = Wall.get_token(host=service.host, **params)
request.session['oauth_token'] = access_token
return callback_url
def _get_wall_data(self):
us = UserService.objects.get(user=self.user, name='ServiceWallabag')
params = dict({'access_token': self.token,
'archive': 0,
'star': 0,
'delete': 0,
'sort': 'created',
'order': 'desc',
'page': 1,
'perPage': 30,
'tags': []})
responses = requests.get(us.host + '/api/entries.json',
params=params)
if responses.status_code == 401:
params['access_token'] = Wall.get_token(host=us.host, **params)
responses = requests.get(us.host + '/api/entries.json',
params=params)
elif responses.status_code != 200:
raise HTTPError(responses.status_code, responses.json())
return responses
def _get_wall_data(self):
us = UserService.objects.get(user=self.user, name='ServiceWallabag')
params = dict({'access_token': self.token,
'archive': 0,
'star': 0,
'delete': 0,
'sort': 'created',
'order': 'desc',
'page': 1,
'perPage': 30,
'tags': []})
responses = requests.get(us.host + '/api/entries.json',
params=params)
if responses.status_code == 401:
params['access_token'] = Wall.get_token(host=us.host, **params)
responses = requests.get(us.host + '/api/entries.json',
params=params)
elif responses.status_code != 200:
raise HTTPError(responses.status_code, responses.json())
return responses
def check_wallabag(self):
params = {
'username': self.config['w_username'],
'password': self.config['w_password'],
'client_id': self.config['w_client_id'],
'client_secret': self.config['w_client_secret']
}
# get token
token = Wallabag.get_token(host=self.config['w_host'], **params)
wall = Wallabag(host=self.config['w_host'],
client_secret=self.config['w_client_secret'],
client_id=self.config['w_client_id'],
token=token)
params = {
'archive': 0,
'star': 0,
'delete': 0,
'sort': 'created',
'order': 'desc',
'page': 1,
'perPage': 30,
'tags': []
}
data = wall.get_entries(**params)
for post in data['_embedded']['items']:
if 'domain_name' in post:
if post['domain_name'] is not None:
if 'boards.4chan.org' in post['domain_name']:
if not db.session.query(
exists().where(db.UrlsTable.url == unicode(
post['url']))).scalar():
self.logger.info("adding {}".format(post['url']))
u = db.UrlsTable()
u.url = unicode(post['url'])
db.session.add(u)
db.session.commit()
wall.delete_entries(post['id'])
else:
self.logger.warning("no domain_name in {}".format(post['url']))
from wallabag_api.wallabag import Wallabag
from pocket import Pocket, PocketException
import os
params = {
'username': os.environ['WALLABAG_USER'],
'password': os.environ['WALLABAG_PASS'],
'client_id': os.environ['WALLABAG_CLIENT'],
'client_secret': os.environ['WALLABAG_SECRET'],
'host': os.environ['WALLABAG_HOST']
}
token = Wallabag.get_token(**params)
wb = Wallabag(params['host'], token, params['client_id'],
params['client_secret'])
entries = wb.get_entries()
urls = []
while entries['page'] <= entries['pages']:
for item in entries['_embedded']['items']:
urls.append(item['url'])
entries = wb.get_entries(page=entries['page'] + 1)
print(len(urls), "urls fetched from wallabag")
p = Pocket(consumer_key=os.environ['POCKET_KEY'],
access_token=os.environ['POCKET_TOKEN'])
for i, url in enumerate(urls):
print(i, url)
p.add(url)
class ServiceWallabag(ServicesMgr):
def __init__(self, token=None):
super(ServiceWallabag, self).__init__(token)
self.token = token
if token:
self.wall = Wallabag(
host=settings.TH_WALLABAG['host'],
client_secret=settings.TH_WALLABAG['client_secret'],
client_id=settings.TH_WALLABAG['client_id'],
token=token)
def read_data(self, **kwargs):
"""
get the data from the service
as the pocket service does not have any date
in its API linked to the note,
add the triggered date to the dict data
thus the service will be triggered when data will be found
:param kwargs: contain keyword args : trigger_id at least
:type kwargs: dict
:rtype: list
"""
data = list()
trigger_id = kwargs['trigger_id']
cache.set('th_wallabag_' + str(trigger_id), data)
def process_data(self, **kwargs):
"""
get the data from the cache
:param trigger_id: trigger ID from which to save data
:type trigger_id: int
"""
kw = {
'cache_stack': 'th_wallabag',
'trigger_id': str(kwargs['trigger_id'])
}
return super(ServiceWallabag, self).process_data(**kw)
def save_data(self, trigger_id, **data):
"""
let's save the data
:param trigger_id: trigger ID from which to save data
:param **data: the data to check to be used and save
:type trigger_id: int
:type **data: dict
:return: the status of the save statement
:rtype: boolean
"""
from th_wallabag.models import Wallabag
status = False
if self.token and 'link' in data and data['link'] is not None\
and len(data['link']) > 0:
# get the data of this trigger
trigger = Wallabag.objects.get(trigger_id=trigger_id)
title = self.set_title(data)
# convert htmlentities
title = HtmlEntities(title).html_entity_decode
try:
self.wall.post_entries(url=data['link'],
title=title,
tags=(trigger.tag.lower()))
sentence = str('wallabag {} created').format(data['link'])
logger.debug(sentence)
status = True
except Exception as e:
logger.critical(e)
status = False
else:
logger.critical(
"no token or link provided for trigger ID {} ".format(
trigger_id))
status = False
return status
def auth(self, request):
"""
let's auth the user to the Service
"""
callback_url = 'http://%s%s' % (request.get_host(),
reverse('wallabag_callback'))
params = {
'username': settings.TH_WALLABAG['username'],
'password': settings.TH_WALLABAG['password'],
'client_id': settings.TH_WALLABAG['client_id'],
'client_secret': settings.TH_WALLABAG['client_secret']
}
acces_token = Wallabag.get_token(host=settings.TH_WALLABAG['host'],
**params)
request.session['oauth_token'] = acces_token
return callback_url
def callback(self, request, **kwargs):
"""
Called from the Service when the user accept to activate it
:param request: request object
:return: callback url
:rtype: string , path to the template
"""
try:
# finally we save the user auth token
# As we already stored the object ServicesActivated
# from the UserServiceCreateView now we update the same
# object to the database so :
# 1) we get the previous objet
us = UserService.objects.get(
user=request.user,
name=ServicesActivated.objects.get(name='ServiceWallabag'))
# 2) Readability API require to use 4 params consumer_key/secret +
# token_key/secret instead of usually get just the token
# from an access_token request. So we need to add a string
# separator for later use to split on this one
us.token = request.session['oauth_token']
# 3) and save everything
us.save()
except KeyError:
return '/'
return 'wallabag/callback.html'
import requests
import yaml
from wallabag_api.wallabag import Wallabag
with open("config.yaml", 'r') as stream:
try:
config = yaml.load(stream)
except (yaml.YAMLError, FileNotFoundError) as exception:
print(exception)
config = None
exit(1)
token = Wallabag.get_token(**config["wallabag"])
wall = Wallabag(host=config["wallabag"]["host"],
client_secret=config["wallabag"]["client_secret"],
client_id=config["wallabag"]["client_id"],
token=token)
a = wall.get_entries(tags=["Golem"])
print(a)
b = a["_embedded"]
c = b["items"]
print(c)
exit()
try:
for i in c[1:]:
print(i["id"])
wall.delete_entries(i["id"])
print(i["id"])