class DLFPModule(Module, CapMessages, CapMessagesPost, CapContent):
NAME = 'dlfp'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '2.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = "Da Linux French Page news website"
CONFIG = BackendConfig(
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''),
ValueBool('get_news', label='Get newspapers', default=True),
ValueBool('get_diaries', label='Get diaries', default=False),
ValueBool('get_polls', label='Get polls', default=False),
ValueBool('get_board', label='Get board', default=False),
ValueBool('get_wiki', label='Get wiki', default=False),
ValueBool('get_tracker', label='Get tracker', default=False))
STORAGE = {'seen': {}}
BROWSER = DLFP
FEEDS = {
'get_news': "https://linuxfr.org/news.atom",
'get_diaries': "https://linuxfr.org/journaux.atom",
'get_polls': "https://linuxfr.org/sondages.atom",
'get_board': "https://linuxfr.org/forums.atom",
'get_wiki': "https://linuxfr.org/wiki.atom",
'get_tracker': "https://linuxfr.org/suivi.atom",
}
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
else:
password = None
return self.create_browser(username, password)
def deinit(self):
# don't need to logout if the browser hasn't been used.
if not self._browser:
return
self.browser.close_session()
#### CapMessages ##############################################
def iter_threads(self):
whats = set()
for param, url in self.FEEDS.items():
if self.config[param].get():
whats.add(url)
for what in whats:
for article in Newsfeed(what, rssid).iter_entries():
if article.datetime and (datetime.now() - article.datetime
) > timedelta(days=60):
continue
thread = Thread(article.id, article.link)
thread.title = article.title
thread._rsscomment = article.rsscomment
if article.datetime:
thread.date = article.datetime
yield thread
def get_thread(self, id, getseen=True):
if not isinstance(id, Thread):
thread = None
else:
thread = id
id = thread.id
if thread.date:
self.storage.set('date', id, thread.date)
self.storage.save()
content = self.browser.get_content(id)
if not content:
return None
if not thread:
thread = Thread(content.id)
flags = Message.IS_HTML
if thread.id not in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(
thread=thread,
id='0', # root message
url=self.browser.absurl(id2url(content.id)),
title=content.title,
sender=content.author or u'',
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % self.browser.absurl(id2url(content.id)),
children=[],
flags=flags)
for com in content.comments:
self._insert_comment(com, thread.root, getseen)
return thread
def _insert_comment(self, com, parent, getseen=True):
""""
Insert 'com' comment and its children in the parent message.
"""
flags = Message.IS_HTML
if com.id not in self.storage.get('seen',
parent.thread.id,
'comments',
default=[]):
flags |= Message.IS_UNREAD
if getseen or flags & Message.IS_UNREAD:
com.parse()
message = Message(
thread=parent.thread,
id=com.id,
url=com.url,
title=com.title,
sender=com.author or u'',
receivers=None,
date=com.date,
parent=parent,
content=com.body,
signature=com.signature +
'<br />'.join(['Score: %d' % com.score,
'URL: %s' % com.url]),
children=[],
flags=flags)
else:
message = Message(thread=parent.thread,
id=com.id,
children=[],
parent=parent,
flags=flags)
parent.children.append(message)
for sub in com.comments:
self._insert_comment(sub, message, getseen)
def iter_unread_messages(self):
for thread in self.iter_threads():
# Check if we have seen all comments of this thread.
oldhash = self.storage.get('hash', thread.id, default="")
newhash = self.browser.get_hash(thread._rsscomment)
if oldhash != newhash:
self.storage.set('hash', thread.id, newhash)
self.storage.save()
self.fill_thread(thread, 'root', getseen=False)
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
def set_message_read(self, message):
self.storage.set(
'seen', message.thread.id, 'comments',
self.storage.get('seen', message.thread.id, 'comments', default=[])
+ [message.id])
self.storage.save()
lastpurge = self.storage.get('lastpurge', default=0)
# 86400 = one day
if time.time() - lastpurge > 86400:
self.storage.set('lastpurge', time.time())
self.storage.save()
# we can't directly delete without a "RuntimeError: dictionary changed size during iteration"
todelete = []
for id in self.storage.get('seen', default={}):
date = self.storage.get('date', id, default=0)
# if no date available, create a new one (compatibility with "old" storage)
if date == 0:
self.storage.set('date', id, datetime.now())
elif datetime.now() - date > timedelta(days=60):
todelete.append(id)
for id in todelete:
self.storage.delete('hash', id)
self.storage.delete('date', id)
self.storage.delete('seen', id)
self.storage.save()
def fill_thread(self, thread, fields, getseen=True):
return self.get_thread(thread, getseen)
#### CapMessagesReply #########################################
def post_message(self, message):
if not self.browser.username:
raise BrowserForbidden()
if not message.parent:
raise CantSendMessage(
'Posting news and diaries on DLFP is not supported yet')
assert message.thread
return self.browser.post_comment(message.thread.id, message.parent.id,
message.title, message.content)
#### CapContent ###############################################
def get_content(self, _id, revision=None):
if isinstance(_id, basestring):
content = Content(_id)
else:
content = _id
_id = content.id
if revision:
raise NotImplementedError(
'Website does not provide access to older revisions sources.')
data = self.browser.get_wiki_content(_id)
if data is None:
return None
content.content = data
return content
def push_content(self, content, message=None, minor=False):
if not self.browser.username:
raise BrowserForbidden()
return self.browser.set_wiki_content(content.id, content.content,
message)
def get_content_preview(self, content):
return self.browser.get_wiki_preview(content.id, content.content)
OBJECTS = {Thread: fill_thread}
class IndeedModule(Module, CapJob):
NAME = 'indeed'
DESCRIPTION = u'indeed website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.1'
BROWSER = IndeedBrowser
type_contrat_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
'all': u'Tous les emplois',
'fulltime': u'Temps plein',
'parttime': u'Temps partiel',
'contract': u'Durée indéterminée',
'internship': u'Stage / Apprentissage',
'temporary': u'Durée déterminée',
}.iteritems())])
limit_date_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
'any': u'à tout moment',
'15': u'depuis 15 jours',
'7': u'depuis 7 jours',
'3': u'depuis 3 jours',
'1': u'depuis hier',
'last': u'depuis ma dernière visite',
}.iteritems())])
radius_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
'0': u'uniquement à cet endroit',
'5': u'dans un rayon de 5 kilomètres',
'10': u'dans un rayon de 10 kilomètres',
'15': u'dans un rayon de 15 kilomètres',
'25': u'dans un rayon de 25 kilomètres',
'50': u'dans un rayon de 50 kilomètres',
'100': u'dans un rayon de 100 kilomètres',
}.iteritems())])
CONFIG = BackendConfig(Value('metier', label=u'Job name', masked=False, default=''),
Value('limit_date', label=u'Date limite', choices=limit_date_choices, default=''),
Value('contrat', label=u'Contract', choices=type_contrat_choices, default=''),
Value('place', label=u'Place', masked=False, default=''),
Value('radius', label=u'Radius', choices=radius_choices, default=''))
def search_job(self, pattern=None):
return self.browser.search_job(metier=pattern)
def advanced_search_job(self):
return self.browser.search_job(metier=self.config['metier'].get(),
limit_date=self.config['limit_date'].get(),
contrat=self.config['contrat'].get(),
place=self.config['place'].get(),
radius=self.config['radius'].get())
def get_job_advert(self, _id, advert=None):
return self.browser.get_job_advert(_id, advert)
def fill_obj(self, advert, fields):
return self.get_job_advert(advert.id, advert)
OBJECTS = {BaseJobAdvert: fill_obj}
class PhpBBModule(Module, CapMessages, CapMessagesPost):
NAME = 'phpbb'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.3'
LICENSE = 'AGPLv3+'
DESCRIPTION = "phpBB forum"
CONFIG = BackendConfig(Value('url', label='URL of forum', regexp='https?://.*'),
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''),
ValueInt('thread_unread_messages', label='Limit number of unread messages to retrieve for a thread', default=500)
)
STORAGE = {'seen': {}}
BROWSER = PhpBB
def create_default_browser(self):
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
else:
password = None
return self.create_browser(self.config['url'].get(),
username, password)
#### CapMessages ##############################################
def _iter_threads(self, root_link=None):
links = list(self.browser.iter_links(root_link.url if root_link else None))
for link in links:
if link.type == link.FORUM:
link.title = '%s[%s]' % (root_link.title if root_link else '', link.title)
for thread in self._iter_threads(link):
yield thread
if link.type == link.TOPIC:
thread = Thread(url2id(link.url))
thread.title = ('%s ' % root_link.title if root_link else '') + link.title
thread.date = link.date
thread.flags = thread.IS_DISCUSSION
yield thread
def iter_threads(self):
return self._iter_threads()
def get_thread(self, id):
thread = None
parent = None
if isinstance(id, Thread):
thread = id
id = thread.id
thread_id = url2id(id, nopost=True) or id
try:
last_seen_id = self.storage.get('seen', default={})[id2topic(thread_id)]
except KeyError:
last_seen_id = 0
for post in self.browser.iter_posts(id):
if not thread:
thread = Thread(thread_id)
thread.title = post.title
m = self._post2message(thread, post)
m.parent = parent
if last_seen_id < post.id:
m.flags |= Message.IS_UNREAD
if parent:
parent.children = [m]
else:
thread.root = m
parent = m
return thread
def _post2message(self, thread, post):
signature = post.signature
if signature:
signature += '<br />'
signature += 'URL: %s' % self.browser.absurl(id2url('%s.%s' % (thread.id, post.id)))
return Message(thread=thread,
id=post.id,
title=post.title,
sender=post.author,
receivers=None,
date=post.date,
parent=None,
content=post.content,
signature=signature,
children=[],
flags=Message.IS_HTML)
def iter_unread_messages(self):
url = self.browser.get_root_feed_url()
for article in Newsfeed(url, rssid).iter_entries():
id = url2id(article.link)
thread = None
try:
last_seen_id = self.storage.get('seen', default={})[id2topic(id)]
except KeyError:
last_seen_id = 0
child = None
iterator = self.browser.riter_posts(id, last_seen_id)
if self.config['thread_unread_messages'].get() > 0:
iterator = limit(iterator, self.config['thread_unread_messages'].get())
for post in iterator:
if not thread:
thread = Thread('%s.%s' % (post.forum_id, post.topic_id))
message = self._post2message(thread, post)
if child:
message.children.append(child)
child.parent = message
if post.parent:
message.parent = Message(thread=thread,
id=post.parent)
else:
thread.root = message
yield message
def set_message_read(self, message):
try:
last_seen_id = self.storage.get('seen', default={})[id2topic(message.thread.id)]
except KeyError:
last_seen_id = 0
if message.id > last_seen_id:
self.storage.set('seen', id2topic(message.thread.id), message.id)
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
#### CapMessagesReply #########################################
def post_message(self, message):
assert message.thread
forum = 0
topic = 0
if message.thread:
try:
if '.' in message.thread.id:
forum, topic = [int(i) for i in message.thread.id.split('.', 1)]
else:
forum = int(message.thread.id)
except ValueError:
raise CantSendMessage('Thread ID must be in form "FORUM_ID[.TOPIC_ID]".')
return self.browser.post_answer(forum,
topic,
message.title,
message.content)
OBJECTS = {Thread: fill_thread}
def setUp(self):
if not self.is_backend_configured():
self.backend.config['resolution'] = Value(value='240')
self.backend.config['format'] = Value(value='mp4')
class AmundiModule(Module, CapBank):
NAME = 'amundi'
DESCRIPTION = u'amundi website'
MAINTAINER = u'James GALT'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.2'
CONFIG = BackendConfig(
ValueBackendPassword('login',
label='Identifiant',
regexp='\d+',
masked=False),
ValueBackendPassword('password', label=u"Mot de passe", regexp='\d+'),
Value('website',
label='Type de compte',
default='ee',
choices={
'ee': 'Amundi Epargne Entreprise',
'tc': 'Amundi Tenue de Compte'
}))
def create_default_browser(self):
b = {'ee': AmundiEEBrowser, 'tc': AmundiTCBrowser}
self.BROWSER = b[self.config['website'].get()]
w = {
'ee': 'https://www.amundi-ee.com',
'tc': 'https://epargnants.amundi-tc.com'
}
return self.create_browser(w[self.config['website'].get()],
self.config['login'].get(),
self.config['password'].get())
def get_account(self, id):
"""
Get an account from its ID.
:param id: ID of the account
:type id: :class:`str`
:rtype: :class:`Account`
:raises: :class:`AccountNotFound`
"""
return find_object(self.iter_accounts(), id=id, error=AccountNotFound)
def iter_accounts(self):
"""
Iter accounts.
:rtype: iter[:class:`Account`]
"""
return self.browser.iter_accounts()
def iter_investment(self, account):
"""
Iter investment of a market account
:param account: account to get investments
:type account: :class:`Account`
:rtype: iter[:class:`Investment`]
:raises: :class:`AccountNotFound`
"""
return self.browser.iter_investments(account)
def iter_history(self, account):
"""
Iter history of transactions on a specific account.
:param account: account to get history
:type account: :class:`Account`
:rtype: iter[:class:`Transaction`]
:raises: :class:`AccountNotFound`
"""
return self.browser.iter_history(account)
def double_auth(self, transfer):
code_needed = CleanText('//label[@for="code_securite"]')(self.doc)
if code_needed:
raise TransferStep(transfer, Value('code', label=code_needed))
class VoyagesSNCFModule(Module, CapTravel):
NAME = 'voyagessncf'
DESCRIPTION = u'Voyages SNCF'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.3'
CONFIG = BackendConfig(
Value('age',
label='Passenger age',
default='ADULT',
choices=OrderedDict(
(('ADULT', '26-59 ans'), ('SENIOR', '60 et +'),
('YOUNG', '12-25 ans'), ('CHILD_UNDER_FOUR', '0-3 ans'),
('CHILDREN', '4-11 ans')))),
Value('card',
label='Passenger card',
default='default',
choices=OrderedDict(
(('default', u'Pas de carte'), ('YOUNGS', u'Carte Jeune'),
('ESCA', u'Carte Escapades'), ('WEEKE', u'Carte Week-end'),
('FQ2ND', u'Abo Fréquence 2e'), ('FQ1ST',
u'Abo Fréquence 1e'),
('FF2ND', u'Abo Forfait 2e'), ('FF1ST', u'Abo Forfait 1e'),
('ACCWE', u'Accompagnant Carte Week-end'),
('ACCCHD',
u'Accompagnant Carte Enfant+'), ('ENFAM',
u'Carte Enfant Famille'),
('FAM30', u'Carte Familles Nombreuses 30%'),
('FAM40', u'Carte Familles Nombreuses 40%'),
('FAM50', u'Carte Familles Nombreuses 50%'),
('FAM75', u'Carte Familles Nombreuses 75%'),
('MI2ND', u'Carte Militaire 2e'), ('MI1ST',
u'Carte Militaire 1e'),
('MIFAM',
u'Carte Famille Militaire'), ('THBIZ',
u'Thalys ThePass Business'),
('THPREM',
u'Thalys ThePass Premium'), ('THWE',
u'Thalys ThePass Weekend')))),
Value('class',
label='Comfort class',
default='2',
choices=OrderedDict((('1', u'1e classe'), ('2', u'2e classe')))))
BROWSER = VoyagesSNCFBrowser
STATIONS = []
def _populate_stations(self):
if len(self.STATIONS) == 0:
with self.browser:
self.STATIONS = self.browser.get_stations()
def iter_station_search(self, pattern):
self._populate_stations()
pattern = pattern.lower()
already = set()
# First stations whose name starts with pattern...
for _id, name in enumerate(self.STATIONS):
if name.lower().startswith(pattern):
already.add(_id)
yield Station(_id, unicode(name))
# ...then ones whose name contains pattern.
for _id, name in enumerate(self.STATIONS):
if pattern in name.lower() and _id not in already:
yield Station(_id, unicode(name))
def iter_station_departures(self, station_id, arrival_id=None, date=None):
self._populate_stations()
if arrival_id is None:
raise UserError('The arrival station is required')
try:
station = self.STATIONS[int(station_id)]
arrival = self.STATIONS[int(arrival_id)]
except (IndexError, ValueError):
try:
station = list(self.iter_station_search(station_id))[0].name
arrival = list(self.iter_station_search(arrival_id))[0].name
except IndexError:
raise UserError('Unknown station')
with self.browser:
for i, d in enumerate(
self.browser.iter_departures(station, arrival, date,
self.config['age'].get(),
self.config['card'].get(),
self.config['class'].get())):
departure = Departure(i, d['type'], d['time'])
departure.departure_station = d['departure']
departure.arrival_station = d['arrival']
departure.arrival_time = d['arrival_time']
departure.price = d['price']
departure.currency = d['currency']
departure.information = d['price_info']
yield departure
class AmazonModule(Module, CapDocument):
NAME = 'amazon'
DESCRIPTION = 'Amazon'
MAINTAINER = 'Théo Dorée'
EMAIL = '*****@*****.**'
LICENSE = 'LGPLv3+'
VERSION = '1.6'
website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({
'www.amazon.com': u'Amazon.com',
'www.amazon.fr': u'Amazon France',
'www.amazon.de': u'Amazon.de',
'www.amazon.co.uk': u'Amazon UK',
}.items())])
BROWSERS = {
'www.amazon.fr': AmazonBrowser,
'www.amazon.com': AmazonEnBrowser,
'www.amazon.de': AmazonDeBrowser,
'www.amazon.co.uk': AmazonUkBrowser,
}
CONFIG = BackendConfig(
Value('website', label=u'Website', choices=website_choices, default='www.amazon.com'),
ValueBackendPassword('email', label='Username', masked=False),
ValueBackendPassword('password', label='Password'),
Value('captcha_response', label='Captcha Response', required=False, default=''),
Value('pin_code', label='OTP response', required=False, default='')
)
accepted_document_types = (DocumentTypes.BILL,)
def create_default_browser(self):
self.BROWSER = self.BROWSERS[self.config['website'].get()]
return self.create_browser(self.config)
def iter_subscription(self):
return self.browser.iter_subscription()
def get_subscription(self, _id):
return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound)
def get_document(self, _id):
subid = _id.rsplit('_', 1)[0]
subscription = self.get_subscription(subid)
return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound)
def iter_documents(self, subscription):
if not isinstance(subscription, Subscription):
subscription = self.get_subscription(subscription)
return self.browser.iter_documents(subscription)
def download_document(self, document):
if not isinstance(document, Document):
document = self.get_document(document)
if document.url is NotAvailable:
return
return self.browser.open(document.url).content
def download_document_pdf(self, document):
if not isinstance(document, Document):
document = self.get_document(document)
if document.url is NotAvailable:
return
if document.format == 'pdf':
return self.browser.open(document.url).content
url = urljoin(self.browser.BASEURL, document.url)
return html_to_pdf(self.browser, url=url)
class LutimModule(Module, CapPaste):
NAME = 'lutim'
DESCRIPTION = u'LUTIm website'
MAINTAINER = u'Vincent A'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.1'
BROWSER = LutimBrowser
CONFIG = BackendConfig(
Value('base_url', label='Hoster base URL', default='http://lut.im/'))
def _base_url(self):
url = self.config['base_url'].get()
if not url.endswith('/'):
url = url + '/'
return url
def create_default_browser(self):
return self.create_browser(self._base_url())
def can_post(self, contents, title=None, public=None, max_age=None):
if re.search(r'[^a-zA-Z0-9=+/\s]', contents):
return 0
elif max_age and max_age < 86400:
return 0 # it cannot be shorter than one day
else:
mime = image_mime(contents, ('gif', 'jpeg', 'png'))
return 20 * int(mime is not None)
def new_paste(self, *a, **kw):
base_url = self._base_url()
class LutImage(BasePaste):
@classmethod
def id2url(cls, id):
return urljoin(base_url, id)
@classmethod
def url2id(cls, url):
if url.startswith(base_url):
return url[len(base_url):]
return LutImage(*a, **kw)
def get_paste(self, id):
paste = self.new_paste(id)
if '/' in id:
paste.id = paste.url2id(id)
if not paste.id:
return None
response = self.browser.readurl(paste.page_url)
if response:
paste.contents = response.encode('base64')
return paste
def post_paste(self, paste, max_age=None):
d = self.browser.post(paste.title or None,
paste.contents.decode('base64'),
(max_age or 0) // 86400)
if d:
paste.id = d['id']
class CreditDuNordModule(Module, CapBank, CapProfile):
NAME = 'creditdunord'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.3'
DESCRIPTION = u'Crédit du Nord, Banque Courtois, Kolb, Nuger, Laydernier, Tarneaud, Société Marseillaise de Crédit'
LICENSE = 'AGPLv3+'
website_choices = OrderedDict([
(k, u'%s (%s)' % (v, k))
for k, v in sorted({
'www.credit-du-nord.fr': u'Crédit du Nord',
'www.banque-courtois.fr': u'Banque Courtois',
'www.banque-kolb.fr': u'Banque Kolb',
'www.banque-laydernier.fr': u'Banque Laydernier',
'www.banque-nuger.fr': u'Banque Nuger',
'www.banque-rhone-alpes.fr': u'Banque Rhône-Alpes',
'www.tarneaud.fr': u'Tarneaud',
'www.smc.fr': u'Société Marseillaise de Crédit',
}.iteritems(),
key=lambda k_v: (k_v[1], k_v[0]))
])
CONFIG = BackendConfig(
Value('website',
label='Banque',
choices=website_choices,
default='www.credit-du-nord.fr'),
ValueBackendPassword('login', label='Identifiant', masked=False),
ValueBackendPassword('password', label='Code confidentiel'))
BROWSER = CreditDuNordBrowser
def create_default_browser(self):
return self.create_browser(self.config['website'].get(),
self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
account = self.get_account(account.id)
for tr in self.browser.get_history(account):
if not tr._is_coming:
yield tr
def iter_coming(self, account):
account = self.get_account(account.id)
for tr in self.browser.get_history(account, coming=True):
if tr._is_coming:
yield tr
def iter_investment(self, account):
account = self.get_account(account.id)
return self.browser.get_investment(account)
def get_profile(self):
return self.browser.get_profile()
class YoutubeModule(Module, CapVideo, CapCollection):
NAME = 'youtube'
MAINTAINER = u'Laurent Bachelier'
EMAIL = '*****@*****.**'
VERSION = '1.4'
DESCRIPTION = 'YouTube video streaming website'
LICENSE = 'AGPLv3+'
BROWSER = None
CONFIG = BackendConfig(
Value('username', label='Email address', default=''),
ValueBackendPassword('password', label='Password', default=''))
URL_RE = re.compile(
r'^https?://(?:\w*\.?youtube(?:|-nocookie)\.com/(?:watch\?v=|embed/|v/)|youtu\.be\/|\w*\.?youtube\.com\/user\/\w+#p\/u\/\d+\/)([^\?&]+)'
)
def create_default_browser(self):
password = None
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
return self.create_browser(username, password)
def _entry2video(self, entry):
"""
Parse an entry returned by googleapi and return a Video object.
"""
snippet = entry['snippet']
id = entry['id']
if isinstance(id, dict):
id = id['videoId']
video = YoutubeVideo(to_unicode(id))
video.title = to_unicode(snippet['title'].strip())
# duration does not seem to be available with api
video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url'])
video.author = to_unicode(snippet['channelTitle'].strip())
return video
def _set_video_attrs(self, video):
new_video = video_info(YoutubeVideo.id2url(video.id))
if not new_video:
return
for k, v in new_video.iter_fields():
if not empty(v) and empty(getattr(video, k)):
setattr(video, k, v)
def get_video(self, _id):
m = self.URL_RE.match(_id)
if m:
_id = m.group(1)
params = {'id': _id, 'part': 'id,snippet'}
youtube = self._build_yt()
response = youtube.videos().list(**params).execute()
items = response.get('items', [])
if not items:
return None
video = self._entry2video(items[0])
self._set_video_attrs(video)
video.set_empty_fields(NotAvailable)
# Youtube video url is https, using ssl encryption
# so we need to use the "play_proxy" method using urllib2 proxy streaming to handle this
video._play_proxy = True
return video
def _build_yt(self):
DEVELOPER_KEY = "AIzaSyApVVeZ03XkKDYHX8T5uOn8Eizfe9CMDbs"
YOUTUBE_API_SERVICE_NAME = "youtube"
YOUTUBE_API_VERSION = "v3"
return ytbuild(YOUTUBE_API_SERVICE_NAME,
YOUTUBE_API_VERSION,
developerKey=DEVELOPER_KEY)
def search_videos(self,
pattern,
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False):
YOUTUBE_MAX_RESULTS = 50
youtube = self._build_yt()
params = {'part': 'id,snippet', 'maxResults': YOUTUBE_MAX_RESULTS}
if pattern is not None:
if isinstance(pattern, unicode):
pattern = pattern.encode('utf-8')
params['q'] = pattern
params['safeSearch'] = 'none' if nsfw else 'strict' # or 'moderate'
params['order'] = ('relevance', 'rating', 'viewCount', 'date')[sortby]
nb_yielded = 0
while True:
search_response = youtube.search().list(**params).execute()
items = search_response.get('items', [])
for entry in items:
if entry["id"]["kind"] != "youtube#video":
continue
yield self._entry2video(entry)
nb_yielded += 1
params['pageToken'] = search_response.get('nextPageToken')
if not params['pageToken']:
return
if nb_yielded < YOUTUBE_MAX_RESULTS:
return
def latest_videos(self):
return self.search_videos(None, CapVideo.SEARCH_DATE)
def fill_video(self, video, fields):
if 'thumbnail' in fields and video.thumbnail:
video.thumbnail.data = requests.get(video.thumbnail.url).content
if 'url' in fields:
self._set_video_attrs(video)
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest'])
if collection.split_path == [u'latest']:
for video in self.latest_videos():
yield video
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and collection.split_path == [u'latest']:
collection.title = u'Latest YouTube videos'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {YoutubeVideo: fill_video}
def new_recipient(self, recipient, **params):
if 'code' in params:
self.validate_rcpt_with_sms(params['code'])
return self.page.rcpt_after_sms(recipient)
data = {
'n_nbOccurences': 1000,
'n_nbOccurences_affichees': 0,
'n_rang': 0,
}
self.recipients.go(data=data)
step_urls = {
'first_recipient_check':
self.absurl(
'/ord-web/ord//ord-valider-destinataire-avant-maj.json',
base=True),
'get_bic':
self.absurl('/ord-web/ord//ord-tiers-calcul-bic.json', base=True),
'get_token':
self.absurl(
'/ord-web/ord//ord-preparer-signature-destinataire.json',
base=True),
'get_sign_info':
self.absurl('/sec/getsigninfo.json', base=True),
'send_otp_to_user':
self.absurl('/sec/csa/send.json', base=True),
}
self.add_recipient.go(
method='POST',
headers={'Content-Type': 'application/json;charset=UTF-8'})
countries = self.page.get_countries()
# first recipient check
data = {
'an_codeAction': 'ajout_tiers',
'an_refSICoordonnee': '',
'an_refSITiers': '',
'cl_iban': recipient.iban,
'cl_raisonSociale': recipient.label,
}
self.location(step_urls['first_recipient_check'], data=data)
# get bic
data = {
'an_activateCMU': 'true',
'an_codePaysBanque': '',
'an_nature': 'C',
'an_numeroCompte': recipient.iban,
'an_topIBAN': 'true',
'cl_adresse': '',
'cl_adresseBanque': '',
'cl_codePays': recipient.iban[:2],
'cl_libellePaysBanque': '',
'cl_libellePaysDestinataire': countries[recipient.iban[:2]],
'cl_nomBanque': '',
'cl_nomRaisonSociale': recipient.label,
'cl_ville': '',
'cl_villeBanque': '',
}
self.location(step_urls['get_bic'], data=data)
bic = self.page.get_response_data()
# get token
data = {
'an_coordonnee_codePaysBanque': '',
'an_coordonnee_nature': 'C',
'an_coordonnee_numeroCompte': recipient.iban,
'an_coordonnee_topConfidentiel': 'false',
'an_coordonnee_topIBAN': 'true',
'an_refSICoordonnee': '',
'an_refSIDestinataire': '',
'cl_adresse': '',
'cl_codePays': recipient.iban[:2],
'cl_coordonnee_adresseBanque': '',
'cl_coordonnee_bic': bic,
'cl_coordonnee_categories_libelle': '',
'cl_coordonnee_categories_refSi': '',
'cl_coordonnee_libellePaysBanque': '',
'cl_coordonnee_nomBanque': '',
'cl_coordonnee_villeBanque': '',
'cl_libellePaysDestinataire': countries[recipient.iban[:2]],
'cl_nomRaisonSociale': recipient.label,
'cl_ville': '',
}
self.location(step_urls['get_token'], data=data)
self.new_rcpt_validate_form = data
payload = self.page.get_response_data()
# get sign info
data = {
'b64_jeton_transaction': payload['jeton'],
'action_level': payload['sensibilite'],
}
self.location(step_urls['get_sign_info'], data=data)
# send otp to user
data = {'context': payload['jeton'], 'csa_op': 'sign'}
self.location(step_urls['send_otp_to_user'], data=data)
self.new_rcpt_validate_form.update(data)
rcpt = self.copy_recipient_obj(recipient)
self.need_reload_state = True
raise AddRecipientStep(
rcpt, Value('code', label='Veuillez entrer le code reçu par SMS.'))
class GDCVaultModule(Module, CapVideo, CapCollection):
NAME = 'gdcvault'
MAINTAINER = u'François Revol'
EMAIL = '*****@*****.**'
VERSION = '1.4'
DESCRIPTION = 'Game Developers Conferences Vault video streaming website'
LICENSE = 'AGPLv3+'
BROWSER = GDCVaultBrowser
CONFIG = BackendConfig(
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''))
def create_default_browser(self):
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
else:
password = None
return self.create_browser(username, password)
def deinit(self):
# don't need to logout if the browser hasn't been used.
if not self._browser:
return
with self.browser:
self.browser.close_session()
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
SORTBY = ['relevance', 'rating', 'views', 'time']
def search_videos(self,
pattern,
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False):
with self.browser:
return self.browser.search_videos(pattern, self.SORTBY[sortby])
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(GDCVaultVideo.id2url(video.id),
video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(
video.thumbnail.url)
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest'])
if collection.split_path == [u'latest']:
for video in self.browser.latest_videos():
yield video
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and collection.split_path == [u'latest']:
collection.title = u'Latest GDCVault videos'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {GDCVaultVideo: fill_video}
class TwitterModule(Module, CapMessages, CapMessagesPost, CapCollection):
NAME = 'twitter'
DESCRIPTION = u'twitter website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.4'
BROWSER = TwitterBrowser
STORAGE = {'seen': {}}
CONFIG = BackendConfig(
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''),
Value('hashtags_subscribe', label='Hashtags subscribe', default=''),
Value('search_subscribe', label='Search subscribe', default=''),
Value('profils_subscribe', label='Profils subscribe', default=''))
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
else:
password = None
return self.create_browser(username, password)
def iter_threads(self):
if self.config['username'].get():
return self.browser.iter_threads()
else:
profils = self.config['profils_subscribe'].get()
hashtags = self.config['hashtags_subscribe'].get()
searchs = self.config['search_subscribe'].get()
tweets = []
if profils:
for profil in profils.split(','):
for tweet in itertools.islice(
self.browser.get_tweets_from_profil(profil), 0,
20):
tweets.append(tweet)
if hashtags:
for hashtag in hashtags.split(','):
for tweet in itertools.islice(
self.browser.get_tweets_from_hashtag(hashtag), 0,
20):
tweets.append(tweet)
if searchs:
for search in searchs.split(','):
for tweet in itertools.islice(
self.browser.get_tweets_from_search(search), 0,
20):
tweets.append(tweet)
tweets.sort(key=lambda o: o.date, reverse=True)
return tweets
def get_thread(self, _id, thread=None, getseen=True):
seen = None
if getseen:
seen = self.storage.get('seen', default={})
return self.browser.get_thread(_id, thread, seen)
def fill_thread(self, thread, fields, getseen=True):
return self.get_thread(thread.id, thread, getseen)
def set_message_read(self, message):
self.storage.set('seen', message.thread.id, message.thread.date)
self.storage.save()
self._purge_message_read()
def _purge_message_read(self):
lastpurge = self.storage.get('lastpurge',
default=datetime.now() -
timedelta(days=60))
if datetime.now() - lastpurge > timedelta(days=60):
self.storage.set('lastpurge', datetime.now() - timedelta(days=60))
self.storage.save()
# we can't directly delete without a "RuntimeError: dictionary changed size during iteration"
todelete = []
for id, date in self.storage.get('seen', default={}).items():
# if no date available, create a new one (compatibility with "old" storage)
if not date:
self.storage.set('seen', id, datetime.now())
elif lastpurge > date:
todelete.append(id)
for id in todelete:
self.storage.delete('seen', id)
self.storage.save()
def post_message(self, message):
if not self.config['username'].get():
raise BrowserForbidden()
self.browser.post(
find_object(self.iter_threads(), id=message.full_id.split('.')[0]),
message.content)
def iter_resources(self, objs, split_path):
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
if self.config['username'].get():
yield Collection([u'me'], u'me')
yield Collection([u'profils'], u'profils')
yield Collection([u'trendy'], u'trendy')
yield Collection([u'hashtags'], u'hashtags')
yield Collection([u'search'], u'search')
if collection.path_level == 1:
if collection.split_path[0] == u'me':
for el in self.browser.get_tweets_from_profil(
self.browser.get_me()):
yield el
if collection.split_path[0] == u'profils':
profils = self.config['profils_subscribe'].get()
if profils:
for profil in profils.split(','):
yield Collection([profil], profil)
if collection.split_path[0] == u'hashtags':
hashtags = self.config['hashtags_subscribe'].get()
if hashtags:
for hashtag in hashtags.split(','):
yield Collection([hashtag], hashtag)
if collection.split_path[0] == u'search':
searchs = self.config['search_subscribe'].get()
if searchs:
for search in searchs.split(','):
yield Collection([search], search)
if collection.split_path[0] == u'trendy':
for obj in self.browser.get_trendy_subjects():
yield Collection([obj.id], obj.id)
if collection.path_level == 2:
if collection.split_path[0] == u'profils':
for el in self.browser.get_tweets_from_profil(
collection.split_path[1]):
yield el
if collection.split_path[0] == u'trendy':
if collection.split_path[1].startswith('#'):
for el in self.browser.get_tweets_from_hashtag(
collection.split_path[1]):
yield el
else:
for el in self.browser.get_tweets_from_search(
collection.split_path[1]):
yield el
if collection.split_path[0] == u'hashtags':
for el in self.browser.get_tweets_from_hashtag(
collection.split_path[1]):
yield el
if collection.split_path[0] == u'search':
for el in self.browser.get_tweets_from_search(
collection.split_path[1]):
yield el
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if collection.path_level == 1 and collection.split_path[0] in \
[u'profils', u'trendy', u'me', u'hashtags', u'search']:
return
if collection.path_level == 2:
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {Thread: fill_thread}
class AgendadulibreModule(Module, CapCalendarEvent):
NAME = 'agendadulibre'
DESCRIPTION = u'agendadulibre website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '2.1'
ASSOCIATED_CATEGORIES = [CATEGORIES.CONF]
BROWSER = AgendadulibreBrowser
region_choices = OrderedDict([
(k, u'%s (%s)' % (v, k))
for k, v in sorted({
"https://www.agendadulibre.org": u'--France--',
"https://www.agendadulibre.org#3": u'Auvergne-Rhône-Alpes',
"https://www.agendadulibre.org#5": u'Bourgogne-Franche-Comté',
"https://www.agendadulibre.org#6": u'Bretagne',
"https://www.agendadulibre.org#7": u'Centre-Val de Loire',
"https://www.agendadulibre.org#30": u'Collectivité sui generis',
"https://www.agendadulibre.org#29": u'Collectivités d\'outre-mer',
"https://www.agendadulibre.org#9": u'Corse',
"https://www.agendadulibre.org#1": u'Grand Est',
"https://www.agendadulibre.org#23": u'Guadeloupe',
"https://www.agendadulibre.org#24": u'Guyane',
"https://www.agendadulibre.org#17": u'Hauts-de-France',
"https://www.agendadulibre.org#12": u'Île-de-France',
"https://www.agendadulibre.org#31": u'Internet',
"https://www.agendadulibre.org#26": u'La Réunion',
"https://www.agendadulibre.org#25": u'Martinique',
"https://www.agendadulibre.org#28": u'Mayotte',
"https://www.agendadulibre.org#4": u'Normandie',
"https://www.agendadulibre.org#2": u'Nouvelle-Aquitaine',
"https://www.agendadulibre.org#13": u'Occitanie',
"https://www.agendadulibre.org#18": u'Pays de la Loire',
"https://www.agendadulibre.org#21": u'Provence-Alpes-Côte d\'Azur',
"https://www.agendadulibre.be": u'--Belgique--',
"https://www.agendadulibre.be#11": u'Antwerpen',
"https://www.agendadulibre.be#10": u'Brabant wallon',
"https://www.agendadulibre.be#9": u'Bruxelles-Capitale',
"https://www.agendadulibre.be#8": u'Hainaut',
"https://www.agendadulibre.be#7": u'Liege',
"https://www.agendadulibre.be#6": u'Limburg',
"https://www.agendadulibre.be#5": u'Luxembourg',
"https://www.agendadulibre.be#4": u'Namur',
"https://www.agendadulibre.be#3": u'Oost-Vlaanderen',
"https://www.agendadulibre.be#2": u'Vlaams-Brabant',
"https://www.agendadulibre.be#1": u'West-Vlaanderen',
"https://www.agendadulibre.ch": u'--Suisse--',
"https://www.agendadulibre.ch#15": u'Appenzell Rhodes-Extérieures',
"https://www.agendadulibre.ch#16": u'Appenzell Rhodes-Intérieures',
"https://www.agendadulibre.ch#19": u'Argovie',
"https://www.agendadulibre.ch#13": u'Bâle-Campagne',
"https://www.agendadulibre.ch#12": u'Bâle-Ville',
"https://www.agendadulibre.ch#2": u'Berne',
"https://www.agendadulibre.ch#10": u'Fribourg',
"https://www.agendadulibre.ch#25": u'Genève',
"https://www.agendadulibre.ch#8": u'Glaris',
"https://www.agendadulibre.ch#18": u'Grisons',
"https://www.agendadulibre.ch#26": u'Jura',
"https://www.agendadulibre.ch#3": u'Lucerne',
"https://www.agendadulibre.ch#24": u'Neuchâtel',
"https://www.agendadulibre.ch#7": u'Nidwald',
"https://www.agendadulibre.ch#6": u'Obwald',
"https://www.agendadulibre.ch#17": u'Saint-Gall',
"https://www.agendadulibre.ch#14": u'Schaffhouse',
"https://www.agendadulibre.ch#5": u'Schwytz',
"https://www.agendadulibre.ch#11": u'Soleure',
"https://www.agendadulibre.ch#21": u'Tessin',
"https://www.agendadulibre.ch#20": u'Thurgovie',
"https://www.agendadulibre.ch#4": u'Uri',
"https://www.agendadulibre.ch#23": u'Valais',
"https://www.agendadulibre.ch#22": u'Vaud',
"https://www.agendadulibre.ch#9": u'Zoug',
"https://www.agendadulibre.ch#1": u'Zurich',
}.items())
])
CONFIG = BackendConfig(
Value('region', label=u'Region', choices=region_choices))
def create_default_browser(self):
choice = self.config['region'].get().split('#')
selected_region = '' if len(choice) < 2 else choice[-1]
return self.create_browser(website=choice[0], region=selected_region)
def search_events(self, query):
return self.browser.list_events(query.start_date, query.end_date,
query.city, query.categories)
def list_events(self, date_from, date_to=None):
return self.browser.list_events(date_from, date_to)
def get_event(self, event_id):
return self.browser.get_event(event_id)
def fill_obj(self, event, fields):
event = self.browser.get_event(event.id, event)
choice = self.config['region'].get().split('#')
selected_region = '' if len(choice) < 2 else choice[-1]
if selected_region == '23':
event.timezone = 'America/Guadeloupe'
elif selected_region == '24':
event.timezone = 'America/Guyana'
elif selected_region == '26':
event.timezone = 'Indian/Reunion'
elif selected_region == '25':
event.timezone = 'America/Martinique'
else:
event.timezone = 'Europe/Paris'
return event
OBJECTS = {AgendadulibreBrowser: fill_obj}
class MonsterModule(Module, CapJob):
NAME = 'monster'
DESCRIPTION = u'monster website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.6'
BROWSER = MonsterBrowser
type_contrat_choices = OrderedDict([
(k, u'%s' % (v))
for k, v in sorted({
'Interim-ou-CDD-ou-mission_8': u'Interim ou CDD ou mission',
'CDI_8': u'CDI',
'Stage-Apprentissage-Alternance_8':
u'Stage/Apprentissage/Alternance',
' ': u'Autres',
'Indépendant-Freelance-Saisonnier-Franchise_8':
u'Indépendant/Freelance/Saisonnier/Franchise',
'Journalier_8': u'Journalier',
'Temps-Partiel_8': u'Temps Partiel',
'Temps-Plein_8': u'Temps Plein',
}.items())
])
date_choices = OrderedDict([(k, u'%s' % (v))
for k, v in sorted({
'-1': u'N importe quelle date',
'000000': u'Aujourd hui',
'1': u'2 derniers jours',
'3': u'3 derniers jours',
'7': u'Les 7 derniers jours',
'14': u'Les 14 derniers jours',
'30': u'30 derniers jours',
}.items())])
CONFIG = BackendConfig(
Value('job_name', label='Job name', masked=False, default=''),
Value('place', label='Place', masked=False, default=''),
Value('contract',
label=u'Contract',
choices=type_contrat_choices,
default=''),
Value('limit_date', label=u'Date', choices=date_choices, default='-1'),
)
def search_job(self, pattern=None):
return self.browser.search_job(pattern)
def advanced_search_job(self):
return self.browser.advanced_search_job(
job_name=self.config['job_name'].get(),
place=self.config['place'].get(),
contract=self.config['contract'].get(),
limit_date=self.config['limit_date'].get())
def get_job_advert(self, _id, advert=None):
return self.browser.get_job_advert(_id, advert)
def fill_obj(self, advert, fields):
return self.get_job_advert(advert.id, advert)
OBJECTS = {BaseJobAdvert: fill_obj}
class NolifeTVBackend(BaseBackend, ICapVideo, ICapCollection):
NAME = 'nolifetv'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '0.e'
DESCRIPTION = 'NolifeTV French video streaming website'
LICENSE = 'AGPLv3+'
BROWSER = NolifeTVBrowser
CONFIG = BackendConfig(
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''))
def create_default_browser(self):
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
else:
password = None
return self.create_browser(username, password)
def get_video(self, _id):
with self.browser:
video = self.browser.get_video(_id)
return video
def search_videos(self,
pattern,
sortby=ICapVideo.SEARCH_RELEVANCE,
nsfw=False,
max_results=None):
with self.browser:
return self.browser.search_videos(pattern)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(NolifeTVVideo.id2url(video.id),
video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(
video.thumbnail.url)
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest'])
if collection.split_path == [u'latest']:
for video in self.browser.latest_videos():
yield video
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and collection.split_path == [u'latest']:
collection.title = u'Latest NoLiveTV videos'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {NolifeTVVideo: fill_video}
class NewspaperFigaroModule(AbstractModule, CapMessages):
MAINTAINER = u'Julien Hebert'
EMAIL = '*****@*****.**'
VERSION = '2.1'
LICENSE = 'AGPLv3+'
STORAGE = {'seen': {}}
NAME = 'lefigaro'
DESCRIPTION = u'Le Figaro French newspaper website'
BROWSER = NewspaperFigaroBrowser
RSS_FEED = 'http://rss.lefigaro.fr/lefigaro/laune?format=xml'
RSSID = staticmethod(rssid)
RSSSIZE = 30
PARENT = 'genericnewspaper'
CONFIG = BackendConfig(
Value('feed',
label='RSS feed',
choices={
'actualites': u'actualites',
'flash-actu': u'flash-actu',
'politique': u'politique',
'international': u'international',
'actualite-france': u'actualite-france',
'hightech': u'hightech',
'sciences': u'sciences',
'sante': u'sante',
'lefigaromagazine': u'lefigaromagazine',
'photos': u'photos',
'economie': u'economie',
'societes': u'societes',
'medias': u'medias',
'immobilier': u'immobilier',
'assurance': u'assurance',
'retraite': u'retraite',
'placement': u'placement',
'impots': u'impots',
'conso': u'conso',
'emploi': u'emploi',
'culture': u'culture',
'cinema': u'cinema',
'musique': u'musique',
'livres': u'livres',
'theatre': u'theatre',
'lifestyle': u'lifestyle',
'automobile': u'automobile',
'gastronomie': u'gastronomie',
'horlogerie': u'horlogerie',
'mode-homme': u'mode-homme',
'sortir-paris': u'sortir-paris',
'vins': u'vins',
'voyages': u'voyages',
'sport': u'sport',
'football': u'football',
'rugby': u'rugby',
'tennis': u'tennis',
'cyclisme': u'cyclisme',
'sport-business': u'sport-business'
}))
def __init__(self, *args, **kwargs):
super(self.__class__, self).__init__(*args, **kwargs)
self.RSS_FEED = "http://www.lefigaro.fr/rss/figaro_%s.xml" % self.config[
'feed'].get()
def iter_threads(self):
for article in Newsfeed(self.RSS_FEED, self.RSSID).iter_entries():
thread = Thread(article.id)
thread.title = article.title
thread.date = article.datetime
yield (thread)
class NewsfeedModule(Module, CapMessages):
NAME = 'newsfeed'
MAINTAINER = u'Clément Schreiner'
EMAIL = "*****@*****.**"
VERSION = '2.1'
DESCRIPTION = "Loads RSS and Atom feeds from any website"
LICENSE = "AGPLv3+"
CONFIG = BackendConfig(Value('url', label="Atom/RSS feed's url", regexp='https?://.*'))
STORAGE = {'seen': []}
def iter_threads(self):
for article in Newsfeed(self.config['url'].get()).iter_entries():
yield self.get_thread(article.id, article)
def get_thread(self, id, entry=None):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = Thread(id)
if entry is None:
entry = Newsfeed(self.config['url'].get()).get_entry(id)
if entry is None:
return None
flags = Message.IS_HTML
if thread.id not in self.storage.get('seen', default=[]):
flags |= Message.IS_UNREAD
if len(entry.content) > 0:
content = u"<p>Link %s</p> %s" % (entry.link, entry.content[0])
else:
content = entry.link
thread.title = entry.title
thread.root = Message(thread=thread,
id=0,
url=entry.link,
title=entry.title,
sender=entry.author,
receivers=None,
date=entry.datetime,
parent=None,
content=content,
children=[],
flags=flags)
return thread
def iter_unread_messages(self):
for thread in self.iter_threads():
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
def set_message_read(self, message):
self.storage.get('seen', default=[]).append(message.thread.id)
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread}
class ArteModule(Module, CapVideo, CapCollection):
NAME = 'arte'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
VERSION = '1.1'
DESCRIPTION = 'Arte French and German TV'
LICENSE = 'AGPLv3+'
order = {
'AIRDATE_DESC': 'Date',
'VIEWS': 'Views',
'ALPHA': 'Alphabetic',
'LAST_CHANCE': 'Last chance'
}
versions_choice = OrderedDict([(k, u'%s' % (v.get('label')))
for k, v in VERSION_VIDEO.items])
format_choice = OrderedDict([(k, u'%s' % (v)) for k, v in FORMATS.items])
lang_choice = OrderedDict([(k, u'%s' % (v.get('label')))
for k, v in LANG.items])
quality_choice = [u'%s' % (k) for k, v in QUALITY.items]
CONFIG = BackendConfig(
Value('lang',
label='Lang of videos',
choices=lang_choice,
default='FRENCH'),
Value('order',
label='Sort order',
choices=order,
default='AIRDATE_DESC'),
Value('quality',
label='Quality of videos',
choices=quality_choice,
default=QUALITY.HD),
Value('format',
label='Format of videos',
choices=format_choice,
default=FORMATS.HTTP_MP4),
Value('version', label='Version of videos', choices=versions_choice))
BROWSER = ArteBrowser
def create_default_browser(self):
return self.create_browser(lang=self.config['lang'].get(),
quality=self.config['quality'].get(),
order=self.config['order'].get(),
format=self.config['format'].get(),
version=self.config['version'].get())
def parse_id(self, _id):
m = re.match('^(\w+)\.(.*)', _id)
if m:
return m.groups()
m = re.match('https?://www.arte.tv/guide/\w+/(?P<id>.+)/(.*)', _id)
if m:
return SITE.PROGRAM.get('id'), m.group(1)
m = re.match(
'https?://(%s).arte.tv/(\w+)/(.*)' %
('|'.join(value.get('id') for value in SITE.values)), _id)
if m:
return m.group(1), '/%s/%s' % (m.group(2), m.group(3))
return 'videos', _id
def get_video(self, _id):
site, _id = self.parse_id(_id)
if site in [value.get('id') for value in SITE.values]:
_site = (value for value in SITE.values
if value.get('id') == site).next()
return getattr(self.browser, _site.get('video'))(_id)
else:
return self.browser.get_video(_id)
def search_videos(self,
pattern,
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False):
return self.browser.search_videos(pattern)
def fill_arte_video(self, video, fields):
if fields != ['thumbnail']:
video = self.browser.get_video(video.id, video)
if 'thumbnail' in fields and video and video.thumbnail:
video.thumbnail.data = self.browser.open(
video.thumbnail.url).content
return video
def fill_site_video(self, video, fields):
if fields != ['thumbnail']:
for site in SITE.values:
m = re.match('%s\.(.*)' % site.get('id'), video.id)
if m:
video = getattr(self.browser,
site.get('video'))(m.group(1), video)
break
if 'thumbnail' in fields and video and video.thumbnail:
video.thumbnail.data = self.browser.open(
video.thumbnail.url).content
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield Collection([u'arte-latest'], u'Latest Arte videos')
for site in SITE.values:
yield Collection([site.get('id')], site.get('label'))
if collection.path_level == 1:
if collection.split_path == [u'arte-latest']:
for video in self.browser.latest_videos():
yield video
else:
for site in SITE.values:
if collection.split_path[0] == site.get(
'id') and collection.path_level in site.keys():
for item in getattr(
self.browser,
site.get(collection.path_level))():
yield item
if collection.path_level >= 2:
for site in SITE.values:
if collection.split_path[0] == site.get(
'id') and collection.path_level in site.keys():
for item in getattr(self.browser,
site.get(collection.path_level))(
collection.split_path):
yield item
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and (collection.split_path == [u'arte-latest']
or collection.split_path[0] in [
value.get('id') for value in SITE.values
]):
return
if BaseVideo in objs and collection.path_level >= 2 and\
collection.split_path[0] in [value.get('id') for value in SITE.values]:
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {ArteVideo: fill_arte_video, ArteSiteVideo: fill_site_video}
class SocieteGeneraleModule(Module, CapBankWealth, CapBankTransferAddRecipient,
CapContact, CapProfile, CapDocument):
NAME = 'societegenerale'
MAINTAINER = u'Jocelyn Jaubert'
EMAIL = '*****@*****.**'
VERSION = '2.1'
LICENSE = 'LGPLv3+'
DESCRIPTION = u'Société Générale'
CONFIG = BackendConfig(
ValueBackendPassword('login', label='Code client', masked=False),
ValueBackendPassword('password', label='Code secret'),
Value('website',
label='Type de compte',
default='par',
choices={
'par': 'Particuliers',
'pro': 'Professionnels',
'ent': 'Entreprises'
}),
# SCA
ValueTransient('code'),
ValueTransient('resume'),
ValueTransient('request_information'),
)
accepted_document_types = (DocumentTypes.STATEMENT, DocumentTypes.RIB)
def create_default_browser(self):
website = self.config['website'].get()
browsers = {
'par': SocieteGenerale,
'pro': SGProfessionalBrowser,
'ent': SGEnterpriseBrowser
}
self.BROWSER = browsers[website]
if website in (
'par',
'pro',
):
return self.create_browser(self.config, self.config['login'].get(),
self.config['password'].get())
else:
return self.create_browser(self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
return find_object(self.browser.get_accounts_list(),
id=_id,
error=AccountNotFound)
def iter_coming(self, account):
if hasattr(self.browser, 'get_cb_operations'):
transactions = list(self.browser.get_cb_operations(account))
return sorted_transactions(transactions)
return self.browser.iter_coming(account)
def iter_history(self, account):
return self.browser.iter_history(account)
def iter_investment(self, account):
return self.browser.iter_investment(account)
def iter_contacts(self):
if not hasattr(self.browser, 'get_advisor'):
raise NotImplementedError()
return self.browser.get_advisor()
def get_profile(self):
if not hasattr(self.browser, 'get_profile'):
raise NotImplementedError()
return self.browser.get_profile()
def iter_transfer_recipients(self, origin_account):
if self.config['website'].get() not in ('par', 'pro'):
raise NotImplementedError()
if not isinstance(origin_account, Account):
origin_account = find_object(self.iter_accounts(),
id=origin_account,
error=AccountNotFound)
return self.browser.iter_recipients(origin_account)
def new_recipient(self, recipient, **params):
if self.config['website'].get() not in ('par', 'pro'):
raise NotImplementedError()
recipient.label = ' '.join(w for w in re.sub(
'[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split())
return self.browser.new_recipient(recipient, **params)
def init_transfer(self, transfer, **params):
if self.config['website'].get() not in ('par', 'pro'):
raise NotImplementedError()
transfer.label = ' '.join(
w for w in re.sub('[^0-9a-zA-Z ]+', '', transfer.label).split())
self.logger.info('Going to do a new transfer')
account = strict_find_object(self.iter_accounts(),
iban=transfer.account_iban)
if not account:
account = strict_find_object(self.iter_accounts(),
id=transfer.account_id,
error=AccountNotFound)
recipient = strict_find_object(self.iter_transfer_recipients(
account.id),
id=transfer.recipient_id)
if not recipient:
recipient = strict_find_object(self.iter_transfer_recipients(
account.id),
iban=transfer.recipient_iban,
error=RecipientNotFound)
transfer.amount = transfer.amount.quantize(Decimal('.01'))
return self.browser.init_transfer(account, recipient, transfer)
def execute_transfer(self, transfer, **params):
if self.config['website'].get() not in ('par', 'pro'):
raise NotImplementedError()
return self.browser.execute_transfer(transfer)
def transfer_check_exec_date(self, old_exec_date, new_exec_date):
return old_exec_date <= new_exec_date <= old_exec_date + timedelta(
days=4)
def iter_resources(self, objs, split_path):
if Account in objs:
self._restrict_level(split_path)
return self.iter_accounts()
if Subscription in objs:
self._restrict_level(split_path)
return self.iter_subscription()
def get_subscription(self, _id):
return find_object(self.iter_subscription(),
id=_id,
error=SubscriptionNotFound)
def get_document(self, _id):
subscription_id = _id.split('_')[0]
subscription = self.get_subscription(subscription_id)
return find_object(self.iter_documents(subscription),
id=_id,
error=DocumentNotFound)
def iter_subscription(self):
return self.browser.iter_subscription()
def iter_documents(self, subscription):
if not isinstance(subscription, Subscription):
subscription = self.get_subscription(subscription)
return self.browser.iter_documents(subscription)
def iter_documents_by_types(self, subscription, accepted_types):
if not isinstance(subscription, Subscription):
subscription = self.get_subscription(subscription)
if self.config['website'].get() not in ('ent', 'pro'):
for doc in self.browser.iter_documents_by_types(
subscription, accepted_types):
yield doc
else:
for doc in self.browser.iter_documents(subscription):
if doc.type in accepted_types:
yield doc
def download_document(self, document):
if not isinstance(document, Document):
document = self.get_document(document)
if document.url is NotAvailable:
return
return self.browser.open(document.url).content
class MailinatorModule(Module, CapMessages):
NAME = 'mailinator'
DESCRIPTION = u'mailinator temp mailbox'
MAINTAINER = u'Vincent A'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.2'
BROWSER = MailinatorBrowser
CONFIG = BackendConfig(Value('inbox', label='Inbox', default=''))
def iter_threads(self):
inbox = self.config['inbox'].get()
if not inbox:
raise NotImplementedError()
else:
for d in self.browser.get_mails(inbox):
thread = Thread(d['id'])
thread.title = d['subject']
thread.flags = thread.IS_DISCUSSION
msg = self.make_message(d, thread)
if not msg.content:
msg.content = self.browser.get_mail_content(msg.id)
thread.root = msg
yield thread
def _get_messages_thread(self, inbox, thread):
first = True
for d in self.browser.get_mails(inbox):
msg = self.make_message(d, thread)
if not msg.content:
msg.content = self.browser.get_mail_content(msg.id)
if first:
first = False
thread.root = msg
else:
msg.parent = thread.root
msg.parent.children.append(msg)
def get_thread(self, _id):
thread = Thread(_id)
thread.title = 'Mail for %s' % _id
thread.flags = thread.IS_DISCUSSION
self._get_messages_thread(_id, thread)
return thread
def make_message(self, d, thread):
msg = Message(thread, d['id'])
msg.children = []
msg.sender = d['from']
msg.flags = 0
if not d.get('read', True):
msg.flags = msg.IS_UNREAD
msg.title = d['subject']
msg.date = d['datetime']
msg.receivers = [d['to']]
return msg
def fill_msg(self, msg, fields):
if 'content' in fields:
msg.content = self.browser.get_mail_content(msg.id)
return msg
OBJECTS = {Message: fill_msg}
class LolixModule(Module, CapJob):
NAME = 'lolix'
DESCRIPTION = u'Lolix French free software employment website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
VERSION = '1.1'
BROWSER = LolixBrowser
region_choices = OrderedDict([
(k, u'%s' % (v)) for k, v in sorted({
'0': u'-- Indifférent --',
'100000000': u'-- France entière',
'100100000': u'-- France métropolitaine',
'100100001': u'-- Alsace',
'100100002': u'-- Auvergne',
'100100003': u'-- Aquitaine',
'100100004': u'-- Bourgogne',
'100100005': u'-- Bretagne',
'100100025': u'-- Centre',
'100100027': u'-- Champagne-Ardenne',
'100100030': u'-- Corse',
'100100037': u'-- Franche-Comté',
'100100040': u'-- Ile de France',
'100100044': u'-- Languedoc-Roussillon',
'100100048': u'-- Limousin',
'100100051': u'-- Lorraine',
'100100055': u'-- Midi-Pyrénées',
'100100060': u'-- Nord-Pas-de-Calais',
'100100073': u'-- Normandie',
'100100076': u'-- Pays-de-Loire',
'100100079': u'-- Picardie',
'100100082': u'-- Poitou-Charentes',
'100100085': u'-- Provence Alpes Cote d\'azur',
'100100090': u'-- Rhône Alpes',
'100200000': u'-- DOM et TOM',
'100200001': u'-- Guadeloupe',
'100200002': u'-- Guyane',
'100200003': u'-- Martinique',
'100200004': u'-- Réunion',
'100200005': u'-- Saint-Pierre et Miquelon',
'200000000': u'-- Etranger',
}.iteritems())
])
poste_choices = OrderedDict([
(k, u'%s' % (v)) for k, v in sorted({
'0': u'-- Indifférent --',
'100000000': u'-- Service Technique',
'100005000': u'-- Administrateur base de données',
'100004000': u'-- Admin. Système/Réseaux',
'100004004': u'-- Administrateur système',
'100004002': u'-- Administrateur réseaux',
'100007000': u'-- Analyste',
'100002000': u'-- Chef de projet',
'100002001': u'-- Chef de projet junior',
'100002002': u'-- Chef de projet senior',
'100021000': u'-- Consultant',
'100003000': u'-- Développeur',
'100003001': u'-- Développeur junior',
'100003002': u'-- Développeur senior',
'100009000': u'-- Directeur technique',
'100006000': u'-- Ingénieur d\'étude',
'100011000': u'-- Ingénieur support',
'100012000': u'-- Responsable R & D',
'100010000': u'-- Technicien',
'100010002': u'-- Technicien hotline',
'100010003': u'-- Technicien maintenance',
'100020000': u'-- Webmaster',
'200000000': u'-- Service Commercial',
'200300000': u'-- Commercial',
'200200000': u'-- Directeur commercial',
'200100000': u'-- Technico commercial',
'400000000': u'-- Service Marketing',
'400100000': u'-- Responsable Marketing',
'300000000': u'-- Service qualité',
'300100000': u'-- Assistant qualité',
'300200000': u'-- Responsable qualité',
'2000000': u'-- Fondateur',
'7000000': u'-- Formateur',
'6000000': u'-- Journaliste',
'500100000': u'-- Assistant(e) de direction',
'4000000': u'-- Stagiaire',
'5000000': u'-- Traducteur',
}.iteritems())
])
'''
'000000' in order to display description in console question
the rule is : len(key) > 5 or ' ' in key:
'''
contrat_choices = OrderedDict([(k, u'%s' % (v))
for k, v in sorted({
'000000': u'-- Indifférent --',
'6': u'Alternance',
'5': u'Apprentissage',
'2': u'CDD',
'1': u'CDI',
'4': u'Freelance',
'3': u'Stage',
}.iteritems())])
limit_date_choices = OrderedDict([(k, u'%s' % (v))
for k, v in sorted({
'2592000': u'30 jours',
'5184000': u'60 jours',
'7776000': u'90 jours',
'0': u'Illimitée',
}.iteritems())])
CONFIG = BackendConfig(
Value('region', label=u'Région', choices=region_choices),
Value('poste', label=u'Poste', choices=poste_choices),
Value('contrat', label=u'Contrat', choices=contrat_choices),
Value('limit_date', label=u'Date limite', choices=limit_date_choices))
def search_job(self, pattern=None):
with self.browser:
for job_advert in self.browser.advanced_search_job(
pattern=pattern):
yield job_advert
def advanced_search_job(self):
for advert in self.browser.advanced_search_job(
region=self.config['region'].get(),
poste=self.config['poste'].get(),
contrat=int(self.config['contrat'].get()),
limit_date=self.config['limit_date'].get()):
yield advert
def get_job_advert(self, _id, advert=None):
with self.browser:
return self.browser.get_job_advert(_id, advert)
def fill_obj(self, advert, fields):
self.get_job_advert(advert.id, advert)
OBJECTS = {LolixJobAdvert: fill_obj}
class FeedlyModule(Module, CapMessages, CapCollection):
NAME = 'feedly'
DESCRIPTION = u'handle the popular RSS reading service Feedly'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.2'
STORAGE = {'seen': []}
CONFIG = BackendConfig(
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''))
BROWSER = FeedlyBrowser
def iter_resources(self, objs, split_path):
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
return self.browser.get_categories()
if collection.path_level == 1:
return self.browser.get_feeds(split_path[0])
if collection.path_level == 2:
url = self.browser.get_feed_url(split_path[0], split_path[1])
threads = []
for article in self.browser.get_unread_feed(url):
thread = self.get_thread(article.id, article)
threads.append(thread)
return threads
def validate_collection(self, objs, collection):
if collection.path_level in [0, 1, 2]:
return
def get_thread(self, id, entry=None):
if isinstance(id, Thread):
thread = id
id = thread.id
else:
thread = Thread(id)
if entry is None:
url = id.split('#')[0]
for article in self.browser.get_unread_feed(url):
if article.id == id:
entry = article
if entry is None:
return None
if thread.id not in self.storage.get('seen', default=[]):
entry.flags = Message.IS_UNREAD
entry.thread = thread
thread.title = entry.title
thread.root = entry
return thread
def iter_unread_messages(self):
for thread in self.iter_threads():
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
def iter_threads(self):
for article in self.browser.iter_threads():
yield self.get_thread(article.id, article)
def set_message_read(self, message):
self.browser.set_message_read(message.thread.id.split('#')[-1])
self.storage.get('seen', default=[]).append(message.thread.id)
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
login_browser = GoogleBrowser(
username, password,
'https://feedly.com/v3/auth/callback&scope=profile+email&state=A8duE2XpzvtgcHt-q29qyBBK2fkpTefgqfzy7SY4GWUOPl3BgrSt4DRS-qKm9MRi_mXJRem8QW7RmNjpc_BIlkWc0JJvpay3UyzIErNvtaZLcsrUy94Ays3gTyispb8R0doguiky8gGxuCFNvJ9iXIB_SlwNhWABm7ut3nIgoMg3wodRgYOPFothhkErchrv076tBwXQA4Z8OIRyrQ'
)
else:
password = None
login_browser = None
return self.create_browser(username, password, login_browser)
OBJECTS = {Thread: fill_thread}
class BanquePopulaireModule(Module, CapBank, CapContact):
NAME = 'banquepopulaire'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.3'
DESCRIPTION = u'Banque Populaire'
LICENSE = 'AGPLv3+'
website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted(
{
'www.ibps.alpes.banquepopulaire.fr': u'Alpes',
'www.ibps.alsace.banquepopulaire.fr': u'Alsace',
'www.ibps.bpaca.banquepopulaire.fr':
u'Aquitaine Centre atlantique',
'www.ibps.atlantique.banquepopulaire.fr': u'Atlantique',
'www.ibps.banquedesavoie.banquepopulaire.fr': u'Banque de Savoie',
'www.ibps.bpbfc.banquepopulaire.fr': u'Bourgogne-Franche Comté',
'www.ibps.bretagnenormandie.cmm.groupe.banquepopulaire.fr':
u'Crédit Maritime Bretagne Normandie',
'www.ibps.atlantique.creditmaritime.groupe.banquepopulaire.fr':
u'Crédit Maritime Atlantique',
'www.ibps.sudouest.creditmaritime.groupe.banquepopulaire.fr':
u'Crédit Maritime du Littoral du Sud-Ouest',
'www.ibps.cotedazur.banquepopulaire.fr': u'Côte d\'azur',
'www.ibps.loirelyonnais.banquepopulaire.fr': u'Loire et Lyonnais',
'www.ibps.lorrainechampagne.banquepopulaire.fr':
u'Lorraine Champagne',
'www.ibps.massifcentral.banquepopulaire.fr': u'Massif central',
'www.ibps.nord.banquepopulaire.fr': u'Nord',
'www.ibps.occitane.banquepopulaire.fr': u'Occitane',
'www.ibps.ouest.banquepopulaire.fr': u'Ouest',
'www.ibps.provencecorse.banquepopulaire.fr': u'Provence et Corse',
'www.ibps.rivesparis.banquepopulaire.fr': u'Rives de Paris',
'www.ibps.sud.banquepopulaire.fr': u'Sud',
'www.ibps.valdefrance.banquepopulaire.fr': u'Val de France',
}.iteritems(),
key=lambda k_v: (k_v[1], k_v[0]))])
CONFIG = BackendConfig(
Value('website', label=u'Région', choices=website_choices),
ValueBackendPassword('login', label='Identifiant', masked=False),
ValueBackendPassword('password', label='Mot de passee'))
BROWSER = BanquePopulaire
def create_default_browser(self):
repls = ('alsace', 'bpalc'), ('lorrainechampagne', 'bpalc')
website = reduce(lambda a, kv: a.replace(*kv), repls,
self.config['website'].get())
return self.create_browser(website, self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
return self.browser.get_accounts_list()
def get_account(self, _id):
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
return self.browser.get_history(account)
def iter_coming(self, account):
return self.browser.get_history(account, coming=True)
def iter_investment(self, account):
return self.browser.get_investment(account)
def iter_contacts(self):
return self.browser.get_advisor()
class OkCModule(Module, CapMessages, CapContact, CapMessagesPost, CapDating):
NAME = 'okc'
MAINTAINER = u'Roger Philibert'
EMAIL = '*****@*****.**'
VERSION = '2.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = u'OkCupid'
CONFIG = BackendConfig(
Value('username', label='Username'),
ValueBackendPassword('password', label='Password'),
ValueBool('facebook',
label='Do you login with Facebook?',
default=False))
STORAGE = {
'profiles_walker': {
'viewed': []
},
's***s': {},
}
BROWSER = OkCBrowser
def create_default_browser(self):
if int(self.config['facebook'].get()):
facebook = self.create_browser(klass=FacebookBrowser)
facebook.login(self.config['username'].get(),
self.config['password'].get())
else:
facebook = None
return self.create_browser(self.config['username'].get(),
self.config['password'].get(), facebook)
# ---- CapDating methods ---------------------
def init_optimizations(self):
self.add_optimization(
'PROFILE_WALKER',
ProfilesWalker(self.weboob.scheduler, self.storage, self.browser))
# ---- CapMessages methods ---------------------
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def iter_threads(self):
threads = self.browser.get_threads_list()
for thread in threads:
t = Thread(thread['user']['userid'])
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % thread['user']['userinfo'][
'displayname']
t.date = datetime.fromtimestamp(thread['time'])
yield t
def get_thread(self, thread):
if not isinstance(thread, Thread):
thread = Thread(thread)
thread.flags = Thread.IS_DISCUSSION
messages = self.browser.get_thread_messages(thread.id)
contact = self.storage.get('s***s',
thread.id,
default={'lastmsg': datetime(1970, 1, 1)})
thread.title = u'Discussion with %s' % messages['fields']['username']
me = OkcContact(self.browser.get_profile(self.browser.me['userid']))
other = OkcContact(self.browser.get_profile(thread.id))
parent = None
for message in messages['messages']:
date = datetime.fromtimestamp(message['timestamp'])
flags = 0
if contact['lastmsg'] < date:
flags = Message.IS_UNREAD
if message['from'] == thread.id:
sender = other
receiver = me
else:
receiver = other
sender = me
if message.get('read', False):
flags |= Message.IS_RECEIVED
# Apply that flag on all previous messages as the 'read'
# attribute is only set on the last read message.
pmsg = parent
while pmsg:
if pmsg.flags & Message.IS_NOT_RECEIVED:
pmsg.flags |= Message.IS_RECEIVED
pmsg.flags &= ~Message.IS_NOT_RECEIVED
pmsg = pmsg.parent
else:
flags |= Message.IS_NOT_RECEIVED
msg = Message(thread=thread,
id=message['id'],
title=thread.title,
sender=sender.name,
receivers=[receiver.name],
date=date,
content=to_unicode(HTMLParser().unescape(
message['body'])),
children=[],
parent=parent,
signature=sender.get_text(),
flags=flags)
if parent:
parent.children = [msg]
else:
thread.root = msg
parent = msg
return thread
def iter_unread_messages(self):
for thread in self.iter_threads():
contact = self.storage.get(
's***s', thread.id, default={'lastmsg': datetime(1970, 1, 1)})
if thread.date <= contact['lastmsg']:
continue
thread = self.get_thread(thread)
for message in thread.iter_all_messages():
if message.flags & message.IS_UNREAD:
yield message
def set_message_read(self, message):
contact = self.storage.get('s***s',
message.thread.id,
default={'lastmsg': datetime(1970, 1, 1)})
if contact['lastmsg'] < message.date:
contact['lastmsg'] = message.date
self.storage.set('s***s', message.thread.id, contact)
self.storage.save()
# ---- CapMessagesPost methods ---------------------
def post_message(self, message):
self.browser.post_message(message.thread.id, message.content)
# ---- CapContact methods ---------------------
def fill_contact(self, contact, fields):
if 'profile' in fields:
contact = self.get_contact(contact)
if contact and 'photos' in fields:
for name, photo in contact.photos.items():
if photo.url and not photo.data:
data = self.browser.open(photo.url).content
contact.set_photo(name, data=data)
if photo.thumbnail_url and not photo.thumbnail_data:
data = self.browser.open(photo.thumbnail_url).content
contact.set_photo(name, thumbnail_data=data)
def fill_photo(self, photo, fields):
if 'data' in fields and photo.url and not photo.data:
photo.data = self.browser.open(photo.url).content
if 'thumbnail_data' in fields and photo.thumbnail_url and not photo.thumbnail_data:
photo.thumbnail_data = self.browser.open(
photo.thumbnail_url).content
return photo
def get_contact(self, user_id):
if isinstance(user_id, Contact):
user_id = user_id.id
info = self.browser.get_profile(user_id)
return OkcContact(info)
def iter_contacts(self, status=Contact.STATUS_ALL, ids=None):
threads = self.browser.get_threads_list()
for thread in threads:
c = self.get_contact(thread['user']['username'])
if c and (c.status & status) and (not ids or c.id in ids):
yield c
OBJECTS = {
Thread: fill_thread,
Contact: fill_contact,
ContactPhoto: fill_photo
}
class EHentaiBackend(BaseBackend, ICapGallery, ICapCollection):
NAME = 'ehentai'
MAINTAINER = u'Roger Philibert'
EMAIL = '*****@*****.**'
VERSION = '0.h'
DESCRIPTION = 'E-Hentai galleries'
LICENSE = 'AGPLv3+'
BROWSER = EHentaiBrowser
CONFIG = BackendConfig(
Value('domain', label='Domain', default='g.e-hentai.org'),
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password'))
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
else:
password = None
return self.create_browser(self.config['domain'].get(), username,
password)
def search_gallery(self, pattern, sortby=None):
with self.browser:
return self.browser.search_gallery(pattern)
def iter_gallery_images(self, gallery):
self.fillobj(gallery, ('url', ))
with self.browser:
return self.browser.iter_gallery_images(gallery)
ID_REGEXP = r'/?\d+/[\dabcdef]+/?'
URL_REGEXP = r'.+/g/(%s)' % ID_REGEXP
def get_gallery(self, _id):
match = re.match(r'^%s$' % self.URL_REGEXP, _id)
if match:
_id = match.group(1)
else:
match = re.match(r'^%s$' % self.ID_REGEXP, _id)
if match:
_id = match.group(0)
else:
return None
gallery = EHentaiGallery(_id)
with self.browser:
if self.browser.gallery_exists(gallery):
return gallery
else:
return None
def fill_gallery(self, gallery, fields):
if not gallery.__iscomplete__():
with self.browser:
self.browser.fill_gallery(gallery, fields)
def fill_image(self, image, fields):
with self.browser:
image.url = self.browser.get_image_url(image)
if 'data' in fields:
ratelimit("ehentai_get", 2)
image.data = self.browser.readurl(image.url)
def iter_resources(self, objs, split_path):
if BaseGallery in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest_nsfw'])
if collection.split_path == [u'latest_nsfw']:
for gallery in self.browser.latest_gallery():
yield gallery
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseGallery in objs and collection.split_path == [u'latest_nsfw']:
collection.title = u'Latest E-Hentai galleries (NSFW)'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {EHentaiGallery: fill_gallery, EHentaiImage: fill_image}
class CragrModule(Module, CapBankTransferAddRecipient, CapContact, CapProfile):
NAME = 'cragr'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.4'
DESCRIPTION = u'Crédit Agricole'
LICENSE = 'AGPLv3+'
website_choices = OrderedDict([
(k, u'%s (%s)' % (v, k))
for k, v in sorted({
'm.ca-alpesprovence.fr': u'Alpes Provence',
'm.ca-alsace-vosges.fr': u'Alsace-Vosges',
'm.ca-anjou-maine.fr': u'Anjou Maine',
'm.ca-aquitaine.fr': u'Aquitaine',
'm.ca-atlantique-vendee.fr': u'Atlantique Vendée',
'm.ca-briepicardie.fr': u'Brie Picardie',
'm.ca-cb.fr': u'Champagne Bourgogne',
'm.ca-centrefrance.fr': u'Centre France',
'm.ca-centreloire.fr': u'Centre Loire',
'm.ca-centreouest.fr': u'Centre Ouest',
'm.ca-centrest.fr': u'Centre Est',
'm.ca-charente-perigord.fr': u'Charente Périgord',
'm.ca-cmds.fr': u'Charente-Maritime Deux-Sèvres',
'm.ca-corse.fr': u'Corse',
'm.ca-cotesdarmor.fr': u'Côtes d\'Armor',
'm.ca-des-savoie.fr': u'Des Savoie',
'm.ca-finistere.fr': u'Finistere',
'm.ca-franchecomte.fr': u'Franche-Comté',
'm.ca-guadeloupe.fr': u'Guadeloupe',
'm.ca-illeetvilaine.fr': u'Ille-et-Vilaine',
'm.ca-languedoc.fr': u'Languedoc',
'm.ca-loirehauteloire.fr': u'Loire Haute Loire',
'm.ca-lorraine.fr': u'Lorraine',
'm.ca-martinique.fr': u'Martinique Guyane',
'm.ca-morbihan.fr': u'Morbihan',
'm.ca-nmp.fr': u'Nord Midi-Pyrénées',
'm.ca-nord-est.fr': u'Nord Est',
'm.ca-norddefrance.fr': u'Nord de France',
'm.ca-normandie-seine.fr': u'Normandie Seine',
'm.ca-normandie.fr': u'Normandie',
'm.ca-paris.fr': u'Ile-de-France',
'm.ca-pca.fr': u'Provence Côte d\'Azur',
'm.ca-reunion.fr': u'Réunion',
'm.ca-sudmed.fr': u'Sud Méditerranée',
'm.ca-sudrhonealpes.fr': u'Sud Rhône Alpes',
'm.ca-toulouse31.fr':
u'Toulouse 31', # m.ca-toulousain.fr redirects here
'm.ca-tourainepoitou.fr': u'Tourraine Poitou',
'm.ca-valdefrance.fr': u'Val de France',
'm.lefil.com': u'Pyrénées Gascogne',
}.items())
])
CONFIG = BackendConfig(
Value('website', label=u'Région', choices=website_choices),
ValueBackendPassword('login', label=u'N° de compte', masked=False),
ValueBackendPassword('password',
label=u'Code personnel',
regexp=r'\d{6}'))
BROWSER = Cragr
COMPAT_DOMAINS = {
'm.lefil.com': 'm.ca-pyrenees-gascogne.fr',
}
def create_default_browser(self):
site_conf = self.config['website'].get()
site_conf = self.COMPAT_DOMAINS.get(site_conf, site_conf)
return self.create_browser(site_conf, self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
return self.browser.get_accounts_list()
def get_account(self, _id):
return find_object(self.iter_accounts(), id=_id, error=AccountNotFound)
def _history_filter(self, account, coming):
today = date.today()
def to_date(obj):
if hasattr(obj, 'date'):
return obj.date()
return obj
for tr in self.browser.get_history(account):
tr_coming = to_date(tr.date) > today
if coming == tr_coming:
yield tr
def iter_history(self, account):
if account.type == Account.TYPE_CARD:
return self._history_filter(account, False)
return self.browser.get_history(account)
def iter_coming(self, account):
if account.type == Account.TYPE_CARD:
return self._history_filter(account, True)
return []
def iter_investment(self, account):
for inv in self.browser.iter_investment(account):
yield inv
def iter_contacts(self):
return self.browser.iter_advisor()
def get_profile(self):
if not hasattr(self.browser, 'get_profile'):
raise NotImplementedError()
return self.browser.get_profile()
def iter_transfer_recipients(self, account):
if not isinstance(account, Account):
account = self.get_account(account)
return self.browser.iter_transfer_recipients(account)
def init_transfer(self, transfer, **params):
def to_ascii(s):
return s.encode('ascii', errors='ignore').decode('ascii')
if transfer.label:
transfer.label = re.sub(r'[+!]', '', to_ascii(transfer.label[:33]))
return self.browser.init_transfer(transfer, **params)
def execute_transfer(self, transfer, **params):
return self.browser.execute_transfer(transfer, **params)
def new_recipient(self, recipient, **params):
return self.browser.new_recipient(recipient, **params)
class FourChanModule(Module, CapMessages):
NAME = 'fourchan'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.3'
LICENSE = 'AGPLv3+'
DESCRIPTION = '4chan image board'
CONFIG = BackendConfig(Value('boards', label='Boards to fetch'))
STORAGE = {'boards': {}}
BROWSER = FourChan
def _splitid(self, id):
return id.split('.', 1)
def get_thread(self, id):
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if '.' not in id:
self.logger.warning('Malformated ID (%s)' % id)
return
board, thread_id = self._splitid(id)
with self.browser:
_thread = self.browser.get_thread(board, thread_id)
flags = 0
if _thread.id not in self.storage.get('boards', board, default={}):
flags |= Message.IS_UNREAD
if not thread:
thread = Thread(id)
thread.title = _thread.filename
thread.root = Message(
thread=thread,
id=0, # root message
title=_thread.filename,
sender=_thread.author,
receivers=None,
date=_thread.datetime,
parent=None,
content=_thread.text,
signature=None,
children=[],
flags=flags | Message.IS_HTML)
for comment in _thread.comments:
flags = 0
if comment.id not in self.storage.get('boards',
board,
_thread.id,
default=[]):
flags |= Message.IS_UNREAD
m = Message(thread=thread,
id=comment.id,
title=_thread.filename,
sender=comment.author,
receivers=None,
date=comment.datetime,
parent=thread.root,
content=comment.text,
signature=None,
children=None,
flags=flags | Message.IS_HTML)
thread.root.children.append(m)
return thread
def iter_threads(self):
for board in self.config['boards'].get().split(' '):
with self.browser:
threads = self.browser.get_threads(board)
for thread in threads:
t = Thread('%s.%s' % (board, thread.id))
t.title = thread.filename
yield t
def iter_unread_messages(self):
for thread in self.iter_threads():
self.fill_thread(thread, 'root')
for m in thread.iter_all_messages():
if m.flags & Message.IS_UNREAD:
yield m
def set_message_read(self, message):
board, thread_id = self._splitid(message.thread.id)
self.storage.set(
'boards', board, thread_id,
self.storage.get('boards', board, thread_id, default=[]) +
[message.id])
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread}
class DailymotionModule(Module, CapVideo, CapCollection):
NAME = 'dailymotion'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.2'
DESCRIPTION = 'Dailymotion video streaming website'
LICENSE = 'AGPLv3+'
BROWSER = DailymotionBrowser
resolution_choice = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({
u'480': u'480p',
u'240': u'240p',
u'380': u'380p',
u'720': u'720p',
u'1080': u'1080p'
}.iteritems())])
format_choice = [u'm3u8', u'mp4']
CONFIG = BackendConfig(Value('resolution', label=u'Resolution', choices=resolution_choice),
Value('format', label=u'Format', choices=format_choice))
SORTBY = ['relevance', 'rated', 'visited', None]
def create_default_browser(self):
resolution = self.config['resolution'].get()
format = self.config['format'].get()
return self.create_browser(resolution=resolution, format=format)
def get_video(self, _id):
m = re.match('http://[w\.]*dailymotion\.com/video/(.*)', _id)
if m:
_id = m.group(1)
if not _id.startswith('http'):
return self.browser.get_video(_id)
def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False):
return self.browser.search_videos(pattern, self.SORTBY[sortby])
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
video = self.browser.get_video(video.id, video)
if 'thumbnail' in fields and video.thumbnail:
video.thumbnail.data = self.browser.open(video.thumbnail.url).content
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest'])
if collection.split_path == [u'latest']:
for video in self.browser.latest_videos():
yield video
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and collection.split_path == [u'latest']:
collection.title = u'Latest Dailymotion videos'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {BaseVideo: fill_video}
