class MediawikiModule(Module, CapContent, CapImage):
NAME = 'mediawiki'
MAINTAINER = u'Clément Schreiner'
EMAIL = '*****@*****.**'
VERSION = '1.2'
LICENSE = 'AGPLv3+'
DESCRIPTION = 'Wikis running MediaWiki, like Wikipedia'
CONFIG = BackendConfig(Value('url', label='URL of the Mediawiki website', default='http://en.wikipedia.org/', regexp='https?://.*'),
Value('apiurl', label='URL of the Mediawiki website\'s API', default='http://en.wikipedia.org/w/api.php', regexp='https?://.*'),
Value('username', label='Login', default=''),
ValueBackendPassword('password', label='Password', default=''))
BROWSER = MediawikiBrowser
def create_default_browser(self):
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
else:
password = ''
return self.create_browser(self.config['url'].get(),
self.config['apiurl'].get(),
username, password)
def get_content(self, _id, revision=None):
_id = _id.replace(' ', '_')
content = Content(_id)
page = _id
rev = revision.id if revision else None
data = self.browser.get_wiki_source(page, rev)
content.content = data
return content
def iter_revisions(self, _id):
return self.browser.iter_wiki_revisions(_id)
def push_content(self, content, message=None, minor=False):
self.browser.set_wiki_source(content, message, minor)
def get_content_preview(self, content):
return self.browser.get_wiki_preview(content)
def _make_image(self, info):
img = WikiImage(info['title'])
if 'thumbnail' in info:
thumb = Thumbnail(info['thumbnail'])
img.thumbnail = thumb
if 'original' in info:
img.url = info['original']
img._canonical_url = info['canonicalurl']
img.size = info['size']
return img
def search_file(self, pattern, sortby=CapFile.SEARCH_RELEVANCE):
for info in self.browser.search_file(pattern):
yield self._make_image(info)
def get_image(self, _id):
_id = _id.replace(' ', '_')
info = self.browser.get_image(_id)
return self._make_image(info)
def fill_img(self, obj, fields):
if set(fields) & set(('url', 'thumbnail')):
new = self.get_image(obj.id)
if not obj.url:
obj.url = new.url
if not obj.thumbnail:
obj.thumbnail = new.thumbnail
if 'data' in fields:
obj.data = self.browser.open(obj.url).content
return obj
OBJECTS = {BaseImage: fill_img, Thumbnail: fill_img}
class FourChanBackend(BaseBackend, ICapMessages):
NAME = 'fourchan'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '0.h'
LICENSE = 'AGPLv3+'
DESCRIPTION = '4chan image board'
CONFIG = BackendConfig(Value('boards', label='Boards to fetch'))
STORAGE = {'boards': {}}
BROWSER = FourChan
def _splitid(self, id):
return id.split('.', 1)
def get_thread(self, id):
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not '.' in id:
self.logger.warning('Malformated ID (%s)' % id)
return
board, thread_id = self._splitid(id)
with self.browser:
_thread = self.browser.get_thread(board, thread_id)
flags = 0
if not _thread.id in self.storage.get('boards', board, default={}):
flags |= Message.IS_UNREAD
if not thread:
thread = Thread(id)
thread.title = _thread.filename
thread.root = Message(
thread=thread,
id=0, # root message
title=_thread.filename,
sender=_thread.author,
receivers=None,
date=_thread.datetime,
parent=None,
content=_thread.text,
signature=None,
children=[],
flags=flags | Message.IS_HTML)
for comment in _thread.comments:
flags = 0
if not comment.id in self.storage.get(
'boards', board, _thread.id, default=[]):
flags |= Message.IS_UNREAD
m = Message(thread=thread,
id=comment.id,
title=_thread.filename,
sender=comment.author,
receivers=None,
date=comment.datetime,
parent=thread.root,
content=comment.text,
signature=None,
children=None,
flags=flags | Message.IS_HTML)
thread.root.children.append(m)
return thread
def iter_threads(self):
for board in self.config['boards'].get().split(' '):
with self.browser:
threads = self.browser.get_threads(board)
for thread in threads:
t = Thread('%s.%s' % (board, thread.id))
t.title = thread.filename
yield t
def iter_unread_messages(self):
for thread in self.iter_threads():
self.fill_thread(thread, 'root')
for m in thread.iter_all_messages():
if m.flags & Message.IS_UNREAD:
yield m
def set_message_read(self, message):
board, thread_id = self._splitid(message.thread.id)
self.storage.set(
'boards', board, thread_id,
self.storage.get('boards', board, thread_id, default=[]) +
[message.id])
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread}
class DailymotionModule(Module, CapVideo, CapCollection):
NAME = 'dailymotion'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.6'
DESCRIPTION = 'Dailymotion video streaming website'
LICENSE = 'AGPLv3+'
BROWSER = DailymotionBrowser
resolution_choice = OrderedDict([(k, u'%s (%s)' % (v, k))
for k, v in sorted({
u'480': u'480p',
u'240': u'240p',
u'380': u'380p',
u'720': u'720p',
u'1080': u'1080p'
}.iteritems())])
format_choice = [u'm3u8', u'mp4']
CONFIG = BackendConfig(
Value('resolution', label=u'Resolution', choices=resolution_choice),
Value('format', label=u'Format', choices=format_choice))
SORTBY = ['relevance', 'rated', 'visited', None]
def create_default_browser(self):
resolution = self.config['resolution'].get()
format = self.config['format'].get()
return self.create_browser(resolution=resolution, format=format)
def get_video(self, _id):
m = re.match('http://[w\.]*dailymotion\.com/video/(.*)', _id)
if m:
_id = m.group(1)
if not _id.startswith('http'):
return self.browser.get_video(_id)
def search_videos(self,
pattern,
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False):
return self.browser.search_videos(pattern, self.SORTBY[sortby])
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
video = self.browser.get_video(video.id, video)
if 'thumbnail' in fields and video.thumbnail:
video.thumbnail.data = self.browser.open(
video.thumbnail.url).content
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest'])
if collection.split_path == [u'latest']:
for video in self.browser.latest_videos():
yield video
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and collection.split_path == [u'latest']:
collection.title = u'Latest Dailymotion videos'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {BaseVideo: fill_video}
class CragrModule(Module, CapBank):
NAME = 'cragr'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.1'
DESCRIPTION = u'Crédit Agricole'
LICENSE = 'AGPLv3+'
website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({
'm.ca-alpesprovence.fr': u'Alpes Provence',
'm.ca-alsace-vosges.fr': u'Alsace-Vosges',
'm.ca-anjou-maine.fr': u'Anjou Maine',
'm.ca-aquitaine.fr': u'Aquitaine',
'm.ca-atlantique-vendee.fr': u'Atlantique Vendée',
'm.ca-briepicardie.fr': u'Brie Picardie',
'm.ca-cb.fr': u'Champagne Bourgogne',
'm.ca-centrefrance.fr': u'Centre France',
'm.ca-centreloire.fr': u'Centre Loire',
'm.ca-centreouest.fr': u'Centre Ouest',
'm.ca-centrest.fr': u'Centre Est',
'm.ca-charente-perigord.fr': u'Charente Périgord',
'm.ca-cmds.fr': u'Charente-Maritime Deux-Sèvres',
'm.ca-corse.fr': u'Corse',
'm.ca-cotesdarmor.fr': u'Côtes d\'Armor',
'm.ca-des-savoie.fr': u'Des Savoie',
'm.ca-finistere.fr': u'Finistere',
'm.ca-franchecomte.fr': u'Franche-Comté',
'm.ca-guadeloupe.fr': u'Guadeloupe',
'm.ca-illeetvilaine.fr': u'Ille-et-Vilaine',
'm.ca-languedoc.fr': u'Languedoc',
'm.ca-loirehauteloire.fr': u'Loire Haute Loire',
'm.ca-lorraine.fr': u'Lorraine',
'm.ca-martinique.fr': u'Martinique Guyane',
'm.ca-morbihan.fr': u'Morbihan',
'm.ca-nmp.fr': u'Nord Midi-Pyrénées',
'm.ca-nord-est.fr': u'Nord Est',
'm.ca-norddefrance.fr': u'Nord de France',
'm.ca-normandie-seine.fr': u'Normandie Seine',
'm.ca-normandie.fr': u'Normandie',
'm.ca-paris.fr': u'Ile-de-France',
'm.ca-pca.fr': u'Provence Côte d\'Azur',
'm.ca-reunion.fr': u'Réunion',
'm.ca-sudmed.fr': u'Sud Méditerranée',
'm.ca-sudrhonealpes.fr': u'Sud Rhône Alpes',
'm.ca-toulouse31.fr': u'Toulouse 31', # m.ca-toulousain.fr redirects here
'm.ca-tourainepoitou.fr': u'Tourraine Poitou',
'm.ca-valdefrance.fr': u'Val de France',
'm.lefil.com': u'Pyrénées Gascogne',
}.iteritems())])
CONFIG = BackendConfig(Value('website', label=u'Region', choices=website_choices),
ValueBackendPassword('login', label=u'N° de compte', masked=False),
ValueBackendPassword('password', label=u'Code personnel', regexp=r'\d{6}'))
BROWSER = Cragr
def create_default_browser(self):
try:
return self.create_browser(self.config['website'].get(),
self.config['login'].get(),
self.config['password'].get())
except Cragr.WebsiteNotSupported:
self.logger.debug('falling-back on mobile version')
self.BROWSER = CragrMobile
return self.create_browser(self.config['website'].get(),
self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
return self.browser.get_accounts_list()
def get_account(self, _id):
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
return self.browser.get_history(account)
def iter_transfer_recipients(self, ignored):
for account in self.browser.get_transfer_accounts().itervalues():
recipient = Recipient()
recipient.id = account.id
recipient.label = account.label
yield recipient
def transfer(self, account, to, amount, reason=None):
return self.browser.do_transfer(account, to, amount, reason)
class ArteBackend(BaseBackend, ICapVideo, ICapCollection):
NAME = 'arte'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '0.e'
DESCRIPTION = 'Arte French and German TV'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(
Value('lang',
label='Lang of videos',
choices={
'fr': 'French',
'de': 'Deutsch',
'en': 'English'
},
default='fr'),
Value('quality',
label='Quality of videos',
choices=['hd', 'sd'],
default='hd'))
BROWSER = ArteBrowser
def create_default_browser(self):
return self.create_browser(lang=self.config['lang'].get(),
quality=self.config['quality'].get())
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
def search_videos(self,
pattern,
sortby=ICapVideo.SEARCH_RELEVANCE,
nsfw=False,
max_results=None):
with self.browser:
return self.browser.search_videos(pattern)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(ArteVideo.id2url(video.id),
video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(
video.thumbnail.url)
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest'])
if collection.split_path == [u'latest']:
for video in self.browser.latest_videos():
yield video
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and collection.split_path == [u'latest']:
collection.title = u'Latest Arte videos'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {ArteVideo: fill_video}
class IndeedModule(Module, CapJob):
NAME = 'indeed'
DESCRIPTION = u'indeed website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.3'
BROWSER = IndeedBrowser
type_contrat_choices = OrderedDict([
(k, u'%s' % (v)) for k, v in sorted({
'all': u'Tous les emplois',
'fulltime': u'Temps plein',
'parttime': u'Temps partiel',
'contract': u'Durée indéterminée',
'internship': u'Stage / Apprentissage',
'temporary': u'Durée déterminée',
}.iteritems())
])
limit_date_choices = OrderedDict([
(k, u'%s' % (v))
for k, v in sorted({
'any': u'à tout moment',
'15': u'depuis 15 jours',
'7': u'depuis 7 jours',
'3': u'depuis 3 jours',
'1': u'depuis hier',
'last': u'depuis ma dernière visite',
}.iteritems())
])
radius_choices = OrderedDict([
(k, u'%s' % (v))
for k, v in sorted({
'0': u'uniquement à cet endroit',
'5': u'dans un rayon de 5 kilomètres',
'10': u'dans un rayon de 10 kilomètres',
'15': u'dans un rayon de 15 kilomètres',
'25': u'dans un rayon de 25 kilomètres',
'50': u'dans un rayon de 50 kilomètres',
'100': u'dans un rayon de 100 kilomètres',
}.iteritems())
])
CONFIG = BackendConfig(
Value('metier', label=u'Job name', masked=False, default=''),
Value('limit_date',
label=u'Date limite',
choices=limit_date_choices,
default=''),
Value('contrat',
label=u'Contract',
choices=type_contrat_choices,
default=''),
Value('place', label=u'Place', masked=False, default=''),
Value('radius', label=u'Radius', choices=radius_choices, default=''))
def search_job(self, pattern=None):
return self.browser.search_job(metier=pattern)
def advanced_search_job(self):
return self.browser.search_job(
metier=self.config['metier'].get(),
limit_date=self.config['limit_date'].get(),
contrat=self.config['contrat'].get(),
place=self.config['place'].get(),
radius=self.config['radius'].get())
def get_job_advert(self, _id, advert=None):
return self.browser.get_job_advert(_id, advert)
def fill_obj(self, advert, fields):
return self.get_job_advert(advert.id, advert)
OBJECTS = {BaseJobAdvert: fill_obj}
class MonsterModule(Module, CapJob):
NAME = 'monster'
DESCRIPTION = u'monster website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.4'
BROWSER = MonsterBrowser
type_contrat_choices = OrderedDict([
(k, u'%s' % (v))
for k, v in sorted({
'Interim-ou-CDD-ou-mission_8': u'Interim ou CDD ou mission',
'CDI_8': u'CDI',
'Stage-Apprentissage-Alternance_8':
u'Stage/Apprentissage/Alternance',
' ': u'Autres',
'Indépendant-Freelance-Saisonnier-Franchise_8':
u'Indépendant/Freelance/Saisonnier/Franchise',
'Journalier_8': u'Journalier',
'Temps-Partiel_8': u'Temps Partiel',
'Temps-Plein_8': u'Temps Plein',
}.items())
])
date_choices = OrderedDict([(k, u'%s' % (v))
for k, v in sorted({
'-1': u'N importe quelle date',
'000000': u'Aujourd hui',
'1': u'2 derniers jours',
'3': u'3 derniers jours',
'7': u'Les 7 derniers jours',
'14': u'Les 14 derniers jours',
'30': u'30 derniers jours',
}.items())])
CONFIG = BackendConfig(
Value('job_name', label='Job name', masked=False, default=''),
Value('place', label='Place', masked=False, default=''),
Value('contract',
label=u'Contract',
choices=type_contrat_choices,
default=''),
Value('limit_date', label=u'Date', choices=date_choices, default='-1'),
)
def search_job(self, pattern=None):
return self.browser.search_job(pattern)
def advanced_search_job(self):
return self.browser.advanced_search_job(
job_name=self.config['job_name'].get(),
place=self.config['place'].get(),
contract=self.config['contract'].get(),
limit_date=self.config['limit_date'].get())
def get_job_advert(self, _id, advert=None):
return self.browser.get_job_advert(_id, advert)
def fill_obj(self, advert, fields):
return self.get_job_advert(advert.id, advert)
OBJECTS = {BaseJobAdvert: fill_obj}
class jcvelauxModule(Module, CapGauge):
NAME = 'jcvelaux'
DESCRIPTION = ('City bike renting availability information.\nCities: %s' %
', '.join(CITIES))
MAINTAINER = 'Herve Werner'
EMAIL = '*****@*****.**'
VERSION = '1.6'
LICENSE = 'AGPLv3'
BROWSER = VelibBrowser
CONFIG = BackendConfig(
Value('city', label='City', default='Lyon',
choices=CITIES + ("ALL", )),
ValueBackendPassword('api_key',
label='Optional API key',
default='',
noprompt=True))
def __init__(self, *a, **kw):
super(jcvelauxModule, self).__init__(*a, **kw)
self.cities = None
def create_default_browser(self):
api_key = self.config['api_key'].get()
return self.create_browser(api_key)
def _make_gauge(self, info):
gauge = Gauge(info['id'])
gauge.name = info['name']
gauge.city = info['city']
gauge.object = 'bikes'
return gauge
def _make_sensor(self, sensor_type, info, gauge):
id = '%s.%s' % (sensor_type, gauge.id)
sensor = GaugeSensor(id)
sensor.gaugeid = gauge.id
sensor.name = SENSOR_TYPES[sensor_type]
sensor.address = '%s' % info['address']
sensor.longitude = info['longitude']
sensor.latitude = info['latitude']
sensor.history = []
return sensor
def _make_measure(self, sensor_type, info, gauge):
id = '%s.%s' % (sensor_type, gauge.id)
measure = BikeMeasure(id)
measure.date = info['last_update']
measure.level = float(info[sensor_type])
return measure
def _parse_gauge(self, info):
gauge = self._make_gauge(info)
gauge.sensors = []
for type in SENSOR_TYPES:
sensor = self._make_sensor(type, info, gauge)
measure = self._make_measure(type, info, gauge)
sensor.lastvalue = measure
gauge.sensors.append(sensor)
return gauge
def _contract(self):
contract = self.config.get('city').get()
if contract.lower() == 'all':
contract = None
return contract
def iter_gauges(self, pattern=None):
if pattern is None:
for jgauge in self.browser.get_station_list(
contract=self._contract()):
yield self._parse_gauge(jgauge)
else:
lowpattern = pattern.lower()
for jgauge in self.browser.get_station_list(
contract=self._contract()):
gauge = self._parse_gauge(jgauge)
if lowpattern in gauge.name.lower(
) or lowpattern in gauge.city.lower():
yield gauge
def iter_sensors(self, gauge, pattern=None):
if not isinstance(gauge, Gauge):
gauge = self._get_gauge_by_id(gauge)
if gauge is None:
raise SensorNotFound()
if pattern is None:
for sensor in gauge.sensors:
yield sensor
else:
lowpattern = pattern.lower()
for sensor in gauge.sensors:
if lowpattern in sensor.name.lower():
yield sensor
def get_last_measure(self, sensor):
if not isinstance(sensor, GaugeSensor):
sensor = self._get_sensor_by_id(sensor)
if sensor is None:
raise SensorNotFound()
return sensor.lastvalue
def _fetch_cities(self):
if self.cities:
return
self.cities = {}
jcontract = self.browser.get_contracts_list()
for jcontract in jcontract:
for city in jcontract['cities']:
self.cities[city.lower()] = jcontract['name']
def _get_gauge_by_id(self, id):
jgauge = self.browser.get_station_infos(id)
if jgauge:
return self._parse_gauge(jgauge)
else:
return None
def _get_sensor_by_id(self, id):
try:
sensor_name, gauge_name, contract = id.split('.')
except ValueError:
raise UserError('Expected format NAME.ID.CITY for sensor: %r' % id)
gauge = self._get_gauge_by_id('%s.%s' % (gauge_name, contract))
if not gauge:
raise SensorNotFound()
for sensor in gauge.sensors:
if sensor.id.lower() == id.lower():
return sensor
class YoutubeModule(Module, CapVideo, CapCollection):
NAME = 'youtube'
MAINTAINER = u'Laurent Bachelier'
EMAIL = '*****@*****.**'
VERSION = '1.4'
DESCRIPTION = 'YouTube video streaming website'
LICENSE = 'AGPLv3+'
BROWSER = None
CONFIG = BackendConfig(
Value('username', label='Email address', default=''),
ValueBackendPassword('password', label='Password', default=''))
URL_RE = re.compile(
r'^https?://(?:\w*\.?youtube(?:|-nocookie)\.com/(?:watch\?v=|embed/|v/)|youtu\.be\/|\w*\.?youtube\.com\/user\/\w+#p\/u\/\d+\/)([^\?&]+)'
)
def create_default_browser(self):
password = None
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
return self.create_browser(username, password)
def _entry2video(self, entry):
"""
Parse an entry returned by googleapi and return a Video object.
"""
snippet = entry['snippet']
id = entry['id']
if isinstance(id, dict):
id = id['videoId']
video = YoutubeVideo(to_unicode(id))
video.title = to_unicode(snippet['title'].strip())
# duration does not seem to be available with api
video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url'])
video.author = to_unicode(snippet['channelTitle'].strip())
return video
def _set_video_attrs(self, video):
new_video = video_info(YoutubeVideo.id2url(video.id))
if not new_video:
return
for k, v in new_video.iter_fields():
if not empty(v) and empty(getattr(video, k)):
setattr(video, k, v)
def get_video(self, _id):
m = self.URL_RE.match(_id)
if m:
_id = m.group(1)
params = {'id': _id, 'part': 'id,snippet'}
youtube = self._build_yt()
response = youtube.videos().list(**params).execute()
items = response.get('items', [])
if not items:
return None
video = self._entry2video(items[0])
self._set_video_attrs(video)
video.set_empty_fields(NotAvailable)
# Youtube video url is https, using ssl encryption
# so we need to use the "play_proxy" method using urllib2 proxy streaming to handle this
video._play_proxy = True
return video
def _build_yt(self):
DEVELOPER_KEY = "AIzaSyApVVeZ03XkKDYHX8T5uOn8Eizfe9CMDbs"
YOUTUBE_API_SERVICE_NAME = "youtube"
YOUTUBE_API_VERSION = "v3"
return ytbuild(YOUTUBE_API_SERVICE_NAME,
YOUTUBE_API_VERSION,
developerKey=DEVELOPER_KEY)
def search_videos(self,
pattern,
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False):
YOUTUBE_MAX_RESULTS = 50
youtube = self._build_yt()
params = {'part': 'id,snippet', 'maxResults': YOUTUBE_MAX_RESULTS}
if pattern is not None:
params['q'] = pattern
params['safeSearch'] = 'none' if nsfw else 'strict' # or 'moderate'
params['order'] = ('relevance', 'rating', 'viewCount', 'date')[sortby]
nb_yielded = 0
while True:
search_response = youtube.search().list(**params).execute()
items = search_response.get('items', [])
for entry in items:
if entry["id"]["kind"] != "youtube#video":
continue
yield self._entry2video(entry)
nb_yielded += 1
params['pageToken'] = search_response.get('nextPageToken')
if not params['pageToken']:
return
if nb_yielded < YOUTUBE_MAX_RESULTS:
return
def latest_videos(self):
return self.search_videos(None, CapVideo.SEARCH_DATE)
def fill_video(self, video, fields):
if 'thumbnail' in fields and video.thumbnail:
video.thumbnail.data = requests.get(video.thumbnail.url).content
if 'url' in fields:
self._set_video_attrs(video)
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest'])
if collection.split_path == [u'latest']:
for video in self.latest_videos():
yield video
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and collection.split_path == [u'latest']:
collection.title = u'Latest YouTube videos'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {YoutubeVideo: fill_video}
class LeclercMobileModule(Module, CapDocument):
NAME = 'leclercmobile'
MAINTAINER = u'Florent Fourcot'
EMAIL = '*****@*****.**'
VERSION = '1.4'
LICENSE = 'AGPLv3+'
DESCRIPTION = 'Leclerc Mobile website'
CONFIG = BackendConfig(ValueBackendPassword('login',
label='Account ID',
masked=False,
regexp='^(\d{10}|)$'),
ValueBackendPassword('password',
label='Password')
)
BROWSER = Leclercmobile
def create_default_browser(self):
return self.create_browser(self.config['login'].get(),
self.config['password'].get())
def iter_subscription(self):
for subscription in self.browser.get_subscription_list():
yield subscription
def get_subscription(self, _id):
if not _id.isdigit():
raise SubscriptionNotFound()
with self.browser:
subscription = self.browser.get_subscription(_id)
if subscription:
return subscription
else:
raise SubscriptionNotFound()
def iter_documents_history(self, subscription):
with self.browser:
for history in self.browser.get_history():
if history.label != "Votre solde":
yield history
def get_document(self, id):
with self.browser:
bill = self.browser.get_document(id)
if bill:
return bill
else:
raise DocumentNotFound()
def iter_documents(self, subscription):
if not isinstance(subscription, Subscription):
subscription = self.get_subscription(subscription)
with self.browser:
for bill in self.browser.iter_documents(subscription.id):
yield bill
# The subscription is actually useless, but maybe for the futur...
def get_details(self, subscription):
with self.browser:
for detail in self.browser.get_details():
yield detail
def download_document(self, bill):
if not isinstance(bill, Bill):
bill = self.get_document(bill)
with self.browser:
return self.browser.readurl(bill.url)
def get_balance(self, subscription):
if not isinstance(subscription, Subscription):
subscription = self.get_subscription(subscription)
with self.browser:
balance = self.browser.get_balance()
balance.label = u"Balance %s" % subscription.id
balance.id = "%s-balance" % subscription.id
balance.currency = u'EUR'
return balance
class jcvelauxModule(Module, CapGauge):
NAME = 'jcvelaux'
DESCRIPTION = (u'City bike renting availability information.\nCities: %s' %
u', '.join(CITIES))
MAINTAINER = u'Herve Werner'
EMAIL = '*****@*****.**'
VERSION = '1.1'
LICENSE = 'AGPLv3'
BROWSER = VelibBrowser
STORAGE = {'boards': {}}
CONFIG = BackendConfig(Value('city', label='City', default='Paris',
choices=CITIES + ("ALL",)))
def __init__(self, *a, **kw):
super(jcvelauxModule, self).__init__(*a, **kw)
self.cities = None
def _make_gauge(self, info):
gauge = Gauge(info['id'])
gauge.name = unicode(info['name'])
gauge.city = unicode(info['city'])
gauge.object = u'bikes'
return gauge
def _make_sensor(self, sensor_type, info, gauge):
id = '%s.%s' % (sensor_type, gauge.id)
sensor = BikeSensor(id)
sensor.gaugeid = gauge.id
sensor.name = SENSOR_TYPES[sensor_type]
sensor.address = unicode(info['address'])
sensor.longitude = info['longitude']
sensor.latitude = info['latitude']
sensor.history = []
return sensor
def _make_measure(self, sensor_type, info):
measure = BikeMeasure()
measure.date = info['last_update']
measure.level = float(info[sensor_type])
return measure
def _parse_gauge(self, info):
gauge = self._make_gauge(info)
gauge.sensors = []
for type in SENSOR_TYPES:
sensor = self._make_sensor(type, info, gauge)
measure = self._make_measure(type, info)
sensor.lastvalue = measure
gauge.sensors.append(sensor)
return gauge
def _contract(self):
contract = self.config.get('city').get()
if contract.lower() == 'all':
contract = None
return contract
def iter_gauges(self, pattern=None):
if pattern is None:
for jgauge in self.browser.get_station_list(contract=self._contract()):
yield self._parse_gauge(jgauge)
else:
lowpattern = pattern.lower()
for jgauge in self.browser.get_station_list(contract=self._contract()):
gauge = self._parse_gauge(jgauge)
if lowpattern in gauge.name.lower() or lowpattern in gauge.city.lower():
yield gauge
def iter_sensors(self, gauge, pattern=None):
if not isinstance(gauge, Gauge):
gauge = self._get_gauge_by_id(gauge)
if gauge is None:
raise SensorNotFound()
if pattern is None:
yield from gauge.sensors
else:
lowpattern = pattern.lower()
for sensor in gauge.sensors:
if lowpattern in sensor.name.lower():
yield sensor
def get_last_measure(self, sensor):
if not isinstance(sensor, GaugeSensor):
sensor = self._get_sensor_by_id(sensor)
if sensor is None:
raise SensorNotFound()
return sensor.lastvalue
def _fetch_cities(self):
if self.cities:
return
self.cities = {}
jcontract = self.browser.get_contracts_list()
for jcontract in jcontract:
for city in jcontract['cities']:
self.cities[city.lower()] = jcontract['name']
def _get_gauge_by_id(self, id):
jgauge = self.browser.get_station_infos(id)
if jgauge:
return self._parse_gauge(jgauge)
else:
return None
def _get_sensor_by_id(self, id):
_, gauge_id = id.split('.', 1)
gauge = self._get_gauge_by_id(gauge_id)
if not gauge:
raise SensorNotFound()
for sensor in gauge.sensors:
if sensor.id.lower() == id.lower():
return sensor
class OkCBackend(BaseBackend, ICapMessages, ICapContact, ICapMessagesPost):
NAME = 'okc'
MAINTAINER = u'Roger Philibert'
EMAIL = '*****@*****.**'
VERSION = '0.e'
LICENSE = 'AGPLv3+'
DESCRIPTION = u'OkCupid dating website'
CONFIG = BackendConfig(Value('username', label='Username'),
ValueBackendPassword('password', label='Password'))
STORAGE = {
's***s': {},
#'notes': {},
}
BROWSER = OkCBrowser
def create_default_browser(self):
return self.create_browser(self.config['username'].get(),
self.config['password'].get())
# ---- ICapMessages methods ---------------------
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def iter_threads(self):
with self.browser:
threads = self.browser.get_threads_list()
for thread in threads:
# Remove messages from user that quit
#if thread['member'].get('isBan', thread['member'].get('dead', False)):
# with self.browser:
# self.browser.delete_thread(thread['member']['id'])
# continue
t = Thread(int(thread['id']))
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % thread['username']
yield t
def get_thread(self, id, contacts=None, get_profiles=False):
"""
Get a thread and its messages.
The 'contacts' parameters is only used for internal calls.
"""
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread:
thread = Thread(int(id))
thread.flags = Thread.IS_DISCUSSION
full = False
else:
full = True
with self.browser:
mails = self.browser.get_thread_mails(id, 100)
my_name = self.browser.get_my_name()
child = None
msg = None
s**t = self._get_slut(mails['member']['pseudo'])
if contacts is None:
contacts = {}
if not thread.title:
thread.title = u'Discussion with %s' % mails['member']['pseudo']
for mail in mails['messages']:
flags = Message.IS_HTML
if parse_dt(mail['date']) > s**t['lastmsg'] and mail[
'id_from'] != self.browser.get_my_name():
flags |= Message.IS_UNREAD
if get_profiles:
if not mail['id_from'] in contacts:
with self.browser:
contacts[mail['id_from']] = self.get_contact(
mail['id_from'])
signature = u''
if mail.get('src', None):
signature += u'Sent from my %s\n\n' % mail['src']
if mail['id_from'] in contacts:
signature += contacts[mail['id_from']].get_text()
msg = Message(thread=thread,
id=int(
time.strftime('%Y%m%d%H%M%S',
parse_dt(
mail['date']).timetuple())),
title=thread.title,
sender=mail['id_from'],
receivers=[
my_name if mail['id_from'] != my_name else
mails['member']['pseudo']
],
date=parse_dt(mail['date']),
content=unescape(mail['message']).strip(),
signature=signature,
children=[],
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if full and msg:
# If we have get all the messages, replace NotLoaded with None as
# parent.
msg.parent = None
if not full and not msg:
# Perhaps there are hidden messages
msg = NotLoaded
thread.root = msg
return thread
#def iter_unread_messages(self, thread=None):
# try:
# contacts = {}
# with self.browser:
# threads = self.browser.get_threads_list()
# for thread in threads:
# if thread['member'].get('isBan', thread['member'].get('dead', False)):
# with self.browser:
# self.browser.delete_thread(int(thread['member']['id']))
# continue
# if self.antispam and not self.antispam.check_thread(thread):
# self.logger.info('Skipped a spam-unread-thread from %s' % thread['member']['pseudo'])
# self.report_spam(thread['member']['pseudo'])
# continue
# s**t = self._get_slut(thread['member']['pseudo'])
# if parse_dt(thread['date']) > s**t['lastmsg']:
# t = self.get_thread(thread['member']['pseudo'], contacts, get_profiles=True)
# for m in t.iter_all_messages():
# if m.flags & m.IS_UNREAD:
# yield m
# except BrowserUnavailable, e:
# self.logger.debug('No messages, browser is unavailable: %s' % e)
# pass # don't care about waiting
def set_message_read(self, message):
if message.sender == self.browser.get_my_name():
return
s**t = self._get_slut(message.sender)
if s**t['lastmsg'] < message.date:
s**t['lastmsg'] = message.date
self.storage.set('s***s', message.sender, s**t)
self.storage.save()
def _get_slut(self, id):
s***s = self.storage.get('s***s')
if not s***s or not id in s***s:
s**t = {'lastmsg': datetime.datetime(1970, 1, 1)}
else:
s**t = self.storage.get('s***s', id)
s**t['lastmsg'] = s**t.get('lastmsg', datetime.datetime(
1970, 1, 1)).replace(tzinfo=tz.tzutc())
return s**t
# ---- ICapMessagesPost methods ---------------------
def post_message(self, message):
content = message.content.replace('\n',
'\r\n').encode('utf-8', 'replace')
with self.browser:
# Check wether we already have a thread with this user
threads = self.browser.get_threads_list()
for thread in threads:
if thread['username'] == message.thread.id:
self.browser.post_reply(thread['id'], content)
break
else:
self.browser.post_mail(message.thread.id, content)
# ---- ICapContact methods ---------------------
def fill_contact(self, contact, fields):
if 'profile' in fields:
contact = self.get_contact(contact)
if contact and 'photos' in fields:
for name, photo in contact.photos.iteritems():
with self.browser:
if photo.url and not photo.data:
data = self.browser.openurl(photo.url).read()
contact.set_photo(name, data=data)
if photo.thumbnail_url and not photo.thumbnail_data:
data = self.browser.openurl(photo.thumbnail_url).read()
contact.set_photo(name, thumbnail_data=data)
def fill_photo(self, photo, fields):
with self.browser:
if 'data' in fields and photo.url and not photo.data:
photo.data = self.browser.readurl(photo.url)
if 'thumbnail_data' in fields and photo.thumbnail_url and not photo.thumbnail_data:
photo.thumbnail_data = self.browser.readurl(
photo.thumbnail_url)
return photo
def get_contact(self, contact):
with self.browser:
if isinstance(contact, Contact):
_id = contact.id
elif isinstance(contact, (int, long, basestring)):
_id = contact
else:
raise TypeError(
"The parameter 'contact' isn't a contact nor a int/long/str/unicode: %s"
% contact)
profile = self.browser.get_profile(_id)
if not profile:
return None
_id = profile['id']
if isinstance(contact, Contact):
contact.id = _id
contact.name = profile['id']
else:
contact = Contact(_id, profile['id'], Contact.STATUS_OFFLINE)
contact.url = 'http://%s/profile/%s' % (self.browser.DOMAIN, _id)
contact.profile = profile['data']
contact.summary = profile['summary']
if contact.profile['details'][
'last_online'].value == u'Online now!':
contact.status = Contact.STATUS_ONLINE
else:
contact.status = Contact.STATUS_OFFLINE
contact.status_msg = contact.profile['details'][
'last_online'].value
for no, photo in enumerate(self.browser.get_photos(_id)):
contact.set_photo(u'image_%i' % no,
url=photo,
thumbnail_url=photo)
return contact
#def _get_partial_contact(self, contact):
# if contact.get('isBan', contact.get('dead', False)):
# with self.browser:
# self.browser.delete_thread(int(contact['id']))
# return None
# s = 0
# if contact.get('isOnline', False):
# s = Contact.STATUS_ONLINE
# else:
# s = Contact.STATUS_OFFLINE
# c = Contact(contact['id'], contact['id'], s)
# c.url = self.browser.id2url(contact['id'])
# if 'birthday' in contact:
# birthday = _parse_dt(contact['birthday'])
# age = int((datetime.datetime.now() - birthday).days / 365.25)
# c.status_msg = u'%s old, %s' % (age, contact['city'])
# if contact['cover'].isdigit() and int(contact['cover']) > 0:
# url = 'http://s%s.adopteunmec.com/%s%%(type)s%s.jpg' % (contact['shard'], contact['path'], contact['cover'])
# else:
# url = 'http://s.adopteunmec.com/www/img/thumb0.gif'
# c.set_photo('image%s' % contact['cover'],
# url=url % {'type': 'image'},
# thumbnail_url=url % {'type': 'thumb0_'})
# return c
def iter_contacts(self, status=Contact.STATUS_ALL, ids=None):
with self.browser:
threads = self.browser.get_threads_list(count=100)
for thread in threads:
c = self._get_partial_contact(thread['member'])
if c and (c.status & status) and (not ids or c.id in ids):
yield c
#def send_query(self, id):
# if isinstance(id, Contact):
# id = id.id
# queries_queue = None
# try:
# queries_queue = self.get_optimization('QUERIES_QUEUE')
# except OptimizationNotFound:
# pass
# if queries_queue and queries_queue.is_running():
# if queries_queue.enqueue_query(id):
# return Query(id, 'A charm has been sent')
# else:
# return Query(id, 'Unable to send charm: it has been enqueued')
# else:
# with self.browser:
# if not self.browser.send_charm(id):
# raise QueryError('No enough charms available')
# return Query(id, 'A charm has been sent')
#def get_notes(self, id):
# if isinstance(id, Contact):
# id = id.id
# return self.storage.get('notes', id)
#def save_notes(self, id, notes):
# if isinstance(id, Contact):
# id = id.id
# self.storage.set('notes', id, notes)
# self.storage.save()
OBJECTS = {
Thread: fill_thread,
Contact: fill_contact,
ContactPhoto: fill_photo
}
class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapContent):
NAME = 'dlfp'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '0.h'
LICENSE = 'AGPLv3+'
DESCRIPTION = "Da Linux French Page news website"
CONFIG = BackendConfig(
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''),
ValueBool('get_news', label='Get newspapers', default=True),
ValueBool('get_diaries', label='Get diaries', default=False),
ValueBool('get_polls', label='Get polls', default=False),
ValueBool('get_board', label='Get board', default=False),
ValueBool('get_wiki', label='Get wiki', default=False),
ValueBool('get_tracker', label='Get tracker', default=False))
STORAGE = {'seen': {}}
BROWSER = DLFP
FEEDS = {
'get_news': "https://linuxfr.org/news.atom",
'get_diaries': "https://linuxfr.org/journaux.atom",
'get_polls': "https://linuxfr.org/sondages.atom",
'get_board': "https://linuxfr.org/forums.atom",
'get_wiki': "https://linuxfr.org/wiki.atom",
'get_tracker': "https://linuxfr.org/suivi.atom",
}
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
else:
password = None
return self.create_browser(username, password)
def deinit(self):
# don't need to logout if the browser hasn't been used.
if not self._browser:
return
with self.browser:
self.browser.close_session()
#### ICapMessages ##############################################
def iter_threads(self):
whats = set()
for param, url in self.FEEDS.iteritems():
if self.config[param].get():
whats.add(url)
for what in whats:
for article in Newsfeed(what, rssid).iter_entries():
if article.datetime and (datetime.now() - article.datetime
) > timedelta(days=60):
continue
thread = Thread(article.id)
thread.title = article.title
thread._rsscomment = article.rsscomment
if article.datetime:
thread.date = article.datetime
yield thread
def get_thread(self, id, getseen=True):
if not isinstance(id, Thread):
thread = None
else:
thread = id
id = thread.id
if thread.date:
self.storage.set('date', id, thread.date)
self.storage.save()
with self.browser:
content = self.browser.get_content(id)
if not content:
return None
if not thread:
thread = Thread(content.id)
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(
thread=thread,
id='0', # root message
title=content.title,
sender=content.author or u'',
receivers=None,
date=thread.date,
parent=None,
content=content.body,
signature='URL: %s' % self.browser.absurl(id2url(content.id)),
children=[],
flags=flags)
for com in content.comments:
self._insert_comment(com, thread.root, getseen)
return thread
def _insert_comment(self, com, parent, getseen=True):
""""
Insert 'com' comment and its children in the parent message.
"""
flags = Message.IS_HTML
if not com.id in self.storage.get(
'seen', parent.thread.id, 'comments', default=[]):
flags |= Message.IS_UNREAD
if getseen or flags & Message.IS_UNREAD:
com.parse()
message = Message(
thread=parent.thread,
id=com.id,
title=com.title,
sender=com.author or u'',
receivers=None,
date=com.date,
parent=parent,
content=com.body,
signature=com.signature +
'<br />'.join(['Score: %d' % com.score,
'URL: %s' % com.url]),
children=[],
flags=flags)
else:
message = Message(thread=parent.thread,
id=com.id,
children=[],
flags=flags)
parent.children.append(message)
for sub in com.comments:
self._insert_comment(sub, message, getseen)
def iter_unread_messages(self):
for thread in self.iter_threads():
# Check if we have seen all comments of this thread.
with self.browser:
oldhash = self.storage.get('hash', thread.id, default="")
newhash = self.browser.get_hash(thread._rsscomment)
if oldhash != newhash:
self.storage.set('hash', thread.id, newhash)
self.storage.save()
self.fill_thread(thread, 'root', getseen=False)
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
def set_message_read(self, message):
self.storage.set(
'seen', message.thread.id, 'comments',
self.storage.get('seen', message.thread.id, 'comments', default=[])
+ [message.id])
self.storage.save()
lastpurge = self.storage.get('lastpurge', default=0)
# 86400 = one day
if time.time() - lastpurge > 86400:
self.storage.set('lastpurge', time.time())
self.storage.save()
# we can't directly delete without a "RuntimeError: dictionary changed size during iteration"
todelete = []
for id in self.storage.get('seen', default={}):
date = self.storage.get('date', id, default=0)
# if no date available, create a new one (compatibility with "old" storage)
if date == 0:
self.storage.set('date', id, datetime.now())
elif datetime.now() - date > timedelta(days=60):
todelete.append(id)
for id in todelete:
self.storage.delete('hash', id)
self.storage.delete('date', id)
self.storage.delete('seen', id)
self.storage.save()
def fill_thread(self, thread, fields, getseen=True):
return self.get_thread(thread, getseen)
#### ICapMessagesReply #########################################
def post_message(self, message):
if not self.browser.username:
raise BrowserForbidden()
if not message.parent:
raise CantSendMessage(
'Posting news and diaries on DLFP is not supported yet')
assert message.thread
with self.browser:
return self.browser.post_comment(message.thread.id,
message.parent.id, message.title,
message.content)
#### ICapContent ###############################################
def get_content(self, _id, revision=None):
if isinstance(_id, basestring):
content = Content(_id)
else:
content = _id
_id = content.id
if revision:
raise NotImplementedError(
'Website does not provide access to older revisions sources.')
with self.browser:
data = self.browser.get_wiki_content(_id)
if data is None:
return None
content.content = data
return content
def push_content(self, content, message=None, minor=False):
if not self.browser.username:
raise BrowserForbidden()
with self.browser:
return self.browser.set_wiki_content(content.id, content.content,
message)
def get_content_preview(self, content):
with self.browser:
return self.browser.get_wiki_preview(content.id, content.content)
OBJECTS = {Thread: fill_thread}
class AdeccoModule(Module, CapJob):
NAME = 'adecco'
DESCRIPTION = u'adecco website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
VERSION = '1.1'
BROWSER = AdeccoBrowser
publicationDate_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
'000000': u'-- Indifférent --',
'1': u'Moins de 48 heures',
'2': u'Moins de 1 semaine',
'4': u'Moins de 2 semaines',
'3': u'Moins de 5 semaines',
}.iteritems())])
type_contract_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
'000000': u'--Indifferent--',
'1': u'CDD',
'2': u'CDI',
'3': u'Intérim',
'4': u'Emploi formation',
'5': u'Emploi saisonnier',
'6': u'Stage',
'7': u'Autre',
}.iteritems())])
places_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
'100|REGION_0|DEPARTEMENT_0': u'--Indifferent--',
'101|REGION_1': u'Alsace',
'102|REGION_1|DEPARTEMENT_1': u'-- Rhin (Bas) (67)',
'103|REGION_1|DEPARTEMENT_2': u'-- Rhin (Haut) (68)',
'104|REGION_2': u'Aquitaine',
'105|REGION_2|DEPARTEMENT_3': u'-- Dordogne (24)',
'106|REGION_2|DEPARTEMENT_4': u'-- Gironde (33)',
'107|REGION_2|DEPARTEMENT_5': u'-- Landes (40)',
'108|REGION_2|DEPARTEMENT_6': u'-- Lot et Garonne (47)',
'109|REGION_2|DEPARTEMENT_7': u'-- Pyrénées Atlantiques (64)',
'110|REGION_3': u'Auvergne',
'111|REGION_3|DEPARTEMENT_8': u'-- Allier (03)',
'112|REGION_3|DEPARTEMENT_9': u'-- Cantal (15)',
'113|REGION_3|DEPARTEMENT_10': u'-- Loire (Haute) (43)',
'114|REGION_3|DEPARTEMENT_11': u'-- Puy de Dôme (63)',
'115|REGION_5': u'Bourgogne',
'116|REGION_5|DEPARTEMENT_15': u'-- Côte d\'Or (21)',
'117|REGION_5|DEPARTEMENT_16': u'-- Nièvre (58)',
'118|REGION_5|DEPARTEMENT_17': u'-- Saône et Loire (71)',
'119|REGION_5|DEPARTEMENT_18': u'-- Yonne (89)',
'120|REGION_6': u'Bretagne',
'121|REGION_6|DEPARTEMENT_19': u'-- Côtes d\'Armor (22)',
'122|REGION_6|DEPARTEMENT_20': u'-- Finistère (29)',
'123|REGION_6|DEPARTEMENT_21': u'-- Ille et Vilaine (35)',
'124|REGION_6|DEPARTEMENT_22': u'-- Morbihan (56)',
'125|REGION_7': u'Centre',
'126|REGION_7|DEPARTEMENT_23': u'-- Cher (18)',
'127|REGION_7|DEPARTEMENT_24': u'-- Eure et Loir (28)',
'128|REGION_7|DEPARTEMENT_25': u'-- Indre (36)',
'129|REGION_7|DEPARTEMENT_26': u'-- Indre et Loire (37)',
'130|REGION_7|DEPARTEMENT_27': u'-- Loir et Cher (41)',
'131|REGION_7|DEPARTEMENT_28': u'-- Loiret (45)',
'132|REGION_8': u'Champagne Ardenne',
'133|REGION_8|DEPARTEMENT_29': u'-- Ardennes (08)',
'134|REGION_8|DEPARTEMENT_30': u'-- Aube (10)',
'135|REGION_8|DEPARTEMENT_31': u'-- Marne (51)',
'136|REGION_8|DEPARTEMENT_32': u'-- Marne (Haute) (52)',
'137|REGION_9': u'Corse',
'138|REGION_9|DEPARTEMENT_33': u'-- Corse du Sud (2A)',
'139|REGION_9|DEPARTEMENT_34': u'-- Haute Corse (2B)',
'140|REGION_11': u'Franche Comté',
'141|REGION_11|DEPARTEMENT_43': u'-- Belfort (Territoire de) (90)',
'142|REGION_11|DEPARTEMENT_40': u'-- Doubs (25)',
'143|REGION_11|DEPARTEMENT_41': u'-- Jura (39)',
'144|REGION_11|DEPARTEMENT_42': u'-- Saône (Haute) (70)',
'145|REGION_13': u'Ile de France',
'146|REGION_13|DEPARTEMENT_49': u'-- Essonne (91)',
'147|REGION_13|DEPARTEMENT_50': u'-- Hauts de Seine (92)',
'148|REGION_13|DEPARTEMENT_46': u'-- Paris (Dept.) (75)',
'149|REGION_13|DEPARTEMENT_51': u'-- Seine Saint Denis (93)',
'150|REGION_13|DEPARTEMENT_47': u'-- Seine et Marne (77)',
'151|REGION_13|DEPARTEMENT_53': u'-- Val d\'Oise (95)',
'152|REGION_13|DEPARTEMENT_52': u'-- Val de Marne (94)',
'153|REGION_13|DEPARTEMENT_48': u'-- Yvelines (78)',
'154|REGION_14': u'Languedoc Roussillon',
'155|REGION_14|DEPARTEMENT_54': u'-- Aude (11)',
'156|REGION_14|DEPARTEMENT_55': u'-- Gard (30)',
'157|REGION_14|DEPARTEMENT_56': u'-- Hérault (34)',
'158|REGION_14|DEPARTEMENT_57': u'-- Lozère (48)',
'159|REGION_14|DEPARTEMENT_58': u'-- Pyrénées Orientales (66)',
'160|REGION_15': u'Limousin',
'161|REGION_15|DEPARTEMENT_59': u'-- Corrèze (19)',
'162|REGION_15|DEPARTEMENT_60': u'-- Creuse (23)',
'163|REGION_15|DEPARTEMENT_61': u'-- Vienne (Haute) (87)',
'164|REGION_16': u'Lorraine',
'165|REGION_16|DEPARTEMENT_62': u'-- Meurthe et Moselle (54)',
'166|REGION_16|DEPARTEMENT_63': u'-- Meuse (55)',
'167|REGION_16|DEPARTEMENT_64': u'-- Moselle (57)',
'168|REGION_16|DEPARTEMENT_65': u'-- Vosges (88)',
'169|REGION_17': u'Midi Pyrénées',
'170|REGION_17|DEPARTEMENT_66': u'-- Ariège (09)',
'171|REGION_17|DEPARTEMENT_67': u'-- Aveyron (12)',
'172|REGION_17|DEPARTEMENT_68': u'-- Garonne (Haute) (31)',
'173|REGION_17|DEPARTEMENT_69': u'-- Gers (32)',
'174|REGION_17|DEPARTEMENT_70': u'-- Lot (46)',
'175|REGION_17|DEPARTEMENT_71': u'-- Pyrénées (Hautes) (65)',
'176|REGION_17|DEPARTEMENT_72': u'-- Tarn (81)',
'177|REGION_17|DEPARTEMENT_73': u'-- Tarn et Garonne (82)',
'178|REGION_18': u'Nord Pas de Calais',
'179|REGION_18|DEPARTEMENT_74': u'-- Nord (59)',
'180|REGION_18|DEPARTEMENT_75': u'-- Pas de Calais (62)',
'181|REGION_4': u'Normandie (Basse)',
'182|REGION_4|DEPARTEMENT_12': u'-- Calvados (14)',
'183|REGION_4|DEPARTEMENT_13': u'-- Manche (50)',
'184|REGION_4|DEPARTEMENT_14': u'-- Orne (61)',
'185|REGION_12': u'Normandie (Haute)',
'186|REGION_12|DEPARTEMENT_44': u'-- Eure (27)',
'187|REGION_12|DEPARTEMENT_47': u'-- Seine Maritime (76)',
'188|REGION_19': u'Pays de la Loire',
'189|REGION_19|DEPARTEMENT_76': u'-- Loire Atlantique (44)',
'190|REGION_19|DEPARTEMENT_77': u'-- Maine et Loire (49)',
'191|REGION_19|DEPARTEMENT_78': u'-- Mayenne (53)',
'192|REGION_19|DEPARTEMENT_79': u'-- Sarthe (72)',
'193|REGION_19|DEPARTEMENT_80': u'-- Vendée (85)',
'194|REGION_20': u'Picardie',
'195|REGION_20|DEPARTEMENT_81': u'-- Aisne (02)',
'196|REGION_20|DEPARTEMENT_83': u'-- Oise (60)',
'197|REGION_20|DEPARTEMENT_84': u'-- Somme (80)',
'198|REGION_21': u'Poitou Charentes',
'199|REGION_21|DEPARTEMENT_85': u'-- Charente (16)',
'200|REGION_21|DEPARTEMENT_86': u'-- Charente Maritime (17)',
'201|REGION_21|DEPARTEMENT_87': u'-- Sèvres (Deux) (79)',
'202|REGION_21|DEPARTEMENT_88': u'-- Vienne (86)',
'203|REGION_22': u'Provence Alpes Côte d\'Azur',
'204|REGION_22|DEPARTEMENT_90': u'-- Alpes (Hautes) (05)',
'205|REGION_22|DEPARTEMENT_91': u'-- Alpes Maritimes (06)',
'206|REGION_22|DEPARTEMENT_89': u'-- Alpes de Haute Provence (04)',
'207|REGION_22|DEPARTEMENT_92': u'-- Bouches du Rhône (13)',
'208|REGION_22|DEPARTEMENT_93': u'-- Var (83)',
'209|REGION_22|DEPARTEMENT_94': u'-- Vaucluse (84)',
'210|REGION_23': u'Rhône Alpes',
'211|REGION_23|DEPARTEMENT_95': u'-- Ain (01)',
'212|REGION_23|DEPARTEMENT_96': u'-- Ardèche (07)',
'213|REGION_23|DEPARTEMENT_97': u'-- Drôme (26)',
'214|REGION_23|DEPARTEMENT_98': u'-- Isère (38)',
'215|REGION_23|DEPARTEMENT_99': u'-- Loire (42)',
'216|REGION_23|DEPARTEMENT_100': u'-- Rhône (69)',
'217|REGION_23|DEPARTEMENT_101': u'-- Savoie (73)',
'218|REGION_23|DEPARTEMENT_102': u'-- Savoie (Haute) (74)',
'219|REGION_10': u'DOM TOM',
'220|REGION_10|DEPARTEMENT_35': u'-- Guadeloupe (971)',
'221|REGION_10|DEPARTEMENT_37': u'-- Guyane (973)',
'222|REGION_10|DEPARTEMENT_38': u'-- La Réunion (974)',
'223|REGION_10|DEPARTEMENT_36': u'-- Martinique (972)',
'224|REGION_10|DEPARTEMENT_108': u'-- Mayotte (976)',
'225|REGION_10|DEPARTEMENT_109': u'-- Nouvelle Calédonie (988)',
'226|REGION_10|DEPARTEMENT_108': u'-- Polynésie (987)',
'227|REGION_10|DEPARTEMENT_107': u'-- Saint Pierre et Miquelon (975)',
'228|REGION_24': u'International',
'229|REGION_24|DEPARTEMENT_104': u'-- Andorre',
'230|REGION_24|DEPARTEMENT_105': u'-- Monaco',
'231|REGION_24|DEPARTEMENT_106': u'-- Suisse',
}.iteritems())])
activityDomain_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({
'100|DOMAIN_0': u'Tous domaines d\'activité',
'101|DOMAIN_1': u'Accueil - Secrétariat - Fonctions Administratives',
'102|DOMAIN_1|ACTIVITY_1': u'-- Accueil',
'103|DOMAIN_1|ACTIVITY_2': u'-- Secrétariat - Assistanat',
'104|DOMAIN_1|ACTIVITY_3': u'-- Autres Fonctions Administratives',
'105|DOMAIN_2': u'Achats - Juridique - Qualité - RH - Direction',
'106|DOMAIN_2|ACTIVITY_4': u'-- Achats ',
'107|DOMAIN_2|ACTIVITY_5': u'-- Juridique',
'108|DOMAIN_2|ACTIVITY_6': u'-- Qualité',
'109|DOMAIN_2|ACTIVITY_7': u'Ressources Humaines - Formation',
'110|DOMAIN_2|ACTIVITY_8': u'-- Direction Générale',
'111|DOMAIN_3': u'Agriculture - Viticulture - Pêche - Espaces Verts',
'112|DOMAIN_3|ACTIVITY_9': u'-- Agriculture - Viticulture - Pêche ',
'113|DOMAIN_3|ACTIVITY_10': u'-- Espaces Verts - Exploitation Forestière',
'114|DOMAIN_4': u'Automobile',
'115|DOMAIN_5': u'Banque - Finance - Gestion Comptabilité - Assurance',
'116|DOMAIN_5|ACTIVITY_11': u'-- Banque - Finance ',
'117|DOMAIN_5|ACTIVITY_12': u'-- Gestion - Comptabilité',
'118|DOMAIN_5|ACTIVITY_13': u'-- Assurance',
'119|DOMAIN_6': u'Bâtiment - Travaux Publics - Architecture - Immobilier',
'120|DOMAIN_6|ACTIVITY_14': u'-- Bâtiment - Travaux Publics',
'121|DOMAIN_6|ACTIVITY_15': u'-- Architecture - Immobilier ',
'122|DOMAIN_13': u'Bureaux d\'Etudes - Méthodes',
'123|DOMAIN_8': u'Commerce - Vente - Grande Distribution',
'124|DOMAIN_8|ACTIVITY_20': u'-- Commerce - Vente',
'125|DOMAIN_8|ACTIVITY_21': u'-- Grande et Moyenne Distribution',
'126|DOMAIN_9': u'Environnement - Nettoyage - Sécurité',
'127|DOMAIN_9|ACTIVITY_22': u'-- Environnement - HSE - Développement durable',
'128|DOMAIN_9|ACTIVITY_23': u'-- Nettoyage - Assainissement - Pressing',
'129|DOMAIN_9|ACTIVITY_24': u'-- Sécurité - Premiers secours',
'130|DOMAIN_10': u'Hôtellerie - Restauration - Métiers de Bouche',
'131|DOMAIN_10|ACTIVITY_25': u'-- Hôtellerie',
'132|DOMAIN_10|ACTIVITY_27': u'-- Métiers de bouche',
'133|DOMAIN_10|ACTIVITY_26': u'-- Restauration',
'134|DOMAIN_11': u'Industrie',
'135|DOMAIN_11|ACTIVITY_32': u'-- Aéronautique - Navale',
'136|DOMAIN_11|ACTIVITY_33': u'-- Agroalimentaire',
'137|DOMAIN_11|ACTIVITY_58': u'-- Chimie - Pétrochimie',
'138|DOMAIN_11|ACTIVITY_28': u'-- Electricité - Electronique - Automatisme',
'139|DOMAIN_11|ACTIVITY_29': u'-- Maintenance - Entretien - SAV ',
'140|DOMAIN_11|ACTIVITY_30': u'-- Mécanique Générale',
'141|DOMAIN_11|ACTIVITY_31': u'-- Production - Fabrication ',
'142|DOMAIN_11|ACTIVITY_36': u'-- Sidérurgie - Métallurgie - Tuyauterie - Soudure',
'143|DOMAIN_11|ACTIVITY_34': u'-- Nucléaire - Production d\'énergie',
'144|DOMAIN_11|ACTIVITY_35': u'-- Plasturgie - Bois - Papier - Verre - Cuir - Textile',
'145|DOMAIN_12': u'Informatique - Technologie de l\'Information',
'146|DOMAIN_12|ACTIVITY_37': u'-- Direction informatique encadrement',
'147|DOMAIN_12|ACTIVITY_38': u'-- Etude et développement',
'148|DOMAIN_12|ACTIVITY_39': u'-- Exploitation, maintenance et support ',
'149|DOMAIN_12|ACTIVITY_40': u'-- Systèmes et réseaux informatique et télécom',
'150|DOMAIN_14': u'Logistique - Manutention - Transport',
'151|DOMAIN_14|ACTIVITY_42': u'-- Conduite de véhicule',
'152|DOMAIN_14|ACTIVITY_43': u'-- Exploitation de logistique - supply chain',
'153|DOMAIN_14|ACTIVITY_44': u'-- Manutention',
'154|DOMAIN_14|ACTIVITY_45': u'-- Transport',
'155|DOMAIN_15': u'Marketing - Communication - Imprimerie - Edition',
'156|DOMAIN_15|ACTIVITY_47': u'-- Imprimerie - Edition - Arts Graphiques',
'157|DOMAIN_15|ACTIVITY_46': u'-- Marketing - Communication - Medias',
'158|DOMAIN_16': u'Médical - Paramédical - Esthétique',
'159|DOMAIN_16|ACTIVITY_59': u'-- Commerce Appareillage',
'160|DOMAIN_16|ACTIVITY_50': u'-- Directions, Cadres et Enseignement',
'161|DOMAIN_16|ACTIVITY_49': u'-- Rééducation, Radiologie, Appareillage, LAM',
'162|DOMAIN_16|ACTIVITY_51': u'-- Secrétariat, Dentaire, Social, Esthétique et Autres',
'163|DOMAIN_16|ACTIVITY_48': u'-- Soignants - Auxiliaires',
'164|DOMAIN_7': u'Pharmacie (Industrie, Officine) - Recherche clinique',
'165|DOMAIN_7|ACTIVITY_16': u'-- Industrie Pharmaceutique / Cosmétologique - Biotech',
'166|DOMAIN_7|ACTIVITY_17': u'-- Recherche Clinique',
'167|DOMAIN_7|ACTIVITY_18': u'-- Pharmacie Officine / Hospit / Para-pharmacie',
'168|DOMAIN_7|ACTIVITY_19': u'-- Vente, information et promotion du médicament',
'169|DOMAIN_17': u'Télémarketing - Téléservices',
'170|DOMAIN_17|ACTIVITY_52': u'-- Téléconseil - Télévente - Autres',
'171|DOMAIN_17|ACTIVITY_53': u'-- Direction, Encadrement',
'172|DOMAIN_18': u'Tourisme - Loisirs - Spectacle - Audiovisuel',
'173|DOMAIN_18|ACTIVITY_54': u'-- Tourisme - Loisirs',
'174|DOMAIN_18|ACTIVITY_55': u'-- Spectacle - Audiovisuel',
}.iteritems())])
CONFIG = BackendConfig(Value('publication_date', label=u'Publication Date', choices=publicationDate_choices),
Value('place', label=u'Place', choices=places_choices),
Value('contract', labe=u'Contract type', choices=type_contract_choices),
Value('activity_domain', label=u'Activity Domain', choices=activityDomain_choices),
)
def search_job(self, pattern=None):
with self.browser:
for advert in self.browser.search_job(pattern):
yield advert
def decode_choice(self, place):
splitted_choice = place.split('|')
part1 = splitted_choice[1].split('_')[1]
if len(splitted_choice) == 3:
part2 = splitted_choice[2].split('_')[1]
return part1, part2
else:
return part1, 0
def advanced_search_job(self):
region, departement = self.decode_choice(self.config['place'].get())
domain, category = self.decode_choice(self.config['activity_domain'].get())
for advert in self.browser.advanced_search_job(publication_date=int(self.config['publication_date'].get()),
contract_type=int(self.config['contract'].get()),
conty=departement,
region=region,
job_category=category,
activity_domain=domain
):
yield advert
def get_job_advert(self, _id, advert=None):
with self.browser:
return self.browser.get_job_advert(_id, advert)
def fill_obj(self, advert, fields):
self.get_job_advert(advert.id, advert)
OBJECTS = {AdeccoJobAdvert: fill_obj}
class AgendadulibreModule(Module, CapCalendarEvent):
NAME = 'agendadulibre'
DESCRIPTION = u'agendadulibre website'
MAINTAINER = u'Bezleputh'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.6'
ASSOCIATED_CATEGORIES = [CATEGORIES.CONF]
BROWSER = AgendadulibreBrowser
region_choices = OrderedDict([
(k, u'%s (%s)' % (v, k))
for k, v in sorted({
"https://www.agendadulibre.org": u'--France--',
"https://www.agendadulibre.org#3": u'Auvergne-Rhône-Alpes',
"https://www.agendadulibre.org#5": u'Bourgogne-Franche-Comté',
"https://www.agendadulibre.org#6": u'Bretagne',
"https://www.agendadulibre.org#7": u'Centre-Val de Loire',
"https://www.agendadulibre.org#30": u'Collectivité sui generis',
"https://www.agendadulibre.org#29": u'Collectivités d\'outre-mer',
"https://www.agendadulibre.org#9": u'Corse',
"https://www.agendadulibre.org#1": u'Grand Est',
"https://www.agendadulibre.org#23": u'Guadeloupe',
"https://www.agendadulibre.org#24": u'Guyane',
"https://www.agendadulibre.org#17": u'Hauts-de-France',
"https://www.agendadulibre.org#12": u'Île-de-France',
"https://www.agendadulibre.org#31": u'Internet',
"https://www.agendadulibre.org#26": u'La Réunion',
"https://www.agendadulibre.org#25": u'Martinique',
"https://www.agendadulibre.org#28": u'Mayotte',
"https://www.agendadulibre.org#4": u'Normandie',
"https://www.agendadulibre.org#2": u'Nouvelle-Aquitaine',
"https://www.agendadulibre.org#13": u'Occitanie',
"https://www.agendadulibre.org#18": u'Pays de la Loire',
"https://www.agendadulibre.org#21": u'Provence-Alpes-Côte d\'Azur',
"https://www.agendadulibre.be": u'--Belgique--',
"https://www.agendadulibre.be#11": u'Antwerpen',
"https://www.agendadulibre.be#10": u'Brabant wallon',
"https://www.agendadulibre.be#9": u'Bruxelles-Capitale',
"https://www.agendadulibre.be#8": u'Hainaut',
"https://www.agendadulibre.be#7": u'Liege',
"https://www.agendadulibre.be#6": u'Limburg',
"https://www.agendadulibre.be#5": u'Luxembourg',
"https://www.agendadulibre.be#4": u'Namur',
"https://www.agendadulibre.be#3": u'Oost-Vlaanderen',
"https://www.agendadulibre.be#2": u'Vlaams-Brabant',
"https://www.agendadulibre.be#1": u'West-Vlaanderen',
"https://www.agendadulibre.ch": u'--Suisse--',
"https://www.agendadulibre.ch#15": u'Appenzell Rhodes-Extérieures',
"https://www.agendadulibre.ch#16": u'Appenzell Rhodes-Intérieures',
"https://www.agendadulibre.ch#19": u'Argovie',
"https://www.agendadulibre.ch#13": u'Bâle-Campagne',
"https://www.agendadulibre.ch#12": u'Bâle-Ville',
"https://www.agendadulibre.ch#2": u'Berne',
"https://www.agendadulibre.ch#10": u'Fribourg',
"https://www.agendadulibre.ch#25": u'Genève',
"https://www.agendadulibre.ch#8": u'Glaris',
"https://www.agendadulibre.ch#18": u'Grisons',
"https://www.agendadulibre.ch#26": u'Jura',
"https://www.agendadulibre.ch#3": u'Lucerne',
"https://www.agendadulibre.ch#24": u'Neuchâtel',
"https://www.agendadulibre.ch#7": u'Nidwald',
"https://www.agendadulibre.ch#6": u'Obwald',
"https://www.agendadulibre.ch#17": u'Saint-Gall',
"https://www.agendadulibre.ch#14": u'Schaffhouse',
"https://www.agendadulibre.ch#5": u'Schwytz',
"https://www.agendadulibre.ch#11": u'Soleure',
"https://www.agendadulibre.ch#21": u'Tessin',
"https://www.agendadulibre.ch#20": u'Thurgovie',
"https://www.agendadulibre.ch#4": u'Uri',
"https://www.agendadulibre.ch#23": u'Valais',
"https://www.agendadulibre.ch#22": u'Vaud',
"https://www.agendadulibre.ch#9": u'Zoug',
"https://www.agendadulibre.ch#1": u'Zurich',
}.items())
])
CONFIG = BackendConfig(
Value('region', label=u'Region', choices=region_choices))
def create_default_browser(self):
choice = self.config['region'].get().split('#')
selected_region = '' if len(choice) < 2 else choice[-1]
return self.create_browser(website=choice[0], region=selected_region)
def search_events(self, query):
return self.browser.list_events(query.start_date, query.end_date,
query.city, query.categories)
def list_events(self, date_from, date_to=None):
return self.browser.list_events(date_from, date_to)
def get_event(self, event_id):
return self.browser.get_event(event_id)
def fill_obj(self, event, fields):
event = self.browser.get_event(event.id, event)
choice = self.config['region'].get().split('#')
selected_region = '' if len(choice) < 2 else choice[-1]
if selected_region == '23':
event.timezone = 'America/Guadeloupe'
elif selected_region == '24':
event.timezone = 'America/Guyana'
elif selected_region == '26':
event.timezone = 'Indian/Reunion'
elif selected_region == '25':
event.timezone = 'America/Martinique'
else:
event.timezone = 'Europe/Paris'
return event
OBJECTS = {AgendadulibreBrowser: fill_obj}
class EHentaiModule(Module, CapGallery, CapCollection):
NAME = 'ehentai'
MAINTAINER = u'Roger Philibert'
EMAIL = '*****@*****.**'
VERSION = '1.1'
DESCRIPTION = 'E-Hentai galleries'
LICENSE = 'AGPLv3+'
BROWSER = EHentaiBrowser
CONFIG = BackendConfig(
Value('domain', label='Domain', default='g.e-hentai.org'),
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password'))
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
else:
password = None
return self.create_browser(self.config['domain'].get(), username,
password)
def search_galleries(self, pattern, sortby=None):
with self.browser:
return self.browser.search_galleries(pattern)
def iter_gallery_images(self, gallery):
self.fillobj(gallery, ('url', ))
with self.browser:
return self.browser.iter_gallery_images(gallery)
ID_REGEXP = r'/?\d+/[\dabcdef]+/?'
URL_REGEXP = r'.+/g/(%s)' % ID_REGEXP
def get_gallery(self, _id):
match = re.match(r'^%s$' % self.URL_REGEXP, _id)
if match:
_id = match.group(1)
else:
match = re.match(r'^%s$' % self.ID_REGEXP, _id)
if match:
_id = match.group(0)
else:
return None
gallery = EHentaiGallery(_id)
with self.browser:
if self.browser.gallery_exists(gallery):
return gallery
else:
return None
def fill_gallery(self, gallery, fields):
if not gallery.__iscomplete__():
with self.browser:
self.browser.fill_gallery(gallery, fields)
def fill_image(self, image, fields):
with self.browser:
image.url = self.browser.get_image_url(image)
if 'data' in fields:
ratelimit("ehentai_get", 2)
image.data = self.browser.readurl(image.url)
def iter_resources(self, objs, split_path):
if BaseGallery in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest_nsfw'])
if collection.split_path == [u'latest_nsfw']:
for gallery in self.browser.latest_gallery():
yield gallery
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseGallery in objs and collection.split_path == [u'latest_nsfw']:
collection.title = u'Latest E-Hentai galleries (NSFW)'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {EHentaiGallery: fill_gallery, EHentaiImage: fill_image}
class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating,
ICapChat, ICapContact, ICapAccount):
NAME = 'aum'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '0.h'
LICENSE = 'AGPLv3+'
DESCRIPTION = u'"Adopte un Mec" French dating website'
CONFIG = BackendConfig(
Value('username', label='Username'),
ValueBackendPassword('password', label='Password'),
ValueBool('antispam', label='Enable anti-spam', default=False),
ValueBool('baskets',
label='Get baskets with new messages',
default=True),
Value('search_query', label='Search query', default=''))
STORAGE = {
'profiles_walker': {
'viewed': []
},
'queries_queue': {
'queue': []
},
's***s': {},
'notes': {},
}
BROWSER = AuMBrowser
MAGIC_ID_BASKET = 1
def __init__(self, *args, **kwargs):
BaseBackend.__init__(self, *args, **kwargs)
if self.config['antispam'].get():
self.antispam = AntiSpam()
else:
self.antispam = None
def create_default_browser(self):
return self.create_browser(self.config['username'].get(),
self.config['password'].get(),
self.config['search_query'].get())
def report_spam(self, id):
with self.browser:
pass
#self.browser.delete_thread(id)
# Do not report fakes to website, to let them to other guys :)
#self.browser.report_fake(id)
# ---- ICapDating methods ---------------------
def init_optimizations(self):
self.add_optimization(
'PROFILE_WALKER',
ProfilesWalker(self.weboob.scheduler, self.storage, self.browser))
self.add_optimization('VISIBILITY',
Visibility(self.weboob.scheduler, self.browser))
self.add_optimization(
'QUERIES_QUEUE',
QueriesQueue(self.weboob.scheduler, self.storage, self.browser))
def iter_events(self):
all_events = {}
with self.browser:
all_events[u'baskets'] = (self.browser.get_baskets,
'You were put into %s\'s basket')
all_events[u'flashs'] = (self.browser.get_flashs,
'You sent a charm to %s')
all_events[u'visits'] = (self.browser.get_visits, 'Visited by %s')
for type, (events, message) in all_events.iteritems():
for event in events():
e = Event(event['who']['id'])
e.date = parse_dt(event['date'])
e.type = type
if 'who' in event:
e.contact = self._get_partial_contact(event['who'])
else:
e.contact = self._get_partial_contact(event)
if not e.contact:
continue
e.message = message % e.contact.name
yield e
# ---- ICapMessages methods ---------------------
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def iter_threads(self):
with self.browser:
threads = self.browser.get_threads_list()
for thread in threads:
#if thread['member'].get('isBan', thread['member'].get('dead', False)):
# with self.browser:
# self.browser.delete_thread(thread['member']['id'])
# continue
if self.antispam and not self.antispam.check_thread(thread):
self.logger.info('Skipped a spam-thread from %s' %
thread['pseudo'])
self.report_spam(thread['who']['id'])
continue
t = Thread(int(thread['who']['id']))
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % to_unicode(
thread['who']['pseudo'])
yield t
def get_thread(self, id, contacts=None, get_profiles=False):
"""
Get a thread and its messages.
The 'contacts' parameters is only used for internal calls.
"""
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread:
thread = Thread(int(id))
thread.flags = Thread.IS_DISCUSSION
full = False
else:
full = True
with self.browser:
mails = self.browser.get_thread_mails(id, 100)
my_name = self.browser.get_my_name()
child = None
msg = None
s**t = self._get_slut(id)
if contacts is None:
contacts = {}
if not thread.title:
thread.title = u'Discussion with %s' % mails['who']['pseudo']
self.storage.set('s***s', int(thread.id), 'status', mails['status'])
self.storage.save()
for mail in mails['results']:
flags = 0
if self.antispam and not self.antispam.check_mail(mail):
self.logger.info('Skipped a spam-mail from %s' %
mails['who']['pseudo'])
self.report_spam(thread.id)
break
if parse_dt(mail['date']) > s**t['lastmsg']:
flags |= Message.IS_UNREAD
if get_profiles:
if not mail['from'] in contacts:
try:
with self.browser:
contacts[mail['from']] = self.get_contact(
mail['from'])
except BrowserHTTPNotFound:
pass
if self.antispam and mail[
'from'] in contacts and not self.antispam.check_contact(
contacts[mail['from']]):
self.logger.info(
'Skipped a spam-mail-profile from %s' %
mails['who']['pseudo'])
self.report_spam(thread.id)
break
if int(mail['from']) == self.browser.my_id:
if mails['remote_status'] == 'new' and msg is None:
flags |= Message.IS_NOT_RECEIVED
else:
flags |= Message.IS_RECEIVED
signature = u''
#if mail.get('src', None):
# signature += u'Sent from my %s\n\n' % mail['src']
if mail['from'] in contacts:
signature += contacts[mail['from']].get_text()
msg = Message(
thread=thread,
id=int(
time.strftime('%Y%m%d%H%M%S',
parse_dt(mail['date']).timetuple())),
title=thread.title,
sender=to_unicode(my_name if int(mail['from']) == self.browser.
my_id else mails['who']['pseudo']),
receivers=[
to_unicode(my_name if int(mail['from']) != self.browser.
my_id else mails['who']['pseudo'])
],
date=parse_dt(mail['date']),
content=to_unicode(unescape(mail['message'] or '').strip()),
signature=signature,
children=[],
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if full and msg:
# If we have get all the messages, replace NotLoaded with None as
# parent.
msg.parent = None
if not full and not msg:
# Perhaps there are hidden messages
msg = NotLoaded
thread.root = msg
return thread
def iter_unread_messages(self):
try:
contacts = {}
with self.browser:
threads = self.browser.get_threads_list()
for thread in threads:
#if thread['member'].get('isBan', thread['member'].get('dead', False)):
# with self.browser:
# self.browser.delete_thread(int(thread['member']['id']))
# continue
if self.antispam and not self.antispam.check_thread(thread):
self.logger.info('Skipped a spam-unread-thread from %s' %
thread['who']['pseudo'])
self.report_spam(thread['member']['id'])
continue
s**t = self._get_slut(thread['who']['id'])
if parse_dt(
thread['date']
) > s**t['lastmsg'] or thread['status'] != s**t['status']:
t = self.get_thread(thread['who']['id'],
contacts,
get_profiles=True)
for m in t.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
if not self.config['baskets'].get():
return
# Send mail when someone added me in her basket.
# XXX possibly race condition if a s**t adds me in her basket
# between the aum.nb_new_baskets() and aum.get_baskets().
with self.browser:
s**t = self._get_slut(-self.MAGIC_ID_BASKET)
new_baskets = self.browser.nb_new_baskets()
if new_baskets > 0:
baskets = self.browser.get_baskets()
my_name = self.browser.get_my_name()
for basket in baskets:
if parse_dt(basket['date']) <= s**t['lastmsg']:
continue
contact = self.get_contact(basket['who']['id'])
if self.antispam and not self.antispam.check_contact(
contact):
self.logger.info('Skipped a spam-basket from %s' %
contact.name)
self.report_spam(basket['who']['id'])
continue
thread = Thread(int(basket['who']['id']))
thread.title = 'Basket of %s' % contact.name
thread.root = Message(
thread=thread,
id=self.MAGIC_ID_BASKET,
title=thread.title,
sender=contact.name,
receivers=[my_name],
date=parse_dt(basket['date']),
content='You are taken in her basket!',
signature=contact.get_text(),
children=[],
flags=Message.IS_UNREAD)
yield thread.root
except BrowserUnavailable as e:
self.logger.debug('No messages, browser is unavailable: %s' % e)
pass # don't care about waiting
def set_message_read(self, message):
if int(message.id) == self.MAGIC_ID_BASKET:
# Save the last baskets checks.
s**t = self._get_slut(-self.MAGIC_ID_BASKET)
if s**t['lastmsg'] < message.date:
s**t['lastmsg'] = message.date
self.storage.set('s***s', -self.MAGIC_ID_BASKET, s**t)
self.storage.save()
return
s**t = self._get_slut(message.thread.id)
if s**t['lastmsg'] < message.date:
s**t['lastmsg'] = message.date
self.storage.set('s***s', int(message.thread.id), s**t)
self.storage.save()
def _get_slut(self, id):
id = int(id)
s***s = self.storage.get('s***s')
if not s***s or not id in s***s:
s**t = {'lastmsg': datetime.datetime(1970, 1, 1), 'status': None}
else:
s**t = self.storage.get('s***s', id)
s**t['lastmsg'] = s**t.get('lastmsg', datetime.datetime(
1970, 1, 1)).replace(tzinfo=tz.tzutc())
s**t['status'] = s**t.get('status', None)
return s**t
# ---- ICapMessagesPost methods ---------------------
def post_message(self, message):
with self.browser:
self.browser.post_mail(message.thread.id, message.content)
# ---- ICapContact methods ---------------------
def fill_contact(self, contact, fields):
if 'profile' in fields:
contact = self.get_contact(contact)
if contact and 'photos' in fields:
for name, photo in contact.photos.iteritems():
with self.browser:
if photo.url and not photo.data:
data = self.browser.openurl(photo.url).read()
contact.set_photo(name, data=data)
if photo.thumbnail_url and not photo.thumbnail_data:
data = self.browser.openurl(photo.thumbnail_url).read()
contact.set_photo(name, thumbnail_data=data)
def fill_photo(self, photo, fields):
with self.browser:
if 'data' in fields and photo.url and not photo.data:
photo.data = self.browser.readurl(photo.url)
if 'thumbnail_data' in fields and photo.thumbnail_url and not photo.thumbnail_data:
photo.thumbnail_data = self.browser.readurl(
photo.thumbnail_url)
return photo
def get_contact(self, contact):
with self.browser:
if isinstance(contact, Contact):
_id = contact.id
elif isinstance(contact, (int, long, basestring)):
_id = contact
else:
raise TypeError(
"The parameter 'contact' isn't a contact nor a int/long/str/unicode: %s"
% contact)
profile = self.browser.get_full_profile(_id)
if not profile:
return None
_id = profile['id']
if isinstance(contact, Contact):
contact.id = _id
contact.name = profile['pseudo']
else:
contact = Contact(_id, profile['pseudo'],
Contact.STATUS_ONLINE)
contact.url = self.browser.id2url(_id)
contact.parse_profile(profile, self.browser.get_consts())
return contact
def _get_partial_contact(self, contact):
s = 0
if contact.get('online', False):
s = Contact.STATUS_ONLINE
else:
s = Contact.STATUS_OFFLINE
c = Contact(contact['id'], to_unicode(contact['pseudo']), s)
c.url = self.browser.id2url(contact['id'])
if 'age' in contact:
c.status_msg = u'%s old, %s' % (contact['age'], contact['city'])
if contact['cover'] is not None:
url = contact['cover'] + '/%(type)s'
else:
url = u'http://s.adopteunmec.com/www/img/thumb0.jpg'
c.set_photo(u'image%s' % contact['cover'],
url=url % {'type': 'full'},
thumbnail_url=url % {'type': 'small'})
return c
def iter_contacts(self, status=Contact.STATUS_ALL, ids=None):
with self.browser:
threads = self.browser.get_threads_list(count=100)
for thread in threads:
c = self._get_partial_contact(thread['who'])
if c and (c.status & status) and (not ids or c.id in ids):
yield c
def send_query(self, id):
if isinstance(id, Contact):
id = id.id
queries_queue = None
try:
queries_queue = self.get_optimization('QUERIES_QUEUE')
except OptimizationNotFound:
pass
if queries_queue and queries_queue.is_running():
if queries_queue.enqueue_query(id):
return Query(id, 'A charm has been sent')
else:
return Query(id, 'Unable to send charm: it has been enqueued')
else:
with self.browser:
if not self.browser.send_charm(id):
raise QueryError('No enough charms available')
return Query(id, 'A charm has been sent')
def get_notes(self, id):
if isinstance(id, Contact):
id = id.id
return self.storage.get('notes', id)
def save_notes(self, id, notes):
if isinstance(id, Contact):
id = id.id
self.storage.set('notes', id, notes)
self.storage.save()
# ---- ICapChat methods ---------------------
def iter_chat_messages(self, _id=None):
with self.browser:
return self.browser.iter_chat_messages(_id)
def send_chat_message(self, _id, message):
with self.browser:
return self.browser.send_chat_message(_id, message)
#def start_chat_polling(self):
#self._profile_walker = ProfilesWalker(self.weboob.scheduler, self.storage, self.browser)
# ---- ICapAccount methods ---------------------
ACCOUNT_REGISTER_PROPERTIES = ValuesDict(
Value('username', label='Email address',
regexp='^[^ ]+@[^ ]+\.[^ ]+$'),
Value('password', label='Password', regexp='^[^ ]+$', masked=True),
Value('sex', label='Sex', choices={
'm': 'Male',
'f': 'Female'
}),
Value('birthday',
label='Birthday (dd/mm/yyyy)',
regexp='^\d+/\d+/\d+$'),
Value('zipcode', label='Zipcode'),
Value('country',
label='Country',
choices={
'fr': 'France',
'be': 'Belgique',
'ch': 'Suisse',
'ca': 'Canada'
},
default='fr'),
Value('godfather', label='Godfather', regexp='^\d*$', default=''),
)
@classmethod
def register_account(klass, account):
"""
Register an account on website
This is a static method, it would be called even if the backend is
instancied.
@param account an Account object which describe the account to create
"""
browser = None
bday, bmonth, byear = account.properties['birthday'].get().split(
'/', 2)
while not browser:
try:
browser = klass.BROWSER(account.properties['username'].get())
browser.register(
password=account.properties['password'].get(),
sex=(0 if account.properties['sex'].get() == 'm' else 1),
birthday_d=int(bday),
birthday_m=int(bmonth),
birthday_y=int(byear),
zipcode=account.properties['zipcode'].get(),
country=account.properties['country'].get(),
godfather=account.properties['godfather'].get())
except CaptchaError:
getLogger('aum').info('Unable to resolve captcha. Retrying...')
browser = None
REGISTER_REGEXP = re.compile(
'.*http://www.adopteunmec.com/register4.php\?([^\' ]*)\'')
def confirm_account(self, mail):
msg = email.message_from_string(mail)
content = u''
for part in msg.walk():
s = part.get_payload(decode=True)
content += unicode(s, 'iso-8859-15')
url = None
for s in content.split():
m = self.REGISTER_REGEXP.match(s)
if m:
url = '/register4.php?' + m.group(1)
break
if url:
browser = self.create_browser('')
browser.openurl(url)
return True
return False
def get_account(self):
"""
Get the current account.
"""
raise NotImplementedError()
def update_account(self, account):
"""
Update the current account.
"""
raise NotImplementedError()
def get_account_status(self):
with self.browser:
return (
StatusField('myname', 'My name', self.browser.get_my_name()),
StatusField('score', 'Score', self.browser.score()),
StatusField('avcharms', 'Available charms',
self.browser.nb_available_charms()),
StatusField('newvisits', 'New visits',
self.browser.nb_new_visites()),
)
OBJECTS = {
Thread: fill_thread,
Contact: fill_contact,
ContactPhoto: fill_photo
}
class PhpBBModule(Module, CapMessages, CapMessagesPost):
NAME = 'phpbb'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.1'
LICENSE = 'AGPLv3+'
DESCRIPTION = "phpBB forum"
CONFIG = BackendConfig(
Value('url', label='URL of forum', regexp='https?://.*'),
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''),
ValueInt(
'thread_unread_messages',
label='Limit number of unread messages to retrieve for a thread',
default=500))
STORAGE = {'seen': {}}
BROWSER = PhpBB
def create_default_browser(self):
username = self.config['username'].get()
if len(username) > 0:
password = self.config['password'].get()
else:
password = None
return self.create_browser(self.config['url'].get(), username,
password)
#### CapMessages ##############################################
def _iter_threads(self, root_link=None):
with self.browser:
links = list(
self.browser.iter_links(root_link.url if root_link else None))
for link in links:
if link.type == link.FORUM:
link.title = '%s[%s]' % (root_link.title if root_link else '',
link.title)
for thread in self._iter_threads(link):
yield thread
if link.type == link.TOPIC:
thread = Thread(url2id(link.url))
thread.title = ('%s ' % root_link.title
if root_link else '') + link.title
thread.date = link.date
thread.flags = thread.IS_DISCUSSION
yield thread
def iter_threads(self):
return self._iter_threads()
def get_thread(self, id):
thread = None
parent = None
if isinstance(id, Thread):
thread = id
id = thread.id
thread_id = url2id(id, nopost=True) or id
try:
last_seen_id = self.storage.get('seen',
default={})[id2topic(thread_id)]
except KeyError:
last_seen_id = 0
with self.browser:
for post in self.browser.iter_posts(id):
if not thread:
thread = Thread(thread_id)
thread.title = post.title
m = self._post2message(thread, post)
m.parent = parent
if last_seen_id < post.id:
m.flags |= Message.IS_UNREAD
if parent:
parent.children = [m]
else:
thread.root = m
parent = m
return thread
def _post2message(self, thread, post):
signature = post.signature
if signature:
signature += '<br />'
signature += 'URL: %s' % self.browser.absurl(
id2url('%s.%s' % (thread.id, post.id)))
return Message(thread=thread,
id=post.id,
title=post.title,
sender=post.author,
receivers=None,
date=post.date,
parent=None,
content=post.content,
signature=signature,
children=[],
flags=Message.IS_HTML)
def iter_unread_messages(self):
with self.browser:
url = self.browser.get_root_feed_url()
for article in Newsfeed(url, rssid).iter_entries():
id = url2id(article.link)
thread = None
try:
last_seen_id = self.storage.get('seen',
default={})[id2topic(id)]
except KeyError:
last_seen_id = 0
child = None
iterator = self.browser.riter_posts(id, last_seen_id)
if self.config['thread_unread_messages'].get() > 0:
iterator = limit(
iterator, self.config['thread_unread_messages'].get())
for post in iterator:
if not thread:
thread = Thread('%s.%s' %
(post.forum_id, post.topic_id))
message = self._post2message(thread, post)
if child:
message.children.append(child)
child.parent = message
if post.parent:
message.parent = Message(thread=thread, id=post.parent)
else:
thread.root = message
yield message
def set_message_read(self, message):
try:
last_seen_id = self.storage.get('seen', default={})[id2topic(
message.thread.id)]
except KeyError:
last_seen_id = 0
if message.id > last_seen_id:
self.storage.set('seen', id2topic(message.thread.id), message.id)
self.storage.save()
def fill_thread(self, thread, fields):
return self.get_thread(thread)
#### CapMessagesReply #########################################
def post_message(self, message):
assert message.thread
forum = 0
topic = 0
if message.thread:
try:
if '.' in message.thread.id:
forum, topic = [
int(i) for i in message.thread.id.split('.', 1)
]
else:
forum = int(message.thread.id)
except ValueError:
raise CantSendMessage(
'Thread ID must be in form "FORUM_ID[.TOPIC_ID]".')
with self.browser:
return self.browser.post_answer(forum, topic, message.title,
message.content)
OBJECTS = {Thread: fill_thread}
class CragrModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, CapProfile):
NAME = 'cragr'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.4'
DESCRIPTION = u'Crédit Agricole'
LICENSE = 'AGPLv3+'
website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({
'm.ca-alpesprovence.fr': u'Alpes Provence',
'm.ca-alsace-vosges.fr': u'Alsace-Vosges',
'm.ca-anjou-maine.fr': u'Anjou Maine',
'm.ca-aquitaine.fr': u'Aquitaine',
'm.ca-atlantique-vendee.fr': u'Atlantique Vendée',
'm.ca-briepicardie.fr': u'Brie Picardie',
'm.ca-cb.fr': u'Champagne Bourgogne',
'm.ca-centrefrance.fr': u'Centre France',
'm.ca-centreloire.fr': u'Centre Loire',
'm.ca-centreouest.fr': u'Centre Ouest',
'm.ca-centrest.fr': u'Centre Est',
'm.ca-charente-perigord.fr': u'Charente Périgord',
'm.ca-cmds.fr': u'Charente-Maritime Deux-Sèvres',
'm.ca-corse.fr': u'Corse',
'm.ca-cotesdarmor.fr': u'Côtes d\'Armor',
'm.ca-des-savoie.fr': u'Des Savoie',
'm.ca-finistere.fr': u'Finistere',
'm.ca-franchecomte.fr': u'Franche-Comté',
'm.ca-guadeloupe.fr': u'Guadeloupe',
'm.ca-illeetvilaine.fr': u'Ille-et-Vilaine',
'm.ca-languedoc.fr': u'Languedoc',
'm.ca-loirehauteloire.fr': u'Loire Haute Loire',
'm.ca-lorraine.fr': u'Lorraine',
'm.ca-martinique.fr': u'Martinique Guyane',
'm.ca-morbihan.fr': u'Morbihan',
'm.ca-nmp.fr': u'Nord Midi-Pyrénées',
'm.ca-nord-est.fr': u'Nord Est',
'm.ca-norddefrance.fr': u'Nord de France',
'm.ca-normandie-seine.fr': u'Normandie Seine',
'm.ca-normandie.fr': u'Normandie',
'm.ca-paris.fr': u'Ile-de-France',
'm.ca-pca.fr': u'Provence Côte d\'Azur',
'm.ca-reunion.fr': u'Réunion',
'm.ca-sudmed.fr': u'Sud Méditerranée',
'm.ca-sudrhonealpes.fr': u'Sud Rhône Alpes',
'm.ca-toulouse31.fr': u'Toulouse 31', # m.ca-toulousain.fr redirects here
'm.ca-tourainepoitou.fr': u'Tourraine Poitou',
'm.ca-valdefrance.fr': u'Val de France',
'm.lefil.com': u'Pyrénées Gascogne',
}.items())])
CONFIG = BackendConfig(Value('website', label=u'Région', choices=website_choices),
ValueBackendPassword('login', label=u'N° de compte', masked=False),
ValueBackendPassword('password', label=u'Code personnel', regexp=r'\d{6}'))
BROWSER = Cragr
COMPAT_DOMAINS = {
'm.lefil.com': 'm.ca-pyrenees-gascogne.fr',
}
def create_default_browser(self):
site_conf = self.config['website'].get()
site_conf = self.COMPAT_DOMAINS.get(site_conf, site_conf)
return self.create_browser(site_conf,
self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
return self.browser.get_accounts_list()
def get_account(self, _id):
return find_object(self.iter_accounts(), id=_id, error=AccountNotFound)
def _history_filter(self, account, coming):
today = date.today()
def to_date(obj):
if hasattr(obj, 'date'):
return obj.date()
return obj
for tr in self.browser.get_history(account):
tr_coming = to_date(tr.date) > today
if coming == tr_coming:
yield tr
def iter_history(self, account):
if account.type == Account.TYPE_CARD:
return self._history_filter(account, False)
return self.browser.get_history(account)
def iter_coming(self, account):
if account.type == Account.TYPE_CARD:
return self._history_filter(account, True)
return []
def iter_investment(self, account):
for inv in self.browser.iter_investment(account):
yield inv
def iter_contacts(self):
return self.browser.iter_advisor()
def get_profile(self):
if not hasattr(self.browser, 'get_profile'):
raise NotImplementedError()
return self.browser.get_profile()
def iter_transfer_recipients(self, account):
if not isinstance(account, Account):
account = self.get_account(account)
return self.browser.iter_transfer_recipients(account)
def init_transfer(self, transfer, **params):
def to_ascii(s):
return s.encode('ascii', errors='ignore').decode('ascii')
if transfer.label:
transfer.label = re.sub(r'[+!]', '', to_ascii(transfer.label[:33]))
return self.browser.init_transfer(transfer, **params)
def execute_transfer(self, transfer, **params):
return self.browser.execute_transfer(transfer, **params)
def new_recipient(self, recipient, **params):
return self.browser.new_recipient(recipient, **params)
class RedditModule(Module, CapImage, CapCollection, CapMessages):
NAME = 'reddit'
DESCRIPTION = u'reddit website'
MAINTAINER = u'Vincent A'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.6'
CONFIG = BackendConfig(
Value('subreddit',
label='Name of the sub-reddit',
regexp='[^/]+',
default='pics'), )
BROWSER = RedditBrowser
def create_default_browser(self):
return self.create_browser(self.config['subreddit'].get())
def get_file(self, _id):
raise NotImplementedError()
def get_image(self, id):
return self.browser.get_image(id)
def search_file(self, pattern, sortby=CapImage.SEARCH_RELEVANCE):
return self.browser.search_images(pattern, sortby, True)
def search_image(self,
pattern,
sortby=CapImage.SEARCH_RELEVANCE,
nsfw=False):
sorting = {
CapImage.SEARCH_RELEVANCE: 'relevance',
CapImage.SEARCH_RATING: 'top',
CapImage.SEARCH_VIEWS: 'top', # not implemented
CapImage.SEARCH_DATE: 'new',
}
sortby = sorting[sortby]
return self.browser.search_images(pattern, sortby, nsfw)
def iter_threads(self):
return self.browser.iter_threads()
def get_thread(self, id):
return self.browser.get_thread(id)
def iter_resources(self, objs, split_path):
for k in self.RESOURCES:
if len(k) == len(split_path) and all(
a is None or a == b for a, b in zip(k, split_path)):
f = self.RESOURCES[k]
return f(self, objs, *split_path)
RESOURCES = {}
@register_resources_handler(RESOURCES)
def iter_resources_root(self, objs):
return [
Collection(['hot'], 'Hot threads'),
Collection(['new'], 'New threads'),
Collection(['rising'], 'Rising threads'),
Collection(['controversial'], 'Controversial threads'),
Collection(['top'], 'Top threads'),
]
@register_resources_handler(RESOURCES, None)
def iter_resources_dir(self, objs, key):
if key == 'hot':
key = ''
if Thread in objs:
return self.iter_threads(cat=key)
if BaseImage in objs:
return self.browser.iter_images(cat=key)
return []
def fill_data(self, obj, fields):
if 'thumbnail' in fields and not obj.thumbnail.data:
obj.thumbnail.data = self.browser.open(obj.thumbnail.url).content
if 'data' in fields:
obj.data = self.browser.open(obj.url).content
def fill_thread(self, obj, fields):
if 'root' in fields:
self.browser.fill_thread(obj)
OBJECTS = {
BaseImage: fill_data,
Thumbnail: fill_data,
Thread: fill_thread,
}
class TinderModule(Module, CapMessages, CapMessagesPost, CapDating):
NAME = 'tinder'
DESCRIPTION = u'Tinder dating mobile application'
MAINTAINER = u'Roger Philibert'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.4'
CONFIG = BackendConfig(
Value('username', label='Facebook email'),
ValueBackendPassword('password', label='Facebook password'),
Value('location',
label='Location (example: 49.6008457,6.129709)',
default=''))
BROWSER = TinderBrowser
STORAGE = {
'contacts': {},
}
def create_default_browser(self):
facebook = self.create_browser(klass=FacebookBrowser)
facebook.login(self.config['username'].get(),
self.config['password'].get())
return self.create_browser(facebook, self.config['location'].get())
# ---- CapDating methods -----------------------
def init_optimizations(self):
self.add_optimization(
'PROFILE_WALKER',
ProfilesWalker(self.weboob.scheduler, self.storage, self.browser))
# ---- CapMessages methods ---------------------
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def iter_threads(self):
for thread in self.browser.get_threads():
if 'person' not in thread:
# The account has been removed, probably because it was a
# spammer.
continue
t = Thread(thread['_id'])
t.flags = Thread.IS_DISCUSSION
t.title = u'Discussion with %s' % thread['person']['name']
contact = self.storage.get('contacts',
t.id,
default={'lastmsg': 0})
birthday = parse_date(thread['person']['birth_date']).date()
signature = u'Age: %d (%s)' % (
(datetime.date.today() - birthday).days / 365.25, birthday)
signature += u'\nLast ping: %s' % parse_date(
thread['person']['ping_time']).strftime('%Y-%m-%d %H:%M:%S')
signature += u'\nPhotos:\n\t%s' % '\n\t'.join(
[photo['url'] for photo in thread['person']['photos']])
signature += u'\n\n%s' % thread['person'].get('bio', '')
t.root = Message(
thread=t,
id=1,
title=t.title,
sender=unicode(thread['person']['name']),
receivers=[self.browser.my_name],
date=parse_date(thread['created_date']),
content=u'Match!',
children=[],
signature=signature,
flags=Message.IS_UNREAD if int(contact['lastmsg']) < 1 else 0)
parent = t.root
for msg in thread['messages']:
flags = 0
if int(contact['lastmsg']) < msg['timestamp']:
flags = Message.IS_UNREAD
msg = Message(
thread=t,
id=msg['timestamp'],
title=t.title,
sender=unicode(
self.browser.my_name if msg['from'] ==
self.browser.my_id else thread['person']['name']),
receivers=[
unicode(self.browser.my_name if msg['to'] == self.
browser.my_id else thread['person']['name'])
],
date=parse_date(msg['sent_date']),
content=unicode(msg['message']),
children=[],
parent=parent,
signature=signature
if msg['to'] == self.browser.my_id else u'',
flags=flags)
parent.children.append(msg)
parent = msg
yield t
def get_thread(self, _id):
for t in self.iter_threads():
if t.id == _id:
return t
def iter_unread_messages(self):
for thread in self.iter_threads():
for message in thread.iter_all_messages():
if message.flags & message.IS_UNREAD:
yield message
def set_message_read(self, message):
contact = self.storage.get('contacts',
message.thread.id,
default={'lastmsg': 0})
if int(contact['lastmsg']) < int(message.id):
contact['lastmsg'] = int(message.id)
self.storage.set('contacts', message.thread.id, contact)
self.storage.save()
# ---- CapMessagesPost methods ---------------------
def post_message(self, message):
self.browser.post_message(message.thread.id, message.content)
OBJECTS = {
Thread: fill_thread,
}
class AferModule(Module, CapBankWealth):
NAME = 'afer'
DESCRIPTION = u'afer website'
MAINTAINER = u'James GALT'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.4'
BROWSER = AferBrowser
CONFIG = BackendConfig(
ValueBackendPassword('login',
label='Username',
regexp='[A-z]\d+',
masked=False),
ValueBackendPassword('password', label=u"mdp", regexp='\d+'))
def create_default_browser(self):
return self.create_browser(self.config['login'].get(),
self.config['password'].get())
def get_account(self, id):
"""
Get an account from its ID.
:param id: ID of the account
:type id: :class:`str`
:rtype: :class:`Account`
:raises: :class:`AccountNotFound`
"""
return find_object(self.iter_accounts(), id=id, error=AccountNotFound)
def iter_accounts(self):
"""
Iter accounts.
:rtype: iter[:class:`Account`]
"""
return self.browser.iter_accounts()
def iter_coming(self, account):
"""
Iter coming transactions on a specific account.
:param account: account to get coming transactions
:type account: :class:`Account`
:rtype: iter[:class:`Transaction`]
:raises: :class:`AccountNotFound`
"""
raise NotImplementedError()
def iter_history(self, account):
"""
Iter history of transactions on a specific account.
:param account: account to get history
:type account: :class:`Account`
:rtype: iter[:class:`Transaction`]
:raises: :class:`AccountNotFound`
"""
return self.browser.iter_history(account)
def iter_investment(self, account):
"""
Iter investment of a market account
:param account: account to get investments
:type account: :class:`Account`
:rtype: iter[:class:`Investment`]
:raises: :class:`AccountNotFound`
"""
return self.browser.iter_investments(account)
class INGModule(Module, CapBank, CapBill):
NAME = 'ing'
MAINTAINER = u'Florent Fourcot'
EMAIL = '*****@*****.**'
VERSION = '1.2'
LICENSE = 'AGPLv3+'
DESCRIPTION = 'ING Direct'
CONFIG = BackendConfig(
ValueBackendPassword('login', label=u'Numéro client', masked=False),
ValueBackendPassword('password',
label='Code secret',
regexp='^(\d{6}|)$'),
ValueBackendPassword('birthday',
label='Date de naissance',
regexp='^(\d{2}[/-]?\d{2}[/-]?\d{4}|)$',
masked=False))
BROWSER = IngBrowser
def create_default_browser(self):
return self.create_browser(self.config['login'].get(),
self.config['password'].get(),
birthday=self.config['birthday'].get())
def iter_resources(self, objs, split_path):
if Account in objs:
self._restrict_level(split_path)
return self.iter_accounts()
if Subscription in objs:
self._restrict_level(split_path)
return self.iter_subscription()
def iter_accounts(self):
return self.browser.get_accounts_list()
def get_account(self, _id):
return self.browser.get_account(_id)
def iter_history(self, account):
if not isinstance(account, Account):
account = self.get_account(account)
return self.browser.get_history(account)
def iter_transfer_recipients(self, account):
if not isinstance(account, Account):
account = self.get_account(account)
return self.browser.get_recipients(account)
def transfer(self, account, recipient, amount, reason):
if not reason:
raise UserError(
'Reason is mandatory to do a transfer on ING website')
if not isinstance(account, Account):
account = self.get_account(account)
if not isinstance(recipient, Recipient):
# Remove weboob identifier prefix (LA-, CC-...)
if "-" in recipient:
recipient = recipient.split('-')[1]
return self.browser.transfer(account, recipient, amount, reason)
def iter_investment(self, account):
if not isinstance(account, Account):
account = self.get_account(account)
return self.browser.get_investments(account)
def iter_coming(self, account):
if not isinstance(account, Account):
account = self.get_account(account)
return self.browser.get_coming(account)
def iter_subscription(self):
return self.browser.get_subscriptions()
def get_subscription(self, _id):
return find_object(self.browser.get_subscriptions(),
id=_id,
error=SubscriptionNotFound)
def get_bill(self, _id):
subscription = self.get_subscription(_id.split('-')[0])
return find_object(self.browser.get_bills(subscription),
id=_id,
error=BillNotFound)
def iter_bills(self, subscription):
if not isinstance(subscription, Subscription):
subscription = self.get_subscription(subscription)
return self.browser.get_bills(subscription)
def download_bill(self, bill):
if not isinstance(bill, Bill):
bill = self.get_bill(bill)
self.browser.predownload(bill)
assert (
self.browser.response.headers['content-type'] == "application/pdf")
return self.browser.response.content
class HappnModule(Module, CapMessages, CapMessagesPost, CapDating, CapContact):
NAME = 'happn'
DESCRIPTION = u'Happn dating mobile application'
MAINTAINER = u'Roger Philibert'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.6'
CONFIG = BackendConfig(
Value('username', label='Facebook email'),
ValueBackendPassword('password', label='Facebook password'),
Value('location', label='Location (example: 49.6008457,6.129709)'),
Value('distance',
label='Distance (in km) around your location you\'re walking'))
BROWSER = HappnBrowser
STORAGE = {
'contacts': {},
}
def create_default_browser(self):
facebook = self.create_browser(klass=FacebookBrowser)
facebook.login(self.config['username'].get(),
self.config['password'].get())
return self.create_browser(facebook)
# ---- CapDating methods -----------------------
def init_optimizations(self):
self.add_optimization(
'PROFILE_WALKER',
ProfilesWalker(self.weboob.scheduler, self.storage, self.browser,
self.config['location'].get(),
self.config['distance'].get()))
# ---- CapMessages methods ---------------------
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def iter_threads(self):
for thread in self.browser.get_threads():
t = Thread(thread['id'])
t.flags = Thread.IS_DISCUSSION
for user in thread['participants']:
if user['user']['id'] != self.browser.my_id:
t.title = u'Discussion with %s' % user['user'][
'display_name']
t.date = parse_date(thread['modification_date'])
yield t
def get_thread(self, thread):
if not isinstance(thread, Thread):
thread = Thread(thread)
thread.flags = Thread.IS_DISCUSSION
info = self.browser.get_thread(thread.id)
for user in info['participants']:
if user['user']['fb_id'] is not None:
user['user']['fb'] = self.browser.get_facebook(
user['user']['fb_id'])
if user['user']['id'] == self.browser.my_id:
me = HappnContact(user['user'])
else:
other = HappnContact(user['user'])
thread.title = u'Discussion with %s' % other.name
contact = self.storage.get(
'contacts',
thread.id,
default={'lastmsg_date': '1970-01-01T01:01:01+00:00'})
child = None
for msg in info['messages']:
flags = 0
if parse_date(contact['lastmsg_date']) < parse_date(
msg['creation_date']):
flags = Message.IS_UNREAD
if msg['sender']['id'] == me.id:
sender = me
receiver = other
else:
sender = other
receiver = me
msg = Message(thread=thread,
id=msg['id'],
title=thread.title,
sender=sender.name,
receivers=[receiver.name],
date=parse_date(msg['creation_date']),
content=msg['message'],
children=[],
parent=None,
signature=sender.get_text(),
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
thread.root = child
return thread
def iter_unread_messages(self):
for thread in self.iter_threads():
thread = self.get_thread(thread)
for message in thread.iter_all_messages():
if message.flags & message.IS_UNREAD:
yield message
def set_message_read(self, message):
contact = self.storage.get(
'contacts',
message.thread.id,
default={'lastmsg_date': '1970-01-01T01:01:01+00:00'})
if parse_date(contact['lastmsg_date']) < message.date:
contact['lastmsg_date'] = str(message.date)
self.storage.set('contacts', message.thread.id, contact)
self.storage.save()
# ---- CapMessagesPost methods ---------------------
def post_message(self, message):
self.browser.post_message(message.thread.id, message.content)
# ---- CapContact methods ---------------------
def get_contact(self, contact_id):
if isinstance(contact_id, Contact):
contact_id = contact_id.id
info = self.browser.get_contact(contact_id)
return HappnContact(info)
OBJECTS = {
Thread: fill_thread,
}
class ArretSurImagesModule(Module, CapVideo, CapCollection):
NAME = 'arretsurimages'
DESCRIPTION = u'arretsurimages website'
MAINTAINER = u'franek'
EMAIL = '*****@*****.**'
VERSION = '1.3'
CONFIG = BackendConfig(
ValueBackendPassword('login', label='email', masked=False),
ValueBackendPassword('password', label='Password'))
BROWSER = ArretSurImagesBrowser
def create_default_browser(self):
return self.create_browser(self.config['login'].get(),
self.config['password'].get(),
get_home=False)
def search_videos(self,
pattern,
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False):
with self.browser:
return self.browser.search_videos(pattern)
# raise UserError('Search does not work on ASI website, use ls latest command')
def get_video(self, _id):
if _id.startswith('http://') and not _id.startswith(
'http://www.arretsurimages.net'):
return None
with self.browser:
return self.browser.get_video(_id)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(
ArretSurImagesVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(
video.thumbnail.url)
return video
def iter_resources(self, objs, split_path):
if BaseVideo in objs:
collection = self.get_collection(objs, split_path)
if collection.path_level == 0:
yield self.get_collection(objs, [u'latest'])
if collection.split_path == [u'latest']:
for video in self.browser.latest_videos():
yield video
def validate_collection(self, objs, collection):
if collection.path_level == 0:
return
if BaseVideo in objs and collection.split_path == [u'latest']:
collection.title = u'Latest ArretSurImages videos'
return
raise CollectionNotFound(collection.split_path)
OBJECTS = {ArretSurImagesVideo: fill_video}
class AmazonModule(Module, CapShop, CapDocument):
NAME = 'amazon'
MAINTAINER = u'Oleg Plakhotniuk'
EMAIL = '*****@*****.**'
VERSION = '1.2'
LICENSE = 'AGPLv3+'
DESCRIPTION = u'Amazon'
website_choices = OrderedDict([
(k, u'%s (%s)' % (v, k))
for k, v in sorted({
'www.amazon.com': u'Amazon.com',
'www.amazon.fr': u'Amazon France',
}.iteritems())
])
BROWSERS = {
'www.amazon.com': Amazon,
'www.amazon.fr': AmazonFR,
}
CONFIG = BackendConfig(
Value('website',
label=u'Website',
choices=website_choices,
default='www.amazon.com'),
ValueBackendPassword('email', label='Username', masked=False),
ValueBackendPassword('password', label='Password'))
def create_default_browser(self):
self.BROWSER = self.BROWSERS[self.config['website'].get()]
return self.create_browser(self.config['email'].get(),
self.config['password'].get())
def get_currency(self):
return self.browser.get_currency()
def get_order(self, id_):
return self.browser.get_order(id_)
def iter_orders(self):
return self.browser.iter_orders()
def iter_payments(self, order):
if not isinstance(order, Order):
order = self.get_order(order)
return self.browser.iter_payments(order)
def iter_items(self, order):
if not isinstance(order, Order):
order = self.get_order(order)
return self.browser.iter_items(order)
def iter_resources(self, objs, split_path):
if Order in objs:
self._restrict_level(split_path)
return self.iter_orders()
if Subscription in objs:
self._restrict_level(split_path)
return self.iter_subscription()
def iter_subscription(self):
return self.browser.get_subscription_list()
def get_subscription(self, _id):
return find_object(self.iter_subscription(),
id=_id,
error=SubscriptionNotFound)
def get_document(self, _id):
subid = _id.split('.')[0]
subscription = self.get_subscription(subid)
return find_object(self.iter_documents(subscription),
id=_id,
error=DocumentNotFound)
def iter_documents(self, subscription):
if not isinstance(subscription, Subscription):
subscription = self.get_subscription(subscription)
return self.browser.iter_documents(subscription)
def download_document(self, bill):
if not isinstance(bill, Bill):
bill = self.get_document(bill)
if bill._url:
return self.browser.open(bill._url).content
return None
from weboob.tools.value import Value, ValueBackendPassword
class MockModule(object):
def __init__(self, name, description, config, backend="manual"):
self.name = name
self.description = description
self.config = config
self.backend = backend
# Officially deprecated modules. They are listed for backwards compatibility and to allow the user to
# look at their past transactions.
DEPRECATED_MODULES = [
MockModule('wellsfargo', 'Wells Fargo',
BackendConfig(Value('login'),
ValueBackendPassword('password'))),
MockModule('citelis', 'City Bank',
BackendConfig(Value('login'),
ValueBackendPassword('password'))),
MockModule('alloresto', 'Allo Resto',
BackendConfig(Value('login'),
ValueBackendPassword('password'))),
]
# The known modules to be ignored either because they are only called by another module,
# or because they are deprecated.
IGNORE_MODULE_LIST = ['s2e', 'linebourse', 'groupama'
] + [m.name for m in DEPRECATED_MODULES]
MANUAL_MODULES = [
MockModule('manual',
class GithubModule(Module, CapBugTracker):
NAME = 'github'
DESCRIPTION = u'GitHub issues tracking'
MAINTAINER = u'Vincent A'
EMAIL = '*****@*****.**'
LICENSE = 'AGPLv3+'
VERSION = '1.2'
CONFIG = BackendConfig(
Value('username', label='Username', default=''),
ValueBackendPassword('password', label='Password', default=''))
BROWSER = GithubBrowser
def create_default_browser(self):
username = self.config['username'].get()
if username:
password = self.config['password'].get()
else:
password = None
return self.create_browser(username, password)
def get_project(self, _id):
d = self.browser.get_project(_id)
project = Project(_id, d['name'])
project.members = list(self._iter_members(project.id))
project.statuses = list(STATUSES.values())
project.categories = []
project.versions = list(self._iter_versions(project.id))
return project
def get_issue(self, _id):
project_id, issue_number = self._extract_issue_id(_id)
project = self.get_project(project_id)
d = self.browser.get_issue(project_id, issue_number)
issue = self._make_issue(d, project)
if d['has_comments']:
self._fetch_comments(issue)
return issue
def iter_issues(self, query):
if ((query.assignee, query.author, query.status,
query.title) == (None, None, None, None)):
it = self.browser.iter_project_issues(query.project)
else:
it = self.browser.iter_issues(query)
project = self.get_project(query.project)
for d in it:
issue = self._make_issue(d, project)
yield issue
def create_issue(self, project_id):
issue = Issue(0)
issue.project = self.get_project(project_id)
return issue
def post_issue(self, issue):
assert not issue.attachments
if issue.id and issue.id != '0':
_, issue_number = self._extract_issue_id(issue.id)
self.browser.edit_issue(issue, issue_number)
else:
self.browser.post_issue(issue)
def update_issue(self, issue_id, update):
assert not update.attachments
self.browser.post_comment(issue_id, update.message)
# iter_projects, remove_issue are impossible
def _iter_members(self, project_id):
for d in self.browser.iter_members(project_id):
yield User(d['id'], d['name'])
def _iter_versions(self, project_id):
for d in self.browser.iter_milestones(project_id):
yield Version(d['id'], d['name'])
def _make_issue(self, d, project):
_id = self._build_issue_id(project.id, d['number'])
issue = Issue(_id)
issue.url = d['url']
issue.project = project
issue.title = d['title']
issue.body = d['body']
issue.creation = d['creation']
issue.updated = d['updated']
issue.author = project.find_user(d['author'], None)
if not issue.author:
# may duplicate users
issue.author = User(d['author'], d['author'])
issue.status = STATUSES[d['status']]
if d['assignee']:
issue.assignee = project.find_user(d['assignee'], None)
else:
issue.assignee = None
if d['version']:
issue.version = project.find_version(d['version'], None)
else:
issue.version = None
issue.category = None
issue.attachments = [
self._make_attachment(dattach) for dattach in d['attachments']
]
return issue
def _fetch_comments(self, issue):
project_id, issue_number = self._extract_issue_id(issue.id)
if not issue.history:
issue.history = []
issue.history += [
self._make_comment(dcomment, issue.project) for dcomment in
self.browser.iter_comments(project_id, issue_number)
]
def _make_attachment(self, d):
a = Attachment(d['url'])
a.url = d['url']
a.filename = d['filename']
return a
def _make_comment(self, d, project):
u = Update(d['id'])
u.message = d['message']
u.author = project.find_user(d['author'], None)
if not u.author:
# may duplicate users
u.author = User(d['author'], d['author'])
u.date = d['date']
u.changes = []
u.attachments = [
self._make_attachment(dattach) for dattach in d['attachments']
]
return u
@staticmethod
def _extract_issue_id(_id):
return _id.rsplit('/', 1)
@staticmethod
def _build_issue_id(project_id, issue_number):
return '%s/%s' % (project_id, issue_number)
class CreditDuNordModule(Module, CapBankWealth, CapProfile):
NAME = 'creditdunord'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '1.6'
DESCRIPTION = u'Crédit du Nord, Banque Courtois, Kolb, Nuger, Laydernier, Tarneaud, Société Marseillaise de Crédit'
LICENSE = 'LGPLv3+'
website_choices = OrderedDict([
(k, u'%s (%s)' % (v, k))
for k, v in sorted({
'www.credit-du-nord.fr': u'Crédit du Nord',
'www.banque-courtois.fr': u'Banque Courtois',
'www.banque-kolb.fr': u'Banque Kolb',
'www.banque-laydernier.fr': u'Banque Laydernier',
'www.banque-nuger.fr': u'Banque Nuger',
'www.banque-rhone-alpes.fr': u'Banque Rhône-Alpes',
'www.tarneaud.fr': u'Tarneaud',
'www.smc.fr': u'Société Marseillaise de Crédit',
}.items(),
key=lambda k_v: (k_v[1], k_v[0]))
])
CONFIG = BackendConfig(
Value('website',
label='Banque',
choices=website_choices,
default='www.credit-du-nord.fr'),
ValueBackendPassword('login', label='Identifiant', masked=False),
ValueBackendPassword('password', label='Code confidentiel'))
BROWSER = CreditDuNordBrowser
def create_default_browser(self):
return self.create_browser(self.config['website'].get(),
self.config['login'].get(),
self.config['password'].get(),
weboob=self.weboob)
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
account = self.browser.get_account(_id)
if account:
return account
raise AccountNotFound()
def get_account_for_history(self, _id):
account = self.browser.get_account_for_history(_id)
if account:
return account
raise AccountNotFound()
def iter_history(self, account):
account = self.get_account_for_history(account.id)
for tr in self.browser.get_history(account):
if not tr._is_coming:
yield tr
def iter_coming(self, account):
account = self.get_account_for_history(account.id)
for tr in self.browser.get_history(account, coming=True):
if tr._is_coming:
yield tr
def iter_investment(self, account):
account = self.get_account(account.id)
return self.browser.get_investment(account)
def get_profile(self):
return self.browser.get_profile()
class BNPorcBackend(BaseBackend, ICapBank, ICapMessages):
NAME = 'bnporc'
MAINTAINER = u'Romain Bignon'
EMAIL = '*****@*****.**'
VERSION = '0.e'
LICENSE = 'AGPLv3+'
DESCRIPTION = 'BNP Paribas French bank website'
CONFIG = BackendConfig(
ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password',
regexp='^(\d{6}|)$'),
ValueBackendPassword(
'rotating_password',
default='',
label=
'Password to set when the allowed uses are exhausted (6 digits)',
regexp='^(\d{6}|)$'))
BROWSER = BNPorc
STORAGE = {'seen': []}
# Store the messages *list* for this duration
CACHE_THREADS = timedelta(seconds=3 * 60 * 60)
def __init__(self, *args, **kwargs):
BaseBackend.__init__(self, *args, **kwargs)
self._threads = None
self._threads_age = datetime.utcnow()
def create_default_browser(self):
if self.config['rotating_password'].get().isdigit() and len(
self.config['rotating_password'].get()) == 6:
rotating_password = self.config['rotating_password'].get()
else:
rotating_password = None
return self.create_browser(
self.config['login'].get(),
self.config['password'].get(),
password_changed_cb=self._password_changed_cb,
rotating_password=rotating_password)
def _password_changed_cb(self, old, new):
self.config['password'].set(new)
self.config['rotating_password'].set(old)
self.config.save()
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
if not _id.isdigit():
raise AccountNotFound()
with self.browser:
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
with self.browser:
return self.browser.iter_history(account._link_id)
def iter_coming(self, account):
with self.browser:
return self.browser.iter_coming_operations(account._link_id)
def iter_transfer_recipients(self, ignored):
for account in self.browser.get_transfer_accounts().itervalues():
recipient = Recipient()
recipient.id = account.id
recipient.label = account.label
yield recipient
def transfer(self, account, to, amount, reason=None):
if isinstance(account, Account):
account = account.id
try:
assert account.isdigit()
assert to.isdigit()
amount = Decimal(amount)
except (AssertionError, ValueError):
raise AccountNotFound()
with self.browser:
return self.browser.transfer(account, to, amount, reason)
def iter_threads(self, cache=False):
"""
If cache is False, always fetch the threads from the website.
"""
old = self._threads_age < datetime.utcnow() - self.CACHE_THREADS
threads = self._threads
if not cache or threads is None or old:
with self.browser:
threads = list(self.browser.iter_threads())
# the website is stupid and does not have the messages in the proper order
threads = sorted(threads, key=lambda t: t.date, reverse=True)
self._threads = threads
seen = self.storage.get('seen', default=[])
for thread in threads:
if thread.id not in seen:
thread.root.flags |= thread.root.IS_UNREAD
else:
thread.root.flags &= ~thread.root.IS_UNREAD
yield thread
def fill_thread(self, thread, fields=None):
if fields is None or 'root' in fields:
return self.get_thread(thread)
def get_thread(self, _id):
if isinstance(_id, Thread):
thread = _id
_id = thread.id
else:
thread = Thread(_id)
with self.browser:
thread = self.browser.get_thread(thread)
return thread
def iter_unread_messages(self):
threads = list(self.iter_threads(cache=True))
for thread in threads:
if thread.root.flags & thread.root.IS_UNREAD:
thread = self.fillobj(thread) or thread
yield thread.root
def set_message_read(self, message):
self.storage.get('seen', default=[]).append(message.thread.id)
self.storage.save()
OBJECTS = {Thread: fill_thread}
