class MediawikiModule(Module, CapContent, CapImage): NAME = 'mediawiki' MAINTAINER = u'Clément Schreiner' EMAIL = '*****@*****.**' VERSION = '1.2' LICENSE = 'AGPLv3+' DESCRIPTION = 'Wikis running MediaWiki, like Wikipedia' CONFIG = BackendConfig(Value('url', label='URL of the Mediawiki website', default='http://en.wikipedia.org/', regexp='https?://.*'), Value('apiurl', label='URL of the Mediawiki website\'s API', default='http://en.wikipedia.org/w/api.php', regexp='https?://.*'), Value('username', label='Login', default=''), ValueBackendPassword('password', label='Password', default='')) BROWSER = MediawikiBrowser def create_default_browser(self): username = self.config['username'].get() if len(username) > 0: password = self.config['password'].get() else: password = '' return self.create_browser(self.config['url'].get(), self.config['apiurl'].get(), username, password) def get_content(self, _id, revision=None): _id = _id.replace(' ', '_') content = Content(_id) page = _id rev = revision.id if revision else None data = self.browser.get_wiki_source(page, rev) content.content = data return content def iter_revisions(self, _id): return self.browser.iter_wiki_revisions(_id) def push_content(self, content, message=None, minor=False): self.browser.set_wiki_source(content, message, minor) def get_content_preview(self, content): return self.browser.get_wiki_preview(content) def _make_image(self, info): img = WikiImage(info['title']) if 'thumbnail' in info: thumb = Thumbnail(info['thumbnail']) img.thumbnail = thumb if 'original' in info: img.url = info['original'] img._canonical_url = info['canonicalurl'] img.size = info['size'] return img def search_file(self, pattern, sortby=CapFile.SEARCH_RELEVANCE): for info in self.browser.search_file(pattern): yield self._make_image(info) def get_image(self, _id): _id = _id.replace(' ', '_') info = self.browser.get_image(_id) return self._make_image(info) def fill_img(self, obj, fields): if set(fields) & set(('url', 'thumbnail')): new = self.get_image(obj.id) if not obj.url: obj.url = new.url if not obj.thumbnail: obj.thumbnail = new.thumbnail if 'data' in fields: obj.data = self.browser.open(obj.url).content return obj OBJECTS = {BaseImage: fill_img, Thumbnail: fill_img}
class FourChanBackend(BaseBackend, ICapMessages): NAME = 'fourchan' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '0.h' LICENSE = 'AGPLv3+' DESCRIPTION = '4chan image board' CONFIG = BackendConfig(Value('boards', label='Boards to fetch')) STORAGE = {'boards': {}} BROWSER = FourChan def _splitid(self, id): return id.split('.', 1) def get_thread(self, id): thread = None if isinstance(id, Thread): thread = id id = thread.id if not '.' in id: self.logger.warning('Malformated ID (%s)' % id) return board, thread_id = self._splitid(id) with self.browser: _thread = self.browser.get_thread(board, thread_id) flags = 0 if not _thread.id in self.storage.get('boards', board, default={}): flags |= Message.IS_UNREAD if not thread: thread = Thread(id) thread.title = _thread.filename thread.root = Message( thread=thread, id=0, # root message title=_thread.filename, sender=_thread.author, receivers=None, date=_thread.datetime, parent=None, content=_thread.text, signature=None, children=[], flags=flags | Message.IS_HTML) for comment in _thread.comments: flags = 0 if not comment.id in self.storage.get( 'boards', board, _thread.id, default=[]): flags |= Message.IS_UNREAD m = Message(thread=thread, id=comment.id, title=_thread.filename, sender=comment.author, receivers=None, date=comment.datetime, parent=thread.root, content=comment.text, signature=None, children=None, flags=flags | Message.IS_HTML) thread.root.children.append(m) return thread def iter_threads(self): for board in self.config['boards'].get().split(' '): with self.browser: threads = self.browser.get_threads(board) for thread in threads: t = Thread('%s.%s' % (board, thread.id)) t.title = thread.filename yield t def iter_unread_messages(self): for thread in self.iter_threads(): self.fill_thread(thread, 'root') for m in thread.iter_all_messages(): if m.flags & Message.IS_UNREAD: yield m def set_message_read(self, message): board, thread_id = self._splitid(message.thread.id) self.storage.set( 'boards', board, thread_id, self.storage.get('boards', board, thread_id, default=[]) + [message.id]) self.storage.save() def fill_thread(self, thread, fields): return self.get_thread(thread) OBJECTS = {Thread: fill_thread}
class DailymotionModule(Module, CapVideo, CapCollection): NAME = 'dailymotion' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.6' DESCRIPTION = 'Dailymotion video streaming website' LICENSE = 'AGPLv3+' BROWSER = DailymotionBrowser resolution_choice = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({ u'480': u'480p', u'240': u'240p', u'380': u'380p', u'720': u'720p', u'1080': u'1080p' }.iteritems())]) format_choice = [u'm3u8', u'mp4'] CONFIG = BackendConfig( Value('resolution', label=u'Resolution', choices=resolution_choice), Value('format', label=u'Format', choices=format_choice)) SORTBY = ['relevance', 'rated', 'visited', None] def create_default_browser(self): resolution = self.config['resolution'].get() format = self.config['format'].get() return self.create_browser(resolution=resolution, format=format) def get_video(self, _id): m = re.match('http://[w\.]*dailymotion\.com/video/(.*)', _id) if m: _id = m.group(1) if not _id.startswith('http'): return self.browser.get_video(_id) def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): return self.browser.search_videos(pattern, self.SORTBY[sortby]) def fill_video(self, video, fields): if fields != ['thumbnail']: # if we don't want only the thumbnail, we probably want also every fields video = self.browser.get_video(video.id, video) if 'thumbnail' in fields and video.thumbnail: video.thumbnail.data = self.browser.open( video.thumbnail.url).content return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest']) if collection.split_path == [u'latest']: for video in self.browser.latest_videos(): yield video def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and collection.split_path == [u'latest']: collection.title = u'Latest Dailymotion videos' return raise CollectionNotFound(collection.split_path) OBJECTS = {BaseVideo: fill_video}
class CragrModule(Module, CapBank): NAME = 'cragr' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.1' DESCRIPTION = u'Crédit Agricole' LICENSE = 'AGPLv3+' website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'm.ca-alpesprovence.fr': u'Alpes Provence', 'm.ca-alsace-vosges.fr': u'Alsace-Vosges', 'm.ca-anjou-maine.fr': u'Anjou Maine', 'm.ca-aquitaine.fr': u'Aquitaine', 'm.ca-atlantique-vendee.fr': u'Atlantique Vendée', 'm.ca-briepicardie.fr': u'Brie Picardie', 'm.ca-cb.fr': u'Champagne Bourgogne', 'm.ca-centrefrance.fr': u'Centre France', 'm.ca-centreloire.fr': u'Centre Loire', 'm.ca-centreouest.fr': u'Centre Ouest', 'm.ca-centrest.fr': u'Centre Est', 'm.ca-charente-perigord.fr': u'Charente Périgord', 'm.ca-cmds.fr': u'Charente-Maritime Deux-Sèvres', 'm.ca-corse.fr': u'Corse', 'm.ca-cotesdarmor.fr': u'Côtes d\'Armor', 'm.ca-des-savoie.fr': u'Des Savoie', 'm.ca-finistere.fr': u'Finistere', 'm.ca-franchecomte.fr': u'Franche-Comté', 'm.ca-guadeloupe.fr': u'Guadeloupe', 'm.ca-illeetvilaine.fr': u'Ille-et-Vilaine', 'm.ca-languedoc.fr': u'Languedoc', 'm.ca-loirehauteloire.fr': u'Loire Haute Loire', 'm.ca-lorraine.fr': u'Lorraine', 'm.ca-martinique.fr': u'Martinique Guyane', 'm.ca-morbihan.fr': u'Morbihan', 'm.ca-nmp.fr': u'Nord Midi-Pyrénées', 'm.ca-nord-est.fr': u'Nord Est', 'm.ca-norddefrance.fr': u'Nord de France', 'm.ca-normandie-seine.fr': u'Normandie Seine', 'm.ca-normandie.fr': u'Normandie', 'm.ca-paris.fr': u'Ile-de-France', 'm.ca-pca.fr': u'Provence Côte d\'Azur', 'm.ca-reunion.fr': u'Réunion', 'm.ca-sudmed.fr': u'Sud Méditerranée', 'm.ca-sudrhonealpes.fr': u'Sud Rhône Alpes', 'm.ca-toulouse31.fr': u'Toulouse 31', # m.ca-toulousain.fr redirects here 'm.ca-tourainepoitou.fr': u'Tourraine Poitou', 'm.ca-valdefrance.fr': u'Val de France', 'm.lefil.com': u'Pyrénées Gascogne', }.iteritems())]) CONFIG = BackendConfig(Value('website', label=u'Region', choices=website_choices), ValueBackendPassword('login', label=u'N° de compte', masked=False), ValueBackendPassword('password', label=u'Code personnel', regexp=r'\d{6}')) BROWSER = Cragr def create_default_browser(self): try: return self.create_browser(self.config['website'].get(), self.config['login'].get(), self.config['password'].get()) except Cragr.WebsiteNotSupported: self.logger.debug('falling-back on mobile version') self.BROWSER = CragrMobile return self.create_browser(self.config['website'].get(), self.config['login'].get(), self.config['password'].get()) def iter_accounts(self): return self.browser.get_accounts_list() def get_account(self, _id): account = self.browser.get_account(_id) if account: return account else: raise AccountNotFound() def iter_history(self, account): return self.browser.get_history(account) def iter_transfer_recipients(self, ignored): for account in self.browser.get_transfer_accounts().itervalues(): recipient = Recipient() recipient.id = account.id recipient.label = account.label yield recipient def transfer(self, account, to, amount, reason=None): return self.browser.do_transfer(account, to, amount, reason)
class ArteBackend(BaseBackend, ICapVideo, ICapCollection): NAME = 'arte' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '0.e' DESCRIPTION = 'Arte French and German TV' LICENSE = 'AGPLv3+' CONFIG = BackendConfig( Value('lang', label='Lang of videos', choices={ 'fr': 'French', 'de': 'Deutsch', 'en': 'English' }, default='fr'), Value('quality', label='Quality of videos', choices=['hd', 'sd'], default='hd')) BROWSER = ArteBrowser def create_default_browser(self): return self.create_browser(lang=self.config['lang'].get(), quality=self.config['quality'].get()) def get_video(self, _id): with self.browser: return self.browser.get_video(_id) def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None): with self.browser: return self.browser.search_videos(pattern) def fill_video(self, video, fields): if fields != ['thumbnail']: # if we don't want only the thumbnail, we probably want also every fields with self.browser: video = self.browser.get_video(ArteVideo.id2url(video.id), video) if 'thumbnail' in fields and video.thumbnail: with self.browser: video.thumbnail.data = self.browser.readurl( video.thumbnail.url) return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest']) if collection.split_path == [u'latest']: for video in self.browser.latest_videos(): yield video def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and collection.split_path == [u'latest']: collection.title = u'Latest Arte videos' return raise CollectionNotFound(collection.split_path) OBJECTS = {ArteVideo: fill_video}
class IndeedModule(Module, CapJob): NAME = 'indeed' DESCRIPTION = u'indeed website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.3' BROWSER = IndeedBrowser type_contrat_choices = OrderedDict([ (k, u'%s' % (v)) for k, v in sorted({ 'all': u'Tous les emplois', 'fulltime': u'Temps plein', 'parttime': u'Temps partiel', 'contract': u'Durée indéterminée', 'internship': u'Stage / Apprentissage', 'temporary': u'Durée déterminée', }.iteritems()) ]) limit_date_choices = OrderedDict([ (k, u'%s' % (v)) for k, v in sorted({ 'any': u'à tout moment', '15': u'depuis 15 jours', '7': u'depuis 7 jours', '3': u'depuis 3 jours', '1': u'depuis hier', 'last': u'depuis ma dernière visite', }.iteritems()) ]) radius_choices = OrderedDict([ (k, u'%s' % (v)) for k, v in sorted({ '0': u'uniquement à cet endroit', '5': u'dans un rayon de 5 kilomètres', '10': u'dans un rayon de 10 kilomètres', '15': u'dans un rayon de 15 kilomètres', '25': u'dans un rayon de 25 kilomètres', '50': u'dans un rayon de 50 kilomètres', '100': u'dans un rayon de 100 kilomètres', }.iteritems()) ]) CONFIG = BackendConfig( Value('metier', label=u'Job name', masked=False, default=''), Value('limit_date', label=u'Date limite', choices=limit_date_choices, default=''), Value('contrat', label=u'Contract', choices=type_contrat_choices, default=''), Value('place', label=u'Place', masked=False, default=''), Value('radius', label=u'Radius', choices=radius_choices, default='')) def search_job(self, pattern=None): return self.browser.search_job(metier=pattern) def advanced_search_job(self): return self.browser.search_job( metier=self.config['metier'].get(), limit_date=self.config['limit_date'].get(), contrat=self.config['contrat'].get(), place=self.config['place'].get(), radius=self.config['radius'].get()) def get_job_advert(self, _id, advert=None): return self.browser.get_job_advert(_id, advert) def fill_obj(self, advert, fields): return self.get_job_advert(advert.id, advert) OBJECTS = {BaseJobAdvert: fill_obj}
class MonsterModule(Module, CapJob): NAME = 'monster' DESCRIPTION = u'monster website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.4' BROWSER = MonsterBrowser type_contrat_choices = OrderedDict([ (k, u'%s' % (v)) for k, v in sorted({ 'Interim-ou-CDD-ou-mission_8': u'Interim ou CDD ou mission', 'CDI_8': u'CDI', 'Stage-Apprentissage-Alternance_8': u'Stage/Apprentissage/Alternance', ' ': u'Autres', 'Indépendant-Freelance-Saisonnier-Franchise_8': u'Indépendant/Freelance/Saisonnier/Franchise', 'Journalier_8': u'Journalier', 'Temps-Partiel_8': u'Temps Partiel', 'Temps-Plein_8': u'Temps Plein', }.items()) ]) date_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '-1': u'N importe quelle date', '000000': u'Aujourd hui', '1': u'2 derniers jours', '3': u'3 derniers jours', '7': u'Les 7 derniers jours', '14': u'Les 14 derniers jours', '30': u'30 derniers jours', }.items())]) CONFIG = BackendConfig( Value('job_name', label='Job name', masked=False, default=''), Value('place', label='Place', masked=False, default=''), Value('contract', label=u'Contract', choices=type_contrat_choices, default=''), Value('limit_date', label=u'Date', choices=date_choices, default='-1'), ) def search_job(self, pattern=None): return self.browser.search_job(pattern) def advanced_search_job(self): return self.browser.advanced_search_job( job_name=self.config['job_name'].get(), place=self.config['place'].get(), contract=self.config['contract'].get(), limit_date=self.config['limit_date'].get()) def get_job_advert(self, _id, advert=None): return self.browser.get_job_advert(_id, advert) def fill_obj(self, advert, fields): return self.get_job_advert(advert.id, advert) OBJECTS = {BaseJobAdvert: fill_obj}
class jcvelauxModule(Module, CapGauge): NAME = 'jcvelaux' DESCRIPTION = ('City bike renting availability information.\nCities: %s' % ', '.join(CITIES)) MAINTAINER = 'Herve Werner' EMAIL = '*****@*****.**' VERSION = '1.6' LICENSE = 'AGPLv3' BROWSER = VelibBrowser CONFIG = BackendConfig( Value('city', label='City', default='Lyon', choices=CITIES + ("ALL", )), ValueBackendPassword('api_key', label='Optional API key', default='', noprompt=True)) def __init__(self, *a, **kw): super(jcvelauxModule, self).__init__(*a, **kw) self.cities = None def create_default_browser(self): api_key = self.config['api_key'].get() return self.create_browser(api_key) def _make_gauge(self, info): gauge = Gauge(info['id']) gauge.name = info['name'] gauge.city = info['city'] gauge.object = 'bikes' return gauge def _make_sensor(self, sensor_type, info, gauge): id = '%s.%s' % (sensor_type, gauge.id) sensor = GaugeSensor(id) sensor.gaugeid = gauge.id sensor.name = SENSOR_TYPES[sensor_type] sensor.address = '%s' % info['address'] sensor.longitude = info['longitude'] sensor.latitude = info['latitude'] sensor.history = [] return sensor def _make_measure(self, sensor_type, info, gauge): id = '%s.%s' % (sensor_type, gauge.id) measure = BikeMeasure(id) measure.date = info['last_update'] measure.level = float(info[sensor_type]) return measure def _parse_gauge(self, info): gauge = self._make_gauge(info) gauge.sensors = [] for type in SENSOR_TYPES: sensor = self._make_sensor(type, info, gauge) measure = self._make_measure(type, info, gauge) sensor.lastvalue = measure gauge.sensors.append(sensor) return gauge def _contract(self): contract = self.config.get('city').get() if contract.lower() == 'all': contract = None return contract def iter_gauges(self, pattern=None): if pattern is None: for jgauge in self.browser.get_station_list( contract=self._contract()): yield self._parse_gauge(jgauge) else: lowpattern = pattern.lower() for jgauge in self.browser.get_station_list( contract=self._contract()): gauge = self._parse_gauge(jgauge) if lowpattern in gauge.name.lower( ) or lowpattern in gauge.city.lower(): yield gauge def iter_sensors(self, gauge, pattern=None): if not isinstance(gauge, Gauge): gauge = self._get_gauge_by_id(gauge) if gauge is None: raise SensorNotFound() if pattern is None: for sensor in gauge.sensors: yield sensor else: lowpattern = pattern.lower() for sensor in gauge.sensors: if lowpattern in sensor.name.lower(): yield sensor def get_last_measure(self, sensor): if not isinstance(sensor, GaugeSensor): sensor = self._get_sensor_by_id(sensor) if sensor is None: raise SensorNotFound() return sensor.lastvalue def _fetch_cities(self): if self.cities: return self.cities = {} jcontract = self.browser.get_contracts_list() for jcontract in jcontract: for city in jcontract['cities']: self.cities[city.lower()] = jcontract['name'] def _get_gauge_by_id(self, id): jgauge = self.browser.get_station_infos(id) if jgauge: return self._parse_gauge(jgauge) else: return None def _get_sensor_by_id(self, id): try: sensor_name, gauge_name, contract = id.split('.') except ValueError: raise UserError('Expected format NAME.ID.CITY for sensor: %r' % id) gauge = self._get_gauge_by_id('%s.%s' % (gauge_name, contract)) if not gauge: raise SensorNotFound() for sensor in gauge.sensors: if sensor.id.lower() == id.lower(): return sensor
class YoutubeModule(Module, CapVideo, CapCollection): NAME = 'youtube' MAINTAINER = u'Laurent Bachelier' EMAIL = '*****@*****.**' VERSION = '1.4' DESCRIPTION = 'YouTube video streaming website' LICENSE = 'AGPLv3+' BROWSER = None CONFIG = BackendConfig( Value('username', label='Email address', default=''), ValueBackendPassword('password', label='Password', default='')) URL_RE = re.compile( r'^https?://(?:\w*\.?youtube(?:|-nocookie)\.com/(?:watch\?v=|embed/|v/)|youtu\.be\/|\w*\.?youtube\.com\/user\/\w+#p\/u\/\d+\/)([^\?&]+)' ) def create_default_browser(self): password = None username = self.config['username'].get() if len(username) > 0: password = self.config['password'].get() return self.create_browser(username, password) def _entry2video(self, entry): """ Parse an entry returned by googleapi and return a Video object. """ snippet = entry['snippet'] id = entry['id'] if isinstance(id, dict): id = id['videoId'] video = YoutubeVideo(to_unicode(id)) video.title = to_unicode(snippet['title'].strip()) # duration does not seem to be available with api video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url']) video.author = to_unicode(snippet['channelTitle'].strip()) return video def _set_video_attrs(self, video): new_video = video_info(YoutubeVideo.id2url(video.id)) if not new_video: return for k, v in new_video.iter_fields(): if not empty(v) and empty(getattr(video, k)): setattr(video, k, v) def get_video(self, _id): m = self.URL_RE.match(_id) if m: _id = m.group(1) params = {'id': _id, 'part': 'id,snippet'} youtube = self._build_yt() response = youtube.videos().list(**params).execute() items = response.get('items', []) if not items: return None video = self._entry2video(items[0]) self._set_video_attrs(video) video.set_empty_fields(NotAvailable) # Youtube video url is https, using ssl encryption # so we need to use the "play_proxy" method using urllib2 proxy streaming to handle this video._play_proxy = True return video def _build_yt(self): DEVELOPER_KEY = "AIzaSyApVVeZ03XkKDYHX8T5uOn8Eizfe9CMDbs" YOUTUBE_API_SERVICE_NAME = "youtube" YOUTUBE_API_VERSION = "v3" return ytbuild(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=DEVELOPER_KEY) def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): YOUTUBE_MAX_RESULTS = 50 youtube = self._build_yt() params = {'part': 'id,snippet', 'maxResults': YOUTUBE_MAX_RESULTS} if pattern is not None: params['q'] = pattern params['safeSearch'] = 'none' if nsfw else 'strict' # or 'moderate' params['order'] = ('relevance', 'rating', 'viewCount', 'date')[sortby] nb_yielded = 0 while True: search_response = youtube.search().list(**params).execute() items = search_response.get('items', []) for entry in items: if entry["id"]["kind"] != "youtube#video": continue yield self._entry2video(entry) nb_yielded += 1 params['pageToken'] = search_response.get('nextPageToken') if not params['pageToken']: return if nb_yielded < YOUTUBE_MAX_RESULTS: return def latest_videos(self): return self.search_videos(None, CapVideo.SEARCH_DATE) def fill_video(self, video, fields): if 'thumbnail' in fields and video.thumbnail: video.thumbnail.data = requests.get(video.thumbnail.url).content if 'url' in fields: self._set_video_attrs(video) return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest']) if collection.split_path == [u'latest']: for video in self.latest_videos(): yield video def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and collection.split_path == [u'latest']: collection.title = u'Latest YouTube videos' return raise CollectionNotFound(collection.split_path) OBJECTS = {YoutubeVideo: fill_video}
class LeclercMobileModule(Module, CapDocument): NAME = 'leclercmobile' MAINTAINER = u'Florent Fourcot' EMAIL = '*****@*****.**' VERSION = '1.4' LICENSE = 'AGPLv3+' DESCRIPTION = 'Leclerc Mobile website' CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', masked=False, regexp='^(\d{10}|)$'), ValueBackendPassword('password', label='Password') ) BROWSER = Leclercmobile def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) def iter_subscription(self): for subscription in self.browser.get_subscription_list(): yield subscription def get_subscription(self, _id): if not _id.isdigit(): raise SubscriptionNotFound() with self.browser: subscription = self.browser.get_subscription(_id) if subscription: return subscription else: raise SubscriptionNotFound() def iter_documents_history(self, subscription): with self.browser: for history in self.browser.get_history(): if history.label != "Votre solde": yield history def get_document(self, id): with self.browser: bill = self.browser.get_document(id) if bill: return bill else: raise DocumentNotFound() def iter_documents(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) with self.browser: for bill in self.browser.iter_documents(subscription.id): yield bill # The subscription is actually useless, but maybe for the futur... def get_details(self, subscription): with self.browser: for detail in self.browser.get_details(): yield detail def download_document(self, bill): if not isinstance(bill, Bill): bill = self.get_document(bill) with self.browser: return self.browser.readurl(bill.url) def get_balance(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) with self.browser: balance = self.browser.get_balance() balance.label = u"Balance %s" % subscription.id balance.id = "%s-balance" % subscription.id balance.currency = u'EUR' return balance
class jcvelauxModule(Module, CapGauge): NAME = 'jcvelaux' DESCRIPTION = (u'City bike renting availability information.\nCities: %s' % u', '.join(CITIES)) MAINTAINER = u'Herve Werner' EMAIL = '*****@*****.**' VERSION = '1.1' LICENSE = 'AGPLv3' BROWSER = VelibBrowser STORAGE = {'boards': {}} CONFIG = BackendConfig(Value('city', label='City', default='Paris', choices=CITIES + ("ALL",))) def __init__(self, *a, **kw): super(jcvelauxModule, self).__init__(*a, **kw) self.cities = None def _make_gauge(self, info): gauge = Gauge(info['id']) gauge.name = unicode(info['name']) gauge.city = unicode(info['city']) gauge.object = u'bikes' return gauge def _make_sensor(self, sensor_type, info, gauge): id = '%s.%s' % (sensor_type, gauge.id) sensor = BikeSensor(id) sensor.gaugeid = gauge.id sensor.name = SENSOR_TYPES[sensor_type] sensor.address = unicode(info['address']) sensor.longitude = info['longitude'] sensor.latitude = info['latitude'] sensor.history = [] return sensor def _make_measure(self, sensor_type, info): measure = BikeMeasure() measure.date = info['last_update'] measure.level = float(info[sensor_type]) return measure def _parse_gauge(self, info): gauge = self._make_gauge(info) gauge.sensors = [] for type in SENSOR_TYPES: sensor = self._make_sensor(type, info, gauge) measure = self._make_measure(type, info) sensor.lastvalue = measure gauge.sensors.append(sensor) return gauge def _contract(self): contract = self.config.get('city').get() if contract.lower() == 'all': contract = None return contract def iter_gauges(self, pattern=None): if pattern is None: for jgauge in self.browser.get_station_list(contract=self._contract()): yield self._parse_gauge(jgauge) else: lowpattern = pattern.lower() for jgauge in self.browser.get_station_list(contract=self._contract()): gauge = self._parse_gauge(jgauge) if lowpattern in gauge.name.lower() or lowpattern in gauge.city.lower(): yield gauge def iter_sensors(self, gauge, pattern=None): if not isinstance(gauge, Gauge): gauge = self._get_gauge_by_id(gauge) if gauge is None: raise SensorNotFound() if pattern is None: yield from gauge.sensors else: lowpattern = pattern.lower() for sensor in gauge.sensors: if lowpattern in sensor.name.lower(): yield sensor def get_last_measure(self, sensor): if not isinstance(sensor, GaugeSensor): sensor = self._get_sensor_by_id(sensor) if sensor is None: raise SensorNotFound() return sensor.lastvalue def _fetch_cities(self): if self.cities: return self.cities = {} jcontract = self.browser.get_contracts_list() for jcontract in jcontract: for city in jcontract['cities']: self.cities[city.lower()] = jcontract['name'] def _get_gauge_by_id(self, id): jgauge = self.browser.get_station_infos(id) if jgauge: return self._parse_gauge(jgauge) else: return None def _get_sensor_by_id(self, id): _, gauge_id = id.split('.', 1) gauge = self._get_gauge_by_id(gauge_id) if not gauge: raise SensorNotFound() for sensor in gauge.sensors: if sensor.id.lower() == id.lower(): return sensor
class OkCBackend(BaseBackend, ICapMessages, ICapContact, ICapMessagesPost): NAME = 'okc' MAINTAINER = u'Roger Philibert' EMAIL = '*****@*****.**' VERSION = '0.e' LICENSE = 'AGPLv3+' DESCRIPTION = u'OkCupid dating website' CONFIG = BackendConfig(Value('username', label='Username'), ValueBackendPassword('password', label='Password')) STORAGE = { 's***s': {}, #'notes': {}, } BROWSER = OkCBrowser def create_default_browser(self): return self.create_browser(self.config['username'].get(), self.config['password'].get()) # ---- ICapMessages methods --------------------- def fill_thread(self, thread, fields): return self.get_thread(thread) def iter_threads(self): with self.browser: threads = self.browser.get_threads_list() for thread in threads: # Remove messages from user that quit #if thread['member'].get('isBan', thread['member'].get('dead', False)): # with self.browser: # self.browser.delete_thread(thread['member']['id']) # continue t = Thread(int(thread['id'])) t.flags = Thread.IS_DISCUSSION t.title = u'Discussion with %s' % thread['username'] yield t def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(mails['member']['pseudo']) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['member']['pseudo'] for mail in mails['messages']: flags = Message.IS_HTML if parse_dt(mail['date']) > s**t['lastmsg'] and mail[ 'id_from'] != self.browser.get_my_name(): flags |= Message.IS_UNREAD if get_profiles: if not mail['id_from'] in contacts: with self.browser: contacts[mail['id_from']] = self.get_contact( mail['id_from']) signature = u'' if mail.get('src', None): signature += u'Sent from my %s\n\n' % mail['src'] if mail['id_from'] in contacts: signature += contacts[mail['id_from']].get_text() msg = Message(thread=thread, id=int( time.strftime('%Y%m%d%H%M%S', parse_dt( mail['date']).timetuple())), title=thread.title, sender=mail['id_from'], receivers=[ my_name if mail['id_from'] != my_name else mails['member']['pseudo'] ], date=parse_dt(mail['date']), content=unescape(mail['message']).strip(), signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread #def iter_unread_messages(self, thread=None): # try: # contacts = {} # with self.browser: # threads = self.browser.get_threads_list() # for thread in threads: # if thread['member'].get('isBan', thread['member'].get('dead', False)): # with self.browser: # self.browser.delete_thread(int(thread['member']['id'])) # continue # if self.antispam and not self.antispam.check_thread(thread): # self.logger.info('Skipped a spam-unread-thread from %s' % thread['member']['pseudo']) # self.report_spam(thread['member']['pseudo']) # continue # s**t = self._get_slut(thread['member']['pseudo']) # if parse_dt(thread['date']) > s**t['lastmsg']: # t = self.get_thread(thread['member']['pseudo'], contacts, get_profiles=True) # for m in t.iter_all_messages(): # if m.flags & m.IS_UNREAD: # yield m # except BrowserUnavailable, e: # self.logger.debug('No messages, browser is unavailable: %s' % e) # pass # don't care about waiting def set_message_read(self, message): if message.sender == self.browser.get_my_name(): return s**t = self._get_slut(message.sender) if s**t['lastmsg'] < message.date: s**t['lastmsg'] = message.date self.storage.set('s***s', message.sender, s**t) self.storage.save() def _get_slut(self, id): s***s = self.storage.get('s***s') if not s***s or not id in s***s: s**t = {'lastmsg': datetime.datetime(1970, 1, 1)} else: s**t = self.storage.get('s***s', id) s**t['lastmsg'] = s**t.get('lastmsg', datetime.datetime( 1970, 1, 1)).replace(tzinfo=tz.tzutc()) return s**t # ---- ICapMessagesPost methods --------------------- def post_message(self, message): content = message.content.replace('\n', '\r\n').encode('utf-8', 'replace') with self.browser: # Check wether we already have a thread with this user threads = self.browser.get_threads_list() for thread in threads: if thread['username'] == message.thread.id: self.browser.post_reply(thread['id'], content) break else: self.browser.post_mail(message.thread.id, content) # ---- ICapContact methods --------------------- def fill_contact(self, contact, fields): if 'profile' in fields: contact = self.get_contact(contact) if contact and 'photos' in fields: for name, photo in contact.photos.iteritems(): with self.browser: if photo.url and not photo.data: data = self.browser.openurl(photo.url).read() contact.set_photo(name, data=data) if photo.thumbnail_url and not photo.thumbnail_data: data = self.browser.openurl(photo.thumbnail_url).read() contact.set_photo(name, thumbnail_data=data) def fill_photo(self, photo, fields): with self.browser: if 'data' in fields and photo.url and not photo.data: photo.data = self.browser.readurl(photo.url) if 'thumbnail_data' in fields and photo.thumbnail_url and not photo.thumbnail_data: photo.thumbnail_data = self.browser.readurl( photo.thumbnail_url) return photo def get_contact(self, contact): with self.browser: if isinstance(contact, Contact): _id = contact.id elif isinstance(contact, (int, long, basestring)): _id = contact else: raise TypeError( "The parameter 'contact' isn't a contact nor a int/long/str/unicode: %s" % contact) profile = self.browser.get_profile(_id) if not profile: return None _id = profile['id'] if isinstance(contact, Contact): contact.id = _id contact.name = profile['id'] else: contact = Contact(_id, profile['id'], Contact.STATUS_OFFLINE) contact.url = 'http://%s/profile/%s' % (self.browser.DOMAIN, _id) contact.profile = profile['data'] contact.summary = profile['summary'] if contact.profile['details'][ 'last_online'].value == u'Online now!': contact.status = Contact.STATUS_ONLINE else: contact.status = Contact.STATUS_OFFLINE contact.status_msg = contact.profile['details'][ 'last_online'].value for no, photo in enumerate(self.browser.get_photos(_id)): contact.set_photo(u'image_%i' % no, url=photo, thumbnail_url=photo) return contact #def _get_partial_contact(self, contact): # if contact.get('isBan', contact.get('dead', False)): # with self.browser: # self.browser.delete_thread(int(contact['id'])) # return None # s = 0 # if contact.get('isOnline', False): # s = Contact.STATUS_ONLINE # else: # s = Contact.STATUS_OFFLINE # c = Contact(contact['id'], contact['id'], s) # c.url = self.browser.id2url(contact['id']) # if 'birthday' in contact: # birthday = _parse_dt(contact['birthday']) # age = int((datetime.datetime.now() - birthday).days / 365.25) # c.status_msg = u'%s old, %s' % (age, contact['city']) # if contact['cover'].isdigit() and int(contact['cover']) > 0: # url = 'http://s%s.adopteunmec.com/%s%%(type)s%s.jpg' % (contact['shard'], contact['path'], contact['cover']) # else: # url = 'http://s.adopteunmec.com/www/img/thumb0.gif' # c.set_photo('image%s' % contact['cover'], # url=url % {'type': 'image'}, # thumbnail_url=url % {'type': 'thumb0_'}) # return c def iter_contacts(self, status=Contact.STATUS_ALL, ids=None): with self.browser: threads = self.browser.get_threads_list(count=100) for thread in threads: c = self._get_partial_contact(thread['member']) if c and (c.status & status) and (not ids or c.id in ids): yield c #def send_query(self, id): # if isinstance(id, Contact): # id = id.id # queries_queue = None # try: # queries_queue = self.get_optimization('QUERIES_QUEUE') # except OptimizationNotFound: # pass # if queries_queue and queries_queue.is_running(): # if queries_queue.enqueue_query(id): # return Query(id, 'A charm has been sent') # else: # return Query(id, 'Unable to send charm: it has been enqueued') # else: # with self.browser: # if not self.browser.send_charm(id): # raise QueryError('No enough charms available') # return Query(id, 'A charm has been sent') #def get_notes(self, id): # if isinstance(id, Contact): # id = id.id # return self.storage.get('notes', id) #def save_notes(self, id, notes): # if isinstance(id, Contact): # id = id.id # self.storage.set('notes', id, notes) # self.storage.save() OBJECTS = { Thread: fill_thread, Contact: fill_contact, ContactPhoto: fill_photo }
class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapContent): NAME = 'dlfp' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '0.h' LICENSE = 'AGPLv3+' DESCRIPTION = "Da Linux French Page news website" CONFIG = BackendConfig( Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default=''), ValueBool('get_news', label='Get newspapers', default=True), ValueBool('get_diaries', label='Get diaries', default=False), ValueBool('get_polls', label='Get polls', default=False), ValueBool('get_board', label='Get board', default=False), ValueBool('get_wiki', label='Get wiki', default=False), ValueBool('get_tracker', label='Get tracker', default=False)) STORAGE = {'seen': {}} BROWSER = DLFP FEEDS = { 'get_news': "https://linuxfr.org/news.atom", 'get_diaries': "https://linuxfr.org/journaux.atom", 'get_polls': "https://linuxfr.org/sondages.atom", 'get_board': "https://linuxfr.org/forums.atom", 'get_wiki': "https://linuxfr.org/wiki.atom", 'get_tracker': "https://linuxfr.org/suivi.atom", } def create_default_browser(self): username = self.config['username'].get() if username: password = self.config['password'].get() else: password = None return self.create_browser(username, password) def deinit(self): # don't need to logout if the browser hasn't been used. if not self._browser: return with self.browser: self.browser.close_session() #### ICapMessages ############################################## def iter_threads(self): whats = set() for param, url in self.FEEDS.iteritems(): if self.config[param].get(): whats.add(url) for what in whats: for article in Newsfeed(what, rssid).iter_entries(): if article.datetime and (datetime.now() - article.datetime ) > timedelta(days=60): continue thread = Thread(article.id) thread.title = article.title thread._rsscomment = article.rsscomment if article.datetime: thread.date = article.datetime yield thread def get_thread(self, id, getseen=True): if not isinstance(id, Thread): thread = None else: thread = id id = thread.id if thread.date: self.storage.set('date', id, thread.date) self.storage.save() with self.browser: content = self.browser.get_content(id) if not content: return None if not thread: thread = Thread(content.id) flags = Message.IS_HTML if not thread.id in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message( thread=thread, id='0', # root message title=content.title, sender=content.author or u'', receivers=None, date=thread.date, parent=None, content=content.body, signature='URL: %s' % self.browser.absurl(id2url(content.id)), children=[], flags=flags) for com in content.comments: self._insert_comment(com, thread.root, getseen) return thread def _insert_comment(self, com, parent, getseen=True): """" Insert 'com' comment and its children in the parent message. """ flags = Message.IS_HTML if not com.id in self.storage.get( 'seen', parent.thread.id, 'comments', default=[]): flags |= Message.IS_UNREAD if getseen or flags & Message.IS_UNREAD: com.parse() message = Message( thread=parent.thread, id=com.id, title=com.title, sender=com.author or u'', receivers=None, date=com.date, parent=parent, content=com.body, signature=com.signature + '<br />'.join(['Score: %d' % com.score, 'URL: %s' % com.url]), children=[], flags=flags) else: message = Message(thread=parent.thread, id=com.id, children=[], flags=flags) parent.children.append(message) for sub in com.comments: self._insert_comment(sub, message, getseen) def iter_unread_messages(self): for thread in self.iter_threads(): # Check if we have seen all comments of this thread. with self.browser: oldhash = self.storage.get('hash', thread.id, default="") newhash = self.browser.get_hash(thread._rsscomment) if oldhash != newhash: self.storage.set('hash', thread.id, newhash) self.storage.save() self.fill_thread(thread, 'root', getseen=False) for m in thread.iter_all_messages(): if m.flags & m.IS_UNREAD: yield m def set_message_read(self, message): self.storage.set( 'seen', message.thread.id, 'comments', self.storage.get('seen', message.thread.id, 'comments', default=[]) + [message.id]) self.storage.save() lastpurge = self.storage.get('lastpurge', default=0) # 86400 = one day if time.time() - lastpurge > 86400: self.storage.set('lastpurge', time.time()) self.storage.save() # we can't directly delete without a "RuntimeError: dictionary changed size during iteration" todelete = [] for id in self.storage.get('seen', default={}): date = self.storage.get('date', id, default=0) # if no date available, create a new one (compatibility with "old" storage) if date == 0: self.storage.set('date', id, datetime.now()) elif datetime.now() - date > timedelta(days=60): todelete.append(id) for id in todelete: self.storage.delete('hash', id) self.storage.delete('date', id) self.storage.delete('seen', id) self.storage.save() def fill_thread(self, thread, fields, getseen=True): return self.get_thread(thread, getseen) #### ICapMessagesReply ######################################### def post_message(self, message): if not self.browser.username: raise BrowserForbidden() if not message.parent: raise CantSendMessage( 'Posting news and diaries on DLFP is not supported yet') assert message.thread with self.browser: return self.browser.post_comment(message.thread.id, message.parent.id, message.title, message.content) #### ICapContent ############################################### def get_content(self, _id, revision=None): if isinstance(_id, basestring): content = Content(_id) else: content = _id _id = content.id if revision: raise NotImplementedError( 'Website does not provide access to older revisions sources.') with self.browser: data = self.browser.get_wiki_content(_id) if data is None: return None content.content = data return content def push_content(self, content, message=None, minor=False): if not self.browser.username: raise BrowserForbidden() with self.browser: return self.browser.set_wiki_content(content.id, content.content, message) def get_content_preview(self, content): with self.browser: return self.browser.get_wiki_preview(content.id, content.content) OBJECTS = {Thread: fill_thread}
class AdeccoModule(Module, CapJob): NAME = 'adecco' DESCRIPTION = u'adecco website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' VERSION = '1.1' BROWSER = AdeccoBrowser publicationDate_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '000000': u'-- Indifférent --', '1': u'Moins de 48 heures', '2': u'Moins de 1 semaine', '4': u'Moins de 2 semaines', '3': u'Moins de 5 semaines', }.iteritems())]) type_contract_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '000000': u'--Indifferent--', '1': u'CDD', '2': u'CDI', '3': u'Intérim', '4': u'Emploi formation', '5': u'Emploi saisonnier', '6': u'Stage', '7': u'Autre', }.iteritems())]) places_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '100|REGION_0|DEPARTEMENT_0': u'--Indifferent--', '101|REGION_1': u'Alsace', '102|REGION_1|DEPARTEMENT_1': u'-- Rhin (Bas) (67)', '103|REGION_1|DEPARTEMENT_2': u'-- Rhin (Haut) (68)', '104|REGION_2': u'Aquitaine', '105|REGION_2|DEPARTEMENT_3': u'-- Dordogne (24)', '106|REGION_2|DEPARTEMENT_4': u'-- Gironde (33)', '107|REGION_2|DEPARTEMENT_5': u'-- Landes (40)', '108|REGION_2|DEPARTEMENT_6': u'-- Lot et Garonne (47)', '109|REGION_2|DEPARTEMENT_7': u'-- Pyrénées Atlantiques (64)', '110|REGION_3': u'Auvergne', '111|REGION_3|DEPARTEMENT_8': u'-- Allier (03)', '112|REGION_3|DEPARTEMENT_9': u'-- Cantal (15)', '113|REGION_3|DEPARTEMENT_10': u'-- Loire (Haute) (43)', '114|REGION_3|DEPARTEMENT_11': u'-- Puy de Dôme (63)', '115|REGION_5': u'Bourgogne', '116|REGION_5|DEPARTEMENT_15': u'-- Côte d\'Or (21)', '117|REGION_5|DEPARTEMENT_16': u'-- Nièvre (58)', '118|REGION_5|DEPARTEMENT_17': u'-- Saône et Loire (71)', '119|REGION_5|DEPARTEMENT_18': u'-- Yonne (89)', '120|REGION_6': u'Bretagne', '121|REGION_6|DEPARTEMENT_19': u'-- Côtes d\'Armor (22)', '122|REGION_6|DEPARTEMENT_20': u'-- Finistère (29)', '123|REGION_6|DEPARTEMENT_21': u'-- Ille et Vilaine (35)', '124|REGION_6|DEPARTEMENT_22': u'-- Morbihan (56)', '125|REGION_7': u'Centre', '126|REGION_7|DEPARTEMENT_23': u'-- Cher (18)', '127|REGION_7|DEPARTEMENT_24': u'-- Eure et Loir (28)', '128|REGION_7|DEPARTEMENT_25': u'-- Indre (36)', '129|REGION_7|DEPARTEMENT_26': u'-- Indre et Loire (37)', '130|REGION_7|DEPARTEMENT_27': u'-- Loir et Cher (41)', '131|REGION_7|DEPARTEMENT_28': u'-- Loiret (45)', '132|REGION_8': u'Champagne Ardenne', '133|REGION_8|DEPARTEMENT_29': u'-- Ardennes (08)', '134|REGION_8|DEPARTEMENT_30': u'-- Aube (10)', '135|REGION_8|DEPARTEMENT_31': u'-- Marne (51)', '136|REGION_8|DEPARTEMENT_32': u'-- Marne (Haute) (52)', '137|REGION_9': u'Corse', '138|REGION_9|DEPARTEMENT_33': u'-- Corse du Sud (2A)', '139|REGION_9|DEPARTEMENT_34': u'-- Haute Corse (2B)', '140|REGION_11': u'Franche Comté', '141|REGION_11|DEPARTEMENT_43': u'-- Belfort (Territoire de) (90)', '142|REGION_11|DEPARTEMENT_40': u'-- Doubs (25)', '143|REGION_11|DEPARTEMENT_41': u'-- Jura (39)', '144|REGION_11|DEPARTEMENT_42': u'-- Saône (Haute) (70)', '145|REGION_13': u'Ile de France', '146|REGION_13|DEPARTEMENT_49': u'-- Essonne (91)', '147|REGION_13|DEPARTEMENT_50': u'-- Hauts de Seine (92)', '148|REGION_13|DEPARTEMENT_46': u'-- Paris (Dept.) (75)', '149|REGION_13|DEPARTEMENT_51': u'-- Seine Saint Denis (93)', '150|REGION_13|DEPARTEMENT_47': u'-- Seine et Marne (77)', '151|REGION_13|DEPARTEMENT_53': u'-- Val d\'Oise (95)', '152|REGION_13|DEPARTEMENT_52': u'-- Val de Marne (94)', '153|REGION_13|DEPARTEMENT_48': u'-- Yvelines (78)', '154|REGION_14': u'Languedoc Roussillon', '155|REGION_14|DEPARTEMENT_54': u'-- Aude (11)', '156|REGION_14|DEPARTEMENT_55': u'-- Gard (30)', '157|REGION_14|DEPARTEMENT_56': u'-- Hérault (34)', '158|REGION_14|DEPARTEMENT_57': u'-- Lozère (48)', '159|REGION_14|DEPARTEMENT_58': u'-- Pyrénées Orientales (66)', '160|REGION_15': u'Limousin', '161|REGION_15|DEPARTEMENT_59': u'-- Corrèze (19)', '162|REGION_15|DEPARTEMENT_60': u'-- Creuse (23)', '163|REGION_15|DEPARTEMENT_61': u'-- Vienne (Haute) (87)', '164|REGION_16': u'Lorraine', '165|REGION_16|DEPARTEMENT_62': u'-- Meurthe et Moselle (54)', '166|REGION_16|DEPARTEMENT_63': u'-- Meuse (55)', '167|REGION_16|DEPARTEMENT_64': u'-- Moselle (57)', '168|REGION_16|DEPARTEMENT_65': u'-- Vosges (88)', '169|REGION_17': u'Midi Pyrénées', '170|REGION_17|DEPARTEMENT_66': u'-- Ariège (09)', '171|REGION_17|DEPARTEMENT_67': u'-- Aveyron (12)', '172|REGION_17|DEPARTEMENT_68': u'-- Garonne (Haute) (31)', '173|REGION_17|DEPARTEMENT_69': u'-- Gers (32)', '174|REGION_17|DEPARTEMENT_70': u'-- Lot (46)', '175|REGION_17|DEPARTEMENT_71': u'-- Pyrénées (Hautes) (65)', '176|REGION_17|DEPARTEMENT_72': u'-- Tarn (81)', '177|REGION_17|DEPARTEMENT_73': u'-- Tarn et Garonne (82)', '178|REGION_18': u'Nord Pas de Calais', '179|REGION_18|DEPARTEMENT_74': u'-- Nord (59)', '180|REGION_18|DEPARTEMENT_75': u'-- Pas de Calais (62)', '181|REGION_4': u'Normandie (Basse)', '182|REGION_4|DEPARTEMENT_12': u'-- Calvados (14)', '183|REGION_4|DEPARTEMENT_13': u'-- Manche (50)', '184|REGION_4|DEPARTEMENT_14': u'-- Orne (61)', '185|REGION_12': u'Normandie (Haute)', '186|REGION_12|DEPARTEMENT_44': u'-- Eure (27)', '187|REGION_12|DEPARTEMENT_47': u'-- Seine Maritime (76)', '188|REGION_19': u'Pays de la Loire', '189|REGION_19|DEPARTEMENT_76': u'-- Loire Atlantique (44)', '190|REGION_19|DEPARTEMENT_77': u'-- Maine et Loire (49)', '191|REGION_19|DEPARTEMENT_78': u'-- Mayenne (53)', '192|REGION_19|DEPARTEMENT_79': u'-- Sarthe (72)', '193|REGION_19|DEPARTEMENT_80': u'-- Vendée (85)', '194|REGION_20': u'Picardie', '195|REGION_20|DEPARTEMENT_81': u'-- Aisne (02)', '196|REGION_20|DEPARTEMENT_83': u'-- Oise (60)', '197|REGION_20|DEPARTEMENT_84': u'-- Somme (80)', '198|REGION_21': u'Poitou Charentes', '199|REGION_21|DEPARTEMENT_85': u'-- Charente (16)', '200|REGION_21|DEPARTEMENT_86': u'-- Charente Maritime (17)', '201|REGION_21|DEPARTEMENT_87': u'-- Sèvres (Deux) (79)', '202|REGION_21|DEPARTEMENT_88': u'-- Vienne (86)', '203|REGION_22': u'Provence Alpes Côte d\'Azur', '204|REGION_22|DEPARTEMENT_90': u'-- Alpes (Hautes) (05)', '205|REGION_22|DEPARTEMENT_91': u'-- Alpes Maritimes (06)', '206|REGION_22|DEPARTEMENT_89': u'-- Alpes de Haute Provence (04)', '207|REGION_22|DEPARTEMENT_92': u'-- Bouches du Rhône (13)', '208|REGION_22|DEPARTEMENT_93': u'-- Var (83)', '209|REGION_22|DEPARTEMENT_94': u'-- Vaucluse (84)', '210|REGION_23': u'Rhône Alpes', '211|REGION_23|DEPARTEMENT_95': u'-- Ain (01)', '212|REGION_23|DEPARTEMENT_96': u'-- Ardèche (07)', '213|REGION_23|DEPARTEMENT_97': u'-- Drôme (26)', '214|REGION_23|DEPARTEMENT_98': u'-- Isère (38)', '215|REGION_23|DEPARTEMENT_99': u'-- Loire (42)', '216|REGION_23|DEPARTEMENT_100': u'-- Rhône (69)', '217|REGION_23|DEPARTEMENT_101': u'-- Savoie (73)', '218|REGION_23|DEPARTEMENT_102': u'-- Savoie (Haute) (74)', '219|REGION_10': u'DOM TOM', '220|REGION_10|DEPARTEMENT_35': u'-- Guadeloupe (971)', '221|REGION_10|DEPARTEMENT_37': u'-- Guyane (973)', '222|REGION_10|DEPARTEMENT_38': u'-- La Réunion (974)', '223|REGION_10|DEPARTEMENT_36': u'-- Martinique (972)', '224|REGION_10|DEPARTEMENT_108': u'-- Mayotte (976)', '225|REGION_10|DEPARTEMENT_109': u'-- Nouvelle Calédonie (988)', '226|REGION_10|DEPARTEMENT_108': u'-- Polynésie (987)', '227|REGION_10|DEPARTEMENT_107': u'-- Saint Pierre et Miquelon (975)', '228|REGION_24': u'International', '229|REGION_24|DEPARTEMENT_104': u'-- Andorre', '230|REGION_24|DEPARTEMENT_105': u'-- Monaco', '231|REGION_24|DEPARTEMENT_106': u'-- Suisse', }.iteritems())]) activityDomain_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '100|DOMAIN_0': u'Tous domaines d\'activité', '101|DOMAIN_1': u'Accueil - Secrétariat - Fonctions Administratives', '102|DOMAIN_1|ACTIVITY_1': u'-- Accueil', '103|DOMAIN_1|ACTIVITY_2': u'-- Secrétariat - Assistanat', '104|DOMAIN_1|ACTIVITY_3': u'-- Autres Fonctions Administratives', '105|DOMAIN_2': u'Achats - Juridique - Qualité - RH - Direction', '106|DOMAIN_2|ACTIVITY_4': u'-- Achats ', '107|DOMAIN_2|ACTIVITY_5': u'-- Juridique', '108|DOMAIN_2|ACTIVITY_6': u'-- Qualité', '109|DOMAIN_2|ACTIVITY_7': u'Ressources Humaines - Formation', '110|DOMAIN_2|ACTIVITY_8': u'-- Direction Générale', '111|DOMAIN_3': u'Agriculture - Viticulture - Pêche - Espaces Verts', '112|DOMAIN_3|ACTIVITY_9': u'-- Agriculture - Viticulture - Pêche ', '113|DOMAIN_3|ACTIVITY_10': u'-- Espaces Verts - Exploitation Forestière', '114|DOMAIN_4': u'Automobile', '115|DOMAIN_5': u'Banque - Finance - Gestion Comptabilité - Assurance', '116|DOMAIN_5|ACTIVITY_11': u'-- Banque - Finance ', '117|DOMAIN_5|ACTIVITY_12': u'-- Gestion - Comptabilité', '118|DOMAIN_5|ACTIVITY_13': u'-- Assurance', '119|DOMAIN_6': u'Bâtiment - Travaux Publics - Architecture - Immobilier', '120|DOMAIN_6|ACTIVITY_14': u'-- Bâtiment - Travaux Publics', '121|DOMAIN_6|ACTIVITY_15': u'-- Architecture - Immobilier ', '122|DOMAIN_13': u'Bureaux d\'Etudes - Méthodes', '123|DOMAIN_8': u'Commerce - Vente - Grande Distribution', '124|DOMAIN_8|ACTIVITY_20': u'-- Commerce - Vente', '125|DOMAIN_8|ACTIVITY_21': u'-- Grande et Moyenne Distribution', '126|DOMAIN_9': u'Environnement - Nettoyage - Sécurité', '127|DOMAIN_9|ACTIVITY_22': u'-- Environnement - HSE - Développement durable', '128|DOMAIN_9|ACTIVITY_23': u'-- Nettoyage - Assainissement - Pressing', '129|DOMAIN_9|ACTIVITY_24': u'-- Sécurité - Premiers secours', '130|DOMAIN_10': u'Hôtellerie - Restauration - Métiers de Bouche', '131|DOMAIN_10|ACTIVITY_25': u'-- Hôtellerie', '132|DOMAIN_10|ACTIVITY_27': u'-- Métiers de bouche', '133|DOMAIN_10|ACTIVITY_26': u'-- Restauration', '134|DOMAIN_11': u'Industrie', '135|DOMAIN_11|ACTIVITY_32': u'-- Aéronautique - Navale', '136|DOMAIN_11|ACTIVITY_33': u'-- Agroalimentaire', '137|DOMAIN_11|ACTIVITY_58': u'-- Chimie - Pétrochimie', '138|DOMAIN_11|ACTIVITY_28': u'-- Electricité - Electronique - Automatisme', '139|DOMAIN_11|ACTIVITY_29': u'-- Maintenance - Entretien - SAV ', '140|DOMAIN_11|ACTIVITY_30': u'-- Mécanique Générale', '141|DOMAIN_11|ACTIVITY_31': u'-- Production - Fabrication ', '142|DOMAIN_11|ACTIVITY_36': u'-- Sidérurgie - Métallurgie - Tuyauterie - Soudure', '143|DOMAIN_11|ACTIVITY_34': u'-- Nucléaire - Production d\'énergie', '144|DOMAIN_11|ACTIVITY_35': u'-- Plasturgie - Bois - Papier - Verre - Cuir - Textile', '145|DOMAIN_12': u'Informatique - Technologie de l\'Information', '146|DOMAIN_12|ACTIVITY_37': u'-- Direction informatique encadrement', '147|DOMAIN_12|ACTIVITY_38': u'-- Etude et développement', '148|DOMAIN_12|ACTIVITY_39': u'-- Exploitation, maintenance et support ', '149|DOMAIN_12|ACTIVITY_40': u'-- Systèmes et réseaux informatique et télécom', '150|DOMAIN_14': u'Logistique - Manutention - Transport', '151|DOMAIN_14|ACTIVITY_42': u'-- Conduite de véhicule', '152|DOMAIN_14|ACTIVITY_43': u'-- Exploitation de logistique - supply chain', '153|DOMAIN_14|ACTIVITY_44': u'-- Manutention', '154|DOMAIN_14|ACTIVITY_45': u'-- Transport', '155|DOMAIN_15': u'Marketing - Communication - Imprimerie - Edition', '156|DOMAIN_15|ACTIVITY_47': u'-- Imprimerie - Edition - Arts Graphiques', '157|DOMAIN_15|ACTIVITY_46': u'-- Marketing - Communication - Medias', '158|DOMAIN_16': u'Médical - Paramédical - Esthétique', '159|DOMAIN_16|ACTIVITY_59': u'-- Commerce Appareillage', '160|DOMAIN_16|ACTIVITY_50': u'-- Directions, Cadres et Enseignement', '161|DOMAIN_16|ACTIVITY_49': u'-- Rééducation, Radiologie, Appareillage, LAM', '162|DOMAIN_16|ACTIVITY_51': u'-- Secrétariat, Dentaire, Social, Esthétique et Autres', '163|DOMAIN_16|ACTIVITY_48': u'-- Soignants - Auxiliaires', '164|DOMAIN_7': u'Pharmacie (Industrie, Officine) - Recherche clinique', '165|DOMAIN_7|ACTIVITY_16': u'-- Industrie Pharmaceutique / Cosmétologique - Biotech', '166|DOMAIN_7|ACTIVITY_17': u'-- Recherche Clinique', '167|DOMAIN_7|ACTIVITY_18': u'-- Pharmacie Officine / Hospit / Para-pharmacie', '168|DOMAIN_7|ACTIVITY_19': u'-- Vente, information et promotion du médicament', '169|DOMAIN_17': u'Télémarketing - Téléservices', '170|DOMAIN_17|ACTIVITY_52': u'-- Téléconseil - Télévente - Autres', '171|DOMAIN_17|ACTIVITY_53': u'-- Direction, Encadrement', '172|DOMAIN_18': u'Tourisme - Loisirs - Spectacle - Audiovisuel', '173|DOMAIN_18|ACTIVITY_54': u'-- Tourisme - Loisirs', '174|DOMAIN_18|ACTIVITY_55': u'-- Spectacle - Audiovisuel', }.iteritems())]) CONFIG = BackendConfig(Value('publication_date', label=u'Publication Date', choices=publicationDate_choices), Value('place', label=u'Place', choices=places_choices), Value('contract', labe=u'Contract type', choices=type_contract_choices), Value('activity_domain', label=u'Activity Domain', choices=activityDomain_choices), ) def search_job(self, pattern=None): with self.browser: for advert in self.browser.search_job(pattern): yield advert def decode_choice(self, place): splitted_choice = place.split('|') part1 = splitted_choice[1].split('_')[1] if len(splitted_choice) == 3: part2 = splitted_choice[2].split('_')[1] return part1, part2 else: return part1, 0 def advanced_search_job(self): region, departement = self.decode_choice(self.config['place'].get()) domain, category = self.decode_choice(self.config['activity_domain'].get()) for advert in self.browser.advanced_search_job(publication_date=int(self.config['publication_date'].get()), contract_type=int(self.config['contract'].get()), conty=departement, region=region, job_category=category, activity_domain=domain ): yield advert def get_job_advert(self, _id, advert=None): with self.browser: return self.browser.get_job_advert(_id, advert) def fill_obj(self, advert, fields): self.get_job_advert(advert.id, advert) OBJECTS = {AdeccoJobAdvert: fill_obj}
class AgendadulibreModule(Module, CapCalendarEvent): NAME = 'agendadulibre' DESCRIPTION = u'agendadulibre website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.6' ASSOCIATED_CATEGORIES = [CATEGORIES.CONF] BROWSER = AgendadulibreBrowser region_choices = OrderedDict([ (k, u'%s (%s)' % (v, k)) for k, v in sorted({ "https://www.agendadulibre.org": u'--France--', "https://www.agendadulibre.org#3": u'Auvergne-Rhône-Alpes', "https://www.agendadulibre.org#5": u'Bourgogne-Franche-Comté', "https://www.agendadulibre.org#6": u'Bretagne', "https://www.agendadulibre.org#7": u'Centre-Val de Loire', "https://www.agendadulibre.org#30": u'Collectivité sui generis', "https://www.agendadulibre.org#29": u'Collectivités d\'outre-mer', "https://www.agendadulibre.org#9": u'Corse', "https://www.agendadulibre.org#1": u'Grand Est', "https://www.agendadulibre.org#23": u'Guadeloupe', "https://www.agendadulibre.org#24": u'Guyane', "https://www.agendadulibre.org#17": u'Hauts-de-France', "https://www.agendadulibre.org#12": u'Île-de-France', "https://www.agendadulibre.org#31": u'Internet', "https://www.agendadulibre.org#26": u'La Réunion', "https://www.agendadulibre.org#25": u'Martinique', "https://www.agendadulibre.org#28": u'Mayotte', "https://www.agendadulibre.org#4": u'Normandie', "https://www.agendadulibre.org#2": u'Nouvelle-Aquitaine', "https://www.agendadulibre.org#13": u'Occitanie', "https://www.agendadulibre.org#18": u'Pays de la Loire', "https://www.agendadulibre.org#21": u'Provence-Alpes-Côte d\'Azur', "https://www.agendadulibre.be": u'--Belgique--', "https://www.agendadulibre.be#11": u'Antwerpen', "https://www.agendadulibre.be#10": u'Brabant wallon', "https://www.agendadulibre.be#9": u'Bruxelles-Capitale', "https://www.agendadulibre.be#8": u'Hainaut', "https://www.agendadulibre.be#7": u'Liege', "https://www.agendadulibre.be#6": u'Limburg', "https://www.agendadulibre.be#5": u'Luxembourg', "https://www.agendadulibre.be#4": u'Namur', "https://www.agendadulibre.be#3": u'Oost-Vlaanderen', "https://www.agendadulibre.be#2": u'Vlaams-Brabant', "https://www.agendadulibre.be#1": u'West-Vlaanderen', "https://www.agendadulibre.ch": u'--Suisse--', "https://www.agendadulibre.ch#15": u'Appenzell Rhodes-Extérieures', "https://www.agendadulibre.ch#16": u'Appenzell Rhodes-Intérieures', "https://www.agendadulibre.ch#19": u'Argovie', "https://www.agendadulibre.ch#13": u'Bâle-Campagne', "https://www.agendadulibre.ch#12": u'Bâle-Ville', "https://www.agendadulibre.ch#2": u'Berne', "https://www.agendadulibre.ch#10": u'Fribourg', "https://www.agendadulibre.ch#25": u'Genève', "https://www.agendadulibre.ch#8": u'Glaris', "https://www.agendadulibre.ch#18": u'Grisons', "https://www.agendadulibre.ch#26": u'Jura', "https://www.agendadulibre.ch#3": u'Lucerne', "https://www.agendadulibre.ch#24": u'Neuchâtel', "https://www.agendadulibre.ch#7": u'Nidwald', "https://www.agendadulibre.ch#6": u'Obwald', "https://www.agendadulibre.ch#17": u'Saint-Gall', "https://www.agendadulibre.ch#14": u'Schaffhouse', "https://www.agendadulibre.ch#5": u'Schwytz', "https://www.agendadulibre.ch#11": u'Soleure', "https://www.agendadulibre.ch#21": u'Tessin', "https://www.agendadulibre.ch#20": u'Thurgovie', "https://www.agendadulibre.ch#4": u'Uri', "https://www.agendadulibre.ch#23": u'Valais', "https://www.agendadulibre.ch#22": u'Vaud', "https://www.agendadulibre.ch#9": u'Zoug', "https://www.agendadulibre.ch#1": u'Zurich', }.items()) ]) CONFIG = BackendConfig( Value('region', label=u'Region', choices=region_choices)) def create_default_browser(self): choice = self.config['region'].get().split('#') selected_region = '' if len(choice) < 2 else choice[-1] return self.create_browser(website=choice[0], region=selected_region) def search_events(self, query): return self.browser.list_events(query.start_date, query.end_date, query.city, query.categories) def list_events(self, date_from, date_to=None): return self.browser.list_events(date_from, date_to) def get_event(self, event_id): return self.browser.get_event(event_id) def fill_obj(self, event, fields): event = self.browser.get_event(event.id, event) choice = self.config['region'].get().split('#') selected_region = '' if len(choice) < 2 else choice[-1] if selected_region == '23': event.timezone = 'America/Guadeloupe' elif selected_region == '24': event.timezone = 'America/Guyana' elif selected_region == '26': event.timezone = 'Indian/Reunion' elif selected_region == '25': event.timezone = 'America/Martinique' else: event.timezone = 'Europe/Paris' return event OBJECTS = {AgendadulibreBrowser: fill_obj}
class EHentaiModule(Module, CapGallery, CapCollection): NAME = 'ehentai' MAINTAINER = u'Roger Philibert' EMAIL = '*****@*****.**' VERSION = '1.1' DESCRIPTION = 'E-Hentai galleries' LICENSE = 'AGPLv3+' BROWSER = EHentaiBrowser CONFIG = BackendConfig( Value('domain', label='Domain', default='g.e-hentai.org'), Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password')) def create_default_browser(self): username = self.config['username'].get() if username: password = self.config['password'].get() else: password = None return self.create_browser(self.config['domain'].get(), username, password) def search_galleries(self, pattern, sortby=None): with self.browser: return self.browser.search_galleries(pattern) def iter_gallery_images(self, gallery): self.fillobj(gallery, ('url', )) with self.browser: return self.browser.iter_gallery_images(gallery) ID_REGEXP = r'/?\d+/[\dabcdef]+/?' URL_REGEXP = r'.+/g/(%s)' % ID_REGEXP def get_gallery(self, _id): match = re.match(r'^%s$' % self.URL_REGEXP, _id) if match: _id = match.group(1) else: match = re.match(r'^%s$' % self.ID_REGEXP, _id) if match: _id = match.group(0) else: return None gallery = EHentaiGallery(_id) with self.browser: if self.browser.gallery_exists(gallery): return gallery else: return None def fill_gallery(self, gallery, fields): if not gallery.__iscomplete__(): with self.browser: self.browser.fill_gallery(gallery, fields) def fill_image(self, image, fields): with self.browser: image.url = self.browser.get_image_url(image) if 'data' in fields: ratelimit("ehentai_get", 2) image.data = self.browser.readurl(image.url) def iter_resources(self, objs, split_path): if BaseGallery in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest_nsfw']) if collection.split_path == [u'latest_nsfw']: for gallery in self.browser.latest_gallery(): yield gallery def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseGallery in objs and collection.split_path == [u'latest_nsfw']: collection.title = u'Latest E-Hentai galleries (NSFW)' return raise CollectionNotFound(collection.split_path) OBJECTS = {EHentaiGallery: fill_gallery, EHentaiImage: fill_image}
class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapChat, ICapContact, ICapAccount): NAME = 'aum' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '0.h' LICENSE = 'AGPLv3+' DESCRIPTION = u'"Adopte un Mec" French dating website' CONFIG = BackendConfig( Value('username', label='Username'), ValueBackendPassword('password', label='Password'), ValueBool('antispam', label='Enable anti-spam', default=False), ValueBool('baskets', label='Get baskets with new messages', default=True), Value('search_query', label='Search query', default='')) STORAGE = { 'profiles_walker': { 'viewed': [] }, 'queries_queue': { 'queue': [] }, 's***s': {}, 'notes': {}, } BROWSER = AuMBrowser MAGIC_ID_BASKET = 1 def __init__(self, *args, **kwargs): BaseBackend.__init__(self, *args, **kwargs) if self.config['antispam'].get(): self.antispam = AntiSpam() else: self.antispam = None def create_default_browser(self): return self.create_browser(self.config['username'].get(), self.config['password'].get(), self.config['search_query'].get()) def report_spam(self, id): with self.browser: pass #self.browser.delete_thread(id) # Do not report fakes to website, to let them to other guys :) #self.browser.report_fake(id) # ---- ICapDating methods --------------------- def init_optimizations(self): self.add_optimization( 'PROFILE_WALKER', ProfilesWalker(self.weboob.scheduler, self.storage, self.browser)) self.add_optimization('VISIBILITY', Visibility(self.weboob.scheduler, self.browser)) self.add_optimization( 'QUERIES_QUEUE', QueriesQueue(self.weboob.scheduler, self.storage, self.browser)) def iter_events(self): all_events = {} with self.browser: all_events[u'baskets'] = (self.browser.get_baskets, 'You were put into %s\'s basket') all_events[u'flashs'] = (self.browser.get_flashs, 'You sent a charm to %s') all_events[u'visits'] = (self.browser.get_visits, 'Visited by %s') for type, (events, message) in all_events.iteritems(): for event in events(): e = Event(event['who']['id']) e.date = parse_dt(event['date']) e.type = type if 'who' in event: e.contact = self._get_partial_contact(event['who']) else: e.contact = self._get_partial_contact(event) if not e.contact: continue e.message = message % e.contact.name yield e # ---- ICapMessages methods --------------------- def fill_thread(self, thread, fields): return self.get_thread(thread) def iter_threads(self): with self.browser: threads = self.browser.get_threads_list() for thread in threads: #if thread['member'].get('isBan', thread['member'].get('dead', False)): # with self.browser: # self.browser.delete_thread(thread['member']['id']) # continue if self.antispam and not self.antispam.check_thread(thread): self.logger.info('Skipped a spam-thread from %s' % thread['pseudo']) self.report_spam(thread['who']['id']) continue t = Thread(int(thread['who']['id'])) t.flags = Thread.IS_DISCUSSION t.title = u'Discussion with %s' % to_unicode( thread['who']['pseudo']) yield t def get_thread(self, id, contacts=None, get_profiles=False): """ Get a thread and its messages. The 'contacts' parameters is only used for internal calls. """ thread = None if isinstance(id, Thread): thread = id id = thread.id if not thread: thread = Thread(int(id)) thread.flags = Thread.IS_DISCUSSION full = False else: full = True with self.browser: mails = self.browser.get_thread_mails(id, 100) my_name = self.browser.get_my_name() child = None msg = None s**t = self._get_slut(id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['who']['pseudo'] self.storage.set('s***s', int(thread.id), 'status', mails['status']) self.storage.save() for mail in mails['results']: flags = 0 if self.antispam and not self.antispam.check_mail(mail): self.logger.info('Skipped a spam-mail from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if parse_dt(mail['date']) > s**t['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: if not mail['from'] in contacts: try: with self.browser: contacts[mail['from']] = self.get_contact( mail['from']) except BrowserHTTPNotFound: pass if self.antispam and mail[ 'from'] in contacts and not self.antispam.check_contact( contacts[mail['from']]): self.logger.info( 'Skipped a spam-mail-profile from %s' % mails['who']['pseudo']) self.report_spam(thread.id) break if int(mail['from']) == self.browser.my_id: if mails['remote_status'] == 'new' and msg is None: flags |= Message.IS_NOT_RECEIVED else: flags |= Message.IS_RECEIVED signature = u'' #if mail.get('src', None): # signature += u'Sent from my %s\n\n' % mail['src'] if mail['from'] in contacts: signature += contacts[mail['from']].get_text() msg = Message( thread=thread, id=int( time.strftime('%Y%m%d%H%M%S', parse_dt(mail['date']).timetuple())), title=thread.title, sender=to_unicode(my_name if int(mail['from']) == self.browser. my_id else mails['who']['pseudo']), receivers=[ to_unicode(my_name if int(mail['from']) != self.browser. my_id else mails['who']['pseudo']) ], date=parse_dt(mail['date']), content=to_unicode(unescape(mail['message'] or '').strip()), signature=signature, children=[], flags=flags) if child: msg.children.append(child) child.parent = msg child = msg if full and msg: # If we have get all the messages, replace NotLoaded with None as # parent. msg.parent = None if not full and not msg: # Perhaps there are hidden messages msg = NotLoaded thread.root = msg return thread def iter_unread_messages(self): try: contacts = {} with self.browser: threads = self.browser.get_threads_list() for thread in threads: #if thread['member'].get('isBan', thread['member'].get('dead', False)): # with self.browser: # self.browser.delete_thread(int(thread['member']['id'])) # continue if self.antispam and not self.antispam.check_thread(thread): self.logger.info('Skipped a spam-unread-thread from %s' % thread['who']['pseudo']) self.report_spam(thread['member']['id']) continue s**t = self._get_slut(thread['who']['id']) if parse_dt( thread['date'] ) > s**t['lastmsg'] or thread['status'] != s**t['status']: t = self.get_thread(thread['who']['id'], contacts, get_profiles=True) for m in t.iter_all_messages(): if m.flags & m.IS_UNREAD: yield m if not self.config['baskets'].get(): return # Send mail when someone added me in her basket. # XXX possibly race condition if a s**t adds me in her basket # between the aum.nb_new_baskets() and aum.get_baskets(). with self.browser: s**t = self._get_slut(-self.MAGIC_ID_BASKET) new_baskets = self.browser.nb_new_baskets() if new_baskets > 0: baskets = self.browser.get_baskets() my_name = self.browser.get_my_name() for basket in baskets: if parse_dt(basket['date']) <= s**t['lastmsg']: continue contact = self.get_contact(basket['who']['id']) if self.antispam and not self.antispam.check_contact( contact): self.logger.info('Skipped a spam-basket from %s' % contact.name) self.report_spam(basket['who']['id']) continue thread = Thread(int(basket['who']['id'])) thread.title = 'Basket of %s' % contact.name thread.root = Message( thread=thread, id=self.MAGIC_ID_BASKET, title=thread.title, sender=contact.name, receivers=[my_name], date=parse_dt(basket['date']), content='You are taken in her basket!', signature=contact.get_text(), children=[], flags=Message.IS_UNREAD) yield thread.root except BrowserUnavailable as e: self.logger.debug('No messages, browser is unavailable: %s' % e) pass # don't care about waiting def set_message_read(self, message): if int(message.id) == self.MAGIC_ID_BASKET: # Save the last baskets checks. s**t = self._get_slut(-self.MAGIC_ID_BASKET) if s**t['lastmsg'] < message.date: s**t['lastmsg'] = message.date self.storage.set('s***s', -self.MAGIC_ID_BASKET, s**t) self.storage.save() return s**t = self._get_slut(message.thread.id) if s**t['lastmsg'] < message.date: s**t['lastmsg'] = message.date self.storage.set('s***s', int(message.thread.id), s**t) self.storage.save() def _get_slut(self, id): id = int(id) s***s = self.storage.get('s***s') if not s***s or not id in s***s: s**t = {'lastmsg': datetime.datetime(1970, 1, 1), 'status': None} else: s**t = self.storage.get('s***s', id) s**t['lastmsg'] = s**t.get('lastmsg', datetime.datetime( 1970, 1, 1)).replace(tzinfo=tz.tzutc()) s**t['status'] = s**t.get('status', None) return s**t # ---- ICapMessagesPost methods --------------------- def post_message(self, message): with self.browser: self.browser.post_mail(message.thread.id, message.content) # ---- ICapContact methods --------------------- def fill_contact(self, contact, fields): if 'profile' in fields: contact = self.get_contact(contact) if contact and 'photos' in fields: for name, photo in contact.photos.iteritems(): with self.browser: if photo.url and not photo.data: data = self.browser.openurl(photo.url).read() contact.set_photo(name, data=data) if photo.thumbnail_url and not photo.thumbnail_data: data = self.browser.openurl(photo.thumbnail_url).read() contact.set_photo(name, thumbnail_data=data) def fill_photo(self, photo, fields): with self.browser: if 'data' in fields and photo.url and not photo.data: photo.data = self.browser.readurl(photo.url) if 'thumbnail_data' in fields and photo.thumbnail_url and not photo.thumbnail_data: photo.thumbnail_data = self.browser.readurl( photo.thumbnail_url) return photo def get_contact(self, contact): with self.browser: if isinstance(contact, Contact): _id = contact.id elif isinstance(contact, (int, long, basestring)): _id = contact else: raise TypeError( "The parameter 'contact' isn't a contact nor a int/long/str/unicode: %s" % contact) profile = self.browser.get_full_profile(_id) if not profile: return None _id = profile['id'] if isinstance(contact, Contact): contact.id = _id contact.name = profile['pseudo'] else: contact = Contact(_id, profile['pseudo'], Contact.STATUS_ONLINE) contact.url = self.browser.id2url(_id) contact.parse_profile(profile, self.browser.get_consts()) return contact def _get_partial_contact(self, contact): s = 0 if contact.get('online', False): s = Contact.STATUS_ONLINE else: s = Contact.STATUS_OFFLINE c = Contact(contact['id'], to_unicode(contact['pseudo']), s) c.url = self.browser.id2url(contact['id']) if 'age' in contact: c.status_msg = u'%s old, %s' % (contact['age'], contact['city']) if contact['cover'] is not None: url = contact['cover'] + '/%(type)s' else: url = u'http://s.adopteunmec.com/www/img/thumb0.jpg' c.set_photo(u'image%s' % contact['cover'], url=url % {'type': 'full'}, thumbnail_url=url % {'type': 'small'}) return c def iter_contacts(self, status=Contact.STATUS_ALL, ids=None): with self.browser: threads = self.browser.get_threads_list(count=100) for thread in threads: c = self._get_partial_contact(thread['who']) if c and (c.status & status) and (not ids or c.id in ids): yield c def send_query(self, id): if isinstance(id, Contact): id = id.id queries_queue = None try: queries_queue = self.get_optimization('QUERIES_QUEUE') except OptimizationNotFound: pass if queries_queue and queries_queue.is_running(): if queries_queue.enqueue_query(id): return Query(id, 'A charm has been sent') else: return Query(id, 'Unable to send charm: it has been enqueued') else: with self.browser: if not self.browser.send_charm(id): raise QueryError('No enough charms available') return Query(id, 'A charm has been sent') def get_notes(self, id): if isinstance(id, Contact): id = id.id return self.storage.get('notes', id) def save_notes(self, id, notes): if isinstance(id, Contact): id = id.id self.storage.set('notes', id, notes) self.storage.save() # ---- ICapChat methods --------------------- def iter_chat_messages(self, _id=None): with self.browser: return self.browser.iter_chat_messages(_id) def send_chat_message(self, _id, message): with self.browser: return self.browser.send_chat_message(_id, message) #def start_chat_polling(self): #self._profile_walker = ProfilesWalker(self.weboob.scheduler, self.storage, self.browser) # ---- ICapAccount methods --------------------- ACCOUNT_REGISTER_PROPERTIES = ValuesDict( Value('username', label='Email address', regexp='^[^ ]+@[^ ]+\.[^ ]+$'), Value('password', label='Password', regexp='^[^ ]+$', masked=True), Value('sex', label='Sex', choices={ 'm': 'Male', 'f': 'Female' }), Value('birthday', label='Birthday (dd/mm/yyyy)', regexp='^\d+/\d+/\d+$'), Value('zipcode', label='Zipcode'), Value('country', label='Country', choices={ 'fr': 'France', 'be': 'Belgique', 'ch': 'Suisse', 'ca': 'Canada' }, default='fr'), Value('godfather', label='Godfather', regexp='^\d*$', default=''), ) @classmethod def register_account(klass, account): """ Register an account on website This is a static method, it would be called even if the backend is instancied. @param account an Account object which describe the account to create """ browser = None bday, bmonth, byear = account.properties['birthday'].get().split( '/', 2) while not browser: try: browser = klass.BROWSER(account.properties['username'].get()) browser.register( password=account.properties['password'].get(), sex=(0 if account.properties['sex'].get() == 'm' else 1), birthday_d=int(bday), birthday_m=int(bmonth), birthday_y=int(byear), zipcode=account.properties['zipcode'].get(), country=account.properties['country'].get(), godfather=account.properties['godfather'].get()) except CaptchaError: getLogger('aum').info('Unable to resolve captcha. Retrying...') browser = None REGISTER_REGEXP = re.compile( '.*http://www.adopteunmec.com/register4.php\?([^\' ]*)\'') def confirm_account(self, mail): msg = email.message_from_string(mail) content = u'' for part in msg.walk(): s = part.get_payload(decode=True) content += unicode(s, 'iso-8859-15') url = None for s in content.split(): m = self.REGISTER_REGEXP.match(s) if m: url = '/register4.php?' + m.group(1) break if url: browser = self.create_browser('') browser.openurl(url) return True return False def get_account(self): """ Get the current account. """ raise NotImplementedError() def update_account(self, account): """ Update the current account. """ raise NotImplementedError() def get_account_status(self): with self.browser: return ( StatusField('myname', 'My name', self.browser.get_my_name()), StatusField('score', 'Score', self.browser.score()), StatusField('avcharms', 'Available charms', self.browser.nb_available_charms()), StatusField('newvisits', 'New visits', self.browser.nb_new_visites()), ) OBJECTS = { Thread: fill_thread, Contact: fill_contact, ContactPhoto: fill_photo }
class PhpBBModule(Module, CapMessages, CapMessagesPost): NAME = 'phpbb' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.1' LICENSE = 'AGPLv3+' DESCRIPTION = "phpBB forum" CONFIG = BackendConfig( Value('url', label='URL of forum', regexp='https?://.*'), Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default=''), ValueInt( 'thread_unread_messages', label='Limit number of unread messages to retrieve for a thread', default=500)) STORAGE = {'seen': {}} BROWSER = PhpBB def create_default_browser(self): username = self.config['username'].get() if len(username) > 0: password = self.config['password'].get() else: password = None return self.create_browser(self.config['url'].get(), username, password) #### CapMessages ############################################## def _iter_threads(self, root_link=None): with self.browser: links = list( self.browser.iter_links(root_link.url if root_link else None)) for link in links: if link.type == link.FORUM: link.title = '%s[%s]' % (root_link.title if root_link else '', link.title) for thread in self._iter_threads(link): yield thread if link.type == link.TOPIC: thread = Thread(url2id(link.url)) thread.title = ('%s ' % root_link.title if root_link else '') + link.title thread.date = link.date thread.flags = thread.IS_DISCUSSION yield thread def iter_threads(self): return self._iter_threads() def get_thread(self, id): thread = None parent = None if isinstance(id, Thread): thread = id id = thread.id thread_id = url2id(id, nopost=True) or id try: last_seen_id = self.storage.get('seen', default={})[id2topic(thread_id)] except KeyError: last_seen_id = 0 with self.browser: for post in self.browser.iter_posts(id): if not thread: thread = Thread(thread_id) thread.title = post.title m = self._post2message(thread, post) m.parent = parent if last_seen_id < post.id: m.flags |= Message.IS_UNREAD if parent: parent.children = [m] else: thread.root = m parent = m return thread def _post2message(self, thread, post): signature = post.signature if signature: signature += '<br />' signature += 'URL: %s' % self.browser.absurl( id2url('%s.%s' % (thread.id, post.id))) return Message(thread=thread, id=post.id, title=post.title, sender=post.author, receivers=None, date=post.date, parent=None, content=post.content, signature=signature, children=[], flags=Message.IS_HTML) def iter_unread_messages(self): with self.browser: url = self.browser.get_root_feed_url() for article in Newsfeed(url, rssid).iter_entries(): id = url2id(article.link) thread = None try: last_seen_id = self.storage.get('seen', default={})[id2topic(id)] except KeyError: last_seen_id = 0 child = None iterator = self.browser.riter_posts(id, last_seen_id) if self.config['thread_unread_messages'].get() > 0: iterator = limit( iterator, self.config['thread_unread_messages'].get()) for post in iterator: if not thread: thread = Thread('%s.%s' % (post.forum_id, post.topic_id)) message = self._post2message(thread, post) if child: message.children.append(child) child.parent = message if post.parent: message.parent = Message(thread=thread, id=post.parent) else: thread.root = message yield message def set_message_read(self, message): try: last_seen_id = self.storage.get('seen', default={})[id2topic( message.thread.id)] except KeyError: last_seen_id = 0 if message.id > last_seen_id: self.storage.set('seen', id2topic(message.thread.id), message.id) self.storage.save() def fill_thread(self, thread, fields): return self.get_thread(thread) #### CapMessagesReply ######################################### def post_message(self, message): assert message.thread forum = 0 topic = 0 if message.thread: try: if '.' in message.thread.id: forum, topic = [ int(i) for i in message.thread.id.split('.', 1) ] else: forum = int(message.thread.id) except ValueError: raise CantSendMessage( 'Thread ID must be in form "FORUM_ID[.TOPIC_ID]".') with self.browser: return self.browser.post_answer(forum, topic, message.title, message.content) OBJECTS = {Thread: fill_thread}
class CragrModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, CapProfile): NAME = 'cragr' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.4' DESCRIPTION = u'Crédit Agricole' LICENSE = 'AGPLv3+' website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'm.ca-alpesprovence.fr': u'Alpes Provence', 'm.ca-alsace-vosges.fr': u'Alsace-Vosges', 'm.ca-anjou-maine.fr': u'Anjou Maine', 'm.ca-aquitaine.fr': u'Aquitaine', 'm.ca-atlantique-vendee.fr': u'Atlantique Vendée', 'm.ca-briepicardie.fr': u'Brie Picardie', 'm.ca-cb.fr': u'Champagne Bourgogne', 'm.ca-centrefrance.fr': u'Centre France', 'm.ca-centreloire.fr': u'Centre Loire', 'm.ca-centreouest.fr': u'Centre Ouest', 'm.ca-centrest.fr': u'Centre Est', 'm.ca-charente-perigord.fr': u'Charente Périgord', 'm.ca-cmds.fr': u'Charente-Maritime Deux-Sèvres', 'm.ca-corse.fr': u'Corse', 'm.ca-cotesdarmor.fr': u'Côtes d\'Armor', 'm.ca-des-savoie.fr': u'Des Savoie', 'm.ca-finistere.fr': u'Finistere', 'm.ca-franchecomte.fr': u'Franche-Comté', 'm.ca-guadeloupe.fr': u'Guadeloupe', 'm.ca-illeetvilaine.fr': u'Ille-et-Vilaine', 'm.ca-languedoc.fr': u'Languedoc', 'm.ca-loirehauteloire.fr': u'Loire Haute Loire', 'm.ca-lorraine.fr': u'Lorraine', 'm.ca-martinique.fr': u'Martinique Guyane', 'm.ca-morbihan.fr': u'Morbihan', 'm.ca-nmp.fr': u'Nord Midi-Pyrénées', 'm.ca-nord-est.fr': u'Nord Est', 'm.ca-norddefrance.fr': u'Nord de France', 'm.ca-normandie-seine.fr': u'Normandie Seine', 'm.ca-normandie.fr': u'Normandie', 'm.ca-paris.fr': u'Ile-de-France', 'm.ca-pca.fr': u'Provence Côte d\'Azur', 'm.ca-reunion.fr': u'Réunion', 'm.ca-sudmed.fr': u'Sud Méditerranée', 'm.ca-sudrhonealpes.fr': u'Sud Rhône Alpes', 'm.ca-toulouse31.fr': u'Toulouse 31', # m.ca-toulousain.fr redirects here 'm.ca-tourainepoitou.fr': u'Tourraine Poitou', 'm.ca-valdefrance.fr': u'Val de France', 'm.lefil.com': u'Pyrénées Gascogne', }.items())]) CONFIG = BackendConfig(Value('website', label=u'Région', choices=website_choices), ValueBackendPassword('login', label=u'N° de compte', masked=False), ValueBackendPassword('password', label=u'Code personnel', regexp=r'\d{6}')) BROWSER = Cragr COMPAT_DOMAINS = { 'm.lefil.com': 'm.ca-pyrenees-gascogne.fr', } def create_default_browser(self): site_conf = self.config['website'].get() site_conf = self.COMPAT_DOMAINS.get(site_conf, site_conf) return self.create_browser(site_conf, self.config['login'].get(), self.config['password'].get()) def iter_accounts(self): return self.browser.get_accounts_list() def get_account(self, _id): return find_object(self.iter_accounts(), id=_id, error=AccountNotFound) def _history_filter(self, account, coming): today = date.today() def to_date(obj): if hasattr(obj, 'date'): return obj.date() return obj for tr in self.browser.get_history(account): tr_coming = to_date(tr.date) > today if coming == tr_coming: yield tr def iter_history(self, account): if account.type == Account.TYPE_CARD: return self._history_filter(account, False) return self.browser.get_history(account) def iter_coming(self, account): if account.type == Account.TYPE_CARD: return self._history_filter(account, True) return [] def iter_investment(self, account): for inv in self.browser.iter_investment(account): yield inv def iter_contacts(self): return self.browser.iter_advisor() def get_profile(self): if not hasattr(self.browser, 'get_profile'): raise NotImplementedError() return self.browser.get_profile() def iter_transfer_recipients(self, account): if not isinstance(account, Account): account = self.get_account(account) return self.browser.iter_transfer_recipients(account) def init_transfer(self, transfer, **params): def to_ascii(s): return s.encode('ascii', errors='ignore').decode('ascii') if transfer.label: transfer.label = re.sub(r'[+!]', '', to_ascii(transfer.label[:33])) return self.browser.init_transfer(transfer, **params) def execute_transfer(self, transfer, **params): return self.browser.execute_transfer(transfer, **params) def new_recipient(self, recipient, **params): return self.browser.new_recipient(recipient, **params)
class RedditModule(Module, CapImage, CapCollection, CapMessages): NAME = 'reddit' DESCRIPTION = u'reddit website' MAINTAINER = u'Vincent A' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.6' CONFIG = BackendConfig( Value('subreddit', label='Name of the sub-reddit', regexp='[^/]+', default='pics'), ) BROWSER = RedditBrowser def create_default_browser(self): return self.create_browser(self.config['subreddit'].get()) def get_file(self, _id): raise NotImplementedError() def get_image(self, id): return self.browser.get_image(id) def search_file(self, pattern, sortby=CapImage.SEARCH_RELEVANCE): return self.browser.search_images(pattern, sortby, True) def search_image(self, pattern, sortby=CapImage.SEARCH_RELEVANCE, nsfw=False): sorting = { CapImage.SEARCH_RELEVANCE: 'relevance', CapImage.SEARCH_RATING: 'top', CapImage.SEARCH_VIEWS: 'top', # not implemented CapImage.SEARCH_DATE: 'new', } sortby = sorting[sortby] return self.browser.search_images(pattern, sortby, nsfw) def iter_threads(self): return self.browser.iter_threads() def get_thread(self, id): return self.browser.get_thread(id) def iter_resources(self, objs, split_path): for k in self.RESOURCES: if len(k) == len(split_path) and all( a is None or a == b for a, b in zip(k, split_path)): f = self.RESOURCES[k] return f(self, objs, *split_path) RESOURCES = {} @register_resources_handler(RESOURCES) def iter_resources_root(self, objs): return [ Collection(['hot'], 'Hot threads'), Collection(['new'], 'New threads'), Collection(['rising'], 'Rising threads'), Collection(['controversial'], 'Controversial threads'), Collection(['top'], 'Top threads'), ] @register_resources_handler(RESOURCES, None) def iter_resources_dir(self, objs, key): if key == 'hot': key = '' if Thread in objs: return self.iter_threads(cat=key) if BaseImage in objs: return self.browser.iter_images(cat=key) return [] def fill_data(self, obj, fields): if 'thumbnail' in fields and not obj.thumbnail.data: obj.thumbnail.data = self.browser.open(obj.thumbnail.url).content if 'data' in fields: obj.data = self.browser.open(obj.url).content def fill_thread(self, obj, fields): if 'root' in fields: self.browser.fill_thread(obj) OBJECTS = { BaseImage: fill_data, Thumbnail: fill_data, Thread: fill_thread, }
class TinderModule(Module, CapMessages, CapMessagesPost, CapDating): NAME = 'tinder' DESCRIPTION = u'Tinder dating mobile application' MAINTAINER = u'Roger Philibert' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.4' CONFIG = BackendConfig( Value('username', label='Facebook email'), ValueBackendPassword('password', label='Facebook password'), Value('location', label='Location (example: 49.6008457,6.129709)', default='')) BROWSER = TinderBrowser STORAGE = { 'contacts': {}, } def create_default_browser(self): facebook = self.create_browser(klass=FacebookBrowser) facebook.login(self.config['username'].get(), self.config['password'].get()) return self.create_browser(facebook, self.config['location'].get()) # ---- CapDating methods ----------------------- def init_optimizations(self): self.add_optimization( 'PROFILE_WALKER', ProfilesWalker(self.weboob.scheduler, self.storage, self.browser)) # ---- CapMessages methods --------------------- def fill_thread(self, thread, fields): return self.get_thread(thread) def iter_threads(self): for thread in self.browser.get_threads(): if 'person' not in thread: # The account has been removed, probably because it was a # spammer. continue t = Thread(thread['_id']) t.flags = Thread.IS_DISCUSSION t.title = u'Discussion with %s' % thread['person']['name'] contact = self.storage.get('contacts', t.id, default={'lastmsg': 0}) birthday = parse_date(thread['person']['birth_date']).date() signature = u'Age: %d (%s)' % ( (datetime.date.today() - birthday).days / 365.25, birthday) signature += u'\nLast ping: %s' % parse_date( thread['person']['ping_time']).strftime('%Y-%m-%d %H:%M:%S') signature += u'\nPhotos:\n\t%s' % '\n\t'.join( [photo['url'] for photo in thread['person']['photos']]) signature += u'\n\n%s' % thread['person'].get('bio', '') t.root = Message( thread=t, id=1, title=t.title, sender=unicode(thread['person']['name']), receivers=[self.browser.my_name], date=parse_date(thread['created_date']), content=u'Match!', children=[], signature=signature, flags=Message.IS_UNREAD if int(contact['lastmsg']) < 1 else 0) parent = t.root for msg in thread['messages']: flags = 0 if int(contact['lastmsg']) < msg['timestamp']: flags = Message.IS_UNREAD msg = Message( thread=t, id=msg['timestamp'], title=t.title, sender=unicode( self.browser.my_name if msg['from'] == self.browser.my_id else thread['person']['name']), receivers=[ unicode(self.browser.my_name if msg['to'] == self. browser.my_id else thread['person']['name']) ], date=parse_date(msg['sent_date']), content=unicode(msg['message']), children=[], parent=parent, signature=signature if msg['to'] == self.browser.my_id else u'', flags=flags) parent.children.append(msg) parent = msg yield t def get_thread(self, _id): for t in self.iter_threads(): if t.id == _id: return t def iter_unread_messages(self): for thread in self.iter_threads(): for message in thread.iter_all_messages(): if message.flags & message.IS_UNREAD: yield message def set_message_read(self, message): contact = self.storage.get('contacts', message.thread.id, default={'lastmsg': 0}) if int(contact['lastmsg']) < int(message.id): contact['lastmsg'] = int(message.id) self.storage.set('contacts', message.thread.id, contact) self.storage.save() # ---- CapMessagesPost methods --------------------- def post_message(self, message): self.browser.post_message(message.thread.id, message.content) OBJECTS = { Thread: fill_thread, }
class AferModule(Module, CapBankWealth): NAME = 'afer' DESCRIPTION = u'afer website' MAINTAINER = u'James GALT' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.4' BROWSER = AferBrowser CONFIG = BackendConfig( ValueBackendPassword('login', label='Username', regexp='[A-z]\d+', masked=False), ValueBackendPassword('password', label=u"mdp", regexp='\d+')) def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) def get_account(self, id): """ Get an account from its ID. :param id: ID of the account :type id: :class:`str` :rtype: :class:`Account` :raises: :class:`AccountNotFound` """ return find_object(self.iter_accounts(), id=id, error=AccountNotFound) def iter_accounts(self): """ Iter accounts. :rtype: iter[:class:`Account`] """ return self.browser.iter_accounts() def iter_coming(self, account): """ Iter coming transactions on a specific account. :param account: account to get coming transactions :type account: :class:`Account` :rtype: iter[:class:`Transaction`] :raises: :class:`AccountNotFound` """ raise NotImplementedError() def iter_history(self, account): """ Iter history of transactions on a specific account. :param account: account to get history :type account: :class:`Account` :rtype: iter[:class:`Transaction`] :raises: :class:`AccountNotFound` """ return self.browser.iter_history(account) def iter_investment(self, account): """ Iter investment of a market account :param account: account to get investments :type account: :class:`Account` :rtype: iter[:class:`Investment`] :raises: :class:`AccountNotFound` """ return self.browser.iter_investments(account)
class INGModule(Module, CapBank, CapBill): NAME = 'ing' MAINTAINER = u'Florent Fourcot' EMAIL = '*****@*****.**' VERSION = '1.2' LICENSE = 'AGPLv3+' DESCRIPTION = 'ING Direct' CONFIG = BackendConfig( ValueBackendPassword('login', label=u'Numéro client', masked=False), ValueBackendPassword('password', label='Code secret', regexp='^(\d{6}|)$'), ValueBackendPassword('birthday', label='Date de naissance', regexp='^(\d{2}[/-]?\d{2}[/-]?\d{4}|)$', masked=False)) BROWSER = IngBrowser def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get(), birthday=self.config['birthday'].get()) def iter_resources(self, objs, split_path): if Account in objs: self._restrict_level(split_path) return self.iter_accounts() if Subscription in objs: self._restrict_level(split_path) return self.iter_subscription() def iter_accounts(self): return self.browser.get_accounts_list() def get_account(self, _id): return self.browser.get_account(_id) def iter_history(self, account): if not isinstance(account, Account): account = self.get_account(account) return self.browser.get_history(account) def iter_transfer_recipients(self, account): if not isinstance(account, Account): account = self.get_account(account) return self.browser.get_recipients(account) def transfer(self, account, recipient, amount, reason): if not reason: raise UserError( 'Reason is mandatory to do a transfer on ING website') if not isinstance(account, Account): account = self.get_account(account) if not isinstance(recipient, Recipient): # Remove weboob identifier prefix (LA-, CC-...) if "-" in recipient: recipient = recipient.split('-')[1] return self.browser.transfer(account, recipient, amount, reason) def iter_investment(self, account): if not isinstance(account, Account): account = self.get_account(account) return self.browser.get_investments(account) def iter_coming(self, account): if not isinstance(account, Account): account = self.get_account(account) return self.browser.get_coming(account) def iter_subscription(self): return self.browser.get_subscriptions() def get_subscription(self, _id): return find_object(self.browser.get_subscriptions(), id=_id, error=SubscriptionNotFound) def get_bill(self, _id): subscription = self.get_subscription(_id.split('-')[0]) return find_object(self.browser.get_bills(subscription), id=_id, error=BillNotFound) def iter_bills(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) return self.browser.get_bills(subscription) def download_bill(self, bill): if not isinstance(bill, Bill): bill = self.get_bill(bill) self.browser.predownload(bill) assert ( self.browser.response.headers['content-type'] == "application/pdf") return self.browser.response.content
class HappnModule(Module, CapMessages, CapMessagesPost, CapDating, CapContact): NAME = 'happn' DESCRIPTION = u'Happn dating mobile application' MAINTAINER = u'Roger Philibert' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.6' CONFIG = BackendConfig( Value('username', label='Facebook email'), ValueBackendPassword('password', label='Facebook password'), Value('location', label='Location (example: 49.6008457,6.129709)'), Value('distance', label='Distance (in km) around your location you\'re walking')) BROWSER = HappnBrowser STORAGE = { 'contacts': {}, } def create_default_browser(self): facebook = self.create_browser(klass=FacebookBrowser) facebook.login(self.config['username'].get(), self.config['password'].get()) return self.create_browser(facebook) # ---- CapDating methods ----------------------- def init_optimizations(self): self.add_optimization( 'PROFILE_WALKER', ProfilesWalker(self.weboob.scheduler, self.storage, self.browser, self.config['location'].get(), self.config['distance'].get())) # ---- CapMessages methods --------------------- def fill_thread(self, thread, fields): return self.get_thread(thread) def iter_threads(self): for thread in self.browser.get_threads(): t = Thread(thread['id']) t.flags = Thread.IS_DISCUSSION for user in thread['participants']: if user['user']['id'] != self.browser.my_id: t.title = u'Discussion with %s' % user['user'][ 'display_name'] t.date = parse_date(thread['modification_date']) yield t def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION info = self.browser.get_thread(thread.id) for user in info['participants']: if user['user']['fb_id'] is not None: user['user']['fb'] = self.browser.get_facebook( user['user']['fb_id']) if user['user']['id'] == self.browser.my_id: me = HappnContact(user['user']) else: other = HappnContact(user['user']) thread.title = u'Discussion with %s' % other.name contact = self.storage.get( 'contacts', thread.id, default={'lastmsg_date': '1970-01-01T01:01:01+00:00'}) child = None for msg in info['messages']: flags = 0 if parse_date(contact['lastmsg_date']) < parse_date( msg['creation_date']): flags = Message.IS_UNREAD if msg['sender']['id'] == me.id: sender = me receiver = other else: sender = other receiver = me msg = Message(thread=thread, id=msg['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=parse_date(msg['creation_date']), content=msg['message'], children=[], parent=None, signature=sender.get_text(), flags=flags) if child: msg.children.append(child) child.parent = msg child = msg thread.root = child return thread def iter_unread_messages(self): for thread in self.iter_threads(): thread = self.get_thread(thread) for message in thread.iter_all_messages(): if message.flags & message.IS_UNREAD: yield message def set_message_read(self, message): contact = self.storage.get( 'contacts', message.thread.id, default={'lastmsg_date': '1970-01-01T01:01:01+00:00'}) if parse_date(contact['lastmsg_date']) < message.date: contact['lastmsg_date'] = str(message.date) self.storage.set('contacts', message.thread.id, contact) self.storage.save() # ---- CapMessagesPost methods --------------------- def post_message(self, message): self.browser.post_message(message.thread.id, message.content) # ---- CapContact methods --------------------- def get_contact(self, contact_id): if isinstance(contact_id, Contact): contact_id = contact_id.id info = self.browser.get_contact(contact_id) return HappnContact(info) OBJECTS = { Thread: fill_thread, }
class ArretSurImagesModule(Module, CapVideo, CapCollection): NAME = 'arretsurimages' DESCRIPTION = u'arretsurimages website' MAINTAINER = u'franek' EMAIL = '*****@*****.**' VERSION = '1.3' CONFIG = BackendConfig( ValueBackendPassword('login', label='email', masked=False), ValueBackendPassword('password', label='Password')) BROWSER = ArretSurImagesBrowser def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get(), get_home=False) def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): with self.browser: return self.browser.search_videos(pattern) # raise UserError('Search does not work on ASI website, use ls latest command') def get_video(self, _id): if _id.startswith('http://') and not _id.startswith( 'http://www.arretsurimages.net'): return None with self.browser: return self.browser.get_video(_id) def fill_video(self, video, fields): if fields != ['thumbnail']: # if we don't want only the thumbnail, we probably want also every fields with self.browser: video = self.browser.get_video( ArretSurImagesVideo.id2url(video.id), video) if 'thumbnail' in fields and video.thumbnail: with self.browser: video.thumbnail.data = self.browser.readurl( video.thumbnail.url) return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest']) if collection.split_path == [u'latest']: for video in self.browser.latest_videos(): yield video def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and collection.split_path == [u'latest']: collection.title = u'Latest ArretSurImages videos' return raise CollectionNotFound(collection.split_path) OBJECTS = {ArretSurImagesVideo: fill_video}
class AmazonModule(Module, CapShop, CapDocument): NAME = 'amazon' MAINTAINER = u'Oleg Plakhotniuk' EMAIL = '*****@*****.**' VERSION = '1.2' LICENSE = 'AGPLv3+' DESCRIPTION = u'Amazon' website_choices = OrderedDict([ (k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'www.amazon.com': u'Amazon.com', 'www.amazon.fr': u'Amazon France', }.iteritems()) ]) BROWSERS = { 'www.amazon.com': Amazon, 'www.amazon.fr': AmazonFR, } CONFIG = BackendConfig( Value('website', label=u'Website', choices=website_choices, default='www.amazon.com'), ValueBackendPassword('email', label='Username', masked=False), ValueBackendPassword('password', label='Password')) def create_default_browser(self): self.BROWSER = self.BROWSERS[self.config['website'].get()] return self.create_browser(self.config['email'].get(), self.config['password'].get()) def get_currency(self): return self.browser.get_currency() def get_order(self, id_): return self.browser.get_order(id_) def iter_orders(self): return self.browser.iter_orders() def iter_payments(self, order): if not isinstance(order, Order): order = self.get_order(order) return self.browser.iter_payments(order) def iter_items(self, order): if not isinstance(order, Order): order = self.get_order(order) return self.browser.iter_items(order) def iter_resources(self, objs, split_path): if Order in objs: self._restrict_level(split_path) return self.iter_orders() if Subscription in objs: self._restrict_level(split_path) return self.iter_subscription() def iter_subscription(self): return self.browser.get_subscription_list() def get_subscription(self, _id): return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) def get_document(self, _id): subid = _id.split('.')[0] subscription = self.get_subscription(subid) return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) def iter_documents(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) return self.browser.iter_documents(subscription) def download_document(self, bill): if not isinstance(bill, Bill): bill = self.get_document(bill) if bill._url: return self.browser.open(bill._url).content return None
from weboob.tools.value import Value, ValueBackendPassword class MockModule(object): def __init__(self, name, description, config, backend="manual"): self.name = name self.description = description self.config = config self.backend = backend # Officially deprecated modules. They are listed for backwards compatibility and to allow the user to # look at their past transactions. DEPRECATED_MODULES = [ MockModule('wellsfargo', 'Wells Fargo', BackendConfig(Value('login'), ValueBackendPassword('password'))), MockModule('citelis', 'City Bank', BackendConfig(Value('login'), ValueBackendPassword('password'))), MockModule('alloresto', 'Allo Resto', BackendConfig(Value('login'), ValueBackendPassword('password'))), ] # The known modules to be ignored either because they are only called by another module, # or because they are deprecated. IGNORE_MODULE_LIST = ['s2e', 'linebourse', 'groupama' ] + [m.name for m in DEPRECATED_MODULES] MANUAL_MODULES = [ MockModule('manual',
class GithubModule(Module, CapBugTracker): NAME = 'github' DESCRIPTION = u'GitHub issues tracking' MAINTAINER = u'Vincent A' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.2' CONFIG = BackendConfig( Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default='')) BROWSER = GithubBrowser def create_default_browser(self): username = self.config['username'].get() if username: password = self.config['password'].get() else: password = None return self.create_browser(username, password) def get_project(self, _id): d = self.browser.get_project(_id) project = Project(_id, d['name']) project.members = list(self._iter_members(project.id)) project.statuses = list(STATUSES.values()) project.categories = [] project.versions = list(self._iter_versions(project.id)) return project def get_issue(self, _id): project_id, issue_number = self._extract_issue_id(_id) project = self.get_project(project_id) d = self.browser.get_issue(project_id, issue_number) issue = self._make_issue(d, project) if d['has_comments']: self._fetch_comments(issue) return issue def iter_issues(self, query): if ((query.assignee, query.author, query.status, query.title) == (None, None, None, None)): it = self.browser.iter_project_issues(query.project) else: it = self.browser.iter_issues(query) project = self.get_project(query.project) for d in it: issue = self._make_issue(d, project) yield issue def create_issue(self, project_id): issue = Issue(0) issue.project = self.get_project(project_id) return issue def post_issue(self, issue): assert not issue.attachments if issue.id and issue.id != '0': _, issue_number = self._extract_issue_id(issue.id) self.browser.edit_issue(issue, issue_number) else: self.browser.post_issue(issue) def update_issue(self, issue_id, update): assert not update.attachments self.browser.post_comment(issue_id, update.message) # iter_projects, remove_issue are impossible def _iter_members(self, project_id): for d in self.browser.iter_members(project_id): yield User(d['id'], d['name']) def _iter_versions(self, project_id): for d in self.browser.iter_milestones(project_id): yield Version(d['id'], d['name']) def _make_issue(self, d, project): _id = self._build_issue_id(project.id, d['number']) issue = Issue(_id) issue.url = d['url'] issue.project = project issue.title = d['title'] issue.body = d['body'] issue.creation = d['creation'] issue.updated = d['updated'] issue.author = project.find_user(d['author'], None) if not issue.author: # may duplicate users issue.author = User(d['author'], d['author']) issue.status = STATUSES[d['status']] if d['assignee']: issue.assignee = project.find_user(d['assignee'], None) else: issue.assignee = None if d['version']: issue.version = project.find_version(d['version'], None) else: issue.version = None issue.category = None issue.attachments = [ self._make_attachment(dattach) for dattach in d['attachments'] ] return issue def _fetch_comments(self, issue): project_id, issue_number = self._extract_issue_id(issue.id) if not issue.history: issue.history = [] issue.history += [ self._make_comment(dcomment, issue.project) for dcomment in self.browser.iter_comments(project_id, issue_number) ] def _make_attachment(self, d): a = Attachment(d['url']) a.url = d['url'] a.filename = d['filename'] return a def _make_comment(self, d, project): u = Update(d['id']) u.message = d['message'] u.author = project.find_user(d['author'], None) if not u.author: # may duplicate users u.author = User(d['author'], d['author']) u.date = d['date'] u.changes = [] u.attachments = [ self._make_attachment(dattach) for dattach in d['attachments'] ] return u @staticmethod def _extract_issue_id(_id): return _id.rsplit('/', 1) @staticmethod def _build_issue_id(project_id, issue_number): return '%s/%s' % (project_id, issue_number)
class CreditDuNordModule(Module, CapBankWealth, CapProfile): NAME = 'creditdunord' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.6' DESCRIPTION = u'Crédit du Nord, Banque Courtois, Kolb, Nuger, Laydernier, Tarneaud, Société Marseillaise de Crédit' LICENSE = 'LGPLv3+' website_choices = OrderedDict([ (k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'www.credit-du-nord.fr': u'Crédit du Nord', 'www.banque-courtois.fr': u'Banque Courtois', 'www.banque-kolb.fr': u'Banque Kolb', 'www.banque-laydernier.fr': u'Banque Laydernier', 'www.banque-nuger.fr': u'Banque Nuger', 'www.banque-rhone-alpes.fr': u'Banque Rhône-Alpes', 'www.tarneaud.fr': u'Tarneaud', 'www.smc.fr': u'Société Marseillaise de Crédit', }.items(), key=lambda k_v: (k_v[1], k_v[0])) ]) CONFIG = BackendConfig( Value('website', label='Banque', choices=website_choices, default='www.credit-du-nord.fr'), ValueBackendPassword('login', label='Identifiant', masked=False), ValueBackendPassword('password', label='Code confidentiel')) BROWSER = CreditDuNordBrowser def create_default_browser(self): return self.create_browser(self.config['website'].get(), self.config['login'].get(), self.config['password'].get(), weboob=self.weboob) def iter_accounts(self): for account in self.browser.get_accounts_list(): yield account def get_account(self, _id): account = self.browser.get_account(_id) if account: return account raise AccountNotFound() def get_account_for_history(self, _id): account = self.browser.get_account_for_history(_id) if account: return account raise AccountNotFound() def iter_history(self, account): account = self.get_account_for_history(account.id) for tr in self.browser.get_history(account): if not tr._is_coming: yield tr def iter_coming(self, account): account = self.get_account_for_history(account.id) for tr in self.browser.get_history(account, coming=True): if tr._is_coming: yield tr def iter_investment(self, account): account = self.get_account(account.id) return self.browser.get_investment(account) def get_profile(self): return self.browser.get_profile()
class BNPorcBackend(BaseBackend, ICapBank, ICapMessages): NAME = 'bnporc' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '0.e' LICENSE = 'AGPLv3+' DESCRIPTION = 'BNP Paribas French bank website' CONFIG = BackendConfig( ValueBackendPassword('login', label='Account ID', masked=False), ValueBackendPassword('password', label='Password', regexp='^(\d{6}|)$'), ValueBackendPassword( 'rotating_password', default='', label= 'Password to set when the allowed uses are exhausted (6 digits)', regexp='^(\d{6}|)$')) BROWSER = BNPorc STORAGE = {'seen': []} # Store the messages *list* for this duration CACHE_THREADS = timedelta(seconds=3 * 60 * 60) def __init__(self, *args, **kwargs): BaseBackend.__init__(self, *args, **kwargs) self._threads = None self._threads_age = datetime.utcnow() def create_default_browser(self): if self.config['rotating_password'].get().isdigit() and len( self.config['rotating_password'].get()) == 6: rotating_password = self.config['rotating_password'].get() else: rotating_password = None return self.create_browser( self.config['login'].get(), self.config['password'].get(), password_changed_cb=self._password_changed_cb, rotating_password=rotating_password) def _password_changed_cb(self, old, new): self.config['password'].set(new) self.config['rotating_password'].set(old) self.config.save() def iter_accounts(self): for account in self.browser.get_accounts_list(): yield account def get_account(self, _id): if not _id.isdigit(): raise AccountNotFound() with self.browser: account = self.browser.get_account(_id) if account: return account else: raise AccountNotFound() def iter_history(self, account): with self.browser: return self.browser.iter_history(account._link_id) def iter_coming(self, account): with self.browser: return self.browser.iter_coming_operations(account._link_id) def iter_transfer_recipients(self, ignored): for account in self.browser.get_transfer_accounts().itervalues(): recipient = Recipient() recipient.id = account.id recipient.label = account.label yield recipient def transfer(self, account, to, amount, reason=None): if isinstance(account, Account): account = account.id try: assert account.isdigit() assert to.isdigit() amount = Decimal(amount) except (AssertionError, ValueError): raise AccountNotFound() with self.browser: return self.browser.transfer(account, to, amount, reason) def iter_threads(self, cache=False): """ If cache is False, always fetch the threads from the website. """ old = self._threads_age < datetime.utcnow() - self.CACHE_THREADS threads = self._threads if not cache or threads is None or old: with self.browser: threads = list(self.browser.iter_threads()) # the website is stupid and does not have the messages in the proper order threads = sorted(threads, key=lambda t: t.date, reverse=True) self._threads = threads seen = self.storage.get('seen', default=[]) for thread in threads: if thread.id not in seen: thread.root.flags |= thread.root.IS_UNREAD else: thread.root.flags &= ~thread.root.IS_UNREAD yield thread def fill_thread(self, thread, fields=None): if fields is None or 'root' in fields: return self.get_thread(thread) def get_thread(self, _id): if isinstance(_id, Thread): thread = _id _id = thread.id else: thread = Thread(_id) with self.browser: thread = self.browser.get_thread(thread) return thread def iter_unread_messages(self): threads = list(self.iter_threads(cache=True)) for thread in threads: if thread.root.flags & thread.root.IS_UNREAD: thread = self.fillobj(thread) or thread yield thread.root def set_message_read(self, message): self.storage.get('seen', default=[]).append(message.thread.id) self.storage.save() OBJECTS = {Thread: fill_thread}