class DLFPModule(Module, CapMessages, CapMessagesPost, CapContent): NAME = 'dlfp' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '2.1' LICENSE = 'AGPLv3+' DESCRIPTION = "Da Linux French Page news website" CONFIG = BackendConfig( Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default=''), ValueBool('get_news', label='Get newspapers', default=True), ValueBool('get_diaries', label='Get diaries', default=False), ValueBool('get_polls', label='Get polls', default=False), ValueBool('get_board', label='Get board', default=False), ValueBool('get_wiki', label='Get wiki', default=False), ValueBool('get_tracker', label='Get tracker', default=False)) STORAGE = {'seen': {}} BROWSER = DLFP FEEDS = { 'get_news': "https://linuxfr.org/news.atom", 'get_diaries': "https://linuxfr.org/journaux.atom", 'get_polls': "https://linuxfr.org/sondages.atom", 'get_board': "https://linuxfr.org/forums.atom", 'get_wiki': "https://linuxfr.org/wiki.atom", 'get_tracker': "https://linuxfr.org/suivi.atom", } def create_default_browser(self): username = self.config['username'].get() if username: password = self.config['password'].get() else: password = None return self.create_browser(username, password) def deinit(self): # don't need to logout if the browser hasn't been used. if not self._browser: return self.browser.close_session() #### CapMessages ############################################## def iter_threads(self): whats = set() for param, url in self.FEEDS.items(): if self.config[param].get(): whats.add(url) for what in whats: for article in Newsfeed(what, rssid).iter_entries(): if article.datetime and (datetime.now() - article.datetime ) > timedelta(days=60): continue thread = Thread(article.id, article.link) thread.title = article.title thread._rsscomment = article.rsscomment if article.datetime: thread.date = article.datetime yield thread def get_thread(self, id, getseen=True): if not isinstance(id, Thread): thread = None else: thread = id id = thread.id if thread.date: self.storage.set('date', id, thread.date) self.storage.save() content = self.browser.get_content(id) if not content: return None if not thread: thread = Thread(content.id) flags = Message.IS_HTML if thread.id not in self.storage.get('seen', default={}): flags |= Message.IS_UNREAD thread.title = content.title if not thread.date: thread.date = content.date thread.root = Message( thread=thread, id='0', # root message url=self.browser.absurl(id2url(content.id)), title=content.title, sender=content.author or u'', receivers=None, date=thread.date, parent=None, content=content.body, signature='URL: %s' % self.browser.absurl(id2url(content.id)), children=[], flags=flags) for com in content.comments: self._insert_comment(com, thread.root, getseen) return thread def _insert_comment(self, com, parent, getseen=True): """" Insert 'com' comment and its children in the parent message. """ flags = Message.IS_HTML if com.id not in self.storage.get('seen', parent.thread.id, 'comments', default=[]): flags |= Message.IS_UNREAD if getseen or flags & Message.IS_UNREAD: com.parse() message = Message( thread=parent.thread, id=com.id, url=com.url, title=com.title, sender=com.author or u'', receivers=None, date=com.date, parent=parent, content=com.body, signature=com.signature + '<br />'.join(['Score: %d' % com.score, 'URL: %s' % com.url]), children=[], flags=flags) else: message = Message(thread=parent.thread, id=com.id, children=[], parent=parent, flags=flags) parent.children.append(message) for sub in com.comments: self._insert_comment(sub, message, getseen) def iter_unread_messages(self): for thread in self.iter_threads(): # Check if we have seen all comments of this thread. oldhash = self.storage.get('hash', thread.id, default="") newhash = self.browser.get_hash(thread._rsscomment) if oldhash != newhash: self.storage.set('hash', thread.id, newhash) self.storage.save() self.fill_thread(thread, 'root', getseen=False) for m in thread.iter_all_messages(): if m.flags & m.IS_UNREAD: yield m def set_message_read(self, message): self.storage.set( 'seen', message.thread.id, 'comments', self.storage.get('seen', message.thread.id, 'comments', default=[]) + [message.id]) self.storage.save() lastpurge = self.storage.get('lastpurge', default=0) # 86400 = one day if time.time() - lastpurge > 86400: self.storage.set('lastpurge', time.time()) self.storage.save() # we can't directly delete without a "RuntimeError: dictionary changed size during iteration" todelete = [] for id in self.storage.get('seen', default={}): date = self.storage.get('date', id, default=0) # if no date available, create a new one (compatibility with "old" storage) if date == 0: self.storage.set('date', id, datetime.now()) elif datetime.now() - date > timedelta(days=60): todelete.append(id) for id in todelete: self.storage.delete('hash', id) self.storage.delete('date', id) self.storage.delete('seen', id) self.storage.save() def fill_thread(self, thread, fields, getseen=True): return self.get_thread(thread, getseen) #### CapMessagesReply ######################################### def post_message(self, message): if not self.browser.username: raise BrowserForbidden() if not message.parent: raise CantSendMessage( 'Posting news and diaries on DLFP is not supported yet') assert message.thread return self.browser.post_comment(message.thread.id, message.parent.id, message.title, message.content) #### CapContent ############################################### def get_content(self, _id, revision=None): if isinstance(_id, basestring): content = Content(_id) else: content = _id _id = content.id if revision: raise NotImplementedError( 'Website does not provide access to older revisions sources.') data = self.browser.get_wiki_content(_id) if data is None: return None content.content = data return content def push_content(self, content, message=None, minor=False): if not self.browser.username: raise BrowserForbidden() return self.browser.set_wiki_content(content.id, content.content, message) def get_content_preview(self, content): return self.browser.get_wiki_preview(content.id, content.content) OBJECTS = {Thread: fill_thread}
class IndeedModule(Module, CapJob): NAME = 'indeed' DESCRIPTION = u'indeed website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.1' BROWSER = IndeedBrowser type_contrat_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ 'all': u'Tous les emplois', 'fulltime': u'Temps plein', 'parttime': u'Temps partiel', 'contract': u'Durée indéterminée', 'internship': u'Stage / Apprentissage', 'temporary': u'Durée déterminée', }.iteritems())]) limit_date_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ 'any': u'à tout moment', '15': u'depuis 15 jours', '7': u'depuis 7 jours', '3': u'depuis 3 jours', '1': u'depuis hier', 'last': u'depuis ma dernière visite', }.iteritems())]) radius_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '0': u'uniquement à cet endroit', '5': u'dans un rayon de 5 kilomètres', '10': u'dans un rayon de 10 kilomètres', '15': u'dans un rayon de 15 kilomètres', '25': u'dans un rayon de 25 kilomètres', '50': u'dans un rayon de 50 kilomètres', '100': u'dans un rayon de 100 kilomètres', }.iteritems())]) CONFIG = BackendConfig(Value('metier', label=u'Job name', masked=False, default=''), Value('limit_date', label=u'Date limite', choices=limit_date_choices, default=''), Value('contrat', label=u'Contract', choices=type_contrat_choices, default=''), Value('place', label=u'Place', masked=False, default=''), Value('radius', label=u'Radius', choices=radius_choices, default='')) def search_job(self, pattern=None): return self.browser.search_job(metier=pattern) def advanced_search_job(self): return self.browser.search_job(metier=self.config['metier'].get(), limit_date=self.config['limit_date'].get(), contrat=self.config['contrat'].get(), place=self.config['place'].get(), radius=self.config['radius'].get()) def get_job_advert(self, _id, advert=None): return self.browser.get_job_advert(_id, advert) def fill_obj(self, advert, fields): return self.get_job_advert(advert.id, advert) OBJECTS = {BaseJobAdvert: fill_obj}
class PhpBBModule(Module, CapMessages, CapMessagesPost): NAME = 'phpbb' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.3' LICENSE = 'AGPLv3+' DESCRIPTION = "phpBB forum" CONFIG = BackendConfig(Value('url', label='URL of forum', regexp='https?://.*'), Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default=''), ValueInt('thread_unread_messages', label='Limit number of unread messages to retrieve for a thread', default=500) ) STORAGE = {'seen': {}} BROWSER = PhpBB def create_default_browser(self): username = self.config['username'].get() if len(username) > 0: password = self.config['password'].get() else: password = None return self.create_browser(self.config['url'].get(), username, password) #### CapMessages ############################################## def _iter_threads(self, root_link=None): links = list(self.browser.iter_links(root_link.url if root_link else None)) for link in links: if link.type == link.FORUM: link.title = '%s[%s]' % (root_link.title if root_link else '', link.title) for thread in self._iter_threads(link): yield thread if link.type == link.TOPIC: thread = Thread(url2id(link.url)) thread.title = ('%s ' % root_link.title if root_link else '') + link.title thread.date = link.date thread.flags = thread.IS_DISCUSSION yield thread def iter_threads(self): return self._iter_threads() def get_thread(self, id): thread = None parent = None if isinstance(id, Thread): thread = id id = thread.id thread_id = url2id(id, nopost=True) or id try: last_seen_id = self.storage.get('seen', default={})[id2topic(thread_id)] except KeyError: last_seen_id = 0 for post in self.browser.iter_posts(id): if not thread: thread = Thread(thread_id) thread.title = post.title m = self._post2message(thread, post) m.parent = parent if last_seen_id < post.id: m.flags |= Message.IS_UNREAD if parent: parent.children = [m] else: thread.root = m parent = m return thread def _post2message(self, thread, post): signature = post.signature if signature: signature += '<br />' signature += 'URL: %s' % self.browser.absurl(id2url('%s.%s' % (thread.id, post.id))) return Message(thread=thread, id=post.id, title=post.title, sender=post.author, receivers=None, date=post.date, parent=None, content=post.content, signature=signature, children=[], flags=Message.IS_HTML) def iter_unread_messages(self): url = self.browser.get_root_feed_url() for article in Newsfeed(url, rssid).iter_entries(): id = url2id(article.link) thread = None try: last_seen_id = self.storage.get('seen', default={})[id2topic(id)] except KeyError: last_seen_id = 0 child = None iterator = self.browser.riter_posts(id, last_seen_id) if self.config['thread_unread_messages'].get() > 0: iterator = limit(iterator, self.config['thread_unread_messages'].get()) for post in iterator: if not thread: thread = Thread('%s.%s' % (post.forum_id, post.topic_id)) message = self._post2message(thread, post) if child: message.children.append(child) child.parent = message if post.parent: message.parent = Message(thread=thread, id=post.parent) else: thread.root = message yield message def set_message_read(self, message): try: last_seen_id = self.storage.get('seen', default={})[id2topic(message.thread.id)] except KeyError: last_seen_id = 0 if message.id > last_seen_id: self.storage.set('seen', id2topic(message.thread.id), message.id) self.storage.save() def fill_thread(self, thread, fields): return self.get_thread(thread) #### CapMessagesReply ######################################### def post_message(self, message): assert message.thread forum = 0 topic = 0 if message.thread: try: if '.' in message.thread.id: forum, topic = [int(i) for i in message.thread.id.split('.', 1)] else: forum = int(message.thread.id) except ValueError: raise CantSendMessage('Thread ID must be in form "FORUM_ID[.TOPIC_ID]".') return self.browser.post_answer(forum, topic, message.title, message.content) OBJECTS = {Thread: fill_thread}
def setUp(self): if not self.is_backend_configured(): self.backend.config['resolution'] = Value(value='240') self.backend.config['format'] = Value(value='mp4')
class AmundiModule(Module, CapBank): NAME = 'amundi' DESCRIPTION = u'amundi website' MAINTAINER = u'James GALT' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.2' CONFIG = BackendConfig( ValueBackendPassword('login', label='Identifiant', regexp='\d+', masked=False), ValueBackendPassword('password', label=u"Mot de passe", regexp='\d+'), Value('website', label='Type de compte', default='ee', choices={ 'ee': 'Amundi Epargne Entreprise', 'tc': 'Amundi Tenue de Compte' })) def create_default_browser(self): b = {'ee': AmundiEEBrowser, 'tc': AmundiTCBrowser} self.BROWSER = b[self.config['website'].get()] w = { 'ee': 'https://www.amundi-ee.com', 'tc': 'https://epargnants.amundi-tc.com' } return self.create_browser(w[self.config['website'].get()], self.config['login'].get(), self.config['password'].get()) def get_account(self, id): """ Get an account from its ID. :param id: ID of the account :type id: :class:`str` :rtype: :class:`Account` :raises: :class:`AccountNotFound` """ return find_object(self.iter_accounts(), id=id, error=AccountNotFound) def iter_accounts(self): """ Iter accounts. :rtype: iter[:class:`Account`] """ return self.browser.iter_accounts() def iter_investment(self, account): """ Iter investment of a market account :param account: account to get investments :type account: :class:`Account` :rtype: iter[:class:`Investment`] :raises: :class:`AccountNotFound` """ return self.browser.iter_investments(account) def iter_history(self, account): """ Iter history of transactions on a specific account. :param account: account to get history :type account: :class:`Account` :rtype: iter[:class:`Transaction`] :raises: :class:`AccountNotFound` """ return self.browser.iter_history(account)
def double_auth(self, transfer): code_needed = CleanText('//label[@for="code_securite"]')(self.doc) if code_needed: raise TransferStep(transfer, Value('code', label=code_needed))
class VoyagesSNCFModule(Module, CapTravel): NAME = 'voyagessncf' DESCRIPTION = u'Voyages SNCF' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.3' CONFIG = BackendConfig( Value('age', label='Passenger age', default='ADULT', choices=OrderedDict( (('ADULT', '26-59 ans'), ('SENIOR', '60 et +'), ('YOUNG', '12-25 ans'), ('CHILD_UNDER_FOUR', '0-3 ans'), ('CHILDREN', '4-11 ans')))), Value('card', label='Passenger card', default='default', choices=OrderedDict( (('default', u'Pas de carte'), ('YOUNGS', u'Carte Jeune'), ('ESCA', u'Carte Escapades'), ('WEEKE', u'Carte Week-end'), ('FQ2ND', u'Abo Fréquence 2e'), ('FQ1ST', u'Abo Fréquence 1e'), ('FF2ND', u'Abo Forfait 2e'), ('FF1ST', u'Abo Forfait 1e'), ('ACCWE', u'Accompagnant Carte Week-end'), ('ACCCHD', u'Accompagnant Carte Enfant+'), ('ENFAM', u'Carte Enfant Famille'), ('FAM30', u'Carte Familles Nombreuses 30%'), ('FAM40', u'Carte Familles Nombreuses 40%'), ('FAM50', u'Carte Familles Nombreuses 50%'), ('FAM75', u'Carte Familles Nombreuses 75%'), ('MI2ND', u'Carte Militaire 2e'), ('MI1ST', u'Carte Militaire 1e'), ('MIFAM', u'Carte Famille Militaire'), ('THBIZ', u'Thalys ThePass Business'), ('THPREM', u'Thalys ThePass Premium'), ('THWE', u'Thalys ThePass Weekend')))), Value('class', label='Comfort class', default='2', choices=OrderedDict((('1', u'1e classe'), ('2', u'2e classe'))))) BROWSER = VoyagesSNCFBrowser STATIONS = [] def _populate_stations(self): if len(self.STATIONS) == 0: with self.browser: self.STATIONS = self.browser.get_stations() def iter_station_search(self, pattern): self._populate_stations() pattern = pattern.lower() already = set() # First stations whose name starts with pattern... for _id, name in enumerate(self.STATIONS): if name.lower().startswith(pattern): already.add(_id) yield Station(_id, unicode(name)) # ...then ones whose name contains pattern. for _id, name in enumerate(self.STATIONS): if pattern in name.lower() and _id not in already: yield Station(_id, unicode(name)) def iter_station_departures(self, station_id, arrival_id=None, date=None): self._populate_stations() if arrival_id is None: raise UserError('The arrival station is required') try: station = self.STATIONS[int(station_id)] arrival = self.STATIONS[int(arrival_id)] except (IndexError, ValueError): try: station = list(self.iter_station_search(station_id))[0].name arrival = list(self.iter_station_search(arrival_id))[0].name except IndexError: raise UserError('Unknown station') with self.browser: for i, d in enumerate( self.browser.iter_departures(station, arrival, date, self.config['age'].get(), self.config['card'].get(), self.config['class'].get())): departure = Departure(i, d['type'], d['time']) departure.departure_station = d['departure'] departure.arrival_station = d['arrival'] departure.arrival_time = d['arrival_time'] departure.price = d['price'] departure.currency = d['currency'] departure.information = d['price_info'] yield departure
class AmazonModule(Module, CapDocument): NAME = 'amazon' DESCRIPTION = 'Amazon' MAINTAINER = 'Théo Dorée' EMAIL = '*****@*****.**' LICENSE = 'LGPLv3+' VERSION = '1.6' website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'www.amazon.com': u'Amazon.com', 'www.amazon.fr': u'Amazon France', 'www.amazon.de': u'Amazon.de', 'www.amazon.co.uk': u'Amazon UK', }.items())]) BROWSERS = { 'www.amazon.fr': AmazonBrowser, 'www.amazon.com': AmazonEnBrowser, 'www.amazon.de': AmazonDeBrowser, 'www.amazon.co.uk': AmazonUkBrowser, } CONFIG = BackendConfig( Value('website', label=u'Website', choices=website_choices, default='www.amazon.com'), ValueBackendPassword('email', label='Username', masked=False), ValueBackendPassword('password', label='Password'), Value('captcha_response', label='Captcha Response', required=False, default=''), Value('pin_code', label='OTP response', required=False, default='') ) accepted_document_types = (DocumentTypes.BILL,) def create_default_browser(self): self.BROWSER = self.BROWSERS[self.config['website'].get()] return self.create_browser(self.config) def iter_subscription(self): return self.browser.iter_subscription() def get_subscription(self, _id): return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) def get_document(self, _id): subid = _id.rsplit('_', 1)[0] subscription = self.get_subscription(subid) return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) def iter_documents(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) return self.browser.iter_documents(subscription) def download_document(self, document): if not isinstance(document, Document): document = self.get_document(document) if document.url is NotAvailable: return return self.browser.open(document.url).content def download_document_pdf(self, document): if not isinstance(document, Document): document = self.get_document(document) if document.url is NotAvailable: return if document.format == 'pdf': return self.browser.open(document.url).content url = urljoin(self.browser.BASEURL, document.url) return html_to_pdf(self.browser, url=url)
class LutimModule(Module, CapPaste): NAME = 'lutim' DESCRIPTION = u'LUTIm website' MAINTAINER = u'Vincent A' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.1' BROWSER = LutimBrowser CONFIG = BackendConfig( Value('base_url', label='Hoster base URL', default='http://lut.im/')) def _base_url(self): url = self.config['base_url'].get() if not url.endswith('/'): url = url + '/' return url def create_default_browser(self): return self.create_browser(self._base_url()) def can_post(self, contents, title=None, public=None, max_age=None): if re.search(r'[^a-zA-Z0-9=+/\s]', contents): return 0 elif max_age and max_age < 86400: return 0 # it cannot be shorter than one day else: mime = image_mime(contents, ('gif', 'jpeg', 'png')) return 20 * int(mime is not None) def new_paste(self, *a, **kw): base_url = self._base_url() class LutImage(BasePaste): @classmethod def id2url(cls, id): return urljoin(base_url, id) @classmethod def url2id(cls, url): if url.startswith(base_url): return url[len(base_url):] return LutImage(*a, **kw) def get_paste(self, id): paste = self.new_paste(id) if '/' in id: paste.id = paste.url2id(id) if not paste.id: return None response = self.browser.readurl(paste.page_url) if response: paste.contents = response.encode('base64') return paste def post_paste(self, paste, max_age=None): d = self.browser.post(paste.title or None, paste.contents.decode('base64'), (max_age or 0) // 86400) if d: paste.id = d['id']
class CreditDuNordModule(Module, CapBank, CapProfile): NAME = 'creditdunord' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.3' DESCRIPTION = u'Crédit du Nord, Banque Courtois, Kolb, Nuger, Laydernier, Tarneaud, Société Marseillaise de Crédit' LICENSE = 'AGPLv3+' website_choices = OrderedDict([ (k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'www.credit-du-nord.fr': u'Crédit du Nord', 'www.banque-courtois.fr': u'Banque Courtois', 'www.banque-kolb.fr': u'Banque Kolb', 'www.banque-laydernier.fr': u'Banque Laydernier', 'www.banque-nuger.fr': u'Banque Nuger', 'www.banque-rhone-alpes.fr': u'Banque Rhône-Alpes', 'www.tarneaud.fr': u'Tarneaud', 'www.smc.fr': u'Société Marseillaise de Crédit', }.iteritems(), key=lambda k_v: (k_v[1], k_v[0])) ]) CONFIG = BackendConfig( Value('website', label='Banque', choices=website_choices, default='www.credit-du-nord.fr'), ValueBackendPassword('login', label='Identifiant', masked=False), ValueBackendPassword('password', label='Code confidentiel')) BROWSER = CreditDuNordBrowser def create_default_browser(self): return self.create_browser(self.config['website'].get(), self.config['login'].get(), self.config['password'].get()) def iter_accounts(self): for account in self.browser.get_accounts_list(): yield account def get_account(self, _id): account = self.browser.get_account(_id) if account: return account else: raise AccountNotFound() def iter_history(self, account): account = self.get_account(account.id) for tr in self.browser.get_history(account): if not tr._is_coming: yield tr def iter_coming(self, account): account = self.get_account(account.id) for tr in self.browser.get_history(account, coming=True): if tr._is_coming: yield tr def iter_investment(self, account): account = self.get_account(account.id) return self.browser.get_investment(account) def get_profile(self): return self.browser.get_profile()
class YoutubeModule(Module, CapVideo, CapCollection): NAME = 'youtube' MAINTAINER = u'Laurent Bachelier' EMAIL = '*****@*****.**' VERSION = '1.4' DESCRIPTION = 'YouTube video streaming website' LICENSE = 'AGPLv3+' BROWSER = None CONFIG = BackendConfig( Value('username', label='Email address', default=''), ValueBackendPassword('password', label='Password', default='')) URL_RE = re.compile( r'^https?://(?:\w*\.?youtube(?:|-nocookie)\.com/(?:watch\?v=|embed/|v/)|youtu\.be\/|\w*\.?youtube\.com\/user\/\w+#p\/u\/\d+\/)([^\?&]+)' ) def create_default_browser(self): password = None username = self.config['username'].get() if len(username) > 0: password = self.config['password'].get() return self.create_browser(username, password) def _entry2video(self, entry): """ Parse an entry returned by googleapi and return a Video object. """ snippet = entry['snippet'] id = entry['id'] if isinstance(id, dict): id = id['videoId'] video = YoutubeVideo(to_unicode(id)) video.title = to_unicode(snippet['title'].strip()) # duration does not seem to be available with api video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url']) video.author = to_unicode(snippet['channelTitle'].strip()) return video def _set_video_attrs(self, video): new_video = video_info(YoutubeVideo.id2url(video.id)) if not new_video: return for k, v in new_video.iter_fields(): if not empty(v) and empty(getattr(video, k)): setattr(video, k, v) def get_video(self, _id): m = self.URL_RE.match(_id) if m: _id = m.group(1) params = {'id': _id, 'part': 'id,snippet'} youtube = self._build_yt() response = youtube.videos().list(**params).execute() items = response.get('items', []) if not items: return None video = self._entry2video(items[0]) self._set_video_attrs(video) video.set_empty_fields(NotAvailable) # Youtube video url is https, using ssl encryption # so we need to use the "play_proxy" method using urllib2 proxy streaming to handle this video._play_proxy = True return video def _build_yt(self): DEVELOPER_KEY = "AIzaSyApVVeZ03XkKDYHX8T5uOn8Eizfe9CMDbs" YOUTUBE_API_SERVICE_NAME = "youtube" YOUTUBE_API_VERSION = "v3" return ytbuild(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=DEVELOPER_KEY) def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): YOUTUBE_MAX_RESULTS = 50 youtube = self._build_yt() params = {'part': 'id,snippet', 'maxResults': YOUTUBE_MAX_RESULTS} if pattern is not None: if isinstance(pattern, unicode): pattern = pattern.encode('utf-8') params['q'] = pattern params['safeSearch'] = 'none' if nsfw else 'strict' # or 'moderate' params['order'] = ('relevance', 'rating', 'viewCount', 'date')[sortby] nb_yielded = 0 while True: search_response = youtube.search().list(**params).execute() items = search_response.get('items', []) for entry in items: if entry["id"]["kind"] != "youtube#video": continue yield self._entry2video(entry) nb_yielded += 1 params['pageToken'] = search_response.get('nextPageToken') if not params['pageToken']: return if nb_yielded < YOUTUBE_MAX_RESULTS: return def latest_videos(self): return self.search_videos(None, CapVideo.SEARCH_DATE) def fill_video(self, video, fields): if 'thumbnail' in fields and video.thumbnail: video.thumbnail.data = requests.get(video.thumbnail.url).content if 'url' in fields: self._set_video_attrs(video) return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest']) if collection.split_path == [u'latest']: for video in self.latest_videos(): yield video def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and collection.split_path == [u'latest']: collection.title = u'Latest YouTube videos' return raise CollectionNotFound(collection.split_path) OBJECTS = {YoutubeVideo: fill_video}
def new_recipient(self, recipient, **params): if 'code' in params: self.validate_rcpt_with_sms(params['code']) return self.page.rcpt_after_sms(recipient) data = { 'n_nbOccurences': 1000, 'n_nbOccurences_affichees': 0, 'n_rang': 0, } self.recipients.go(data=data) step_urls = { 'first_recipient_check': self.absurl( '/ord-web/ord//ord-valider-destinataire-avant-maj.json', base=True), 'get_bic': self.absurl('/ord-web/ord//ord-tiers-calcul-bic.json', base=True), 'get_token': self.absurl( '/ord-web/ord//ord-preparer-signature-destinataire.json', base=True), 'get_sign_info': self.absurl('/sec/getsigninfo.json', base=True), 'send_otp_to_user': self.absurl('/sec/csa/send.json', base=True), } self.add_recipient.go( method='POST', headers={'Content-Type': 'application/json;charset=UTF-8'}) countries = self.page.get_countries() # first recipient check data = { 'an_codeAction': 'ajout_tiers', 'an_refSICoordonnee': '', 'an_refSITiers': '', 'cl_iban': recipient.iban, 'cl_raisonSociale': recipient.label, } self.location(step_urls['first_recipient_check'], data=data) # get bic data = { 'an_activateCMU': 'true', 'an_codePaysBanque': '', 'an_nature': 'C', 'an_numeroCompte': recipient.iban, 'an_topIBAN': 'true', 'cl_adresse': '', 'cl_adresseBanque': '', 'cl_codePays': recipient.iban[:2], 'cl_libellePaysBanque': '', 'cl_libellePaysDestinataire': countries[recipient.iban[:2]], 'cl_nomBanque': '', 'cl_nomRaisonSociale': recipient.label, 'cl_ville': '', 'cl_villeBanque': '', } self.location(step_urls['get_bic'], data=data) bic = self.page.get_response_data() # get token data = { 'an_coordonnee_codePaysBanque': '', 'an_coordonnee_nature': 'C', 'an_coordonnee_numeroCompte': recipient.iban, 'an_coordonnee_topConfidentiel': 'false', 'an_coordonnee_topIBAN': 'true', 'an_refSICoordonnee': '', 'an_refSIDestinataire': '', 'cl_adresse': '', 'cl_codePays': recipient.iban[:2], 'cl_coordonnee_adresseBanque': '', 'cl_coordonnee_bic': bic, 'cl_coordonnee_categories_libelle': '', 'cl_coordonnee_categories_refSi': '', 'cl_coordonnee_libellePaysBanque': '', 'cl_coordonnee_nomBanque': '', 'cl_coordonnee_villeBanque': '', 'cl_libellePaysDestinataire': countries[recipient.iban[:2]], 'cl_nomRaisonSociale': recipient.label, 'cl_ville': '', } self.location(step_urls['get_token'], data=data) self.new_rcpt_validate_form = data payload = self.page.get_response_data() # get sign info data = { 'b64_jeton_transaction': payload['jeton'], 'action_level': payload['sensibilite'], } self.location(step_urls['get_sign_info'], data=data) # send otp to user data = {'context': payload['jeton'], 'csa_op': 'sign'} self.location(step_urls['send_otp_to_user'], data=data) self.new_rcpt_validate_form.update(data) rcpt = self.copy_recipient_obj(recipient) self.need_reload_state = True raise AddRecipientStep( rcpt, Value('code', label='Veuillez entrer le code reçu par SMS.'))
class GDCVaultModule(Module, CapVideo, CapCollection): NAME = 'gdcvault' MAINTAINER = u'François Revol' EMAIL = '*****@*****.**' VERSION = '1.4' DESCRIPTION = 'Game Developers Conferences Vault video streaming website' LICENSE = 'AGPLv3+' BROWSER = GDCVaultBrowser CONFIG = BackendConfig( Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default='')) def create_default_browser(self): username = self.config['username'].get() if len(username) > 0: password = self.config['password'].get() else: password = None return self.create_browser(username, password) def deinit(self): # don't need to logout if the browser hasn't been used. if not self._browser: return with self.browser: self.browser.close_session() def get_video(self, _id): with self.browser: return self.browser.get_video(_id) SORTBY = ['relevance', 'rating', 'views', 'time'] def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): with self.browser: return self.browser.search_videos(pattern, self.SORTBY[sortby]) def fill_video(self, video, fields): if fields != ['thumbnail']: # if we don't want only the thumbnail, we probably want also every fields with self.browser: video = self.browser.get_video(GDCVaultVideo.id2url(video.id), video) if 'thumbnail' in fields and video.thumbnail: with self.browser: video.thumbnail.data = self.browser.readurl( video.thumbnail.url) return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest']) if collection.split_path == [u'latest']: for video in self.browser.latest_videos(): yield video def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and collection.split_path == [u'latest']: collection.title = u'Latest GDCVault videos' return raise CollectionNotFound(collection.split_path) OBJECTS = {GDCVaultVideo: fill_video}
class TwitterModule(Module, CapMessages, CapMessagesPost, CapCollection): NAME = 'twitter' DESCRIPTION = u'twitter website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.4' BROWSER = TwitterBrowser STORAGE = {'seen': {}} CONFIG = BackendConfig( Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default=''), Value('hashtags_subscribe', label='Hashtags subscribe', default=''), Value('search_subscribe', label='Search subscribe', default=''), Value('profils_subscribe', label='Profils subscribe', default='')) def create_default_browser(self): username = self.config['username'].get() if username: password = self.config['password'].get() else: password = None return self.create_browser(username, password) def iter_threads(self): if self.config['username'].get(): return self.browser.iter_threads() else: profils = self.config['profils_subscribe'].get() hashtags = self.config['hashtags_subscribe'].get() searchs = self.config['search_subscribe'].get() tweets = [] if profils: for profil in profils.split(','): for tweet in itertools.islice( self.browser.get_tweets_from_profil(profil), 0, 20): tweets.append(tweet) if hashtags: for hashtag in hashtags.split(','): for tweet in itertools.islice( self.browser.get_tweets_from_hashtag(hashtag), 0, 20): tweets.append(tweet) if searchs: for search in searchs.split(','): for tweet in itertools.islice( self.browser.get_tweets_from_search(search), 0, 20): tweets.append(tweet) tweets.sort(key=lambda o: o.date, reverse=True) return tweets def get_thread(self, _id, thread=None, getseen=True): seen = None if getseen: seen = self.storage.get('seen', default={}) return self.browser.get_thread(_id, thread, seen) def fill_thread(self, thread, fields, getseen=True): return self.get_thread(thread.id, thread, getseen) def set_message_read(self, message): self.storage.set('seen', message.thread.id, message.thread.date) self.storage.save() self._purge_message_read() def _purge_message_read(self): lastpurge = self.storage.get('lastpurge', default=datetime.now() - timedelta(days=60)) if datetime.now() - lastpurge > timedelta(days=60): self.storage.set('lastpurge', datetime.now() - timedelta(days=60)) self.storage.save() # we can't directly delete without a "RuntimeError: dictionary changed size during iteration" todelete = [] for id, date in self.storage.get('seen', default={}).items(): # if no date available, create a new one (compatibility with "old" storage) if not date: self.storage.set('seen', id, datetime.now()) elif lastpurge > date: todelete.append(id) for id in todelete: self.storage.delete('seen', id) self.storage.save() def post_message(self, message): if not self.config['username'].get(): raise BrowserForbidden() self.browser.post( find_object(self.iter_threads(), id=message.full_id.split('.')[0]), message.content) def iter_resources(self, objs, split_path): collection = self.get_collection(objs, split_path) if collection.path_level == 0: if self.config['username'].get(): yield Collection([u'me'], u'me') yield Collection([u'profils'], u'profils') yield Collection([u'trendy'], u'trendy') yield Collection([u'hashtags'], u'hashtags') yield Collection([u'search'], u'search') if collection.path_level == 1: if collection.split_path[0] == u'me': for el in self.browser.get_tweets_from_profil( self.browser.get_me()): yield el if collection.split_path[0] == u'profils': profils = self.config['profils_subscribe'].get() if profils: for profil in profils.split(','): yield Collection([profil], profil) if collection.split_path[0] == u'hashtags': hashtags = self.config['hashtags_subscribe'].get() if hashtags: for hashtag in hashtags.split(','): yield Collection([hashtag], hashtag) if collection.split_path[0] == u'search': searchs = self.config['search_subscribe'].get() if searchs: for search in searchs.split(','): yield Collection([search], search) if collection.split_path[0] == u'trendy': for obj in self.browser.get_trendy_subjects(): yield Collection([obj.id], obj.id) if collection.path_level == 2: if collection.split_path[0] == u'profils': for el in self.browser.get_tweets_from_profil( collection.split_path[1]): yield el if collection.split_path[0] == u'trendy': if collection.split_path[1].startswith('#'): for el in self.browser.get_tweets_from_hashtag( collection.split_path[1]): yield el else: for el in self.browser.get_tweets_from_search( collection.split_path[1]): yield el if collection.split_path[0] == u'hashtags': for el in self.browser.get_tweets_from_hashtag( collection.split_path[1]): yield el if collection.split_path[0] == u'search': for el in self.browser.get_tweets_from_search( collection.split_path[1]): yield el def validate_collection(self, objs, collection): if collection.path_level == 0: return if collection.path_level == 1 and collection.split_path[0] in \ [u'profils', u'trendy', u'me', u'hashtags', u'search']: return if collection.path_level == 2: return raise CollectionNotFound(collection.split_path) OBJECTS = {Thread: fill_thread}
class AgendadulibreModule(Module, CapCalendarEvent): NAME = 'agendadulibre' DESCRIPTION = u'agendadulibre website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '2.1' ASSOCIATED_CATEGORIES = [CATEGORIES.CONF] BROWSER = AgendadulibreBrowser region_choices = OrderedDict([ (k, u'%s (%s)' % (v, k)) for k, v in sorted({ "https://www.agendadulibre.org": u'--France--', "https://www.agendadulibre.org#3": u'Auvergne-Rhône-Alpes', "https://www.agendadulibre.org#5": u'Bourgogne-Franche-Comté', "https://www.agendadulibre.org#6": u'Bretagne', "https://www.agendadulibre.org#7": u'Centre-Val de Loire', "https://www.agendadulibre.org#30": u'Collectivité sui generis', "https://www.agendadulibre.org#29": u'Collectivités d\'outre-mer', "https://www.agendadulibre.org#9": u'Corse', "https://www.agendadulibre.org#1": u'Grand Est', "https://www.agendadulibre.org#23": u'Guadeloupe', "https://www.agendadulibre.org#24": u'Guyane', "https://www.agendadulibre.org#17": u'Hauts-de-France', "https://www.agendadulibre.org#12": u'Île-de-France', "https://www.agendadulibre.org#31": u'Internet', "https://www.agendadulibre.org#26": u'La Réunion', "https://www.agendadulibre.org#25": u'Martinique', "https://www.agendadulibre.org#28": u'Mayotte', "https://www.agendadulibre.org#4": u'Normandie', "https://www.agendadulibre.org#2": u'Nouvelle-Aquitaine', "https://www.agendadulibre.org#13": u'Occitanie', "https://www.agendadulibre.org#18": u'Pays de la Loire', "https://www.agendadulibre.org#21": u'Provence-Alpes-Côte d\'Azur', "https://www.agendadulibre.be": u'--Belgique--', "https://www.agendadulibre.be#11": u'Antwerpen', "https://www.agendadulibre.be#10": u'Brabant wallon', "https://www.agendadulibre.be#9": u'Bruxelles-Capitale', "https://www.agendadulibre.be#8": u'Hainaut', "https://www.agendadulibre.be#7": u'Liege', "https://www.agendadulibre.be#6": u'Limburg', "https://www.agendadulibre.be#5": u'Luxembourg', "https://www.agendadulibre.be#4": u'Namur', "https://www.agendadulibre.be#3": u'Oost-Vlaanderen', "https://www.agendadulibre.be#2": u'Vlaams-Brabant', "https://www.agendadulibre.be#1": u'West-Vlaanderen', "https://www.agendadulibre.ch": u'--Suisse--', "https://www.agendadulibre.ch#15": u'Appenzell Rhodes-Extérieures', "https://www.agendadulibre.ch#16": u'Appenzell Rhodes-Intérieures', "https://www.agendadulibre.ch#19": u'Argovie', "https://www.agendadulibre.ch#13": u'Bâle-Campagne', "https://www.agendadulibre.ch#12": u'Bâle-Ville', "https://www.agendadulibre.ch#2": u'Berne', "https://www.agendadulibre.ch#10": u'Fribourg', "https://www.agendadulibre.ch#25": u'Genève', "https://www.agendadulibre.ch#8": u'Glaris', "https://www.agendadulibre.ch#18": u'Grisons', "https://www.agendadulibre.ch#26": u'Jura', "https://www.agendadulibre.ch#3": u'Lucerne', "https://www.agendadulibre.ch#24": u'Neuchâtel', "https://www.agendadulibre.ch#7": u'Nidwald', "https://www.agendadulibre.ch#6": u'Obwald', "https://www.agendadulibre.ch#17": u'Saint-Gall', "https://www.agendadulibre.ch#14": u'Schaffhouse', "https://www.agendadulibre.ch#5": u'Schwytz', "https://www.agendadulibre.ch#11": u'Soleure', "https://www.agendadulibre.ch#21": u'Tessin', "https://www.agendadulibre.ch#20": u'Thurgovie', "https://www.agendadulibre.ch#4": u'Uri', "https://www.agendadulibre.ch#23": u'Valais', "https://www.agendadulibre.ch#22": u'Vaud', "https://www.agendadulibre.ch#9": u'Zoug', "https://www.agendadulibre.ch#1": u'Zurich', }.items()) ]) CONFIG = BackendConfig( Value('region', label=u'Region', choices=region_choices)) def create_default_browser(self): choice = self.config['region'].get().split('#') selected_region = '' if len(choice) < 2 else choice[-1] return self.create_browser(website=choice[0], region=selected_region) def search_events(self, query): return self.browser.list_events(query.start_date, query.end_date, query.city, query.categories) def list_events(self, date_from, date_to=None): return self.browser.list_events(date_from, date_to) def get_event(self, event_id): return self.browser.get_event(event_id) def fill_obj(self, event, fields): event = self.browser.get_event(event.id, event) choice = self.config['region'].get().split('#') selected_region = '' if len(choice) < 2 else choice[-1] if selected_region == '23': event.timezone = 'America/Guadeloupe' elif selected_region == '24': event.timezone = 'America/Guyana' elif selected_region == '26': event.timezone = 'Indian/Reunion' elif selected_region == '25': event.timezone = 'America/Martinique' else: event.timezone = 'Europe/Paris' return event OBJECTS = {AgendadulibreBrowser: fill_obj}
class MonsterModule(Module, CapJob): NAME = 'monster' DESCRIPTION = u'monster website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.6' BROWSER = MonsterBrowser type_contrat_choices = OrderedDict([ (k, u'%s' % (v)) for k, v in sorted({ 'Interim-ou-CDD-ou-mission_8': u'Interim ou CDD ou mission', 'CDI_8': u'CDI', 'Stage-Apprentissage-Alternance_8': u'Stage/Apprentissage/Alternance', ' ': u'Autres', 'Indépendant-Freelance-Saisonnier-Franchise_8': u'Indépendant/Freelance/Saisonnier/Franchise', 'Journalier_8': u'Journalier', 'Temps-Partiel_8': u'Temps Partiel', 'Temps-Plein_8': u'Temps Plein', }.items()) ]) date_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '-1': u'N importe quelle date', '000000': u'Aujourd hui', '1': u'2 derniers jours', '3': u'3 derniers jours', '7': u'Les 7 derniers jours', '14': u'Les 14 derniers jours', '30': u'30 derniers jours', }.items())]) CONFIG = BackendConfig( Value('job_name', label='Job name', masked=False, default=''), Value('place', label='Place', masked=False, default=''), Value('contract', label=u'Contract', choices=type_contrat_choices, default=''), Value('limit_date', label=u'Date', choices=date_choices, default='-1'), ) def search_job(self, pattern=None): return self.browser.search_job(pattern) def advanced_search_job(self): return self.browser.advanced_search_job( job_name=self.config['job_name'].get(), place=self.config['place'].get(), contract=self.config['contract'].get(), limit_date=self.config['limit_date'].get()) def get_job_advert(self, _id, advert=None): return self.browser.get_job_advert(_id, advert) def fill_obj(self, advert, fields): return self.get_job_advert(advert.id, advert) OBJECTS = {BaseJobAdvert: fill_obj}
class NolifeTVBackend(BaseBackend, ICapVideo, ICapCollection): NAME = 'nolifetv' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '0.e' DESCRIPTION = 'NolifeTV French video streaming website' LICENSE = 'AGPLv3+' BROWSER = NolifeTVBrowser CONFIG = BackendConfig( Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default='')) def create_default_browser(self): username = self.config['username'].get() if len(username) > 0: password = self.config['password'].get() else: password = None return self.create_browser(username, password) def get_video(self, _id): with self.browser: video = self.browser.get_video(_id) return video def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None): with self.browser: return self.browser.search_videos(pattern) def fill_video(self, video, fields): if fields != ['thumbnail']: # if we don't want only the thumbnail, we probably want also every fields with self.browser: video = self.browser.get_video(NolifeTVVideo.id2url(video.id), video) if 'thumbnail' in fields and video.thumbnail: with self.browser: video.thumbnail.data = self.browser.readurl( video.thumbnail.url) return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest']) if collection.split_path == [u'latest']: for video in self.browser.latest_videos(): yield video def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and collection.split_path == [u'latest']: collection.title = u'Latest NoLiveTV videos' return raise CollectionNotFound(collection.split_path) OBJECTS = {NolifeTVVideo: fill_video}
class NewspaperFigaroModule(AbstractModule, CapMessages): MAINTAINER = u'Julien Hebert' EMAIL = '*****@*****.**' VERSION = '2.1' LICENSE = 'AGPLv3+' STORAGE = {'seen': {}} NAME = 'lefigaro' DESCRIPTION = u'Le Figaro French newspaper website' BROWSER = NewspaperFigaroBrowser RSS_FEED = 'http://rss.lefigaro.fr/lefigaro/laune?format=xml' RSSID = staticmethod(rssid) RSSSIZE = 30 PARENT = 'genericnewspaper' CONFIG = BackendConfig( Value('feed', label='RSS feed', choices={ 'actualites': u'actualites', 'flash-actu': u'flash-actu', 'politique': u'politique', 'international': u'international', 'actualite-france': u'actualite-france', 'hightech': u'hightech', 'sciences': u'sciences', 'sante': u'sante', 'lefigaromagazine': u'lefigaromagazine', 'photos': u'photos', 'economie': u'economie', 'societes': u'societes', 'medias': u'medias', 'immobilier': u'immobilier', 'assurance': u'assurance', 'retraite': u'retraite', 'placement': u'placement', 'impots': u'impots', 'conso': u'conso', 'emploi': u'emploi', 'culture': u'culture', 'cinema': u'cinema', 'musique': u'musique', 'livres': u'livres', 'theatre': u'theatre', 'lifestyle': u'lifestyle', 'automobile': u'automobile', 'gastronomie': u'gastronomie', 'horlogerie': u'horlogerie', 'mode-homme': u'mode-homme', 'sortir-paris': u'sortir-paris', 'vins': u'vins', 'voyages': u'voyages', 'sport': u'sport', 'football': u'football', 'rugby': u'rugby', 'tennis': u'tennis', 'cyclisme': u'cyclisme', 'sport-business': u'sport-business' })) def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) self.RSS_FEED = "http://www.lefigaro.fr/rss/figaro_%s.xml" % self.config[ 'feed'].get() def iter_threads(self): for article in Newsfeed(self.RSS_FEED, self.RSSID).iter_entries(): thread = Thread(article.id) thread.title = article.title thread.date = article.datetime yield (thread)
class NewsfeedModule(Module, CapMessages): NAME = 'newsfeed' MAINTAINER = u'Clément Schreiner' EMAIL = "*****@*****.**" VERSION = '2.1' DESCRIPTION = "Loads RSS and Atom feeds from any website" LICENSE = "AGPLv3+" CONFIG = BackendConfig(Value('url', label="Atom/RSS feed's url", regexp='https?://.*')) STORAGE = {'seen': []} def iter_threads(self): for article in Newsfeed(self.config['url'].get()).iter_entries(): yield self.get_thread(article.id, article) def get_thread(self, id, entry=None): if isinstance(id, Thread): thread = id id = thread.id else: thread = Thread(id) if entry is None: entry = Newsfeed(self.config['url'].get()).get_entry(id) if entry is None: return None flags = Message.IS_HTML if thread.id not in self.storage.get('seen', default=[]): flags |= Message.IS_UNREAD if len(entry.content) > 0: content = u"<p>Link %s</p> %s" % (entry.link, entry.content[0]) else: content = entry.link thread.title = entry.title thread.root = Message(thread=thread, id=0, url=entry.link, title=entry.title, sender=entry.author, receivers=None, date=entry.datetime, parent=None, content=content, children=[], flags=flags) return thread def iter_unread_messages(self): for thread in self.iter_threads(): for m in thread.iter_all_messages(): if m.flags & m.IS_UNREAD: yield m def set_message_read(self, message): self.storage.get('seen', default=[]).append(message.thread.id) self.storage.save() def fill_thread(self, thread, fields): return self.get_thread(thread) OBJECTS = {Thread: fill_thread}
class ArteModule(Module, CapVideo, CapCollection): NAME = 'arte' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' VERSION = '1.1' DESCRIPTION = 'Arte French and German TV' LICENSE = 'AGPLv3+' order = { 'AIRDATE_DESC': 'Date', 'VIEWS': 'Views', 'ALPHA': 'Alphabetic', 'LAST_CHANCE': 'Last chance' } versions_choice = OrderedDict([(k, u'%s' % (v.get('label'))) for k, v in VERSION_VIDEO.items]) format_choice = OrderedDict([(k, u'%s' % (v)) for k, v in FORMATS.items]) lang_choice = OrderedDict([(k, u'%s' % (v.get('label'))) for k, v in LANG.items]) quality_choice = [u'%s' % (k) for k, v in QUALITY.items] CONFIG = BackendConfig( Value('lang', label='Lang of videos', choices=lang_choice, default='FRENCH'), Value('order', label='Sort order', choices=order, default='AIRDATE_DESC'), Value('quality', label='Quality of videos', choices=quality_choice, default=QUALITY.HD), Value('format', label='Format of videos', choices=format_choice, default=FORMATS.HTTP_MP4), Value('version', label='Version of videos', choices=versions_choice)) BROWSER = ArteBrowser def create_default_browser(self): return self.create_browser(lang=self.config['lang'].get(), quality=self.config['quality'].get(), order=self.config['order'].get(), format=self.config['format'].get(), version=self.config['version'].get()) def parse_id(self, _id): m = re.match('^(\w+)\.(.*)', _id) if m: return m.groups() m = re.match('https?://www.arte.tv/guide/\w+/(?P<id>.+)/(.*)', _id) if m: return SITE.PROGRAM.get('id'), m.group(1) m = re.match( 'https?://(%s).arte.tv/(\w+)/(.*)' % ('|'.join(value.get('id') for value in SITE.values)), _id) if m: return m.group(1), '/%s/%s' % (m.group(2), m.group(3)) return 'videos', _id def get_video(self, _id): site, _id = self.parse_id(_id) if site in [value.get('id') for value in SITE.values]: _site = (value for value in SITE.values if value.get('id') == site).next() return getattr(self.browser, _site.get('video'))(_id) else: return self.browser.get_video(_id) def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): return self.browser.search_videos(pattern) def fill_arte_video(self, video, fields): if fields != ['thumbnail']: video = self.browser.get_video(video.id, video) if 'thumbnail' in fields and video and video.thumbnail: video.thumbnail.data = self.browser.open( video.thumbnail.url).content return video def fill_site_video(self, video, fields): if fields != ['thumbnail']: for site in SITE.values: m = re.match('%s\.(.*)' % site.get('id'), video.id) if m: video = getattr(self.browser, site.get('video'))(m.group(1), video) break if 'thumbnail' in fields and video and video.thumbnail: video.thumbnail.data = self.browser.open( video.thumbnail.url).content return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield Collection([u'arte-latest'], u'Latest Arte videos') for site in SITE.values: yield Collection([site.get('id')], site.get('label')) if collection.path_level == 1: if collection.split_path == [u'arte-latest']: for video in self.browser.latest_videos(): yield video else: for site in SITE.values: if collection.split_path[0] == site.get( 'id') and collection.path_level in site.keys(): for item in getattr( self.browser, site.get(collection.path_level))(): yield item if collection.path_level >= 2: for site in SITE.values: if collection.split_path[0] == site.get( 'id') and collection.path_level in site.keys(): for item in getattr(self.browser, site.get(collection.path_level))( collection.split_path): yield item def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and (collection.split_path == [u'arte-latest'] or collection.split_path[0] in [ value.get('id') for value in SITE.values ]): return if BaseVideo in objs and collection.path_level >= 2 and\ collection.split_path[0] in [value.get('id') for value in SITE.values]: return raise CollectionNotFound(collection.split_path) OBJECTS = {ArteVideo: fill_arte_video, ArteSiteVideo: fill_site_video}
class SocieteGeneraleModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, CapProfile, CapDocument): NAME = 'societegenerale' MAINTAINER = u'Jocelyn Jaubert' EMAIL = '*****@*****.**' VERSION = '2.1' LICENSE = 'LGPLv3+' DESCRIPTION = u'Société Générale' CONFIG = BackendConfig( ValueBackendPassword('login', label='Code client', masked=False), ValueBackendPassword('password', label='Code secret'), Value('website', label='Type de compte', default='par', choices={ 'par': 'Particuliers', 'pro': 'Professionnels', 'ent': 'Entreprises' }), # SCA ValueTransient('code'), ValueTransient('resume'), ValueTransient('request_information'), ) accepted_document_types = (DocumentTypes.STATEMENT, DocumentTypes.RIB) def create_default_browser(self): website = self.config['website'].get() browsers = { 'par': SocieteGenerale, 'pro': SGProfessionalBrowser, 'ent': SGEnterpriseBrowser } self.BROWSER = browsers[website] if website in ( 'par', 'pro', ): return self.create_browser(self.config, self.config['login'].get(), self.config['password'].get()) else: return self.create_browser(self.config['login'].get(), self.config['password'].get()) def iter_accounts(self): for account in self.browser.get_accounts_list(): yield account def get_account(self, _id): return find_object(self.browser.get_accounts_list(), id=_id, error=AccountNotFound) def iter_coming(self, account): if hasattr(self.browser, 'get_cb_operations'): transactions = list(self.browser.get_cb_operations(account)) return sorted_transactions(transactions) return self.browser.iter_coming(account) def iter_history(self, account): return self.browser.iter_history(account) def iter_investment(self, account): return self.browser.iter_investment(account) def iter_contacts(self): if not hasattr(self.browser, 'get_advisor'): raise NotImplementedError() return self.browser.get_advisor() def get_profile(self): if not hasattr(self.browser, 'get_profile'): raise NotImplementedError() return self.browser.get_profile() def iter_transfer_recipients(self, origin_account): if self.config['website'].get() not in ('par', 'pro'): raise NotImplementedError() if not isinstance(origin_account, Account): origin_account = find_object(self.iter_accounts(), id=origin_account, error=AccountNotFound) return self.browser.iter_recipients(origin_account) def new_recipient(self, recipient, **params): if self.config['website'].get() not in ('par', 'pro'): raise NotImplementedError() recipient.label = ' '.join(w for w in re.sub( '[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) return self.browser.new_recipient(recipient, **params) def init_transfer(self, transfer, **params): if self.config['website'].get() not in ('par', 'pro'): raise NotImplementedError() transfer.label = ' '.join( w for w in re.sub('[^0-9a-zA-Z ]+', '', transfer.label).split()) self.logger.info('Going to do a new transfer') account = strict_find_object(self.iter_accounts(), iban=transfer.account_iban) if not account: account = strict_find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) recipient = strict_find_object(self.iter_transfer_recipients( account.id), id=transfer.recipient_id) if not recipient: recipient = strict_find_object(self.iter_transfer_recipients( account.id), iban=transfer.recipient_iban, error=RecipientNotFound) transfer.amount = transfer.amount.quantize(Decimal('.01')) return self.browser.init_transfer(account, recipient, transfer) def execute_transfer(self, transfer, **params): if self.config['website'].get() not in ('par', 'pro'): raise NotImplementedError() return self.browser.execute_transfer(transfer) def transfer_check_exec_date(self, old_exec_date, new_exec_date): return old_exec_date <= new_exec_date <= old_exec_date + timedelta( days=4) def iter_resources(self, objs, split_path): if Account in objs: self._restrict_level(split_path) return self.iter_accounts() if Subscription in objs: self._restrict_level(split_path) return self.iter_subscription() def get_subscription(self, _id): return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) def get_document(self, _id): subscription_id = _id.split('_')[0] subscription = self.get_subscription(subscription_id) return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) def iter_subscription(self): return self.browser.iter_subscription() def iter_documents(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) return self.browser.iter_documents(subscription) def iter_documents_by_types(self, subscription, accepted_types): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) if self.config['website'].get() not in ('ent', 'pro'): for doc in self.browser.iter_documents_by_types( subscription, accepted_types): yield doc else: for doc in self.browser.iter_documents(subscription): if doc.type in accepted_types: yield doc def download_document(self, document): if not isinstance(document, Document): document = self.get_document(document) if document.url is NotAvailable: return return self.browser.open(document.url).content
class MailinatorModule(Module, CapMessages): NAME = 'mailinator' DESCRIPTION = u'mailinator temp mailbox' MAINTAINER = u'Vincent A' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.2' BROWSER = MailinatorBrowser CONFIG = BackendConfig(Value('inbox', label='Inbox', default='')) def iter_threads(self): inbox = self.config['inbox'].get() if not inbox: raise NotImplementedError() else: for d in self.browser.get_mails(inbox): thread = Thread(d['id']) thread.title = d['subject'] thread.flags = thread.IS_DISCUSSION msg = self.make_message(d, thread) if not msg.content: msg.content = self.browser.get_mail_content(msg.id) thread.root = msg yield thread def _get_messages_thread(self, inbox, thread): first = True for d in self.browser.get_mails(inbox): msg = self.make_message(d, thread) if not msg.content: msg.content = self.browser.get_mail_content(msg.id) if first: first = False thread.root = msg else: msg.parent = thread.root msg.parent.children.append(msg) def get_thread(self, _id): thread = Thread(_id) thread.title = 'Mail for %s' % _id thread.flags = thread.IS_DISCUSSION self._get_messages_thread(_id, thread) return thread def make_message(self, d, thread): msg = Message(thread, d['id']) msg.children = [] msg.sender = d['from'] msg.flags = 0 if not d.get('read', True): msg.flags = msg.IS_UNREAD msg.title = d['subject'] msg.date = d['datetime'] msg.receivers = [d['to']] return msg def fill_msg(self, msg, fields): if 'content' in fields: msg.content = self.browser.get_mail_content(msg.id) return msg OBJECTS = {Message: fill_msg}
class LolixModule(Module, CapJob): NAME = 'lolix' DESCRIPTION = u'Lolix French free software employment website' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' VERSION = '1.1' BROWSER = LolixBrowser region_choices = OrderedDict([ (k, u'%s' % (v)) for k, v in sorted({ '0': u'-- Indifférent --', '100000000': u'-- France entière', '100100000': u'-- France métropolitaine', '100100001': u'-- Alsace', '100100002': u'-- Auvergne', '100100003': u'-- Aquitaine', '100100004': u'-- Bourgogne', '100100005': u'-- Bretagne', '100100025': u'-- Centre', '100100027': u'-- Champagne-Ardenne', '100100030': u'-- Corse', '100100037': u'-- Franche-Comté', '100100040': u'-- Ile de France', '100100044': u'-- Languedoc-Roussillon', '100100048': u'-- Limousin', '100100051': u'-- Lorraine', '100100055': u'-- Midi-Pyrénées', '100100060': u'-- Nord-Pas-de-Calais', '100100073': u'-- Normandie', '100100076': u'-- Pays-de-Loire', '100100079': u'-- Picardie', '100100082': u'-- Poitou-Charentes', '100100085': u'-- Provence Alpes Cote d\'azur', '100100090': u'-- Rhône Alpes', '100200000': u'-- DOM et TOM', '100200001': u'-- Guadeloupe', '100200002': u'-- Guyane', '100200003': u'-- Martinique', '100200004': u'-- Réunion', '100200005': u'-- Saint-Pierre et Miquelon', '200000000': u'-- Etranger', }.iteritems()) ]) poste_choices = OrderedDict([ (k, u'%s' % (v)) for k, v in sorted({ '0': u'-- Indifférent --', '100000000': u'-- Service Technique', '100005000': u'-- Administrateur base de données', '100004000': u'-- Admin. Système/Réseaux', '100004004': u'-- Administrateur système', '100004002': u'-- Administrateur réseaux', '100007000': u'-- Analyste', '100002000': u'-- Chef de projet', '100002001': u'-- Chef de projet junior', '100002002': u'-- Chef de projet senior', '100021000': u'-- Consultant', '100003000': u'-- Développeur', '100003001': u'-- Développeur junior', '100003002': u'-- Développeur senior', '100009000': u'-- Directeur technique', '100006000': u'-- Ingénieur d\'étude', '100011000': u'-- Ingénieur support', '100012000': u'-- Responsable R & D', '100010000': u'-- Technicien', '100010002': u'-- Technicien hotline', '100010003': u'-- Technicien maintenance', '100020000': u'-- Webmaster', '200000000': u'-- Service Commercial', '200300000': u'-- Commercial', '200200000': u'-- Directeur commercial', '200100000': u'-- Technico commercial', '400000000': u'-- Service Marketing', '400100000': u'-- Responsable Marketing', '300000000': u'-- Service qualité', '300100000': u'-- Assistant qualité', '300200000': u'-- Responsable qualité', '2000000': u'-- Fondateur', '7000000': u'-- Formateur', '6000000': u'-- Journaliste', '500100000': u'-- Assistant(e) de direction', '4000000': u'-- Stagiaire', '5000000': u'-- Traducteur', }.iteritems()) ]) ''' '000000' in order to display description in console question the rule is : len(key) > 5 or ' ' in key: ''' contrat_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '000000': u'-- Indifférent --', '6': u'Alternance', '5': u'Apprentissage', '2': u'CDD', '1': u'CDI', '4': u'Freelance', '3': u'Stage', }.iteritems())]) limit_date_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ '2592000': u'30 jours', '5184000': u'60 jours', '7776000': u'90 jours', '0': u'Illimitée', }.iteritems())]) CONFIG = BackendConfig( Value('region', label=u'Région', choices=region_choices), Value('poste', label=u'Poste', choices=poste_choices), Value('contrat', label=u'Contrat', choices=contrat_choices), Value('limit_date', label=u'Date limite', choices=limit_date_choices)) def search_job(self, pattern=None): with self.browser: for job_advert in self.browser.advanced_search_job( pattern=pattern): yield job_advert def advanced_search_job(self): for advert in self.browser.advanced_search_job( region=self.config['region'].get(), poste=self.config['poste'].get(), contrat=int(self.config['contrat'].get()), limit_date=self.config['limit_date'].get()): yield advert def get_job_advert(self, _id, advert=None): with self.browser: return self.browser.get_job_advert(_id, advert) def fill_obj(self, advert, fields): self.get_job_advert(advert.id, advert) OBJECTS = {LolixJobAdvert: fill_obj}
class FeedlyModule(Module, CapMessages, CapCollection): NAME = 'feedly' DESCRIPTION = u'handle the popular RSS reading service Feedly' MAINTAINER = u'Bezleputh' EMAIL = '*****@*****.**' LICENSE = 'AGPLv3+' VERSION = '1.2' STORAGE = {'seen': []} CONFIG = BackendConfig( Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default='')) BROWSER = FeedlyBrowser def iter_resources(self, objs, split_path): collection = self.get_collection(objs, split_path) if collection.path_level == 0: return self.browser.get_categories() if collection.path_level == 1: return self.browser.get_feeds(split_path[0]) if collection.path_level == 2: url = self.browser.get_feed_url(split_path[0], split_path[1]) threads = [] for article in self.browser.get_unread_feed(url): thread = self.get_thread(article.id, article) threads.append(thread) return threads def validate_collection(self, objs, collection): if collection.path_level in [0, 1, 2]: return def get_thread(self, id, entry=None): if isinstance(id, Thread): thread = id id = thread.id else: thread = Thread(id) if entry is None: url = id.split('#')[0] for article in self.browser.get_unread_feed(url): if article.id == id: entry = article if entry is None: return None if thread.id not in self.storage.get('seen', default=[]): entry.flags = Message.IS_UNREAD entry.thread = thread thread.title = entry.title thread.root = entry return thread def iter_unread_messages(self): for thread in self.iter_threads(): for m in thread.iter_all_messages(): if m.flags & m.IS_UNREAD: yield m def iter_threads(self): for article in self.browser.iter_threads(): yield self.get_thread(article.id, article) def set_message_read(self, message): self.browser.set_message_read(message.thread.id.split('#')[-1]) self.storage.get('seen', default=[]).append(message.thread.id) self.storage.save() def fill_thread(self, thread, fields): return self.get_thread(thread) def create_default_browser(self): username = self.config['username'].get() if username: password = self.config['password'].get() login_browser = GoogleBrowser( username, password, 'https://feedly.com/v3/auth/callback&scope=profile+email&state=A8duE2XpzvtgcHt-q29qyBBK2fkpTefgqfzy7SY4GWUOPl3BgrSt4DRS-qKm9MRi_mXJRem8QW7RmNjpc_BIlkWc0JJvpay3UyzIErNvtaZLcsrUy94Ays3gTyispb8R0doguiky8gGxuCFNvJ9iXIB_SlwNhWABm7ut3nIgoMg3wodRgYOPFothhkErchrv076tBwXQA4Z8OIRyrQ' ) else: password = None login_browser = None return self.create_browser(username, password, login_browser) OBJECTS = {Thread: fill_thread}
class BanquePopulaireModule(Module, CapBank, CapContact): NAME = 'banquepopulaire' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.3' DESCRIPTION = u'Banque Populaire' LICENSE = 'AGPLv3+' website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted( { 'www.ibps.alpes.banquepopulaire.fr': u'Alpes', 'www.ibps.alsace.banquepopulaire.fr': u'Alsace', 'www.ibps.bpaca.banquepopulaire.fr': u'Aquitaine Centre atlantique', 'www.ibps.atlantique.banquepopulaire.fr': u'Atlantique', 'www.ibps.banquedesavoie.banquepopulaire.fr': u'Banque de Savoie', 'www.ibps.bpbfc.banquepopulaire.fr': u'Bourgogne-Franche Comté', 'www.ibps.bretagnenormandie.cmm.groupe.banquepopulaire.fr': u'Crédit Maritime Bretagne Normandie', 'www.ibps.atlantique.creditmaritime.groupe.banquepopulaire.fr': u'Crédit Maritime Atlantique', 'www.ibps.sudouest.creditmaritime.groupe.banquepopulaire.fr': u'Crédit Maritime du Littoral du Sud-Ouest', 'www.ibps.cotedazur.banquepopulaire.fr': u'Côte d\'azur', 'www.ibps.loirelyonnais.banquepopulaire.fr': u'Loire et Lyonnais', 'www.ibps.lorrainechampagne.banquepopulaire.fr': u'Lorraine Champagne', 'www.ibps.massifcentral.banquepopulaire.fr': u'Massif central', 'www.ibps.nord.banquepopulaire.fr': u'Nord', 'www.ibps.occitane.banquepopulaire.fr': u'Occitane', 'www.ibps.ouest.banquepopulaire.fr': u'Ouest', 'www.ibps.provencecorse.banquepopulaire.fr': u'Provence et Corse', 'www.ibps.rivesparis.banquepopulaire.fr': u'Rives de Paris', 'www.ibps.sud.banquepopulaire.fr': u'Sud', 'www.ibps.valdefrance.banquepopulaire.fr': u'Val de France', }.iteritems(), key=lambda k_v: (k_v[1], k_v[0]))]) CONFIG = BackendConfig( Value('website', label=u'Région', choices=website_choices), ValueBackendPassword('login', label='Identifiant', masked=False), ValueBackendPassword('password', label='Mot de passee')) BROWSER = BanquePopulaire def create_default_browser(self): repls = ('alsace', 'bpalc'), ('lorrainechampagne', 'bpalc') website = reduce(lambda a, kv: a.replace(*kv), repls, self.config['website'].get()) return self.create_browser(website, self.config['login'].get(), self.config['password'].get()) def iter_accounts(self): return self.browser.get_accounts_list() def get_account(self, _id): account = self.browser.get_account(_id) if account: return account else: raise AccountNotFound() def iter_history(self, account): return self.browser.get_history(account) def iter_coming(self, account): return self.browser.get_history(account, coming=True) def iter_investment(self, account): return self.browser.get_investment(account) def iter_contacts(self): return self.browser.get_advisor()
class OkCModule(Module, CapMessages, CapContact, CapMessagesPost, CapDating): NAME = 'okc' MAINTAINER = u'Roger Philibert' EMAIL = '*****@*****.**' VERSION = '2.1' LICENSE = 'AGPLv3+' DESCRIPTION = u'OkCupid' CONFIG = BackendConfig( Value('username', label='Username'), ValueBackendPassword('password', label='Password'), ValueBool('facebook', label='Do you login with Facebook?', default=False)) STORAGE = { 'profiles_walker': { 'viewed': [] }, 's***s': {}, } BROWSER = OkCBrowser def create_default_browser(self): if int(self.config['facebook'].get()): facebook = self.create_browser(klass=FacebookBrowser) facebook.login(self.config['username'].get(), self.config['password'].get()) else: facebook = None return self.create_browser(self.config['username'].get(), self.config['password'].get(), facebook) # ---- CapDating methods --------------------- def init_optimizations(self): self.add_optimization( 'PROFILE_WALKER', ProfilesWalker(self.weboob.scheduler, self.storage, self.browser)) # ---- CapMessages methods --------------------- def fill_thread(self, thread, fields): return self.get_thread(thread) def iter_threads(self): threads = self.browser.get_threads_list() for thread in threads: t = Thread(thread['user']['userid']) t.flags = Thread.IS_DISCUSSION t.title = u'Discussion with %s' % thread['user']['userinfo'][ 'displayname'] t.date = datetime.fromtimestamp(thread['time']) yield t def get_thread(self, thread): if not isinstance(thread, Thread): thread = Thread(thread) thread.flags = Thread.IS_DISCUSSION messages = self.browser.get_thread_messages(thread.id) contact = self.storage.get('s***s', thread.id, default={'lastmsg': datetime(1970, 1, 1)}) thread.title = u'Discussion with %s' % messages['fields']['username'] me = OkcContact(self.browser.get_profile(self.browser.me['userid'])) other = OkcContact(self.browser.get_profile(thread.id)) parent = None for message in messages['messages']: date = datetime.fromtimestamp(message['timestamp']) flags = 0 if contact['lastmsg'] < date: flags = Message.IS_UNREAD if message['from'] == thread.id: sender = other receiver = me else: receiver = other sender = me if message.get('read', False): flags |= Message.IS_RECEIVED # Apply that flag on all previous messages as the 'read' # attribute is only set on the last read message. pmsg = parent while pmsg: if pmsg.flags & Message.IS_NOT_RECEIVED: pmsg.flags |= Message.IS_RECEIVED pmsg.flags &= ~Message.IS_NOT_RECEIVED pmsg = pmsg.parent else: flags |= Message.IS_NOT_RECEIVED msg = Message(thread=thread, id=message['id'], title=thread.title, sender=sender.name, receivers=[receiver.name], date=date, content=to_unicode(HTMLParser().unescape( message['body'])), children=[], parent=parent, signature=sender.get_text(), flags=flags) if parent: parent.children = [msg] else: thread.root = msg parent = msg return thread def iter_unread_messages(self): for thread in self.iter_threads(): contact = self.storage.get( 's***s', thread.id, default={'lastmsg': datetime(1970, 1, 1)}) if thread.date <= contact['lastmsg']: continue thread = self.get_thread(thread) for message in thread.iter_all_messages(): if message.flags & message.IS_UNREAD: yield message def set_message_read(self, message): contact = self.storage.get('s***s', message.thread.id, default={'lastmsg': datetime(1970, 1, 1)}) if contact['lastmsg'] < message.date: contact['lastmsg'] = message.date self.storage.set('s***s', message.thread.id, contact) self.storage.save() # ---- CapMessagesPost methods --------------------- def post_message(self, message): self.browser.post_message(message.thread.id, message.content) # ---- CapContact methods --------------------- def fill_contact(self, contact, fields): if 'profile' in fields: contact = self.get_contact(contact) if contact and 'photos' in fields: for name, photo in contact.photos.items(): if photo.url and not photo.data: data = self.browser.open(photo.url).content contact.set_photo(name, data=data) if photo.thumbnail_url and not photo.thumbnail_data: data = self.browser.open(photo.thumbnail_url).content contact.set_photo(name, thumbnail_data=data) def fill_photo(self, photo, fields): if 'data' in fields and photo.url and not photo.data: photo.data = self.browser.open(photo.url).content if 'thumbnail_data' in fields and photo.thumbnail_url and not photo.thumbnail_data: photo.thumbnail_data = self.browser.open( photo.thumbnail_url).content return photo def get_contact(self, user_id): if isinstance(user_id, Contact): user_id = user_id.id info = self.browser.get_profile(user_id) return OkcContact(info) def iter_contacts(self, status=Contact.STATUS_ALL, ids=None): threads = self.browser.get_threads_list() for thread in threads: c = self.get_contact(thread['user']['username']) if c and (c.status & status) and (not ids or c.id in ids): yield c OBJECTS = { Thread: fill_thread, Contact: fill_contact, ContactPhoto: fill_photo }
class EHentaiBackend(BaseBackend, ICapGallery, ICapCollection): NAME = 'ehentai' MAINTAINER = u'Roger Philibert' EMAIL = '*****@*****.**' VERSION = '0.h' DESCRIPTION = 'E-Hentai galleries' LICENSE = 'AGPLv3+' BROWSER = EHentaiBrowser CONFIG = BackendConfig( Value('domain', label='Domain', default='g.e-hentai.org'), Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password')) def create_default_browser(self): username = self.config['username'].get() if username: password = self.config['password'].get() else: password = None return self.create_browser(self.config['domain'].get(), username, password) def search_gallery(self, pattern, sortby=None): with self.browser: return self.browser.search_gallery(pattern) def iter_gallery_images(self, gallery): self.fillobj(gallery, ('url', )) with self.browser: return self.browser.iter_gallery_images(gallery) ID_REGEXP = r'/?\d+/[\dabcdef]+/?' URL_REGEXP = r'.+/g/(%s)' % ID_REGEXP def get_gallery(self, _id): match = re.match(r'^%s$' % self.URL_REGEXP, _id) if match: _id = match.group(1) else: match = re.match(r'^%s$' % self.ID_REGEXP, _id) if match: _id = match.group(0) else: return None gallery = EHentaiGallery(_id) with self.browser: if self.browser.gallery_exists(gallery): return gallery else: return None def fill_gallery(self, gallery, fields): if not gallery.__iscomplete__(): with self.browser: self.browser.fill_gallery(gallery, fields) def fill_image(self, image, fields): with self.browser: image.url = self.browser.get_image_url(image) if 'data' in fields: ratelimit("ehentai_get", 2) image.data = self.browser.readurl(image.url) def iter_resources(self, objs, split_path): if BaseGallery in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest_nsfw']) if collection.split_path == [u'latest_nsfw']: for gallery in self.browser.latest_gallery(): yield gallery def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseGallery in objs and collection.split_path == [u'latest_nsfw']: collection.title = u'Latest E-Hentai galleries (NSFW)' return raise CollectionNotFound(collection.split_path) OBJECTS = {EHentaiGallery: fill_gallery, EHentaiImage: fill_image}
class CragrModule(Module, CapBankTransferAddRecipient, CapContact, CapProfile): NAME = 'cragr' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.4' DESCRIPTION = u'Crédit Agricole' LICENSE = 'AGPLv3+' website_choices = OrderedDict([ (k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'm.ca-alpesprovence.fr': u'Alpes Provence', 'm.ca-alsace-vosges.fr': u'Alsace-Vosges', 'm.ca-anjou-maine.fr': u'Anjou Maine', 'm.ca-aquitaine.fr': u'Aquitaine', 'm.ca-atlantique-vendee.fr': u'Atlantique Vendée', 'm.ca-briepicardie.fr': u'Brie Picardie', 'm.ca-cb.fr': u'Champagne Bourgogne', 'm.ca-centrefrance.fr': u'Centre France', 'm.ca-centreloire.fr': u'Centre Loire', 'm.ca-centreouest.fr': u'Centre Ouest', 'm.ca-centrest.fr': u'Centre Est', 'm.ca-charente-perigord.fr': u'Charente Périgord', 'm.ca-cmds.fr': u'Charente-Maritime Deux-Sèvres', 'm.ca-corse.fr': u'Corse', 'm.ca-cotesdarmor.fr': u'Côtes d\'Armor', 'm.ca-des-savoie.fr': u'Des Savoie', 'm.ca-finistere.fr': u'Finistere', 'm.ca-franchecomte.fr': u'Franche-Comté', 'm.ca-guadeloupe.fr': u'Guadeloupe', 'm.ca-illeetvilaine.fr': u'Ille-et-Vilaine', 'm.ca-languedoc.fr': u'Languedoc', 'm.ca-loirehauteloire.fr': u'Loire Haute Loire', 'm.ca-lorraine.fr': u'Lorraine', 'm.ca-martinique.fr': u'Martinique Guyane', 'm.ca-morbihan.fr': u'Morbihan', 'm.ca-nmp.fr': u'Nord Midi-Pyrénées', 'm.ca-nord-est.fr': u'Nord Est', 'm.ca-norddefrance.fr': u'Nord de France', 'm.ca-normandie-seine.fr': u'Normandie Seine', 'm.ca-normandie.fr': u'Normandie', 'm.ca-paris.fr': u'Ile-de-France', 'm.ca-pca.fr': u'Provence Côte d\'Azur', 'm.ca-reunion.fr': u'Réunion', 'm.ca-sudmed.fr': u'Sud Méditerranée', 'm.ca-sudrhonealpes.fr': u'Sud Rhône Alpes', 'm.ca-toulouse31.fr': u'Toulouse 31', # m.ca-toulousain.fr redirects here 'm.ca-tourainepoitou.fr': u'Tourraine Poitou', 'm.ca-valdefrance.fr': u'Val de France', 'm.lefil.com': u'Pyrénées Gascogne', }.items()) ]) CONFIG = BackendConfig( Value('website', label=u'Région', choices=website_choices), ValueBackendPassword('login', label=u'N° de compte', masked=False), ValueBackendPassword('password', label=u'Code personnel', regexp=r'\d{6}')) BROWSER = Cragr COMPAT_DOMAINS = { 'm.lefil.com': 'm.ca-pyrenees-gascogne.fr', } def create_default_browser(self): site_conf = self.config['website'].get() site_conf = self.COMPAT_DOMAINS.get(site_conf, site_conf) return self.create_browser(site_conf, self.config['login'].get(), self.config['password'].get()) def iter_accounts(self): return self.browser.get_accounts_list() def get_account(self, _id): return find_object(self.iter_accounts(), id=_id, error=AccountNotFound) def _history_filter(self, account, coming): today = date.today() def to_date(obj): if hasattr(obj, 'date'): return obj.date() return obj for tr in self.browser.get_history(account): tr_coming = to_date(tr.date) > today if coming == tr_coming: yield tr def iter_history(self, account): if account.type == Account.TYPE_CARD: return self._history_filter(account, False) return self.browser.get_history(account) def iter_coming(self, account): if account.type == Account.TYPE_CARD: return self._history_filter(account, True) return [] def iter_investment(self, account): for inv in self.browser.iter_investment(account): yield inv def iter_contacts(self): return self.browser.iter_advisor() def get_profile(self): if not hasattr(self.browser, 'get_profile'): raise NotImplementedError() return self.browser.get_profile() def iter_transfer_recipients(self, account): if not isinstance(account, Account): account = self.get_account(account) return self.browser.iter_transfer_recipients(account) def init_transfer(self, transfer, **params): def to_ascii(s): return s.encode('ascii', errors='ignore').decode('ascii') if transfer.label: transfer.label = re.sub(r'[+!]', '', to_ascii(transfer.label[:33])) return self.browser.init_transfer(transfer, **params) def execute_transfer(self, transfer, **params): return self.browser.execute_transfer(transfer, **params) def new_recipient(self, recipient, **params): return self.browser.new_recipient(recipient, **params)
class FourChanModule(Module, CapMessages): NAME = 'fourchan' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.3' LICENSE = 'AGPLv3+' DESCRIPTION = '4chan image board' CONFIG = BackendConfig(Value('boards', label='Boards to fetch')) STORAGE = {'boards': {}} BROWSER = FourChan def _splitid(self, id): return id.split('.', 1) def get_thread(self, id): thread = None if isinstance(id, Thread): thread = id id = thread.id if '.' not in id: self.logger.warning('Malformated ID (%s)' % id) return board, thread_id = self._splitid(id) with self.browser: _thread = self.browser.get_thread(board, thread_id) flags = 0 if _thread.id not in self.storage.get('boards', board, default={}): flags |= Message.IS_UNREAD if not thread: thread = Thread(id) thread.title = _thread.filename thread.root = Message( thread=thread, id=0, # root message title=_thread.filename, sender=_thread.author, receivers=None, date=_thread.datetime, parent=None, content=_thread.text, signature=None, children=[], flags=flags | Message.IS_HTML) for comment in _thread.comments: flags = 0 if comment.id not in self.storage.get('boards', board, _thread.id, default=[]): flags |= Message.IS_UNREAD m = Message(thread=thread, id=comment.id, title=_thread.filename, sender=comment.author, receivers=None, date=comment.datetime, parent=thread.root, content=comment.text, signature=None, children=None, flags=flags | Message.IS_HTML) thread.root.children.append(m) return thread def iter_threads(self): for board in self.config['boards'].get().split(' '): with self.browser: threads = self.browser.get_threads(board) for thread in threads: t = Thread('%s.%s' % (board, thread.id)) t.title = thread.filename yield t def iter_unread_messages(self): for thread in self.iter_threads(): self.fill_thread(thread, 'root') for m in thread.iter_all_messages(): if m.flags & Message.IS_UNREAD: yield m def set_message_read(self, message): board, thread_id = self._splitid(message.thread.id) self.storage.set( 'boards', board, thread_id, self.storage.get('boards', board, thread_id, default=[]) + [message.id]) self.storage.save() def fill_thread(self, thread, fields): return self.get_thread(thread) OBJECTS = {Thread: fill_thread}
class DailymotionModule(Module, CapVideo, CapCollection): NAME = 'dailymotion' MAINTAINER = u'Romain Bignon' EMAIL = '*****@*****.**' VERSION = '1.2' DESCRIPTION = 'Dailymotion video streaming website' LICENSE = 'AGPLv3+' BROWSER = DailymotionBrowser resolution_choice = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({ u'480': u'480p', u'240': u'240p', u'380': u'380p', u'720': u'720p', u'1080': u'1080p' }.iteritems())]) format_choice = [u'm3u8', u'mp4'] CONFIG = BackendConfig(Value('resolution', label=u'Resolution', choices=resolution_choice), Value('format', label=u'Format', choices=format_choice)) SORTBY = ['relevance', 'rated', 'visited', None] def create_default_browser(self): resolution = self.config['resolution'].get() format = self.config['format'].get() return self.create_browser(resolution=resolution, format=format) def get_video(self, _id): m = re.match('http://[w\.]*dailymotion\.com/video/(.*)', _id) if m: _id = m.group(1) if not _id.startswith('http'): return self.browser.get_video(_id) def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): return self.browser.search_videos(pattern, self.SORTBY[sortby]) def fill_video(self, video, fields): if fields != ['thumbnail']: # if we don't want only the thumbnail, we probably want also every fields video = self.browser.get_video(video.id, video) if 'thumbnail' in fields and video.thumbnail: video.thumbnail.data = self.browser.open(video.thumbnail.url).content return video def iter_resources(self, objs, split_path): if BaseVideo in objs: collection = self.get_collection(objs, split_path) if collection.path_level == 0: yield self.get_collection(objs, [u'latest']) if collection.split_path == [u'latest']: for video in self.browser.latest_videos(): yield video def validate_collection(self, objs, collection): if collection.path_level == 0: return if BaseVideo in objs and collection.split_path == [u'latest']: collection.title = u'Latest Dailymotion videos' return raise CollectionNotFound(collection.split_path) OBJECTS = {BaseVideo: fill_video}