Example #1
0
 def iter_station_departures(self, station_id, arrival_id=None):
     url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/index.php?gare=%s' % unicode(
         station_id)
     result = self.openurl(url.encode('utf-8')).read()
     result = result
     departure = ''
     for line in result.split('&'):
         if not '=' in line:
             raise BrokenPageError('Unable to parse result: %s' % line)
         key, value = line.split('=', 1)
         if key == 'nomgare':
             departure = value
         elif key.startswith('ligne'):
             _type, unknown, _time, arrival, served, late, late_reason = value.split(
                 ';', 6)
             yield {
                 'type':
                 to_unicode(_type),
                 'time':
                 datetime.combine(date.today(),
                                  time(*[int(x)
                                         for x in _time.split(':')])),
                 'departure':
                 to_unicode(departure),
                 'arrival':
                 to_unicode(arrival).strip(),
                 'late':
                 late and time(0, int(late.split()[0])) or time(),
                 'late_reason':
                 to_unicode(late_reason).replace('\n', '').strip()
             }
Example #2
0
 def retrieveContact_eb(self, backend, error, backtrace):
     content = unicode(
         self.tr('Unable to get contact:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += u'\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while getting contact'),
                          content, QMessageBox.Ok)
Example #3
0
    def get_video(self, _id):
        video = QuviVideo(_id)

        parser = LibQuvi()
        if not parser.load():
            raise UserError('Make sure libquvi 0.4 is installed')

        try:
            info = parser.get_info(video.page_url)
        except QuviError as qerror:
            raise UserError(qerror.message)

        video.url = to_unicode(info.get('url'))
        if not video.url:
            raise NotImplementedError()

        video.ext = to_unicode(info.get('suffix'))
        video.title = to_unicode(info.get('title'))
        video.page = to_unicode(info.get('page'))
        duration = int(info.get('duration', 0))
        if duration:
            video.duration = datetime.timedelta(milliseconds=duration)
        if info.get('thumbnail'):
            video.thumbnail = Thumbnail(info.get('thumbnail'))
            video.thumbnail.url = video.thumbnail.id
        return video
Example #4
0
    def get_video(self, _id):
        video = QuviVideo(_id)

        parser = LibQuvi()
        if not parser.load():
            raise UserError('Make sure libquvi 0.4 is installed')

        try:
            info = parser.get_info(video.page_url)
        except QuviError as qerror:
            raise UserError(qerror.message)

        video.url = to_unicode(info.get('url'))
        if not video.url:
            raise NotImplementedError()

        video.ext = to_unicode(info.get('suffix'))
        video.title = to_unicode(info.get('title'))
        video.page = to_unicode(info.get('page'))
        duration = int(info.get('duration', 0))
        if duration:
            video.duration = datetime.timedelta(milliseconds=duration)
        if info.get('thumbnail'):
            video.thumbnail = BaseImage(info.get('thumbnail'))
            video.thumbnail.url = video.thumbnail.id
        return video
Example #5
0
 def _postReply_eb(self, backend, error, backtrace):
     content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while posting reply'),
                          content, QMessageBox.Ok)
     self.process_reply = None
Example #6
0
    def iter_routes(self):
        try:
            table = self.parser.select(self.document.getroot(), 'table.horaires3', 1)
        except BrokenPageError:
            raise StationNotFound('Station not found')

        departure = self.parser.select(table, 'td.caption strong', 1).text
        for tr in table.findall('tr'):
            if len(tr.findall('td')) != 4:
                continue

            code_mission = self.parser.select(tr, 'td[headers=Code_de_mission] a', 1).text.strip()
            time_s = self.parser.select(tr, 'td[headers=Heure_de_passage]', 1).text.strip().rstrip(u'\xa0*')
            destination = self.parser.select(tr, 'td[headers=Destination]', 1).text.strip()
            plateform = self.parser.select(tr, 'td[headers=Voie]', 1).text.strip()

            late_reason = None
            time = None
            try :
                time = datetime.datetime.combine(datetime.date.today(), datetime.time(*[int(x) for x in time_s.split(':')]))
            except ValueError:
                late_reason = time_s
                self.logger.warning('Unable to parse datetime "%s"' % time_s)

            yield {'type':        to_unicode(code_mission),
                   'time':        time,
                   'departure':   to_unicode(departure),
                   'arrival':     to_unicode(destination),
                   'late':        datetime.time(),
                   'late_reason': late_reason,
                   'plateform':   to_unicode(plateform)}
Example #7
0
    def get_video(self, video=None):
        _id = to_unicode(self.group_dict['id'])
        if video is None:
            video = YoujizzVideo(_id)
        title_el = self.parser.select(self.document.getroot(), 'title', 1)
        video.title = to_unicode(title_el.text.strip())

        # youjizz HTML is crap, we must parse it with regexps
        data = lxml.html.tostring(self.document.getroot())
        m = re.search(r'<strong>.*?Runtime.*?</strong> (.+?)</div>', data)
        if m:
            txt = m.group(1).strip()
            if txt == 'Unknown':
                video.duration = NotAvailable
            else:
                minutes, seconds = (int(v) for v in to_unicode(txt).split(':'))
                video.duration = datetime.timedelta(minutes=minutes, seconds=seconds)
        else:
            raise BrokenPageError('Unable to retrieve video duration')

        real_id = int(_id.split('-')[-1])
        data = self.browser.readurl('http://www.youjizz.com/videos/embed/%s' % real_id)

        video_file_urls = re.findall(r'"(http://[^",]+\.youjizz\.com[^",]+\.flv(?:\?[^"]*)?)"', data)
        if len(video_file_urls) == 0:
            raise BrokenPageError('Video URL not found')
        elif len(video_file_urls) > 1:
            raise BrokenPageError('Many video file URL found')
        else:
            video.url = to_unicode(video_file_urls[0])

        return video
Example #8
0
    def get_video(self, video=None):
        _id = to_unicode(self.group_dict["id"])
        if video is None:
            video = YoujizzVideo(_id)
        title_el = select(self.document.getroot(), "title", 1)
        video.title = to_unicode(title_el.text.strip())

        # youjizz HTML is crap, we must parse it with regexps
        data = lxml.html.tostring(self.document.getroot())
        m = re.search(r"<strong>.*?Runtime.*?</strong> (.+?)<br.*>", data)
        try:
            if m:
                minutes, seconds = (int(v) for v in to_unicode(m.group(1).strip()).split(":"))
                video.duration = datetime.timedelta(minutes=minutes, seconds=seconds)
            else:
                raise Exception()
        except Exception:
            raise SelectElementException("Could not retrieve video duration")

        video_file_urls = re.findall(r'"(http://media[^ ,]+\.flv)"', data)
        if len(video_file_urls) == 0:
            raise SelectElementException("Video URL not found")
        elif len(video_file_urls) > 1:
            raise SelectElementException("Many video file URL found")
        else:
            video.url = video_file_urls[0]

        return video
Example #9
0
    def get_steps(self):
        errors = []
        for p in self.parser.select(self.document.getroot(), 'p.errors'):
            if p.text:
                errors.append(p.text.strip())

        if len(errors) > 0:
            raise RoadmapError('Unable to establish a roadmap: %s' % ', '.join(errors))

        current_step = None
        i = 0
        for tr in self.parser.select(self.document.getroot(), 'table.horaires2 tbody tr'):
            if not 'class' in tr.attrib:
                continue
            elif tr.attrib['class'] == 'trHautTroncon':
                current_step = {}
                current_step['id'] = i
                i += 1
                current_step['start_time'] = self.parse_time(self.parser.select(tr, 'td.formattedHeureDepart p', 1).text.strip())
                current_step['line'] = to_unicode(self.parser.select(tr, 'td.rechercheResultatColumnMode img')[-1].attrib['alt'])
                current_step['departure'] = to_unicode(self.parser.select(tr, 'td.descDepart p strong', 1).text.strip())
                current_step['duration'] = self.parse_duration(self.parser.select(tr, 'td.rechercheResultatVertAlign', 1).text.strip())
            elif tr.attrib['class'] == 'trBasTroncon':
                current_step['end_time'] = self.parse_time(self.parser.select(tr, 'td.formattedHeureArrivee p', 1).text.strip())
                current_step['arrival'] = to_unicode(self.parser.select(tr, 'td.descArrivee p strong', 1).text.strip())
                yield current_step
Example #10
0
    def get_video(self, video=None):
        _id = to_unicode(self.group_dict['id'])
        if video is None:
            video = YoujizzVideo(_id)
        title_el = self.parser.select(self.document.getroot(), 'title', 1)
        video.title = to_unicode(title_el.text.strip())

        # youjizz HTML is crap, we must parse it with regexps
        data = lxml.html.tostring(self.document.getroot())
        m = re.search(r'<strong>.*?Runtime.*?</strong> (.+?)</div>', data)
        if m:
            txt = m.group(1).strip()
            if txt == 'Unknown':
                video.duration = NotAvailable
            else:
                minutes, seconds = (int(v) for v in to_unicode(txt).split(':'))
                video.duration = datetime.timedelta(minutes=minutes,
                                                    seconds=seconds)
        else:
            raise BrokenPageError('Unable to retrieve video duration')

        real_id = int(_id.split('-')[-1])
        data = self.browser.readurl('http://www.youjizz.com/videos/embed/%s' %
                                    real_id)

        video_file_urls = re.findall(
            r'"(http://[^",]+\.youjizz\.com[^",]+\.flv(?:\?[^"]*)?)"', data)
        if len(video_file_urls) == 0:
            raise BrokenPageError('Video URL not found')
        elif len(video_file_urls) > 1:
            raise BrokenPageError('Many video file URL found')
        else:
            video.url = to_unicode(video_file_urls[0])

        return video
Example #11
0
    def get_list(self):
        table = self.find_table()
        for tr in self.parser.select(table, 'tr', 'many'):
            tds = self.parser.select(tr, 'td')
            if len(tds) != 6:
                continue
            tdlabel, tdid, tdcur, tdupdated, tdbal, tdbalcur = tds

            account = Account()
            account.label = to_unicode(tdlabel.text_content().strip())
            # this is important - and is also the last part of the id (considering spaces)
            # we can't use only the link as it does not goes where we want
            try:
                link = self.parser.select(tdlabel, 'a', 1)
            except BrokenPageError:
                # probably an account we can't display the history
                account._link_id = None
            else:
                account._link_id = parse_qs(link.attrib['href'])['ch4'][0]
            account.id = to_unicode(tdid.text.strip().replace(' ', ''))
            account.iban = 'FR76' + account.id
            # just in case we are showing the converted balances
            account._main_currency = Account.get_currency(tdcur.text)
            # we have to ignore those accounts, because using NotAvailable
            # makes boobank and probably many others crash
            if tdbal.text_content().strip() == 'indisponible':
                continue
            account.balance = Decimal(
                Transaction.clean_amount(tdbal.text_content()))
            account.currency = Account.get_currency(tdbalcur.text)
            account._updated = datetime.strptime(tdupdated.text, '%d/%m/%Y')
            yield account
Example #12
0
    def bcall_error_handler(self, backend, error, backtrace):
        if isinstance(error, TransferStep):
            params = {}
            for field in error.fields:
                v = self.ask(field)
                params[field.id] = v
            #backend.config['accept_transfer'].set(v)
            params['backends'] = backend
            self.start_format()
            for transfer in self.do('transfer', error.transfer, **params):
                self.format(transfer)
        elif isinstance(error, AddRecipientStep):
            params = {}
            params['backends'] = backend
            for field in error.fields:
                v = self.ask(field)
                params[field.id] = v
            try:
                next(iter(self.do('add_recipient', error.recipient, **params)))
            except CallErrors as e:
                self.bcall_errors_handler(e)
        elif isinstance(error, TransferInvalidAmount):
            print(u'Error(%s): %s' % (backend.name, to_unicode(error)
                                      or 'The transfer amount is invalid'),
                  file=self.stderr)
        elif isinstance(error, TransferInvalidLabel):
            print(u'Error(%s): %s' % (backend.name, to_unicode(error)
                                      or 'The transfer label is invalid'),
                  file=self.stderr)
        elif isinstance(error, TransferInvalidEmitter):
            print(u'Error(%s): %s' % (backend.name, to_unicode(error)
                                      or 'The transfer emitter is invalid'),
                  file=self.stderr)
        elif isinstance(error, TransferInvalidRecipient):
            print(u'Error(%s): %s' % (backend.name, to_unicode(error)
                                      or 'The transfer recipient is invalid'),
                  file=self.stderr)
        elif isinstance(error, TransferInvalidDate):
            print(u'Error(%s): %s' %
                  (backend.name, to_unicode(error)
                   or 'The transfer execution date is invalid'),
                  file=self.stderr)
        elif isinstance(error, CaptchaQuestion):
            if not self.captcha_weboob.count_backends():
                print(
                    'Error(%s): Site requires solving a CAPTCHA but no CapCaptchaSolver backends were configured'
                    % backend.name,
                    file=self.stderr)
                return False

            print(
                'Info(%s): Encountered CAPTCHA, please wait for its resolution, it can take dozens of seconds'
                % backend.name,
                file=self.stderr)
            job = exception_to_job(error)
            self.solve_captcha(job, backend)
            return False
        else:
            return super(Boobank,
                         self).bcall_error_handler(backend, error, backtrace)
Example #13
0
 def add_cookie(self, name, value):
     # httplib/cookielib don't seem to like unicode cookies...
     if sys.version_info.major < 3:
         name = to_unicode(name).encode('utf-8')
         value = to_unicode(value).encode('utf-8')
     self.browser.logger.debug('adding cookie %r=%r', name, value)
     self.browser.session.cookies.set(name, value, domain=urlsplit(self.url).hostname)
Example #14
0
    def get_list(self):
        table = self.find_table()
        for tr in self.parser.select(table, 'tr', 'many'):
            tds = self.parser.select(tr, 'td')
            if len(tds) != 6:
                continue
            tdlabel, tdid, tdcur, tdupdated, tdbal, tdbalcur = tds

            account = Account()
            account.label = to_unicode(tdlabel.text_content().strip())
            # this is important - and is also the last part of the id (considering spaces)
            # we can't use only the link as it does not goes where we want
            try:
                link = self.parser.select(tdlabel, 'a', 1)
            except BrokenPageError:
                # probably an account we can't display the history
                account._link_id = None
            else:
                account._link_id = parse_qs(link.attrib['href'])['ch4'][0]
            account.id = to_unicode(tdid.text.strip().replace(' ', ''))
            account.iban = 'FR76' + account.id
            # just in case we are showing the converted balances
            account._main_currency = Account.get_currency(tdcur.text)
            # we have to ignore those accounts, because using NotAvailable
            # makes boobank and probably many others crash
            if tdbal.text_content().strip() == 'indisponible':
                continue
            account.balance = Decimal(Transaction.clean_amount(tdbal.text_content()))
            account.currency = Account.get_currency(tdbalcur.text)
            account._updated = datetime.strptime(tdupdated.text, '%d/%m/%Y')
            yield account
Example #15
0
 def _postReply_eb(self, backend, error, backtrace):
     content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while posting reply'),
                          content, QMessageBox.Ok)
     self.process_reply = None
Example #16
0
    def get_stream_info(self, radio, url):
        stream = BaseAudioStream(0)
        current = StreamInfo(0)

        r = self.browser.open(url, stream=True, headers={'Icy-Metadata':'1'})

        stream.bitrate = int(r.headers['icy-br'].split(',')[0])

        r.raw.read(int(r.headers['icy-metaint']))
        size = ord(r.raw.read(1))
        content = r.raw.read(size*16)
        r.close()

        for s in content.split("\x00")[0].split(";"):
            a = s.split("=")
            if a[0] == "StreamTitle":
                stream.title = to_unicode(a[1].split("'")[1])
                res = stream.title.split(" - ")
                current.who = to_unicode(res[0])
                if(len(res) == 1):
                    current.what = ""
                else:
                    current.what = to_unicode(res[1])

        stream.format=u'mp3'
        stream.url = url
        return [stream], current
Example #17
0
 def add_cookie(self, name, value):
     # httplib/cookielib don't seem to like unicode cookies...
     if sys.version_info.major < 3:
         name = to_unicode(name).encode('utf-8')
         value = to_unicode(value).encode('utf-8')
     self.browser.logger.debug('adding cookie %r=%r', name, value)
     self.browser.session.cookies.set(name, value)
Example #18
0
    def get_stream_info(self, radio, url):
        stream = BaseAudioStream(0)
        current = StreamInfo(0)

        r = self.browser.open(url, stream=True, headers={'Icy-Metadata': 1})

        stream.bitrate = int(r.headers['icy-br'].split(',')[0])

        r.raw.read(int(r.headers['icy-metaint']))
        size = ord(r.raw.read(1))
        content = r.raw.read(size * 16)
        r.close()

        for s in content.split("\x00")[0].split(";"):
            a = s.split("=")
            if a[0] == "StreamTitle":
                stream.title = to_unicode(a[1].split("'")[1])
                res = stream.title.split(" - ")
                current.who = to_unicode(res[0])
                if (len(res) == 1):
                    current.what = ""
                else:
                    current.what = to_unicode(res[1])

        stream.format = u'mp3'
        stream.url = url
        return [stream], current
Example #19
0
    def bcall_error_handler(self, backend, error, backtrace):
        """
        Handler for an exception inside the CallErrors exception.

        This method can be overrided to support more exceptions types.
        """
        if isinstance(error, BrowserQuestion):
            for field in error.fields:
                v = self.ask(field)
                if v:
                    backend.config[field.id].set(v)
        elif isinstance(error, BrowserIncorrectPassword):
            msg = unicode(error)
            if not msg:
                msg = 'invalid login/password.'
            print('Error(%s): %s' % (backend.name, msg), file=self.stderr)
            if self.ask('Do you want to reconfigure this backend?', default=True):
                self.unload_backends(names=[backend.name])
                self.edit_backend(backend.name)
                self.load_backends(names=[backend.name])
        elif isinstance(error, BrowserSSLError):
            print(u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC, file=self.stderr)
        elif isinstance(error, BrowserForbidden):
            msg = unicode(error)
            if not msg:
                msg = 'access denied'
            print(u'Error(%s): %s' % (backend.name, msg or 'Forbidden'), file=self.stderr)
        elif isinstance(error, BrowserUnavailable):
            msg = unicode(error)
            if not msg:
                msg = 'website is unavailable.'
            print(u'Error(%s): %s' % (backend.name, msg), file=self.stderr)
        elif isinstance(error, NotImplementedError):
            print(u'Error(%s): this feature is not supported yet by this backend.' % backend.name, file=self.stderr)
            print(u'      %s   To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name)), file=self.stderr)
            print(u'      %s   please contact: %s <*****@*****.**>' % (' ' * len(backend.name), backend.MAINTAINER, backend.NAME), file=self.stderr)
        elif isinstance(error, UserError):
            print(u'Error(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr)
        elif isinstance(error, MoreResultsAvailable):
            print(u'Hint: There are more results for backend %s' % (backend.name), file=self.stderr)
        else:
            print(u'Bug(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr)

            minfo = self.weboob.repositories.get_module_info(backend.NAME)
            if minfo and not minfo.is_local():
                self.weboob.repositories.update_repositories(ConsoleProgress(self))

                # minfo of the new available module
                minfo = self.weboob.repositories.get_module_info(backend.NAME)
                if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \
                   self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \
                   self.install_module(minfo):
                    print('New version of module %s has been installed. Retry to call the command.' % minfo.name)
                    return

            if logging.root.level <= logging.DEBUG:
                print(backtrace, file=self.stderr)
            else:
                return True
Example #20
0
 def load(self, items):
     self.version = int(items['version'])
     self.capabilities = items['capabilities'].split()
     self.description = to_unicode(items['description'])
     self.maintainer = to_unicode(items['maintainer'])
     self.license = to_unicode(items['license'])
     self.icon = items['icon'].strip() or None
     self.urls = items['urls']
Example #21
0
 def load(self, items):
     self.version = int(items['version'])
     self.capabilities = items['capabilities'].split()
     self.description = to_unicode(items['description'])
     self.maintainer = to_unicode(items['maintainer'])
     self.license = to_unicode(items['license'])
     self.icon = items['icon'].strip() or None
     self.urls = items['urls']
Example #22
0
 def load(self, items):
     self.version = int(items["version"])
     self.capabilities = items["capabilities"].split()
     self.description = to_unicode(items["description"])
     self.maintainer = to_unicode(items["maintainer"])
     self.license = to_unicode(items["license"])
     self.icon = items["icon"].strip() or None
     self.urls = items["urls"]
Example #23
0
 def _saveNotes_eb(self, backend, error, backtrace):
     self.ui.saveButton.setEnabled(True)
     self.ui.textEdit.setEnabled(True)
     content = unicode(self.tr('Unable to save notes:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
         QMessageBox.critical(self, self.tr('Error while saving notes'),
         content, QMessageBox.Ok)
Example #24
0
 def _saveNotes_eb(self, backend, error, backtrace):
     self.ui.saveButton.setEnabled(True)
     self.ui.textEdit.setEnabled(True)
     content = unicode(self.tr('Unable to save notes:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
         QMessageBox.critical(self, self.tr('Error while saving notes'),
         content, QMessageBox.Ok)
Example #25
0
 def _errorSavePage(self, backend, error, backtrace):
     """ """
     content = unicode(self.tr('Unable to save page:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while saving page'),
                          content, QMessageBox.Ok)
     self.ui.saveButton.setEnabled(True)
     self.ui.saveButton.setText("Save")
Example #26
0
 def _errorLoadPage(self, backend, error, backtrace):
     """ Error callback for loadPage """
     content = unicode(self.tr('Unable to load page:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while loading page'),
                          content, QMessageBox.Ok)
     self.ui.loadButton.setEnabled(True)
     self.ui.loadButton.setText("Load")
Example #27
0
 def _errorLoadPage(self, backend, error, backtrace):
     """ Error callback for loadPage """
     content = self.tr('Unable to load page:\n%s\n') % to_unicode(error)
     if logging.root.level <= logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while loading page'),
                          content, QMessageBox.Ok)
     self.ui.loadButton.setEnabled(True)
     self.ui.loadButton.setText("Load")
Example #28
0
 def _errorSavePage(self, backend, error, backtrace):
     """ """
     content = self.tr('Unable to save page:\n%s\n') % to_unicode(error)
     if logging.root.level <= logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while saving page'),
                          content, QMessageBox.Ok)
     self.ui.saveButton.setEnabled(True)
     self.ui.saveButton.setText("Save")
Example #29
0
    def bcall_error_handler(self, backend, error, backtrace):
        """
        Handler for an exception inside the CallErrors exception.

        This method can be overrided to support more exceptions types.
        """
        if isinstance(error, BrowserIncorrectPassword):
            msg = unicode(error)
            if not msg:
                msg = 'invalid login/password.'
            # TODO ask to reconfigure backend
            print >> sys.stderr, 'Error(%s): %s' % (backend.name, msg)
            if self.ask('Do you want to reconfigure this backend?',
                        default=True):
                self.unload_backends(names=[backend.name])
                self.edit_backend(backend.name)
                self.load_backends(names=[backend.name])
        elif isinstance(error, BrowserUnavailable):
            msg = unicode(error)
            if not msg:
                msg = 'website is unavailable.'
            print >> sys.stderr, u'Error(%s): %s' % (backend.name, msg)
        elif isinstance(error, BrowserForbidden):
            print >> sys.stderr, u'Error(%s): %s' % (backend.name, msg
                                                     or 'Forbidden')
        elif isinstance(error, NotImplementedError):
            print >> sys.stderr, u'Error(%s): this feature is not supported yet by this backend.' % backend.name
            print >> sys.stderr, u'      %s   To help the maintainer of this backend implement this feature,' % (
                ' ' * len(backend.name))
            print >> sys.stderr, u'      %s   please contact: %s <%s>' % (
                ' ' * len(backend.name), backend.MAINTAINER, backend.EMAIL)
        elif isinstance(error, UserError):
            print >> sys.stderr, u'Error(%s): %s' % (backend.name,
                                                     to_unicode(error))
        elif isinstance(error, SSLError):
            print >> sys.stderr, u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC
        else:
            print >> sys.stderr, u'Bug(%s): %s' % (backend.name,
                                                   to_unicode(error))

            minfo = self.weboob.repositories.get_module_info(backend.NAME)
            if minfo and not minfo.is_local():
                self.weboob.repositories.update_repositories()

                # minfo of the new available module
                minfo = self.weboob.repositories.get_module_info(backend.NAME)
                if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \
                   self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \
                   self.install_module(minfo):
                    print 'New version of module %s has been installed. Retry to call the command.' % minfo.name
                    return

            if logging.root.level == logging.DEBUG:
                print >> sys.stderr, backtrace
            else:
                return True
Example #30
0
    def write_dict(self, item, fp):
        writer = csv.writer(fp, delimiter=self.field_separator)
        if not self.started:
            writer.writerow([to_unicode(v) for v in item.keys()])
            self.started = True

        if sys.version_info.major >= 3:
            writer.writerow([str(v) for v in item.values()])
        else:
            writer.writerow([to_unicode(v).encode('utf-8') for v in item.values()])
Example #31
0
 def get_profile(self):
     from weboob.tools.misc import to_unicode
     profile = Profile()
     if len([k for k in self.session.cookies.keys() if k == 'CTX']) > 1:
         del self.session.cookies['CTX']
     elif 'username='******'CTX', ''):
         profile.name = to_unicode(re.search('username=([^&]+)', self.session.cookies['CTX']).group(1))
     elif 'nomusager=' in self.session.cookies.get('headerdei'):
         profile.name = to_unicode(re.search('nomusager=(?:[^&]+/ )?([^&]+)', self.session.cookies['headerdei']).group(1))
     return profile
Example #32
0
 def _errorHistory(self, backend, error, backtrace):
     """ Loading the history has failed """
     content = unicode(self.tr('Unable to load history:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while loading history'),
                          content, QMessageBox.Ok)
     
     self.ui.loadHistoryButton.setEnabled(True)
     self.ui.loadHistoryButton.setText("Reload")
Example #33
0
    def parse(self):
        emonths = [
            'January', 'February', 'March', 'April', 'May', 'June', 'July',
            'August', 'September', 'October', 'November', 'December'
        ]
        date_format, time_format, months = self.guess_format()
        for row in self.document.xpath(
                '//table[@id="transactionTable"]/tbody/tr'):
            if len(row.xpath('.//td')) < 5:
                continue

            amount = row.xpath(
                './/td[@headers="gross"]')[-1].text_content().strip()
            if re.search('\d', amount):
                currency = Account.get_currency(amount)
                amount = AmTr.decimal_amount(amount)
            else:
                continue

            idtext = row.xpath('.//td[@class="detailsNoPrint"]//span[@class="accessAid"]')[0] \
                .text_content().replace(u'\xa0', u' ').strip().rpartition(' ')[-1]
            trans = Transaction(idtext)
            trans.amount = amount
            trans._currency = currency

            datetext = row.xpath(
                './/td[@class="dateInfo"]')[0].text_content().strip()
            for i in range(0, 12):
                datetext = datetext.replace(months[i], emonths[i])
            date = dateutil.parser.parse(datetext)
            trans.date = date
            trans.rdate = date

            trans.label = to_unicode(
                row.xpath('.//td[@class="emailInfo"]')
                [0].text_content().strip())
            info = to_unicode(
                row.xpath('.//td[@class="paymentTypeInfo"]')
                [0].text_content().strip())
            trans.raw = info + u' ' + trans.label

            if u'Authorization' in info or u'Autorisation' in info or \
               u'Order' in info:
                continue

            if u'Credit Card' in trans.label or u'Carte bancaire' in trans.label:
                trans.type = Transaction.TYPE_CARD
            elif info.startswith(u'Payment') or info.startswith(u'Paiement'):
                trans.type = Transaction.TYPE_ORDER
            elif u'Currency Conversion' in info or u'Conversion de devise' in info:
                trans.type = Transaction.TYPE_BANK
            else:
                trans.type = Transaction.TYPE_UNKNOWN

            yield trans
Example #34
0
 def _entry2video(self, entry):
     """
     Parse an entry returned by googleapi and return a Video object.
     """
     snippet = entry['snippet']
     video = YoutubeVideo(to_unicode(entry['id']['videoId']))
     video.title = to_unicode(snippet['title'].strip())
     # duration does not seem to be available with api
     video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url'])
     video.author = to_unicode(snippet['channelTitle'].strip())
     return video
Example #35
0
    def _getNotes_eb(self, backend, error, backtrace):
        if isinstance(error, NotImplementedError):
            return

        self.ui.textEdit.setEnabled(True)
        self.ui.saveButton.setEnabled(True)
        content = self.tr('Unable to load notes:\n%s\n') % to_unicode(error)
        if logging.root.level <= logging.DEBUG:
            content += '\n%s\n' % to_unicode(backtrace)
            QMessageBox.critical(self, self.tr('Error while loading notes'),
            content, QMessageBox.Ok)
Example #36
0
 def _entry2video(self, entry):
     """
     Parse an entry returned by googleapi and return a Video object.
     """
     snippet = entry['snippet']
     video = YoutubeVideo(to_unicode(entry['id']['videoId']))
     video.title = to_unicode(snippet['title'].strip())
     # duration does not seem to be available with api
     video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url'])
     video.author = to_unicode(snippet['channelTitle'].strip())
     return video
Example #37
0
    def _errorHistory(self, backend, error, backtrace):
        """ Loading the history has failed """
        content = unicode(
            self.tr('Unable to load history:\n%s\n')) % to_unicode(error)
        if logging.root.level == logging.DEBUG:
            content += '\n%s\n' % to_unicode(backtrace)
        QMessageBox.critical(self, self.tr('Error while loading history'),
                             content, QMessageBox.Ok)

        self.ui.loadHistoryButton.setEnabled(True)
        self.ui.loadHistoryButton.setText("Reload")
Example #38
0
    def get_current(self, radio):
        document = self.browser.location('http://rock.ouifm.fr/dynamic-menu.json')
        suffix = ''
        if radio != 'general':
            suffix = '_%s' % radio

        last = document['last%s' % suffix][0]

        artist = to_unicode(last.get('artiste%s' % suffix, '').strip())
        title = to_unicode(last.get('titre%s' % suffix, '').strip())
        return artist, title
Example #39
0
    def write_dict(self, item, fp):
        writer = csv.writer(fp, delimiter=self.field_separator)
        if not self.started:
            writer.writerow([to_unicode(v) for v in item.keys()])
            self.started = True

        if sys.version_info.major >= 3:
            writer.writerow([str(v) for v in item.values()])
        else:
            writer.writerow(
                [to_unicode(v).encode('utf-8') for v in item.values()])
Example #40
0
    def _getNotes_eb(self, backend, error, backtrace):
        if isinstance(error, NotImplementedError):
            return

        self.ui.textEdit.setEnabled(True)
        self.ui.saveButton.setEnabled(True)
        content = unicode(self.tr('Unable to load notes:\n%s\n')) % to_unicode(error)
        if logging.root.level == logging.DEBUG:
            content += '\n%s\n' % to_unicode(backtrace)
            QMessageBox.critical(self, self.tr('Error while loading notes'),
            content, QMessageBox.Ok)
Example #41
0
    def get_video(self, video=None):
        _id = to_unicode(self.group_dict['id'])
        if video is None:
            video = CappedVideo(_id)
            video.set_empty_fields(NotAvailable)

        title_tmp = self.parser.select(self.document.getroot(), 'title', 1)
        video.title = to_unicode(title_tmp.text.strip())

        # Videopages doesn't have duration information (only results pages)
        video.url = u'http://cdn.capped.tv/vhq/%s.mp4' % _id
        return video
Example #42
0
    def get_current(self, radio):
        document = self.browser.location(
            'http://rock.ouifm.fr/dynamic-menu.json')
        suffix = ''
        if radio != 'general':
            suffix = '_%s' % radio

        last = document['last%s' % suffix][0]

        artist = to_unicode(last.get('artiste%s' % suffix, '').strip())
        title = to_unicode(last.get('titre%s' % suffix, '').strip())
        return artist, title
Example #43
0
 def get_profile(self):
     from weboob.tools.misc import to_unicode
     profile = Profile()
     if 'username='******'CTX', ''):
         profile.name = to_unicode(
             re.search('username=([^&]+)',
                       self.session.cookies['CTX']).group(1))
     elif 'nomusager=' in self.session.cookies.get('headerdei'):
         profile.name = to_unicode(
             re.search('nomusager=(?:[^&]+/ )?([^&]+)',
                       self.session.cookies['headerdei']).group(1))
     return profile
Example #44
0
    def get_current(self, radio):
        document = self.browser.location("http://www.ouifm.fr/onair.json")
        rad = ""
        if radio == "general":
            rad = "rock"
        else:
            rad = radio

        last = document[rad][0]

        artist = to_unicode(last.get("artist", "").strip())
        title = to_unicode(last.get("title", "").strip())
        return artist, title
Example #45
0
 def _postReply_eb(self, backend, error, backtrace):
     content = self.tr('Unable to send message:\n%s\n') % to_unicode(error)
     if logging.root.level <= logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while posting reply'),
                          content, QMessageBox.Ok)
     self.ui.backendsList.setEnabled(True)
     self.ui.threadsList.setEnabled(True)
     self.ui.messagesTree.setEnabled(True)
     self.ui.replyButton.setEnabled(True)
     self.ui.replyWidget.setEnabled(True)
     self.ui.sendButton.setText(self.tr('Send'))
     self.process_reply = None
Example #46
0
 def _postReply_eb(self, backend, error, backtrace):
     content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error)
     if logging.root.level == logging.DEBUG:
         content += '\n%s\n' % to_unicode(backtrace)
     QMessageBox.critical(self, self.tr('Error while posting reply'),
                          content, QMessageBox.Ok)
     self.ui.backendsList.setEnabled(True)
     self.ui.threadsList.setEnabled(True)
     self.ui.messagesTree.setEnabled(True)
     self.ui.replyButton.setEnabled(True)
     self.ui.replyWidget.setEnabled(True)
     self.ui.sendButton.setText(self.tr('Send'))
     self.process_reply = None
Example #47
0
    def get_current(self, radio):
        document = self.browser.request('http://www.ouifm.fr/onair.json')
        rad = ''
        if radio == 'general':
            rad = 'rock'
        else:
            rad = radio

        last = document[rad][0]

        artist = to_unicode(last.get('artist', '').strip())
        title = to_unicode(last.get('title', '').strip())
        return artist, title
Example #48
0
    def get_current(self, radio):
        document = self.browser.request('http://www.ouifm.fr/onair.json')
        rad = ''
        if radio == 'general':
            rad = 'rock'
        else:
            rad = radio

        last = document[rad][0]

        artist = to_unicode(last.get('artist', '').strip())
        title = to_unicode(last.get('title', '').strip())
        return artist, title
Example #49
0
    def iter_videos(self):
        # When no results are found, the website returns random results
        sb = self.parser.select(self.document.getroot(),
                                'div.search form input.searchbox', 1)
        if sb.value == 'No Results Found':
            return

        #Extracting meta data from results page
        vidbackdrop_list = self.parser.select(self.document.getroot(),
                                              'div.vidBackdrop    ')
        for vidbackdrop in vidbackdrop_list:
            url = self.parser.select(vidbackdrop, 'a', 1).attrib['href']
            _id = url[2:]

            video = CappedVideo(_id)
            video.set_empty_fields(NotAvailable, ('url', ))

            video.title = to_unicode(
                self.parser.select(vidbackdrop, 'div.vidTitle a', 1).text)
            video.author = to_unicode(
                self.parser.select(vidbackdrop, 'div.vidAuthor a', 1).text)

            thumbnail_url = 'http://cdn.capped.tv/pre/%s.png' % _id
            video.thumbnail = Thumbnail(thumbnail_url)
            video.thumbnail.url = to_unicode(video.thumbnail.id)

            #we get the description field
            duration_tmp = self.parser.select(vidbackdrop, 'div.vidInfo', 1)
            #we remove tabs and spaces
            duration_tmp2 = duration_tmp.text[7:]
            #we remove all fields exept time
            duration_tmp3 = duration_tmp2.split(' ')[0]
            #we transform it in datetime format
            parts = duration_tmp3.split(':')
            if len(parts) == 1:
                hours = minutes = 0
                seconds = parts[0]
            elif len(parts) == 2:
                hours = 0
                minutes, seconds = parts
            elif len(parts) == 3:
                hours, minutes, seconds = parts
            else:
                raise BrokenPageError('Unable to parse duration %r' %
                                      duration_tmp)

            video.duration = datetime.timedelta(hours=int(hours),
                                                minutes=int(minutes),
                                                seconds=int(seconds))

            yield video
Example #50
0
    def _entry2video(self, entry):
        """
        Parse an entry returned by gdata and return a Video object.
        """
        video = YoutubeVideo(to_unicode(entry.id.text.split('/')[-1].strip()))
        video.title = to_unicode(entry.media.title.text.strip())
        video.duration = datetime.timedelta(seconds=int(entry.media.duration.seconds.strip()))
        video.thumbnail = Thumbnail(to_unicode(entry.media.thumbnail[0].url.strip()))

        if entry.author[0].name.text:
            video.author = to_unicode(entry.author[0].name.text.strip())
        if entry.media.name:
            video.author = to_unicode(entry.media.name.text.strip())
        return video
Example #51
0
 def get_list(self):
     table = self.parser.select(self.document.getroot(), "#tab-corps", 1)
     for tr in self.parser.select(table, "tr", "many"):
         tdname, tdid, tdagency, tdbalance = [td.text_content().strip() for td in self.parser.select(tr, "td", 4)]
         # it has empty rows - ignore those without the necessary info
         if all((tdname, tdid, tdbalance)):
             account = Account()
             account.label = to_unicode(tdname)
             account.id = to_unicode(tdid.replace(u"\xa0", "").replace(" ", ""))
             account._agency = to_unicode(tdagency)
             account._is_card = False
             account.balance = Decimal(Transaction.clean_amount(tdbalance))
             account.currency = account.get_currency(tdbalance)
             yield account
Example #52
0
    def bcall_error_handler(self, backend, error, backtrace):
        """
        Handler for an exception inside the CallErrors exception.

        This method can be overrided to support more exceptions types.
        """
        if isinstance(error, BrowserIncorrectPassword):
            msg = unicode(error)
            if not msg:
                msg = 'invalid login/password.'
            # TODO ask to reconfigure backend
            print >>sys.stderr, 'Error(%s): %s' % (backend.name, msg)
            if self.ask('Do you want to reconfigure this backend?', default=True):
                self.unload_backends(names=[backend.name])
                self.edit_backend(backend.name)
                self.load_backends(names=[backend.name])
        elif isinstance(error, BrowserUnavailable):
            msg = unicode(error)
            if not msg:
                msg = 'website is unavailable.'
            print >>sys.stderr, u'Error(%s): %s' % (backend.name, msg)
        elif isinstance(error, BrowserForbidden):
            print >>sys.stderr, u'Error(%s): %s' % (backend.name, msg or 'Forbidden')
        elif isinstance(error, NotImplementedError):
            print >>sys.stderr, u'Error(%s): this feature is not supported yet by this backend.' % backend.name
            print >>sys.stderr, u'      %s   To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name))
            print >>sys.stderr, u'      %s   please contact: %s <%s>' % (' ' * len(backend.name), backend.MAINTAINER, backend.EMAIL)
        elif isinstance(error, UserError):
            print >>sys.stderr, u'Error(%s): %s' % (backend.name, to_unicode(error))
        elif isinstance(error, SSLError):
            print >>sys.stderr, u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC
        else:
            print >>sys.stderr, u'Bug(%s): %s' % (backend.name, to_unicode(error))

            minfo = self.weboob.repositories.get_module_info(backend.NAME)
            if minfo and not minfo.is_local():
                self.weboob.repositories.update_repositories()

                # minfo of the new available module
                minfo = self.weboob.repositories.get_module_info(backend.NAME)
                if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \
                   self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \
                   self.install_module(minfo):
                    print 'New version of module %s has been installed. Retry to call the command.' % minfo.name
                    return

            if logging.root.level == logging.DEBUG:
                print >>sys.stderr, backtrace
            else:
                return True
Example #53
0
    def _entry2video(self, entry):
        """
        Parse an entry returned by gdata and return a Video object.
        """
        video = YoutubeVideo(to_unicode(entry.id.text.split("/")[-1].strip()))
        video.title = to_unicode(entry.media.title.text.strip())
        video.duration = datetime.timedelta(seconds=int(entry.media.duration.seconds.strip()))
        video.thumbnail = Thumbnail(to_unicode(entry.media.thumbnail[0].url.strip()))

        if entry.author[0].name.text:
            video.author = to_unicode(entry.author[0].name.text.strip())
        if entry.media.name:
            video.author = to_unicode(entry.media.name.text.strip())
        return video
Example #54
0
 def get_profile(self):
     if not self.is_cenet_website:
         from weboob.tools.misc import to_unicode
         profile = Profile()
         if 'username='******'CTX', ''):
             profile.name = to_unicode(
                 re.search('username=([^&]+)',
                           self.session.cookies['CTX']).group(1))
         elif 'nomusager=' in self.session.cookies.get('headerdei'):
             profile.name = to_unicode(
                 re.search('nomusager=(?:[^&]+/ )?([^&]+)',
                           self.session.cookies['headerdei']).group(1))
     else:
         profile = self.cenet_home.stay_or_go().get_profile()
     return profile
Example #55
0
def fixurl(url):
    url = to_unicode(url)

    # remove javascript crap
    url = url.replace('/#!/', '/')

    # parse it
    parsed = urlparse.urlsplit(url)

    # divide the netloc further
    userpass, at, hostport = parsed.netloc.rpartition('@')
    user, colon1, pass_ = userpass.partition(':')
    host, colon2, port = hostport.partition(':')

    # encode each component
    scheme = parsed.scheme.encode('utf8')
    user = urllib.quote(user.encode('utf8'))
    colon1 = colon1.encode('utf8')
    pass_ = urllib.quote(pass_.encode('utf8'))
    at = at.encode('utf8')
    host = host.encode('idna')
    colon2 = colon2.encode('utf8')
    port = port.encode('utf8')
    path = '/'.join(pce.encode('utf8') for pce in parsed.path.split('/'))
    # while valid, it is most likely an error
    path = path.replace('//', '/')
    query = parsed.query.encode('utf8')
    fragment = parsed.fragment.encode('utf8')

    # put it back together
    netloc = ''.join((user, colon1, pass_, at, host, colon2, port))
    return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
Example #56
0
    def onecmd(self, line):
        """
        This REPL method is overrided to catch some particular exceptions.
        """
        line = to_unicode(line)
        cmd, arg, ignored = self.parseline(line)

        # Set the right formatter for the command.
        try:
            formatter_name = self.commands_formatters[cmd]
        except KeyError:
            formatter_name = self.DEFAULT_FORMATTER
        self.set_formatter(formatter_name)

        try:
            try:
                return super(ReplApplication, self).onecmd(line)
            except CallErrors as e:
                self.bcall_errors_handler(e)
            except BackendNotGiven as e:
                print >>sys.stderr, 'Error: %s' % str(e)
            except NotEnoughArguments as e:
                print >>sys.stderr, 'Error: not enough arguments. %s' % str(e)
            except (KeyboardInterrupt, EOFError):
                # ^C during a command process doesn't exit application.
                print '\nAborted.'
        finally:
            self.flush()
Example #57
0
    def api_request(self, command, **kwargs):
        if 'data' in kwargs:
            data = to_unicode(kwargs.pop('data')).encode('utf-8', 'replace')
        else:
            data = None

        headers = {}
        if not command.startswith('applications'):
            today = local2utc(datetime.now()).strftime('%Y-%m-%d')
            token = sha256(self.username + self.APITOKEN + today).hexdigest()

            headers['Authorization'] = 'Basic %s' % (b64encode('%s:%s' % (self.username, self.password)))
            headers['X-Platform'] = 'android'
            headers['X-Client-Version'] = self.APIVERSION
            headers['X-AUM-Token'] = token

        url = self.buildurl(self.absurl('/api/%s' % command), **kwargs)
        if isinstance(url, unicode):
            url = url.encode('utf-8')
        req = self.request_class(url, data, headers)
        buf = self.openurl(req).read()

        try:
            r = json.loads(buf)
        except ValueError:
            raise ValueError(buf)

        return r
Example #58
0
 def is_empty(self):
     for td in self.parser.select(self.document.getroot(),
                                  'td.V11vertGras'):
         if u'Aucune opération enregistrée' in to_unicode(
                 td.text_content()):
             return True
     return False