def iter_station_departures(self, station_id, arrival_id=None): url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/index.php?gare=%s' % unicode( station_id) result = self.openurl(url.encode('utf-8')).read() result = result departure = '' for line in result.split('&'): if not '=' in line: raise BrokenPageError('Unable to parse result: %s' % line) key, value = line.split('=', 1) if key == 'nomgare': departure = value elif key.startswith('ligne'): _type, unknown, _time, arrival, served, late, late_reason = value.split( ';', 6) yield { 'type': to_unicode(_type), 'time': datetime.combine(date.today(), time(*[int(x) for x in _time.split(':')])), 'departure': to_unicode(departure), 'arrival': to_unicode(arrival).strip(), 'late': late and time(0, int(late.split()[0])) or time(), 'late_reason': to_unicode(late_reason).replace('\n', '').strip() }
def retrieveContact_eb(self, backend, error, backtrace): content = unicode( self.tr('Unable to get contact:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += u'\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while getting contact'), content, QMessageBox.Ok)
def get_video(self, _id): video = QuviVideo(_id) parser = LibQuvi() if not parser.load(): raise UserError('Make sure libquvi 0.4 is installed') try: info = parser.get_info(video.page_url) except QuviError as qerror: raise UserError(qerror.message) video.url = to_unicode(info.get('url')) if not video.url: raise NotImplementedError() video.ext = to_unicode(info.get('suffix')) video.title = to_unicode(info.get('title')) video.page = to_unicode(info.get('page')) duration = int(info.get('duration', 0)) if duration: video.duration = datetime.timedelta(milliseconds=duration) if info.get('thumbnail'): video.thumbnail = Thumbnail(info.get('thumbnail')) video.thumbnail.url = video.thumbnail.id return video
def get_video(self, _id): video = QuviVideo(_id) parser = LibQuvi() if not parser.load(): raise UserError('Make sure libquvi 0.4 is installed') try: info = parser.get_info(video.page_url) except QuviError as qerror: raise UserError(qerror.message) video.url = to_unicode(info.get('url')) if not video.url: raise NotImplementedError() video.ext = to_unicode(info.get('suffix')) video.title = to_unicode(info.get('title')) video.page = to_unicode(info.get('page')) duration = int(info.get('duration', 0)) if duration: video.duration = datetime.timedelta(milliseconds=duration) if info.get('thumbnail'): video.thumbnail = BaseImage(info.get('thumbnail')) video.thumbnail.url = video.thumbnail.id return video
def _postReply_eb(self, backend, error, backtrace): content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while posting reply'), content, QMessageBox.Ok) self.process_reply = None
def iter_routes(self): try: table = self.parser.select(self.document.getroot(), 'table.horaires3', 1) except BrokenPageError: raise StationNotFound('Station not found') departure = self.parser.select(table, 'td.caption strong', 1).text for tr in table.findall('tr'): if len(tr.findall('td')) != 4: continue code_mission = self.parser.select(tr, 'td[headers=Code_de_mission] a', 1).text.strip() time_s = self.parser.select(tr, 'td[headers=Heure_de_passage]', 1).text.strip().rstrip(u'\xa0*') destination = self.parser.select(tr, 'td[headers=Destination]', 1).text.strip() plateform = self.parser.select(tr, 'td[headers=Voie]', 1).text.strip() late_reason = None time = None try : time = datetime.datetime.combine(datetime.date.today(), datetime.time(*[int(x) for x in time_s.split(':')])) except ValueError: late_reason = time_s self.logger.warning('Unable to parse datetime "%s"' % time_s) yield {'type': to_unicode(code_mission), 'time': time, 'departure': to_unicode(departure), 'arrival': to_unicode(destination), 'late': datetime.time(), 'late_reason': late_reason, 'plateform': to_unicode(plateform)}
def get_video(self, video=None): _id = to_unicode(self.group_dict['id']) if video is None: video = YoujizzVideo(_id) title_el = self.parser.select(self.document.getroot(), 'title', 1) video.title = to_unicode(title_el.text.strip()) # youjizz HTML is crap, we must parse it with regexps data = lxml.html.tostring(self.document.getroot()) m = re.search(r'<strong>.*?Runtime.*?</strong> (.+?)</div>', data) if m: txt = m.group(1).strip() if txt == 'Unknown': video.duration = NotAvailable else: minutes, seconds = (int(v) for v in to_unicode(txt).split(':')) video.duration = datetime.timedelta(minutes=minutes, seconds=seconds) else: raise BrokenPageError('Unable to retrieve video duration') real_id = int(_id.split('-')[-1]) data = self.browser.readurl('http://www.youjizz.com/videos/embed/%s' % real_id) video_file_urls = re.findall(r'"(http://[^",]+\.youjizz\.com[^",]+\.flv(?:\?[^"]*)?)"', data) if len(video_file_urls) == 0: raise BrokenPageError('Video URL not found') elif len(video_file_urls) > 1: raise BrokenPageError('Many video file URL found') else: video.url = to_unicode(video_file_urls[0]) return video
def get_video(self, video=None): _id = to_unicode(self.group_dict["id"]) if video is None: video = YoujizzVideo(_id) title_el = select(self.document.getroot(), "title", 1) video.title = to_unicode(title_el.text.strip()) # youjizz HTML is crap, we must parse it with regexps data = lxml.html.tostring(self.document.getroot()) m = re.search(r"<strong>.*?Runtime.*?</strong> (.+?)<br.*>", data) try: if m: minutes, seconds = (int(v) for v in to_unicode(m.group(1).strip()).split(":")) video.duration = datetime.timedelta(minutes=minutes, seconds=seconds) else: raise Exception() except Exception: raise SelectElementException("Could not retrieve video duration") video_file_urls = re.findall(r'"(http://media[^ ,]+\.flv)"', data) if len(video_file_urls) == 0: raise SelectElementException("Video URL not found") elif len(video_file_urls) > 1: raise SelectElementException("Many video file URL found") else: video.url = video_file_urls[0] return video
def get_steps(self): errors = [] for p in self.parser.select(self.document.getroot(), 'p.errors'): if p.text: errors.append(p.text.strip()) if len(errors) > 0: raise RoadmapError('Unable to establish a roadmap: %s' % ', '.join(errors)) current_step = None i = 0 for tr in self.parser.select(self.document.getroot(), 'table.horaires2 tbody tr'): if not 'class' in tr.attrib: continue elif tr.attrib['class'] == 'trHautTroncon': current_step = {} current_step['id'] = i i += 1 current_step['start_time'] = self.parse_time(self.parser.select(tr, 'td.formattedHeureDepart p', 1).text.strip()) current_step['line'] = to_unicode(self.parser.select(tr, 'td.rechercheResultatColumnMode img')[-1].attrib['alt']) current_step['departure'] = to_unicode(self.parser.select(tr, 'td.descDepart p strong', 1).text.strip()) current_step['duration'] = self.parse_duration(self.parser.select(tr, 'td.rechercheResultatVertAlign', 1).text.strip()) elif tr.attrib['class'] == 'trBasTroncon': current_step['end_time'] = self.parse_time(self.parser.select(tr, 'td.formattedHeureArrivee p', 1).text.strip()) current_step['arrival'] = to_unicode(self.parser.select(tr, 'td.descArrivee p strong', 1).text.strip()) yield current_step
def get_video(self, video=None): _id = to_unicode(self.group_dict['id']) if video is None: video = YoujizzVideo(_id) title_el = self.parser.select(self.document.getroot(), 'title', 1) video.title = to_unicode(title_el.text.strip()) # youjizz HTML is crap, we must parse it with regexps data = lxml.html.tostring(self.document.getroot()) m = re.search(r'<strong>.*?Runtime.*?</strong> (.+?)</div>', data) if m: txt = m.group(1).strip() if txt == 'Unknown': video.duration = NotAvailable else: minutes, seconds = (int(v) for v in to_unicode(txt).split(':')) video.duration = datetime.timedelta(minutes=minutes, seconds=seconds) else: raise BrokenPageError('Unable to retrieve video duration') real_id = int(_id.split('-')[-1]) data = self.browser.readurl('http://www.youjizz.com/videos/embed/%s' % real_id) video_file_urls = re.findall( r'"(http://[^",]+\.youjizz\.com[^",]+\.flv(?:\?[^"]*)?)"', data) if len(video_file_urls) == 0: raise BrokenPageError('Video URL not found') elif len(video_file_urls) > 1: raise BrokenPageError('Many video file URL found') else: video.url = to_unicode(video_file_urls[0]) return video
def get_list(self): table = self.find_table() for tr in self.parser.select(table, 'tr', 'many'): tds = self.parser.select(tr, 'td') if len(tds) != 6: continue tdlabel, tdid, tdcur, tdupdated, tdbal, tdbalcur = tds account = Account() account.label = to_unicode(tdlabel.text_content().strip()) # this is important - and is also the last part of the id (considering spaces) # we can't use only the link as it does not goes where we want try: link = self.parser.select(tdlabel, 'a', 1) except BrokenPageError: # probably an account we can't display the history account._link_id = None else: account._link_id = parse_qs(link.attrib['href'])['ch4'][0] account.id = to_unicode(tdid.text.strip().replace(' ', '')) account.iban = 'FR76' + account.id # just in case we are showing the converted balances account._main_currency = Account.get_currency(tdcur.text) # we have to ignore those accounts, because using NotAvailable # makes boobank and probably many others crash if tdbal.text_content().strip() == 'indisponible': continue account.balance = Decimal( Transaction.clean_amount(tdbal.text_content())) account.currency = Account.get_currency(tdbalcur.text) account._updated = datetime.strptime(tdupdated.text, '%d/%m/%Y') yield account
def bcall_error_handler(self, backend, error, backtrace): if isinstance(error, TransferStep): params = {} for field in error.fields: v = self.ask(field) params[field.id] = v #backend.config['accept_transfer'].set(v) params['backends'] = backend self.start_format() for transfer in self.do('transfer', error.transfer, **params): self.format(transfer) elif isinstance(error, AddRecipientStep): params = {} params['backends'] = backend for field in error.fields: v = self.ask(field) params[field.id] = v try: next(iter(self.do('add_recipient', error.recipient, **params))) except CallErrors as e: self.bcall_errors_handler(e) elif isinstance(error, TransferInvalidAmount): print(u'Error(%s): %s' % (backend.name, to_unicode(error) or 'The transfer amount is invalid'), file=self.stderr) elif isinstance(error, TransferInvalidLabel): print(u'Error(%s): %s' % (backend.name, to_unicode(error) or 'The transfer label is invalid'), file=self.stderr) elif isinstance(error, TransferInvalidEmitter): print(u'Error(%s): %s' % (backend.name, to_unicode(error) or 'The transfer emitter is invalid'), file=self.stderr) elif isinstance(error, TransferInvalidRecipient): print(u'Error(%s): %s' % (backend.name, to_unicode(error) or 'The transfer recipient is invalid'), file=self.stderr) elif isinstance(error, TransferInvalidDate): print(u'Error(%s): %s' % (backend.name, to_unicode(error) or 'The transfer execution date is invalid'), file=self.stderr) elif isinstance(error, CaptchaQuestion): if not self.captcha_weboob.count_backends(): print( 'Error(%s): Site requires solving a CAPTCHA but no CapCaptchaSolver backends were configured' % backend.name, file=self.stderr) return False print( 'Info(%s): Encountered CAPTCHA, please wait for its resolution, it can take dozens of seconds' % backend.name, file=self.stderr) job = exception_to_job(error) self.solve_captcha(job, backend) return False else: return super(Boobank, self).bcall_error_handler(backend, error, backtrace)
def add_cookie(self, name, value): # httplib/cookielib don't seem to like unicode cookies... if sys.version_info.major < 3: name = to_unicode(name).encode('utf-8') value = to_unicode(value).encode('utf-8') self.browser.logger.debug('adding cookie %r=%r', name, value) self.browser.session.cookies.set(name, value, domain=urlsplit(self.url).hostname)
def get_list(self): table = self.find_table() for tr in self.parser.select(table, 'tr', 'many'): tds = self.parser.select(tr, 'td') if len(tds) != 6: continue tdlabel, tdid, tdcur, tdupdated, tdbal, tdbalcur = tds account = Account() account.label = to_unicode(tdlabel.text_content().strip()) # this is important - and is also the last part of the id (considering spaces) # we can't use only the link as it does not goes where we want try: link = self.parser.select(tdlabel, 'a', 1) except BrokenPageError: # probably an account we can't display the history account._link_id = None else: account._link_id = parse_qs(link.attrib['href'])['ch4'][0] account.id = to_unicode(tdid.text.strip().replace(' ', '')) account.iban = 'FR76' + account.id # just in case we are showing the converted balances account._main_currency = Account.get_currency(tdcur.text) # we have to ignore those accounts, because using NotAvailable # makes boobank and probably many others crash if tdbal.text_content().strip() == 'indisponible': continue account.balance = Decimal(Transaction.clean_amount(tdbal.text_content())) account.currency = Account.get_currency(tdbalcur.text) account._updated = datetime.strptime(tdupdated.text, '%d/%m/%Y') yield account
def get_stream_info(self, radio, url): stream = BaseAudioStream(0) current = StreamInfo(0) r = self.browser.open(url, stream=True, headers={'Icy-Metadata':'1'}) stream.bitrate = int(r.headers['icy-br'].split(',')[0]) r.raw.read(int(r.headers['icy-metaint'])) size = ord(r.raw.read(1)) content = r.raw.read(size*16) r.close() for s in content.split("\x00")[0].split(";"): a = s.split("=") if a[0] == "StreamTitle": stream.title = to_unicode(a[1].split("'")[1]) res = stream.title.split(" - ") current.who = to_unicode(res[0]) if(len(res) == 1): current.what = "" else: current.what = to_unicode(res[1]) stream.format=u'mp3' stream.url = url return [stream], current
def add_cookie(self, name, value): # httplib/cookielib don't seem to like unicode cookies... if sys.version_info.major < 3: name = to_unicode(name).encode('utf-8') value = to_unicode(value).encode('utf-8') self.browser.logger.debug('adding cookie %r=%r', name, value) self.browser.session.cookies.set(name, value)
def get_stream_info(self, radio, url): stream = BaseAudioStream(0) current = StreamInfo(0) r = self.browser.open(url, stream=True, headers={'Icy-Metadata': 1}) stream.bitrate = int(r.headers['icy-br'].split(',')[0]) r.raw.read(int(r.headers['icy-metaint'])) size = ord(r.raw.read(1)) content = r.raw.read(size * 16) r.close() for s in content.split("\x00")[0].split(";"): a = s.split("=") if a[0] == "StreamTitle": stream.title = to_unicode(a[1].split("'")[1]) res = stream.title.split(" - ") current.who = to_unicode(res[0]) if (len(res) == 1): current.what = "" else: current.what = to_unicode(res[1]) stream.format = u'mp3' stream.url = url return [stream], current
def bcall_error_handler(self, backend, error, backtrace): """ Handler for an exception inside the CallErrors exception. This method can be overrided to support more exceptions types. """ if isinstance(error, BrowserQuestion): for field in error.fields: v = self.ask(field) if v: backend.config[field.id].set(v) elif isinstance(error, BrowserIncorrectPassword): msg = unicode(error) if not msg: msg = 'invalid login/password.' print('Error(%s): %s' % (backend.name, msg), file=self.stderr) if self.ask('Do you want to reconfigure this backend?', default=True): self.unload_backends(names=[backend.name]) self.edit_backend(backend.name) self.load_backends(names=[backend.name]) elif isinstance(error, BrowserSSLError): print(u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC, file=self.stderr) elif isinstance(error, BrowserForbidden): msg = unicode(error) if not msg: msg = 'access denied' print(u'Error(%s): %s' % (backend.name, msg or 'Forbidden'), file=self.stderr) elif isinstance(error, BrowserUnavailable): msg = unicode(error) if not msg: msg = 'website is unavailable.' print(u'Error(%s): %s' % (backend.name, msg), file=self.stderr) elif isinstance(error, NotImplementedError): print(u'Error(%s): this feature is not supported yet by this backend.' % backend.name, file=self.stderr) print(u' %s To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name)), file=self.stderr) print(u' %s please contact: %s <*****@*****.**>' % (' ' * len(backend.name), backend.MAINTAINER, backend.NAME), file=self.stderr) elif isinstance(error, UserError): print(u'Error(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr) elif isinstance(error, MoreResultsAvailable): print(u'Hint: There are more results for backend %s' % (backend.name), file=self.stderr) else: print(u'Bug(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr) minfo = self.weboob.repositories.get_module_info(backend.NAME) if minfo and not minfo.is_local(): self.weboob.repositories.update_repositories(ConsoleProgress(self)) # minfo of the new available module minfo = self.weboob.repositories.get_module_info(backend.NAME) if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \ self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \ self.install_module(minfo): print('New version of module %s has been installed. Retry to call the command.' % minfo.name) return if logging.root.level <= logging.DEBUG: print(backtrace, file=self.stderr) else: return True
def load(self, items): self.version = int(items['version']) self.capabilities = items['capabilities'].split() self.description = to_unicode(items['description']) self.maintainer = to_unicode(items['maintainer']) self.license = to_unicode(items['license']) self.icon = items['icon'].strip() or None self.urls = items['urls']
def load(self, items): self.version = int(items["version"]) self.capabilities = items["capabilities"].split() self.description = to_unicode(items["description"]) self.maintainer = to_unicode(items["maintainer"]) self.license = to_unicode(items["license"]) self.icon = items["icon"].strip() or None self.urls = items["urls"]
def _saveNotes_eb(self, backend, error, backtrace): self.ui.saveButton.setEnabled(True) self.ui.textEdit.setEnabled(True) content = unicode(self.tr('Unable to save notes:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while saving notes'), content, QMessageBox.Ok)
def _errorSavePage(self, backend, error, backtrace): """ """ content = unicode(self.tr('Unable to save page:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while saving page'), content, QMessageBox.Ok) self.ui.saveButton.setEnabled(True) self.ui.saveButton.setText("Save")
def _errorLoadPage(self, backend, error, backtrace): """ Error callback for loadPage """ content = unicode(self.tr('Unable to load page:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while loading page'), content, QMessageBox.Ok) self.ui.loadButton.setEnabled(True) self.ui.loadButton.setText("Load")
def _errorLoadPage(self, backend, error, backtrace): """ Error callback for loadPage """ content = self.tr('Unable to load page:\n%s\n') % to_unicode(error) if logging.root.level <= logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while loading page'), content, QMessageBox.Ok) self.ui.loadButton.setEnabled(True) self.ui.loadButton.setText("Load")
def _errorSavePage(self, backend, error, backtrace): """ """ content = self.tr('Unable to save page:\n%s\n') % to_unicode(error) if logging.root.level <= logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while saving page'), content, QMessageBox.Ok) self.ui.saveButton.setEnabled(True) self.ui.saveButton.setText("Save")
def bcall_error_handler(self, backend, error, backtrace): """ Handler for an exception inside the CallErrors exception. This method can be overrided to support more exceptions types. """ if isinstance(error, BrowserIncorrectPassword): msg = unicode(error) if not msg: msg = 'invalid login/password.' # TODO ask to reconfigure backend print >> sys.stderr, 'Error(%s): %s' % (backend.name, msg) if self.ask('Do you want to reconfigure this backend?', default=True): self.unload_backends(names=[backend.name]) self.edit_backend(backend.name) self.load_backends(names=[backend.name]) elif isinstance(error, BrowserUnavailable): msg = unicode(error) if not msg: msg = 'website is unavailable.' print >> sys.stderr, u'Error(%s): %s' % (backend.name, msg) elif isinstance(error, BrowserForbidden): print >> sys.stderr, u'Error(%s): %s' % (backend.name, msg or 'Forbidden') elif isinstance(error, NotImplementedError): print >> sys.stderr, u'Error(%s): this feature is not supported yet by this backend.' % backend.name print >> sys.stderr, u' %s To help the maintainer of this backend implement this feature,' % ( ' ' * len(backend.name)) print >> sys.stderr, u' %s please contact: %s <%s>' % ( ' ' * len(backend.name), backend.MAINTAINER, backend.EMAIL) elif isinstance(error, UserError): print >> sys.stderr, u'Error(%s): %s' % (backend.name, to_unicode(error)) elif isinstance(error, SSLError): print >> sys.stderr, u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC else: print >> sys.stderr, u'Bug(%s): %s' % (backend.name, to_unicode(error)) minfo = self.weboob.repositories.get_module_info(backend.NAME) if minfo and not minfo.is_local(): self.weboob.repositories.update_repositories() # minfo of the new available module minfo = self.weboob.repositories.get_module_info(backend.NAME) if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \ self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \ self.install_module(minfo): print 'New version of module %s has been installed. Retry to call the command.' % minfo.name return if logging.root.level == logging.DEBUG: print >> sys.stderr, backtrace else: return True
def write_dict(self, item, fp): writer = csv.writer(fp, delimiter=self.field_separator) if not self.started: writer.writerow([to_unicode(v) for v in item.keys()]) self.started = True if sys.version_info.major >= 3: writer.writerow([str(v) for v in item.values()]) else: writer.writerow([to_unicode(v).encode('utf-8') for v in item.values()])
def get_profile(self): from weboob.tools.misc import to_unicode profile = Profile() if len([k for k in self.session.cookies.keys() if k == 'CTX']) > 1: del self.session.cookies['CTX'] elif 'username='******'CTX', ''): profile.name = to_unicode(re.search('username=([^&]+)', self.session.cookies['CTX']).group(1)) elif 'nomusager=' in self.session.cookies.get('headerdei'): profile.name = to_unicode(re.search('nomusager=(?:[^&]+/ )?([^&]+)', self.session.cookies['headerdei']).group(1)) return profile
def _errorHistory(self, backend, error, backtrace): """ Loading the history has failed """ content = unicode(self.tr('Unable to load history:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while loading history'), content, QMessageBox.Ok) self.ui.loadHistoryButton.setEnabled(True) self.ui.loadHistoryButton.setText("Reload")
def parse(self): emonths = [ 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December' ] date_format, time_format, months = self.guess_format() for row in self.document.xpath( '//table[@id="transactionTable"]/tbody/tr'): if len(row.xpath('.//td')) < 5: continue amount = row.xpath( './/td[@headers="gross"]')[-1].text_content().strip() if re.search('\d', amount): currency = Account.get_currency(amount) amount = AmTr.decimal_amount(amount) else: continue idtext = row.xpath('.//td[@class="detailsNoPrint"]//span[@class="accessAid"]')[0] \ .text_content().replace(u'\xa0', u' ').strip().rpartition(' ')[-1] trans = Transaction(idtext) trans.amount = amount trans._currency = currency datetext = row.xpath( './/td[@class="dateInfo"]')[0].text_content().strip() for i in range(0, 12): datetext = datetext.replace(months[i], emonths[i]) date = dateutil.parser.parse(datetext) trans.date = date trans.rdate = date trans.label = to_unicode( row.xpath('.//td[@class="emailInfo"]') [0].text_content().strip()) info = to_unicode( row.xpath('.//td[@class="paymentTypeInfo"]') [0].text_content().strip()) trans.raw = info + u' ' + trans.label if u'Authorization' in info or u'Autorisation' in info or \ u'Order' in info: continue if u'Credit Card' in trans.label or u'Carte bancaire' in trans.label: trans.type = Transaction.TYPE_CARD elif info.startswith(u'Payment') or info.startswith(u'Paiement'): trans.type = Transaction.TYPE_ORDER elif u'Currency Conversion' in info or u'Conversion de devise' in info: trans.type = Transaction.TYPE_BANK else: trans.type = Transaction.TYPE_UNKNOWN yield trans
def _entry2video(self, entry): """ Parse an entry returned by googleapi and return a Video object. """ snippet = entry['snippet'] video = YoutubeVideo(to_unicode(entry['id']['videoId'])) video.title = to_unicode(snippet['title'].strip()) # duration does not seem to be available with api video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url']) video.author = to_unicode(snippet['channelTitle'].strip()) return video
def _getNotes_eb(self, backend, error, backtrace): if isinstance(error, NotImplementedError): return self.ui.textEdit.setEnabled(True) self.ui.saveButton.setEnabled(True) content = self.tr('Unable to load notes:\n%s\n') % to_unicode(error) if logging.root.level <= logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while loading notes'), content, QMessageBox.Ok)
def _errorHistory(self, backend, error, backtrace): """ Loading the history has failed """ content = unicode( self.tr('Unable to load history:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while loading history'), content, QMessageBox.Ok) self.ui.loadHistoryButton.setEnabled(True) self.ui.loadHistoryButton.setText("Reload")
def get_current(self, radio): document = self.browser.location('http://rock.ouifm.fr/dynamic-menu.json') suffix = '' if radio != 'general': suffix = '_%s' % radio last = document['last%s' % suffix][0] artist = to_unicode(last.get('artiste%s' % suffix, '').strip()) title = to_unicode(last.get('titre%s' % suffix, '').strip()) return artist, title
def write_dict(self, item, fp): writer = csv.writer(fp, delimiter=self.field_separator) if not self.started: writer.writerow([to_unicode(v) for v in item.keys()]) self.started = True if sys.version_info.major >= 3: writer.writerow([str(v) for v in item.values()]) else: writer.writerow( [to_unicode(v).encode('utf-8') for v in item.values()])
def _getNotes_eb(self, backend, error, backtrace): if isinstance(error, NotImplementedError): return self.ui.textEdit.setEnabled(True) self.ui.saveButton.setEnabled(True) content = unicode(self.tr('Unable to load notes:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while loading notes'), content, QMessageBox.Ok)
def get_video(self, video=None): _id = to_unicode(self.group_dict['id']) if video is None: video = CappedVideo(_id) video.set_empty_fields(NotAvailable) title_tmp = self.parser.select(self.document.getroot(), 'title', 1) video.title = to_unicode(title_tmp.text.strip()) # Videopages doesn't have duration information (only results pages) video.url = u'http://cdn.capped.tv/vhq/%s.mp4' % _id return video
def get_current(self, radio): document = self.browser.location( 'http://rock.ouifm.fr/dynamic-menu.json') suffix = '' if radio != 'general': suffix = '_%s' % radio last = document['last%s' % suffix][0] artist = to_unicode(last.get('artiste%s' % suffix, '').strip()) title = to_unicode(last.get('titre%s' % suffix, '').strip()) return artist, title
def get_profile(self): from weboob.tools.misc import to_unicode profile = Profile() if 'username='******'CTX', ''): profile.name = to_unicode( re.search('username=([^&]+)', self.session.cookies['CTX']).group(1)) elif 'nomusager=' in self.session.cookies.get('headerdei'): profile.name = to_unicode( re.search('nomusager=(?:[^&]+/ )?([^&]+)', self.session.cookies['headerdei']).group(1)) return profile
def get_current(self, radio): document = self.browser.location("http://www.ouifm.fr/onair.json") rad = "" if radio == "general": rad = "rock" else: rad = radio last = document[rad][0] artist = to_unicode(last.get("artist", "").strip()) title = to_unicode(last.get("title", "").strip()) return artist, title
def _postReply_eb(self, backend, error, backtrace): content = self.tr('Unable to send message:\n%s\n') % to_unicode(error) if logging.root.level <= logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while posting reply'), content, QMessageBox.Ok) self.ui.backendsList.setEnabled(True) self.ui.threadsList.setEnabled(True) self.ui.messagesTree.setEnabled(True) self.ui.replyButton.setEnabled(True) self.ui.replyWidget.setEnabled(True) self.ui.sendButton.setText(self.tr('Send')) self.process_reply = None
def _postReply_eb(self, backend, error, backtrace): content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error) if logging.root.level == logging.DEBUG: content += '\n%s\n' % to_unicode(backtrace) QMessageBox.critical(self, self.tr('Error while posting reply'), content, QMessageBox.Ok) self.ui.backendsList.setEnabled(True) self.ui.threadsList.setEnabled(True) self.ui.messagesTree.setEnabled(True) self.ui.replyButton.setEnabled(True) self.ui.replyWidget.setEnabled(True) self.ui.sendButton.setText(self.tr('Send')) self.process_reply = None
def get_current(self, radio): document = self.browser.request('http://www.ouifm.fr/onair.json') rad = '' if radio == 'general': rad = 'rock' else: rad = radio last = document[rad][0] artist = to_unicode(last.get('artist', '').strip()) title = to_unicode(last.get('title', '').strip()) return artist, title
def iter_videos(self): # When no results are found, the website returns random results sb = self.parser.select(self.document.getroot(), 'div.search form input.searchbox', 1) if sb.value == 'No Results Found': return #Extracting meta data from results page vidbackdrop_list = self.parser.select(self.document.getroot(), 'div.vidBackdrop ') for vidbackdrop in vidbackdrop_list: url = self.parser.select(vidbackdrop, 'a', 1).attrib['href'] _id = url[2:] video = CappedVideo(_id) video.set_empty_fields(NotAvailable, ('url', )) video.title = to_unicode( self.parser.select(vidbackdrop, 'div.vidTitle a', 1).text) video.author = to_unicode( self.parser.select(vidbackdrop, 'div.vidAuthor a', 1).text) thumbnail_url = 'http://cdn.capped.tv/pre/%s.png' % _id video.thumbnail = Thumbnail(thumbnail_url) video.thumbnail.url = to_unicode(video.thumbnail.id) #we get the description field duration_tmp = self.parser.select(vidbackdrop, 'div.vidInfo', 1) #we remove tabs and spaces duration_tmp2 = duration_tmp.text[7:] #we remove all fields exept time duration_tmp3 = duration_tmp2.split(' ')[0] #we transform it in datetime format parts = duration_tmp3.split(':') if len(parts) == 1: hours = minutes = 0 seconds = parts[0] elif len(parts) == 2: hours = 0 minutes, seconds = parts elif len(parts) == 3: hours, minutes, seconds = parts else: raise BrokenPageError('Unable to parse duration %r' % duration_tmp) video.duration = datetime.timedelta(hours=int(hours), minutes=int(minutes), seconds=int(seconds)) yield video
def _entry2video(self, entry): """ Parse an entry returned by gdata and return a Video object. """ video = YoutubeVideo(to_unicode(entry.id.text.split('/')[-1].strip())) video.title = to_unicode(entry.media.title.text.strip()) video.duration = datetime.timedelta(seconds=int(entry.media.duration.seconds.strip())) video.thumbnail = Thumbnail(to_unicode(entry.media.thumbnail[0].url.strip())) if entry.author[0].name.text: video.author = to_unicode(entry.author[0].name.text.strip()) if entry.media.name: video.author = to_unicode(entry.media.name.text.strip()) return video
def get_list(self): table = self.parser.select(self.document.getroot(), "#tab-corps", 1) for tr in self.parser.select(table, "tr", "many"): tdname, tdid, tdagency, tdbalance = [td.text_content().strip() for td in self.parser.select(tr, "td", 4)] # it has empty rows - ignore those without the necessary info if all((tdname, tdid, tdbalance)): account = Account() account.label = to_unicode(tdname) account.id = to_unicode(tdid.replace(u"\xa0", "").replace(" ", "")) account._agency = to_unicode(tdagency) account._is_card = False account.balance = Decimal(Transaction.clean_amount(tdbalance)) account.currency = account.get_currency(tdbalance) yield account
def bcall_error_handler(self, backend, error, backtrace): """ Handler for an exception inside the CallErrors exception. This method can be overrided to support more exceptions types. """ if isinstance(error, BrowserIncorrectPassword): msg = unicode(error) if not msg: msg = 'invalid login/password.' # TODO ask to reconfigure backend print >>sys.stderr, 'Error(%s): %s' % (backend.name, msg) if self.ask('Do you want to reconfigure this backend?', default=True): self.unload_backends(names=[backend.name]) self.edit_backend(backend.name) self.load_backends(names=[backend.name]) elif isinstance(error, BrowserUnavailable): msg = unicode(error) if not msg: msg = 'website is unavailable.' print >>sys.stderr, u'Error(%s): %s' % (backend.name, msg) elif isinstance(error, BrowserForbidden): print >>sys.stderr, u'Error(%s): %s' % (backend.name, msg or 'Forbidden') elif isinstance(error, NotImplementedError): print >>sys.stderr, u'Error(%s): this feature is not supported yet by this backend.' % backend.name print >>sys.stderr, u' %s To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name)) print >>sys.stderr, u' %s please contact: %s <%s>' % (' ' * len(backend.name), backend.MAINTAINER, backend.EMAIL) elif isinstance(error, UserError): print >>sys.stderr, u'Error(%s): %s' % (backend.name, to_unicode(error)) elif isinstance(error, SSLError): print >>sys.stderr, u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC else: print >>sys.stderr, u'Bug(%s): %s' % (backend.name, to_unicode(error)) minfo = self.weboob.repositories.get_module_info(backend.NAME) if minfo and not minfo.is_local(): self.weboob.repositories.update_repositories() # minfo of the new available module minfo = self.weboob.repositories.get_module_info(backend.NAME) if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \ self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \ self.install_module(minfo): print 'New version of module %s has been installed. Retry to call the command.' % minfo.name return if logging.root.level == logging.DEBUG: print >>sys.stderr, backtrace else: return True
def _entry2video(self, entry): """ Parse an entry returned by gdata and return a Video object. """ video = YoutubeVideo(to_unicode(entry.id.text.split("/")[-1].strip())) video.title = to_unicode(entry.media.title.text.strip()) video.duration = datetime.timedelta(seconds=int(entry.media.duration.seconds.strip())) video.thumbnail = Thumbnail(to_unicode(entry.media.thumbnail[0].url.strip())) if entry.author[0].name.text: video.author = to_unicode(entry.author[0].name.text.strip()) if entry.media.name: video.author = to_unicode(entry.media.name.text.strip()) return video
def get_profile(self): if not self.is_cenet_website: from weboob.tools.misc import to_unicode profile = Profile() if 'username='******'CTX', ''): profile.name = to_unicode( re.search('username=([^&]+)', self.session.cookies['CTX']).group(1)) elif 'nomusager=' in self.session.cookies.get('headerdei'): profile.name = to_unicode( re.search('nomusager=(?:[^&]+/ )?([^&]+)', self.session.cookies['headerdei']).group(1)) else: profile = self.cenet_home.stay_or_go().get_profile() return profile
def fixurl(url): url = to_unicode(url) # remove javascript crap url = url.replace('/#!/', '/') # parse it parsed = urlparse.urlsplit(url) # divide the netloc further userpass, at, hostport = parsed.netloc.rpartition('@') user, colon1, pass_ = userpass.partition(':') host, colon2, port = hostport.partition(':') # encode each component scheme = parsed.scheme.encode('utf8') user = urllib.quote(user.encode('utf8')) colon1 = colon1.encode('utf8') pass_ = urllib.quote(pass_.encode('utf8')) at = at.encode('utf8') host = host.encode('idna') colon2 = colon2.encode('utf8') port = port.encode('utf8') path = '/'.join(pce.encode('utf8') for pce in parsed.path.split('/')) # while valid, it is most likely an error path = path.replace('//', '/') query = parsed.query.encode('utf8') fragment = parsed.fragment.encode('utf8') # put it back together netloc = ''.join((user, colon1, pass_, at, host, colon2, port)) return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def onecmd(self, line): """ This REPL method is overrided to catch some particular exceptions. """ line = to_unicode(line) cmd, arg, ignored = self.parseline(line) # Set the right formatter for the command. try: formatter_name = self.commands_formatters[cmd] except KeyError: formatter_name = self.DEFAULT_FORMATTER self.set_formatter(formatter_name) try: try: return super(ReplApplication, self).onecmd(line) except CallErrors as e: self.bcall_errors_handler(e) except BackendNotGiven as e: print >>sys.stderr, 'Error: %s' % str(e) except NotEnoughArguments as e: print >>sys.stderr, 'Error: not enough arguments. %s' % str(e) except (KeyboardInterrupt, EOFError): # ^C during a command process doesn't exit application. print '\nAborted.' finally: self.flush()
def api_request(self, command, **kwargs): if 'data' in kwargs: data = to_unicode(kwargs.pop('data')).encode('utf-8', 'replace') else: data = None headers = {} if not command.startswith('applications'): today = local2utc(datetime.now()).strftime('%Y-%m-%d') token = sha256(self.username + self.APITOKEN + today).hexdigest() headers['Authorization'] = 'Basic %s' % (b64encode('%s:%s' % (self.username, self.password))) headers['X-Platform'] = 'android' headers['X-Client-Version'] = self.APIVERSION headers['X-AUM-Token'] = token url = self.buildurl(self.absurl('/api/%s' % command), **kwargs) if isinstance(url, unicode): url = url.encode('utf-8') req = self.request_class(url, data, headers) buf = self.openurl(req).read() try: r = json.loads(buf) except ValueError: raise ValueError(buf) return r
def is_empty(self): for td in self.parser.select(self.document.getroot(), 'td.V11vertGras'): if u'Aucune opération enregistrée' in to_unicode( td.text_content()): return True return False