def iter_station_departures(self, station_id, arrival_id=None):
url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/index.php?gare=%s' % unicode(
station_id)
result = self.openurl(url.encode('utf-8')).read()
result = result
departure = ''
for line in result.split('&'):
if not '=' in line:
raise BrokenPageError('Unable to parse result: %s' % line)
key, value = line.split('=', 1)
if key == 'nomgare':
departure = value
elif key.startswith('ligne'):
_type, unknown, _time, arrival, served, late, late_reason = value.split(
';', 6)
yield {
'type':
to_unicode(_type),
'time':
datetime.combine(date.today(),
time(*[int(x)
for x in _time.split(':')])),
'departure':
to_unicode(departure),
'arrival':
to_unicode(arrival).strip(),
'late':
late and time(0, int(late.split()[0])) or time(),
'late_reason':
to_unicode(late_reason).replace('\n', '').strip()
}
def retrieveContact_eb(self, backend, error, backtrace):
content = unicode(
self.tr('Unable to get contact:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += u'\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while getting contact'),
content, QMessageBox.Ok)
def get_video(self, _id):
video = QuviVideo(_id)
parser = LibQuvi()
if not parser.load():
raise UserError('Make sure libquvi 0.4 is installed')
try:
info = parser.get_info(video.page_url)
except QuviError as qerror:
raise UserError(qerror.message)
video.url = to_unicode(info.get('url'))
if not video.url:
raise NotImplementedError()
video.ext = to_unicode(info.get('suffix'))
video.title = to_unicode(info.get('title'))
video.page = to_unicode(info.get('page'))
duration = int(info.get('duration', 0))
if duration:
video.duration = datetime.timedelta(milliseconds=duration)
if info.get('thumbnail'):
video.thumbnail = Thumbnail(info.get('thumbnail'))
video.thumbnail.url = video.thumbnail.id
return video
def get_video(self, _id):
video = QuviVideo(_id)
parser = LibQuvi()
if not parser.load():
raise UserError('Make sure libquvi 0.4 is installed')
try:
info = parser.get_info(video.page_url)
except QuviError as qerror:
raise UserError(qerror.message)
video.url = to_unicode(info.get('url'))
if not video.url:
raise NotImplementedError()
video.ext = to_unicode(info.get('suffix'))
video.title = to_unicode(info.get('title'))
video.page = to_unicode(info.get('page'))
duration = int(info.get('duration', 0))
if duration:
video.duration = datetime.timedelta(milliseconds=duration)
if info.get('thumbnail'):
video.thumbnail = BaseImage(info.get('thumbnail'))
video.thumbnail.url = video.thumbnail.id
return video
def _postReply_eb(self, backend, error, backtrace):
content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while posting reply'),
content, QMessageBox.Ok)
self.process_reply = None
def iter_routes(self):
try:
table = self.parser.select(self.document.getroot(), 'table.horaires3', 1)
except BrokenPageError:
raise StationNotFound('Station not found')
departure = self.parser.select(table, 'td.caption strong', 1).text
for tr in table.findall('tr'):
if len(tr.findall('td')) != 4:
continue
code_mission = self.parser.select(tr, 'td[headers=Code_de_mission] a', 1).text.strip()
time_s = self.parser.select(tr, 'td[headers=Heure_de_passage]', 1).text.strip().rstrip(u'\xa0*')
destination = self.parser.select(tr, 'td[headers=Destination]', 1).text.strip()
plateform = self.parser.select(tr, 'td[headers=Voie]', 1).text.strip()
late_reason = None
time = None
try :
time = datetime.datetime.combine(datetime.date.today(), datetime.time(*[int(x) for x in time_s.split(':')]))
except ValueError:
late_reason = time_s
self.logger.warning('Unable to parse datetime "%s"' % time_s)
yield {'type': to_unicode(code_mission),
'time': time,
'departure': to_unicode(departure),
'arrival': to_unicode(destination),
'late': datetime.time(),
'late_reason': late_reason,
'plateform': to_unicode(plateform)}
def get_video(self, video=None):
_id = to_unicode(self.group_dict['id'])
if video is None:
video = YoujizzVideo(_id)
title_el = self.parser.select(self.document.getroot(), 'title', 1)
video.title = to_unicode(title_el.text.strip())
# youjizz HTML is crap, we must parse it with regexps
data = lxml.html.tostring(self.document.getroot())
m = re.search(r'<strong>.*?Runtime.*?</strong> (.+?)</div>', data)
if m:
txt = m.group(1).strip()
if txt == 'Unknown':
video.duration = NotAvailable
else:
minutes, seconds = (int(v) for v in to_unicode(txt).split(':'))
video.duration = datetime.timedelta(minutes=minutes, seconds=seconds)
else:
raise BrokenPageError('Unable to retrieve video duration')
real_id = int(_id.split('-')[-1])
data = self.browser.readurl('http://www.youjizz.com/videos/embed/%s' % real_id)
video_file_urls = re.findall(r'"(http://[^",]+\.youjizz\.com[^",]+\.flv(?:\?[^"]*)?)"', data)
if len(video_file_urls) == 0:
raise BrokenPageError('Video URL not found')
elif len(video_file_urls) > 1:
raise BrokenPageError('Many video file URL found')
else:
video.url = to_unicode(video_file_urls[0])
return video
def get_video(self, video=None):
_id = to_unicode(self.group_dict["id"])
if video is None:
video = YoujizzVideo(_id)
title_el = select(self.document.getroot(), "title", 1)
video.title = to_unicode(title_el.text.strip())
# youjizz HTML is crap, we must parse it with regexps
data = lxml.html.tostring(self.document.getroot())
m = re.search(r"<strong>.*?Runtime.*?</strong> (.+?)<br.*>", data)
try:
if m:
minutes, seconds = (int(v) for v in to_unicode(m.group(1).strip()).split(":"))
video.duration = datetime.timedelta(minutes=minutes, seconds=seconds)
else:
raise Exception()
except Exception:
raise SelectElementException("Could not retrieve video duration")
video_file_urls = re.findall(r'"(http://media[^ ,]+\.flv)"', data)
if len(video_file_urls) == 0:
raise SelectElementException("Video URL not found")
elif len(video_file_urls) > 1:
raise SelectElementException("Many video file URL found")
else:
video.url = video_file_urls[0]
return video
def get_steps(self):
errors = []
for p in self.parser.select(self.document.getroot(), 'p.errors'):
if p.text:
errors.append(p.text.strip())
if len(errors) > 0:
raise RoadmapError('Unable to establish a roadmap: %s' % ', '.join(errors))
current_step = None
i = 0
for tr in self.parser.select(self.document.getroot(), 'table.horaires2 tbody tr'):
if not 'class' in tr.attrib:
continue
elif tr.attrib['class'] == 'trHautTroncon':
current_step = {}
current_step['id'] = i
i += 1
current_step['start_time'] = self.parse_time(self.parser.select(tr, 'td.formattedHeureDepart p', 1).text.strip())
current_step['line'] = to_unicode(self.parser.select(tr, 'td.rechercheResultatColumnMode img')[-1].attrib['alt'])
current_step['departure'] = to_unicode(self.parser.select(tr, 'td.descDepart p strong', 1).text.strip())
current_step['duration'] = self.parse_duration(self.parser.select(tr, 'td.rechercheResultatVertAlign', 1).text.strip())
elif tr.attrib['class'] == 'trBasTroncon':
current_step['end_time'] = self.parse_time(self.parser.select(tr, 'td.formattedHeureArrivee p', 1).text.strip())
current_step['arrival'] = to_unicode(self.parser.select(tr, 'td.descArrivee p strong', 1).text.strip())
yield current_step
def get_video(self, video=None):
_id = to_unicode(self.group_dict['id'])
if video is None:
video = YoujizzVideo(_id)
title_el = self.parser.select(self.document.getroot(), 'title', 1)
video.title = to_unicode(title_el.text.strip())
# youjizz HTML is crap, we must parse it with regexps
data = lxml.html.tostring(self.document.getroot())
m = re.search(r'<strong>.*?Runtime.*?</strong> (.+?)</div>', data)
if m:
txt = m.group(1).strip()
if txt == 'Unknown':
video.duration = NotAvailable
else:
minutes, seconds = (int(v) for v in to_unicode(txt).split(':'))
video.duration = datetime.timedelta(minutes=minutes,
seconds=seconds)
else:
raise BrokenPageError('Unable to retrieve video duration')
real_id = int(_id.split('-')[-1])
data = self.browser.readurl('http://www.youjizz.com/videos/embed/%s' %
real_id)
video_file_urls = re.findall(
r'"(http://[^",]+\.youjizz\.com[^",]+\.flv(?:\?[^"]*)?)"', data)
if len(video_file_urls) == 0:
raise BrokenPageError('Video URL not found')
elif len(video_file_urls) > 1:
raise BrokenPageError('Many video file URL found')
else:
video.url = to_unicode(video_file_urls[0])
return video
def get_list(self):
table = self.find_table()
for tr in self.parser.select(table, 'tr', 'many'):
tds = self.parser.select(tr, 'td')
if len(tds) != 6:
continue
tdlabel, tdid, tdcur, tdupdated, tdbal, tdbalcur = tds
account = Account()
account.label = to_unicode(tdlabel.text_content().strip())
# this is important - and is also the last part of the id (considering spaces)
# we can't use only the link as it does not goes where we want
try:
link = self.parser.select(tdlabel, 'a', 1)
except BrokenPageError:
# probably an account we can't display the history
account._link_id = None
else:
account._link_id = parse_qs(link.attrib['href'])['ch4'][0]
account.id = to_unicode(tdid.text.strip().replace(' ', ''))
account.iban = 'FR76' + account.id
# just in case we are showing the converted balances
account._main_currency = Account.get_currency(tdcur.text)
# we have to ignore those accounts, because using NotAvailable
# makes boobank and probably many others crash
if tdbal.text_content().strip() == 'indisponible':
continue
account.balance = Decimal(
Transaction.clean_amount(tdbal.text_content()))
account.currency = Account.get_currency(tdbalcur.text)
account._updated = datetime.strptime(tdupdated.text, '%d/%m/%Y')
yield account
def bcall_error_handler(self, backend, error, backtrace):
if isinstance(error, TransferStep):
params = {}
for field in error.fields:
v = self.ask(field)
params[field.id] = v
#backend.config['accept_transfer'].set(v)
params['backends'] = backend
self.start_format()
for transfer in self.do('transfer', error.transfer, **params):
self.format(transfer)
elif isinstance(error, AddRecipientStep):
params = {}
params['backends'] = backend
for field in error.fields:
v = self.ask(field)
params[field.id] = v
try:
next(iter(self.do('add_recipient', error.recipient, **params)))
except CallErrors as e:
self.bcall_errors_handler(e)
elif isinstance(error, TransferInvalidAmount):
print(u'Error(%s): %s' % (backend.name, to_unicode(error)
or 'The transfer amount is invalid'),
file=self.stderr)
elif isinstance(error, TransferInvalidLabel):
print(u'Error(%s): %s' % (backend.name, to_unicode(error)
or 'The transfer label is invalid'),
file=self.stderr)
elif isinstance(error, TransferInvalidEmitter):
print(u'Error(%s): %s' % (backend.name, to_unicode(error)
or 'The transfer emitter is invalid'),
file=self.stderr)
elif isinstance(error, TransferInvalidRecipient):
print(u'Error(%s): %s' % (backend.name, to_unicode(error)
or 'The transfer recipient is invalid'),
file=self.stderr)
elif isinstance(error, TransferInvalidDate):
print(u'Error(%s): %s' %
(backend.name, to_unicode(error)
or 'The transfer execution date is invalid'),
file=self.stderr)
elif isinstance(error, CaptchaQuestion):
if not self.captcha_weboob.count_backends():
print(
'Error(%s): Site requires solving a CAPTCHA but no CapCaptchaSolver backends were configured'
% backend.name,
file=self.stderr)
return False
print(
'Info(%s): Encountered CAPTCHA, please wait for its resolution, it can take dozens of seconds'
% backend.name,
file=self.stderr)
job = exception_to_job(error)
self.solve_captcha(job, backend)
return False
else:
return super(Boobank,
self).bcall_error_handler(backend, error, backtrace)
def add_cookie(self, name, value):
# httplib/cookielib don't seem to like unicode cookies...
if sys.version_info.major < 3:
name = to_unicode(name).encode('utf-8')
value = to_unicode(value).encode('utf-8')
self.browser.logger.debug('adding cookie %r=%r', name, value)
self.browser.session.cookies.set(name, value, domain=urlsplit(self.url).hostname)
def get_list(self):
table = self.find_table()
for tr in self.parser.select(table, 'tr', 'many'):
tds = self.parser.select(tr, 'td')
if len(tds) != 6:
continue
tdlabel, tdid, tdcur, tdupdated, tdbal, tdbalcur = tds
account = Account()
account.label = to_unicode(tdlabel.text_content().strip())
# this is important - and is also the last part of the id (considering spaces)
# we can't use only the link as it does not goes where we want
try:
link = self.parser.select(tdlabel, 'a', 1)
except BrokenPageError:
# probably an account we can't display the history
account._link_id = None
else:
account._link_id = parse_qs(link.attrib['href'])['ch4'][0]
account.id = to_unicode(tdid.text.strip().replace(' ', ''))
account.iban = 'FR76' + account.id
# just in case we are showing the converted balances
account._main_currency = Account.get_currency(tdcur.text)
# we have to ignore those accounts, because using NotAvailable
# makes boobank and probably many others crash
if tdbal.text_content().strip() == 'indisponible':
continue
account.balance = Decimal(Transaction.clean_amount(tdbal.text_content()))
account.currency = Account.get_currency(tdbalcur.text)
account._updated = datetime.strptime(tdupdated.text, '%d/%m/%Y')
yield account
def _postReply_eb(self, backend, error, backtrace):
content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while posting reply'),
content, QMessageBox.Ok)
self.process_reply = None
def get_stream_info(self, radio, url):
stream = BaseAudioStream(0)
current = StreamInfo(0)
r = self.browser.open(url, stream=True, headers={'Icy-Metadata':'1'})
stream.bitrate = int(r.headers['icy-br'].split(',')[0])
r.raw.read(int(r.headers['icy-metaint']))
size = ord(r.raw.read(1))
content = r.raw.read(size*16)
r.close()
for s in content.split("\x00")[0].split(";"):
a = s.split("=")
if a[0] == "StreamTitle":
stream.title = to_unicode(a[1].split("'")[1])
res = stream.title.split(" - ")
current.who = to_unicode(res[0])
if(len(res) == 1):
current.what = ""
else:
current.what = to_unicode(res[1])
stream.format=u'mp3'
stream.url = url
return [stream], current
def add_cookie(self, name, value):
# httplib/cookielib don't seem to like unicode cookies...
if sys.version_info.major < 3:
name = to_unicode(name).encode('utf-8')
value = to_unicode(value).encode('utf-8')
self.browser.logger.debug('adding cookie %r=%r', name, value)
self.browser.session.cookies.set(name, value)
def get_stream_info(self, radio, url):
stream = BaseAudioStream(0)
current = StreamInfo(0)
r = self.browser.open(url, stream=True, headers={'Icy-Metadata': 1})
stream.bitrate = int(r.headers['icy-br'].split(',')[0])
r.raw.read(int(r.headers['icy-metaint']))
size = ord(r.raw.read(1))
content = r.raw.read(size * 16)
r.close()
for s in content.split("\x00")[0].split(";"):
a = s.split("=")
if a[0] == "StreamTitle":
stream.title = to_unicode(a[1].split("'")[1])
res = stream.title.split(" - ")
current.who = to_unicode(res[0])
if (len(res) == 1):
current.what = ""
else:
current.what = to_unicode(res[1])
stream.format = u'mp3'
stream.url = url
return [stream], current
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overrided to support more exceptions types.
"""
if isinstance(error, BrowserQuestion):
for field in error.fields:
v = self.ask(field)
if v:
backend.config[field.id].set(v)
elif isinstance(error, BrowserIncorrectPassword):
msg = unicode(error)
if not msg:
msg = 'invalid login/password.'
print('Error(%s): %s' % (backend.name, msg), file=self.stderr)
if self.ask('Do you want to reconfigure this backend?', default=True):
self.unload_backends(names=[backend.name])
self.edit_backend(backend.name)
self.load_backends(names=[backend.name])
elif isinstance(error, BrowserSSLError):
print(u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC, file=self.stderr)
elif isinstance(error, BrowserForbidden):
msg = unicode(error)
if not msg:
msg = 'access denied'
print(u'Error(%s): %s' % (backend.name, msg or 'Forbidden'), file=self.stderr)
elif isinstance(error, BrowserUnavailable):
msg = unicode(error)
if not msg:
msg = 'website is unavailable.'
print(u'Error(%s): %s' % (backend.name, msg), file=self.stderr)
elif isinstance(error, NotImplementedError):
print(u'Error(%s): this feature is not supported yet by this backend.' % backend.name, file=self.stderr)
print(u' %s To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name)), file=self.stderr)
print(u' %s please contact: %s <*****@*****.**>' % (' ' * len(backend.name), backend.MAINTAINER, backend.NAME), file=self.stderr)
elif isinstance(error, UserError):
print(u'Error(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr)
elif isinstance(error, MoreResultsAvailable):
print(u'Hint: There are more results for backend %s' % (backend.name), file=self.stderr)
else:
print(u'Bug(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr)
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and not minfo.is_local():
self.weboob.repositories.update_repositories(ConsoleProgress(self))
# minfo of the new available module
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \
self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \
self.install_module(minfo):
print('New version of module %s has been installed. Retry to call the command.' % minfo.name)
return
if logging.root.level <= logging.DEBUG:
print(backtrace, file=self.stderr)
else:
return True
def load(self, items):
self.version = int(items['version'])
self.capabilities = items['capabilities'].split()
self.description = to_unicode(items['description'])
self.maintainer = to_unicode(items['maintainer'])
self.license = to_unicode(items['license'])
self.icon = items['icon'].strip() or None
self.urls = items['urls']
def load(self, items):
self.version = int(items['version'])
self.capabilities = items['capabilities'].split()
self.description = to_unicode(items['description'])
self.maintainer = to_unicode(items['maintainer'])
self.license = to_unicode(items['license'])
self.icon = items['icon'].strip() or None
self.urls = items['urls']
def load(self, items):
self.version = int(items["version"])
self.capabilities = items["capabilities"].split()
self.description = to_unicode(items["description"])
self.maintainer = to_unicode(items["maintainer"])
self.license = to_unicode(items["license"])
self.icon = items["icon"].strip() or None
self.urls = items["urls"]
def _saveNotes_eb(self, backend, error, backtrace):
self.ui.saveButton.setEnabled(True)
self.ui.textEdit.setEnabled(True)
content = unicode(self.tr('Unable to save notes:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while saving notes'),
content, QMessageBox.Ok)
def _saveNotes_eb(self, backend, error, backtrace):
self.ui.saveButton.setEnabled(True)
self.ui.textEdit.setEnabled(True)
content = unicode(self.tr('Unable to save notes:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while saving notes'),
content, QMessageBox.Ok)
def _errorSavePage(self, backend, error, backtrace):
""" """
content = unicode(self.tr('Unable to save page:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while saving page'),
content, QMessageBox.Ok)
self.ui.saveButton.setEnabled(True)
self.ui.saveButton.setText("Save")
def _errorLoadPage(self, backend, error, backtrace):
""" Error callback for loadPage """
content = unicode(self.tr('Unable to load page:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while loading page'),
content, QMessageBox.Ok)
self.ui.loadButton.setEnabled(True)
self.ui.loadButton.setText("Load")
def _errorLoadPage(self, backend, error, backtrace):
""" Error callback for loadPage """
content = self.tr('Unable to load page:\n%s\n') % to_unicode(error)
if logging.root.level <= logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while loading page'),
content, QMessageBox.Ok)
self.ui.loadButton.setEnabled(True)
self.ui.loadButton.setText("Load")
def _errorSavePage(self, backend, error, backtrace):
""" """
content = self.tr('Unable to save page:\n%s\n') % to_unicode(error)
if logging.root.level <= logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while saving page'),
content, QMessageBox.Ok)
self.ui.saveButton.setEnabled(True)
self.ui.saveButton.setText("Save")
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overrided to support more exceptions types.
"""
if isinstance(error, BrowserIncorrectPassword):
msg = unicode(error)
if not msg:
msg = 'invalid login/password.'
# TODO ask to reconfigure backend
print >> sys.stderr, 'Error(%s): %s' % (backend.name, msg)
if self.ask('Do you want to reconfigure this backend?',
default=True):
self.unload_backends(names=[backend.name])
self.edit_backend(backend.name)
self.load_backends(names=[backend.name])
elif isinstance(error, BrowserUnavailable):
msg = unicode(error)
if not msg:
msg = 'website is unavailable.'
print >> sys.stderr, u'Error(%s): %s' % (backend.name, msg)
elif isinstance(error, BrowserForbidden):
print >> sys.stderr, u'Error(%s): %s' % (backend.name, msg
or 'Forbidden')
elif isinstance(error, NotImplementedError):
print >> sys.stderr, u'Error(%s): this feature is not supported yet by this backend.' % backend.name
print >> sys.stderr, u' %s To help the maintainer of this backend implement this feature,' % (
' ' * len(backend.name))
print >> sys.stderr, u' %s please contact: %s <%s>' % (
' ' * len(backend.name), backend.MAINTAINER, backend.EMAIL)
elif isinstance(error, UserError):
print >> sys.stderr, u'Error(%s): %s' % (backend.name,
to_unicode(error))
elif isinstance(error, SSLError):
print >> sys.stderr, u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC
else:
print >> sys.stderr, u'Bug(%s): %s' % (backend.name,
to_unicode(error))
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and not minfo.is_local():
self.weboob.repositories.update_repositories()
# minfo of the new available module
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \
self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \
self.install_module(minfo):
print 'New version of module %s has been installed. Retry to call the command.' % minfo.name
return
if logging.root.level == logging.DEBUG:
print >> sys.stderr, backtrace
else:
return True
def write_dict(self, item, fp):
writer = csv.writer(fp, delimiter=self.field_separator)
if not self.started:
writer.writerow([to_unicode(v) for v in item.keys()])
self.started = True
if sys.version_info.major >= 3:
writer.writerow([str(v) for v in item.values()])
else:
writer.writerow([to_unicode(v).encode('utf-8') for v in item.values()])
def get_profile(self):
from weboob.tools.misc import to_unicode
profile = Profile()
if len([k for k in self.session.cookies.keys() if k == 'CTX']) > 1:
del self.session.cookies['CTX']
elif 'username='******'CTX', ''):
profile.name = to_unicode(re.search('username=([^&]+)', self.session.cookies['CTX']).group(1))
elif 'nomusager=' in self.session.cookies.get('headerdei'):
profile.name = to_unicode(re.search('nomusager=(?:[^&]+/ )?([^&]+)', self.session.cookies['headerdei']).group(1))
return profile
def _errorHistory(self, backend, error, backtrace):
""" Loading the history has failed """
content = unicode(self.tr('Unable to load history:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while loading history'),
content, QMessageBox.Ok)
self.ui.loadHistoryButton.setEnabled(True)
self.ui.loadHistoryButton.setText("Reload")
def parse(self):
emonths = [
'January', 'February', 'March', 'April', 'May', 'June', 'July',
'August', 'September', 'October', 'November', 'December'
]
date_format, time_format, months = self.guess_format()
for row in self.document.xpath(
'//table[@id="transactionTable"]/tbody/tr'):
if len(row.xpath('.//td')) < 5:
continue
amount = row.xpath(
'.//td[@headers="gross"]')[-1].text_content().strip()
if re.search('\d', amount):
currency = Account.get_currency(amount)
amount = AmTr.decimal_amount(amount)
else:
continue
idtext = row.xpath('.//td[@class="detailsNoPrint"]//span[@class="accessAid"]')[0] \
.text_content().replace(u'\xa0', u' ').strip().rpartition(' ')[-1]
trans = Transaction(idtext)
trans.amount = amount
trans._currency = currency
datetext = row.xpath(
'.//td[@class="dateInfo"]')[0].text_content().strip()
for i in range(0, 12):
datetext = datetext.replace(months[i], emonths[i])
date = dateutil.parser.parse(datetext)
trans.date = date
trans.rdate = date
trans.label = to_unicode(
row.xpath('.//td[@class="emailInfo"]')
[0].text_content().strip())
info = to_unicode(
row.xpath('.//td[@class="paymentTypeInfo"]')
[0].text_content().strip())
trans.raw = info + u' ' + trans.label
if u'Authorization' in info or u'Autorisation' in info or \
u'Order' in info:
continue
if u'Credit Card' in trans.label or u'Carte bancaire' in trans.label:
trans.type = Transaction.TYPE_CARD
elif info.startswith(u'Payment') or info.startswith(u'Paiement'):
trans.type = Transaction.TYPE_ORDER
elif u'Currency Conversion' in info or u'Conversion de devise' in info:
trans.type = Transaction.TYPE_BANK
else:
trans.type = Transaction.TYPE_UNKNOWN
yield trans
def _entry2video(self, entry):
"""
Parse an entry returned by googleapi and return a Video object.
"""
snippet = entry['snippet']
video = YoutubeVideo(to_unicode(entry['id']['videoId']))
video.title = to_unicode(snippet['title'].strip())
# duration does not seem to be available with api
video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url'])
video.author = to_unicode(snippet['channelTitle'].strip())
return video
def _getNotes_eb(self, backend, error, backtrace):
if isinstance(error, NotImplementedError):
return
self.ui.textEdit.setEnabled(True)
self.ui.saveButton.setEnabled(True)
content = self.tr('Unable to load notes:\n%s\n') % to_unicode(error)
if logging.root.level <= logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while loading notes'),
content, QMessageBox.Ok)
def _entry2video(self, entry):
"""
Parse an entry returned by googleapi and return a Video object.
"""
snippet = entry['snippet']
video = YoutubeVideo(to_unicode(entry['id']['videoId']))
video.title = to_unicode(snippet['title'].strip())
# duration does not seem to be available with api
video.thumbnail = Thumbnail(snippet['thumbnails']['default']['url'])
video.author = to_unicode(snippet['channelTitle'].strip())
return video
def _errorHistory(self, backend, error, backtrace):
""" Loading the history has failed """
content = unicode(
self.tr('Unable to load history:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while loading history'),
content, QMessageBox.Ok)
self.ui.loadHistoryButton.setEnabled(True)
self.ui.loadHistoryButton.setText("Reload")
def get_current(self, radio):
document = self.browser.location('http://rock.ouifm.fr/dynamic-menu.json')
suffix = ''
if radio != 'general':
suffix = '_%s' % radio
last = document['last%s' % suffix][0]
artist = to_unicode(last.get('artiste%s' % suffix, '').strip())
title = to_unicode(last.get('titre%s' % suffix, '').strip())
return artist, title
def write_dict(self, item, fp):
writer = csv.writer(fp, delimiter=self.field_separator)
if not self.started:
writer.writerow([to_unicode(v) for v in item.keys()])
self.started = True
if sys.version_info.major >= 3:
writer.writerow([str(v) for v in item.values()])
else:
writer.writerow(
[to_unicode(v).encode('utf-8') for v in item.values()])
def _getNotes_eb(self, backend, error, backtrace):
if isinstance(error, NotImplementedError):
return
self.ui.textEdit.setEnabled(True)
self.ui.saveButton.setEnabled(True)
content = unicode(self.tr('Unable to load notes:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while loading notes'),
content, QMessageBox.Ok)
def get_video(self, video=None):
_id = to_unicode(self.group_dict['id'])
if video is None:
video = CappedVideo(_id)
video.set_empty_fields(NotAvailable)
title_tmp = self.parser.select(self.document.getroot(), 'title', 1)
video.title = to_unicode(title_tmp.text.strip())
# Videopages doesn't have duration information (only results pages)
video.url = u'http://cdn.capped.tv/vhq/%s.mp4' % _id
return video
def get_current(self, radio):
document = self.browser.location(
'http://rock.ouifm.fr/dynamic-menu.json')
suffix = ''
if radio != 'general':
suffix = '_%s' % radio
last = document['last%s' % suffix][0]
artist = to_unicode(last.get('artiste%s' % suffix, '').strip())
title = to_unicode(last.get('titre%s' % suffix, '').strip())
return artist, title
def get_profile(self):
from weboob.tools.misc import to_unicode
profile = Profile()
if 'username='******'CTX', ''):
profile.name = to_unicode(
re.search('username=([^&]+)',
self.session.cookies['CTX']).group(1))
elif 'nomusager=' in self.session.cookies.get('headerdei'):
profile.name = to_unicode(
re.search('nomusager=(?:[^&]+/ )?([^&]+)',
self.session.cookies['headerdei']).group(1))
return profile
def get_current(self, radio):
document = self.browser.location("http://www.ouifm.fr/onair.json")
rad = ""
if radio == "general":
rad = "rock"
else:
rad = radio
last = document[rad][0]
artist = to_unicode(last.get("artist", "").strip())
title = to_unicode(last.get("title", "").strip())
return artist, title
def _postReply_eb(self, backend, error, backtrace):
content = self.tr('Unable to send message:\n%s\n') % to_unicode(error)
if logging.root.level <= logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while posting reply'),
content, QMessageBox.Ok)
self.ui.backendsList.setEnabled(True)
self.ui.threadsList.setEnabled(True)
self.ui.messagesTree.setEnabled(True)
self.ui.replyButton.setEnabled(True)
self.ui.replyWidget.setEnabled(True)
self.ui.sendButton.setText(self.tr('Send'))
self.process_reply = None
def _postReply_eb(self, backend, error, backtrace):
content = unicode(self.tr('Unable to send message:\n%s\n')) % to_unicode(error)
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % to_unicode(backtrace)
QMessageBox.critical(self, self.tr('Error while posting reply'),
content, QMessageBox.Ok)
self.ui.backendsList.setEnabled(True)
self.ui.threadsList.setEnabled(True)
self.ui.messagesTree.setEnabled(True)
self.ui.replyButton.setEnabled(True)
self.ui.replyWidget.setEnabled(True)
self.ui.sendButton.setText(self.tr('Send'))
self.process_reply = None
def get_current(self, radio):
document = self.browser.request('http://www.ouifm.fr/onair.json')
rad = ''
if radio == 'general':
rad = 'rock'
else:
rad = radio
last = document[rad][0]
artist = to_unicode(last.get('artist', '').strip())
title = to_unicode(last.get('title', '').strip())
return artist, title
def get_current(self, radio):
document = self.browser.request('http://www.ouifm.fr/onair.json')
rad = ''
if radio == 'general':
rad = 'rock'
else:
rad = radio
last = document[rad][0]
artist = to_unicode(last.get('artist', '').strip())
title = to_unicode(last.get('title', '').strip())
return artist, title
def iter_videos(self):
# When no results are found, the website returns random results
sb = self.parser.select(self.document.getroot(),
'div.search form input.searchbox', 1)
if sb.value == 'No Results Found':
return
#Extracting meta data from results page
vidbackdrop_list = self.parser.select(self.document.getroot(),
'div.vidBackdrop ')
for vidbackdrop in vidbackdrop_list:
url = self.parser.select(vidbackdrop, 'a', 1).attrib['href']
_id = url[2:]
video = CappedVideo(_id)
video.set_empty_fields(NotAvailable, ('url', ))
video.title = to_unicode(
self.parser.select(vidbackdrop, 'div.vidTitle a', 1).text)
video.author = to_unicode(
self.parser.select(vidbackdrop, 'div.vidAuthor a', 1).text)
thumbnail_url = 'http://cdn.capped.tv/pre/%s.png' % _id
video.thumbnail = Thumbnail(thumbnail_url)
video.thumbnail.url = to_unicode(video.thumbnail.id)
#we get the description field
duration_tmp = self.parser.select(vidbackdrop, 'div.vidInfo', 1)
#we remove tabs and spaces
duration_tmp2 = duration_tmp.text[7:]
#we remove all fields exept time
duration_tmp3 = duration_tmp2.split(' ')[0]
#we transform it in datetime format
parts = duration_tmp3.split(':')
if len(parts) == 1:
hours = minutes = 0
seconds = parts[0]
elif len(parts) == 2:
hours = 0
minutes, seconds = parts
elif len(parts) == 3:
hours, minutes, seconds = parts
else:
raise BrokenPageError('Unable to parse duration %r' %
duration_tmp)
video.duration = datetime.timedelta(hours=int(hours),
minutes=int(minutes),
seconds=int(seconds))
yield video
def _entry2video(self, entry):
"""
Parse an entry returned by gdata and return a Video object.
"""
video = YoutubeVideo(to_unicode(entry.id.text.split('/')[-1].strip()))
video.title = to_unicode(entry.media.title.text.strip())
video.duration = datetime.timedelta(seconds=int(entry.media.duration.seconds.strip()))
video.thumbnail = Thumbnail(to_unicode(entry.media.thumbnail[0].url.strip()))
if entry.author[0].name.text:
video.author = to_unicode(entry.author[0].name.text.strip())
if entry.media.name:
video.author = to_unicode(entry.media.name.text.strip())
return video
def get_list(self):
table = self.parser.select(self.document.getroot(), "#tab-corps", 1)
for tr in self.parser.select(table, "tr", "many"):
tdname, tdid, tdagency, tdbalance = [td.text_content().strip() for td in self.parser.select(tr, "td", 4)]
# it has empty rows - ignore those without the necessary info
if all((tdname, tdid, tdbalance)):
account = Account()
account.label = to_unicode(tdname)
account.id = to_unicode(tdid.replace(u"\xa0", "").replace(" ", ""))
account._agency = to_unicode(tdagency)
account._is_card = False
account.balance = Decimal(Transaction.clean_amount(tdbalance))
account.currency = account.get_currency(tdbalance)
yield account
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overrided to support more exceptions types.
"""
if isinstance(error, BrowserIncorrectPassword):
msg = unicode(error)
if not msg:
msg = 'invalid login/password.'
# TODO ask to reconfigure backend
print >>sys.stderr, 'Error(%s): %s' % (backend.name, msg)
if self.ask('Do you want to reconfigure this backend?', default=True):
self.unload_backends(names=[backend.name])
self.edit_backend(backend.name)
self.load_backends(names=[backend.name])
elif isinstance(error, BrowserUnavailable):
msg = unicode(error)
if not msg:
msg = 'website is unavailable.'
print >>sys.stderr, u'Error(%s): %s' % (backend.name, msg)
elif isinstance(error, BrowserForbidden):
print >>sys.stderr, u'Error(%s): %s' % (backend.name, msg or 'Forbidden')
elif isinstance(error, NotImplementedError):
print >>sys.stderr, u'Error(%s): this feature is not supported yet by this backend.' % backend.name
print >>sys.stderr, u' %s To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name))
print >>sys.stderr, u' %s please contact: %s <%s>' % (' ' * len(backend.name), backend.MAINTAINER, backend.EMAIL)
elif isinstance(error, UserError):
print >>sys.stderr, u'Error(%s): %s' % (backend.name, to_unicode(error))
elif isinstance(error, SSLError):
print >>sys.stderr, u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC
else:
print >>sys.stderr, u'Bug(%s): %s' % (backend.name, to_unicode(error))
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and not minfo.is_local():
self.weboob.repositories.update_repositories()
# minfo of the new available module
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \
self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \
self.install_module(minfo):
print 'New version of module %s has been installed. Retry to call the command.' % minfo.name
return
if logging.root.level == logging.DEBUG:
print >>sys.stderr, backtrace
else:
return True
def _entry2video(self, entry):
"""
Parse an entry returned by gdata and return a Video object.
"""
video = YoutubeVideo(to_unicode(entry.id.text.split("/")[-1].strip()))
video.title = to_unicode(entry.media.title.text.strip())
video.duration = datetime.timedelta(seconds=int(entry.media.duration.seconds.strip()))
video.thumbnail = Thumbnail(to_unicode(entry.media.thumbnail[0].url.strip()))
if entry.author[0].name.text:
video.author = to_unicode(entry.author[0].name.text.strip())
if entry.media.name:
video.author = to_unicode(entry.media.name.text.strip())
return video
def get_profile(self):
if not self.is_cenet_website:
from weboob.tools.misc import to_unicode
profile = Profile()
if 'username='******'CTX', ''):
profile.name = to_unicode(
re.search('username=([^&]+)',
self.session.cookies['CTX']).group(1))
elif 'nomusager=' in self.session.cookies.get('headerdei'):
profile.name = to_unicode(
re.search('nomusager=(?:[^&]+/ )?([^&]+)',
self.session.cookies['headerdei']).group(1))
else:
profile = self.cenet_home.stay_or_go().get_profile()
return profile
def fixurl(url):
url = to_unicode(url)
# remove javascript crap
url = url.replace('/#!/', '/')
# parse it
parsed = urlparse.urlsplit(url)
# divide the netloc further
userpass, at, hostport = parsed.netloc.rpartition('@')
user, colon1, pass_ = userpass.partition(':')
host, colon2, port = hostport.partition(':')
# encode each component
scheme = parsed.scheme.encode('utf8')
user = urllib.quote(user.encode('utf8'))
colon1 = colon1.encode('utf8')
pass_ = urllib.quote(pass_.encode('utf8'))
at = at.encode('utf8')
host = host.encode('idna')
colon2 = colon2.encode('utf8')
port = port.encode('utf8')
path = '/'.join(pce.encode('utf8') for pce in parsed.path.split('/'))
# while valid, it is most likely an error
path = path.replace('//', '/')
query = parsed.query.encode('utf8')
fragment = parsed.fragment.encode('utf8')
# put it back together
netloc = ''.join((user, colon1, pass_, at, host, colon2, port))
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def onecmd(self, line):
"""
This REPL method is overrided to catch some particular exceptions.
"""
line = to_unicode(line)
cmd, arg, ignored = self.parseline(line)
# Set the right formatter for the command.
try:
formatter_name = self.commands_formatters[cmd]
except KeyError:
formatter_name = self.DEFAULT_FORMATTER
self.set_formatter(formatter_name)
try:
try:
return super(ReplApplication, self).onecmd(line)
except CallErrors as e:
self.bcall_errors_handler(e)
except BackendNotGiven as e:
print >>sys.stderr, 'Error: %s' % str(e)
except NotEnoughArguments as e:
print >>sys.stderr, 'Error: not enough arguments. %s' % str(e)
except (KeyboardInterrupt, EOFError):
# ^C during a command process doesn't exit application.
print '\nAborted.'
finally:
self.flush()
def api_request(self, command, **kwargs):
if 'data' in kwargs:
data = to_unicode(kwargs.pop('data')).encode('utf-8', 'replace')
else:
data = None
headers = {}
if not command.startswith('applications'):
today = local2utc(datetime.now()).strftime('%Y-%m-%d')
token = sha256(self.username + self.APITOKEN + today).hexdigest()
headers['Authorization'] = 'Basic %s' % (b64encode('%s:%s' % (self.username, self.password)))
headers['X-Platform'] = 'android'
headers['X-Client-Version'] = self.APIVERSION
headers['X-AUM-Token'] = token
url = self.buildurl(self.absurl('/api/%s' % command), **kwargs)
if isinstance(url, unicode):
url = url.encode('utf-8')
req = self.request_class(url, data, headers)
buf = self.openurl(req).read()
try:
r = json.loads(buf)
except ValueError:
raise ValueError(buf)
return r
def is_empty(self):
for td in self.parser.select(self.document.getroot(),
'td.V11vertGras'):
if u'Aucune opération enregistrée' in to_unicode(
td.text_content()):
return True
return False