def main(filename):
weboob = Weboob()
try:
hds = weboob.build_backend('hds')
except ModuleLoadError, e:
print >>sys.stderr, 'Unable to load "hds" module: %s' % e
return 1
class BackendTest(TestCase):
MODULE = None
def __init__(self, *args, **kwargs):
super(BackendTest, self).__init__(*args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
# Skip tests when passwords are missing
self.weboob.requests.register('login', self.login_cb)
if self.weboob.load_backends(modules=[self.MODULE]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
def login_cb(self, backend_name, value):
raise SkipTest('missing config \'%s\' is required for this test' %
value.label)
def run(self, result):
"""
Call the parent run() for each backend instance.
Skip the test if we have no backends.
"""
# This is a hack to fix an issue with nosetests running
# with many tests. The default is 1000.
sys.setrecursionlimit(10000)
try:
if not len(self.backends):
self.backend = self.weboob.build_backend(self.MODULE,
nofail=True)
TestCase.run(self, result)
else:
# Run for all backend
for backend_instance in self.backends.keys():
print(backend_instance)
self.backend = self.backends[backend_instance]
TestCase.run(self, result)
finally:
self.weboob.deinit()
def shortDescription(self):
"""
Generate a description with the backend instance name.
"""
# do not use TestCase.shortDescription as it returns None
return '%s [%s]' % (str(self), self.backend_instance)
def is_backend_configured(self):
"""
Check if the backend is in the user configuration file
"""
return self.weboob.backends_config.backend_exists(
self.backend.config.instname)
def weboob_download(self, credentials, logs):
logger.info(
'Start weboob operations with module %s',
self.weboob_module_id.name)
w = Weboob()
back = w.build_backend(
self.weboob_module_id.name, params=credentials, name='odoo')
try:
sub = back.iter_subscription().next()
except BrowserIncorrectPassword:
logs['msg'].append(_('Wrong password.'))
logs['result'] = 'failure'
return []
bills = back.iter_bills(sub)
start_date = self.download_start_date
invoices = []
for bill in bills:
logger.debug('bill.id=%s, bill.fullid=%s', bill.id, bill.fullid)
inv_details = bill.to_dict()
logger.info('bill.to_dict=%s', inv_details)
# bill.to_dict=OrderedDict([
# ('id', u'60006530609_216421161'),
# ('url', u'https://api.bouyguestelecom.fr/comptes-facturat...'),
# ('date', date(2018, 7, 16)),
# ('format', u'pdf'),
# ('label', u'Juillet 2018'),
# ('type', u'bill'),
# ('transactions', []),
# ('price', Decimal('30.99')),
# ('currency', u'EUR'),
# ('vat', NotLoaded),
# ('duedate', NotLoaded), ('startdate', NotLoaded),
# ('finishdate', NotLoaded), ('income', False)])
# Do we have invoice number here ? NO
logger.info("Found invoice dated %s", inv_details.get('date'))
if (
start_date and
inv_details.get('date') and
fields.Date.to_string(inv_details['date']) < start_date):
logger.info(
'Skipping invoice %s dated %s dated before '
'download_start_date %s',
inv_details.get('label'), inv_details['date'], start_date)
continue
logger.info('Start to download bill with full ID %s', bill.fullid)
pdf_inv = back.download_document(bill.id)
filename = 'invoice_%s_%s.%s' % (
self.weboob_module_id.name,
inv_details.get('label') and
inv_details['label'].replace(' ', '_'),
inv_details.get('format', 'pdf'))
invoices.append((pdf_inv.encode('base64'), filename))
return invoices
class BackendTest(TestCase):
MODULE = None
def __init__(self, *args, **kwargs):
super(BackendTest, self).__init__(*args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
# Skip tests when passwords are missing
self.weboob.requests.register('login', self.login_cb)
if self.weboob.load_backends(modules=[self.MODULE]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
def login_cb(self, backend_name, value):
raise SkipTest('missing config \'%s\' is required for this test' % value.label)
def run(self, result):
"""
Call the parent run() for each backend instance.
Skip the test if we have no backends.
"""
# This is a hack to fix an issue with nosetests running
# with many tests. The default is 1000.
sys.setrecursionlimit(10000)
try:
if not len(self.backends):
self.backend = self.weboob.build_backend(self.MODULE, nofail=True)
TestCase.run(self, result)
else:
# Run for all backend
for backend_instance in self.backends.keys():
print(backend_instance)
self.backend = self.backends[backend_instance]
TestCase.run(self, result)
finally:
self.weboob.deinit()
def shortDescription(self):
"""
Generate a description with the backend instance name.
"""
# do not use TestCase.shortDescription as it returns None
return '%s [%s]' % (str(self), self.backend_instance)
def is_backend_configured(self):
"""
Check if the backend is in the user configuration file
"""
return self.weboob.backends_config.backend_exists(self.backend.config.instname)
class Connector(object):
"""
Connector is a tool that connects to common websites like bank website,
phone operator website... and that grabs personal data from there.
Credentials are required to make this operation.
Technically, connectors are weboob backend wrappers.
"""
@staticmethod
def version():
"""
Get the version of the installed Weboob.
"""
return Weboob.VERSION
def __init__(self, weboob_data_path):
"""
Create a Weboob instance.
:param weboob_data_path: Weboob path to use.
"""
# By default, consider we don't need to update the repositories.
self.needs_update = False
if not os.path.isdir(weboob_data_path):
os.makedirs(weboob_data_path)
# Set weboob data directory and sources.list file.
self.weboob_data_path = weboob_data_path
self.write_weboob_sources_list()
# Create a Weboob object.
self.weboob = Weboob(workdir=weboob_data_path,
datadir=weboob_data_path)
self.backends = collections.defaultdict(dict)
# Update the weboob repos only if new repos are included.
if self.needs_update:
self.update()
def write_weboob_sources_list(self):
"""
Ensure the Weboob sources.list file contains the required entries from
Kresus.
"""
sources_list_path = os.path.join(self.weboob_data_path, 'sources.list')
# Read the content of existing sources.list, if it exists.
original_sources_list_content = []
if os.path.isfile(sources_list_path):
with io.open(sources_list_path, encoding="utf-8") as fh:
original_sources_list_content = fh.read().splitlines()
# Determine the new content of the sources.list
new_sources_list_content = []
if ('WEBOOB_SOURCES_LIST' in os.environ
and os.path.isfile(os.environ['WEBOOB_SOURCES_LIST'])):
# Read the new content from the sources.list provided as env variable.
with io.open(os.environ['WEBOOB_SOURCES_LIST'],
encoding="utf-8") as fh:
new_sources_list_content = fh.read().splitlines()
else:
# The default content of the sources.list
new_sources_list_content = [
unicode('https://updates.weboob.org/%(version)s/main/'),
unicode('file://%s/fakemodules/' %
(os.path.dirname(os.path.abspath(__file__))))
]
# Update the source.list content and update the repository, only if the
# content has changed.
if set(original_sources_list_content) != set(new_sources_list_content):
with io.open(sources_list_path, 'w',
encoding="utf-8") as sources_list_file:
sources_list_file.write('\n'.join(new_sources_list_content))
self.needs_update = True
def update(self):
"""
Update Weboob modules.
"""
# Weboob has an offending print statement when it "Rebuilds index",
# which happen at every run if the user has a local repository. We need
# to silence it, hence the temporary redirect of stdout.
sys.stdout = open(os.devnull, "w")
try:
self.weboob.update(progress=DummyProgress())
except ConnectionError as exc:
# Do not delete the repository if there is a connection error.
raise exc
except Exception:
# Try to remove the data directory, to see if it changes a thing.
# This is especially useful when a new version of Weboob is
# published and/or the keyring changes.
shutil.rmtree(self.weboob_data_path)
os.makedirs(self.weboob_data_path)
# Recreate the Weboob object as the directories are created
# on creating the Weboob object.
self.weboob = Weboob(workdir=self.weboob_data_path,
datadir=self.weboob_data_path)
# Rewrite sources.list file
self.write_weboob_sources_list()
# Retry update
self.weboob.update(progress=DummyProgress())
finally:
# Restore stdout
sys.stdout = sys.__stdout__
def create_backend(self, modulename, parameters):
"""
Create a Weboob backend for a given module, ready to be used to fetch
data.
:param modulename: The name of the module from which backend should be
created.
:param parameters: A dict of parameters to pass to the module. It
should at least contain ``login`` and ``password`` fields, but can
contain additional values depending on the module.
"""
# Install the module if required.
repositories = self.weboob.repositories
minfo = repositories.get_module_info(modulename)
if (minfo is not None and not minfo.is_installed()
and not minfo.is_local()):
# We cannot install a locally available module, this would
# result in a ModuleInstallError.
try:
repositories.install(minfo, progress=DummyProgress())
except ModuleInstallError:
fail(GENERIC_EXCEPTION,
"Unable to install module %s." % bank_module,
traceback.format_exc())
# Initialize the backend.
login = parameters['login']
self.backends[modulename][login] = self.weboob.build_backend(
modulename, parameters)
def delete_backend(self, modulename, login=None):
"""
Delete a created backend for the given module.
:param modulename: The name of the module from which backend should be
deleted.
:param login: An optional login to delete only a specific backend.
Otherwise delete all the backends from the given module name.
"""
def _deinit_backend(backend):
"""
Deinitialize a given Weboob loaded backend object.
"""
# This code comes directly from Weboob core code. As we are
# building backends on our side, we are responsible for
# deinitialization.
with backend:
backend.deinit()
try:
# Deinit matching backend objects and remove them from loaded
# backends dict.
if login:
_deinit_backend(self.backends[modulename][login])
del self.backends[modulename][login]
else:
for backend in self.backends:
_deinit_backend(backend[modulename])
del self.backends[modulename]
gc.collect() # Force GC collection, better than nothing.
except KeyError:
logging.warning('No matching backends for module %s and login %s.',
modulename, login)
def get_all_backends(self):
"""
Get all the available built backends.
:returns: A list of backends.
"""
backends = []
for modules_backends in self.backends.values():
backends.extend(modules_backends.values())
return backends
def get_bank_backends(self, modulename):
"""
Get all the built backends for a given bank module.
:param modulename: The name of the module from which the backend should
be created.
:returns: A list of backends.
"""
if modulename in self.backends:
return self.backends[modulename].values()
logging.warning('No matching built backends for bank module %s.',
modulename)
return []
def get_backend(self, modulename, login):
"""
Get a specific backend associated to a specific login with a specific
bank module.
:param modulename: The name of the module from which the backend should
be created.
:param login: The login to further filter on the available backends.
:returns: A list of backends (with a single item).
"""
if not modulename:
# Module name is mandatory in this case.
logging.error('Missing bank module name.')
return []
if modulename in self.backends and login in self.backends[modulename]:
return [self.backends[modulename][login]]
logging.warning(
'No matching built backends for bank module %s with login %s.',
modulename, login)
return []
def get_backends(self, modulename=None, login=None):
"""
Get a list of backends matching criterions.
:param modulename: The name of the module from which the backend should
be created.
:param login: The login to further filter on the available backends. If
passed, ``modulename`` cannot be empty.
:returns: A list of backends.
"""
if login:
# If login is provided, only return backends matching the
# module name and login (at most one).
return self.get_backend(modulename, login)
if modulename:
# If only modulename is provided, returns all matching
# backends.
return self.get_bank_backends(modulename)
# Just return all available backends.
return self.get_all_backends()
@staticmethod
def get_accounts(backend):
"""
Fetch accounts data from Weboob.
:param backend: The Weboob built backend to fetch data from.
:returns: A list of dicts representing the available accounts.
"""
results = []
for account in backend.iter_accounts():
iban = None
if not empty(account.iban):
iban = account.iban
currency = None
if not empty(account.currency):
currency = unicode(account.currency)
results.append({
'accountNumber': account.id,
'title': account.label,
'balance': unicode(account.balance),
'iban': iban,
'currency': currency
})
return results
@staticmethod
def get_operations(backend):
"""
Fetch operations data from Weboob.
:param backend: The Weboob built backend to fetch data from.
:returns: A list of dicts representing the available operations.
"""
results = []
for account in list(backend.iter_accounts()):
# Get operations for all accounts available.
try:
history = backend.iter_history(account)
# Build an operation dict for each operation.
for line in history:
# Handle date
if line.rdate:
# Use date of the payment (real date) if available.
date = line.rdate
elif line.date:
# Otherwise, use debit date, on the bank statement.
date = line.date
else:
logging.error(
'No known date property in operation line: %s.',
unicode(line.raw))
date = datetime.now()
if line.label:
title = unicode(line.label)
else:
title = unicode(line.raw)
isodate = date.isoformat()
debit_date = line.date.isoformat()
results.append({
'account': account.id,
'amount': unicode(line.amount),
'raw': unicode(line.raw),
'type': line.type,
'date': isodate,
'debit_date': debit_date,
'title': title
})
except NotImplementedError:
# Weboob raises a NotImplementedError upon iteration, not upon
# method call. Hence, this exception should wrap the whole
# iteration.
logging.error(('This account type has not been implemented by '
'weboob: %s.'), account.id)
return results
def fetch(self, which, modulename=None, login=None):
"""
Wrapper to fetch data from the Weboob connector.
This wrapper fetches the required data from Weboob and returns it. It
handles the translation between Weboob exceptions and Kresus error
codes stored in the JSON response.
:param which: The type of data to fetch. Can be either ``accounts`` or
``operations``.
:param modulename: The name of the module from which data should be
fetched. Optional, if not provided all available backends are used.
:param login: The login to further filter on the available backends.
Optional, if not provided all matching backends are used.
:returns: A dict of the fetched data, in a ``values`` keys. Errors are
described under ``error_code``, ``error_short`` and ``error_content``
keys.
"""
results = {}
try:
results['values'] = []
backends = self.get_backends(modulename, login)
if which == 'accounts':
fetch_function = self.get_accounts
elif which == 'operations':
fetch_function = self.get_operations
else:
raise Exception('Invalid fetch command.')
for backend in backends:
with backend: # Acquire lock on backend
results['values'].extend(fetch_function(backend))
except NoAccountsException:
results['error_code'] = NO_ACCOUNTS
except ModuleLoadError:
results['error_code'] = UNKNOWN_MODULE
except BrowserPasswordExpired:
results['error_code'] = EXPIRED_PASSWORD
except ActionNeeded as exc:
# This `except` clause is not in alphabetic order and cannot be,
# because BrowserPasswordExpired (above) inherits from it in
# Weboob 1.4.
results['error_code'] = ACTION_NEEDED
results['error_content'] = unicode(exc)
except BrowserIncorrectPassword:
# This `except` clause is not in alphabetic order and cannot be,
# because BrowserPasswordExpired (above) inherits from it in
# Weboob 1.3.
results['error_code'] = INVALID_PASSWORD
except Module.ConfigError as exc:
results['error_code'] = INVALID_PARAMETERS
results['error_content'] = unicode(exc)
except ConnectionError as exc:
results['error_code'] = CONNECTION_ERROR
results['error_content'] = unicode(exc)
except Exception as exc:
fail(GENERIC_EXCEPTION, 'Unknown error: %s.' % unicode(exc),
traceback.format_exc())
return results
def main(filename):
weboob = Weboob()
try:
hds = weboob.build_backend('hds')
except ModuleLoadError as e:
print('Unable to load "hds" module: %s' % e, file=sys.stderr)
return 1
try:
db = sqlite.connect(database=filename, timeout=10.0)
except sqlite.OperationalError as err:
print('Unable to open %s database: %s' % (filename, err),
file=sys.stderr)
return 1
sys.stdout.write('Reading database... ')
sys.stdout.flush()
try:
results = db.execute('SELECT id, author FROM stories')
except sqlite.OperationalError as err:
print('fail!\nUnable to read database: %s' % err, file=sys.stderr)
return 1
stored = set()
authors = set()
for r in results:
stored.add(r[0])
authors.add(r[1])
stored_authors = {s[0] for s in db.execute('SELECT name FROM authors')}
sys.stdout.write('ok\n')
br = hds.browser
to_fetch = set()
sys.stdout.write('Getting stories list from website... ')
sys.stdout.flush()
for story in br.iter_stories():
if int(story.id) in stored:
break
to_fetch.add(story.id)
authors.add(story.author.name)
sys.stdout.write(' ok\n')
sys.stdout.write('Getting %d new storiese... ' % len(to_fetch))
sys.stdout.flush()
for id in to_fetch:
story = br.get_story(id)
if not story:
logging.warning('Story #%d unavailable' % id)
continue
db.execute(
"""INSERT INTO stories (id, title, date, category, author, body)
VALUES (?, ?, ?, ?, ?, ?)""",
(story.id, story.title, story.date, story.category,
story.author.name, story.body))
db.commit()
sys.stdout.write('ok\n')
authors = authors.difference(stored_authors)
sys.stdout.write('Getting %d new authors... ' % len(authors))
sys.stdout.flush()
for a in authors:
author = br.get_author(a)
if not author:
logging.warning('Author %s unavailable\n' % id)
continue
db.execute(
"INSERT INTO authors (name, sex, description) VALUES (?, ?, ?)",
(a, author.sex, author.description))
db.commit()
sys.stdout.write(' ok\n')
return 0
def main(filename):
weboob = Weboob()
try:
hds = weboob.build_backend('hds')
except ModuleLoadError as e:
print('Unable to load "hds" module: %s' % e, file=sys.stderr)
return 1
try:
db = sqlite.connect(database=filename, timeout=10.0)
except sqlite.OperationalError as err:
print('Unable to open %s database: %s' % (filename, err), file=sys.stderr)
return 1
sys.stdout.write('Reading database... ')
sys.stdout.flush()
try:
results = db.execute('SELECT id, author FROM stories')
except sqlite.OperationalError as err:
print('fail!\nUnable to read database: %s' % err, file=sys.stderr)
return 1
stored = set()
authors = set()
for r in results:
stored.add(r[0])
authors.add(r[1])
stored_authors = set([s[0] for s in db.execute('SELECT name FROM authors')])
sys.stdout.write('ok\n')
br = hds.browser
to_fetch = set()
sys.stdout.write('Getting stories list from website... ')
sys.stdout.flush()
for story in br.iter_stories():
if int(story.id) in stored:
break
to_fetch.add(story.id)
authors.add(story.author.name)
sys.stdout.write(' ok\n')
sys.stdout.write('Getting %d new storiese... ' % len(to_fetch))
sys.stdout.flush()
for id in to_fetch:
story = br.get_story(id)
if not story:
logging.warning('Story #%d unavailable' % id)
continue
db.execute("""INSERT INTO stories (id, title, date, category, author, body)
VALUES (?, ?, ?, ?, ?, ?)""",
(story.id, story.title, story.date, story.category,
story.author.name, story.body))
db.commit()
sys.stdout.write('ok\n')
authors = authors.difference(stored_authors)
sys.stdout.write('Getting %d new authors... ' % len(authors))
sys.stdout.flush()
for a in authors:
author = br.get_author(a)
if not author:
logging.warning('Author %s unavailable\n' % id)
continue
db.execute("INSERT INTO authors (name, sex, description) VALUES (?, ?, ?)",
(a, author.sex, author.description))
db.commit()
sys.stdout.write(' ok\n')
return 0
class Connector(object):
"""
Connector is a tool that connects to common websites like bank website,
phone operator website... and that grabs personal data from there.
Credentials are required to make this operation.
Technically, connectors are weboob backend wrappers.
"""
@staticmethod
def version():
"""
Get the version of the installed Weboob.
"""
return Weboob.VERSION
def __init__(self, weboob_data_path, fakemodules_path, sources_list_content, is_prod):
"""
Create a Weboob instance.
:param weboob_data_path: Weboob path to use.
:param fakemodules_path: Path to the fake modules directory in user
data.
:param sources_list_content: Optional content of the sources.list file,
as an array of lines, or None if not present.
:param is_prod: whether we're running in production or not.
"""
# By default, consider we don't need to update the repositories.
self.needs_update = False
self.fakemodules_path = fakemodules_path
self.sources_list_content = sources_list_content
if not os.path.isdir(weboob_data_path):
os.makedirs(weboob_data_path)
# Set weboob data directory and sources.list file.
self.weboob_data_path = weboob_data_path
self.write_weboob_sources_list()
# Create a Weboob object.
self.weboob = Weboob(workdir=weboob_data_path,
datadir=weboob_data_path)
self.backend = None
self.storage = None
# To make development more pleasant, always copy the fake modules in
# non-production modes.
if not is_prod:
self.copy_fakemodules()
# Update the weboob repos only if new repos are included.
if self.needs_update:
self.update()
def copy_fakemodules(self):
"""
Copies the fake modules files into the default fakemodules user-data
directory.
When Weboob updates modules, it might want to write within the
fakemodules directory, which might not be writable by the current
user. To prevent this, first copy the fakemodules directory in
a directory we have write access to, and then use that directory
in the sources list file.
"""
fakemodules_src = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fakemodules')
if os.path.isdir(self.fakemodules_path):
shutil.rmtree(self.fakemodules_path)
shutil.copytree(fakemodules_src, self.fakemodules_path)
def write_weboob_sources_list(self):
"""
Ensure the Weboob sources.list file contains the required entries from
Kresus.
"""
sources_list_path = os.path.join(self.weboob_data_path, 'sources.list')
# Determine the new content of the sources.list file.
new_sources_list_content = []
if self.sources_list_content is not None:
new_sources_list_content = self.sources_list_content
else:
# Default content of the sources.list file.
new_sources_list_content = [
unicode('https://updates.weboob.org/%(version)s/main/'),
unicode('file://%s' % self.fakemodules_path)
]
# Read the content of existing sources.list, if it exists.
original_sources_list_content = []
if os.path.isfile(sources_list_path):
with io.open(sources_list_path, encoding="utf-8") as fh:
original_sources_list_content = fh.read().splitlines()
# Update the source.list content and update the repository, only if the
# content has changed.
if set(original_sources_list_content) != set(new_sources_list_content):
with io.open(sources_list_path, 'w', encoding="utf-8") as sources_list_file:
sources_list_file.write('\n'.join(new_sources_list_content))
self.needs_update = True
def update(self):
"""
Update Weboob modules.
"""
self.copy_fakemodules()
# Weboob has an offending print statement when it "Rebuilds index",
# which happen at every run if the user has a local repository. We need
# to silence it, hence the temporary redirect of stdout.
sys.stdout = open(os.devnull, "w")
try:
self.weboob.update(progress=DummyProgress())
except ConnectionError as exc:
# Do not delete the repository if there is a connection error.
raise exc
except Exception:
# Try to remove the data directory, to see if it changes a thing.
# This is especially useful when a new version of Weboob is
# published and/or the keyring changes.
shutil.rmtree(self.weboob_data_path)
os.makedirs(self.weboob_data_path)
# Recreate the Weboob object as the directories are created
# on creating the Weboob object.
self.weboob = Weboob(workdir=self.weboob_data_path,
datadir=self.weboob_data_path)
# Rewrite sources.list file
self.write_weboob_sources_list()
# Retry update
self.weboob.update(progress=DummyProgress())
finally:
# Restore stdout
sys.stdout = sys.__stdout__
def create_backend(self, modulename, parameters, session):
"""
Create a Weboob backend for a given module, ready to be used to fetch
data.
:param modulename: The name of the module from which backend should be
created.
:param parameters: A dict of parameters to pass to the module. It
should at least contain ``login`` and ``password`` fields, but can
contain additional values depending on the module.
:param session: an object representing the browser state.
"""
# Install the module if required.
repositories = self.weboob.repositories
minfo = repositories.get_module_info(modulename)
if (
minfo is not None and not minfo.is_installed() and
not minfo.is_local()
):
# We cannot install a locally available module, this would
# result in a ModuleInstallError.
try:
repositories.install(minfo, progress=DummyProgress())
except ModuleInstallError:
fail(
GENERIC_EXCEPTION,
"Unable to install module %s." % modulename,
traceback.format_exc()
)
# Initialize the Storage.
self.storage = DictStorage(session)
# Initialize the backend.
self.backend = self.weboob.build_backend(
modulename,
parameters,
storage=self.storage
)
def delete_backend(self):
"""
Delete a created backend for the given module.
"""
if self.backend:
with self.backend:
self.backend.deinit()
self.backend = None
self.storage = None
def get_accounts(self):
"""
Fetch accounts data from Weboob.
:param backend: The Weboob built backend to fetch data from.
:returns: A list of dicts representing the available accounts.
"""
results = []
with self.backend:
for account in list(self.backend.iter_accounts()):
# The minimum dict keys for an account are :
# 'id', 'label', 'balance' and 'type'
# Retrieve extra information for the account.
account = self.backend.fillobj(account, ['iban', 'currency'])
iban = None
if not empty(account.iban):
iban = account.iban
currency = None
if not empty(account.currency):
currency = unicode(account.currency)
results.append({
'vendorAccountId': account.id,
'label': account.label,
'balance': account.balance,
'iban': iban,
'currency': currency,
'type': account.type,
})
return results
def get_operations(self):
"""
Fetch operations data from Weboob.
:param backend: The Weboob built backend to fetch data from.
:returns: A list of dicts representing the available operations.
"""
results = []
with self.backend:
for account in list(self.backend.iter_accounts()):
# Get all operations for this account.
nyi_methods = []
operations = []
try:
operations += list(self.backend.iter_history(account))
except NotImplementedError:
nyi_methods.append('iter_history')
try:
operations += [
op for op in self.backend.iter_coming(account)
if op.type in [
Transaction.TYPE_DEFERRED_CARD,
Transaction.TYPE_CARD_SUMMARY
]
]
except NotImplementedError:
nyi_methods.append('iter_coming')
for method_name in nyi_methods:
logging.error(
('%s not implemented for this account: %s.'),
method_name,
account.id
)
# Build an operation dict for each operation.
for operation in operations:
label = None
if not empty(operation.label):
label = unicode(operation.label)
raw_label = None
if not empty(operation.raw):
raw_label = unicode(operation.raw)
elif label:
raw_label = label
if raw_label and not label:
label = raw_label
# Handle date
if operation.rdate:
# Use date of the payment (real date) if available.
date = operation.rdate
elif operation.date:
# Otherwise, use debit date, on the bank statement.
date = operation.date
else:
logging.error(
'No known date property in operation line: %s.',
raw_label or "no label"
)
date = datetime.now()
isodate = date.isoformat()
debit_date = operation.date.isoformat()
results.append({
'account': account.id,
'amount': operation.amount,
'rawLabel': raw_label,
'type': operation.type,
'date': isodate,
'debit_date': debit_date,
'label': label
})
return results
def fetch(self, which):
"""
Wrapper to fetch data from the Weboob connector.
This wrapper fetches the required data from Weboob and returns it. It
handles the translation between Weboob exceptions and Kresus error
codes stored in the JSON response.
:param which: The type of data to fetch. Can be either ``accounts`` or
``operations``.
:param modulename: The name of the module from which data should be
fetched. Optional, if not provided all available backends are used.
:param login: The login to further filter on the available backends.
Optional, if not provided all matching backends are used.
:returns: A dict of the fetched data, in a ``values`` keys. Errors are
described under ``error_code``, ``error_short`` and ``error_message``
keys.
"""
results = {}
try:
if which == 'accounts':
results['values'] = self.get_accounts()
elif which == 'operations':
results['values'] = self.get_operations()
else:
raise Exception('Invalid fetch command.')
except NoAccountsException:
results['error_code'] = NO_ACCOUNTS
except ModuleLoadError:
results['error_code'] = UNKNOWN_MODULE
except BrowserPasswordExpired:
results['error_code'] = EXPIRED_PASSWORD
except BrowserQuestion:
results['error_code'] = BROWSER_QUESTION
except AuthMethodNotImplemented:
results['error_code'] = AUTH_METHOD_NYI
except ActionNeeded as exc:
# This `except` clause is not in alphabetic order and cannot be,
# because BrowserPasswordExpired and AuthMethodNotImplemented
# (above) inherits from it in Weboob 1.4.
results['error_code'] = ACTION_NEEDED
results['error_message'] = unicode(exc)
except BrowserIncorrectPassword:
# This `except` clause is not in alphabetic order and cannot be,
# because BrowserPasswordExpired (above) inherits from it in
# Weboob 1.3.
results['error_code'] = INVALID_PASSWORD
except Module.ConfigError as exc:
results['error_code'] = INVALID_PARAMETERS
results['error_message'] = unicode(exc)
except ConnectionError as exc:
results['error_code'] = CONNECTION_ERROR
results['error_message'] = unicode(exc)
except Exception as exc:
fail(
GENERIC_EXCEPTION,
'Unknown error: %s.' % unicode(exc),
traceback.format_exc()
)
# Return session information for future use.
results['session'] = self.storage.dump()
return results
class Connector():
"""
Connector is a tool that connects to common websites like bank website,
phone operator website... and that grabs personal data from there.
Credentials are required to make this operation.
Technically, connectors are weboob backend wrappers.
"""
@staticmethod
def version():
"""
Get the version of the installed Weboob.
"""
return Weboob.VERSION
def __init__(self, weboob_data_path, fakemodules_path, sources_list_content, is_prod):
"""
Create a Weboob instance.
:param weboob_data_path: Weboob path to use.
:param fakemodules_path: Path to the fake modules directory in user
data.
:param sources_list_content: Optional content of the sources.list file,
as an array of lines, or None if not present.
:param is_prod: whether we're running in production or not.
"""
# By default, consider we don't need to update the repositories.
self.needs_update = False
self.fakemodules_path = fakemodules_path
self.sources_list_content = sources_list_content
if not os.path.isdir(weboob_data_path):
os.makedirs(weboob_data_path)
# Set weboob data directory and sources.list file.
self.weboob_data_path = weboob_data_path
self.weboob_backup_path = os.path.normpath('%s.bak' % weboob_data_path)
self.write_weboob_sources_list()
# Create a Weboob object.
self.weboob = Weboob(workdir=weboob_data_path,
datadir=weboob_data_path)
self.backend = None
self.storage = None
# To make development more pleasant, always copy the fake modules in
# non-production modes.
if not is_prod:
self.copy_fakemodules()
# Update the weboob repos only if new repos are included.
if self.needs_update:
self.update()
def copy_fakemodules(self):
"""
Copies the fake modules files into the default fakemodules user-data
directory.
When Weboob updates modules, it might want to write within the
fakemodules directory, which might not be writable by the current
user. To prevent this, first copy the fakemodules directory in
a directory we have write access to, and then use that directory
in the sources list file.
"""
fakemodules_src = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fakemodules')
if os.path.isdir(self.fakemodules_path):
shutil.rmtree(self.fakemodules_path)
shutil.copytree(fakemodules_src, self.fakemodules_path)
def write_weboob_sources_list(self):
"""
Ensure the Weboob sources.list file contains the required entries from
Kresus.
"""
sources_list_path = os.path.join(self.weboob_data_path, 'sources.list')
# Determine the new content of the sources.list file.
new_sources_list_content = []
if self.sources_list_content is not None:
new_sources_list_content = self.sources_list_content
else:
# Default content of the sources.list file.
new_sources_list_content = [
unicode('https://updates.weboob.org/%(version)s/main/'),
unicode('file://%s' % self.fakemodules_path)
]
# Read the content of existing sources.list, if it exists.
original_sources_list_content = []
if os.path.isfile(sources_list_path):
with io.open(sources_list_path, encoding="utf-8") as fh:
original_sources_list_content = fh.read().splitlines()
# Update the source.list content and update the repository, only if the
# content has changed.
if set(original_sources_list_content) != set(new_sources_list_content):
with io.open(sources_list_path, 'w', encoding="utf-8") as sources_list_file:
sources_list_file.write('\n'.join(new_sources_list_content))
self.needs_update = True
def backup_data_dir(self):
"""
Backups modules.
"""
# shutil.copytree expects the destination path to not exist.
if os.path.isdir(self.weboob_backup_path):
shutil.rmtree(self.weboob_backup_path)
shutil.copytree(self.weboob_data_path, self.weboob_backup_path)
def restore_data_dir(self):
"""
Restores modules to their initial path.
"""
if os.path.isdir(self.weboob_backup_path):
# Ensure the target directory is clean.
if os.path.isdir(self.weboob_data_path):
shutil.rmtree(self.weboob_data_path)
# Replace the invalid data with the backup.
shutil.move(os.path.join(self.weboob_backup_path), self.weboob_data_path)
def clean_data_dir_backup(self):
"""
Cleans the backup.
"""
if os.path.isdir(self.weboob_backup_path):
shutil.rmtree(self.weboob_backup_path)
def update(self):
"""
Update Weboob modules.
"""
self.copy_fakemodules()
# Weboob has an offending print statement when it "Rebuilds index",
# which happen at every run if the user has a local repository. We need
# to silence it, hence the temporary redirect of stdout.
sys.stdout = open(os.devnull, "w")
# Create the backup before doing anything.
self.backup_data_dir()
try:
self.weboob.update(progress=DummyProgress())
except (ConnectionError, HTTPError) as exc:
# Do not delete the repository if there is a connection error or the repo has problems.
raise exc
except Exception:
# Try to remove the data directory, to see if it changes a thing.
# This is especially useful when a new version of Weboob is
# published and/or the keyring changes.
shutil.rmtree(self.weboob_data_path)
os.makedirs(self.weboob_data_path)
# Recreate the Weboob object as the directories are created
# on creating the Weboob object.
self.weboob = Weboob(workdir=self.weboob_data_path,
datadir=self.weboob_data_path)
# Rewrite sources.list file
self.write_weboob_sources_list()
# Retry update
try:
self.weboob.update(progress=DummyProgress())
except Exception as exc:
# If it still fails, just restore the previous state.
self.restore_data_dir()
# Re-throw the exception so that the user is warned of the problem.
raise exc
finally:
# Restore stdout
sys.stdout = sys.__stdout__
# Clean the backup.
self.clean_data_dir_backup()
def create_backend(self, modulename, parameters, session):
"""
Create a Weboob backend for a given module, ready to be used to fetch
data.
:param modulename: The name of the module from which backend should be
created.
:param parameters: A dict of parameters to pass to the module. It
should at least contain ``login`` and ``password`` fields, but can
contain additional values depending on the module.
:param session: an object representing the browser state.
"""
# Install the module if required.
repositories = self.weboob.repositories
minfo = repositories.get_module_info(modulename)
if (
minfo is not None and not minfo.is_installed() and
not minfo.is_local()
):
# We cannot install a locally available module, this would
# result in a ModuleInstallError.
try:
repositories.install(minfo, progress=DummyProgress())
except ModuleInstallError:
fail(
GENERIC_EXCEPTION,
"Unable to install module %s." % modulename,
traceback.format_exc()
)
# Initialize the Storage.
self.storage = DictStorage(session)
# Initialize the backend.
self.backend = self.weboob.build_backend(
modulename,
parameters,
storage=self.storage
)
def delete_backend(self):
"""
Delete a created backend for the given module.
"""
if self.backend:
with self.backend:
self.backend.deinit()
self.backend = None
self.storage = None
def get_accounts(self):
"""
Fetch accounts data from Weboob.
:param backend: The Weboob built backend to fetch data from.
:returns: A list of dicts representing the available accounts.
"""
results = []
with self.backend:
for account in list(self.backend.iter_accounts()):
# The minimum dict keys for an account are :
# 'id', 'label', 'balance' and 'type'
# Retrieve extra information for the account.
account = self.backend.fillobj(account, ['iban', 'currency'])
iban = None
if not empty(account.iban):
iban = account.iban
currency = None
if not empty(account.currency):
currency = unicode(account.currency)
results.append({
'vendorAccountId': account.id,
'label': account.label,
'balance': account.balance,
'iban': iban,
'currency': currency,
'type': account.type,
})
return results
def get_operations(self, from_date=None):
"""
Fetch operations data from Weboob.
:param from_date: The date until (in the past) which the transactions should be fetched.
Optional, if not provided all transactions are returned.
:returns: A list of dicts representing the available operations.
"""
results = []
with self.backend:
for account in list(self.backend.iter_accounts()):
# Get all operations for this account.
nyi_methods = []
operations = []
try:
for histop in self.backend.iter_history(account):
operations.append(histop)
# Ensure all the dates are datetime objects, so that we can compare them.
op_date = histop.date
if isinstance(op_date, date):
op_date = datetime(op_date.year, op_date.month, op_date.day)
op_rdate = histop.rdate
if isinstance(op_rdate, date):
op_rdate = datetime(op_rdate.year, op_rdate.month, op_rdate.day)
if op_rdate and op_rdate > op_date:
op_date = op_rdate
if from_date and op_date and op_date < from_date:
logging.debug(
'Stopped fetch because op date (%s) is before from_date (%s)',
op_date.isoformat(),
from_date.isoformat()
)
break
except NotImplementedError:
nyi_methods.append('iter_history')
try:
operations += [
op for op in self.backend.iter_coming(account)
if op.type in [
Transaction.TYPE_DEFERRED_CARD,
Transaction.TYPE_CARD_SUMMARY
]
]
except NotImplementedError:
nyi_methods.append('iter_coming')
for method_name in nyi_methods:
logging.error(
('%s not implemented for this account: %s.'),
method_name,
account.id
)
# Build an operation dict for each operation.
for operation in operations:
label = None
if not empty(operation.label):
label = unicode(operation.label)
raw_label = None
if not empty(operation.raw):
raw_label = unicode(operation.raw)
elif label:
raw_label = label
if raw_label and not label:
label = raw_label
# Handle date
if operation.rdate:
# Use date of the payment (real date) if available.
op_date = operation.rdate
elif operation.date:
# Otherwise, use debit date, on the bank statement.
op_date = operation.date
else:
logging.error(
'No known date property in operation line: %s.',
raw_label or "no label"
)
op_date = datetime.now()
isodate = op_date.isoformat()
debit_date = operation.date.isoformat()
results.append({
'account': account.id,
'amount': operation.amount,
'rawLabel': raw_label,
'type': operation.type,
'date': isodate,
'debit_date': debit_date,
'label': label
})
return results
def fetch(self, which, from_date=None):
"""
Wrapper to fetch data from the Weboob connector.
This wrapper fetches the required data from Weboob and returns it. It
handles the translation between Weboob exceptions and Kresus error
codes stored in the JSON response.
:param which: The type of data to fetch. Can be either ``accounts`` or
``operations``.
:param from_date: The date until (in the past) which the transactions should be fetched.
Optional, if not provided all transactions are returned.
:returns: A dict of the fetched data, in a ``values`` keys. Errors are
described under ``error_code``, ``error_short`` and ``error_message``
keys.
"""
results = {}
try:
if which == 'accounts':
results['values'] = self.get_accounts()
elif which == 'operations':
results['values'] = self.get_operations(from_date)
else:
raise Exception('Invalid fetch command.')
except NoAccountsException:
results['error_code'] = NO_ACCOUNTS
except ModuleLoadError:
results['error_code'] = UNKNOWN_MODULE
except BrowserPasswordExpired:
results['error_code'] = EXPIRED_PASSWORD
except BrowserQuestion as question:
results['action_kind'] = "browser_question"
# Fields are Weboob Value()s: has fields id/label?/description.
results['fields'] = [{"id": f.id, "label": f.label} for f in question.fields]
except AuthMethodNotImplemented:
results['error_code'] = AUTH_METHOD_NYI
except ActionNeeded as exc:
# This `except` clause is not in alphabetic order and cannot be,
# because BrowserPasswordExpired and AuthMethodNotImplemented
# (above) inherits from it in Weboob 1.4.
results['error_code'] = ACTION_NEEDED
results['error_message'] = unicode(exc)
except BrowserIncorrectPassword:
# This `except` clause is not in alphabetic order and cannot be,
# because BrowserPasswordExpired (above) inherits from it in
# Weboob 1.3.
results['error_code'] = INVALID_PASSWORD
except NeedInteractiveFor2FA:
results['error_code'] = REQUIRES_INTERACTIVE
except DecoupledValidation as validation:
results['action_kind'] = "decoupled_validation"
results['message'] = unicode(validation.message)
results['fields'] = []
except Module.ConfigError as exc:
results['error_code'] = INVALID_PARAMETERS
results['error_message'] = unicode(exc)
except ConnectionError as exc:
results['error_code'] = CONNECTION_ERROR
results['error_message'] = unicode(exc)
except Exception as exc:
fail(
GENERIC_EXCEPTION,
'Unknown error: %s.' % unicode(exc),
traceback.format_exc()
)
# Return session information for future use.
results['session'] = self.storage.dump()
return results
