def __init__(self):
logger.debug("class init")
self.peg_perco_names = []
self.weboob = Weboob()
self.weboob.load_backends(CapBank)
logger.debug('backends loaded: %s' % list(self.weboob.iter_backends()))
self.investments = {}
def retrieve_accounts(bank_name):
w = Weboob()
w.load_backends(caps=(CapBank, ))
bank_backend = w.get_backend(bank_name)
if bank_backend:
bank = Bank.get_or_create(name=bank_name)
accounts = bank_backend.iter_accounts()
for account in accounts:
logger.info(
f'[Accounts] Retrieving account {account.label} - {account.balance} from {bank_name}'
)
db_account = Account.get_or_none(bank=bank[0],
account_id=account.id)
if db_account is None:
db_account = Account.create(bank=bank[0],
account_id=account.id,
user=1,
label=account.label,
balance=account.balance)
else:
db_account.label = account.label
db_account.balance = account.balance
db_account.save()
transactions = bank_backend.iter_history(account)
for transaction in transactions:
db_transanction = Transaction.get_or_create(
account=db_account.id,
label=transaction.label,
category=transaction.category,
amount=transaction.amount,
date=transaction.date)
def get_app_dictionary(self):
if self.enabled:
# we wan't to update every VALUES_UPDATE_INTERVAL minutes
if self.status is None or timezone.now(
) >= self.last_activity + timedelta(
minutes=settings.VALUES_UPDATE_INTERVAL):
# Weboob 1.2 example http://dev.weboob.org/
weboob = Weboob()
weboob.load_backends(CapParcel)
for backend in list(weboob.iter_backends()):
if backend.name == self.parcel_carrier:
parcel = backend.get_parcel_tracking(self.parcel)
parcel_dict = parcel.to_dict()
self.status = parcel_dict['status']
if parcel_dict['arrival'] == 'Not loaded':
self.arrival = None
else:
pass
#parcel_dict['arrival']
self.url = parcel_dict['url']
self.info = parcel_dict['info']
self.save()
return {
'parcel': self.parcel,
'parcel_carrier': self.parcel_carrier,
'arrival': self.arrival,
'status': self.status,
'info': self.info
}
def update(self):
"""
Update Weboob modules.
"""
# Weboob has an offending print statement when it "Rebuilds index",
# which happen at every run if the user has a local repository. We need
# to silence it, hence the temporary redirect of stdout.
sys.stdout = open(os.devnull, "w")
try:
self.weboob.update(progress=DummyProgress())
except ConnectionError as exc:
# Do not delete the repository if there is a connection error.
raise exc
except Exception:
# Try to remove the data directory, to see if it changes a thing.
# This is especially useful when a new version of Weboob is
# published and/or the keyring changes.
shutil.rmtree(self.weboob_data_path)
os.makedirs(self.weboob_data_path)
# Recreate the Weboob object as the directories are created
# on creating the Weboob object.
self.weboob = Weboob(workdir=self.weboob_data_path,
datadir=self.weboob_data_path)
# Rewrite sources.list file
self.write_weboob_sources_list()
# Retry update
self.weboob.update(progress=DummyProgress())
finally:
# Restore stdout
sys.stdout = sys.__stdout__
def main(filename):
weboob = Weboob()
try:
hds = weboob.build_backend('hds')
except ModuleLoadError, e:
print >>sys.stderr, 'Unable to load "hds" module: %s' % e
return 1
def __init__(self, backend_name, download_directory, links_directory):
self.download_directory = download_directory
self.links_directory = links_directory
self.backend_name = backend_name
self.backend = None
self.weboob = Weboob()
self.weboob.load_backends(modules=[self.backend_name])
self.backend=self.weboob.get_backend(self.backend_name)
class BackendTest(TestCase):
MODULE = None
def __init__(self, *args, **kwargs):
super(BackendTest, self).__init__(*args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
# Skip tests when passwords are missing
self.weboob.requests.register('login', self.login_cb)
if self.weboob.load_backends(modules=[self.MODULE]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
def login_cb(self, backend_name, value):
raise SkipTest('missing config \'%s\' is required for this test' %
value.label)
def run(self, result):
"""
Call the parent run() for each backend instance.
Skip the test if we have no backends.
"""
# This is a hack to fix an issue with nosetests running
# with many tests. The default is 1000.
sys.setrecursionlimit(10000)
try:
if not len(self.backends):
self.backend = self.weboob.build_backend(self.MODULE,
nofail=True)
TestCase.run(self, result)
else:
# Run for all backend
for backend_instance in self.backends.keys():
print(backend_instance)
self.backend = self.backends[backend_instance]
TestCase.run(self, result)
finally:
self.weboob.deinit()
def shortDescription(self):
"""
Generate a description with the backend instance name.
"""
# do not use TestCase.shortDescription as it returns None
return '%s [%s]' % (str(self), self.backend_instance)
def is_backend_configured(self):
"""
Check if the backend is in the user configuration file
"""
return self.weboob.backends_config.backend_exists(
self.backend.config.instname)
def weboob_download(self, credentials, logs):
logger.info(
'Start weboob operations with module %s',
self.weboob_module_id.name)
w = Weboob()
back = w.build_backend(
self.weboob_module_id.name, params=credentials, name='odoo')
try:
sub = back.iter_subscription().next()
except BrowserIncorrectPassword:
logs['msg'].append(_('Wrong password.'))
logs['result'] = 'failure'
return []
bills = back.iter_bills(sub)
start_date = self.download_start_date
invoices = []
for bill in bills:
logger.debug('bill.id=%s, bill.fullid=%s', bill.id, bill.fullid)
inv_details = bill.to_dict()
logger.info('bill.to_dict=%s', inv_details)
# bill.to_dict=OrderedDict([
# ('id', u'60006530609_216421161'),
# ('url', u'https://api.bouyguestelecom.fr/comptes-facturat...'),
# ('date', date(2018, 7, 16)),
# ('format', u'pdf'),
# ('label', u'Juillet 2018'),
# ('type', u'bill'),
# ('transactions', []),
# ('price', Decimal('30.99')),
# ('currency', u'EUR'),
# ('vat', NotLoaded),
# ('duedate', NotLoaded), ('startdate', NotLoaded),
# ('finishdate', NotLoaded), ('income', False)])
# Do we have invoice number here ? NO
logger.info("Found invoice dated %s", inv_details.get('date'))
if (
start_date and
inv_details.get('date') and
fields.Date.to_string(inv_details['date']) < start_date):
logger.info(
'Skipping invoice %s dated %s dated before '
'download_start_date %s',
inv_details.get('label'), inv_details['date'], start_date)
continue
logger.info('Start to download bill with full ID %s', bill.fullid)
pdf_inv = back.download_document(bill.id)
filename = 'invoice_%s_%s.%s' % (
self.weboob_module_id.name,
inv_details.get('label') and
inv_details['label'].replace(' ', '_'),
inv_details.get('format', 'pdf'))
invoices.append((pdf_inv.encode('base64'), filename))
return invoices
def __init__(self, name, backend_name, my_download_directory,
my_links_directory):
self.download_directory = my_download_directory
self.links_directory = my_links_directory
self.backend_name = backend_name
self.backend = None
self.weboob = Weboob()
self.weboob.load_backends(modules=self.backend_name, )
self.backend = self.weboob.get_backend(self.backend_name)
self.name = name
class BackendTest(TestCase):
MODULE = None
def __init__(self, *args, **kwargs):
super(BackendTest, self).__init__(*args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
# Skip tests when passwords are missing
self.weboob.requests.register('login', self.login_cb)
if self.weboob.load_backends(modules=[self.MODULE]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
def login_cb(self, backend_name, value):
raise SkipTest('missing config \'%s\' is required for this test' % value.label)
def run(self, result):
"""
Call the parent run() for each backend instance.
Skip the test if we have no backends.
"""
# This is a hack to fix an issue with nosetests running
# with many tests. The default is 1000.
sys.setrecursionlimit(10000)
try:
if not len(self.backends):
self.backend = self.weboob.build_backend(self.MODULE, nofail=True)
TestCase.run(self, result)
else:
# Run for all backend
for backend_instance in self.backends.keys():
print(backend_instance)
self.backend = self.backends[backend_instance]
TestCase.run(self, result)
finally:
self.weboob.deinit()
def shortDescription(self):
"""
Generate a description with the backend instance name.
"""
# do not use TestCase.shortDescription as it returns None
return '%s [%s]' % (str(self), self.backend_instance)
def is_backend_configured(self):
"""
Check if the backend is in the user configuration file
"""
return self.weboob.backends_config.backend_exists(self.backend.config.instname)
def __init__(self, *args, **kwargs):
TestCase.__init__(self, *args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
if self.weboob.load_backends(modules=[self.BACKEND]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
# chose one backend (enough for most tests)
self.backend_instance = choice(self.backends.keys())
self.backend = self.backends[self.backend_instance]
def __init__(self, *args, **kwargs):
super(BackendTest, self).__init__(*args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
# Skip tests when passwords are missing
self.weboob.requests.register('login', self.login_cb)
if self.weboob.load_backends(modules=[self.MODULE]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
def get_accounts(bname):
w = Weboob()
w.load_backends(names=[bname])
backend = w.get_backend(bname)
results = {}
for account in backend.iter_accounts():
# a unicode bigger than 8 characters used as key of the table make some bugs in the C++ code
# convert to string before to use it
results[str(account.id)] = {'name': account.label,
'balance': int(account.balance * 100),
'type': int(account.type),
}
return results
def __init__(self):
self.ind = appindicator.Indicator.new(APPINDICATOR_ID,
os.path.abspath(resource_filename('boobank_indicator.data',
'indicator-boobank.png')),
appindicator.IndicatorCategory.APPLICATION_STATUS)
self.menu = Gtk.Menu()
self.ind.set_menu(self.menu)
logging.basicConfig()
if 'weboob_path' in os.environ:
self.weboob = Weboob(os.environ['weboob_path'])
else:
self.weboob = Weboob()
self.weboob.load_backends(CapBank)
def update_module_list(self):
w = Weboob()
logger.info('Weboob: calling update_repositories')
w.repositories.update_repositories()
weboob_modules = w.repositories.get_all_modules_info('CapDocument')
module_name2obj = {}
for module in self.search_read([], ['name']):
module_name2obj[module['name']] = self.browse(module['id'])
for name, info in weboob_modules.iteritems():
vals = {
'name': name,
'maintainer': info.maintainer,
'license': info.license,
'description': info.description,
'available_version': info.version,
'state': 'uninstalled',
}
if info.is_installed():
vals.update({
'has_parameters': self.has_parameter(w, name),
'state': 'installed',
'installed_version': info.version,
})
if name in module_name2obj:
module_name2obj[name].write(vals)
else:
self.create(vals)
def update(self):
"""
Update Weboob modules.
"""
self.copy_fakemodules()
# Weboob has an offending print statement when it "Rebuilds index",
# which happen at every run if the user has a local repository. We need
# to silence it, hence the temporary redirect of stdout.
sys.stdout = open(os.devnull, "w")
try:
self.weboob.update(progress=DummyProgress())
except ConnectionError as exc:
# Do not delete the repository if there is a connection error.
raise exc
except Exception:
# Try to remove the data directory, to see if it changes a thing.
# This is especially useful when a new version of Weboob is
# published and/or the keyring changes.
shutil.rmtree(self.weboob_data_path)
os.makedirs(self.weboob_data_path)
# Recreate the Weboob object as the directories are created
# on creating the Weboob object.
self.weboob = Weboob(workdir=self.weboob_data_path,
datadir=self.weboob_data_path)
# Rewrite sources.list file
self.write_weboob_sources_list()
# Retry update
self.weboob.update(progress=DummyProgress())
finally:
# Restore stdout
sys.stdout = sys.__stdout__
def get_accounts(bname):
w = Weboob()
w.load_backends(names=[bname])
backend = w.get_backend(bname)
results = {}
for account in backend.iter_accounts():
# a unicode bigger than 8 characters used as key of the table make some bugs in the C++ code
# convert to string before to use it
results[str(account.id)] = {
'name': account.label,
'balance': int(account.balance * 100),
'type': int(account.type),
}
return results
def update(self):
"""
Update Weboob modules.
"""
self.copy_fakemodules()
# Weboob has an offending print statement when it "Rebuilds index",
# which happen at every run if the user has a local repository. We need
# to silence it, hence the temporary redirect of stdout.
sys.stdout = open(os.devnull, "w")
# Create the backup before doing anything.
self.backup_data_dir()
try:
self.weboob.update(progress=DummyProgress())
except (ConnectionError, HTTPError) as exc:
# Do not delete the repository if there is a connection error or the repo has problems.
raise exc
except Exception:
# Try to remove the data directory, to see if it changes a thing.
# This is especially useful when a new version of Weboob is
# published and/or the keyring changes.
shutil.rmtree(self.weboob_data_path)
os.makedirs(self.weboob_data_path)
# Recreate the Weboob object as the directories are created
# on creating the Weboob object.
self.weboob = Weboob(workdir=self.weboob_data_path,
datadir=self.weboob_data_path)
# Rewrite sources.list file
self.write_weboob_sources_list()
# Retry update
try:
self.weboob.update(progress=DummyProgress())
except Exception as exc:
# If it still fails, just restore the previous state.
self.restore_data_dir()
# Re-throw the exception so that the user is warned of the problem.
raise exc
finally:
# Restore stdout
sys.stdout = sys.__stdout__
# Clean the backup.
self.clean_data_dir_backup()
def get_transactions(bname, accid, maximum):
w = Weboob()
w.load_backends(names=[bname])
backend = w.get_backend(bname)
acc = backend.get_account(accid)
results = {}
results['id'] = acc.id
results['name'] = acc.label
results['balance'] = int(acc.balance * 100)
results['type'] = int(acc.type)
results['transactions'] = []
try:
count = int(maximum)
if count < 1:
count = 0
except:
count = 0
i = 0
first = True
rewriteid = False
seen = set()
for tr in backend.iter_history(acc):
if first:
if tr.id == u'0' or tr.id == u'':
rewriteid = True
first = False
if rewriteid:
tr.id = tr.unique_id(seen)
t = {
'id': tr.id,
'date': tr.date.strftime('%Y-%m-%d'),
'rdate': tr.rdate.strftime('%Y-%m-%d'),
'type': int(tr.type),
'raw': tr.raw,
'category': tr.category,
'label': tr.label,
'amount': int(tr.amount * 100),
}
results['transactions'].append(t)
i += 1
if count != 0 and i >= count:
break
return results
def get_transactions(bname, accid, maximum):
w = Weboob()
w.load_backends(names=[bname])
backend = w.get_backend(bname)
acc = backend.get_account(accid)
results = {}
results['id'] = acc.id
results['name'] = acc.label
results['balance'] = int(acc.balance * 100)
results['type'] = int(acc.type)
results['transactions'] = []
try:
count = int(maximum)
if count < 1:
count = 0
except:
count = 0
i = 0
first = True
rewriteid = False
seen = set()
for tr in backend.iter_history(acc):
if first:
if tr.id == u'0' or tr.id == u'':
rewriteid = True
first = False
if rewriteid:
tr.id = tr.unique_id(seen)
t = {'id': tr.id,
'date': tr.date.strftime('%Y-%m-%d'),
'rdate': tr.rdate.strftime('%Y-%m-%d'),
'type': int(tr.type),
'raw': tr.raw,
'category': tr.category,
'label': tr.label,
'amount': int(tr.amount * 100),
}
results['transactions'].append(t)
i += 1
if count != 0 and i >= count:
break
return results
def __init__(self, *args, **kwargs):
TestCase.__init__(self, *args, **kwargs)
self.backend = None
self.weboob = Weboob()
if self.weboob.load_backends(modules=[self.BACKEND]):
self.backend = choice(self.weboob.backend_instances.values())
def _get_all_transactions(self):
weboob = Weboob()
backend = weboob.load_backends(
names=['bp'])['bp'] # TODO Make it generic
account_transactions = []
for account in backend.iter_accounts():
if account.label != 'COMPTE BANCAIRE': # TODO Make it generic
continue
for weboob_transaction in backend.iter_history(account):
transaction = Transaction(date=weboob_transaction.date,
amount=float(
weboob_transaction.amount),
label=weboob_transaction.label)
account_transactions.append(transaction)
return account_transactions
def __init__(self, backend_name, download_directory, links_directory):
self.download_directory = download_directory
self.links_directory = links_directory
self.backend_name = backend_name
self.backend = None
self.weboob = Weboob()
self.weboob.load_backends(modules=[self.backend_name])
self.backend=self.weboob.get_backend(self.backend_name)
def new_jobs():
w = Weboob()
w.load_backends(CapJob)
#w['popolemploi']
words = u'python'
jobs = w.search_job(words)
jobs = list(jobs)
Poleemploi.query.delete()
for job in jobs:
new = Poleemploi(society_name=job.society_name,
place=job.place,
title=job.title,
contract_type=job.contract_type,
publication_date=job.publication_date)
db.session.add(new)
db.session.commit()
return redirect(url_for('jobs_index'))
def __init__(self, weboob_data_path, fakemodules_path,
sources_list_content, is_prod):
"""
Create a Weboob instance.
:param weboob_data_path: Weboob path to use.
:param fakemodules_path: Path to the fake modules directory in user
data.
:param sources_list_content: Optional content of the sources.list file,
as an array of lines, or None if not present.
:param is_prod: whether we're running in production or not.
"""
# By default, consider we don't need to update the repositories.
self.needs_update = False
self.fakemodules_path = fakemodules_path
self.sources_list_content = sources_list_content
if not os.path.isdir(weboob_data_path):
os.makedirs(weboob_data_path)
# Set weboob data directory and sources.list file.
self.weboob_data_path = weboob_data_path
self.weboob_backup_path = os.path.normpath('%s.bak' % weboob_data_path)
self.write_weboob_sources_list()
# Create a Weboob object.
self.weboob = Weboob(workdir=weboob_data_path,
datadir=weboob_data_path)
self.backend = None
self.storage = None
# To make development more pleasant, always copy the fake modules in
# non-production modes.
if not is_prod:
self.copy_fakemodules()
# To make development more pleasant, always copy the fake modules in
# non-production modes.
if not is_prod:
self.copy_fakemodules()
# Update the weboob repos only if new repos are included.
if self.needs_update:
self.update()
def bank_connection_and_load_transactions(request, pk):
bank = get_object_or_404(Banks, pk=pk)
w = Weboob()
# TODO: Vérifier que les champs ne sont pas vide avant de lancer la connexion
# TODO: si erreur de connexion, alors raise une erreur et revenir vers une autre page ?
# TODO: affecter un owner_of_account lorsqu'un account est créé
if request.method == 'POST':
form = BankConnectionForm(instance=bank, data=request.POST)
if form.is_valid():
# TODO: vérifier que le mot de passe est identique à celui stocké en base if any (sous forme cryptée)
logger.warning(
'============= bank_connection_and_load_transactions __ bank.module_weboob.name_of_module = :: %s',
bank.module_weboob.name_of_module)
wb = w.load_backend(
bank.module_weboob.name_of_module, bank.name_of_bank, {
'login': form.cleaned_data['bank_login'],
'password': form.cleaned_data['bank_password']
})
# Get list of available account(s) in the bank
list_of_accounts = list(wb.iter_accounts())
# Get list of transactions coming from bank history
context = load_transactions(request, wb, bank, list_of_accounts)
return render(request,
'ManageGesfi/load_transactions_from_account.html',
context)
else:
form = BankConnectionForm(instance=bank)
context = {
'bank': bank,
'form': form,
}
return render(
request, 'banksandaccounts/bank_connection_and_load_transactions.html',
context)
def load_list_of_modules_in_database(request):
'''
function to load list of weboob modules in database
:param request:
:return:
'''
w = Weboob()
w.update()
listbanks = w.repositories.get_all_modules_info(CapBank)
db_wm_list = WeboobModules.objects.all()
list_of_db_modules = []
for key in db_wm_list:
list_of_db_modules.append(key.name_of_module)
list_of_banks = []
for key, val in listbanks.items():
wm = WeboobModules()
data_bank = {}
data_bank['module'] = key
wm.name_of_module = key
data_bank['description'] = val.description
wm.description_of_module = val.description
list_of_banks.append(data_bank)
if wm.name_of_module not in list_of_db_modules:
wm.save()
list_of_banks.sort(key=lambda k: k['module'])
db_wm_list = WeboobModules.objects.all()
# context = {'list_of_banks': list_of_banks}
context = {'list_of_banks': db_wm_list, 'load': True}
return render(request, 'ManageWeboob/list_of_available_modules.html', context)
class BackendTest(TestCase):
MODULE = None
def __init__(self, *args, **kwargs):
TestCase.__init__(self, *args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
if self.weboob.load_backends(modules=[self.MODULE]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
# chose one backend (enough for most tests)
self.backend_instance = choice(self.backends.keys())
self.backend = self.backends[self.backend_instance]
def run(self, result):
"""
Call the parent run() for each backend instance.
Skip the test if we have no backends.
"""
# This is a hack to fix an issue with nosetests running
# with many tests. The default is 1000.
sys.setrecursionlimit(10000)
try:
if not len(self.backends):
result.startTest(self)
result.stopTest(self)
raise SkipTest('No backends configured for this module.')
TestCase.run(self, result)
finally:
self.weboob.deinit()
def shortDescription(self):
"""
Generate a description with the backend instance name.
"""
# do not use TestCase.shortDescription as it returns None
return '%s [%s]' % (str(self), self.backend_instance)
def banks_supported():
"""
Return all the banks supported by weboob.
:return: [(module_name, Module_description)]
"""
banks_supported = []
weboob = Weboob()
modules = weboob.repositories.get_all_modules_info(CapBank)
for bank_module_name, info in modules.items():
banks_supported.append((bank_module_name, info.description))
return banks_supported
class Boobpeg:
""" Class that performs requests using weboob capabitilities """
def __init__(self):
logger.debug("class init")
self.peg_perco_names = []
self.weboob = Weboob()
self.weboob.load_backends(CapBank)
logger.debug('backends loaded: %s' % list(self.weboob.iter_backends()))
self.investments = {}
def set_peg_perco_names(self, peg_perco_names):
logger.debug("set_peg_perco_names(): peg_perco_names: %s" %
peg_perco_names)
self.peg_perco_names = peg_perco_names
def retrieve_investments(self, account):
logger.debug("retrieve_investments(): account: %s" % account)
account_investments = {}
for investment in list(self.weboob.iter_investment(account)):
account_investments[investment.code] = str(investment.unitvalue)
logger.debug("retrieve_investments(): return: %s" %
account_investments)
return account_investments
def update_investments(self):
logger.debug("update_investments()")
accounts = list(self.weboob.iter_accounts())
logger.debug("update_investments(): accounts: %s" % accounts)
for account in accounts:
if account.label in self.peg_perco_names:
logger.debug('update_investments(): account "%s" is in the '
'account list to check' % account.label)
self.investments.update(self.retrieve_investments(account))
else:
logger.debug('update_investments(): account "%s" is NOT in the'
' account list to check' % account.label)
logger.debug("update_investments(): return: %s" % self.investments)
return self.investments
def get_backends():
w = Weboob()
result = {}
for instance_name, name, params in sorted(
w.backends_config.iter_backends()):
module = w.modules_loader.get_or_load_module(name)
if not module.has_caps(CapBank):
continue
result[instance_name] = {'module': name}
return result
def __init__(self, *args, **kwargs):
super(BackendTest, self).__init__(*args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
# Skip tests when passwords are missing
self.weboob.requests.register('login', self.login_cb)
if self.weboob.load_backends(modules=[self.MODULE]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
def install(self):
w = Weboob()
for module in self:
if module.state == 'uninstalled':
logger.info('Starting to install weboob module %s',
module.name)
w.repositories.install(module.name)
has_parameter = self.has_parameter(w, module.name)
module.write({
'state': 'installed',
'installed_version': module.available_version,
'has_parameter': has_parameter,
})
logger.info('Weboob module %s is now installed', module.name)
def __init__(self, *args, **kwargs):
TestCase.__init__(self, *args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
if self.weboob.load_backends(modules=[self.BACKEND]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
# chose one backend (enough for most tests)
self.backend_instance = choice(self.backends.keys())
self.backend = self.backends[self.backend_instance]
class RboorrentDownload(object):
def __init__(self, _id, no_tracker):
self.id, self.backend_name = _id.split("@")
self.no_tracker = no_tracker
self.weboob = Weboob()
self.backend = self.weboob.load_backends(
modules=[self.backend_name])[self.backend_name]
def get_magnet(self, torrent):
if self.no_tracker:
return "&".join([
_ for _ in torrent.magnet.split("&") if not _.startswith("tr=")
])
else:
return torrent.magnet
def write_meta(self, torrent):
dest = "meta-%s-%s.torrent" % (torrent.id, torrent.name)
magnet = self.get_magnet(torrent)
buf = "d10:magnet-uri%d:%se" % (len(magnet), magnet)
try:
with open(dest, 'w') as f:
f.write(buf)
except IOError as e:
print('Unable to write "%s": %s' % (dest, e.message))
def write_torrent(self, torrent):
dest = "%s-%s.torrent" % (torrent.id, torrent.name)
try:
buf = self.backend.get_torrent_file(torrent.id)
if buf:
try:
with open(dest, 'w') as f:
f.write(buf)
except IOError as e:
print('Unable to write "%s": %s' % (dest, e))
except Exception as e:
print("Could not get torrent file for %s@%s" %
(self.id, self.backend_name))
def run(self):
try:
torrent = self.backend.get_torrent(self.id)
if torrent.magnet:
self.write_meta(torrent)
else:
self.write_torrent(torrent)
except HTTPNotFound:
print("Could not find %s@%s" % (self.id, self.backend_name))
def bank_create_with_weboob_module(request, pk):
w = Weboob()
if request.method == 'POST':
form = BankForm(data=request.POST)
if form.is_valid():
bank = form.save(commit=False)
bank.bank_password = make_password(
form.cleaned_data['bank_password'])
bank.save()
w.load_backend(
bank.module_weboob.name_of_module, bank.name_of_bank, {
'login': form.cleaned_data['bank_login'],
'password': form.cleaned_data['bank_password']
})
# Get list of available account(s) in the bank
list_of_accounts = list(w.iter_accounts())
# Get list of transactions coming from bank history
context = load_transactions(request, w, bank, list_of_accounts)
return render(request,
'ManageGesfi/load_transactions_from_account.html',
context)
else:
form = BankForm()
context = {
'form': form,
'create': True
# TODO: ajouter un flag "create with weboob" pour rediriger vers load_transactions (et créer les comptes)
}
return render(request, 'banksandaccounts/bank_edit.html', context)
def __init__(self, weboob_data_path):
"""
Create a Weboob instance.
:param weboob_data_path: Weboob path to use.
"""
# By default, consider we don't need to update the repositories.
self.needs_update = False
if not os.path.isdir(weboob_data_path):
os.makedirs(weboob_data_path)
# Set weboob data directory and sources.list file.
self.weboob_data_path = weboob_data_path
self.write_weboob_sources_list()
# Create a Weboob object.
self.weboob = Weboob(workdir=weboob_data_path,
datadir=weboob_data_path)
self.backends = collections.defaultdict(dict)
# Update the weboob repos only if new repos are included.
if self.needs_update:
self.update()
class RboorrentDownload(object):
def __init__(self, _id, no_tracker):
self.id, self.backend_name = _id.split("@")
self.no_tracker = no_tracker
self.weboob = Weboob()
self.backend = self.weboob.load_backends(modules=[self.backend_name])[self.backend_name]
def get_magnet(self, torrent):
if self.no_tracker:
return "&".join([_ for _ in torrent.magnet.split("&") if not _.startswith("tr=")])
else:
return torrent.magnet
def write_meta(self, torrent):
dest = "meta-%s-%s.torrent" % (torrent.id, torrent.name)
magnet = self.get_magnet(torrent)
buf = "d10:magnet-uri%d:%se" % (len(magnet), magnet)
try:
with open(dest, 'w') as f:
f.write(buf)
except IOError as e:
print('Unable to write "%s": %s' % (dest, e.message))
def write_torrent(self, torrent):
dest = "%s-%s.torrent" % (torrent.id, torrent.name)
try:
buf = self.backend.get_torrent_file(torrent.id)
if buf:
try:
with open(dest, 'w') as f:
f.write(buf)
except IOError as e:
print('Unable to write "%s": %s' % (dest, e))
except Exception as e:
print("Could not get torrent file for %s@%s" % (self.id, self.backend_name))
def run(self):
try:
torrent = self.backend.get_torrent(self.id)
if torrent.magnet:
self.write_meta(torrent)
else:
self.write_torrent(torrent)
except HTTPNotFound:
print("Could not find %s@%s" % (self.id, self.backend_name))
class BackendTest(TestCase):
BACKEND = None
def __init__(self, *args, **kwargs):
TestCase.__init__(self, *args, **kwargs)
self.backend = None
self.weboob = Weboob()
if self.weboob.load_backends(modules=[self.BACKEND]):
self.backend = choice(self.weboob.backend_instances.values())
def run(self, result):
if not self.backend:
result.startTest(self)
result.stopTest(self)
raise SkipTest()
return TestCase.run(self, result)
def weboob_module_id_change(self):
if self.weboob_module_id and self.weboob_module_id.has_parameters:
w = Weboob()
bmod = w.modules_loader.get_or_load_module(
self.weboob_module_id.name)
new_params = self.env['weboob.parameter']
for key, value in bmod.config.iteritems():
if key not in ['login', 'password']:
note = value.label or ''
if value.choices:
options = ', '.join([
"'%s' (%s)" % (key_opt, help_opt)
for (key_opt, help_opt) in value.choices.items()])
note = _("%s. Possible values: %s.") % (note, options)
param = new_params.new({
'key': key,
'note': note,
})
new_params += param
self.weboob_parameter_ids = new_params
def __init__(self, weboob_data_path, fakemodules_path, sources_list_content, is_prod):
"""
Create a Weboob instance.
:param weboob_data_path: Weboob path to use.
:param fakemodules_path: Path to the fake modules directory in user
data.
:param sources_list_content: Optional content of the sources.list file,
as an array of lines, or None if not present.
:param is_prod: whether we're running in production or not.
"""
# By default, consider we don't need to update the repositories.
self.needs_update = False
self.fakemodules_path = fakemodules_path
self.sources_list_content = sources_list_content
if not os.path.isdir(weboob_data_path):
os.makedirs(weboob_data_path)
# Set weboob data directory and sources.list file.
self.weboob_data_path = weboob_data_path
self.write_weboob_sources_list()
# Create a Weboob object.
self.weboob = Weboob(workdir=weboob_data_path,
datadir=weboob_data_path)
self.backend = None
self.storage = None
# To make development more pleasant, always copy the fake modules in
# non-production modes.
if not is_prod:
self.copy_fakemodules()
# Update the weboob repos only if new repos are included.
if self.needs_update:
self.update()
class BoobankChecker():
def __init__(self):
self.ind = appindicator.Indicator.new(APPINDICATOR_ID,
os.path.abspath(resource_filename('boobank_indicator.data',
'indicator-boobank.png')),
appindicator.IndicatorCategory.APPLICATION_STATUS)
self.menu = Gtk.Menu()
self.ind.set_menu(self.menu)
logging.basicConfig()
if 'weboob_path' in os.environ:
self.weboob = Weboob(os.environ['weboob_path'])
else:
self.weboob = Weboob()
self.weboob.load_backends(CapBank)
def clean_menu(self, menu):
for i in menu.get_children():
submenu = i.get_submenu()
if submenu:
self.clean_menu(i)
menu.remove(i)
def check_boobank(self):
self.ind.set_status(appindicator.IndicatorStatus.ACTIVE)
self.clean_menu(self.menu)
total = 0
currency = ''
threads = []
try:
for account in self.weboob.do('iter_accounts'):
balance = account.balance
if account.coming:
balance += account.coming
if account.type != Account.TYPE_LOAN:
total += balance
image = "green_light.png" if balance > 0 else "red_light.png"
else:
image = "personal-loan.png"
currency = account.currency_text
label = "%s: %s%s" % (account.label, balance, account.currency_text)
account_item = create_image_menu_item(label, image)
thread = BoobankTransactionsChecker(self.weboob, account_item, account)
thread.start()
threads.append(thread)
except CallErrors as errors:
self.bcall_errors_handler(errors)
for thread in threads:
thread.join()
for thread in threads:
self.menu.append(thread.menu)
thread.menu.show()
if len(self.menu.get_children()) == 0:
Notify.Notification.new('<b>Boobank</b>',
'No Bank account found\n Please configure one by running boobank',
'notification-message-im').show()
sep = Gtk.SeparatorMenuItem()
self.menu.append(sep)
sep.show()
total_item = Gtk.MenuItem("%s: %s%s" % ("Total", total, currency))
self.menu.append(total_item)
total_item.show()
sep = Gtk.SeparatorMenuItem()
self.menu.append(sep)
sep.show()
btnQuit = Gtk.ImageMenuItem()
image = Gtk.Image()
image.set_from_stock(Gtk.STOCK_QUIT, Gtk.IconSize.BUTTON)
btnQuit.set_image(image)
btnQuit.set_label('Quit')
btnQuit.set_always_show_image(True)
btnQuit.connect("activate", self.quit)
self.menu.append(btnQuit)
btnQuit.show()
def quit(self, widget):
Gtk.main_quit()
def bcall_errors_handler(self, errors):
"""
Handler for the CallErrors exception.
"""
self.ind.set_status(appindicator.IndicatorStatus.ATTENTION)
for backend, error, backtrace in errors.errors:
notify = True
if isinstance(error, BrowserIncorrectPassword):
msg = 'invalid login/password.'
elif isinstance(error, BrowserSSLError):
msg = '/!\ SERVER CERTIFICATE IS INVALID /!\\'
elif isinstance(error, BrowserForbidden):
msg = unicode(error) or 'Forbidden'
elif isinstance(error, BrowserUnavailable):
msg = unicode(error)
if not msg:
msg = 'website is unavailable.'
elif isinstance(error, NotImplementedError):
notify = False
elif isinstance(error, UserError):
msg = unicode(error)
elif isinstance(error, MoreResultsAvailable):
notify = False
else:
msg = unicode(error)
if notify:
Notify.Notification.new('<b>Error Boobank: %s</b>' % backend.name,
msg,
'notification-message-im').show()
def main(self):
self.check_boobank()
GObject.timeout_add(PING_FREQUENCY * 1000, self.check_boobank)
Gtk.main()
def main(filename):
weboob = Weboob()
try:
hds = weboob.build_backend('hds')
except ModuleLoadError as e:
print('Unable to load "hds" module: %s' % e, file=sys.stderr)
return 1
try:
db = sqlite.connect(database=filename, timeout=10.0)
except sqlite.OperationalError as err:
print('Unable to open %s database: %s' % (filename, err), file=sys.stderr)
return 1
sys.stdout.write('Reading database... ')
sys.stdout.flush()
try:
results = db.execute('SELECT id, author FROM stories')
except sqlite.OperationalError as err:
print('fail!\nUnable to read database: %s' % err, file=sys.stderr)
return 1
stored = set()
authors = set()
for r in results:
stored.add(r[0])
authors.add(r[1])
stored_authors = set([s[0] for s in db.execute('SELECT name FROM authors')])
sys.stdout.write('ok\n')
br = hds.browser
to_fetch = set()
sys.stdout.write('Getting stories list from website... ')
sys.stdout.flush()
for story in br.iter_stories():
if int(story.id) in stored:
break
to_fetch.add(story.id)
authors.add(story.author.name)
sys.stdout.write(' ok\n')
sys.stdout.write('Getting %d new storiese... ' % len(to_fetch))
sys.stdout.flush()
for id in to_fetch:
story = br.get_story(id)
if not story:
logging.warning('Story #%d unavailable' % id)
continue
db.execute("""INSERT INTO stories (id, title, date, category, author, body)
VALUES (?, ?, ?, ?, ?, ?)""",
(story.id, story.title, story.date, story.category,
story.author.name, story.body))
db.commit()
sys.stdout.write('ok\n')
authors = authors.difference(stored_authors)
sys.stdout.write('Getting %d new authors... ' % len(authors))
sys.stdout.flush()
for a in authors:
author = br.get_author(a)
if not author:
logging.warning('Author %s unavailable\n' % id)
continue
db.execute("INSERT INTO authors (name, sex, description) VALUES (?, ?, ?)",
(a, author.sex, author.description))
db.commit()
sys.stdout.write(' ok\n')
return 0
class MyThread(Thread):
daemon = True
def __init__(self, bot):
Thread.__init__(self)
self.weboob = Weboob(storage=StandardStorage(STORAGE_FILE))
self.weboob.load_backends()
self.bot = bot
self.bot.set_weboob(self.weboob)
def run(self):
for ev in self.bot.joined.itervalues():
ev.wait()
self.weboob.repeat(300, self.check_board)
self.weboob.repeat(600, self.check_dlfp)
self.weboob.repeat(600, self.check_twitter)
self.weboob.loop()
def find_keywords(self, text):
for word in [
'weboob', 'videoob', 'havesex', 'havedate', 'monboob', 'boobmsg',
'flatboob', 'boobill', 'pastoob', 'radioob', 'translaboob', 'traveloob', 'handjoob',
'boobathon', 'boobank', 'boobtracker', 'comparoob', 'wetboobs',
'webcontentedit', 'weboorrents', 'assnet', 'budget insight', 'budget-insight', 'budgetinsight', 'budgea']:
if word in text.lower():
return word
return None
def check_twitter(self):
nb_tweets = 10
for backend in self.weboob.iter_backends(module='twitter'):
for thread in list(itertools.islice(backend.iter_resources(None, ['search', 'weboob']),
0,
nb_tweets)):
if not backend.storage.get('lastpurge'):
backend.storage.set('lastpurge', datetime.now() - timedelta(days=60))
backend.storage.save()
if thread.id not in backend.storage.get('seen', default={}) and\
thread.date > backend.storage.get('lastpurge'):
_item = thread.id.split('#')
url = 'https://twitter.com/%s/status/%s' % (_item[0], _item[1])
for msg in self.bot.on_url(url):
self.bot.send_message('%s: %s' % (_item[0], url))
self.bot.send_message(msg)
backend.set_message_read(backend.fill_thread(thread, ['root']).root)
def check_dlfp(self):
for msg in self.weboob.do('iter_unread_messages', backends=['dlfp']):
word = self.find_keywords(msg.content)
if word is not None:
url = msg.signature[msg.signature.find('https://linuxfr'):]
self.bot.send_message('[DLFP] %s talks about %s: %s' % (
msg.sender, word, url))
self.weboob[msg.backend].set_message_read(msg)
def check_board(self):
def iter_messages(backend):
with backend.browser:
return backend.browser.iter_new_board_messages()
for msg in self.weboob.do(iter_messages, backends=['dlfp']):
word = self.find_keywords(msg.message)
if word is not None and msg.login != 'moules':
message = msg.message.replace(word, '\002%s\002' % word)
self.bot.send_message('[DLFP] <%s> %s' % (msg.login, message))
def stop(self):
self.weboob.want_stop()
self.weboob.deinit()
def __init__(self, bot):
Thread.__init__(self)
self.weboob = Weboob(storage=StandardStorage(STORAGE_FILE))
self.weboob.load_backends()
self.bot = bot
self.bot.set_weboob(self.weboob)
class DownloadBoob(object):
"""
Search and download
:param name:
:param backend_name:
:param my_download_directory:
:param my_links_directory:
"""
def __init__(self, name, backend_name, my_download_directory,
my_links_directory):
self.download_directory = my_download_directory
self.links_directory = my_links_directory
self.backend_name = backend_name
self.backend = None
self.weboob = Weboob()
self.weboob.load_backends(modules=self.backend_name, )
self.backend = self.weboob.get_backend(self.backend_name)
self.name = name
def get_filename(self, video, relative=False, m3u=False):
"""
Generate filename for the video
:param relative:
:param m3u:
:param video:
:rtype : string
"""
if relative:
directory = os.path.join("..", DOWNLOAD_DIRECTORY,
self.backend_name)
else:
directory = os.path.join(self.download_directory, self.backend_name)
if not os.path.exists(directory.encode('utf8')):
os.makedirs(directory.encode('utf8'))
if not m3u:
ext = video.ext
if not ext:
ext = 'avi'
else:
ext = 'm3u'
return "%s/%s.%s" % (directory, removenonascii(video.id), ext)
def get_linkname(self, video, m3u=False):
"""
Generate filename for the link
:param m3u:
:param video:
:rtype : string
"""
if not os.path.exists(self.links_directory.encode('utf8')):
os.makedirs(self.links_directory.encode('utf8'))
if not m3u:
ext = video.ext
if not ext:
ext = 'avi'
else:
ext = 'm3u'
if not kodi:
misc = video.date.strftime("%y-%m-%d")
if not misc:
misc = video.id
return "%s/%s (%s).%s" % (self.links_directory,
removespecial(video.title),
removespecial(misc), ext)
else:
return "%s/%s.%s" % (self.links_directory,
removespecial(video.title), ext)
def is_downloaded(self, video):
"""
Check if the video has already been downloaded
:param video:
:rtype : bool
"""
if (os.path.isfile(self.get_filename(video).encode('utf8')) or
os.path.isfile(self.get_filename(video,
m3u=True).encode('utf8'))):
logging.info("%s Already downloaded : %s" %
(video.id, video.title))
return True
logging.debug("%s To be downloaded : %s" %
(video.id, video.title))
return False
def init_dir(self):
"""
create directory
"""
if not os.path.isdir(self.links_directory.encode('utf8')):
logging.debug(" create link directory : %s" % self.links_directory)
os.makedirs(self.links_directory.encode('utf8'))
else:
logging.debug(" existing link directory : %s" % self.links_directory)
if kodi:
file_name = os.path.join(self.links_directory, 'tvshow.nfo')
show_name = self.links_directory.split("/")[-1]
if not os.path.isfile(file_name.encode('utf8')):
logging.debug(" create %s" % file_name)
f = codecs.open(file_name, "w", "utf-8")
f.write(u"<tvshow>\n")
f.write(u" <title>" + show_name + u"</title>\n")
f.write(u"</tvshow>\n")
f.close()
else:
logging.debug(" existing %s" % file_name)
def do_search(self,
pattern=None,
pattern_type="search",
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False):
"""
Search for videos
:param pattern:
:param pattern_type:
:param sortby:
:param nsfw:
:return:
"""
logging.debug(" Searching for videos for %s" % self.name)
list_videos = []
if pattern_type == "search":
list_videos = self.backend.search_videos(pattern, sortby, nsfw)
elif pattern_type == "ls":
sys.path.insert(
0, backend_directory + "/" + self.backend.name + "/"
) # HACK
if 'video' in sys.modules:
del sys.modules['video']
if self.backend.name == "arte":
from video import ArteVideo as Video_Init
elif self.backend.name == "canalplus":
from video import CanalplusVideo as Video_Init
elif self.backend.name == "arretsurimages":
from video import ArretSurImagesVideo as Video_Init
elif self.backend.name == "dailymotion":
from video import DailymotionVideo as Video_Init
elif self.backend.name == "nolifetv":
from video import NolifeTVVideo as Video_Init
elif self.backend.name == "youtube":
from video import YoutubeVideo as Video_Init
else:
from weboob.capabilities.video import BaseVideo as Video_Init # END OF HACK
for videoid in videoob_list_rep(pattern, self.backend):
list_videos.append(Video_Init(videoid))
if list_videos:
logging.debug(" found videos for %s" % self.name)
else:
logging.error(" did not found videos for %s" % self.name)
return list_videos
def filter_list(self, list_videos, title_regexp, id_regexp, author_regexp,
title_exclude, max_results):
"""
Filter the list after the search
:param list_videos:
:param title_regexp:
:param id_regexp:
:param author_regexp:
:param title_exclude:
:param max_results:
:return:
"""
logging.debug(" filtering list of found video for %s" % self.name)
videos = []
num_videos = 0
for video in list_videos:
if is_ok(video, title_regexp, id_regexp, author_regexp,
title_exclude):
if not self.is_downloaded(video):
videoob_get_info(self.backend, video)
if not video:
logging.error('Error in Video: %s' % video)
elif not video.url:
logging.error(
'Error: the URL is not available : %s (%s)' %
(video.url, video.id))
else:
if is_ok(video, title_regexp, id_regexp, author_regexp,
title_exclude):
num_videos += 1
if not self.is_downloaded(video):
videos.append(video)
print("New Video : %s" % video.title)
print(" Description:%s" % video.description)
print(" Author:%s" % video.author)
print(" Id:%s" % video.id)
print(" Duration:%s" % video.duration)
print(" Date:%s" % video.date)
if num_videos == max_results:
break
return videos
def write_m3u(self, video):
"""
Write m3u file for streaming files
:param video:
:return:
"""
logging.debug(" Write m3u for %s" % video.title)
if video.ext == "m3u" or video.ext == "m3u8":
return do_download(video, self.get_filename(video))
else:
if matched(video.url, "\.m3u") or matched(video.url, "\.m3u8"):
return do_download(video, self.get_filename(video))
else:
dest = self.get_filename(video, m3u=True)
show_name = self.links_directory.split("/")[-1]
logging.debug(" create %s" % dest)
f = codecs.open(dest, "w", "utf-8")
f.write("#EXTINF: ")
if video.duration:
f.write(str(video.duration))
else:
f.write(str(-1))
f.write(", " + show_name + " - " + video.title + "\n")
f.write(video.url)
f.close()
return 0
def set_link(self, video, m3u=False):
"""
Create link file
:param video:
:param m3u:
"""
linkname = self.get_linkname(video, m3u=m3u)
idname = self.get_filename(video, relative=True, m3u=m3u)
absolute_idname = self.get_filename(video, m3u=m3u)
if not os.path.islink(linkname.encode('utf8')) and os.path.isfile(absolute_idname.encode('utf8')):
logging.info(" %s -> %s" % (linkname, idname))
os.symlink(idname.encode('utf8'), linkname.encode('utf8'))
else:
logging.debug(" Not generating link for %s" % video.title)
def do_mv(self, video, m3u=False):
"""
move video file after download
:param video:
:param m3u:
"""
linkname = self.get_linkname(video, m3u=m3u)
absolute_idname = self.get_filename(video, m3u=m3u)
if not os.path.isfile(linkname.encode('utf8')) and os.path.isfile(absolute_idname.encode('utf8')):
logging.info(" %s => %s" % (absolute_idname, linkname))
os.rename(absolute_idname.encode('utf8'), linkname.encode('utf8'))
open(absolute_idname, 'w').close()
else:
logging.debug(" Not moving file %s" % linkname)
def download(self,
pattern=None,
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False,
max_results=20,
title_regexp=None,
id_regexp=None,
pattern_type="search",
author_regexp=None,
title_exclude=None):
# create directory for links
"""
Main process
:param pattern:
:param sortby:
:param nsfw:
:param max_results:
:param title_regexp:
:param id_regexp:
:param pattern_type:
:param author_regexp:
:param title_exclude:
"""
self.init_dir()
# search for videos
list_videos = self.do_search(pattern, pattern_type, sortby, nsfw)
# Filter the list of videos
videos = self.filter_list(list_videos, title_regexp, id_regexp,
author_regexp, title_exclude, max_results)
# download videos
if videos:
for video in videos:
print("Downloading... " + video.title)
if kodi: # THE "TITLE" BECOME "S00E00 - TITLE (ID)"
rewrite_title(video)
if down_live: # CREATE LIVE STREAM
ret = self.write_m3u(video)
else: # DOWNLOAD VIDEO
ret = do_download(video, self.get_filename(video)
) # FOR DIRECT LINKS
if not ret:
ret = do_conv(video, self.get_filename(video)
) # FOR INDIRECT LINKS
if not ret:
if not kodi:
self.set_link(
video
) # CREATE LINKS FOR A BEAUTIFULL LIBRARY
else:
self.do_mv(video) # MOVE FILES FOR A BEAUTIFULL LIBRARY
# CREATE NFO FILES FOR KODI
write_nfo(self.get_linkname(video),
self.links_directory, self.backend_name,
video)
print("Downloaded : " + video.title)
else:
assert isinstance(video.title, basestring)
print("Failed download :" + video.title)
class Downloadboob(object):
def __init__(self, backend_name, download_directory, links_directory):
self.download_directory = download_directory
self.links_directory = links_directory
self.backend_name = backend_name
self.backend = None
self.weboob = Weboob()
self.weboob.load_backends(modules=[self.backend_name])
self.backend=self.weboob.get_backend(self.backend_name)
def purge(self):
if not os.path.isdir(self.links_directory):
return
dirList=os.listdir(self.links_directory)
for local_link_name in dirList:
link_name = self.links_directory + "/" + local_link_name
if not self.check_link(link_name):
print u"Remove %s" % link_name
os.remove(link_name)
else:
print u"Keep %s" % link_name
def check_link(self, link_name):
if os.path.islink(link_name):
file_name = os.readlink(link_name)
absolute_file_name = os.path.join(self.links_directory, file_name)
if os.path.isfile(absolute_file_name):
return True
return False
else:
return True
def download(self, pattern=None, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None, title_exclude=[], id_regexp=None):
print "For backend %s, search for '%s'" % (backend_name, pattern)
# create directory for links
print " create link to %s" % self.links_directory
if not os.path.isdir(self.links_directory):
os.makedirs(self.links_directory)
# search for videos
videos = []
for i, video in enumerate(self.backend.search_videos(pattern, sortby, nsfw)):
if i == max_results:
break
if not self.is_downloaded(video):
self.backend.fill_video(video, ('url','title', 'url', 'duration'))
if not(self.is_excluded(video.title, title_exclude)) and self.id_regexp_matched(video.id, id_regexp):
print " %s\n Id:%s\n Duration:%s" % (video.title, video.id, video.duration)
videos.append(video)
else:
print "Already downloaded, check %s" % video.id
self.backend.fill_video(video, ('url','title', 'url', 'duration'))
linkname = self.get_linkname(video)
if not os.path.exists(linkname):
self.remove_download(video)
# download videos
print "Downloading..."
for video in videos:
self.do_download(video)
def is_excluded(self, title, title_exclude):
for exclude in title_exclude:
if title.find(exclude) > -1:
return True
return False
def id_regexp_matched(self, video_id, id_regexp):
if id_regexp:
return re.search(id_regexp, video_id) is not None
return True
def get_filename(self, video, relative=False):
if relative:
directory = os.path.join("..", DOWNLOAD_DIRECTORY, self.backend_name)
else:
directory = os.path.join(self.download_directory, self.backend_name)
if not os.path.exists(directory):
os.makedirs(directory)
ext = video.ext
if not ext:
ext = 'avi'
return u"%s/%s.%s" % (directory, removeNonAscii(video.id), ext)
def get_linkname(self, video):
if not os.path.exists(self.links_directory):
os.makedirs(self.links_directory)
ext = video.ext
if not ext:
ext = 'avi'
misc = video.date
if not misc:
misc = video.id
return u"%s/%s (%s).%s" % (self.links_directory, removeSpecial(video.title), removeSpecial(misc), ext)
def is_downloaded(self, video):
# check if the file is 0 byte
return os.path.isfile(self.get_filename(video))
def remove_download(self, video):
path = self.get_filename(video)
if os.stat(path).st_size == 0:
# already empty
return
print 'Remove video %s' % video.title
# Empty it to keep information we have already downloaded it.
with open(path, 'w'):
pass
def set_linkname(self, video):
linkname = self.get_linkname(video)
idname = self.get_filename(video, relative=True)
absolute_idname = self.get_filename(video, relative=False)
if not os.path.islink(linkname) and os.path.isfile(absolute_idname):
print "%s -> %s" % (linkname, idname)
os.symlink(idname, linkname)
def do_download(self, video):
if not video:
print >>sys.stderr, 'Video not found: %s' % video
return 3
if not video.url:
print >>sys.stderr, 'Error: the direct URL is not available.'
return 4
def check_exec(executable):
with open('/dev/null', 'w') as devnull:
process = subprocess.Popen(['which', executable], stdout=devnull)
if process.wait() != 0:
print >>sys.stderr, 'Please install "%s"' % executable
return False
return True
dest = self.get_filename(video)
if video.url.startswith('rtmp'):
if not check_exec('rtmpdump'):
return 1
args = ('rtmpdump', '-e', '-r', video.url, '-o', dest)
elif video.url.startswith('mms'):
if not check_exec('mimms'):
return 1
args = ('mimms', video.url, dest)
else:
if not check_exec('wget'):
return 1
args = ('wget', video.url, '-O', dest)
os.spawnlp(os.P_WAIT, args[0], *args)
self.set_linkname(video)
def __init__(self, _id, no_tracker):
self.id, self.backend_name = _id.split("@")
self.no_tracker = no_tracker
self.weboob = Weboob()
self.backend = self.weboob.load_backends(modules=[self.backend_name])[self.backend_name]
class Connector(object):
"""
Connector is a tool that connects to common websites like bank website,
phone operator website... and that grabs personal data from there.
Credentials are required to make this operation.
Technically, connectors are weboob backend wrappers.
"""
@staticmethod
def version():
"""
Get the version of the installed Weboob.
"""
return Weboob.VERSION
def __init__(self, weboob_data_path, fakemodules_path, sources_list_content, is_prod):
"""
Create a Weboob instance.
:param weboob_data_path: Weboob path to use.
:param fakemodules_path: Path to the fake modules directory in user
data.
:param sources_list_content: Optional content of the sources.list file,
as an array of lines, or None if not present.
:param is_prod: whether we're running in production or not.
"""
# By default, consider we don't need to update the repositories.
self.needs_update = False
self.fakemodules_path = fakemodules_path
self.sources_list_content = sources_list_content
if not os.path.isdir(weboob_data_path):
os.makedirs(weboob_data_path)
# Set weboob data directory and sources.list file.
self.weboob_data_path = weboob_data_path
self.write_weboob_sources_list()
# Create a Weboob object.
self.weboob = Weboob(workdir=weboob_data_path,
datadir=weboob_data_path)
self.backend = None
self.storage = None
# To make development more pleasant, always copy the fake modules in
# non-production modes.
if not is_prod:
self.copy_fakemodules()
# Update the weboob repos only if new repos are included.
if self.needs_update:
self.update()
def copy_fakemodules(self):
"""
Copies the fake modules files into the default fakemodules user-data
directory.
When Weboob updates modules, it might want to write within the
fakemodules directory, which might not be writable by the current
user. To prevent this, first copy the fakemodules directory in
a directory we have write access to, and then use that directory
in the sources list file.
"""
fakemodules_src = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fakemodules')
if os.path.isdir(self.fakemodules_path):
shutil.rmtree(self.fakemodules_path)
shutil.copytree(fakemodules_src, self.fakemodules_path)
def write_weboob_sources_list(self):
"""
Ensure the Weboob sources.list file contains the required entries from
Kresus.
"""
sources_list_path = os.path.join(self.weboob_data_path, 'sources.list')
# Determine the new content of the sources.list file.
new_sources_list_content = []
if self.sources_list_content is not None:
new_sources_list_content = self.sources_list_content
else:
# Default content of the sources.list file.
new_sources_list_content = [
unicode('https://updates.weboob.org/%(version)s/main/'),
unicode('file://%s' % self.fakemodules_path)
]
# Read the content of existing sources.list, if it exists.
original_sources_list_content = []
if os.path.isfile(sources_list_path):
with io.open(sources_list_path, encoding="utf-8") as fh:
original_sources_list_content = fh.read().splitlines()
# Update the source.list content and update the repository, only if the
# content has changed.
if set(original_sources_list_content) != set(new_sources_list_content):
with io.open(sources_list_path, 'w', encoding="utf-8") as sources_list_file:
sources_list_file.write('\n'.join(new_sources_list_content))
self.needs_update = True
def update(self):
"""
Update Weboob modules.
"""
self.copy_fakemodules()
# Weboob has an offending print statement when it "Rebuilds index",
# which happen at every run if the user has a local repository. We need
# to silence it, hence the temporary redirect of stdout.
sys.stdout = open(os.devnull, "w")
try:
self.weboob.update(progress=DummyProgress())
except ConnectionError as exc:
# Do not delete the repository if there is a connection error.
raise exc
except Exception:
# Try to remove the data directory, to see if it changes a thing.
# This is especially useful when a new version of Weboob is
# published and/or the keyring changes.
shutil.rmtree(self.weboob_data_path)
os.makedirs(self.weboob_data_path)
# Recreate the Weboob object as the directories are created
# on creating the Weboob object.
self.weboob = Weboob(workdir=self.weboob_data_path,
datadir=self.weboob_data_path)
# Rewrite sources.list file
self.write_weboob_sources_list()
# Retry update
self.weboob.update(progress=DummyProgress())
finally:
# Restore stdout
sys.stdout = sys.__stdout__
def create_backend(self, modulename, parameters, session):
"""
Create a Weboob backend for a given module, ready to be used to fetch
data.
:param modulename: The name of the module from which backend should be
created.
:param parameters: A dict of parameters to pass to the module. It
should at least contain ``login`` and ``password`` fields, but can
contain additional values depending on the module.
:param session: an object representing the browser state.
"""
# Install the module if required.
repositories = self.weboob.repositories
minfo = repositories.get_module_info(modulename)
if (
minfo is not None and not minfo.is_installed() and
not minfo.is_local()
):
# We cannot install a locally available module, this would
# result in a ModuleInstallError.
try:
repositories.install(minfo, progress=DummyProgress())
except ModuleInstallError:
fail(
GENERIC_EXCEPTION,
"Unable to install module %s." % modulename,
traceback.format_exc()
)
# Initialize the Storage.
self.storage = DictStorage(session)
# Initialize the backend.
self.backend = self.weboob.build_backend(
modulename,
parameters,
storage=self.storage
)
def delete_backend(self):
"""
Delete a created backend for the given module.
"""
if self.backend:
with self.backend:
self.backend.deinit()
self.backend = None
self.storage = None
def get_accounts(self):
"""
Fetch accounts data from Weboob.
:param backend: The Weboob built backend to fetch data from.
:returns: A list of dicts representing the available accounts.
"""
results = []
with self.backend:
for account in list(self.backend.iter_accounts()):
# The minimum dict keys for an account are :
# 'id', 'label', 'balance' and 'type'
# Retrieve extra information for the account.
account = self.backend.fillobj(account, ['iban', 'currency'])
iban = None
if not empty(account.iban):
iban = account.iban
currency = None
if not empty(account.currency):
currency = unicode(account.currency)
results.append({
'vendorAccountId': account.id,
'label': account.label,
'balance': account.balance,
'iban': iban,
'currency': currency,
'type': account.type,
})
return results
def get_operations(self):
"""
Fetch operations data from Weboob.
:param backend: The Weboob built backend to fetch data from.
:returns: A list of dicts representing the available operations.
"""
results = []
with self.backend:
for account in list(self.backend.iter_accounts()):
# Get all operations for this account.
nyi_methods = []
operations = []
try:
operations += list(self.backend.iter_history(account))
except NotImplementedError:
nyi_methods.append('iter_history')
try:
operations += [
op for op in self.backend.iter_coming(account)
if op.type in [
Transaction.TYPE_DEFERRED_CARD,
Transaction.TYPE_CARD_SUMMARY
]
]
except NotImplementedError:
nyi_methods.append('iter_coming')
for method_name in nyi_methods:
logging.error(
('%s not implemented for this account: %s.'),
method_name,
account.id
)
# Build an operation dict for each operation.
for operation in operations:
label = None
if not empty(operation.label):
label = unicode(operation.label)
raw_label = None
if not empty(operation.raw):
raw_label = unicode(operation.raw)
elif label:
raw_label = label
if raw_label and not label:
label = raw_label
# Handle date
if operation.rdate:
# Use date of the payment (real date) if available.
date = operation.rdate
elif operation.date:
# Otherwise, use debit date, on the bank statement.
date = operation.date
else:
logging.error(
'No known date property in operation line: %s.',
raw_label or "no label"
)
date = datetime.now()
isodate = date.isoformat()
debit_date = operation.date.isoformat()
results.append({
'account': account.id,
'amount': operation.amount,
'rawLabel': raw_label,
'type': operation.type,
'date': isodate,
'debit_date': debit_date,
'label': label
})
return results
def fetch(self, which):
"""
Wrapper to fetch data from the Weboob connector.
This wrapper fetches the required data from Weboob and returns it. It
handles the translation between Weboob exceptions and Kresus error
codes stored in the JSON response.
:param which: The type of data to fetch. Can be either ``accounts`` or
``operations``.
:param modulename: The name of the module from which data should be
fetched. Optional, if not provided all available backends are used.
:param login: The login to further filter on the available backends.
Optional, if not provided all matching backends are used.
:returns: A dict of the fetched data, in a ``values`` keys. Errors are
described under ``error_code``, ``error_short`` and ``error_message``
keys.
"""
results = {}
try:
if which == 'accounts':
results['values'] = self.get_accounts()
elif which == 'operations':
results['values'] = self.get_operations()
else:
raise Exception('Invalid fetch command.')
except NoAccountsException:
results['error_code'] = NO_ACCOUNTS
except ModuleLoadError:
results['error_code'] = UNKNOWN_MODULE
except BrowserPasswordExpired:
results['error_code'] = EXPIRED_PASSWORD
except BrowserQuestion:
results['error_code'] = BROWSER_QUESTION
except AuthMethodNotImplemented:
results['error_code'] = AUTH_METHOD_NYI
except ActionNeeded as exc:
# This `except` clause is not in alphabetic order and cannot be,
# because BrowserPasswordExpired and AuthMethodNotImplemented
# (above) inherits from it in Weboob 1.4.
results['error_code'] = ACTION_NEEDED
results['error_message'] = unicode(exc)
except BrowserIncorrectPassword:
# This `except` clause is not in alphabetic order and cannot be,
# because BrowserPasswordExpired (above) inherits from it in
# Weboob 1.3.
results['error_code'] = INVALID_PASSWORD
except Module.ConfigError as exc:
results['error_code'] = INVALID_PARAMETERS
results['error_message'] = unicode(exc)
except ConnectionError as exc:
results['error_code'] = CONNECTION_ERROR
results['error_message'] = unicode(exc)
except Exception as exc:
fail(
GENERIC_EXCEPTION,
'Unknown error: %s.' % unicode(exc),
traceback.format_exc()
)
# Return session information for future use.
results['session'] = self.storage.dump()
return results
class MyThread(Thread):
daemon = True
def __init__(self, bot):
Thread.__init__(self)
self.weboob = Weboob(storage=StandardStorage(STORAGE_FILE))
self.weboob.load_backends()
self.bot = bot
self.bot.set_weboob(self.weboob)
def run(self):
for ev in self.bot.joined.itervalues():
ev.wait()
self.weboob.repeat(300, self.check_board)
self.weboob.repeat(600, self.check_dlfp)
self.weboob.loop()
def find_keywords(self, text):
for word in [
'weboob', 'videoob', 'havesex', 'havedate', 'monboob', 'boobmsg',
'flatboob', 'boobill', 'pastoob', 'radioob', 'translaboob', 'traveloob', 'handjoob',
'boobathon', 'boobank', 'boobtracker', 'comparoob', 'wetboobs',
'webcontentedit', 'weboorrents', u'sàt', u'salut à toi', 'assnet',
'budget insight', 'budget-insight', 'budgetinsight', 'budgea']:
if word in text.lower():
return word
return None
def check_dlfp(self):
for backend, msg in self.weboob.do('iter_unread_messages', backends=['dlfp']):
word = self.find_keywords(msg.content)
if word is not None:
url = msg.signature[msg.signature.find('https://linuxfr'):]
self.bot.send_message('[DLFP] %s talks about %s: %s' % (
msg.sender, word, url))
backend.set_message_read(msg)
def check_board(self):
def iter_messages(backend):
with backend.browser:
return backend.browser.iter_new_board_messages()
for backend, msg in self.weboob.do(iter_messages, backends=['dlfp']):
word = self.find_keywords(msg.message)
if word is not None and msg.login != 'moules':
message = msg.message.replace(word, '\002%s\002' % word)
self.bot.send_message('[DLFP] <%s> %s' % (msg.login, message))
def stop(self):
self.weboob.want_stop()
self.weboob.deinit()
