def register_account(klass, account):
"""
Register an account on website
This is a static method, it would be called even if the backend is
instancied.
@param account an Account object which describe the account to create
"""
browser = None
bday, bmonth, byear = account.properties['birthday'].get().split('/', 2)
while not browser:
try:
browser = klass.BROWSER(account.properties['username'].get())
browser.register(password= account.properties['password'].get(),
sex= (0 if account.properties['sex'].get() == 'm' else 1),
birthday_d= int(bday),
birthday_m= int(bmonth),
birthday_y= int(byear),
zipcode= account.properties['zipcode'].get(),
country= account.properties['country'].get(),
godfather= account.properties['godfather'].get())
except CaptchaError:
getLogger('aum').info('Unable to resolve captcha. Retrying...')
browser = None
def ICON(klass):
try:
import xdg.IconTheme
except ImportError:
getLogger(klass.NAME).debug(u'Python xdg module was not found. Please install it to read icon files.')
else:
return xdg.IconTheme.getIconPath(klass.NAME)
def register_account(klass, account):
"""
Register an account on website
This is a static method, it would be called even if the backend is
instancied.
@param account an Account object which describe the account to create
"""
browser = None
bday, bmonth, byear = account.properties['birthday'].get().split(
'/', 2)
while not browser:
try:
browser = klass.BROWSER(account.properties['username'].get())
browser.register(
password=account.properties['password'].get(),
sex=(0 if account.properties['sex'].get() == 'm' else 1),
birthday_d=int(bday),
birthday_m=int(bmonth),
birthday_y=int(byear),
zipcode=account.properties['zipcode'].get(),
country=account.properties['country'].get(),
godfather=account.properties['godfather'].get())
except CaptchaError:
getLogger('aum').info('Unable to resolve captcha. Retrying...')
browser = None
def __init__(self, sched, storage, browser):
self._sched = sched
self._storage = storage
self._browser = browser
self._logger = getLogger('walker', browser.logger)
self._view_cron = None
def __init__(self, workdir, datadir, version):
self.logger = getLogger('repositories')
self.version = version
WeboobBrowser.set_version(version)
self.workdir = workdir
self.datadir = datadir
self.sources_list = os.path.join(self.workdir, self.SOURCES_LIST)
self.modules_dir = os.path.join(self.datadir, self.MODULES_DIR)
self.repos_dir = os.path.join(self.datadir, self.REPOS_DIR)
self.keyrings_dir = os.path.join(self.datadir, self.KEYRINGS_DIR)
self.icons_dir = os.path.join(self.datadir, self.ICONS_DIR)
self.create_dir(self.datadir)
self.create_dir(self.modules_dir)
self.create_dir(self.repos_dir)
self.create_dir(self.keyrings_dir)
self.create_dir(self.icons_dir)
self.versions = Versions(self.modules_dir)
self.repositories = []
if not os.path.exists(self.sources_list):
with open(self.sources_list, 'w') as f:
f.write(DEFAULT_SOURCES_LIST)
self.update()
else:
self.load()
def __init__(self, option_parser=None):
super(Application, self).__init__()
self.encoding = self.guess_encoding()
self.logger = getLogger(self.APPNAME)
self.weboob = self.create_weboob()
if self.CONFDIR is None:
self.CONFDIR = self.weboob.workdir
self.config = None
self.options = None
self.condition = None
self.storage = None
if option_parser is None:
self._parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version())
else:
self._parser = option_parser
if self.DESCRIPTION:
self._parser.description = self.DESCRIPTION
app_options = OptionGroup(self._parser, '%s Options' % self.APPNAME.capitalize())
self.add_application_options(app_options)
if len(app_options.option_list) > 0:
self._parser.add_option_group(app_options)
self._parser.add_option('-b', '--backends', help='what backend(s) to enable (comma separated)')
self._parser.add_option('-e', '--exclude-backends', help='what backend(s) to exclude (comma separated)')
self._parser.add_option('-I', '--insecure', action='store_true', help='do not validate SSL')
self._parser.add_option('--nss', action='store_true', help='Use NSS instead of OpenSSL')
logging_options = OptionGroup(self._parser, 'Logging Options')
logging_options.add_option('-d', '--debug', action='count', help='display debug messages. Set up it twice to more verbosity', default=0)
logging_options.add_option('-q', '--quiet', action='store_true', help='display only error messages')
logging_options.add_option('-v', '--verbose', action='store_true', help='display info messages')
logging_options.add_option('--logging-file', action='store', type='string', dest='logging_file', help='file to save logs')
logging_options.add_option('-a', '--save-responses', action='store_true', help='save every response')
logging_options.add_option('--export-session', action='store_true', help='log browser session cookies after login')
self._parser.add_option_group(logging_options)
self._parser.add_option('--shell-completion', action='store_true', help=optparse.SUPPRESS_HELP)
self._is_default_count = True
def __init__(self, backends, condition, function, *args, **kwargs):
"""
@param backends list of backends to call.
@param condition a IResultsCondition object. Can be None.
@param function backends' method name, or callable object.
@param args, kwargs arguments given to called functions.
"""
self.logger = getLogger('bcall')
# Store if a backend is finished
self.backends = {}
for backend in backends:
self.backends[backend.name] = False
# Condition
self.condition = condition
# Global mutex on object
self.mutex = RLock()
# Event set when every backends have give their data
self.finish_event = Event()
# Event set when there are new responses
self.response_event = Event()
# Waiting responses
self.responses = []
# Errors
self.errors = []
# Threads
self.threads = []
# Create jobs for each backend
with self.mutex:
for backend in backends:
self.logger.debug('Creating a new thread for %s' % backend)
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
if not backends:
self.finish_event.set()
def __init__(self, option_parser=None):
self.logger = getLogger(self.APPNAME)
self.weboob = self.create_weboob()
if self.CONFDIR is None:
self.CONFDIR = self.weboob.workdir
self.config = None
self.options = None
self.condition = None
if option_parser is None:
self._parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version())
else:
self._parser = option_parser
if self.DESCRIPTION:
self._parser.description = self.DESCRIPTION
app_options = OptionGroup(self._parser, '%s Options' % self.APPNAME.capitalize())
self.add_application_options(app_options)
if len(app_options.option_list) > 0:
self._parser.add_option_group(app_options)
self._parser.add_option('-b', '--backends', help='what backend(s) to enable (comma separated)')
self._parser.add_option('-e', '--exclude-backends', help='what backend(s) to exclude (comma separated)')
self._parser.add_option('-I', '--insecure', action='store_true', help='do not validate SSL')
logging_options = OptionGroup(self._parser, 'Logging Options')
logging_options.add_option('-d', '--debug', action='store_true', help='display debug messages')
logging_options.add_option('-q', '--quiet', action='store_true', help='display only error messages')
logging_options.add_option('-v', '--verbose', action='store_true', help='display info messages')
logging_options.add_option('--logging-file', action='store', type='string', dest='logging_file', help='file to save logs')
logging_options.add_option('-a', '--save-responses', action='store_true', help='save every response')
self._parser.add_option_group(logging_options)
self._parser.add_option('--shell-completion', action='store_true', help=optparse.SUPPRESS_HELP)
def __init__(self, option_parser=None):
self.logger = getLogger(self.APPNAME)
self.weboob = self.create_weboob()
self.config = None
self.options = None
if option_parser is None:
self._parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version())
else:
self._parser = option_parser
if self.DESCRIPTION:
self._parser.description = self.DESCRIPTION
app_options = OptionGroup(self._parser, "%s Options" % self.APPNAME.capitalize())
self.add_application_options(app_options)
if len(app_options.option_list) > 0:
self._parser.add_option_group(app_options)
self._parser.add_option("-b", "--backends", help="what backend(s) to enable (comma separated)")
logging_options = OptionGroup(self._parser, "Logging Options")
logging_options.add_option("-d", "--debug", action="store_true", help="display debug messages")
logging_options.add_option("-q", "--quiet", action="store_true", help="display only error messages")
logging_options.add_option("-v", "--verbose", action="store_true", help="display info messages")
logging_options.add_option(
"--logging-file", action="store", type="string", dest="logging_file", help="file to save logs"
)
logging_options.add_option("-a", "--save-responses", action="store_true", help="save every response")
self._parser.add_option_group(logging_options)
self._parser.add_option("--shell-completion", action="store_true", help=optparse.SUPPRESS_HELP)
def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None, storage=None):
self.logger = getLogger('weboob')
self.workdir = workdir
self.backend_instances = {}
# Scheduler
if scheduler is None:
scheduler = Scheduler()
self.scheduler = scheduler
# Create WORKDIR
if not os.path.exists(self.workdir):
os.mkdir(self.workdir, 0700)
elif not os.path.isdir(self.workdir):
self.logger.warning(u'"%s" is not a directory' % self.workdir)
# Backends loader
self.modules_loader = ModulesLoader()
# Backend instances config
if not backends_filename:
backends_filename = os.path.join(self.workdir, self.BACKENDS_FILENAME)
elif not backends_filename.startswith('/'):
backends_filename = os.path.join(self.workdir, backends_filename)
self.backends_config = BackendsConfig(backends_filename)
# Storage
self.storage = storage
def __init__(self, option_parser=None):
self.encoding = self.guess_encoding()
self.logger = getLogger(self.APPNAME)
self.weboob = self.create_weboob()
if self.CONFDIR is None:
self.CONFDIR = self.weboob.workdir
self.config = None
self.options = None
self.condition = None
self.storage = None
if option_parser is None:
self._parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version())
else:
self._parser = option_parser
if self.DESCRIPTION:
self._parser.description = self.DESCRIPTION
app_options = OptionGroup(self._parser, '%s Options' % self.APPNAME.capitalize())
self.add_application_options(app_options)
if len(app_options.option_list) > 0:
self._parser.add_option_group(app_options)
self._parser.add_option('-b', '--backends', help='what backend(s) to enable (comma separated)')
self._parser.add_option('-e', '--exclude-backends', help='what backend(s) to exclude (comma separated)')
self._parser.add_option('-I', '--insecure', action='store_true', help='do not validate SSL')
logging_options = OptionGroup(self._parser, 'Logging Options')
logging_options.add_option('-d', '--debug', action='count', help='display debug messages. Set up it twice to more verbosity')
logging_options.add_option('-q', '--quiet', action='store_true', help='display only error messages')
logging_options.add_option('-v', '--verbose', action='store_true', help='display info messages')
logging_options.add_option('--logging-file', action='store', type='string', dest='logging_file', help='file to save logs')
logging_options.add_option('-a', '--save-responses', action='store_true', help='save every response')
self._parser.add_option_group(logging_options)
self._parser.add_option('--shell-completion', action='store_true', help=optparse.SUPPRESS_HELP)
self._is_default_count = True
def __init__(self, backends, function, *args, **kwargs):
"""
:param backends: List of backends to call
:type backends: list[:class:`BaseBackend`]
:param function: backends' method name, or callable object.
:type function: :class:`str` or :class:`callable`
"""
self.logger = getLogger('bcall')
# Store if a backend is finished
self.backends = {}
for backend in backends:
self.backends[backend.name] = False
# Global mutex on object
self.mutex = RLock()
# Event set when every backends have give their data
self.finish_event = Event()
# Event set when there are new responses
self.response_event = Event()
# Waiting responses
self.responses = []
# Errors
self.errors = []
# Threads
self.threads = []
# Create jobs for each backend
with self.mutex:
for backend in backends:
self.threads.append(
Timer(0, self._caller,
(backend, function, args, kwargs)).start())
if not backends:
self.finish_event.set()
def __init__(self, workdir, datadir, version):
self.logger = getLogger('repositories')
self.version = version
self.browser = None
self.workdir = workdir
self.datadir = datadir
self.sources_list = os.path.join(self.workdir, self.SOURCES_LIST)
self.modules_dir = os.path.join(self.datadir, self.MODULES_DIR, self.version)
self.repos_dir = os.path.join(self.datadir, self.REPOS_DIR)
self.keyrings_dir = os.path.join(self.datadir, self.KEYRINGS_DIR)
self.icons_dir = os.path.join(self.datadir, self.ICONS_DIR)
self.create_dir(self.datadir)
self.create_dir(self.modules_dir)
self.create_dir(self.repos_dir)
self.create_dir(self.keyrings_dir)
self.create_dir(self.icons_dir)
self.versions = Versions(self.modules_dir)
self.repositories = []
if not os.path.exists(self.sources_list):
with open(self.sources_list, 'w') as f:
f.write(DEFAULT_SOURCES_LIST)
self.update()
else:
self.load()
def __init__(self, browser, document, url='', groups=None, group_dict=None, logger=None):
self.browser = browser
self.document = document
self.url = url
self.groups = groups
self.group_dict = group_dict
self.logger = getLogger('page', logger)
def __init__(self, backends, function, *args, **kwargs):
"""
:param backends: List of backends to call
:type backends: list[:class:`BaseBackend`]
:param function: backends' method name, or callable object.
:type function: :class:`str` or :class:`callable`
"""
self.logger = getLogger('bcall')
# Store if a backend is finished
self.backends = {}
for backend in backends:
self.backends[backend.name] = False
# Global mutex on object
self.mutex = RLock()
# Event set when every backends have give their data
self.finish_event = Event()
# Event set when there are new responses
self.response_event = Event()
# Waiting responses
self.responses = []
# Errors
self.errors = []
# Threads
self.threads = []
# Create jobs for each backend
with self.mutex:
for backend in backends:
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
if not backends:
self.finish_event.set()
def __init__(self, sched, storage, browser):
self._sched = sched
self._storage = storage
self._browser = browser
self._logger = getLogger('walker', browser.logger)
self._view_cron = None
def print_debug(self, value):
logger = getLogger('b2filters')
result = ''
outputvalue = value
if isinstance(value, list):
from lxml import etree
outputvalue = ''
first = True
for element in value:
if first:
first = False
else:
outputvalue += ', '
if isinstance(element, etree.ElementBase):
outputvalue += "%s" % etree.tostring(element,
encoding=unicode)
else:
outputvalue += "%r" % element
if self._obj is not None:
result += "%s" % self._obj._random_id
if self._key is not None:
result += ".%s" % self._key
name = str(self)
result += " %s(%r" % (name, outputvalue)
for arg in self.__dict__:
if arg.startswith('_') or arg == u"selector":
continue
if arg == u'default' and getattr(self, arg) == _NO_DEFAULT:
continue
result += ", %s=%r" % (arg, getattr(self, arg))
result += u')'
logger.log(DEBUG_FILTERS, result)
res = function(self, value)
return res
def __init__(self,
weboob,
name,
config=None,
storage=None,
logger=None,
nofail=False):
self.logger = getLogger(name, parent=logger)
self.weboob = weboob
self.name = name
self.lock = RLock()
if config is None:
config = {}
# Private fields (which start with '_')
self._private_config = dict((key, value)
for key, value in config.iteritems()
if key.startswith('_'))
# Load configuration of backend.
self.config = self.CONFIG.load(weboob, self.NAME, self.name, config,
nofail)
self.storage = BackendStorage(self.name, storage)
self.storage.load(self.STORAGE)
def __init__(self,
logger=None,
proxy=None,
responses_dirname=None,
weboob=None,
proxy_headers=None):
super(SeleniumBrowser, self).__init__()
self.responses_dirname = responses_dirname
self.responses_count = 0
self.weboob = weboob
self.logger = getLogger('browser', logger)
self.proxy = proxy or {}
# We set the default value of selenium logger to ERROR to avoid
# spamming logs with useless information.
# Also, the data we send to the browser using selenium (with send_keys)
# can be displayed clearly in the log, if the log level is
# set to DEBUG.
logging.getLogger('selenium').setLevel(logging.ERROR)
self.implicit_timeout = 0
self.last_page_hash = None
self._setup_driver()
self._urls = []
cls = type(self)
for attr in dir(cls):
val = getattr(cls, attr)
if isinstance(val, URL):
val = deepcopy(val)
val.browser = self
setattr(self, attr, val)
self._urls.append(val)
self._urls.sort(key=lambda u: u._creation_counter)
def __init__(self, browser):
super(SeleniumPage, self).__init__()
self.params = {}
self.browser = browser
self.driver = browser.driver
self.logger = getLogger(self.__class__.__name__.lower(),
browser.logger)
def __init__(self, weboob, name, config, storage, logger=None):
self.logger = getLogger(name, parent=logger)
self.weboob = weboob
self.name = name
self.lock = RLock()
# Private fields (which start with '_')
self._private_config = dict([(key, value) for key, value in config.iteritems() if key.startswith('_')])
# Configuration of backend
self.config = {}
for name, field in self.CONFIG.iteritems():
value = config.get(name, None)
if value is None:
if field.required:
raise BaseBackend.ConfigError('Backend(%s): Configuration error: Missing parameter "%s" (%s)' % (self.name, name, field.description))
value = field.default
try:
field.set_value(value)
except ValueError, v:
raise BaseBackend.ConfigError('Backend(%s): Configuration error for field "%s": %s' % (self.name, name, v))
# field.value is a property which converts string to right type (bool/int/float)
self.config[name] = field.value
def print_debug(self, value):
logger = getLogger('b2filters')
result = ''
outputvalue = value
if isinstance(value, list):
from lxml import etree
outputvalue = ''
first = True
for element in value:
if first:
first = False
else:
outputvalue += ', '
if isinstance(element, etree.ElementBase):
outputvalue += "%s" % etree.tostring(element, encoding=unicode)
else:
outputvalue += "%r" % element
if self._obj is not None:
result += "%s" % self._obj._random_id
if self._key is not None:
result += ".%s" % self._key
name = str(self)
result += " %s(%r" % (name, outputvalue)
for arg in self.__dict__:
if arg.startswith('_') or arg == u"selector":
continue
if arg == u'default' and getattr(self, arg) == _NO_DEFAULT:
continue
result += ", %s=%r" % (arg, getattr(self, arg))
result += u')'
logger.log(DEBUG_FILTERS, result)
res = function(self, value)
return res
def __init__(self, browser, document, url='', groups=None, group_dict=None, logger=None):
self.browser = browser
self.parser = browser.parser
self.document = document
self.url = url
self.groups = groups
self.group_dict = group_dict
self.logger = getLogger('page', logger)
def __init__(self, sched, storage, browser):
super(ProfilesWalker, self).__init__()
self._sched = sched
self._storage = storage
self._browser = browser
self._logger = getLogger('walker', browser.logger)
self._last_position_update = None
self._view_cron = None
def __init__(self, logger=None, proxy=None, responses_dirname=None):
self.logger = getLogger('browser', logger)
self.PROXIES = proxy
self._setup_session(self.PROFILE)
self.url = None
self.response = None
self.responses_dirname = responses_dirname
self.responses_count = 1
def __init__(self, sched, storage, browser):
super(ProfilesWalker, self).__init__()
self._sched = sched
self._storage = storage
self._browser = browser
self._logger = getLogger('walker', browser.logger)
self._last_position_update = None
self._view_cron = None
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('queriesqueue', browser.logger)
self.queue = storage.get('queries_queue', 'queue', default=[])
self.check_cron = None
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('queriesqueue', browser.logger)
self.queue = storage.get('queries_queue', 'queue', default=[])
self.check_cron = None
def __init__(self, package):
self.logger = getLogger('backend')
self.package = package
self.klass = None
for attrname in dir(self.package):
attr = getattr(self.package, attrname)
if isinstance(attr, type) and issubclass(attr, Module) and attr != Module:
self.klass = attr
if not self.klass:
raise ImportError('%s is not a backend (no Module class found)' % package)
def __init__(self, sched, storage, browser):
super(QueriesQueue, self).__init__()
self._sched = sched
self._storage = storage
self._browser = browser
self._logger = getLogger('queriesqueue', browser.logger)
self._queue = storage.get('queries_queue', 'queue', default=[])
self._check_cron = None
def __init__(self, script, logger=None):
try:
import execjs
except ImportError:
raise ImportError('Please install PyExecJS')
self.runner = execjs.get()
self.logger = getLogger('js', logger)
self.ctx = self.runner.compile(self.HEADER + script)
def __init__(self, package):
self.logger = getLogger("backend")
self.package = package
self.klass = None
for attrname in dir(self.package):
attr = getattr(self.package, attrname)
if isinstance(attr, type) and issubclass(attr, BaseBackend) and attr != BaseBackend:
self.klass = attr
if not self.klass:
raise ImportError("%s is not a backend (no BaseBackend class found)" % package)
def __init__(self, script, logger=None):
try:
import execjs
except ImportError:
raise ImportError('Please install PyExecJS')
self.runner = execjs.get()
self.logger = getLogger('js', logger)
self.ctx = self.runner.compile(self.HEADER + script)
def handle_attr(self, key, func):
try:
value = self.use_selector(func, key=key)
except Exception as e:
# Help debugging as tracebacks do not give us the key
self.logger.warning('Attribute %s raises %s' % (key, repr(e)))
raise
logger = getLogger('b2filters')
logger.log(DEBUG_FILTERS, "%s.%s = %r" % (self._random_id, key, value))
setattr(self.obj, key, value)
def handle_attr(self, key, func):
try:
value = self.use_selector(func, key=key)
except Exception as e:
# Help debugging as tracebacks do not give us the key
self.logger.warning('Attribute %s raises %s' % (key, repr(e)))
raise
logger = getLogger('b2filters')
logger.log(DEBUG_FILTERS, "%s.%s = %r" % (self._random_id, key, value))
setattr(self.obj, key, value)
def __init__(self, package):
self.logger = getLogger('backend')
self.package = package
self.klass = None
for attrname in dir(self.package):
attr = getattr(self.package, attrname)
if isinstance(attr, type) and issubclass(attr, Module) and attr != Module:
self.klass = attr
if not self.klass:
raise ImportError('%s is not a backend (no Module class found)' % package)
def __init__(self, username=None, password=None, firefox_cookies=None,
parser=None, history=NoHistory(), proxy=None, logger=None):
"""
Constructor of Browser.
@param username [str] username on website.
@param password [str] password on website. If it is None, Browser will
not try to login.
@param filefox_cookies [str] Path to cookies' sqlite file.
@param parser [IParser] parser to use on HTML files.
@param hisory [object] History manager. Default value is an object
which does not keep history.
@param proxy [str] proxy URL to use.
"""
mechanize.Browser.__init__(self, history=history)
self.logger = getLogger('browser', logger)
self.addheaders = [
['User-agent', self.USER_AGENT]
]
# Use a proxy
self.proxy = proxy
if proxy:
proto = 'http'
if proxy.find('://') >= 0:
proto, domain = proxy.split('://', 1)
else:
domain = proxy
self.set_proxies({proto: domain})
# Share cookies with firefox
if firefox_cookies and HAVE_COOKIES:
self._cookie = FirefoxCookieJar(self.DOMAIN, firefox_cookies)
self._cookie.load()
self.set_cookiejar(self._cookie)
else:
self._cookie = None
if parser is None:
parser = get_parser()()
elif isinstance(parser, (tuple,list)):
parser = get_parser(parser)()
self.parser = parser
self.page = None
self.last_update = 0.0
self.username = username
self.password = password
self.lock = RLock()
if self.password:
try:
self.home()
# Do not abort the build of browser when the website is down.
except BrowserUnavailable:
pass
def __init__(self, workdir=None, backends_filename=None, scheduler=None, storage=None):
self.logger = getLogger('weboob')
self.backend_instances = {}
self.callbacks = {'login': lambda backend_name, value: None,
'captcha': lambda backend_name, image: None,
}
# Scheduler
if scheduler is None:
scheduler = Scheduler()
self.scheduler = scheduler
# Create WORKDIR
if workdir is not None:
datadir = workdir
elif 'WEBOOB_WORKDIR' in os.environ:
datadir = workdir = os.environ.get('WEBOOB_WORKDIR')
else:
old_workdir = os.path.join(os.path.expanduser('~'), '.weboob')
xdg_config_home = os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob')
xdg_data_home = os.path.join(os.environ.get('XDG_DATA_HOME', os.path.join(os.path.expanduser('~'), '.local', 'share')), 'weboob')
if os.path.isdir(old_workdir):
self.logger.warning('You are using "%s" as working directory. Files are moved into %s and %s.'
% (old_workdir, xdg_config_home, xdg_data_home))
self._create_dir(xdg_config_home)
self._create_dir(xdg_data_home)
for f in os.listdir(old_workdir):
if f in Repositories.SHARE_DIRS:
dest = xdg_data_home
else:
dest = xdg_config_home
shutil.move(os.path.join(old_workdir, f), dest)
shutil.rmtree(old_workdir)
workdir = xdg_config_home
datadir = xdg_data_home
self.workdir = os.path.realpath(workdir)
self._create_dir(workdir)
# Repositories management
self.repositories = Repositories(workdir, datadir, self.VERSION)
# Backends loader
self.modules_loader = ModulesLoader(self.repositories)
# Backend instances config
if not backends_filename:
backends_filename = os.environ.get('WEBOOB_BACKENDS', os.path.join(self.workdir, self.BACKENDS_FILENAME))
elif not backends_filename.startswith('/'):
backends_filename = os.path.join(self.workdir, backends_filename)
self.backends_config = BackendsConfig(backends_filename)
# Storage
self.storage = storage
def do_budgea(self, line):
"""
budgea USERNAME PASSWORD
Export your bank accounts and transactions to Budgea.
Budgea is an online web and mobile application to manage your bank
accounts. To avoid giving your credentials to this service, you can use
this command.
https://www.budgea.com
"""
username, password = self.parse_command_args(line, 2, 2)
client = APIBrowser(baseurl='https://budgea.biapi.pro/2.0/',
proxy=getproxies(),
logger=getLogger('apibrowser', self.logger))
client.set_profile(Weboob(self.VERSION))
client.TIMEOUT = 60
try:
r = client.request('auth/token', data={'username': username, 'password': password, 'application': 'weboob'})
except BrowserHTTPError as r:
error = r.response.json()
print('Error: {}'.format(error.get('message', error['code'])), file=self.stderr)
return 1
client.session.headers['Authorization'] = 'Bearer %s' % r['token']
accounts = {}
for account in client.request('users/me/accounts')['accounts']:
if account['id_connection'] is None:
accounts[account['number']] = account
for account in self.do('iter_accounts'):
if account.id not in accounts:
r = client.request('users/me/accounts', data={'name': account.label,
'balance': account.balance,
'number': account.id,
})
self.logger.debug(r)
account_id = r['id']
else:
account_id = accounts[account.id]['id']
transactions = []
for tr in self.do('iter_history', account, backends=account.backend):
transactions.append({'original_wording': tr.raw,
'simplified_wording': tr.label,
'value': tr.amount,
'date': tr.date.strftime('%Y-%m-%d'),
})
r = client.request('users/me/accounts/%s/transactions' % account_id,
data={'transactions': transactions})
client.request('users/me/accounts/%s' % account_id, data={'balance': account.balance})
print('- %s (%s%s): %s new transactions' % (account.label, account.balance, account.currency_text, len(r)))
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('walker', browser.logger)
self.walk_cron = None
self.view_cron = None
self.visited_profiles = set(storage.get('profiles_walker', 'viewed'))
self.logger.info(u'Loaded %d already visited profiles from storage.' % len(self.visited_profiles))
self.profiles_queue = set()
def load_browser(self):
from weboob.browser.browsers import Browser
from weboob.browser.profiles import Weboob as WeboobProfile
from weboob.tools.compat import getproxies
class WeboobBrowser(Browser):
PROFILE = WeboobProfile(self.version)
if self.browser is None:
self.browser = WeboobBrowser(
logger=getLogger('browser', parent=self.logger),
proxy=getproxies())
def __init__(
self,
firefox_cookies=None,
parser=None,
history=NoHistory(),
proxy=None,
logger=None,
factory=None,
responses_dirname=None,
):
mechanize.Browser.__init__(self, history=history, factory=factory)
self.logger = getLogger("browser", logger)
self.addheaders = [["User-agent", self.USER_AGENT]]
# Use a proxy
self.proxy = proxy
if proxy is not None:
self.set_proxies(proxy)
# Share cookies with firefox
if firefox_cookies:
# Try to load cookies
try:
from .firefox_cookies import FirefoxCookieJar
self._cookie = FirefoxCookieJar(self.DOMAIN, firefox_cookies)
self._cookie.load()
self.set_cookiejar(self._cookie)
except ImportError as e:
logging.warning("Unable to store Firefox cookies: %s", e)
self._cookie = None
else:
self._cookie = None
if parser is None:
parser = get_parser()()
elif isinstance(parser, (tuple, list, basestring)):
parser = get_parser(parser)()
self.parser = parser
self.lock = RLock()
if self.DEBUG_HTTP:
# display messages from httplib
self.set_debug_http(True)
if logging.root.level <= logging.DEBUG:
# Enable log messages from mechanize.Browser
self.set_debug_redirects(True)
mech_logger = logging.getLogger("mechanize")
mech_logger.setLevel(logging.INFO)
self.responses_dirname = responses_dirname
self.responses_count = 0
def load_browser(self):
from weboob.browser.browsers import Browser
from weboob.browser.profiles import Weboob as WeboobProfile
from weboob.tools.compat import getproxies
class WeboobBrowser(Browser):
PROFILE = WeboobProfile(self.version)
if self.browser is None:
self.browser = WeboobBrowser(logger=getLogger('browser',
parent=self.logger),
proxy=getproxies())
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('priorityconn', browser.logger)
self.config = storage.get('priority_connection', 'config', default=None)
if self.config == {}:
self.config = None
self.check_cron = None
self.activity_cron = None
def __init__(self, sched, storage, browser, location, distance):
super(ProfilesWalker, self).__init__()
self._sched = sched
self._storage = storage
self._browser = browser
self._logger = getLogger('walker', browser.logger)
self._last_position_update = None
self._view_cron = None
self._location = geopy.Point(map(float, location.split(',')))
self._max_distance = int(distance)
def __init__(self, logger=None, proxy=None, responses_dirname=None, weboob=None):
self.logger = getLogger('browser', logger)
self.responses_dirname = responses_dirname
self.responses_count = 1
if isinstance(self.VERIFY, basestring):
self.VERIFY = self.asset(self.VERIFY)
self.PROXIES = proxy
self._setup_session(self.PROFILE)
self.url = None
self.response = None
def __init__(self, sched, storage, browser, location, distance):
super(ProfilesWalker, self).__init__()
self._sched = sched
self._storage = storage
self._browser = browser
self._logger = getLogger('walker', browser.logger)
self._last_position_update = None
self._view_cron = None
self._location = geopy.Point(map(float, location.split(',')))
self._max_distance = int(distance)
def __init__(self, logger=None, proxy=None, responses_dirname=None):
self.logger = getLogger('browser', logger)
self.PROXIES = proxy
self._setup_session(self.PROFILE)
self.url = None
self.response = None
self.responses_dirname = responses_dirname
self.responses_count = 1
if isinstance(self.VERIFY, basestring):
self.VERIFY = self.asset(self.VERIFY)
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('priorityconn', browser.logger)
self.config = storage.get('priority_connection', 'config', default=None)
if self.config == {}:
self.config = None
self.check_cron = None
self.activity_cron = None
def __init__(self,
firefox_cookies=None,
parser=None,
history=NoHistory(),
proxy=None,
logger=None,
factory=None,
responses_dirname=None):
mechanize.Browser.__init__(self, history=history, factory=factory)
self.logger = getLogger('browser', logger)
self.responses_dirname = responses_dirname
self.save_responses = responses_dirname is not None
self.responses_count = 0
self.addheaders = [['User-agent', self.USER_AGENT]]
# Use a proxy
self.proxy = proxy
if proxy is not None:
self.set_proxies(proxy)
# Share cookies with firefox
if firefox_cookies:
# Try to load cookies
try:
from .firefox_cookies import FirefoxCookieJar
self._cookie = FirefoxCookieJar(self.DOMAIN, firefox_cookies)
self._cookie.load()
self.set_cookiejar(self._cookie)
except ImportError as e:
logging.warning("Unable to store Firefox cookies: %s", e)
self._cookie = None
else:
self._cookie = None
if parser is None:
parser = get_parser()()
elif isinstance(parser, (tuple, list, basestring)):
parser = get_parser(parser)()
self.parser = parser
self.lock = RLock()
if self.DEBUG_HTTP:
# display messages from httplib
self.set_debug_http(True)
if logging.root.level <= logging.DEBUG:
# Enable log messages from mechanize.Browser
self.set_debug_redirects(True)
mech_logger = logging.getLogger("mechanize")
mech_logger.setLevel(logging.INFO)
def handle_attr(self, key, func):
try:
value = self.use_selector(func, key=key)
except SkipItem as e:
# Help debugging as tracebacks do not give us the key
self.logger.debug("Attribute %s raises a %r", key, e)
raise
except Exception as e:
# If we are here, we have probably a real parsing issue
self.logger.warning('Attribute %s raises %s', key, repr(e))
raise
logger = getLogger('b2filters')
logger.log(DEBUG_FILTERS, "%s.%s = %r" % (self._random_id, key, value))
setattr(self.obj, key, value)
def __init__(self,
firefox_cookies=None,
parser=None,
history=NoHistory(),
proxy=None,
logger=None,
factory=None,
responses_dirname=None):
mechanize.Browser.__init__(self, history=history, factory=factory)
self.logger = getLogger('browser', logger)
self.addheaders = [['User-agent', self.USER_AGENT]]
# Use a proxy
self.proxy = proxy
if proxy:
proto = 'http'
if '://' in proxy:
v = urlsplit(proxy)
proto = v.scheme
domain = v.netloc
else:
domain = proxy
self.set_proxies({proto: domain})
# Share cookies with firefox
if firefox_cookies and HAVE_COOKIES:
self._cookie = FirefoxCookieJar(self.DOMAIN, firefox_cookies)
self._cookie.load()
self.set_cookiejar(self._cookie)
else:
self._cookie = None
if parser is None:
parser = get_parser()()
elif isinstance(parser, (tuple, list, basestring)):
parser = get_parser(parser)()
self.parser = parser
self.lock = RLock()
if self.DEBUG_HTTP:
# display messages from httplib
self.set_debug_http(True)
if self.DEBUG_MECHANIZE:
# Enable log messages from mechanize.Browser
self.set_debug_redirects(True)
self.responses_dirname = responses_dirname
def __init__(self, records, main_cats, eps=0.1, min_elements=3):
"""Initialize clustering.
records -- list of user transactions
eps -- neighborhood radius
min_elements -- minimum elements to form a cluster
"""
self.records = records
self.main_cats = main_cats
self.min_elements = min_elements
self.eps = eps
self.clusters = []
self.logger = getLogger('forecasts.clusterer')
def __init__(self, sched, storage, browser):
self._sched = sched
self._storage = storage
self._browser = browser
self._logger = getLogger('walker', browser.logger)
self._config = storage.get('profile_walker', 'config', default=None)
if self._config == {}:
self._config = None
self._view_cron = None
self._visited_profiles = set(storage.get('profiles_walker', 'viewed'))
self._logger.info(u'Loaded %d already visited profiles from storage.' %
len(self._visited_profiles))
self._profiles_queue = set()
def __init__(self, backends, function, *args, **kwargs):
"""
:param backends: List of backends to call
:type backends: list[:class:`Module`]
:param function: backends' method name, or callable object.
:type function: :class:`str` or :class:`callable`
"""
self.logger = getLogger('bcall')
self.responses = Queue.Queue()
self.errors = []
self.tasks = Queue.Queue()
for backend in backends:
Thread(target=self.backend_process,
args=(function, args, kwargs)).start()
self.tasks.put(backend)
def __init__(self, num_days=31, num_months=12, threshold_amount=100):
assert num_days <= 31
assert num_months <= 12
self.num_days = num_days
self.num_months = num_months
self.threshold_amount = int(threshold_amount)
self.month_ids = defaultdict(int)
self.records_cum = 0
self.records_mean = 0
self.records_std = 0
self.records_grouped = np.zeros((self.num_days, self.num_months))
self.logger = getLogger('forecasts.prediction')
