def iter_investment(self, account):
if not account.url or account.type not in (Account.TYPE_MARKET, Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE):
return
if account._perimeter != self.current_perimeter:
self.go_perimeter(account._perimeter)
if account.type == Account.TYPE_PEA and account._liquidity_url:
yield create_french_liquidity(account.balance)
return
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
new_location = self.moveto_market_website(account)
# Detail unavailable
try:
self.location(new_location)
except BrowserHTTPError:
return
elif account.type == Account.TYPE_LIFE_INSURANCE:
new_location = self.moveto_insurance_website(account)
if not new_location:
return
self.location(new_location, data={})
if self.bgpi.is_here():
if self.page.cgu_needed() or not self.page.go_detail():
return
# Going the the life insurances details is not enough,
# Once we are there we need to select the correct one:
data = self.page.get_params(account.id)
if not data:
return
self.location('https://bgpi-gestionprivee.credit-agricole.fr/bgpi/CompteDetail.do', data=data)
if self.lifeinsurance.is_here():
self.page.go_on_detail(account.id)
# We scrape the non-invested part as liquidities:
if self.page.has_liquidities():
valuation = self.page.get_liquidities()
yield create_french_liquidity(valuation)
for inv in self.page.iter_investment():
yield inv
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
self.quit_market_website()
elif account.type == Account.TYPE_LIFE_INSURANCE:
self.quit_insurance_website()
def get_liquidity(self):
# Not all accounts have a Liquidity element
liquidity_element = CleanDecimal.French(
'//td[contains(text(), "Solde espèces en euros")]//following-sibling::td[position()=1]',
default=None)(self.doc)
if liquidity_element:
return create_french_liquidity(liquidity_element)
def iter_investment(self, account):
if account.balance == 0:
return
# Start with liquidities:
if account._liquidity:
yield create_french_liquidity(account._liquidity)
if self.old_website_connection:
self.old_accounts.stay_or_go().go_to_account(account.id)
if account._invpage:
for inv in account._invpage.iter_investment(currency=account.currency):
if not inv.code:
params = {'securityId': inv._security_id}
self.invest_detail.go(params=params)
if self.invest_detail.is_here():
inv.code, inv.code_type = self.page.get_isin_code_and_type()
yield inv
return
self.switch_account(account.id)
token = self.page.get_token()
try:
data = {'grouping': 'SecurityCategory'}
self.investment.go(data=data, headers=token)
except HTTPNotFound:
return
for inv in self.page.iter_investment(currency=account.currency):
yield inv
def get_investment(self, account):
if account.type == Account.TYPE_LIFE_INSURANCE:
if not account._external_website:
self.logger.warning(
'This account is limited, there is no available investment.'
)
return
self.assurancevie.stay_or_go()
if account._is_calie_account:
calie_details = self.open(account.url)
for inv in calie_details.page.iter_investment():
yield inv
else:
self.go_life_insurance_website()
assert self.av_list.is_here(
), 'Something went wrong during iter life insurance investments'
self.av_investments.go(life_insurance_id=account.id)
for inv in self.page.iter_investment():
yield inv
self.go_back_from_life_insurance_website()
elif hasattr(account, '_market_link') and account._market_link:
self.connexion_bourse()
for inv in self.location(
account._market_link).page.iter_investment():
yield inv
self.deconnexion_bourse()
elif account.id in self.get_bourse_accounts_ids():
yield create_french_liquidity(account.balance)
def iter_investment(self, account):
self.transactions.go()
if account._acctype == 'bank' and account.type in (Account.TYPE_PEA, Account.TYPE_MARKET):
if 'Liquidités' in account.label:
return [create_french_liquidity(account.balance)]
account = self.get_netfinca_account(account)
self.location(account._market_link)
assert self.bourse.is_here()
return self.page.iter_investment()
if account.id not in self.cache['invs']:
# do we still need it ?...
if account._acctype == "bank" and account._hasinv:
self.go_account_pages(account, "investment")
elif account._acctype == "investment":
self.go_wealth_pages(account)
investment_url = self.page.get_investment_url()
if investment_url is None:
self.logger.warning('no investment link for account %s, returning empty', account)
# fake data, don't cache it
return []
self.location(investment_url)
self.cache['invs'][account.id] = list(self.page.iter_investment(currency=account.currency))
return self.cache['invs'][account.id]
def get_investments(self, account):
if account is not None:
# the balance is highly dynamic, fetch it along with the investments to grab a snapshot
account.balance = CleanDecimal(None, replace_dots=True).filter(self.get_balance(account.type))
for line in self.doc.xpath('//table[@id="t_intraday"]/tbody/tr'):
if line.find_class('categorie') or line.find_class('detail') or line.find_class('detail02'):
continue
cols = line.findall('td')
inv = Investment()
inv.label = CleanText(None).filter(cols[self.COL_LABEL])
link = cols[self.COL_LABEL].xpath('./a[contains(@href, "cdReferentiel")]')[0]
inv.id = re.search('cdReferentiel=(.*)', link.attrib['href']).group(1)
inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1)
inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY], True)
inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE], True)
inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE], False)
inv.valuation = self.parse_decimal(cols[self.COL_VALUATION], True)
diff = cols[self.COL_PERF].text.strip()
if diff == "-":
inv.diff = NotAvailable
else:
inv.diff = CleanDecimal(None, replace_dots=True).filter(diff)
if is_isin_valid(inv.code):
inv.code_type = Investment.CODE_TYPE_ISIN
yield inv
if account.type != account.TYPE_MARKET:
valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]'))
yield create_french_liquidity(valuation)
def iter_investment(self, account):
today = datetime.date.today()
self.coming.go()
# unfortunately there doesn't seem to be a page indicating what's
# left to be repaid on each project, so let's sum...
valuations = {}
commissions = {}
for tr in self.page.iter_transactions():
if tr.date <= today:
continue
if tr.raw not in valuations:
valuations[tr.raw] = tr.amount
commissions[tr.raw] = tr.commission
else:
valuations[tr.raw] += tr.amount
commissions[tr.raw] += tr.commission
for label, value in valuations.items():
inv = Investment()
inv.label = label
inv.valuation = value
inv.diff = commissions[label]
yield inv
yield create_french_liquidity(account._liquidities)
def iter_investment(self, account):
# Start with liquidities:
if account._liquidity:
yield create_french_liquidity(account._liquidity)
''' Delete this part when old website is obsolete '''
if self.old_website_connection:
self.old_accounts.stay_or_go().go_to_account(account.id)
if account._invpage:
for inv in account._invpage.iter_investment(currency=account.currency):
if not inv.code:
params = {'securityId': inv._security_id}
self.invest_detail.go(params=params)
if self.invest_detail.is_here():
inv.code, inv.code_type = self.page.get_isin_code_and_type()
yield inv
return
self.switch_account(account.id)
token = self.page.get_token()
try:
data = {'grouping': 'SecurityCategory'}
self.investment.go(data=data, headers=token)
except HTTPNotFound:
return
for inv in self.page.iter_investment(currency=account.currency):
yield inv
def get_investments(self, account):
if account is not None:
# the balance is highly dynamic, fetch it along with the investments to grab a snapshot
account.balance = CleanDecimal(None, replace_dots=True).filter(self.get_balance(account.type))
for line in self.doc.xpath('//table[@id="t_intraday"]/tbody/tr'):
if line.find_class('categorie') or line.find_class('detail') or line.find_class('detail02'):
continue
cols = line.findall('td')
inv = Investment()
inv.label = CleanText(None).filter(cols[self.COL_LABEL])
link = cols[self.COL_LABEL].xpath('./a[contains(@href, "cdReferentiel")]')[0]
inv.id = re.search('cdReferentiel=(.*)', link.attrib['href']).group(1)
inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1)
inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY], True)
inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE], True)
inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE], False)
inv.valuation = self.parse_decimal(cols[self.COL_VALUATION], True)
diff = cols[self.COL_PERF].text.strip()
if diff == "-":
inv.diff = NotAvailable
else:
inv.diff = CleanDecimal(None, replace_dots=True).filter(diff)
if is_isin_valid(inv.code):
inv.code_type = Investment.CODE_TYPE_ISIN
yield inv
if account.type != account.TYPE_MARKET:
valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]'))
yield create_french_liquidity(valuation)
def iter_investment(self, account):
self.transactions.go()
if account._acctype == 'bank' and account.type in (Account.TYPE_PEA, Account.TYPE_MARKET):
if 'Liquidités' in account.label:
return [create_french_liquidity(account.balance)]
account = self.get_netfinca_account(account)
self.location(account._market_link)
assert self.bourse.is_here()
return self.page.iter_investment()
if account.id not in self.cache['invs']:
# do we still need it ?...
if account._acctype == "bank" and account._hasinv:
self.go_account_pages(account, "investment")
elif account._acctype == "investment":
self.go_wealth_pages(account)
investment_url = self.page.get_investment_url()
if investment_url is None:
self.logger.warning('no investment link for account %s, returning empty', account)
# fake data, don't cache it
return []
self.location(investment_url)
self.cache['invs'][account.id] = list(self.page.iter_investment(currency=account.currency))
return self.cache['invs'][account.id]
def iter_investment(self):
# Xpath can be h3/h4 or div/span; in both cases
# the first node contains "Solde Espèces":
valuation = CleanDecimal('//li/*[contains(text(), "Solde Espèces")]/following-sibling::*', replace_dots=True, default=None)(self.doc)
if valuation:
yield create_french_liquidity(valuation)
for inv in self.get_investment():
yield inv
def iter_investment(self, account):
if account.type in (Account.TYPE_PERP, Account.TYPE_PERCO, Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION):
if account.label == "Vers l'avenir":
# Website crashes when clicking on these Life Insurances...
return
self.go_to_account_space(account._contract)
token = self.token_page.go().get_token()
data = {
'situation_travail': 'CONTRAT',
'idelco': account.id,
':cq_csrf_token': token,
}
self.predica_redirection.go(data=data)
self.predica_investments.go()
for inv in self.page.iter_investments():
yield inv
elif account.type == Account.TYPE_PEA and account.label == 'Compte espèce PEA':
yield create_french_liquidity(account.balance)
return
elif account.type in (Account.TYPE_PEA, Account.TYPE_MARKET):
# Do not try to get to Netfinca if there is no money
# on the account or the server will return an error 500
if account.balance == 0:
return
self.go_to_account_space(account._contract)
token = self.token_page.go().get_token()
data = {
'situation_travail': 'BANCAIRE',
'num_compte': account.id,
'code_fam_produit': account._fam_product_code,
'code_fam_contrat_compte': account._fam_contract_code,
':cq_csrf_token': token,
}
# For some market accounts, investments are not even accessible,
# and the only way to know if there are investments is to try
# to go to the Netfinca space with the accounts parameters.
try:
self.netfinca_redirection.go(data=data)
except BrowserHTTPNotFound:
self.logger.info('Investments are not available for this account.')
self.go_to_account_space(account._contract)
return
url = self.page.get_url()
if 'netfinca' in url:
self.location(url)
self.netfinca.session.cookies.update(self.session.cookies)
self.netfinca.accounts.go()
for inv in self.netfinca.iter_investments(account):
if inv.code == 'XX-liquidity' and account.type == Account.TYPE_PEA:
# Liquidities are already fetched on the "PEA espèces"
continue
yield inv
def get_investment(self, account):
if account.type in (Account.TYPE_LOAN, ):
self.investments[account.id] = []
return []
# Add "Liquidities" investment if the account is a "Compte titres PEA":
if account.type == Account.TYPE_PEA and account.id.startswith('CPT'):
self.investments[account.id] = [
create_french_liquidity(account.balance)
]
return self.investments[account.id]
if account.id in self.investments.keys(
) and self.investments[account.id] is False:
raise NotImplementedError()
if account.id not in self.investments.keys():
self.investments[account.id] = []
try:
if self.go_investments(account, get_account=True):
# Redirection URL is https://www.linebourse.fr/ReroutageSJR
if "linebourse" in self.url:
# Eliminating the 3 letters prefix to match IDs on Linebourse:
linebourse_id = account.id[3:]
for inv in self.linebourse.iter_investment(
linebourse_id):
self.investments[account.id].append(inv)
if self.etna.is_here():
params = self.page.params
elif self.natixis_redirect.is_here():
# the url may contain a "#", so we cannot make a request to it, the params after "#" would be dropped
url = self.page.get_redirect()
self.logger.debug('using redirect url %s', url)
m = self.etna.match(url)
if not m:
# url can be contratPrev which is not investments
self.logger.debug(
'Unable to handle this kind of contract')
raise NotImplementedError()
params = m.groupdict()
if self.natixis_redirect.is_here() or self.etna.is_here():
try:
self.natixis_invest.go(**params)
except ServerError:
# Broken website .. nothing to do.
self.investments[account.id] = iter([])
return self.investments[account.id]
self.investments[account.id] = list(
self.page.get_investments())
except NotImplementedError:
self.investments[account.id] = []
return self.investments[account.id]
def iter_investments(self, account):
if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PEA,
Account.TYPE_MARKET, Account.TYPE_PERP):
return
# Add "Liquidities" investment if the account is a "Compte titres PEA":
if account.type == Account.TYPE_PEA and account.id.startswith('CPT'):
yield create_french_liquidity(account.balance)
return
if self.go_investments(account, get_account=True):
# Redirection URL is https://www.linebourse.fr/ReroutageSJR
if 'linebourse' in self.url:
self.logger.warning(
'Going to Linebourse space to fetch investments.')
# Eliminating the 3 letters prefix to match IDs on Linebourse:
linebourse_id = account.id[3:]
for inv in self.linebourse.iter_investment(linebourse_id):
yield inv
return
if self.etna.is_here():
self.logger.warning(
'Going to Etna space to fetch investments.')
params = self.page.params
elif self.natixis_redirect.is_here():
self.logger.warning(
'Going to Natixis space to fetch investments.')
# the url may contain a "#", so we cannot make a request to it, the params after "#" would be dropped
url = self.page.get_redirect()
self.logger.debug('using redirect url %s', url)
m = self.etna.match(url)
if not m:
# URL can be contratPrev which is not investments
self.logger.warning(
'Unable to handle this kind of contract.')
return
params = m.groupdict()
if self.natixis_redirect.is_here() or self.etna.is_here():
try:
self.natixis_invest.go(**params)
except ServerError:
# Broken website... nothing to do.
return
for inv in self.page.iter_investments():
yield inv
def get_investment(self, account):
if account._is_inv:
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
self.por.go(subbank=self.currentSubBank)
self.page.send_form(account)
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not account._link_inv:
return iter([])
self.location(account._link_inv)
return self.page.iter_investment()
if account.type is Account.TYPE_PEA:
liquidities = create_french_liquidity(account.balance)
liquidities.label = account.label
return [liquidities]
return iter([])
def get_investment(self, account):
if account._is_inv:
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
self.por.go(subbank=self.currentSubBank)
self.page.send_form(account)
elif account.type == Account.TYPE_LIFE_INSURANCE:
if not account._link_inv:
return iter([])
self.location(account._link_inv)
return self.page.iter_investment()
if account.type is Account.TYPE_PEA:
liquidities = create_french_liquidity(account.balance)
liquidities.label = account.label
return [liquidities]
return iter([])
def iter_investment(self, account):
if account.id not in self.invs:
staging = '_s' if 'staging' in self.sessionId else ''
self.accounts.stay_or_go(staging=staging,
accountId=self.intAccount,
sessionId=self.sessionId)
invests = list(
self.page.iter_investment(currency=account.currency))
# Replace as liquidities investments that are cash
self.invs[account.id] = [
create_french_liquidity(inv.valuation) if len(inv.label) < 4
and Currency.get_currency(inv.label) else inv
for inv in invests
]
return self.invs[account.id]
def iter_investment(self, account):
if account.type == Account.TYPE_PEA and 'espèces' in account.label.lower(
):
return [create_french_liquidity(account.balance)]
# Life insurances and PERP may be scraped from the API or from the "Assurance Vie" space,
# so we need to discriminate between both using account._details:
if account.type in (account.TYPE_LIFE_INSURANCE, account.TYPE_PERP,
account.TYPE_CAPITALISATION):
if hasattr(account, '_details'):
# Going to the "Assurances Vie" page
natiovie_params = self.natio_vie_pro.go().get_params()
self.capitalisation_page.go(params=natiovie_params)
# Fetching the form to get the contract investments:
capitalisation_params = self.page.get_params(account)
self.capitalisation_page.go(params=capitalisation_params)
return self.page.iter_investments()
else:
# No capitalisation contract has yet been found in the API:
assert account.type != account.TYPE_CAPITALISATION
self.lifeinsurances.go(json={
"ibanCrypte": account.id,
})
return self.page.iter_investments()
elif account.type in (account.TYPE_MARKET, account.TYPE_PEA):
try:
self.market_list.go(json={}, method='POST')
except ServerError:
self.logger.warning("An Internal Server Error occurred")
return iter([])
for market_acc in self.page.get_list():
if account.number[-4:] == market_acc['securityAccountNumber'][
-4:] and not account.iban:
# Sometimes generate an Internal Server Error ...
try:
self.market.go(
json={
"securityAccountNumber":
market_acc['securityAccountNumber'],
})
except ServerError:
self.logger.warning(
"An Internal Server Error occurred")
break
return self.page.iter_investments()
return iter([])
def get_investment(self, account):
if 'LIQUIDIT' in account.label:
return [create_french_liquidity(account.balance)]
if not account._inv:
return []
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
self.location(account._link, data=account._args)
if self.page.can_iter_investments() and self.page.not_restrained():
return self.page.get_market_investment()
elif account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION):
self.location(account._link, data=account._args)
self.location(account._link.replace("_attente", "_detail_contrat_rep"), data=account._args)
if self.page.can_iter_investments():
return self.page.get_li_investments()
return []
def get_investment(self, account):
if account.type == Account.TYPE_LIFE_INSURANCE:
if not account._form:
self.logger.warning('This account is limited, there is no available investment.')
return
self.assurancevie.stay_or_go()
# The website often returns an error so we try again:
# "L’accès au service est momentanément indisponible."
try:
account._form.submit()
except BrowserUnavailable:
self.logger.warning("Service unavailable, we submit the form again.")
self.assurancevie.stay_or_go()
account._form.submit()
if self.calie.is_here():
for inv in self.page.iter_investment():
yield inv
# Get back to Life Insurance space
self.assurancevie.go()
return
# Some users will get a message : "Ne détenant pas de compte dépôt
# chez LCL, l'accès à ce service vous est indisponible."
if self.form2.is_here() and self.page.assurancevie_hist_not_available():
return
self.avdetail.go()
self.av_investments.go()
for inv in self.page.iter_investment():
yield inv
self.avdetail.go()
self.page.come_back()
elif hasattr(account, '_market_link') and account._market_link:
self.connexion_bourse()
for inv in self.location(account._market_link).page.iter_investment():
yield inv
self.deconnexion_bourse()
elif account.id in self.get_bourse_accounts_ids():
yield create_french_liquidity(account.balance)
def iter_investment(self, account):
if account.balance == 0:
return
handled_invest_accounts = (
Account.TYPE_MARKET,
Account.TYPE_PEA,
Account.TYPE_LIFE_INSURANCE,
Account.TYPE_CAPITALISATION,
Account.TYPE_PERP,
Account.TYPE_PERCO,
)
if account.type not in handled_invest_accounts:
self.unhandled_method(account.id)
return
if account.label == 'DAV PEA':
# 'PEA Espèces'
yield create_french_liquidity(account.balance)
return
if account.url:
if 'PREDICA' in account.url:
# Fetch investments on Predica space
for inv in self.get_predica_investments(account):
yield inv
elif 'CATITRES' in account.url:
# Fetch investments on Netfinca space
for inv in self.get_netfinca_investments(account):
yield inv
elif 'bgpi' in account.url:
# Fetch investments on BGPI space
self.location(account.url)
if self.bgpi_investments.is_here():
for inv in self.page.iter_investments():
yield inv
# Go back to the main Cragr website afterwards
self.accounts.stay_or_go()
def get_investment(self, account):
if 'LIQUIDIT' in account.label:
return [create_french_liquidity(account.balance)]
if not account._inv:
return []
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
self.location(account._link, data=account._args)
if self.page.can_iter_investments() and self.page.not_restrained():
return self.page.get_market_investment()
elif account.type in (Account.TYPE_LIFE_INSURANCE,
Account.TYPE_CAPITALISATION):
self.location(account._link, data=account._args)
self.location(account._link.replace("_attente",
"_detail_contrat_rep"),
data=account._args)
if self.page.can_iter_investments():
return self.page.get_li_investments()
return []
def iter_investment(self, account):
if account.type == Account.TYPE_LIFE_INSURANCE:
if not self.goto_spirica(account):
return iter([])
return self.spirica.iter_investment(account)
elif account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
bourse_account = self.get_bourse_account(account)
if not bourse_account:
return iter([])
self.location(bourse_account._market_link)
assert self.bourse.is_here()
invs = list(self.page.iter_investment())
# _especes is set during BoursePage accounts parsing. BoursePage
# inherits from lcl module BoursePage
if bourse_account._especes:
invs.append(create_french_liquidity(bourse_account._especes))
self.leave_espace_bourse()
return invs
raise NotImplementedError()
def iter_investment(self, account):
if account.type == Account.TYPE_LIFE_INSURANCE:
if not self.goto_spirica(account):
return iter([])
return self.spirica.iter_investment(account)
elif account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
bourse_account = self.get_bourse_account(account)
if not bourse_account:
return iter([])
self.location(bourse_account._market_link)
assert self.bourse.is_here()
invs = list(self.page.iter_investment())
# _especes is set during BoursePage accounts parsing. BoursePage
# inherits from lcl module BoursePage
if bourse_account._especes:
invs.append(create_french_liquidity(bourse_account._especes))
self.leave_espace_bourse()
return invs
raise NotImplementedError()
def get_investment(self, account):
self.deleteCTX()
if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION, Account.TYPE_MARKET, Account.TYPE_PEA) or 'measure_id' in account._info:
raise NotImplementedError()
if account.type == Account.TYPE_PEA and account.label == 'PEA NUMERAIRE':
yield create_french_liquidity(account.balance)
return
if self.home.is_here():
self.page.go_list()
else:
self.home.go()
self.page.go_history(account._info)
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
# Some users may not have access to this.
if not self.market.is_here():
return
self.page.submit()
if 'offrebourse.com' in self.url:
# Some users may not have access to this.
if self.page.is_error():
return
self.update_linebourse_token()
for investment in self.linebourse.iter_investments(account.id):
yield investment
return
elif account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION):
if self.page.is_account_inactive(account.id):
self.logger.warning('Account %s %s is inactive.' % (account.label, account.id))
return
if "MILLEVIE" in account.label:
self.page.go_life_insurance(account)
label = account.label.split()[-1]
self.natixis_life_ins_inv.go(id1=label[:3], id2=label[3:5], id3=account.id)
for tr in self.page.get_investments():
yield tr
return
try:
self.page.go_life_insurance(account)
if not self.market.is_here() and not self.message.is_here():
# life insurance website is not always available
raise BrowserUnavailable()
self.page.submit()
self.location('https://www.extranet2.caisse-epargne.fr%s' % self.page.get_cons_repart())
except (IndexError, AttributeError) as e:
self.logger.error(e)
return
if self.garbage.is_here():
self.page.come_back()
return
for i in self.page.iter_investment():
yield i
if self.market.is_here():
self.page.come_back()
def get_liquidities(self):
value = CleanDecimal.French(CleanText('//a[starts-with(text(),"Compte de paiement")]'))(self.doc)
return create_french_liquidity(value)
def get_liquidity(self):
liquidity = CleanDecimal(
'//table//tr[@class="titreAvant"]/td[contains(text(), "Liquidit")]/following-sibling::td',
replace_dots=True)(self.doc)
if liquidity:
return create_french_liquidity(liquidity)
def get_investment(self, account):
self.deleteCTX()
if account.type not in (
Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION,
Account.TYPE_MARKET,
Account.TYPE_PEA) or 'measure_id' in account._info:
raise NotImplementedError()
if account.type == Account.TYPE_PEA and account.label == 'PEA NUMERAIRE':
yield create_french_liquidity(account.balance)
return
if self.home.is_here():
self.page.go_list()
else:
self.home.go()
self.page.go_history(account._info)
if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
# Some users may not have access to this.
if not self.market.is_here():
return
self.page.submit()
if 'offrebourse.com' in self.url:
# Some users may not have access to this.
if self.page.is_error():
return
self.update_linebourse_token()
for investment in self.linebourse.iter_investments(account.id):
yield investment
# We need to go back to the synthesis, else we can not go home later
self.home_tache.go(tache='CPTSYNT0')
return
elif account.type in (Account.TYPE_LIFE_INSURANCE,
Account.TYPE_CAPITALISATION):
if self.page.is_account_inactive(account.id):
self.logger.warning('Account %s %s is inactive.' %
(account.label, account.id))
return
if "MILLEVIE" in account.label:
try:
self.page.go_life_insurance(account)
except ServerError as ex:
if ex.response.status_code == 500 and 'MILLEVIE PREMIUM' in account.label:
self.logger.info(
"Can not reach investment page for MILLEVIE PREMIUM account"
)
return
raise
label = account.label.split()[-1]
self.natixis_life_ins_inv.go(id1=label[:3],
id2=label[3:5],
id3=account.id)
for tr in self.page.get_investments():
yield tr
return
try:
self.page.go_life_insurance(account)
if not self.market.is_here() and not self.message.is_here():
# life insurance website is not always available
raise BrowserUnavailable()
self.page.submit()
self.location('https://www.extranet2.caisse-epargne.fr%s' %
self.page.get_cons_repart())
except (IndexError, AttributeError) as e:
self.logger.error(e)
return
if self.garbage.is_here():
self.page.come_back()
return
for i in self.page.iter_investment():
yield i
if self.market.is_here():
self.page.come_back()
def iter_investments(self, account):
yield (create_french_liquidity(account.balance))
def iter_investments(self, account):
yield (create_french_liquidity(account.balance))
def get_liquidity(self):
liquidity_element = self.doc.xpath('//td[contains(text(), "Solde espèces en euros")]//following-sibling::td[position()=1]')
assert len(liquidity_element) <= 1
if liquidity_element:
valuation = CleanDecimal(replace_dots=True).filter(liquidity_element[0])
return create_french_liquidity(valuation)
def get_liquidity(self):
liquidity = CleanDecimal('//table//tr[@class="titreAvant"]/td[contains(text(), "Liquidit")]/following-sibling::td', replace_dots=True)(self.doc)
if liquidity:
return create_french_liquidity(liquidity)
def iter_investment(self, account):
if account.type == Account.TYPE_LIFE_INSURANCE and (
'rothschild' in account.label.lower() or re.match(
r'^open (perspective|strat)', account.label, re.I)):
self.life_insurance_investments.go(space=self.space,
idx=account._index,
category=account._category)
# TODO
#for inv in self.page.iter_investments():
# yield inv
elif account.type in (Account.TYPE_PERP, Account.TYPE_PERCO,
Account.TYPE_LIFE_INSURANCE,
Account.TYPE_CAPITALISATION):
if account.label == "Vers l'avenir":
# Website crashes when clicking on these Life Insurances...
return
self.go_to_account_space(account._contract)
token = self.token_page.go().get_token()
data = {
'situation_travail': 'CONTRAT',
'idelco': account.id,
':cq_csrf_token': token,
}
try:
self.predica_redirection.go(space=self.space, data=data)
except ServerError:
self.logger.warning(
'Got ServerError when fetching investments for account id %s',
account.id)
else:
self.predica_investments.go()
for inv in self.page.iter_investments():
yield inv
elif account.type == Account.TYPE_PEA and account.label == 'Compte espèce PEA':
yield create_french_liquidity(account.balance)
return
elif account.type in (Account.TYPE_PEA, Account.TYPE_MARKET):
# Do not try to get to Netfinca if there is no money
# on the account or the server will return an error 500
if account.balance == 0:
return
self.go_to_account_space(account._contract)
token = self.token_page.go().get_token()
data = {
'situation_travail': 'BANCAIRE',
'num_compte': account.id,
'code_fam_produit': account._fam_product_code,
'code_fam_contrat_compte': account._fam_contract_code,
':cq_csrf_token': token,
}
# For some market accounts, investments are not even accessible,
# and the only way to know if there are investments is to try
# to go to the Netfinca space with the accounts parameters.
try:
self.netfinca_redirection.go(space=self.space, data=data)
except BrowserHTTPNotFound:
self.logger.info(
'Investments are not available for this account.')
self.go_to_account_space(account._contract)
return
url = self.page.get_url()
if 'netfinca' in url:
self.location(url)
self.netfinca.session.cookies.update(self.session.cookies)
self.netfinca.accounts.go()
for inv in self.netfinca.iter_investments(account):
if inv.code == 'XX-liquidity' and account.type == Account.TYPE_PEA:
# Liquidities are already fetched on the "PEA espèces"
continue
yield inv
def iter_investments(self):
self.portfolio.go()
yield create_french_liquidity(self.page.get_liquidity())
for inv in self.page.iter_investments():
yield inv
def iter_investments(self, account):
# We did not get some html, but something like that (XX is a quantity, YY a price):
# "message='<total> €{<total> €{0,01 €{<liquidity> €{0,00{{05/17{{03/05/2017{11:06{-XX €{710TI81000029397EUR{XX €{XX €{|OPHTHOTECH(NASDAQ)#cotationValeur.php?val=OPHT&pl=11&nc=2&
# popup=2{6{E:ALO{PAR{{reel{695{380{ALSTOM REGROUPT#XX#YY,YY €#YY,YY €#1 YYY,YY €#-YYY,YY €#-42,42%#-0,98 %#42,42 %#|1|AXA#cotationValeur.php?val=E:CS&pl=6&nc=1&
# popup=2{6{E:CS{PAR{{reel{695{380{AXA#XX#YY,YY €#YY,YYY €#YYY,YY €#YY,YY €#3,70%#42,42 %#42,42 %#|1|blablablab #cotationValeur.php?val=P:CODE&pl=6&nc=1&
# [...]
data = self.browser.cache["investments_data"].get(account.id, self.doc)
lines = data.split("|1|")
message = lines[0]
if len(lines) > 1:
start = 1
lines[0] = lines[0].split("|")[1]
else:
start = 0
lines = data.split("popup=2")
lines.pop(0)
invests = []
for line in lines:
_id, _pl = None, None
columns = line.split('#')
if columns[1] != '':
_pl = columns[start].split('{')[1]
_id = columns[start].split('{')[2]
invest = Investment()
# If the link with the label and ISIN code is present we use it to fill the label and code.
# If not, the label can still be found in the first column of the row but the ISIN is unavailable.
invest.label = columns[start].split('{')[-1] or columns[0]
invest.code = _id or NotAvailable
if invest.code and ':' in invest.code:
invest.code = self.browser.titrevalue.open(val=invest.code,
pl=_pl).get_isin()
# The code we got is not a real ISIN code.
if invest.code and not re.match(
'^[A-Z]{2}[\d]{10}$|^[A-Z]{2}[\d]{5}[A-Z]{1}[\d]{4}$',
invest.code):
m = re.search(
'\{([A-Z]{2}[\d]{10})\{|\{([A-Z]{2}[\d]{5}[A-Z]{1}[\d]{4})\{',
line)
if m:
invest.code = m.group(1) or m.group(2)
for x, attr in enumerate(
['quantity', 'unitprice', 'unitvalue', 'valuation', 'diff'],
1):
currency = FrenchTransaction.Currency().filter(columns[start +
x])
amount = CleanDecimal(default=NotAvailable).filter(
FrenchTransaction.clean_amount(columns[start + x]))
if currency and currency != account.currency:
invest.original_currency = currency
attr = "original_" + attr
setattr(invest, attr, amount)
# valuation is not nullable, use 0 as default value
if not invest.valuation:
invest.valuation = Decimal('0')
# On some case we have a multine investment with a total column
# for now we have only see this on 2 lines, we will need to adapt it when o
if columns[9 if start ==
0 else 0] == '|Total' and _id == 'fichevaleur':
prev_inv = invest
invest = invests.pop(-1)
if prev_inv.quantity:
invest.quantity = invest.quantity + prev_inv.quantity
if prev_inv.valuation:
invest.valuation = invest.valuation + prev_inv.valuation
if prev_inv.diff:
invest.diff = invest.diff + prev_inv.diff
invests.append(invest)
# There is no investment on life insurance in the process to be created.
if len(message.split('&')) >= 4:
# We also have to get the liquidity as an investment.
valuation = CleanDecimal(None, True).filter(
message.split('&')[3].replace('euro;{', '').strip())
invests.append(create_french_liquidity(valuation))
for invest in invests:
yield invest
def iter_investments(self, account):
# We did not get some html, but something like that (XX is a quantity, YY a price):
# "message='<total> €{<total> €{0,01 €{<liquidity> €{0,00{{05/17{{03/05/2017{11:06{-XX €{710TI81000029397EUR{XX €{XX €{|OPHTHOTECH(NASDAQ)#cotationValeur.php?val=OPHT&pl=11&nc=2&
# popup=2{6{E:ALO{PAR{{reel{695{380{ALSTOM REGROUPT#XX#YY,YY €#YY,YY €#1 YYY,YY €#-YYY,YY €#-42,42%#-0,98 %#42,42 %#|1|AXA#cotationValeur.php?val=E:CS&pl=6&nc=1&
# popup=2{6{E:CS{PAR{{reel{695{380{AXA#XX#YY,YY €#YY,YYY €#YYY,YY €#YY,YY €#3,70%#42,42 %#42,42 %#|1|blablablab #cotationValeur.php?val=P:CODE&pl=6&nc=1&
# [...]
data = self.browser.cache["investments_data"].get(account.id, self.doc)
lines = data.split("|1|")
message = lines[0]
if len(lines) > 1:
start = 1
lines[0] = lines[0].split("|")[1]
else:
start = 0
lines = data.split("popup=2")
lines.pop(0)
invests = []
for line in lines:
_id, _pl = None, None
columns = line.split('#')
if columns[1] != '':
_pl = columns[start].split('{')[1]
_id = columns[start].split('{')[2]
invest = Investment()
invest.label = columns[start].split('{')[-1]
invest.code = _id or NotAvailable
if invest.code and ':' in invest.code:
invest.code = self.browser.titrevalue.open(val=invest.code,pl=_pl).get_isin()
# The code we got is not a real ISIN code.
if invest.code and not re.match('^[A-Z]{2}[\d]{10}$|^[A-Z]{2}[\d]{5}[A-Z]{1}[\d]{4}$', invest.code):
m = re.search('\{([A-Z]{2}[\d]{10})\{|\{([A-Z]{2}[\d]{5}[A-Z]{1}[\d]{4})\{', line)
if m:
invest.code = m.group(1) or m.group(2)
for x, attr in enumerate(['quantity', 'unitprice', 'unitvalue', 'valuation', 'diff'], 1):
currency = FrenchTransaction.Currency().filter(columns[start + x])
amount = CleanDecimal(default=NotAvailable).filter(FrenchTransaction.clean_amount(columns[start + x]))
if currency and currency != account.currency:
invest.original_currency = currency
attr = "original_" + attr
setattr(invest, attr, amount)
# valuation is not nullable, use 0 as default value
if not invest.valuation:
invest.valuation = Decimal('0')
# On some case we have a multine investment with a total column
# for now we have only see this on 2 lines, we will need to adapt it when o
if columns[9 if start == 0 else 0] == u'|Total' and _id == 'fichevaleur':
prev_inv = invest
invest = invests.pop(-1)
if prev_inv.quantity:
invest.quantity = invest.quantity + prev_inv.quantity
if prev_inv.valuation:
invest.valuation = invest.valuation + prev_inv.valuation
if prev_inv.diff:
invest.diff = invest.diff + prev_inv.diff
invests.append(invest)
# There is no investment on life insurance in the process to be created.
if len(message.split('&')) >= 4:
# We also have to get the liquidity as an investment.
valuation = CleanDecimal(None, True).filter(message.split('&')[3].replace('euro;{','').strip())
invests.append(create_french_liquidity(valuation))
for invest in invests:
yield invest
