def get_investments(self, account):
for line in self.doc.xpath('//table[@id="tableau_support"]/tbody/tr'):
cols = line.findall('td')
inv = Investment()
inv.id = re.search(
'cdReferentiel=(.*)',
cols[self.COL_LABEL].find('a').attrib['href']).group(1)
inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1)
inv.label = CleanText(None).filter(cols[self.COL_LABEL])
inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY])
inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE])
inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE])
inv.vdate = Date(CleanText(cols[self.COL_DATE],
default=NotAvailable),
dayfirst=True,
default=NotAvailable)(self.doc)
inv.valuation = self.parse_decimal(cols[self.COL_VALUATION])
inv.diff = self.parse_decimal(cols[self.COL_PERF])
diff_percent = self.parse_decimal(cols[self.COL_PERF_PERCENT])
inv.diff_percent = diff_percent / 100 if diff_percent else NotAvailable
if is_isin_valid(inv.code):
inv.code_type = Investment.CODE_TYPE_ISIN
yield inv
def get_investments(self, account):
if account is not None:
# the balance is highly dynamic, fetch it along with the investments to grab a snapshot
account.balance = CleanDecimal(None, replace_dots=True).filter(self.get_balance(account.type))
for line in self.doc.xpath('//table[@id="t_intraday"]/tbody/tr'):
if line.find_class('categorie') or line.find_class('detail') or line.find_class('detail02'):
continue
cols = line.findall('td')
inv = Investment()
inv.label = CleanText(None).filter(cols[self.COL_LABEL])
link = cols[self.COL_LABEL].xpath('./a[contains(@href, "cdReferentiel")]')[0]
inv.id = re.search('cdReferentiel=(.*)', link.attrib['href']).group(1)
inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1)
inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY], True)
inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE], True)
inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE], False)
inv.valuation = self.parse_decimal(cols[self.COL_VALUATION], True)
diff = cols[self.COL_PERF].text.strip()
if diff == "-":
inv.diff = NotAvailable
else:
inv.diff = CleanDecimal(None, replace_dots=True).filter(diff)
if is_isin_valid(inv.code):
inv.code_type = Investment.CODE_TYPE_ISIN
yield inv
if account.type != account.TYPE_MARKET:
valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]'))
yield create_french_liquidity(valuation)
def get_investments(self, account):
if account is not None:
# the balance is highly dynamic, fetch it along with the investments to grab a snapshot
account.balance = CleanDecimal(None, replace_dots=True).filter(self.get_balance(account.type))
for line in self.doc.xpath('//table[@id="t_intraday"]/tbody/tr'):
if line.find_class('categorie') or line.find_class('detail') or line.find_class('detail02'):
continue
cols = line.findall('td')
inv = Investment()
inv.label = CleanText(None).filter(cols[self.COL_LABEL])
link = cols[self.COL_LABEL].xpath('./a[contains(@href, "cdReferentiel")]')[0]
inv.id = re.search('cdReferentiel=(.*)', link.attrib['href']).group(1)
inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1)
inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY], True)
inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE], True)
inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE], False)
inv.valuation = self.parse_decimal(cols[self.COL_VALUATION], True)
diff = cols[self.COL_PERF].text.strip()
if diff == "-":
inv.diff = NotAvailable
else:
inv.diff = CleanDecimal(None, replace_dots=True).filter(diff)
if is_isin_valid(inv.code):
inv.code_type = Investment.CODE_TYPE_ISIN
yield inv
if account.type != account.TYPE_MARKET:
valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]'))
yield create_french_liquidity(valuation)
def obj_code(self):
code = CleanText(TableCell('code'))(self)
# there is no example of invests without valid ISIN code
# wait for it to retrieve them corretly
assert is_isin_valid(
code
), 'This code is not a valid ISIN, please check what invest is it.'
return code
def iter_investment(self):
not_rounded_valuations = self.get_not_rounded_valuations()
doc = self.browser.open('/brs/fisc/fisca10a.html').page.doc
num_page = None
try:
num_page = int(CleanText('.')(doc.xpath(u'.//tr[contains(td[1], "Relevé des plus ou moins values latentes")]/td[2]')[0]).split('/')[1])
except IndexError:
pass
docs = [doc]
if num_page:
for n in range(2, num_page + 1):
docs.append(self.browser.open('%s%s' % ('/brs/fisc/fisca10a.html?action=12&numPage=', str(n))).page.doc)
for doc in docs:
# There are two different tables possible depending on the market account type.
is_detailed = bool(doc.xpath(u'//span[contains(text(), "Années d\'acquisition")]'))
tr_xpath = '//tr[@height and td[@colspan="6"]]' if is_detailed else '//tr[count(td)>5]'
for tr in doc.xpath(tr_xpath):
cells = tr.findall('td')
inv = Investment()
title_split = cells[self.COL_LABEL].xpath('.//span')[0].attrib['title'].split(' - ')
inv.label = unicode(title_split[0])
for code in title_split[1:]:
if is_isin_valid(code):
inv.code = unicode(code)
inv.code_type = Investment.CODE_TYPE_ISIN
break
else:
inv.code = NotAvailable
inv.code_type = NotAvailable
if is_detailed:
inv.quantity = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[2]')[0])
inv.unitprice = MyDecimal('.', replace_dots=True)(tr.xpath('./following-sibling::tr/td[3]')[1])
inv.unitvalue = MyDecimal('.', replace_dots=True)(tr.xpath('./following-sibling::tr/td[3]')[0])
try: # try to get not rounded value
inv.valuation = not_rounded_valuations[inv.label]
except KeyError: # ok.. take it from the page
inv.valuation = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[4]')[0])
inv.diff = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[5]')[0]) or \
MyDecimal('.')(tr.xpath('./following-sibling::tr/td[6]')[0])
else:
inv.quantity = MyDecimal('.')(cells[self.COL_QUANTITY])
inv.diff = MyDecimal('.')(cells[self.COL_DIFF])
inv.unitprice = MyDecimal('.')(cells[self.COL_UNITPRICE].xpath('.//tr[1]/td[2]')[0])
inv.unitvalue = MyDecimal('.')(cells[self.COL_VALUATION].xpath('.//tr[1]/td[2]')[0])
inv.valuation = MyDecimal('.')(cells[self.COL_VALUATION].xpath('.//tr[2]/td[2]')[0])
yield inv
def check_investment(self, account, inv):
self.assertTrue(inv.label, 'investment %r has no label' % inv)
self.assertFalse(empty(inv.valuation), 'investment %r has no valuation' % inv)
if inv.code and inv.code != 'XX-liquidity':
self.assertTrue(inv.code_type, 'investment %r has code but no code type' % inv)
if inv.code_type == inv.CODE_TYPE_ISIN and inv.code and not inv.code.startswith('XX'):
self.assertTrue(is_isin_valid(inv.code), 'investment %r has invalid ISIN: %r' % (inv, inv.code))
if not empty(inv.portfolio_share):
self.assertTrue(0 < inv.portfolio_share <= 1, 'investment %r has invalid portfolio_share' % inv)
def check_investment(self, account, inv):
self.assertTrue(inv.label, 'investment %r has no label' % inv)
self.assertTrue(inv.valuation, 'investment %r has no valuation' % inv)
if inv.code and inv.code != 'XX-liquidity':
self.assertTrue(inv.code_type, 'investment %r has code but no code type' % inv)
if inv.code_type == inv.CODE_TYPE_ISIN and inv.code and not inv.code.startswith('XX'):
self.assertTrue(is_isin_valid(inv.code), 'investment %r has invalid ISIN: %r' % (inv, inv.code))
if not empty(inv.portfolio_share):
self.assertTrue(0 < inv.portfolio_share <= 1, 'investment %r has invalid portfolio_share' % inv)
class item(ItemElement):
klass = Investment
def condition(self):
# Some rows do not contain an expected item format,
# There is no valuation (mnt) because some buy/sell orders are not yet finished.
# We want invalid values to fail in the CleanDecimal filter so we catch only when mnt is missing
return Dict('mnt',
default=NotAvailable)(self) is not NotAvailable
obj_label = Dict('libval')
obj_code = Dict('codval')
obj_code_type = Eval(
lambda x: Investment.CODE_TYPE_ISIN
if is_isin_valid(x) else NotAvailable, Field('code'))
obj_quantity = CleanDecimal(Dict('qttit'))
obj_unitvalue = CleanDecimal(Dict('crs'))
obj_valuation = CleanDecimal(Dict('mnt'))
obj_vdate = Env('date')
def parse(self, el):
symbols = {
'+': 1,
'-': -1,
'\u0000': None, # "NULL" character
}
self.env['sign'] = symbols.get(Dict('signePlv')(self), None)
def obj_diff(self):
if Dict('plv', default=None)(self) and Env('sign')(self):
return CleanDecimal(Dict('plv'),
sign=lambda x: Env('sign')(self))(self)
return NotAvailable
def obj_unitprice(self):
if Dict('pam', default=None)(self):
return CleanDecimal(Dict('pam'))(self)
return NotAvailable
def obj_diff_percent(self):
if not Env('sign')(self):
return NotAvailable
# obj_diff_percent key can have several names:
if Dict('plvPourcentage', default=None)(self):
return CleanDecimal(Dict('plvPourcentage'),
sign=lambda x: Env('sign')(self))(self)
elif Dict('pourcentagePlv', default=None)(self):
return CleanDecimal(Dict('pourcentagePlv'),
sign=lambda x: Env('sign')(self))(self)
def obj_portfolio_share(self):
active_percent = Dict('pourcentageActif',
default=NotAvailable)(self)
if empty(active_percent):
return NotAvailable
return Eval(lambda x: x / 100,
CleanDecimal(active_percent))(self)
def obj_code(self):
code = self._product()['isin']
if is_isin_valid(code):
# Prefix CFD (Contrats for difference) ISIN codes with "XX-"
# to avoid id_security duplicates in the database
if "- CFD" in Field('label')(self):
return "XX-" + code
return code
return NotAvailable
def obj_code(self):
# We try to get the code from <a> div. If we didn't find code in url,
# we try to find it in the cell text
tablecell = TableCell('label', colspan=True)(self)[0]
# url find try
code_match = Regexp(
Link(tablecell.xpath('./following-sibling::td[position()=1]/div/a')),
r'sico=([A-Z0-9]*)',
default=None)(self)
if is_isin_valid(code_match):
return code_match
# cell text find try
text = CleanText(tablecell.xpath('./following-sibling::td[position()=1]/div')[0])(self)
for code in text.split(' '):
if is_isin_valid(code):
return code
return NotAvailable
def obj_code(self):
# We try to get the code from <a> div. If we didn't find code in url,
# we try to find it in the cell text
tablecell = TableCell('label', colspan=True)(self)[0]
# url find try
url = tablecell.xpath('./following-sibling::td[position()=1]/div/a')[0].attrib['href']
code_match = re.search(r'sico=([A-Z0-9]*)', url)
if code_match:
if is_isin_valid(code_match.group(1)):
return code_match.group(1)
# cell text find try
text = CleanText(tablecell.xpath('./following-sibling::td[position()=1]/div')[0])(self)
for code in text.split(' '):
if is_isin_valid(code):
return code
return NotAvailable
def get_investments(self, account):
for line in self.doc.xpath('//table[@id="tableau_support"]/tbody/tr'):
cols = line.findall('td')
inv = Investment()
inv.id = re.search('cdReferentiel=(.*)', cols[self.COL_LABEL].find('a').attrib['href']).group(1)
inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1)
inv.label = CleanText(None).filter(cols[self.COL_LABEL])
inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY])
inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE])
inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE])
inv.vdate = Date(CleanText(cols[self.COL_DATE], default=NotAvailable), default=NotAvailable)(self.doc)
inv.valuation = self.parse_decimal(cols[self.COL_VALUATION])
inv.diff = self.parse_decimal(cols[self.COL_PERF])
diff_percent = self.parse_decimal(cols[self.COL_PERF_PERCENT])
inv.diff_percent = diff_percent / 100 if diff_percent else NotAvailable
if is_isin_valid(inv.code):
inv.code_type = Investment.CODE_TYPE_ISIN
yield inv
def iter_investment(self, account, invs=None):
if account.id not in self.investments and invs is not None:
self.investments[account.id] = []
for inv in invs:
i = Investment()
# If nothing is given to make the label, we use the ISIN instead
# We let it crash if the ISIN is not available either.
if all([inv['classification'], inv['description']]):
i.label = "%s - %s" % (inv['classification'],
inv['description'])
else:
i.label = Coalesce().filter((
inv['classification'],
inv['description'],
inv['isin'],
))
i.code = inv['isin']
if not is_isin_valid(i.code):
i.code = NotAvailable
i.code_type = NotAvailable
if u'Solde Espèces' in i.label:
i.code = 'XX-liquidity'
else:
i.code_type = Investment.CODE_TYPE_ISIN
i.quantity = CleanDecimal(default=NotAvailable).filter(
inv['nombreParts'])
i.unitprice = CleanDecimal(default=NotAvailable).filter(
inv['prixMoyenAchat'])
i.unitvalue = CleanDecimal(default=NotAvailable).filter(
inv['valeurCotation'])
i.valuation = CleanDecimal().filter(inv['montantEuro'])
# For some invests the vdate returned is None
# Consequently we set the default value at NotAvailable
i.vdate = Date(default=NotAvailable).filter(
inv['datePosition'])
i.diff = CleanDecimal(default=NotAvailable).filter(
inv['performanceEuro'])
self.investments[account.id].append(i)
return self.investments[account.id]
def obj_code(self):
code = Regexp(Link('./th/a', default=''),
r'isin=(\w+)|/(\w+)\.pdf',
default=NotAvailable)(self)
return code if is_isin_valid(code) else NotAvailable
def obj_code(self):
code = CleanText(Dict('code'))(self)
if is_isin_valid(code):
return code
return NotAvailable
def obj_code(self):
code = Dict('codeISIN')(self)
if is_isin_valid(code):
return code
return NotAvailable
def obj_code(self):
code = Regexp(Link('./th/a', default=''), r'isin=(\w+)|/(\w+)\.pdf', default=NotAvailable)(self)
return code if is_isin_valid(code) else NotAvailable
def obj_code(self):
for code in Field('label')(self).split():
if is_isin_valid(code):
return code
return NotAvailable
def obj_code_type(self):
if is_isin_valid(CleanText(Dict('code'))(self)):
return Investment.CODE_TYPE_ISIN
return NotAvailable
def obj_code(self):
code = CleanText(Dict('code'))(self)
if is_isin_valid(code):
return code
return NotAvailable
def obj_code_type(self):
if is_isin_valid(Field('code')(self)):
return Investment.CODE_TYPE_ISIN
return NotAvailable
def obj_code_type(self):
code = Field('code')(self)
if code and is_isin_valid(code):
return Investment.CODE_TYPE_ISIN
return NotAvailable
def obj_code(self):
if is_isin_valid(Dict('IsinCode')(self)):
return Dict('IsinCode')(self)
elif "espèces" in Field('label')(self).lower():
return "XX-liquidity"
return NotAvailable
def obj_code(self):
code = Dict('cdsptisn')(self)
if is_isin_valid(code):
return code
return NotAvailable
def obj_code_type(self):
if is_isin_valid(self.obj_code()):
return Investment.CODE_TYPE_ISIN
return NotAvailable
def obj_code(self):
for code in Field('label')(self).split():
if is_isin_valid(code):
return code
return NotAvailable
def get_isin_code_and_type(self):
code = CleanText('//td[strong[text()="ISIN"]]/following-sibling::td[1]', default=NotAvailable)(self.doc)
if is_isin_valid(code):
return code, Investment.CODE_TYPE_ISIN
return NotAvailable, NotAvailable
def obj_code_type(self):
return Investment.CODE_TYPE_ISIN if is_isin_valid(
Field('code')(self)) else NotAvailable
def obj_code(self):
if Field('label')(self) == "LIQUIDITES":
return 'XX-liquidity'
code = CleanText(TableCell('code'))(self)
return code if is_isin_valid(code) else NotAvailable
def get_isin_code_and_type(self):
code = CleanText('//td[strong[text()="ISIN"]]/following-sibling::td[1]', default=NotAvailable)(self.doc)
if is_isin_valid(code):
return code, Investment.CODE_TYPE_ISIN
return NotAvailable, NotAvailable
def obj_code_type(self):
if is_isin_valid(CleanText(Dict('code'))(self)):
return Investment.CODE_TYPE_ISIN
return NotAvailable
def obj_code_type(self):
if is_isin_valid(self.obj_code()):
return Investment.CODE_TYPE_ISIN
return NotAvailable
def obj_code(self):
if Field('label')(self) == "LIQUIDITES":
return 'XX-liquidity'
code = CleanText(TableCell('code'))(self)
return code if is_isin_valid(code) else NotAvailable
def parse(self, el):
# Trying to find vdate and unitvalue
unitvalue, vdate = None, None
for span in TableCell('label')(self)[0].xpath('.//span'):
if unitvalue is None:
unitvalue = Regexp(CleanText('.'), '^([\d,]+)$', default=None)(span)
if vdate is None:
vdate = None if any(x in CleanText('./parent::div')(span) for x in ["échéance", "Maturity"]) else \
Regexp(CleanText('.'), '^([\d\/]+)$', default=None)(span)
self.env['unitvalue'] = MyDecimal().filter(unitvalue) if unitvalue else NotAvailable
self.env['vdate'] = Date(dayfirst=True).filter(vdate) if vdate else NotAvailable
self.env['_link'] = None
self.env['asset_category'] = NotAvailable
page = None
link_id = Attr(u'.//a[contains(@title, "détail du fonds")]', 'id', default=None)(self)
inv_id = Attr('.//a[contains(@id, "linkpdf")]', 'id', default=None)(self)
if link_id and inv_id:
form = self.page.get_form('//div[@id="operation"]//form')
form['idFonds'] = inv_id.split('-', 1)[-1]
form['org.richfaces.ajax.component'] = form[link_id] = link_id
page = self.page.browser.open(form['javax.faces.encodedURL'], data=dict(form)).page
if 'hsbc.fr' in self.page.browser.BASEURL:
# Special space for HSBC, does not contain any information related to performances.
m = re.search(r'fundid=(\w+).+SH=(\w+)', CleanText('//complete', default='')(page.doc))
if m: # had to put full url to skip redirections.
page = page.browser.open('https://www.assetmanagement.hsbc.com/feedRequest?feed_data=gfcFundData&cod=FR&client=FCPE&fId=%s&SH=%s&lId=fr' % m.groups()).page
elif not self.page.browser.history.is_here():
url = page.get_invest_url()
if empty(url):
self.env['code'] = NotAvailable
self.env['code_type'] = NotAvailable
return
# URLs used in browser.py to access investments performance history:
if url.startswith('https://optimisermon.epargne-retraite-entreprises'):
# This URL can be used to access the BNP Wealth API to fetch investment performance and ISIN code
self.env['_link'] = url
self.env['code'] = NotAvailable
self.env['code_type'] = NotAvailable
return
elif (url.startswith('http://sggestion-ede.com/product') or
url.startswith('https://www.lyxorfunds.com/part') or
url.startswith('https://www.societegeneralegestion.fr') or
url.startswith('http://www.etoile-gestion.com/productsheet')):
self.env['_link'] = url
# Try to fetch ISIN code from URL with re.match
match = re.match(r'http://www.cpr-am.fr/fr/fonds_detail.php\?isin=([A-Z0-9]+)', url)
match = match or re.match(r'http://www.cpr-am.fr/particuliers/product/view/([A-Z0-9]+)', url)
if match:
self.env['code'] = match.group(1)
if is_isin_valid(match.group(1)):
self.env['code_type'] = Investment.CODE_TYPE_ISIN
else:
self.env['code_type'] = Investment.CODE_TYPE_AMF
return
# Try to fetch ISIN code from URL with re.search
m = re.search(r'&ISIN=([^&]+)', url)
m = m or re.search(r'&isin=([^&]+)', url)
m = m or re.search(r'&codeIsin=([^&]+)', url)
m = m or re.search(r'lyxorfunds\.com/part/([^/]+)', url)
if m:
self.env['code'] = m.group(1)
if is_isin_valid(m.group(1)):
self.env['code_type'] = Investment.CODE_TYPE_ISIN
else:
self.env['code_type'] = Investment.CODE_TYPE_AMF
return
useless_urls = (
# pdf... http://docfinder.is.bnpparibas-ip.com/api/files/040d05b3-1776-4991-aa49-f0cd8717dab8/1536
'http://docfinder.is.bnpparibas-ip.com/',
# The AXA website displays performance graphs but everything is calculated using JS scripts.
# There is an API but it only contains risk data and performances per year, not 1-3-5 years.
'https://epargne-salariale.axa-im.fr/fr/',
# Redirection to the Rothschild Gestion website, which doesn't exist anymore...
'https://www.rothschildgestion.com',
# URL to the Morningstar website does not contain any useful information
'http://doc.morningstar.com',
)
for useless_url in useless_urls:
if url.startswith(useless_url):
self.env['code'] = NotAvailable
self.env['code_type'] = NotAvailable
return
if url.startswith('http://fr.swisslife-am.com/fr/'):
self.page.browser.session.cookies.set('location', 'fr')
self.page.browser.session.cookies.set('prof', 'undefined')
try:
page = self.page.browser.open(url).page
except HTTPNotFound:
# Some pages lead to a 404 so we must avoid unnecessary crash
self.logger.warning('URL %s was not found, investment details will be skipped.', url)
if isinstance(page, CodePage):
self.env['code'] = page.get_code()
self.env['code_type'] = page.CODE_TYPE
self.env['asset_category'] = page.get_asset_category()
else:
# The page is not handled and does not have a get_code method.
self.env['code'] = NotAvailable
self.env['code_type'] = NotAvailable
self.env['asset_category'] = NotAvailable
def obj_code(self):
code = CleanText('.//span[@class="cl-secondary"]')(self)
if is_isin_valid(code):
return code
return NotAvailable
def obj_code(self):
if is_isin_valid(Dict('IsinCode')(self)):
return Dict('IsinCode')(self)
elif "espèces" in Field('label')(self).lower():
return "XX-liquidity"
return NotAvailable
