def get_investments(self, account): for line in self.doc.xpath('//table[@id="tableau_support"]/tbody/tr'): cols = line.findall('td') inv = Investment() inv.id = re.search( 'cdReferentiel=(.*)', cols[self.COL_LABEL].find('a').attrib['href']).group(1) inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1) inv.label = CleanText(None).filter(cols[self.COL_LABEL]) inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY]) inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE]) inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE]) inv.vdate = Date(CleanText(cols[self.COL_DATE], default=NotAvailable), dayfirst=True, default=NotAvailable)(self.doc) inv.valuation = self.parse_decimal(cols[self.COL_VALUATION]) inv.diff = self.parse_decimal(cols[self.COL_PERF]) diff_percent = self.parse_decimal(cols[self.COL_PERF_PERCENT]) inv.diff_percent = diff_percent / 100 if diff_percent else NotAvailable if is_isin_valid(inv.code): inv.code_type = Investment.CODE_TYPE_ISIN yield inv
def get_investments(self, account): if account is not None: # the balance is highly dynamic, fetch it along with the investments to grab a snapshot account.balance = CleanDecimal(None, replace_dots=True).filter(self.get_balance(account.type)) for line in self.doc.xpath('//table[@id="t_intraday"]/tbody/tr'): if line.find_class('categorie') or line.find_class('detail') or line.find_class('detail02'): continue cols = line.findall('td') inv = Investment() inv.label = CleanText(None).filter(cols[self.COL_LABEL]) link = cols[self.COL_LABEL].xpath('./a[contains(@href, "cdReferentiel")]')[0] inv.id = re.search('cdReferentiel=(.*)', link.attrib['href']).group(1) inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1) inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY], True) inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE], True) inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE], False) inv.valuation = self.parse_decimal(cols[self.COL_VALUATION], True) diff = cols[self.COL_PERF].text.strip() if diff == "-": inv.diff = NotAvailable else: inv.diff = CleanDecimal(None, replace_dots=True).filter(diff) if is_isin_valid(inv.code): inv.code_type = Investment.CODE_TYPE_ISIN yield inv if account.type != account.TYPE_MARKET: valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]')) yield create_french_liquidity(valuation)
def obj_code(self): code = CleanText(TableCell('code'))(self) # there is no example of invests without valid ISIN code # wait for it to retrieve them corretly assert is_isin_valid( code ), 'This code is not a valid ISIN, please check what invest is it.' return code
def iter_investment(self): not_rounded_valuations = self.get_not_rounded_valuations() doc = self.browser.open('/brs/fisc/fisca10a.html').page.doc num_page = None try: num_page = int(CleanText('.')(doc.xpath(u'.//tr[contains(td[1], "Relevé des plus ou moins values latentes")]/td[2]')[0]).split('/')[1]) except IndexError: pass docs = [doc] if num_page: for n in range(2, num_page + 1): docs.append(self.browser.open('%s%s' % ('/brs/fisc/fisca10a.html?action=12&numPage=', str(n))).page.doc) for doc in docs: # There are two different tables possible depending on the market account type. is_detailed = bool(doc.xpath(u'//span[contains(text(), "Années d\'acquisition")]')) tr_xpath = '//tr[@height and td[@colspan="6"]]' if is_detailed else '//tr[count(td)>5]' for tr in doc.xpath(tr_xpath): cells = tr.findall('td') inv = Investment() title_split = cells[self.COL_LABEL].xpath('.//span')[0].attrib['title'].split(' - ') inv.label = unicode(title_split[0]) for code in title_split[1:]: if is_isin_valid(code): inv.code = unicode(code) inv.code_type = Investment.CODE_TYPE_ISIN break else: inv.code = NotAvailable inv.code_type = NotAvailable if is_detailed: inv.quantity = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[2]')[0]) inv.unitprice = MyDecimal('.', replace_dots=True)(tr.xpath('./following-sibling::tr/td[3]')[1]) inv.unitvalue = MyDecimal('.', replace_dots=True)(tr.xpath('./following-sibling::tr/td[3]')[0]) try: # try to get not rounded value inv.valuation = not_rounded_valuations[inv.label] except KeyError: # ok.. take it from the page inv.valuation = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[4]')[0]) inv.diff = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[5]')[0]) or \ MyDecimal('.')(tr.xpath('./following-sibling::tr/td[6]')[0]) else: inv.quantity = MyDecimal('.')(cells[self.COL_QUANTITY]) inv.diff = MyDecimal('.')(cells[self.COL_DIFF]) inv.unitprice = MyDecimal('.')(cells[self.COL_UNITPRICE].xpath('.//tr[1]/td[2]')[0]) inv.unitvalue = MyDecimal('.')(cells[self.COL_VALUATION].xpath('.//tr[1]/td[2]')[0]) inv.valuation = MyDecimal('.')(cells[self.COL_VALUATION].xpath('.//tr[2]/td[2]')[0]) yield inv
def check_investment(self, account, inv): self.assertTrue(inv.label, 'investment %r has no label' % inv) self.assertFalse(empty(inv.valuation), 'investment %r has no valuation' % inv) if inv.code and inv.code != 'XX-liquidity': self.assertTrue(inv.code_type, 'investment %r has code but no code type' % inv) if inv.code_type == inv.CODE_TYPE_ISIN and inv.code and not inv.code.startswith('XX'): self.assertTrue(is_isin_valid(inv.code), 'investment %r has invalid ISIN: %r' % (inv, inv.code)) if not empty(inv.portfolio_share): self.assertTrue(0 < inv.portfolio_share <= 1, 'investment %r has invalid portfolio_share' % inv)
def check_investment(self, account, inv): self.assertTrue(inv.label, 'investment %r has no label' % inv) self.assertTrue(inv.valuation, 'investment %r has no valuation' % inv) if inv.code and inv.code != 'XX-liquidity': self.assertTrue(inv.code_type, 'investment %r has code but no code type' % inv) if inv.code_type == inv.CODE_TYPE_ISIN and inv.code and not inv.code.startswith('XX'): self.assertTrue(is_isin_valid(inv.code), 'investment %r has invalid ISIN: %r' % (inv, inv.code)) if not empty(inv.portfolio_share): self.assertTrue(0 < inv.portfolio_share <= 1, 'investment %r has invalid portfolio_share' % inv)
class item(ItemElement): klass = Investment def condition(self): # Some rows do not contain an expected item format, # There is no valuation (mnt) because some buy/sell orders are not yet finished. # We want invalid values to fail in the CleanDecimal filter so we catch only when mnt is missing return Dict('mnt', default=NotAvailable)(self) is not NotAvailable obj_label = Dict('libval') obj_code = Dict('codval') obj_code_type = Eval( lambda x: Investment.CODE_TYPE_ISIN if is_isin_valid(x) else NotAvailable, Field('code')) obj_quantity = CleanDecimal(Dict('qttit')) obj_unitvalue = CleanDecimal(Dict('crs')) obj_valuation = CleanDecimal(Dict('mnt')) obj_vdate = Env('date') def parse(self, el): symbols = { '+': 1, '-': -1, '\u0000': None, # "NULL" character } self.env['sign'] = symbols.get(Dict('signePlv')(self), None) def obj_diff(self): if Dict('plv', default=None)(self) and Env('sign')(self): return CleanDecimal(Dict('plv'), sign=lambda x: Env('sign')(self))(self) return NotAvailable def obj_unitprice(self): if Dict('pam', default=None)(self): return CleanDecimal(Dict('pam'))(self) return NotAvailable def obj_diff_percent(self): if not Env('sign')(self): return NotAvailable # obj_diff_percent key can have several names: if Dict('plvPourcentage', default=None)(self): return CleanDecimal(Dict('plvPourcentage'), sign=lambda x: Env('sign')(self))(self) elif Dict('pourcentagePlv', default=None)(self): return CleanDecimal(Dict('pourcentagePlv'), sign=lambda x: Env('sign')(self))(self) def obj_portfolio_share(self): active_percent = Dict('pourcentageActif', default=NotAvailable)(self) if empty(active_percent): return NotAvailable return Eval(lambda x: x / 100, CleanDecimal(active_percent))(self)
def obj_code(self): code = self._product()['isin'] if is_isin_valid(code): # Prefix CFD (Contrats for difference) ISIN codes with "XX-" # to avoid id_security duplicates in the database if "- CFD" in Field('label')(self): return "XX-" + code return code return NotAvailable
def obj_code(self): # We try to get the code from <a> div. If we didn't find code in url, # we try to find it in the cell text tablecell = TableCell('label', colspan=True)(self)[0] # url find try code_match = Regexp( Link(tablecell.xpath('./following-sibling::td[position()=1]/div/a')), r'sico=([A-Z0-9]*)', default=None)(self) if is_isin_valid(code_match): return code_match # cell text find try text = CleanText(tablecell.xpath('./following-sibling::td[position()=1]/div')[0])(self) for code in text.split(' '): if is_isin_valid(code): return code return NotAvailable
def obj_code(self): # We try to get the code from <a> div. If we didn't find code in url, # we try to find it in the cell text tablecell = TableCell('label', colspan=True)(self)[0] # url find try url = tablecell.xpath('./following-sibling::td[position()=1]/div/a')[0].attrib['href'] code_match = re.search(r'sico=([A-Z0-9]*)', url) if code_match: if is_isin_valid(code_match.group(1)): return code_match.group(1) # cell text find try text = CleanText(tablecell.xpath('./following-sibling::td[position()=1]/div')[0])(self) for code in text.split(' '): if is_isin_valid(code): return code return NotAvailable
def get_investments(self, account): for line in self.doc.xpath('//table[@id="tableau_support"]/tbody/tr'): cols = line.findall('td') inv = Investment() inv.id = re.search('cdReferentiel=(.*)', cols[self.COL_LABEL].find('a').attrib['href']).group(1) inv.code = re.match('^[A-Z]+[0-9]+(.*)$', inv.id).group(1) inv.label = CleanText(None).filter(cols[self.COL_LABEL]) inv.quantity = self.parse_decimal(cols[self.COL_QUANTITY]) inv.unitprice = self.parse_decimal(cols[self.COL_UNITPRICE]) inv.unitvalue = self.parse_decimal(cols[self.COL_UNITVALUE]) inv.vdate = Date(CleanText(cols[self.COL_DATE], default=NotAvailable), default=NotAvailable)(self.doc) inv.valuation = self.parse_decimal(cols[self.COL_VALUATION]) inv.diff = self.parse_decimal(cols[self.COL_PERF]) diff_percent = self.parse_decimal(cols[self.COL_PERF_PERCENT]) inv.diff_percent = diff_percent / 100 if diff_percent else NotAvailable if is_isin_valid(inv.code): inv.code_type = Investment.CODE_TYPE_ISIN yield inv
def iter_investment(self, account, invs=None): if account.id not in self.investments and invs is not None: self.investments[account.id] = [] for inv in invs: i = Investment() # If nothing is given to make the label, we use the ISIN instead # We let it crash if the ISIN is not available either. if all([inv['classification'], inv['description']]): i.label = "%s - %s" % (inv['classification'], inv['description']) else: i.label = Coalesce().filter(( inv['classification'], inv['description'], inv['isin'], )) i.code = inv['isin'] if not is_isin_valid(i.code): i.code = NotAvailable i.code_type = NotAvailable if u'Solde Espèces' in i.label: i.code = 'XX-liquidity' else: i.code_type = Investment.CODE_TYPE_ISIN i.quantity = CleanDecimal(default=NotAvailable).filter( inv['nombreParts']) i.unitprice = CleanDecimal(default=NotAvailable).filter( inv['prixMoyenAchat']) i.unitvalue = CleanDecimal(default=NotAvailable).filter( inv['valeurCotation']) i.valuation = CleanDecimal().filter(inv['montantEuro']) # For some invests the vdate returned is None # Consequently we set the default value at NotAvailable i.vdate = Date(default=NotAvailable).filter( inv['datePosition']) i.diff = CleanDecimal(default=NotAvailable).filter( inv['performanceEuro']) self.investments[account.id].append(i) return self.investments[account.id]
def obj_code(self): code = Regexp(Link('./th/a', default=''), r'isin=(\w+)|/(\w+)\.pdf', default=NotAvailable)(self) return code if is_isin_valid(code) else NotAvailable
def obj_code(self): code = CleanText(Dict('code'))(self) if is_isin_valid(code): return code return NotAvailable
def obj_code(self): code = Dict('codeISIN')(self) if is_isin_valid(code): return code return NotAvailable
def obj_code(self): for code in Field('label')(self).split(): if is_isin_valid(code): return code return NotAvailable
def obj_code_type(self): if is_isin_valid(CleanText(Dict('code'))(self)): return Investment.CODE_TYPE_ISIN return NotAvailable
def obj_code_type(self): if is_isin_valid(Field('code')(self)): return Investment.CODE_TYPE_ISIN return NotAvailable
def obj_code_type(self): code = Field('code')(self) if code and is_isin_valid(code): return Investment.CODE_TYPE_ISIN return NotAvailable
def obj_code(self): if is_isin_valid(Dict('IsinCode')(self)): return Dict('IsinCode')(self) elif "espèces" in Field('label')(self).lower(): return "XX-liquidity" return NotAvailable
def obj_code(self): code = Dict('cdsptisn')(self) if is_isin_valid(code): return code return NotAvailable
def obj_code_type(self): if is_isin_valid(self.obj_code()): return Investment.CODE_TYPE_ISIN return NotAvailable
def get_isin_code_and_type(self): code = CleanText('//td[strong[text()="ISIN"]]/following-sibling::td[1]', default=NotAvailable)(self.doc) if is_isin_valid(code): return code, Investment.CODE_TYPE_ISIN return NotAvailable, NotAvailable
def obj_code_type(self): return Investment.CODE_TYPE_ISIN if is_isin_valid( Field('code')(self)) else NotAvailable
def obj_code(self): if Field('label')(self) == "LIQUIDITES": return 'XX-liquidity' code = CleanText(TableCell('code'))(self) return code if is_isin_valid(code) else NotAvailable
def parse(self, el): # Trying to find vdate and unitvalue unitvalue, vdate = None, None for span in TableCell('label')(self)[0].xpath('.//span'): if unitvalue is None: unitvalue = Regexp(CleanText('.'), '^([\d,]+)$', default=None)(span) if vdate is None: vdate = None if any(x in CleanText('./parent::div')(span) for x in ["échéance", "Maturity"]) else \ Regexp(CleanText('.'), '^([\d\/]+)$', default=None)(span) self.env['unitvalue'] = MyDecimal().filter(unitvalue) if unitvalue else NotAvailable self.env['vdate'] = Date(dayfirst=True).filter(vdate) if vdate else NotAvailable self.env['_link'] = None self.env['asset_category'] = NotAvailable page = None link_id = Attr(u'.//a[contains(@title, "détail du fonds")]', 'id', default=None)(self) inv_id = Attr('.//a[contains(@id, "linkpdf")]', 'id', default=None)(self) if link_id and inv_id: form = self.page.get_form('//div[@id="operation"]//form') form['idFonds'] = inv_id.split('-', 1)[-1] form['org.richfaces.ajax.component'] = form[link_id] = link_id page = self.page.browser.open(form['javax.faces.encodedURL'], data=dict(form)).page if 'hsbc.fr' in self.page.browser.BASEURL: # Special space for HSBC, does not contain any information related to performances. m = re.search(r'fundid=(\w+).+SH=(\w+)', CleanText('//complete', default='')(page.doc)) if m: # had to put full url to skip redirections. page = page.browser.open('https://www.assetmanagement.hsbc.com/feedRequest?feed_data=gfcFundData&cod=FR&client=FCPE&fId=%s&SH=%s&lId=fr' % m.groups()).page elif not self.page.browser.history.is_here(): url = page.get_invest_url() if empty(url): self.env['code'] = NotAvailable self.env['code_type'] = NotAvailable return # URLs used in browser.py to access investments performance history: if url.startswith('https://optimisermon.epargne-retraite-entreprises'): # This URL can be used to access the BNP Wealth API to fetch investment performance and ISIN code self.env['_link'] = url self.env['code'] = NotAvailable self.env['code_type'] = NotAvailable return elif (url.startswith('http://sggestion-ede.com/product') or url.startswith('https://www.lyxorfunds.com/part') or url.startswith('https://www.societegeneralegestion.fr') or url.startswith('http://www.etoile-gestion.com/productsheet')): self.env['_link'] = url # Try to fetch ISIN code from URL with re.match match = re.match(r'http://www.cpr-am.fr/fr/fonds_detail.php\?isin=([A-Z0-9]+)', url) match = match or re.match(r'http://www.cpr-am.fr/particuliers/product/view/([A-Z0-9]+)', url) if match: self.env['code'] = match.group(1) if is_isin_valid(match.group(1)): self.env['code_type'] = Investment.CODE_TYPE_ISIN else: self.env['code_type'] = Investment.CODE_TYPE_AMF return # Try to fetch ISIN code from URL with re.search m = re.search(r'&ISIN=([^&]+)', url) m = m or re.search(r'&isin=([^&]+)', url) m = m or re.search(r'&codeIsin=([^&]+)', url) m = m or re.search(r'lyxorfunds\.com/part/([^/]+)', url) if m: self.env['code'] = m.group(1) if is_isin_valid(m.group(1)): self.env['code_type'] = Investment.CODE_TYPE_ISIN else: self.env['code_type'] = Investment.CODE_TYPE_AMF return useless_urls = ( # pdf... http://docfinder.is.bnpparibas-ip.com/api/files/040d05b3-1776-4991-aa49-f0cd8717dab8/1536 'http://docfinder.is.bnpparibas-ip.com/', # The AXA website displays performance graphs but everything is calculated using JS scripts. # There is an API but it only contains risk data and performances per year, not 1-3-5 years. 'https://epargne-salariale.axa-im.fr/fr/', # Redirection to the Rothschild Gestion website, which doesn't exist anymore... 'https://www.rothschildgestion.com', # URL to the Morningstar website does not contain any useful information 'http://doc.morningstar.com', ) for useless_url in useless_urls: if url.startswith(useless_url): self.env['code'] = NotAvailable self.env['code_type'] = NotAvailable return if url.startswith('http://fr.swisslife-am.com/fr/'): self.page.browser.session.cookies.set('location', 'fr') self.page.browser.session.cookies.set('prof', 'undefined') try: page = self.page.browser.open(url).page except HTTPNotFound: # Some pages lead to a 404 so we must avoid unnecessary crash self.logger.warning('URL %s was not found, investment details will be skipped.', url) if isinstance(page, CodePage): self.env['code'] = page.get_code() self.env['code_type'] = page.CODE_TYPE self.env['asset_category'] = page.get_asset_category() else: # The page is not handled and does not have a get_code method. self.env['code'] = NotAvailable self.env['code_type'] = NotAvailable self.env['asset_category'] = NotAvailable
def obj_code(self): code = CleanText('.//span[@class="cl-secondary"]')(self) if is_isin_valid(code): return code return NotAvailable