class item(ItemElement):
klass = Transaction
obj_id = None # will be overwrited by the browser
# we use lower for compatibility with the old website
obj_raw = Transaction.Raw(Lower('.//td[@class="lbl"]'))
obj_amount = CleanDecimal('.//td[starts-with(@class, "amount")]',
replace_dots=True)
obj_date = INGDate(CleanText('.//td[@class="date"]'),
dayfirst=True)
obj_rdate = Field('date')
obj__hash = PreHashmd5(Field('date'), Field('raw'),
Field('amount'))
obj_category = INGCategory(
Attr('.//td[@class="picto"]/span', 'class'))
def condition(self):
if self.el.find('.//td[@class="date"]') is None:
return False
if 'index' in self.env and self.env[
'index'] > 0 and self.page.i < self.env['index']:
self.page.i += 1
return False
return True
class item(ItemElement):
klass = Bill
obj__ref = CleanText('//input[@id="noref"]/@value')
obj_id = Format('%s_%s', Env('subid'), CleanText('./@facture-id'))
obj_url = Format(
'http://www.bouyguestelecom.fr/parcours/facture/download/index?id=%s&no_reference=%s',
CleanText('./@facture-id'), CleanText('./@facture-ligne'))
obj_date = Env('date')
obj_format = u"pdf"
obj_label = CleanText('./text()')
obj_type = u"bill"
obj_price = CleanDecimal(
CleanText('./span', replace=[(u' € ', '.')]))
obj_currency = u"€"
def parse(self, el):
self.env['date'] = parse_french_date(
'01 %s' % CleanText('./text()')(self)).date()
def condition(self):
# XXX ugly fix to avoid duplicate bills
return CleanText('./@facture-id')(self.el) != CleanText(
'./following-sibling::div[1]/@facture-id')(self.el)
def handle_response(self, account, recipient, amount, reason):
account_txt = CleanText(
'//form//dl/dt[span[contains(text(), "biter")]]/following::dd[1]',
replace=[(' ', '')])(self.doc)
recipient_txt = CleanText(
'//form//dl/dt[span[contains(text(), "diter")]]/following::dd[1]',
replace=[(' ', '')])(self.doc)
assert account.id in account_txt or ''.join(
account.label.split()) == account_txt, 'Something went wrong'
assert recipient.id in recipient_txt or ''.join(
recipient.label.split()) == recipient_txt, 'Something went wrong'
r_amount = CleanDecimal(
'//form//dl/dt[span[contains(text(), "Montant")]]/following::dd[1]',
replace_dots=True)(self.doc)
exec_date = Date(CleanText(
'//form//dl/dt[span[contains(text(), "Date")]]/following::dd[1]'),
dayfirst=True)(self.doc)
currency = FrenchTransaction.Currency(
'//form//dl/dt[span[contains(text(), "Montant")]]/following::dd[1]'
)(self.doc)
transfer = Transfer()
transfer.currency = currency
transfer.amount = r_amount
transfer.account_iban = account.iban
transfer.recipient_iban = recipient.iban
transfer.account_id = account.id
transfer.recipient_id = recipient.id
transfer.exec_date = exec_date
transfer.label = reason
transfer.account_label = account.label
transfer.recipient_label = recipient.label
transfer.account_balance = account.balance
return transfer
class item(ItemElement):
klass = Account
TYPE = {
'Livret': Account.TYPE_SAVINGS,
'Compte': Account.TYPE_CHECKING,
'PEA': Account.TYPE_PEA,
'PEA-PME': Account.TYPE_PEA,
'Compte-titres': Account.TYPE_MARKET,
'Assurance-vie': Account.TYPE_LIFE_INSURANCE,
'Crédit': Account.TYPE_LOAN,
}
obj_id = CleanText(
'./td//div[contains(@class, "-synthese-title") or contains(@class, "-synthese-text")]'
) & Regexp(pattern=r'(\d+)')
obj_label = CleanText(
'./td//div[contains(@class, "-synthese-title")]')
obj_balance = MyDecimal(
'./td//div[contains(@class, "-synthese-num")]',
replace_dots=True)
obj_currency = FrenchTransaction.Currency(
'./td//div[contains(@class, "-synthese-num")]')
obj_type = Map(Regexp(Field('label'), r'^([^ ]*)'),
TYPE,
default=Account.TYPE_UNKNOWN)
def obj_url(self):
return urljoin(self.page.url, CleanText('./@data-href')(self))
obj__card_balance = CleanDecimal(
'./td//div[@class="synthese-encours"][last()]/div[2]',
default=None)
def condition(self):
return not len(self.el.xpath('./td[@class="chart"]'))
def obj_performance_history(self):
perfs = {}
if CleanDecimal.French('(//tr[th[text()="1 an"]]/td[1])[1]',
default=None)(self):
perfs[1] = Eval(
lambda x: x / 100,
CleanDecimal.French('(//tr[th[text()="1 an"]]/td[1])[1]'))(
self)
if CleanDecimal.French('(//tr[th[text()="3 ans"]]/td[1])[1]',
default=None)(self):
perfs[3] = Eval(
lambda x: x / 100,
CleanDecimal.French(
'(//tr[th[text()="3 ans"]]/td[1])[1]'))(self)
if CleanDecimal.French('(//tr[th[text()="5 ans"]]/td[1])[1]',
default=None)(self):
perfs[5] = Eval(
lambda x: x / 100,
CleanDecimal.French(
'(//tr[th[text()="5 ans"]]/td[1])[1]'))(self)
return perfs
def get_next_payment_amount(self):
return CleanDecimal(Regexp(
CleanText(u'//p[@id="c_prochaineEcheance"]//strong'), u'(.*) le'),
replace_dots=True)(self.doc)
def get_list(self):
accounts = []
for cpt in self.doc.xpath(
'//div[contains(@class, " compte") and not(contains(@class, "compte_selected"))]'
):
# ignore auto assurance accounts
if 'aut' in cpt.get('class'):
continue
account = Account()
account._history_link = Link(
'./ul/li/a[contains(@id, "consulter_solde") '
'or contains(@id, "historique") '
'or contains(@id, "contrat") '
'or contains(@id, "assurance_vie_operations")]')(cpt)
# this is to test if access to the accounts info is blocked for different reasons
page = self.browser.open(account._history_link).page
if isinstance(page, LoanPage):
account = Loan()
account._history_link = Link(
'./ul/li/a[contains(@id, "consulter_solde") '
'or contains(@id, "historique") '
'or contains(@id, "contrat") '
'or contains(@id, "assurance_vie_operations")]')(cpt)
if isinstance(page, LoanPage):
account.id = CleanText(
'(//p[@id="c_montantEmprunte"]//span[@class="valStatic"]//strong)[1]'
)(cpt)
account.label = CleanText(
'(//p[@id="c_montantEmprunte"]//span[@class="valStatic"]//strong)[1]'
)(cpt)
account.type = Account.TYPE_LOAN
account_history_page = self.browser.open(
account._history_link).page
account.total_amount = account_history_page.get_total_amount()
account.next_payment_amount = account_history_page.get_next_payment_amount(
)
account.next_payment_date = account_history_page.get_next_payment_date(
)
account.account_label = account_history_page.get_account_label(
)
account.subscription_date = account_history_page.get_subscription_date(
)
account.maturity_date = account_history_page.get_maturity_date(
)
if len(accounts) == 0:
global_error_message = page.doc.xpath(
'//div[@id="as_renouvellementMIFID.do_"]/div[contains(text(), "Bonjour")] '
'| //div[@id="as_afficherMessageBloquantMigration.do_"]//div[@class="content_message"] '
'| //p[contains(text(), "Et si vous faisiez de Fortuneo votre banque principale")] '
'| //div[@id="as_renouvellementMotDePasse.do_"]//p[contains(text(), "votre mot de passe")]'
'| //div[@id="as_afficherSecuriteForteOTPIdentification.do_"]//span[contains(text(), "Pour valider ")]'
)
if global_error_message:
raise ActionNeeded(CleanText('.')(global_error_message[0]))
local_error_message = page.doc.xpath(
'//div[@id="error"]/p[@class="erreur_texte1"]')
if local_error_message:
raise BrowserUnavailable(
CleanText('.')(local_error_message[0]))
number = RawText('./a[contains(@class, "numero_compte")]')(
cpt).replace(u'N° ', '')
account.id = CleanText(None).filter(number).replace(u'N°', '')
account._card_links = []
card_link = Link('./ul/li/a[contains(text(), "Carte bancaire")]',
default='')(cpt)
if len(card_link) > 0:
account._card_links.append(card_link)
account.label = CleanText(
'./a[contains(@class, "numero_compte")]/@title')(cpt)
for pattern, type in self.ACCOUNT_TYPES.items():
if pattern in account._history_link:
account.type = type
break
if account.type in {
Account.TYPE_PEA, Account.TYPE_MARKET,
Account.TYPE_LIFE_INSURANCE
}:
account._investment_link = Link(
'./ul/li/a[contains(@id, "portefeuille")]')(cpt)
balance = self.browser.open(
account._investment_link).page.get_balance(account.type)
if account.type in {Account.TYPE_PEA, Account.TYPE_MARKET}:
self.browser.investments[account.id] = list(
self.browser.open(
account._investment_link).page.get_investments(
account))
else:
balance = self.browser.open(
account._history_link).page.get_balance()
if account.type is not Account.TYPE_LOAN:
account.coming = self.browser.open(
account._history_link).page.get_coming()
if account.type in {Account.TYPE_PEA, Account.TYPE_MARKET}:
account.currency = self.browser.open(
account._investment_link).page.get_currency()
else:
account.currency = account.get_currency(balance)
account.balance = CleanDecimal(None,
replace_dots=True).filter(balance)
if account.type in (Account.TYPE_CHECKING, Account.TYPE_SAVINGS):
# Need a token sent by SMS to customers
account.iban = NotAvailable
if (account.label, account.id, account.balance) not in [
(a.label, a.id, a.balance) for a in accounts
]:
accounts.append(account)
return accounts
def MyDecimal(*args, **kwargs):
kwargs.update(replace_dots=True, default=NotAvailable)
return CleanDecimal(*args, **kwargs)
def MyDecimal(*args, **kwargs):
kwargs.update(replace_dots=True, default=Decimal(0))
return CleanDecimal(*args, **kwargs)
class get_housing(ItemElement):
klass = Housing
obj_id = Env('_id')
obj_title = CleanText(CleanHTML('//meta[@itemprop="name"]/@content'))
obj_area = CleanDecimal(Regexp(CleanText(CleanHTML('//meta[@itemprop="name"]/@content')),
'(.*?)(\d*) m\xb2(.*?)', '\\2', default=NotAvailable),
default=NotAvailable)
obj_rooms = CleanDecimal('//div[has-class("offer-info")]//span[has-class("offer-rooms-number")]',
default=NotAvailable)
obj_cost = CleanDecimal('//*[@itemprop="price"]', default=0)
obj_currency = Regexp(
CleanText('//*[@itemprop="price"]'),
'.*([%s%s%s])' % (u'€', u'$', u'£'),
default=u'€'
)
def obj_utilities(self):
notes = CleanText('//p[@class="offer-description-notes"]')(self)
if "Loyer mensuel charges comprises" in notes:
return UTILITIES.INCLUDED
else:
return UTILITIES.UNKNOWN
obj_price_per_meter = PricePerMeterFilter()
obj_date = Date(Regexp(CleanText('//p[@class="offer-description-notes"]|//p[has-class("darkergrey")]'),
u'.* Mis à jour : (\d{2}/\d{2}/\d{4}).*'),
dayfirst=True)
obj_text = CleanHTML('//div[has-class("offer-description-text")]/meta[@itemprop="description"]/@content')
obj_location = CleanText('//*[@itemprop="address"]')
obj_station = CleanText(
'//div[has-class("offer-description-metro")]',
default=NotAvailable
)
obj_url = BrowserURL('housing', _id=Env('_id'))
def obj_photos(self):
photos = []
for img in XPath('//div[@class="carousel-content"]/ul/li/a/img/@src|//div[@class="carousel"]/ul/li/a/img/@src')(self):
photos.append(HousingPhoto(u'%s' % img.replace('75x75', '800x600')))
return photos
def obj_details(self):
details = {}
energy_value = CleanText(
'//div[has-class("offer-energy-greenhouseeffect-summary")]//div[has-class("energy-summary")]',
default=None
)(self)
if energy_value and len(energy_value) > 1:
energy_value = energy_value.replace("DPE", "").strip()[0]
if energy_value not in ["A", "B", "C", "D", "E", "F", "G"]:
energy_value = None
if energy_value is None:
energy_value = NotAvailable
details["DPE"] = energy_value
greenhouse_value = CleanText(
'//div[has-class("offer-energy-greenhouseeffect-summary")]//div[has-class("greenhouse-summary")]',
default=None
)(self)
if greenhouse_value and len(greenhouse_value) > 1:
greenhouse_value = greenhouse_value.replace("GES", "").strip()[0]
if greenhouse_value not in ["A", "B", "C", "D", "E", "F", "G"]:
greenhouse_value = None
if greenhouse_value is None:
greenhouse_value = NotAvailable
details["GES"] = greenhouse_value
details["creationDate"] = Date(
Regexp(
CleanText(
'//p[@class="offer-description-notes"]|//p[has-class("darkergrey")]'
),
u'.*Mis en ligne : (\d{2}/\d{2}/\d{4}).*'
),
dayfirst=True
)(self)
honoraires = CleanText(
(
'//div[has-class("offer-price")]/span[has-class("lbl-agencyfees")]'
),
default=None
)(self)
if honoraires:
details["Honoraires"] = (
"{} (TTC, en sus)".format(
honoraires.split(":")[1].strip()
)
)
for li in XPath('//ul[@itemprop="description"]/li')(self):
label = CleanText('./div[has-class("criteria-label")]')(li)
value = CleanText('./div[has-class("criteria-value")]')(li)
details[label] = value
return details
class get_housing(ItemElement):
klass = Housing
def parse(self, el):
details = dict()
self.env['area'] = NotAvailable
self.env['GES'] = NotAvailable
self.env['DPE'] = NotAvailable
self.env['typeBien'] = NotAvailable
for item in el.xpath('//div[@class="line"]/h2'):
property = CleanText('./span[@class="property"]')(item)
if 'Surface' in property:
self.env['area'] = CleanDecimal(
Regexp(CleanText('./span[@class="value"]'), '(.*)m.*'),
replace_dots=(',', '.'))(item)
elif 'Type de bien' in property:
value = CleanText('./span[@class="value"]')(item).lower()
if value == 'parking':
self.env['typeBien'] = HOUSE_TYPES.PARKING
elif value == 'appartement':
self.env['typeBien'] = HOUSE_TYPES.APART
elif value == 'maison':
self.env['typeBien'] = HOUSE_TYPES.HOUSE
elif value == 'terrain':
self.env['typeBien'] = HOUSE_TYPES.LAND
else:
self.env['typeBien'] = HOUSE_TYPES.OTHER
elif 'Meublé' in property:
value = CleanText('./span[@class="value"]')(item).lower()
self.env['isFurnished'] = (value == 'meublé')
else:
key = u'%s' % CleanText('./span[@class="property"]')(item)
if 'GES' in key or 'Classe' in key:
if 'Classe' in key:
key = 'DPE'
value = (
CleanText('./span[@class="value"]')(item).strip())
if len(value):
self.env[key] = getattr(ENERGY_CLASS, value[0],
NotAvailable)
else:
details[key] = CleanText('./span[@class="value"]')(
item)
self.env['details'] = details
obj_id = Env('_id')
def obj_type(self):
breadcrumb = Link(
'(//nav[has-class("breadcrumbsNav")]//a)[last()]')(self)
if 'colocations' in breadcrumb:
return POSTS_TYPES.SHARING
elif 'locations' in breadcrumb:
if self.env['isFurnished']:
return POSTS_TYPES.FURNISHED_RENT
else:
return POSTS_TYPES.RENT
else:
return POSTS_TYPES.SALE
def obj_advert_type(self):
line_pro = XPath('.//span[has-class("ispro")]', default=None)(self)
if line_pro:
return ADVERT_TYPES.PROFESSIONAL
else:
return ADVERT_TYPES.PERSONAL
obj_house_type = Env('typeBien')
obj_title = CleanText('//h1[@itemprop="name"]')
obj_cost = CleanDecimal('//h2[@itemprop="price"]/@content',
default=Decimal(0))
obj_currency = Currency('//h2[@itemprop="price"]/span[@class="value"]')
def obj_utilities(self):
utilities = Regexp(
CleanText('//h2[@itemprop="price"]/span[@class="value"]'),
'.*[%s%s%s](.*)' % (u'€', u'$', u'£'),
default=u'')(self)
if "C.C." in utilities:
return UTILITIES.INCLUDED
elif "H.C." in utilities:
return UTILITIES.EXCLUDED
else:
return UTILITIES.UNKNOWN
obj_DPE = Env('DPE')
obj_GES = Env('GES')
obj_text = CleanText('//p[@itemprop="description"]')
obj_location = CleanText('//span[@itemprop="address"]')
obj_details = Env('details')
def obj_rooms(self):
rooms = self.env["details"].get(u"Pièces", None)
return Decimal(rooms) if rooms else NotAvailable
obj_area = Env('area')
obj_price_per_meter = PricePerMeterFilter()
obj_url = BrowserURL('housing', _id=Env('_id'))
def obj_date(self):
_date = Regexp(
CleanText('//p[has-class("line")]', replace=[(u'à', '')]),
'.*Mise en ligne le (.*)')(self)
for fr, en in DATE_TRANSLATE_FR:
_date = fr.sub(en, _date)
self.env['tmp'] = _date
return DateTime(Env('tmp'), LinearDateGuesser())(self)
def obj_photos(self):
items = re.findall(r'images\[\d\]\s*=\s*"([\w:\/\.-]*\.jpg)";',
CleanText('//script')(self))
photos = [HousingPhoto(unicode(item)) for item in items]
if not photos:
img = CleanText('//meta[@itemprop="image"]/@content',
default=None)(self)
if img:
photos.append(HousingPhoto(img))
return photos
def obj_balance(self):
balance = CleanDecimal(Dict('soldeEuro', default="0"))(self)
return -abs(balance) if Field('type')(
self) == Account.TYPE_LOAN else balance
def get_list(self):
accounts = []
noaccounts = self.get_from_js('_js_noMvts =', ';')
if noaccounts is not None:
assert 'avez aucun compte' in noaccounts
return []
txt = self.get_from_js('_data = new Array(', ');', is_list=True)
if txt is None:
raise BrowserUnavailable('Unable to find accounts list in scripts')
data = json.loads('[%s]' % txt.replace("'", '"'))
for line in data:
a = Account()
a.id = line[self.COL_ID].replace(' ', '')
if re.match(
r'Classement=(.*?):::Banque=(.*?):::Agence=(.*?):::SScompte=(.*?):::Serie=(.*)',
a.id):
a.id = str(CleanDecimal().filter(a.id))
a._acc_nb = a.id.split('_')[0] if len(
a.id.split('_')) > 1 else None
a.label = MyStrip(line[self.COL_LABEL],
xpath='.//div[@class="libelleCompteTDB"]')
# This account can be multiple life insurance accounts
if a.label == 'ASSURANCE VIE-BON CAPI-SCPI-DIVERS *':
continue
a.balance = Decimal(
FrenchTransaction.clean_amount(line[self.COL_BALANCE]))
a.currency = a.get_currency(line[self.COL_BALANCE])
a.type = self.get_account_type(a.label)
if line[self.COL_HISTORY] == 'true':
a._inv = False
a._link = self.get_history_link()
a._args = {
'_eventId': 'clicDetailCompte',
'_ipc_eventValue': '',
'_ipc_fireEvent': '',
'deviseAffichee': 'DEVISE',
'execution': self.get_execution(),
'idCompteClique': line[self.COL_ID],
}
else:
a._inv = True
a._args = {
'_ipc_eventValue': line[self.COL_ID],
'_ipc_fireEvent': line[self.COL_FIRE_EVENT],
}
a._link = self.doc.xpath(
'//form[@name="changePageForm"]')[0].attrib['action']
if a.type is Account.TYPE_CARD:
a.coming = a.balance
a.balance = Decimal('0.0')
accounts.append(a)
return accounts
def obj_price(self):
if CleanText(TableCell('amount')(self))(self):
return CleanDecimal(Regexp(CleanText(TableCell('amount')), '.*?([\d,]+).*', default=NotAvailable), replace_dots=True, default=NotAvailable)(self)
else:
return Field('_ht')(self)
def obj_amount(self):
amount = CleanDecimal(Dict('Montant/Valeur'))(self)
return -amount if Dict('Montant/CodeSens')(
self) == "D" else amount
def parse(self, el):
link = el.xpath('./td[1]/a')[0].get('href', '')
if 'POR_SyntheseLst' in link:
raise SkipItem()
url = urlparse(link)
p = parse_qs(url.query)
if 'rib' not in p and 'webid' not in p:
raise SkipItem()
for td in el.xpath('./td[2] | ./td[3]'):
try:
balance = CleanDecimal('.', replace_dots=True)(td)
except InvalidOperation:
continue
else:
break
else:
raise ParseError('Unable to find balance for account %s' %
CleanText('./td[1]/a')(el))
self.env['_is_webid'] = False
if self.page.browser.is_new_website:
id = CleanText(
'./td[1]/a/node()[contains(@class, "doux")]',
replace=[(' ', '')])(el)
else:
if 'rib' in p:
id = p['rib'][0]
else:
id = p['webid'][0]
self.env['_is_webid'] = True
page = self.page.browser.open(link).page
# Handle cards
if id in self.parent.objects:
if page.is_fleet() or id in self.page.browser.fleet_pages:
if not id in self.page.browser.fleet_pages:
self.page.browser.fleet_pages[id] = []
self.page.browser.fleet_pages[id].append(page)
else:
account = self.parent.objects[id]
if not account.coming:
account.coming = Decimal('0.0')
account.coming += balance
account._card_links.append(link)
raise SkipItem()
self.env['id'] = id
# Handle real balances
coming = page.find_amount(
u"Opérations à venir") if page else None
accounting = page.find_amount(
u"Solde comptable") if page else None
if accounting is not None and accounting + (
coming or Decimal('0')) != balance:
self.page.logger.warning('%s + %s != %s' %
(accounting, coming, balance))
if accounting is not None:
balance = accounting
self.env['balance'] = balance
self.env['coming'] = coming or NotAvailable
class item(ItemElement):
klass = Housing
def validate(self, obj):
return obj.id is not None
obj_url = Format(u'http:%s', Link('.'))
obj_id = Regexp(
Link('.'),
'//www.leboncoin.fr/(ventes_immobilieres|locations|colocations)/(.*).htm.*',
'\\2',
default=None)
obj_type = Env('query_type')
def obj_advert_type(self):
ispro = XPath('.//span[has-class("ispro")]',
default=None)(self)
if ispro:
return ADVERT_TYPES.PROFESSIONAL
else:
return ADVERT_TYPES.PERSONAL
obj_house_type = NotAvailable
obj_title = CleanText('./@title|./section/p[@class="item_title"]')
obj_cost = CleanDecimal(
'./section[@class="item_infos"]/*[@class="item_price"]/text()',
replace_dots=(',', '.'),
default=Decimal(0))
obj_location = CleanText(
'./section[@class="item_infos"]/*[@itemtype="http://schema.org/Place"]/text()'
)
obj_currency = Currency(
'./section[@class="item_infos"]/*[@class="item_price"]')
def obj_utilities(self):
utilities = Regexp(CleanText(
'./section[@class="item_infos"]/*[@class="item_price"]'),
'\d+ [%s%s%s](.*)' % (u'€', u'$', u'£'),
default=u'')(self)
if "C.C." in utilities:
return UTILITIES.INCLUDED
elif "H.C." in utilities:
return UTILITIES.EXCLUDED
else:
return UTILITIES.UNKNOWN
obj_text = Join(' - ', './/p[@class="item_supp"]')
def obj_date(self):
_date = CleanText(
'./section[@class="item_infos"]/aside/p[@class="item_supp"]/text()',
replace=[('Aujourd\'hui', str(date.today())),
('Hier', str(
(date.today() - timedelta(1))))])(self)
if not _date:
return NotAvailable
for fr, en in DATE_TRANSLATE_FR:
_date = fr.sub(en, _date)
self.env['tmp'] = _date
return DateTime(Env('tmp'), LinearDateGuesser())(self)
def obj_photos(self):
photos = []
url = Attr(
'./div[@class="item_image"]/span/span[@class="lazyload"]',
'data-imgsrc',
default=None)(self)
if url:
photos.append(
HousingPhoto(url.replace("ad-thumb", "ad-image")))
return photos
def obj__total_amount(self):
return CleanDecimal(Dict('grantedAmount', default=None),
default=NotAvailable)(self)
class item(ItemElement):
offer_details_wrapper = (
'./div/div/div[has-class("offer-details-wrapper")]'
)
klass = Housing
obj_id = Format(
'%s-%s',
Regexp(Env('type'), '(.*)-.*'),
CleanText('./@id', replace=[('header-offer-', '')])
)
obj_title = Attr(
offer_details_wrapper + '/div/div/p[@class="offer-type"]/a',
'title'
)
obj_url = Format(
"http://www.logic-immo.com/%s.htm",
CleanText(
'./@id',
replace=[('header-offer-', 'detail-location-')]
)
)
obj_area = CleanDecimal(
(
offer_details_wrapper +
'/div/div/div[has-class("offer-details-second")]' +
'/div/h3[has-class("offer-attributes")]/span' +
'/span[has-class("offer-area-number")]'
),
default=NotAvailable
)
obj_rooms = CleanDecimal(
(
offer_details_wrapper +
'/div/div/div[has-class("offer-details-second")]' +
'/div/h3[has-class("offer-attributes")]' +
'/span[has-class("offer-rooms")]' +
'/span[has-class("offer-rooms-number")]'
),
default=NotAvailable
)
obj_price_per_meter = PricePerMeterFilter()
obj_cost = CleanDecimal(
Regexp(
CleanText(
(
offer_details_wrapper +
'/div/div/p[@class="offer-price"]/span'
),
default=NotAvailable
),
'(.*) [%s%s%s]' % (u'€', u'$', u'£'),
default=NotAvailable
),
default=NotAvailable
)
obj_currency = Regexp(
CleanText(
offer_details_wrapper + '/div/div/p[has-class("offer-price")]/span',
default=NotAvailable
),
'.*([%s%s%s])' % (u'€', u'$', u'£'), default=u'€'
)
obj_utilities = UTILITIES.UNKNOWN
obj_date = Date(
Regexp(
CleanText(
'./div/div/div[has-class("offer-picture-more")]/div/p[has-class("offer-update")]'
),
".*(\d{2}/\d{2}/\d{4}).*")
)
obj_text = CleanText(
offer_details_wrapper + '/div/div/div/p[has-class("offer-description")]/span'
)
obj_location = CleanText(
offer_details_wrapper +
'//div[has-class("offer-places-block")]'
)
def obj_photos(self):
photos = []
url = Attr(
'./div/div/div/div[has-class("picture-wrapper")]/div/img',
'src'
)(self)
if url:
photos.append(HousingPhoto(url))
return photos
def obj_details(self):
details = {}
honoraires = CleanText(
(
self.offer_details_wrapper +
'/div/div/p[@class="offer-agency-fees"]'
),
default=None
)(self)
if honoraires:
details["Honoraires"] = (
"{} (TTC, en sus)".format(
honoraires.split(":")[1].strip()
)
)
return details
class item(ItemElement):
klass = Account
obj_label = Upper(Dict('libelleContrat'))
obj_balance = CleanDecimal(Dict('solde', default="0"))
obj_currency = 'EUR'
obj_coming = CleanDecimal(Dict('AVenir', default=None),
default=NotAvailable)
obj__index = Dict('index')
obj__owner = Dict('nomTitulaire')
def obj_id(self):
type = Field('type')(self)
if type == Account.TYPE_LIFE_INSURANCE:
number = self.get_lifenumber()
if number:
return number
elif type in (Account.TYPE_PEA, Account.TYPE_MARKET):
number = self.get_market_number()
if number:
return number
try:
return Env('numbers')(self)[Dict('index')(self)]
except KeyError:
# index often changes, so we can't use it... and have to do something ugly
return Slugify(
Format('%s-%s', Dict('libelleContrat'),
Dict('nomTitulaire')))(self)
def obj_type(self):
for key in self.page.TYPES:
if key in Env('type_label')(self).lower():
return self.page.TYPES[key]
return Account.TYPE_UNKNOWN
def obj_ownership(self):
if Dict('nomCotitulaire', default=None)(self):
return AccountOwnership.CO_OWNER
owner = Dict('nomTitulaire', default=None)(self)
if owner and all(n in owner.upper()
for n in self.env['name'].split()):
return AccountOwnership.OWNER
return AccountOwnership.ATTORNEY
def get_market_number(self):
label = Field('label')(self)
page = self.page.browser._go_market_history()
return page.get_account_id(label, Field('_owner')(self))
def get_lifenumber(self):
index = Dict('index')(self)
data = json.loads(
self.page.browser.redirect_insurance.open(
accid=index).text)
if not data:
raise SkipItem('account seems unavailable')
url = data['url']
page = self.page.browser.open(url).page
return page.get_account_id()
def get_history(self, currency):
self.MONTHS = self.FR_MONTHS if currency == 'EUR' else self.US_MONTHS
#checking if the card is still valid
if self.document.xpath('//div[@id="errorbox"]'):
return
#adding a time delta because amex have hard time to put the date in a good interval
beginning_date = self.get_beginning_debit_date() - datetime.timedelta(
days=300)
end_date = self.get_end_debit_date()
guesser = ChaoticDateGuesser(beginning_date, end_date)
for tr in reversed(
self.document.xpath(
'//div[@id="txnsSection"]//tr[@class="tableStandardText"]')
):
cols = tr.findall('td')
t = Transaction()
day, month = self.parser.tocleanstring(cols[self.COL_DATE]).split(
' ', 1)
day = int(day)
month = self.MONTHS.index(month.rstrip('.')) + 1
date = guesser.guess_date(day, month)
rdate = None
try:
detail = self.parser.select(cols[self.COL_TEXT],
'div.hiddenROC', 1)
except BrokenPageError:
pass
else:
m = re.search(r' (\d{2} \D{3,4})', (' '.join(
[txt.strip() for txt in detail.itertext()])).strip())
if m:
rday, rmonth = m.group(1).split(' ')
rday = int(rday)
rmonth = self.MONTHS.index(rmonth.rstrip('.')) + 1
rdate = guesser.guess_date(rday, rmonth)
detail.drop_tree()
raw = (' '.join([
txt.strip() for txt in cols[self.COL_TEXT].itertext()
])).strip()
credit = self.parser.tocleanstring(cols[self.COL_CREDIT])
debit = self.parser.tocleanstring(cols[self.COL_DEBIT])
t.date = date
t.rdate = rdate or date
t.raw = re.sub(r'[ ]+', ' ', raw)
t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip()
t.amount = CleanDecimal(replace_dots=currency == 'EUR').filter(
credit or debit) * (1 if credit else -1)
if t.amount > 0:
t.type = t.TYPE_ORDER
else:
t.type = t.TYPE_CARD
yield t
def obj_total_amount(self):
# Json key change depending on loan type, consumer credit or revolving credit
return CleanDecimal(
Dict('montantEmprunte', default=None)(self)
or Dict('montantUtilise'))(self)
def get_loan_balance(self):
return CleanDecimal.US('//*[@id="table-detail"]/tbody/tr/td[@class="capital"]', default=NotAvailable)(self.doc)
def obj_balance(self):
return -abs(CleanDecimal().filter(
self.el.get('montantRestant',
self.el.get('montantUtilise'))))
def parse_decimal(self, string):
string = CleanText(None).filter(string)
if string == '-' or string == '*':
return NotAvailable
return CleanDecimal(None, replace_dots=True).filter(string)
def obj_balance(self):
if Field('type')(self) == Account.TYPE_LOAN:
return -abs(CleanDecimal('.//span[@class="number"]', replace_dots=True)(self))
return CleanDecimal('.//span[@class="number"]', replace_dots=True, default=NotAvailable)(self)
def get_total_amount(self):
return CleanDecimal(u'(//p[@id="c_montantEmprunte"]//strong)[2]',
replace_dots=True)(self.doc)
def obj_balance(self):
balance = CleanDecimal('.//div[contains(@class, "right_col")]//h2[1]',
replace_dots=True)(self)
return (-balance
if Field('type')(self) in (Account.TYPE_LOAN, ) else balance)
def obj_quantity(self):
# default for euro funds
return CleanDecimal(TableCell('quantity'), default=CleanDecimal(TableCell('support_value'))(self))(self)
def obj_balance(self):
available = CleanDecimal(
'.//p[contains(., "encours depuis le")]//preceding-sibling::h2',
default=None,
replace_dots=True)(self)
return NotAvailable if not available else -available