def handle_response(self, transfer): self.check_errors() transfer_data = self.doc['data']['enregistrementVirement'] transfer.id = transfer_data['reference'] transfer.exec_date = parse_french_date(self.doc['data']['enregistrementVirement']['dateExecution']).date() # Timestamp at which the bank registered the transfer register_date = re.sub(' 24:', ' 00:', self.doc['data']['enregistrementVirement']['dateEnregistrement']) transfer._register_date = parse_french_date(register_date) return transfer
def get_history(self, date_guesser): seen = set() lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr') for line in lines[1:]: # first line is balance is_balance = line.xpath('./td/@class="cel-texte cel-neg"') [date, label, _, amount] = [self.parser.tocleanstring(td) for td in line.xpath('./td')] t = Transaction(0) t.set_amount(amount) t.label = t.raw = label if is_balance: m = re.search('(\d+ [^ ]+ \d+)', label) if not m: raise BrokenPageError('Unable to read card balance in history: %r' % label) t.date = parse_french_date(m.group(1)) t.amount = -t.amount else: day, month = map(int, date.split('/', 1)) t.date = date_guesser.guess_date(day, month) t.type = t.TYPE_CARD t.rdate = t.date try: t.id = t.unique_id(seen) except UnicodeEncodeError: print t print t.label raise yield t
def parse_datetime(self, text): m = re.match('(\d+)/(\d+)/(\d+) (\d+):(\d+) (\w+)', text) if m: date = datetime.datetime(int(m.group(3)), int(m.group(1)), int(m.group(2)), int(m.group(4)), int(m.group(5))) if m.group(6) == 'pm': date += datetime.timedelta(0,12*3600) return date m = re.match('(\d+)-(\d+)-(\d+) (\d+):(\d+)', text) if m: return datetime.datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) m = re.match('(\d+) (\w+) (\d+) (\d+):(\d+)', text) if m: return parse_french_date(text) self.logger.warning('Unable to parse "%s"' % text) return text
def parse(self, el): label = CleanText('//div[contains(@class, "lister")]//p[@class="c"]')(el) if not label: return label = re.findall('(\d+ [^ ]+ \d+)', label)[-1] # use the trick of relativedelta to get the last day of month. self.env['debit_date'] = parse_french_date(label) + relativedelta(day=31)
def iter_coming(self): for op in self.path('data.listerOperations.compte.operationAvenir.*.operation.*'): codeOperation = cast(op.get('codeOperation'), int, 0) # Coming transactions don't have real id tr = Transaction.from_dict({ 'type': self.COMING_TYPE_TO_TYPE.get(codeOperation) or Transaction.TYPE_UNKNOWN, 'amount': op.get('montant'), 'card': op.get('numeroPorteurCarte'), }) tr.parse(date=parse_french_date(op.get('dateOperation')), vdate=parse_french_date(op.get('valueDate')), raw=op.get('libelle')) if tr.type == Transaction.TYPE_CARD: tr.type = self.browser.card_to_transaction_type.get(op.get('keyCarte'), Transaction.TYPE_DEFERRED_CARD) yield tr
def parse_transaction(self, transaction, account): trans = [] if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif', u'Payé', u'En attente', u'Rejeté', u'Expiré', \ u'Created']: return [] if transaction['transactionDescription'].startswith(u'Offre de remboursement') or transaction['transactionDescription'].startswith(u'Commande à'): return [] t = FrenchTransaction(transaction['transactionId']) if not transaction['transactionAmount']['currencyCode'] == account.currency: cc = self.browser.convert_amount(account, transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId']) if not cc: return [] t.original_amount = Decimal('%.2f' % transaction['transactionAmount']['currencyDoubleValue']) t.original_currency = u'' + transaction['transactionAmount']['currencyCode'] t.amount = abs(cc) if not transaction['debit'] else -abs(cc) else: t.amount = Decimal('%.2f' % transaction['net']['currencyDoubleValue']) date = parse_french_date(transaction['transactionTime']) raw = transaction['transactionDescription'] if raw.startswith(u'Paiement \xe0') or raw.startswith('Achat de'): payback_id, payback_raw, payback_amount, payback_currency = self.browser.check_for_payback(transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId']) if payback_id and payback_raw and payback_amount and payback_currency: t_payback = FrenchTransaction(payback_id) t_payback.amount = payback_amount t_payback.original_currency = payback_currency t_payback.type = FrenchTransaction.TYPE_TRANSFER t_payback.parse(date=date, raw=u'Prélèvement pour %s' % raw) trans.append(t_payback) t.commission = Decimal('%.2f' % transaction['fee']['currencyDoubleValue']) t.parse(date=date, raw=raw) trans.append(t) return trans
def recap(self, origin, recipient, transfer): error = CleanText(u'//div[@id="transfer_form:moveMoneyDetailsBody"]//span[@class="error"]', default=None)(self.doc) or \ CleanText(u'//p[contains(text(), "Nous sommes désolés. Le solde de votre compte ne doit pas être inférieur au montant de votre découvert autorisé. Veuillez saisir un montant inférieur.")]', default=None)(self.doc) if error: raise TransferInvalidAmount(message=error) t = Transfer() t.label = transfer.label assert transfer.amount == CleanDecimal('//div[@id="transferSummary"]/div[@id="virementLabel"]\ //label[@class="digits positive"]', replace_dots=True)(self.doc) t.amount = transfer.amount t.currency = FrenchTransaction.Currency('//div[@id="transferSummary"]/div[@id="virementLabel"]\ //label[@class="digits positive"]')(self.doc) assert origin.label == CleanText('//div[@id="transferSummary"]/div[has-class("debit")]//span[@class="title"]')(self.doc) assert origin.balance == CleanDecimal('//div[@id="transferSummary"]/div[has-class("debit")]\ //label[@class="digits positive"]', replace_dots=True)(self.doc) t.account_balance = origin.balance t.account_label = origin.label t.account_iban = origin.iban t.account_id = origin.id assert recipient.label == CleanText('//div[@id="transferSummary"]/div[has-class("credit")]//span[@class="title"]')(self.doc) t.recipient_label = recipient.label t.recipient_iban = recipient.iban t.recipient_id = recipient.id t.exec_date = parse_french_date(CleanText('//p[has-class("exec-date")]', children=False, replace=[('le', ''), (u'exécuté', ''), ('demain', ''), ('(', ''), (')', ''), ("aujourd'hui", '')])(self.doc)).date() return t
def parse_transaction(self, transaction, account): t = FrenchTransaction(transaction['id']) if not transaction['isPrimaryCurrency']: original_currency = unicode(transaction['amounts']['txnCurrency']) if original_currency in self.browser.account_currencies: return [] if 'conversionFrom' in transaction['amounts'] and account.currency == transaction['amounts']['conversionFrom']['currency']: cc = self.format_amount(transaction['amounts']['conversionFrom']['value'], transaction['isCredit']) else: try: cc = self.browser.convert_amount(account, transaction, transaction['detailsLink']) except ServerError: self.logger.warning('Unable to go on detail, transaction skipped.') return [] if not cc: return [] t.original_amount = self.format_amount(transaction['amounts']['net']['value'], transaction["isCredit"]) t.original_currency = original_currency t.amount = self.format_amount(cc, transaction['isCredit']) else: t.amount = self.format_amount(transaction['amounts']['net']['value'], transaction['isCredit']) date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year']) raw = transaction.get('counterparty', transaction['displayType']) t.parse(date=date, raw=raw) return [t]
def obj_date(self): date = CleanText( './div/p[@class="item-date"]' )(self).split(" / ") if len(date) > 1: return parse_french_date(date[1].strip()) else: return NotLoaded
def obj_rdate(self): raw = self.obj_raw() mtc = re.search(r'\bDU (\d{2})\.?(\d{2})\.?(\d{2})\b', raw) if mtc: date = '%s/%s/%s' % (mtc.group(1), mtc.group(2), mtc.group(3)) return parse_french_date(date) return fromtimestamp(Dict('dateCreation')(self))
def obj_rdate(self): raw = self.obj_raw() mtc = re.search(r'\bDU (\d{6}|\d{8})\b', raw) if mtc: date = mtc.group(1) date = '%s/%s/%s' % (date[0:2], date[2:4], date[4:]) return parse_french_date(date) return fromtimestamp(Dict('dateCreation')(self))
def parse_transaction(self, transaction): t = FrenchTransaction(transaction['activityId']) date = parse_french_date(transaction['date']) raw = transaction['counterparty'] t.parse(date=date, raw=raw) amount = transaction['displayAmount'] t.set_amount(amount) t._currency = transaction['currencyCode'] return t
def obj_rdate(self): raw = self.obj_raw() mtc = re.search(r'\bDU (\d{6})\b', raw) if mtc: date = mtc.group(1) date = '%s/%s/%s' % (date[0:2], date[2:4], date[4:]) return parse_french_date(date) return fromtimestamp(self, Dict('dateCreation'))
def parse(self, el): label = CleanText( '//div[contains(@class, "lister")]//p[@class="c"]')(el) if not label: return label = re.findall('(\d+ [^ ]+ \d+)', label)[-1] # use the trick of relativedelta to get the last day of month. self.env['debit_date'] = parse_french_date( label) + relativedelta(day=31)
def obj_date(self): year = Regexp( CleanText( './preceding-sibling::li[@class="rowdate"]//span[@class="mois"]' ), r'(\d+)')(self) day_month = CleanText('.//div[has-class("col-date")]/span')( self) return parse_french_date(day_month + ' ' + year)
def iter_history(self): for op in self.get('data.listerOperations.compte.operationPassee') or []: codeFamille = cast(self.one('operationType.codeFamille', op), int) tr = Transaction.from_dict({ 'id': op.get('idOperation'), 'type': self.CODE_TO_TYPE.get(codeFamille) or Transaction.TYPE_UNKNOWN, 'category': op.get('categorie'), 'amount': self.one('montant.montant', op), }) tr.parse(raw=op.get('libelleOperation'), date=parse_french_date(op.get('dateOperation')), vdate=parse_french_date(self.one('montant.valueDate', op))) raw_is_summary = re.match(r'FACTURE CARTE SELON RELEVE DU\b|FACTURE CARTE CARTE AFFAIRES \d{4}X{8}\d{4} SUIVANT\b', tr.raw) if tr.type == Transaction.TYPE_CARD and raw_is_summary: tr.type = Transaction.TYPE_CARD_SUMMARY tr.deleted = True yield tr
def get_operations(self): for tr in self._get_operations(self)(): tr.type = tr.TYPE_DEFERRED_CARD deferred_date = Regexp( CleanText('//div[@class="date"][contains(text(), "Carte")]'), r'le ([^:]+)', default=None)(self.doc) if deferred_date: tr.date = parse_french_date(deferred_date) yield tr
def obj_rdate(self): raw = self.obj_raw() mtc = re.search(r'\bDU (\d{2})\.?(\d{2})\.?(\d{2})\b', raw) if mtc: date = '%s/%s/%s' % (mtc.group(1), mtc.group(2), mtc.group(3)) return parse_french_date(date) # The date can be truncated, so it is not retrieved if 'dateCreation' in self.el: return fromtimestamp(Dict('dateCreation')(self))
def parse(self, el): self.env['date'] = Date(Regexp(CleanText(u'//td[contains(text(), "Total prélevé")]'), ' (\d{2}/\d{2}/\d{4})', \ default=NotAvailable), default=NotAvailable)(self) \ or (parse_french_date('%s %s' % ('1', CleanText(u'//select[@id="moi"]/option[@selected]')(self))) + relativedelta(day=31)).date() self.env['_is_coming'] = date.today() < self.env['date'] amount = CleanText( TableCell('amount'))(self).split('dont frais') self.env['amount'] = amount[0] self.env['commission'] = amount[1] if len( amount) > 1 else NotAvailable
def get_ongoing_coming(self): # The title of the coming is usually 'Opérations débitées' but if # the coming is positive, it will become 'Opérations créditées' raw_date = Regexp(CleanText( '//table[@class="ca-table"]//tr[1]//b[contains(text(), "Opérations débitées") or contains(text(), "Opérations créditées")]' ), r'le (.*) :', default=None)(self.doc) if not raw_date: return None return parse_french_date(raw_date).date()
def get_operations(self): for tr in self._get_operations(self)(): tr.type = tr.TYPE_DEFERRED_CARD deferred_date = Regexp(CleanText('//div[@class="date"][contains(text(), "Carte")]'), r'le ([^:]+)', default=None)(self.doc) if deferred_date: tr.date = parse_french_date(deferred_date).date() # rdate > date doesn't make any sense but it seems that lcl website can do shitty things sometimes if tr.date >= tr.rdate: yield tr else: self.logger.error('skipping transaction with rdate > date')
def get_profile(self): profile = Person() content = self.get_content() profile.name = content['prenom'] + ' ' + content['nom'] profile.address = content['adresse'] + ' ' + content['codePostal'] + ' ' + content['ville'] profile.country = content['pays'] profile.birth_date = parse_french_date(content['dateNaissance']).date() return profile
def get_contact(self): profile_a = self.document.find("a", href=re.compile(r"profil_read.php\?.*")) _id = re.search(r"\?(.*)", profile_a["href"]).group(1) # not available in the 'boosted' version contact = Contact(_id, _id, Contact.STATUS_OFFLINE) contact.url = self.url contact.profile = {} thumbnail_url = "http://photos.onvasortir.com/%s/photos/%s_resize.png" % (self.browser.city, _id) if self.document.find("img", attrs={"src": thumbnail_url}): photo_url = thumbnail_url.replace("_resize", "") contact.set_photo("main", thumbnail_url=thumbnail_url, url=photo_url, hidden=False) location_a = self.document.find("a", href=re.compile(r"vue_profil_carte\.php\?")) if location_a: lat = float(re.search(r"Lat=([\d.]+)", location_a["href"]).group(1)) self._set_profile(contact, "latitude", lat) lng = float(re.search(r"Lng=([\d.]+)", location_a["href"]).group(1)) self._set_profile(contact, "longitude", lng) div = self.document.find("div", attrs={"class": "PADtitreBlanc_txt"}, text=re.compile("Personal Info")) td = div.findParent("tr").findNextSibling("tr").td infos_text = td.getText(separator="\n").strip() it = iter(infos_text.split("\n")) infos = dict(zip(it, it)) if infos["Sex :"] == "Man": self._set_profile(contact, "sex", "M") elif infos["Sex :"] == "Woman": self._set_profile(contact, "sex", "F") if infos["Birthday :"] != "Unknown": self._set_profile( contact, "birthday", parse_french_date(re.search(r"(\d+ \w+ \d+)", infos["Birthday :"]).group(1)) ) self._try_attr(contact, infos, "First Name :", "first_name") self._try_attr(contact, infos, "Status :", "marriage") self._try_attr(contact, infos, "Area :", "area") div = self.document.find("div", attrs={"class": "PADtitreBlanc_txt"}, text=re.compile("A few words")) td = div.findParent("tr").findNextSibling("tr").td summary = td.getText(separator="\n").strip() if summary == "Unknown": contact.summary = u"" else: contact.summary = summary div = self.document.find("div", style=re.compile("dashed")) if div: # TODO handle html, links and smileys contact.status_msg = div.getText() else: contact.status_msg = u"" return contact
def get_profile(self): profile = Person() content = self.get_content() profile.name = content['prenom'] + ' ' + content['nom'] profile.address = content['adresse'] + ' ' + content[ 'codePostal'] + ' ' + content['ville'] profile.country = content['pays'] profile.birth_date = parse_french_date(content['dateNaissance']).date() return profile
def iter_coming(self): for op in self.path( 'data.listerOperations.compte.operationAvenir.*.operation.*'): codeOperation = cast(op.get('codeOperation'), int, 0) # Coming transactions don't have real id tr = Transaction.from_dict({ 'type': self.COMING_TYPE_TO_TYPE.get(codeOperation) or Transaction.TYPE_UNKNOWN, 'amount': op.get('montant'), 'card': op.get('numeroPorteurCarte'), }) tr.parse(date=parse_french_date(op.get('dateOperation')), vdate=parse_french_date(op.get('valueDate')), raw=op.get('libelle')) if tr.type == Transaction.TYPE_CARD: tr.type = self.browser.card_to_transaction_type.get( op.get('keyCarte'), Transaction.TYPE_DEFERRED_CARD) yield tr
def get_contact(self): profile_a = self.document.find('a', href=re.compile(r'profil_read.php\?.*')) _id = re.search(r'\?(.*)', profile_a['href']).group(1) # not available in the 'boosted' version contact = Contact(_id, _id, Contact.STATUS_OFFLINE) contact.url = self.url contact.profile = {} thumbnail_url = 'http://photos.onvasortir.com/%s/photos/%s_resize.png' % (self.browser.city, _id) if self.document.find('img', attrs={'src': thumbnail_url}): photo_url = thumbnail_url.replace('_resize', '') contact.set_photo('main', thumbnail_url=thumbnail_url, url=photo_url, hidden=False) location_a = self.document.find('a', href=re.compile(r'vue_profil_carte\.php\?')) if location_a: lat = float(re.search(r'Lat=([\d.]+)', location_a['href']).group(1)) self._set_profile(contact, 'latitude', lat) lng = float(re.search(r'Lng=([\d.]+)', location_a['href']).group(1)) self._set_profile(contact, 'longitude', lng) div = self.document.find('div', attrs={'class': 'PADtitreBlanc_txt'}, text=re.compile('Personal Info')) td = div.findParent('tr').findNextSibling('tr').td infos_text = td.getText(separator='\n').strip() it = iter(infos_text.split('\n')) infos = dict(zip(it, it)) if infos['Sex :'] == 'Man': self._set_profile(contact, 'sex', 'M') elif infos['Sex :'] == 'Woman': self._set_profile(contact, 'sex', 'F') if infos['Birthday :'] != 'Unknown': self._set_profile(contact, 'birthday', parse_french_date(re.search(r'(\d+ \w+ \d+)', infos['Birthday :']).group(1))) self._try_attr(contact, infos, 'First Name :', 'first_name') self._try_attr(contact, infos, 'Status :', 'marriage') self._try_attr(contact, infos, 'Area :', 'area') div = self.document.find('div', attrs={'class': 'PADtitreBlanc_txt'}, text=re.compile('A few words')) td = div.findParent('tr').findNextSibling('tr').td summary = td.getText(separator='\n').strip() if summary == 'Unknown': contact.summary = u'' else: contact.summary = summary div = self.document.find('div', style=re.compile('dashed')) if div: # TODO handle html, links and smileys contact.status_msg = div.getText() else: contact.status_msg = u'' return contact
def iter_history(self): header, lines = self.doc[0], self.doc[1:][::-1] assert header == ['Date', "Libellé de l'opération ", ' Débit', 'Credit'], "wrong columns" for line in lines: tr = Transaction() tr.raw = line[1] assert not (line[2] and line[3]), "cannot have both debit and credit" amount = float(line[3] or 0) - abs(float(line[2] or 0)) tr.amount = Decimal(str(amount)) tr.date = parse_french_date(line[0]) yield tr
def parse_date(self, b): content = parse_b(b) a_date = content[1:content.index('-')] for fr, en in date_util.DATE_TRANSLATE_FR: a_date[1] = fr.sub(en, a_date[1]) if (datetime.now().month > datetime.strptime(a_date[1], "%B").month): a_date.append(u'%i' % (datetime.now().year + 1)) else: a_date.append(u'%i' % (datetime.now().year)) return date_util.parse_french_date(" ".join(a_date))
def iter_history(self, coming): for op in self.get('data.listerMouvements.listeMouvements') or []: #We have not date for this statut so we just skit it if op.get('statut') == u'En cours': continue tr = Transaction.from_dict({ 'type': Transaction.TYPE_BANK, '_state': op.get('statut'), 'amount': op.get('montantNet'), }) if op.get('statut') == 'Sans suite': continue tr.parse(date=parse_french_date(op.get('dateSaisie')), vdate = parse_french_date(op.get('dateEffet')) if op.get('dateEffet') else None, raw='%s %s' % (op.get('libelleMouvement'), op.get('canalSaisie') or '')) tr._op = op if (op.get('statut') == u'Traité') ^ coming: yield tr
def get_operations(self): for tr in self._get_operations(self)(): tr.type = tr.TYPE_DEFERRED_CARD deferred_date = Regexp( CleanText('//div[@class="date"][contains(text(), "Carte")]'), r'le ([^:]+)', default=None)(self.doc) if deferred_date: tr.date = parse_french_date(deferred_date).date() # rdate > date doesn't make any sense but it seems that lcl website can do shitty things sometimes if tr.date >= tr.rdate: yield tr else: self.logger.error('skipping transaction with rdate > date')
def get_profile(self): d = {el[0]: el[1] for el in self.doc} profile = Person() profile.name = '%s %s' % (d['Nom'], d['Prénom']) profile.birth_date = parse_french_date(d['Date de naissance']).date() profile.address = '%s %s %s' % (d['Adresse de correspondance'], d['Code postal résidence fiscale'], d['Ville adresse de correspondance']) profile.country = d['Pays adresse de correspondance'] profile.email = d['Adresse e-mail'] profile.phone = d.get('Téléphone portable') profile.job_activity_area = d.get('Secteur d\'activité') profile.job = d.get('Situation professionnelle') profile.company_name = d.get('Employeur') profile.family_situation = d.get('Situation familiale') return profile
def parse(self): for i, tr in enumerate(self.document.xpath('//tr')): t = FrenchTransaction(tr.xpath('./td[@class="transactionId"]/span')[0].text.strip()) date = parse_french_date(tr.xpath('./td[@class="date"]')[0].text.strip()) status = tr.xpath('./td[@class="desc"]/ul/li[@class="first"]')[0].text.strip() #We pass this because it's not transaction if status == u'Créé' or status == u'Annulé': continue raw = tr.xpath('./td[@class="desc"]/strong')[0].text.strip() t.parse(date=date, raw=raw) amount = tr.xpath('./td[@class="price"]/span')[0].text.strip() t.set_amount(amount) t._currency = Account.get_currency(amount) yield t
def get_history(self, date_guesser, state=None): seen = set() lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr') debit_date = None for i, line in enumerate(lines): is_balance = line.xpath('./td/@class="cel-texte cel-neg"') # It is possible to have three or four columns. cols = [self.parser.tocleanstring(td) for td in line.xpath('./td')] date = cols[0] label = cols[1] amount = cols[-1] t = Transaction() t.set_amount(amount) t.label = t.raw = label if is_balance: m = re.search('(\d+ [^ ]+ \d+)', label) if not m: raise BrokenPageError( 'Unable to read card balance in history: %r' % label) if state is None: debit_date = parse_french_date(m.group(1)) else: debit_date = state # Skip the first line because it is balance if i == 0: continue t.date = t.rdate = debit_date # Consider the second one as a positive amount to reset balance to 0. t.amount = -t.amount state = t.date else: day, month = map(int, date.split('/', 1)) t.rdate = date_guesser.guess_date(day, month) t.date = debit_date t.type = t.TYPE_CARD try: t.id = t.unique_id(seen) except UnicodeEncodeError: self.logger.debug(t) self.logger.debug(t.label) raise yield state, t
def parse_transaction(self, transaction, account): page = None if 'id' not in transaction or not transaction['date']: return [] t = FrenchTransaction(transaction['id']) if not transaction['isPrimaryCurrency']: if not 'txnCurrency' in transaction['amounts']: return [] original_currency = unicode(transaction['amounts']['txnCurrency']) if original_currency in self.browser.account_currencies: return [] if 'conversionFrom' in transaction[ 'amounts'] and 'value' in transaction['amounts'][ 'conversionFrom'] and account.currency == transaction[ 'amounts']['conversionFrom']['currency']: cc = self.format_amount( transaction['amounts']['conversionFrom']['value'], transaction['isCredit']) else: try: page = self.return_detail_page(transaction['detailsLink']) cc = page.get_converted_amount() if isinstance( page, HistoryDetailsPage) else None except ServerError: self.logger.warning( 'Unable to go on detail, transaction skipped.') return [] if not cc: return [] t.original_amount = self.format_amount( transaction['amounts']['net']['value'], transaction["isCredit"]) t.original_currency = original_currency t.amount = self.format_amount(cc, transaction['isCredit']) else: t.amount = self.format_amount( transaction['amounts']['net']['value'], transaction['isCredit']) date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year']).date() raw = transaction.get('counterparty', transaction['displayType']) t.parse(date=date, raw=raw) if page is None and t.amount < 0: page = self.return_detail_page(transaction['detailsLink']) funding_src = page.get_funding_src(t) if isinstance( page, HistoryDetailsPage) else None return [t] if funding_src is None else ([t] + [funding_src])
def get_history(self, date_guesser, state=None): seen = set() lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr') debit_date = None for i, line in enumerate(lines): is_balance = line.xpath('./td/@class="cel-texte cel-neg"') # It is possible to have three or four columns. cols = [self.parser.tocleanstring(td) for td in line.xpath('./td')] date = cols[0] label = cols[1] amount = cols[-1] t = Transaction() t.set_amount(amount) t.label = t.raw = label if is_balance: m = re.search('(\d+ [^ ]+ \d+)', label) if not m: raise BrokenPageError('Unable to read card balance in history: %r' % label) if state is None: debit_date = parse_french_date(m.group(1)) else: debit_date = state # Skip the first line because it is balance if i == 0: continue t.date = t.rdate = debit_date # Consider the second one as a positive amount to reset balance to 0. t.amount = -t.amount state = t.date else: day, month = map(int, date.split('/', 1)) t.rdate = date_guesser.guess_date(day, month) t.date = debit_date t.type = t.TYPE_CARD try: t.id = t.unique_id(seen) except UnicodeEncodeError: self.logger.debug(t) self.logger.debug(t.label) raise yield state, t
def parse_transaction(self, transaction, account): trans = [] if 'transactionStatus' in transaction and transaction[ 'transactionStatus'] in [ u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif', u'Payé', u'En attente', u'Rejeté', u'Expiré', u'Created', u'Brouillon', u'Paid', u'Pending', u'Canceled' ]: return [] for pattern in [u'Commande à', u'Offre de remboursement', u'Bill to']: if 'description' not in transaction[ 'transactionDescription'] or transaction[ 'transactionDescription']['description'].startswith( pattern): return [] t = FrenchTransaction(transaction['transactionId']) # Those are not really transactions. if not 'currency' in transaction['grossAmount'] \ or transaction['transactionDescription']['description'].startswith("Conversion de devise"): return [] original_currency = unicode(transaction['grossAmount']['currency']) if not original_currency == account.currency: if original_currency in self.browser.account_currencies: return [] cc = [tr['grossAmount']['amountUnformatted'] for tr in transaction['secondaryTransactions'] \ if account.currency == tr['grossAmount']['currency'] \ and (tr['grossAmount']['amountUnformatted'] < 0) == (transaction['grossAmount']['amountUnformatted'] < 0) \ and tr['transactionDescription']['description'].startswith('Conversion de devise')] if not cc: return [] assert len(cc) == 1 t.original_amount = Decimal( str(transaction['grossAmount']['amountUnformatted'])) t.original_currency = original_currency t.amount = Decimal(str(cc[0])) else: t.amount = Decimal( str(transaction['netAmount']['amountUnformatted'])) date = parse_french_date(transaction['transactionTime']) raw = "%s %s" % (transaction['transactionDescription']['description'], transaction['transactionDescription']['name']) if raw == "Transfert de Compte bancaire": t.type = FrenchTransaction.TYPE_TRANSFER t.commission = Decimal( str(transaction['feeAmount']['amountUnformatted'])) t.parse(date=date, raw=raw) trans.append(t) return trans
def iter_history(self, coming): for op in self.get('data.listerMouvements.listeMouvements') or []: #We have not date for this statut so we just skit it if op.get('statut') in self.IGNORED_STATUSES: continue tr = Transaction.from_dict({ 'type': Transaction.TYPE_BANK, '_state': op.get('statut'), 'amount': op.get('montantNet'), }) tr.parse(date=parse_french_date(op.get('dateSaisie')), vdate = parse_french_date(op.get('dateEffet')) if op.get('dateEffet') else None, raw='%s %s' % (op.get('libelleMouvement'), op.get('canalSaisie') or '')) tr._op = op if not tr.amount: if op.get('rib', {}).get('codeBanque') == 'null': self.logger.info('ignoring non-transaction with label %r', tr.raw) continue if (op.get('statut') == u'Traité') ^ coming: yield tr
def get_profile(self): self.get_universes() profile = Person() content = self.location('/transactionnel/services/rest/User/user').json()['content'] profile.name = content['prenom'] + ' ' + content['nom'] profile.address = content['adresse'] + ' ' + content['codePostal'] + ' ' + content['ville'] profile.country = content['pays'] profile.birth_date = parse_french_date(content['dateNaissance']).date() content = self.location('/transactionnel/services/applications/gestionEmail/getAdressesMails').json()['content'] profile.email = content['emailPart'] return profile
def parse_transaction(self, transaction, account): trans = [] if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif', u'Payé', u'En attente', u'Rejeté', u'Expiré', \ u'Created', u'Canceled']: return [] if transaction['transactionDescription'].startswith( u'Offre de remboursement' ) or transaction['transactionDescription'].startswith(u'Commande à'): return [] t = FrenchTransaction(transaction['transactionId']) original_currency = unicode( transaction['transactionAmount']['currencyCode']) if not original_currency == account.currency: if original_currency in self.browser.account_currencies: return [] cc = self.browser.convert_amount( account, transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId']) if not cc: return [] t.original_amount = Decimal( '%.2f' % transaction['transactionAmount']['currencyDoubleValue']) t.original_currency = original_currency t.amount = abs(cc) if not transaction['debit'] else -abs(cc) else: t.amount = Decimal('%.2f' % transaction['net']['currencyDoubleValue']) date = parse_french_date(transaction['transactionTime']) raw = transaction['transactionDescription'] if raw.startswith(u'Paiement \xe0') or raw.startswith('Achat de'): payback_id, payback_raw, payback_amount, payback_currency = self.browser.check_for_payback( transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId']) if payback_id and payback_raw and payback_amount and payback_currency: t_payback = FrenchTransaction(payback_id) t_payback.amount = payback_amount t_payback.original_currency = payback_currency t_payback.type = FrenchTransaction.TYPE_TRANSFER t_payback.parse(date=date, raw=u'Prélèvement pour %s' % raw) trans.append(t_payback) t.commission = Decimal('%.2f' % transaction['fee']['currencyDoubleValue']) t.parse(date=date, raw=raw) trans.append(t) return trans
def parse_transaction(self, transaction, account): t = FrenchTransaction(transaction['id']) if not transaction['isPrimaryCurrency']: cc = self.browser.convert_amount(account, transaction, transaction['detailsLink']) if not cc: return [] t.original_amount = self.format_amount(transaction['amounts']['net']['value'], transaction["isCredit"]) t.original_currency = u'' + transaction['amounts']['txnCurrency'] t.amount = self.format_amount(cc, transaction['isCredit']) else: t.amount = self.format_amount(transaction['amounts']['net']['value'], transaction['isCredit']) date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year']) raw = transaction.get('counterparty', transaction['displayType']) t.parse(date=date, raw=raw) return [t]
def parse(self): for i, tr in enumerate(self.document.xpath('//tr')): t = FrenchTransaction( tr.xpath('./td[@class="transactionId"]/span')[0].text.strip()) date = parse_french_date( tr.xpath('./td[@class="date"]')[0].text.strip()) status = tr.xpath( './td[@class="desc"]/ul/li[@class="first"]')[0].text.strip() #We pass this because it's not transaction if status == u'Créé' or status == u'Annulé': continue raw = tr.xpath('./td[@class="desc"]/strong')[0].text.strip() t.parse(date=date, raw=raw) amount = tr.xpath('./td[@class="price"]/span')[0].text.strip() t.set_amount(amount) t._currency = Account.get_currency(amount) yield t
def extract_concert(self, concert_table): d = {} date_h3 = concert_table.iter('h3').next() d['date'] = parse_french_date(date_h3.text) cancel_h2 = next(date_h3.itersiblings('h2'), None) if cancel_h2 is not None and cancel_h2.text.startswith('ANNUL'): d['active'] = False else: d['active'] = True performers_table = concert_table.iterdescendants('table').next() d['performers'] = list(self.extract_performers(performers_table)) d['summary'] = ' + '.join(p['name'] for p in d['performers']) d['description'] = d['summary'] return d
def extract_concert(self, concert_table): d = {} date_h3 = next(concert_table.iter('h3')) d['date'] = parse_french_date(date_h3.text) cancel_h2 = next(date_h3.itersiblings('h2'), None) if cancel_h2 is not None and cancel_h2.text.startswith('ANNUL'): d['active'] = False else: d['active'] = True performers_table = next(concert_table.iterdescendants('table')) d['performers'] = list(self.extract_performers(performers_table)) d['summary'] = ' + '.join(p['name'] for p in d['performers']) d['description'] = d['summary'] return d
def obj_date(self): if Env('is_card', default=False)(self): month = CleanText('//label[@for="movementSearch_period_%s"]' % ('1' if Env('is_previous', default=False)(self) else '0'), replace=[(u'Débit ', '')])(self) date_text = CleanText(u'//li[h3]/h4[@class="summary__title" and contains(text(), "Solde débité au")and contains(text(), "%s")]' % month, replace=[(u'Solde débité au ', '')])(self) if not date_text: date = Date(Attr('.//time', 'datetime'))(self) if self.page.browser.deferred_card_calendar is None: self.page.browser.location(Link('//a[contains(text(), "calendrier")]')(self)) closest = self.page.browser.get_debit_date(date) if closest: return closest return date debit_date = parse_french_date(date_text) return debit_date.date() return Date(Attr('.//time', 'datetime'))(self)
def extract_concert(self, concert_table): d = {} date_h3 = concert_table.iter("h3").next() d["date"] = parse_french_date(date_h3.text) cancel_h2 = next(date_h3.itersiblings("h2"), None) if cancel_h2 is not None and cancel_h2.text.startswith("ANNUL"): d["active"] = False else: d["active"] = True performers_table = concert_table.iterdescendants("table").next() d["performers"] = list(self.extract_performers(performers_table)) d["summary"] = " + ".join(p["name"] for p in d["performers"]) d["description"] = d["summary"] return d
def parse_transaction(self, transaction, account): t = FrenchTransaction(transaction['activityId']) date = parse_french_date(transaction['date']) raw = transaction.get('counterparty', transaction['displayType']) t.parse(date=date, raw=raw) try: if transaction['currencyCode'] != account.currency: transaction = self.browser.convert_amount(account, transaction) t.original_amount = self.format_amount(transaction['originalAmount'], transaction["isCredit"]) t.original_currency = u'' + transaction["currencyCode"] t.amount = self.format_amount(transaction['netAmount'], transaction["isCredit"]) except KeyError: return t._currency = transaction['currencyCode'] return t
def parse_transaction(self, transaction, account): if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif']: return t = FrenchTransaction(transaction['transactionId']) if not transaction['transactionAmount']['currencyCode'] == account.currency: cc = self.browser.convert_amount(account, transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId']) if not cc: return t.original_amount = Decimal(transaction['transactionAmount']['currencyDoubleValue']) t.original_currency = u'' + transaction['transactionAmount']['currencyCode'] t.set_amount(cc) else: t.amount = Decimal(transaction['transactionAmount']['currencyDoubleValue']) date = parse_french_date(transaction['transactionTime']) raw = transaction['transactionDescription'] t.commission = Decimal(transaction['fee']['currencyDoubleValue']) t.parse(date=date, raw=raw) return t
def _get_messages(self, thread): thread_div = self.document.find(True, 'PADpost_txt') used_ids = set() rcpt = self.document.find('input', attrs={'type': 'hidden', 'name': 'Dest'})['value'] sender_to_receiver = {rcpt: self.browser.username, self.browser.username: rcpt} # site is sorted from latest to oldest message for message_table in reversed(thread_div.findAll('table')): for td in message_table.findAll('td'): profile_a = td.find('a', href=re.compile(r'profil_read.php\?.*')) if not profile_a: continue first_br = td.find('br') assert first_br.nextSibling.name == 'br' text_nodes = ovsparse.all_next_siblings(first_br.nextSibling.nextSibling) # TODO #~ print text_nodes # date will be used as id sitedate = profile_a.findParent('div').find(text=re.compile(',.*')).replace(', ', '') sysdate = parse_french_date(sitedate) compactdate = datetime.datetime.strftime(sysdate, '%Y%m%dT%H%M%S') # but make it unique msg_id = ovsparse.create_unique_id(compactdate, used_ids) used_ids.add(msg_id) message = Message(thread, msg_id) message.sender = re.search(r'\?(.+)', profile_a['href']).group(1) message.receivers = [sender_to_receiver[message.sender]] message.date = sysdate message.content = ovsparse.html_message_to_text(text_nodes) notread_self = bool(td.find('span', 'ColorSurligne')) notread_other = bool(td.find('span', 'new_sortiepartenaire')) if notread_other or notread_self: message.flags |= Message.IS_NOT_RECEIVED else: message.flags |= Message.IS_RECEIVED yield message
def parse_transaction(self, transaction): t = FrenchTransaction(transaction['activityId']) date = parse_french_date(transaction['date']) try: raw = transaction['counterparty'] except KeyError: raw = transaction['displayType'] t.parse(date=date, raw=raw) try: amount = transaction['netAmount'] except KeyError: return if transaction['isCredit']: t.set_amount(credit=amount) else: t.set_amount(debit=amount) t._currency = transaction['currencyCode'] return t
def get_history(self): index = 0 # Check if this is a multi-cards page pages = [] for a in self.document.xpath('//table[@class="liste"]/tbody/tr/td/a'): card_link = a.get('href') history_url = 'https://%s/%s/fr/banque/%s' % ( self.browser.DOMAIN, self.browser.currentSubBank, card_link) page = self.browser.get_document(self.browser.openurl(history_url)) pages.append(page) if len(pages) == 0: # If not, add this page as transactions list pages.append(self.document) for page in pages: label = self.parser.tocleanstring( self.parser.select(page.getroot(), 'div.lister p.c', 1)) label = re.findall('(\d+ [^ ]+ \d+)', label)[-1] # use the trick of relativedelta to get the last day of month. debit_date = parse_french_date(label) + relativedelta(day=31) for tr in page.xpath('//table[@class="liste"]/tbody/tr'): tds = tr.findall('td')[:4] if len(tds) < 4: continue tr = Transaction(index) parts = [ txt.strip() for txt in list(tds[-3].itertext()) + list(tds[-2].itertext()) if len(txt.strip()) > 0 ] tr.parse(date=tds[0].text.strip(' \xa0'), raw=u' '.join(parts)) tr.date = debit_date tr.type = tr.TYPE_CARD # Don't take all of the content (with tocleanstring for example), # because there is a span.aide. tr.set_amount(tds[-1].text) yield tr
def handle_response(self, account, recipient, amount, reason): self.check_errors() transfer_data = self.doc['data']['validationVirement'] self.abort_if_unknown(transfer_data) if 'idBeneficiaire' in transfer_data and transfer_data[ 'idBeneficiaire'] is not None: assert transfer_data['idBeneficiaire'] == recipient._transfer_id elif transfer_data.get('ibanCompteCrediteur'): assert transfer_data['ibanCompteCrediteur'] == recipient.iban transfer = Transfer() transfer.currency = transfer_data['devise'] transfer.amount = Decimal(transfer_data['montantEuros']) transfer.account_iban = transfer_data['ibanCompteDebiteur'] transfer.account_id = account.id try: transfer.recipient_iban = transfer_data[ 'ibanCompteCrediteur'] or recipient.iban except KeyError: # In last version, json contains a key 'idBeneficiaire' containing: # "idBeneficiaire" : "00003##00001####FR7610278123456789028070101", transfer.recipient_id = transfer_data['idBeneficiaire'] transfer.recipient_iban = transfer.recipient_id.split( '#')[-1] or recipient.iban else: transfer.recipient_id = recipient.id transfer.exec_date = parse_french_date( transfer_data['dateExecution']).date() transfer.fees = Decimal(transfer_data.get('montantFrais', '0')) transfer.label = transfer_data['motifVirement'] transfer.account_label = account.label transfer.recipient_label = recipient.label transfer.id = transfer_data['reference'] # This is true if a transfer with the same metadata has already been done recently transfer._doublon = transfer_data['doublon'] transfer.account_balance = account.balance return transfer
def parse(self): for tr in self.document.xpath('//tbody/tr'): tlink = tr.xpath('./td[@class="desc"]/a[@class="rowClick"]')[0].attrib['href'].strip() t = FrenchTransaction(tlink[tlink.find('&id=')+4:]) date = parse_french_date(tr.xpath('./td[@class="date"]')[0].text.strip()) raw = tr.xpath('./td[@class="desc"]/a[@class="rowClick"]')[0].tail.strip() # Filter lines that do not actually modify the balance if raw.startswith('Autorisation ') or raw.endswith(' en attente par PayPal'): continue t.parse(date=date, raw=raw) amount = tr.xpath('./td[@class="price-value net"]')[0].text.strip() t.set_amount(amount) commission = tr.xpath('./td[@class="price-value fee"]')[0].text.strip() t.commission = Decimal(t.clean_amount(commission)) t.label = t.raw if t.commission: t.label += " (%s)" % tr.xpath('./td[@class="price-value gross"]')[0].text.strip() t._currency = Account.get_currency(amount) yield t
def parse_transaction(self, transaction, account): if 'id' not in transaction or not transaction['date']: return [] t = FrenchTransaction(transaction['id']) if not transaction['isPrimaryCurrency']: original_currency = unicode(transaction['amounts']['txnCurrency']) if original_currency in self.browser.account_currencies: return [] if 'conversionFrom' in transaction[ 'amounts'] and 'value' in transaction['amounts'][ 'conversionFrom'] and account.currency == transaction[ 'amounts']['conversionFrom']['currency']: cc = self.format_amount( transaction['amounts']['conversionFrom']['value'], transaction['isCredit']) else: try: cc = self.browser.convert_amount( account, transaction, transaction['detailsLink']) except ServerError: self.logger.warning( 'Unable to go on detail, transaction skipped.') return [] if not cc: return [] t.original_amount = self.format_amount( transaction['amounts']['net']['value'], transaction["isCredit"]) t.original_currency = original_currency t.amount = self.format_amount(cc, transaction['isCredit']) else: t.amount = self.format_amount( transaction['amounts']['net']['value'], transaction['isCredit']) date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year']) raw = transaction.get('counterparty', transaction['displayType']) t.parse(date=date, raw=raw) return [t]
def parse_datetime(self, text): m = re.match('(\d+)/(\d+)/(\d+) (\d+):(\d+) (\w+)', text) if m: date = datetime.datetime(int(m.group(3)), int(m.group(1)), int(m.group(2)), int(m.group(4)), int(m.group(5))) if m.group(6) == 'pm': date += datetime.timedelta(0, 12 * 3600) return date m = re.match('(\d+)-(\d+)-(\d+) (\d+):(\d+)', text) if m: return datetime.datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) m = re.match('(\d+) (\w+) (\d+) (\d+):(\d+)', text) if m: return parse_french_date(text) self.logger.warning('Unable to parse "%s"' % text) return text
def parse_transaction(self, transaction, account): t = FrenchTransaction(transaction['id']) if not transaction['isPrimaryCurrency']: cc = self.browser.convert_amount(account, transaction, transaction['detailsLink']) if not cc: return [] t.original_amount = self.format_amount( transaction['amounts']['net']['value'], transaction["isCredit"]) t.original_currency = u'' + transaction['amounts']['txnCurrency'] t.amount = self.format_amount(cc, transaction['isCredit']) else: t.amount = self.format_amount( transaction['amounts']['net']['value'], transaction['isCredit']) date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year']) raw = transaction.get('counterparty', transaction['displayType']) t.parse(date=date, raw=raw) return [t]
def text2issue(self, issue, m): # XXX HACK to support real incoming emails if 'Subject' in m: m['Title'] = m['Subject'] for key, (list_name, is_list_object) in self.ISSUE_FIELDS: value = m.get(key) if value is None: continue new_value = u'' for part in decode_header(value): if part[1]: new_value += unicode(part[0], part[1]) else: new_value += part[0].decode('utf-8') value = new_value if is_list_object: objects_list = getattr(issue.project, list_name) value = self.get_list_item(objects_list, value) # FIXME: autodetect if key in ['start', 'due']: if len(value) > 0: #value = datetime.strptime(value, "%Y-%m-%d %H:%M:%S") value = parse_french_date(value) else: value = None setattr(issue, key, value) for key in issue.fields.keys(): value = m.get(self.sanitize_key(key)) if value is not None: issue.fields[key] = value.decode('utf-8') content = u'' for part in m.walk(): if part.get_content_type() == 'text/plain': s = part.get_payload(decode=True) charsets = part.get_charsets() + m.get_charsets() for charset in charsets: try: if charset is not None: content += unicode(s, charset) else: content += unicode(s, encoding='utf-8') except UnicodeError as e: self.logger.warning('Unicode error: %s' % e) continue except Exception as e: self.logger.exception(e) continue else: break issue.body = content m = re.search('([^< ]+@[^ >]+)', m['From'] or '') if m: return m.group(1)