def handle_response(self, transfer):
self.check_errors()
transfer_data = self.doc['data']['enregistrementVirement']
transfer.id = transfer_data['reference']
transfer.exec_date = parse_french_date(self.doc['data']['enregistrementVirement']['dateExecution']).date()
# Timestamp at which the bank registered the transfer
register_date = re.sub(' 24:', ' 00:', self.doc['data']['enregistrementVirement']['dateEnregistrement'])
transfer._register_date = parse_french_date(register_date)
return transfer
def get_history(self, date_guesser):
seen = set()
lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr')
for line in lines[1:]: # first line is balance
is_balance = line.xpath('./td/@class="cel-texte cel-neg"')
[date, label, _, amount] = [self.parser.tocleanstring(td)
for td in line.xpath('./td')]
t = Transaction(0)
t.set_amount(amount)
t.label = t.raw = label
if is_balance:
m = re.search('(\d+ [^ ]+ \d+)', label)
if not m:
raise BrokenPageError('Unable to read card balance in history: %r' % label)
t.date = parse_french_date(m.group(1))
t.amount = -t.amount
else:
day, month = map(int, date.split('/', 1))
t.date = date_guesser.guess_date(day, month)
t.type = t.TYPE_CARD
t.rdate = t.date
try:
t.id = t.unique_id(seen)
except UnicodeEncodeError:
print t
print t.label
raise
yield t
def parse_datetime(self, text):
m = re.match('(\d+)/(\d+)/(\d+) (\d+):(\d+) (\w+)', text)
if m:
date = datetime.datetime(int(m.group(3)),
int(m.group(1)),
int(m.group(2)),
int(m.group(4)),
int(m.group(5)))
if m.group(6) == 'pm':
date += datetime.timedelta(0,12*3600)
return date
m = re.match('(\d+)-(\d+)-(\d+) (\d+):(\d+)', text)
if m:
return datetime.datetime(int(m.group(1)),
int(m.group(2)),
int(m.group(3)),
int(m.group(4)),
int(m.group(5)))
m = re.match('(\d+) (\w+) (\d+) (\d+):(\d+)', text)
if m:
return parse_french_date(text)
self.logger.warning('Unable to parse "%s"' % text)
return text
def parse(self, el):
label = CleanText('//div[contains(@class, "lister")]//p[@class="c"]')(el)
if not label:
return
label = re.findall('(\d+ [^ ]+ \d+)', label)[-1]
# use the trick of relativedelta to get the last day of month.
self.env['debit_date'] = parse_french_date(label) + relativedelta(day=31)
def iter_coming(self):
for op in self.path('data.listerOperations.compte.operationAvenir.*.operation.*'):
codeOperation = cast(op.get('codeOperation'), int, 0)
# Coming transactions don't have real id
tr = Transaction.from_dict({
'type': self.COMING_TYPE_TO_TYPE.get(codeOperation) or Transaction.TYPE_UNKNOWN,
'amount': op.get('montant'),
'card': op.get('numeroPorteurCarte'),
})
tr.parse(date=parse_french_date(op.get('dateOperation')),
vdate=parse_french_date(op.get('valueDate')),
raw=op.get('libelle'))
if tr.type == Transaction.TYPE_CARD:
tr.type = self.browser.card_to_transaction_type.get(op.get('keyCarte'),
Transaction.TYPE_DEFERRED_CARD)
yield tr
def parse_transaction(self, transaction, account):
trans = []
if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif', u'Payé', u'En attente', u'Rejeté', u'Expiré', \
u'Created']:
return []
if transaction['transactionDescription'].startswith(u'Offre de remboursement') or transaction['transactionDescription'].startswith(u'Commande à'):
return []
t = FrenchTransaction(transaction['transactionId'])
if not transaction['transactionAmount']['currencyCode'] == account.currency:
cc = self.browser.convert_amount(account, transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId'])
if not cc:
return []
t.original_amount = Decimal('%.2f' % transaction['transactionAmount']['currencyDoubleValue'])
t.original_currency = u'' + transaction['transactionAmount']['currencyCode']
t.amount = abs(cc) if not transaction['debit'] else -abs(cc)
else:
t.amount = Decimal('%.2f' % transaction['net']['currencyDoubleValue'])
date = parse_french_date(transaction['transactionTime'])
raw = transaction['transactionDescription']
if raw.startswith(u'Paiement \xe0') or raw.startswith('Achat de'):
payback_id, payback_raw, payback_amount, payback_currency = self.browser.check_for_payback(transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId'])
if payback_id and payback_raw and payback_amount and payback_currency:
t_payback = FrenchTransaction(payback_id)
t_payback.amount = payback_amount
t_payback.original_currency = payback_currency
t_payback.type = FrenchTransaction.TYPE_TRANSFER
t_payback.parse(date=date, raw=u'Prélèvement pour %s' % raw)
trans.append(t_payback)
t.commission = Decimal('%.2f' % transaction['fee']['currencyDoubleValue'])
t.parse(date=date, raw=raw)
trans.append(t)
return trans
def recap(self, origin, recipient, transfer):
error = CleanText(u'//div[@id="transfer_form:moveMoneyDetailsBody"]//span[@class="error"]', default=None)(self.doc) or \
CleanText(u'//p[contains(text(), "Nous sommes désolés. Le solde de votre compte ne doit pas être inférieur au montant de votre découvert autorisé. Veuillez saisir un montant inférieur.")]', default=None)(self.doc)
if error:
raise TransferInvalidAmount(message=error)
t = Transfer()
t.label = transfer.label
assert transfer.amount == CleanDecimal('//div[@id="transferSummary"]/div[@id="virementLabel"]\
//label[@class="digits positive"]', replace_dots=True)(self.doc)
t.amount = transfer.amount
t.currency = FrenchTransaction.Currency('//div[@id="transferSummary"]/div[@id="virementLabel"]\
//label[@class="digits positive"]')(self.doc)
assert origin.label == CleanText('//div[@id="transferSummary"]/div[has-class("debit")]//span[@class="title"]')(self.doc)
assert origin.balance == CleanDecimal('//div[@id="transferSummary"]/div[has-class("debit")]\
//label[@class="digits positive"]', replace_dots=True)(self.doc)
t.account_balance = origin.balance
t.account_label = origin.label
t.account_iban = origin.iban
t.account_id = origin.id
assert recipient.label == CleanText('//div[@id="transferSummary"]/div[has-class("credit")]//span[@class="title"]')(self.doc)
t.recipient_label = recipient.label
t.recipient_iban = recipient.iban
t.recipient_id = recipient.id
t.exec_date = parse_french_date(CleanText('//p[has-class("exec-date")]', children=False,
replace=[('le', ''), (u'exécuté', ''), ('demain', ''), ('(', ''), (')', ''),
("aujourd'hui", '')])(self.doc)).date()
return t
def parse_transaction(self, transaction, account):
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
original_currency = unicode(transaction['amounts']['txnCurrency'])
if original_currency in self.browser.account_currencies:
return []
if 'conversionFrom' in transaction['amounts'] and account.currency == transaction['amounts']['conversionFrom']['currency']:
cc = self.format_amount(transaction['amounts']['conversionFrom']['value'], transaction['isCredit'])
else:
try:
cc = self.browser.convert_amount(account, transaction, transaction['detailsLink'])
except ServerError:
self.logger.warning('Unable to go on detail, transaction skipped.')
return []
if not cc:
return []
t.original_amount = self.format_amount(transaction['amounts']['net']['value'], transaction["isCredit"])
t.original_currency = original_currency
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(transaction['amounts']['net']['value'], transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
return [t]
def obj_date(self):
date = CleanText(
'./div/p[@class="item-date"]'
)(self).split(" / ")
if len(date) > 1:
return parse_french_date(date[1].strip())
else:
return NotLoaded
def obj_rdate(self):
raw = self.obj_raw()
mtc = re.search(r'\bDU (\d{2})\.?(\d{2})\.?(\d{2})\b', raw)
if mtc:
date = '%s/%s/%s' % (mtc.group(1), mtc.group(2), mtc.group(3))
return parse_french_date(date)
return fromtimestamp(Dict('dateCreation')(self))
def obj_rdate(self):
raw = self.obj_raw()
mtc = re.search(r'\bDU (\d{6}|\d{8})\b', raw)
if mtc:
date = mtc.group(1)
date = '%s/%s/%s' % (date[0:2], date[2:4], date[4:])
return parse_french_date(date)
return fromtimestamp(Dict('dateCreation')(self))
def parse_transaction(self, transaction):
t = FrenchTransaction(transaction['activityId'])
date = parse_french_date(transaction['date'])
raw = transaction['counterparty']
t.parse(date=date, raw=raw)
amount = transaction['displayAmount']
t.set_amount(amount)
t._currency = transaction['currencyCode']
return t
def obj_rdate(self):
raw = self.obj_raw()
mtc = re.search(r'\bDU (\d{6})\b', raw)
if mtc:
date = mtc.group(1)
date = '%s/%s/%s' % (date[0:2], date[2:4], date[4:])
return parse_french_date(date)
return fromtimestamp(self, Dict('dateCreation'))
def parse(self, el):
label = CleanText(
'//div[contains(@class, "lister")]//p[@class="c"]')(el)
if not label:
return
label = re.findall('(\d+ [^ ]+ \d+)', label)[-1]
# use the trick of relativedelta to get the last day of month.
self.env['debit_date'] = parse_french_date(
label) + relativedelta(day=31)
def obj_date(self):
year = Regexp(
CleanText(
'./preceding-sibling::li[@class="rowdate"]//span[@class="mois"]'
), r'(\d+)')(self)
day_month = CleanText('.//div[has-class("col-date")]/span')(
self)
return parse_french_date(day_month + ' ' + year)
def iter_history(self):
for op in self.get('data.listerOperations.compte.operationPassee') or []:
codeFamille = cast(self.one('operationType.codeFamille', op), int)
tr = Transaction.from_dict({
'id': op.get('idOperation'),
'type': self.CODE_TO_TYPE.get(codeFamille) or Transaction.TYPE_UNKNOWN,
'category': op.get('categorie'),
'amount': self.one('montant.montant', op),
})
tr.parse(raw=op.get('libelleOperation'),
date=parse_french_date(op.get('dateOperation')),
vdate=parse_french_date(self.one('montant.valueDate', op)))
raw_is_summary = re.match(r'FACTURE CARTE SELON RELEVE DU\b|FACTURE CARTE CARTE AFFAIRES \d{4}X{8}\d{4} SUIVANT\b', tr.raw)
if tr.type == Transaction.TYPE_CARD and raw_is_summary:
tr.type = Transaction.TYPE_CARD_SUMMARY
tr.deleted = True
yield tr
def get_operations(self):
for tr in self._get_operations(self)():
tr.type = tr.TYPE_DEFERRED_CARD
deferred_date = Regexp(
CleanText('//div[@class="date"][contains(text(), "Carte")]'),
r'le ([^:]+)',
default=None)(self.doc)
if deferred_date:
tr.date = parse_french_date(deferred_date)
yield tr
def obj_rdate(self):
raw = self.obj_raw()
mtc = re.search(r'\bDU (\d{2})\.?(\d{2})\.?(\d{2})\b', raw)
if mtc:
date = '%s/%s/%s' % (mtc.group(1), mtc.group(2), mtc.group(3))
return parse_french_date(date)
# The date can be truncated, so it is not retrieved
if 'dateCreation' in self.el:
return fromtimestamp(Dict('dateCreation')(self))
def parse(self, el):
self.env['date'] = Date(Regexp(CleanText(u'//td[contains(text(), "Total prélevé")]'), ' (\d{2}/\d{2}/\d{4})', \
default=NotAvailable), default=NotAvailable)(self) \
or (parse_french_date('%s %s' % ('1', CleanText(u'//select[@id="moi"]/option[@selected]')(self))) + relativedelta(day=31)).date()
self.env['_is_coming'] = date.today() < self.env['date']
amount = CleanText(
TableCell('amount'))(self).split('dont frais')
self.env['amount'] = amount[0]
self.env['commission'] = amount[1] if len(
amount) > 1 else NotAvailable
def get_ongoing_coming(self):
# The title of the coming is usually 'Opérations débitées' but if
# the coming is positive, it will become 'Opérations créditées'
raw_date = Regexp(CleanText(
'//table[@class="ca-table"]//tr[1]//b[contains(text(), "Opérations débitées") or contains(text(), "Opérations créditées")]'
),
r'le (.*) :',
default=None)(self.doc)
if not raw_date:
return None
return parse_french_date(raw_date).date()
def get_operations(self):
for tr in self._get_operations(self)():
tr.type = tr.TYPE_DEFERRED_CARD
deferred_date = Regexp(CleanText('//div[@class="date"][contains(text(), "Carte")]'), r'le ([^:]+)', default=None)(self.doc)
if deferred_date:
tr.date = parse_french_date(deferred_date).date()
# rdate > date doesn't make any sense but it seems that lcl website can do shitty things sometimes
if tr.date >= tr.rdate:
yield tr
else:
self.logger.error('skipping transaction with rdate > date')
def get_profile(self):
profile = Person()
content = self.get_content()
profile.name = content['prenom'] + ' ' + content['nom']
profile.address = content['adresse'] + ' ' + content['codePostal'] + ' ' + content['ville']
profile.country = content['pays']
profile.birth_date = parse_french_date(content['dateNaissance']).date()
return profile
def get_contact(self):
profile_a = self.document.find("a", href=re.compile(r"profil_read.php\?.*"))
_id = re.search(r"\?(.*)", profile_a["href"]).group(1)
# not available in the 'boosted' version
contact = Contact(_id, _id, Contact.STATUS_OFFLINE)
contact.url = self.url
contact.profile = {}
thumbnail_url = "http://photos.onvasortir.com/%s/photos/%s_resize.png" % (self.browser.city, _id)
if self.document.find("img", attrs={"src": thumbnail_url}):
photo_url = thumbnail_url.replace("_resize", "")
contact.set_photo("main", thumbnail_url=thumbnail_url, url=photo_url, hidden=False)
location_a = self.document.find("a", href=re.compile(r"vue_profil_carte\.php\?"))
if location_a:
lat = float(re.search(r"Lat=([\d.]+)", location_a["href"]).group(1))
self._set_profile(contact, "latitude", lat)
lng = float(re.search(r"Lng=([\d.]+)", location_a["href"]).group(1))
self._set_profile(contact, "longitude", lng)
div = self.document.find("div", attrs={"class": "PADtitreBlanc_txt"}, text=re.compile("Personal Info"))
td = div.findParent("tr").findNextSibling("tr").td
infos_text = td.getText(separator="\n").strip()
it = iter(infos_text.split("\n"))
infos = dict(zip(it, it))
if infos["Sex :"] == "Man":
self._set_profile(contact, "sex", "M")
elif infos["Sex :"] == "Woman":
self._set_profile(contact, "sex", "F")
if infos["Birthday :"] != "Unknown":
self._set_profile(
contact, "birthday", parse_french_date(re.search(r"(\d+ \w+ \d+)", infos["Birthday :"]).group(1))
)
self._try_attr(contact, infos, "First Name :", "first_name")
self._try_attr(contact, infos, "Status :", "marriage")
self._try_attr(contact, infos, "Area :", "area")
div = self.document.find("div", attrs={"class": "PADtitreBlanc_txt"}, text=re.compile("A few words"))
td = div.findParent("tr").findNextSibling("tr").td
summary = td.getText(separator="\n").strip()
if summary == "Unknown":
contact.summary = u""
else:
contact.summary = summary
div = self.document.find("div", style=re.compile("dashed"))
if div:
# TODO handle html, links and smileys
contact.status_msg = div.getText()
else:
contact.status_msg = u""
return contact
def get_profile(self):
profile = Person()
content = self.get_content()
profile.name = content['prenom'] + ' ' + content['nom']
profile.address = content['adresse'] + ' ' + content[
'codePostal'] + ' ' + content['ville']
profile.country = content['pays']
profile.birth_date = parse_french_date(content['dateNaissance']).date()
return profile
def iter_coming(self):
for op in self.path(
'data.listerOperations.compte.operationAvenir.*.operation.*'):
codeOperation = cast(op.get('codeOperation'), int, 0)
# Coming transactions don't have real id
tr = Transaction.from_dict({
'type':
self.COMING_TYPE_TO_TYPE.get(codeOperation)
or Transaction.TYPE_UNKNOWN,
'amount':
op.get('montant'),
'card':
op.get('numeroPorteurCarte'),
})
tr.parse(date=parse_french_date(op.get('dateOperation')),
vdate=parse_french_date(op.get('valueDate')),
raw=op.get('libelle'))
if tr.type == Transaction.TYPE_CARD:
tr.type = self.browser.card_to_transaction_type.get(
op.get('keyCarte'), Transaction.TYPE_DEFERRED_CARD)
yield tr
def get_contact(self):
profile_a = self.document.find('a', href=re.compile(r'profil_read.php\?.*'))
_id = re.search(r'\?(.*)', profile_a['href']).group(1)
# not available in the 'boosted' version
contact = Contact(_id, _id, Contact.STATUS_OFFLINE)
contact.url = self.url
contact.profile = {}
thumbnail_url = 'http://photos.onvasortir.com/%s/photos/%s_resize.png' % (self.browser.city, _id)
if self.document.find('img', attrs={'src': thumbnail_url}):
photo_url = thumbnail_url.replace('_resize', '')
contact.set_photo('main', thumbnail_url=thumbnail_url, url=photo_url, hidden=False)
location_a = self.document.find('a', href=re.compile(r'vue_profil_carte\.php\?'))
if location_a:
lat = float(re.search(r'Lat=([\d.]+)', location_a['href']).group(1))
self._set_profile(contact, 'latitude', lat)
lng = float(re.search(r'Lng=([\d.]+)', location_a['href']).group(1))
self._set_profile(contact, 'longitude', lng)
div = self.document.find('div', attrs={'class': 'PADtitreBlanc_txt'}, text=re.compile('Personal Info'))
td = div.findParent('tr').findNextSibling('tr').td
infos_text = td.getText(separator='\n').strip()
it = iter(infos_text.split('\n'))
infos = dict(zip(it, it))
if infos['Sex :'] == 'Man':
self._set_profile(contact, 'sex', 'M')
elif infos['Sex :'] == 'Woman':
self._set_profile(contact, 'sex', 'F')
if infos['Birthday :'] != 'Unknown':
self._set_profile(contact, 'birthday', parse_french_date(re.search(r'(\d+ \w+ \d+)', infos['Birthday :']).group(1)))
self._try_attr(contact, infos, 'First Name :', 'first_name')
self._try_attr(contact, infos, 'Status :', 'marriage')
self._try_attr(contact, infos, 'Area :', 'area')
div = self.document.find('div', attrs={'class': 'PADtitreBlanc_txt'}, text=re.compile('A few words'))
td = div.findParent('tr').findNextSibling('tr').td
summary = td.getText(separator='\n').strip()
if summary == 'Unknown':
contact.summary = u''
else:
contact.summary = summary
div = self.document.find('div', style=re.compile('dashed'))
if div:
# TODO handle html, links and smileys
contact.status_msg = div.getText()
else:
contact.status_msg = u''
return contact
def iter_history(self):
header, lines = self.doc[0], self.doc[1:][::-1]
assert header == ['Date', "Libellé de l'opération ", ' Débit', 'Credit'], "wrong columns"
for line in lines:
tr = Transaction()
tr.raw = line[1]
assert not (line[2] and line[3]), "cannot have both debit and credit"
amount = float(line[3] or 0) - abs(float(line[2] or 0))
tr.amount = Decimal(str(amount))
tr.date = parse_french_date(line[0])
yield tr
def parse_date(self, b):
content = parse_b(b)
a_date = content[1:content.index('-')]
for fr, en in date_util.DATE_TRANSLATE_FR:
a_date[1] = fr.sub(en, a_date[1])
if (datetime.now().month > datetime.strptime(a_date[1], "%B").month):
a_date.append(u'%i' % (datetime.now().year + 1))
else:
a_date.append(u'%i' % (datetime.now().year))
return date_util.parse_french_date(" ".join(a_date))
def iter_history(self, coming):
for op in self.get('data.listerMouvements.listeMouvements') or []:
#We have not date for this statut so we just skit it
if op.get('statut') == u'En cours':
continue
tr = Transaction.from_dict({
'type': Transaction.TYPE_BANK,
'_state': op.get('statut'),
'amount': op.get('montantNet'),
})
if op.get('statut') == 'Sans suite':
continue
tr.parse(date=parse_french_date(op.get('dateSaisie')),
vdate = parse_french_date(op.get('dateEffet')) if op.get('dateEffet') else None,
raw='%s %s' % (op.get('libelleMouvement'), op.get('canalSaisie') or ''))
tr._op = op
if (op.get('statut') == u'Traité') ^ coming:
yield tr
def iter_history(self):
header, lines = self.doc[0], self.doc[1:][::-1]
assert header == ['Date', "Libellé de l'opération ", ' Débit', 'Credit'], "wrong columns"
for line in lines:
tr = Transaction()
tr.raw = line[1]
assert not (line[2] and line[3]), "cannot have both debit and credit"
amount = float(line[3] or 0) - abs(float(line[2] or 0))
tr.amount = Decimal(str(amount))
tr.date = parse_french_date(line[0])
yield tr
def get_operations(self):
for tr in self._get_operations(self)():
tr.type = tr.TYPE_DEFERRED_CARD
deferred_date = Regexp(
CleanText('//div[@class="date"][contains(text(), "Carte")]'),
r'le ([^:]+)',
default=None)(self.doc)
if deferred_date:
tr.date = parse_french_date(deferred_date).date()
# rdate > date doesn't make any sense but it seems that lcl website can do shitty things sometimes
if tr.date >= tr.rdate:
yield tr
else:
self.logger.error('skipping transaction with rdate > date')
def get_profile(self):
d = {el[0]: el[1] for el in self.doc}
profile = Person()
profile.name = '%s %s' % (d['Nom'], d['Prénom'])
profile.birth_date = parse_french_date(d['Date de naissance']).date()
profile.address = '%s %s %s' % (d['Adresse de correspondance'], d['Code postal résidence fiscale'], d['Ville adresse de correspondance'])
profile.country = d['Pays adresse de correspondance']
profile.email = d['Adresse e-mail']
profile.phone = d.get('Téléphone portable')
profile.job_activity_area = d.get('Secteur d\'activité')
profile.job = d.get('Situation professionnelle')
profile.company_name = d.get('Employeur')
profile.family_situation = d.get('Situation familiale')
return profile
def get_profile(self):
d = {el[0]: el[1] for el in self.doc}
profile = Person()
profile.name = '%s %s' % (d['Nom'], d['Prénom'])
profile.birth_date = parse_french_date(d['Date de naissance']).date()
profile.address = '%s %s %s' % (d['Adresse de correspondance'], d['Code postal résidence fiscale'], d['Ville adresse de correspondance'])
profile.country = d['Pays adresse de correspondance']
profile.email = d['Adresse e-mail']
profile.phone = d.get('Téléphone portable')
profile.job_activity_area = d.get('Secteur d\'activité')
profile.job = d.get('Situation professionnelle')
profile.company_name = d.get('Employeur')
profile.family_situation = d.get('Situation familiale')
return profile
def parse(self):
for i, tr in enumerate(self.document.xpath('//tr')):
t = FrenchTransaction(tr.xpath('./td[@class="transactionId"]/span')[0].text.strip())
date = parse_french_date(tr.xpath('./td[@class="date"]')[0].text.strip())
status = tr.xpath('./td[@class="desc"]/ul/li[@class="first"]')[0].text.strip()
#We pass this because it's not transaction
if status == u'Créé' or status == u'Annulé':
continue
raw = tr.xpath('./td[@class="desc"]/strong')[0].text.strip()
t.parse(date=date, raw=raw)
amount = tr.xpath('./td[@class="price"]/span')[0].text.strip()
t.set_amount(amount)
t._currency = Account.get_currency(amount)
yield t
def get_history(self, date_guesser, state=None):
seen = set()
lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr')
debit_date = None
for i, line in enumerate(lines):
is_balance = line.xpath('./td/@class="cel-texte cel-neg"')
# It is possible to have three or four columns.
cols = [self.parser.tocleanstring(td) for td in line.xpath('./td')]
date = cols[0]
label = cols[1]
amount = cols[-1]
t = Transaction()
t.set_amount(amount)
t.label = t.raw = label
if is_balance:
m = re.search('(\d+ [^ ]+ \d+)', label)
if not m:
raise BrokenPageError(
'Unable to read card balance in history: %r' % label)
if state is None:
debit_date = parse_french_date(m.group(1))
else:
debit_date = state
# Skip the first line because it is balance
if i == 0:
continue
t.date = t.rdate = debit_date
# Consider the second one as a positive amount to reset balance to 0.
t.amount = -t.amount
state = t.date
else:
day, month = map(int, date.split('/', 1))
t.rdate = date_guesser.guess_date(day, month)
t.date = debit_date
t.type = t.TYPE_CARD
try:
t.id = t.unique_id(seen)
except UnicodeEncodeError:
self.logger.debug(t)
self.logger.debug(t.label)
raise
yield state, t
def parse_transaction(self, transaction, account):
page = None
if 'id' not in transaction or not transaction['date']:
return []
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
if not 'txnCurrency' in transaction['amounts']:
return []
original_currency = unicode(transaction['amounts']['txnCurrency'])
if original_currency in self.browser.account_currencies:
return []
if 'conversionFrom' in transaction[
'amounts'] and 'value' in transaction['amounts'][
'conversionFrom'] and account.currency == transaction[
'amounts']['conversionFrom']['currency']:
cc = self.format_amount(
transaction['amounts']['conversionFrom']['value'],
transaction['isCredit'])
else:
try:
page = self.return_detail_page(transaction['detailsLink'])
cc = page.get_converted_amount() if isinstance(
page, HistoryDetailsPage) else None
except ServerError:
self.logger.warning(
'Unable to go on detail, transaction skipped.')
return []
if not cc:
return []
t.original_amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction["isCredit"])
t.original_currency = original_currency
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' +
transaction['date']['year']).date()
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
if page is None and t.amount < 0:
page = self.return_detail_page(transaction['detailsLink'])
funding_src = page.get_funding_src(t) if isinstance(
page, HistoryDetailsPage) else None
return [t] if funding_src is None else ([t] + [funding_src])
def get_history(self, date_guesser, state=None):
seen = set()
lines = self.document.xpath('(//table[@class="ca-table"])[2]/tr')
debit_date = None
for i, line in enumerate(lines):
is_balance = line.xpath('./td/@class="cel-texte cel-neg"')
# It is possible to have three or four columns.
cols = [self.parser.tocleanstring(td) for td in line.xpath('./td')]
date = cols[0]
label = cols[1]
amount = cols[-1]
t = Transaction()
t.set_amount(amount)
t.label = t.raw = label
if is_balance:
m = re.search('(\d+ [^ ]+ \d+)', label)
if not m:
raise BrokenPageError('Unable to read card balance in history: %r' % label)
if state is None:
debit_date = parse_french_date(m.group(1))
else:
debit_date = state
# Skip the first line because it is balance
if i == 0:
continue
t.date = t.rdate = debit_date
# Consider the second one as a positive amount to reset balance to 0.
t.amount = -t.amount
state = t.date
else:
day, month = map(int, date.split('/', 1))
t.rdate = date_guesser.guess_date(day, month)
t.date = debit_date
t.type = t.TYPE_CARD
try:
t.id = t.unique_id(seen)
except UnicodeEncodeError:
self.logger.debug(t)
self.logger.debug(t.label)
raise
yield state, t
def parse_transaction(self, transaction, account):
trans = []
if 'transactionStatus' in transaction and transaction[
'transactionStatus'] in [
u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif',
u'Payé', u'En attente', u'Rejeté', u'Expiré', u'Created',
u'Brouillon', u'Paid', u'Pending', u'Canceled'
]:
return []
for pattern in [u'Commande à', u'Offre de remboursement', u'Bill to']:
if 'description' not in transaction[
'transactionDescription'] or transaction[
'transactionDescription']['description'].startswith(
pattern):
return []
t = FrenchTransaction(transaction['transactionId'])
# Those are not really transactions.
if not 'currency' in transaction['grossAmount'] \
or transaction['transactionDescription']['description'].startswith("Conversion de devise"):
return []
original_currency = unicode(transaction['grossAmount']['currency'])
if not original_currency == account.currency:
if original_currency in self.browser.account_currencies:
return []
cc = [tr['grossAmount']['amountUnformatted'] for tr in transaction['secondaryTransactions'] \
if account.currency == tr['grossAmount']['currency'] \
and (tr['grossAmount']['amountUnformatted'] < 0) == (transaction['grossAmount']['amountUnformatted'] < 0) \
and tr['transactionDescription']['description'].startswith('Conversion de devise')]
if not cc:
return []
assert len(cc) == 1
t.original_amount = Decimal(
str(transaction['grossAmount']['amountUnformatted']))
t.original_currency = original_currency
t.amount = Decimal(str(cc[0]))
else:
t.amount = Decimal(
str(transaction['netAmount']['amountUnformatted']))
date = parse_french_date(transaction['transactionTime'])
raw = "%s %s" % (transaction['transactionDescription']['description'],
transaction['transactionDescription']['name'])
if raw == "Transfert de Compte bancaire":
t.type = FrenchTransaction.TYPE_TRANSFER
t.commission = Decimal(
str(transaction['feeAmount']['amountUnformatted']))
t.parse(date=date, raw=raw)
trans.append(t)
return trans
def iter_history(self, coming):
for op in self.get('data.listerMouvements.listeMouvements') or []:
#We have not date for this statut so we just skit it
if op.get('statut') in self.IGNORED_STATUSES:
continue
tr = Transaction.from_dict({
'type': Transaction.TYPE_BANK,
'_state': op.get('statut'),
'amount': op.get('montantNet'),
})
tr.parse(date=parse_french_date(op.get('dateSaisie')),
vdate = parse_french_date(op.get('dateEffet')) if op.get('dateEffet') else None,
raw='%s %s' % (op.get('libelleMouvement'), op.get('canalSaisie') or ''))
tr._op = op
if not tr.amount:
if op.get('rib', {}).get('codeBanque') == 'null':
self.logger.info('ignoring non-transaction with label %r', tr.raw)
continue
if (op.get('statut') == u'Traité') ^ coming:
yield tr
def get_profile(self):
self.get_universes()
profile = Person()
content = self.location('/transactionnel/services/rest/User/user').json()['content']
profile.name = content['prenom'] + ' ' + content['nom']
profile.address = content['adresse'] + ' ' + content['codePostal'] + ' ' + content['ville']
profile.country = content['pays']
profile.birth_date = parse_french_date(content['dateNaissance']).date()
content = self.location('/transactionnel/services/applications/gestionEmail/getAdressesMails').json()['content']
profile.email = content['emailPart']
return profile
def parse_transaction(self, transaction, account):
trans = []
if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif', u'Payé', u'En attente', u'Rejeté', u'Expiré', \
u'Created', u'Canceled']:
return []
if transaction['transactionDescription'].startswith(
u'Offre de remboursement'
) or transaction['transactionDescription'].startswith(u'Commande à'):
return []
t = FrenchTransaction(transaction['transactionId'])
original_currency = unicode(
transaction['transactionAmount']['currencyCode'])
if not original_currency == account.currency:
if original_currency in self.browser.account_currencies:
return []
cc = self.browser.convert_amount(
account, transaction,
'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id='
+ transaction['transactionId'])
if not cc:
return []
t.original_amount = Decimal(
'%.2f' %
transaction['transactionAmount']['currencyDoubleValue'])
t.original_currency = original_currency
t.amount = abs(cc) if not transaction['debit'] else -abs(cc)
else:
t.amount = Decimal('%.2f' %
transaction['net']['currencyDoubleValue'])
date = parse_french_date(transaction['transactionTime'])
raw = transaction['transactionDescription']
if raw.startswith(u'Paiement \xe0') or raw.startswith('Achat de'):
payback_id, payback_raw, payback_amount, payback_currency = self.browser.check_for_payback(
transaction,
'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id='
+ transaction['transactionId'])
if payback_id and payback_raw and payback_amount and payback_currency:
t_payback = FrenchTransaction(payback_id)
t_payback.amount = payback_amount
t_payback.original_currency = payback_currency
t_payback.type = FrenchTransaction.TYPE_TRANSFER
t_payback.parse(date=date, raw=u'Prélèvement pour %s' % raw)
trans.append(t_payback)
t.commission = Decimal('%.2f' %
transaction['fee']['currencyDoubleValue'])
t.parse(date=date, raw=raw)
trans.append(t)
return trans
def parse_transaction(self, transaction, account):
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
cc = self.browser.convert_amount(account, transaction, transaction['detailsLink'])
if not cc:
return []
t.original_amount = self.format_amount(transaction['amounts']['net']['value'], transaction["isCredit"])
t.original_currency = u'' + transaction['amounts']['txnCurrency']
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(transaction['amounts']['net']['value'], transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' + transaction['date']['year'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
return [t]
def parse(self):
for i, tr in enumerate(self.document.xpath('//tr')):
t = FrenchTransaction(
tr.xpath('./td[@class="transactionId"]/span')[0].text.strip())
date = parse_french_date(
tr.xpath('./td[@class="date"]')[0].text.strip())
status = tr.xpath(
'./td[@class="desc"]/ul/li[@class="first"]')[0].text.strip()
#We pass this because it's not transaction
if status == u'Créé' or status == u'Annulé':
continue
raw = tr.xpath('./td[@class="desc"]/strong')[0].text.strip()
t.parse(date=date, raw=raw)
amount = tr.xpath('./td[@class="price"]/span')[0].text.strip()
t.set_amount(amount)
t._currency = Account.get_currency(amount)
yield t
def extract_concert(self, concert_table):
d = {}
date_h3 = concert_table.iter('h3').next()
d['date'] = parse_french_date(date_h3.text)
cancel_h2 = next(date_h3.itersiblings('h2'), None)
if cancel_h2 is not None and cancel_h2.text.startswith('ANNUL'):
d['active'] = False
else:
d['active'] = True
performers_table = concert_table.iterdescendants('table').next()
d['performers'] = list(self.extract_performers(performers_table))
d['summary'] = ' + '.join(p['name'] for p in d['performers'])
d['description'] = d['summary']
return d
def extract_concert(self, concert_table):
d = {}
date_h3 = next(concert_table.iter('h3'))
d['date'] = parse_french_date(date_h3.text)
cancel_h2 = next(date_h3.itersiblings('h2'), None)
if cancel_h2 is not None and cancel_h2.text.startswith('ANNUL'):
d['active'] = False
else:
d['active'] = True
performers_table = next(concert_table.iterdescendants('table'))
d['performers'] = list(self.extract_performers(performers_table))
d['summary'] = ' + '.join(p['name'] for p in d['performers'])
d['description'] = d['summary']
return d
def obj_date(self):
if Env('is_card', default=False)(self):
month = CleanText('//label[@for="movementSearch_period_%s"]' % ('1' if Env('is_previous', default=False)(self) else '0'),
replace=[(u'Débit ', '')])(self)
date_text = CleanText(u'//li[h3]/h4[@class="summary__title" and contains(text(), "Solde débité au")and contains(text(), "%s")]' % month,
replace=[(u'Solde débité au ', '')])(self)
if not date_text:
date = Date(Attr('.//time', 'datetime'))(self)
if self.page.browser.deferred_card_calendar is None:
self.page.browser.location(Link('//a[contains(text(), "calendrier")]')(self))
closest = self.page.browser.get_debit_date(date)
if closest:
return closest
return date
debit_date = parse_french_date(date_text)
return debit_date.date()
return Date(Attr('.//time', 'datetime'))(self)
def extract_concert(self, concert_table):
d = {}
date_h3 = concert_table.iter("h3").next()
d["date"] = parse_french_date(date_h3.text)
cancel_h2 = next(date_h3.itersiblings("h2"), None)
if cancel_h2 is not None and cancel_h2.text.startswith("ANNUL"):
d["active"] = False
else:
d["active"] = True
performers_table = concert_table.iterdescendants("table").next()
d["performers"] = list(self.extract_performers(performers_table))
d["summary"] = " + ".join(p["name"] for p in d["performers"])
d["description"] = d["summary"]
return d
def parse_transaction(self, transaction, account):
t = FrenchTransaction(transaction['activityId'])
date = parse_french_date(transaction['date'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
try:
if transaction['currencyCode'] != account.currency:
transaction = self.browser.convert_amount(account, transaction)
t.original_amount = self.format_amount(transaction['originalAmount'], transaction["isCredit"])
t.original_currency = u'' + transaction["currencyCode"]
t.amount = self.format_amount(transaction['netAmount'], transaction["isCredit"])
except KeyError:
return
t._currency = transaction['currencyCode']
return t
def parse_transaction(self, transaction, account):
if transaction['transactionStatus'] in [u'Créé', u'Annulé', u'Suspendu', u'Mis à jour', u'Actif']:
return
t = FrenchTransaction(transaction['transactionId'])
if not transaction['transactionAmount']['currencyCode'] == account.currency:
cc = self.browser.convert_amount(account, transaction, 'https://www.paypal.com/cgi-bin/webscr?cmd=_history-details-from-hub&id=' + transaction['transactionId'])
if not cc:
return
t.original_amount = Decimal(transaction['transactionAmount']['currencyDoubleValue'])
t.original_currency = u'' + transaction['transactionAmount']['currencyCode']
t.set_amount(cc)
else:
t.amount = Decimal(transaction['transactionAmount']['currencyDoubleValue'])
date = parse_french_date(transaction['transactionTime'])
raw = transaction['transactionDescription']
t.commission = Decimal(transaction['fee']['currencyDoubleValue'])
t.parse(date=date, raw=raw)
return t
def _get_messages(self, thread):
thread_div = self.document.find(True, 'PADpost_txt')
used_ids = set()
rcpt = self.document.find('input', attrs={'type': 'hidden', 'name': 'Dest'})['value']
sender_to_receiver = {rcpt: self.browser.username, self.browser.username: rcpt}
# site is sorted from latest to oldest message
for message_table in reversed(thread_div.findAll('table')):
for td in message_table.findAll('td'):
profile_a = td.find('a', href=re.compile(r'profil_read.php\?.*'))
if not profile_a:
continue
first_br = td.find('br')
assert first_br.nextSibling.name == 'br'
text_nodes = ovsparse.all_next_siblings(first_br.nextSibling.nextSibling) # TODO
#~ print text_nodes
# date will be used as id
sitedate = profile_a.findParent('div').find(text=re.compile(',.*')).replace(', ', '')
sysdate = parse_french_date(sitedate)
compactdate = datetime.datetime.strftime(sysdate, '%Y%m%dT%H%M%S')
# but make it unique
msg_id = ovsparse.create_unique_id(compactdate, used_ids)
used_ids.add(msg_id)
message = Message(thread, msg_id)
message.sender = re.search(r'\?(.+)', profile_a['href']).group(1)
message.receivers = [sender_to_receiver[message.sender]]
message.date = sysdate
message.content = ovsparse.html_message_to_text(text_nodes)
notread_self = bool(td.find('span', 'ColorSurligne'))
notread_other = bool(td.find('span', 'new_sortiepartenaire'))
if notread_other or notread_self:
message.flags |= Message.IS_NOT_RECEIVED
else:
message.flags |= Message.IS_RECEIVED
yield message
def parse_transaction(self, transaction):
t = FrenchTransaction(transaction['activityId'])
date = parse_french_date(transaction['date'])
try:
raw = transaction['counterparty']
except KeyError:
raw = transaction['displayType']
t.parse(date=date, raw=raw)
try:
amount = transaction['netAmount']
except KeyError:
return
if transaction['isCredit']:
t.set_amount(credit=amount)
else:
t.set_amount(debit=amount)
t._currency = transaction['currencyCode']
return t
def get_history(self):
index = 0
# Check if this is a multi-cards page
pages = []
for a in self.document.xpath('//table[@class="liste"]/tbody/tr/td/a'):
card_link = a.get('href')
history_url = 'https://%s/%s/fr/banque/%s' % (
self.browser.DOMAIN, self.browser.currentSubBank, card_link)
page = self.browser.get_document(self.browser.openurl(history_url))
pages.append(page)
if len(pages) == 0:
# If not, add this page as transactions list
pages.append(self.document)
for page in pages:
label = self.parser.tocleanstring(
self.parser.select(page.getroot(), 'div.lister p.c', 1))
label = re.findall('(\d+ [^ ]+ \d+)', label)[-1]
# use the trick of relativedelta to get the last day of month.
debit_date = parse_french_date(label) + relativedelta(day=31)
for tr in page.xpath('//table[@class="liste"]/tbody/tr'):
tds = tr.findall('td')[:4]
if len(tds) < 4:
continue
tr = Transaction(index)
parts = [
txt.strip() for txt in list(tds[-3].itertext()) +
list(tds[-2].itertext()) if len(txt.strip()) > 0
]
tr.parse(date=tds[0].text.strip(' \xa0'), raw=u' '.join(parts))
tr.date = debit_date
tr.type = tr.TYPE_CARD
# Don't take all of the content (with tocleanstring for example),
# because there is a span.aide.
tr.set_amount(tds[-1].text)
yield tr
def handle_response(self, account, recipient, amount, reason):
self.check_errors()
transfer_data = self.doc['data']['validationVirement']
self.abort_if_unknown(transfer_data)
if 'idBeneficiaire' in transfer_data and transfer_data[
'idBeneficiaire'] is not None:
assert transfer_data['idBeneficiaire'] == recipient._transfer_id
elif transfer_data.get('ibanCompteCrediteur'):
assert transfer_data['ibanCompteCrediteur'] == recipient.iban
transfer = Transfer()
transfer.currency = transfer_data['devise']
transfer.amount = Decimal(transfer_data['montantEuros'])
transfer.account_iban = transfer_data['ibanCompteDebiteur']
transfer.account_id = account.id
try:
transfer.recipient_iban = transfer_data[
'ibanCompteCrediteur'] or recipient.iban
except KeyError:
# In last version, json contains a key 'idBeneficiaire' containing:
# "idBeneficiaire" : "00003##00001####FR7610278123456789028070101",
transfer.recipient_id = transfer_data['idBeneficiaire']
transfer.recipient_iban = transfer.recipient_id.split(
'#')[-1] or recipient.iban
else:
transfer.recipient_id = recipient.id
transfer.exec_date = parse_french_date(
transfer_data['dateExecution']).date()
transfer.fees = Decimal(transfer_data.get('montantFrais', '0'))
transfer.label = transfer_data['motifVirement']
transfer.account_label = account.label
transfer.recipient_label = recipient.label
transfer.id = transfer_data['reference']
# This is true if a transfer with the same metadata has already been done recently
transfer._doublon = transfer_data['doublon']
transfer.account_balance = account.balance
return transfer
def parse(self):
for tr in self.document.xpath('//tbody/tr'):
tlink = tr.xpath('./td[@class="desc"]/a[@class="rowClick"]')[0].attrib['href'].strip()
t = FrenchTransaction(tlink[tlink.find('&id=')+4:])
date = parse_french_date(tr.xpath('./td[@class="date"]')[0].text.strip())
raw = tr.xpath('./td[@class="desc"]/a[@class="rowClick"]')[0].tail.strip()
# Filter lines that do not actually modify the balance
if raw.startswith('Autorisation ') or raw.endswith(' en attente par PayPal'):
continue
t.parse(date=date, raw=raw)
amount = tr.xpath('./td[@class="price-value net"]')[0].text.strip()
t.set_amount(amount)
commission = tr.xpath('./td[@class="price-value fee"]')[0].text.strip()
t.commission = Decimal(t.clean_amount(commission))
t.label = t.raw
if t.commission:
t.label += " (%s)" % tr.xpath('./td[@class="price-value gross"]')[0].text.strip()
t._currency = Account.get_currency(amount)
yield t
def parse_transaction(self, transaction, account):
if 'id' not in transaction or not transaction['date']:
return []
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
original_currency = unicode(transaction['amounts']['txnCurrency'])
if original_currency in self.browser.account_currencies:
return []
if 'conversionFrom' in transaction[
'amounts'] and 'value' in transaction['amounts'][
'conversionFrom'] and account.currency == transaction[
'amounts']['conversionFrom']['currency']:
cc = self.format_amount(
transaction['amounts']['conversionFrom']['value'],
transaction['isCredit'])
else:
try:
cc = self.browser.convert_amount(
account, transaction, transaction['detailsLink'])
except ServerError:
self.logger.warning(
'Unable to go on detail, transaction skipped.')
return []
if not cc:
return []
t.original_amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction["isCredit"])
t.original_currency = original_currency
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' +
transaction['date']['year'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
return [t]
def parse_datetime(self, text):
m = re.match('(\d+)/(\d+)/(\d+) (\d+):(\d+) (\w+)', text)
if m:
date = datetime.datetime(int(m.group(3)), int(m.group(1)),
int(m.group(2)), int(m.group(4)),
int(m.group(5)))
if m.group(6) == 'pm':
date += datetime.timedelta(0, 12 * 3600)
return date
m = re.match('(\d+)-(\d+)-(\d+) (\d+):(\d+)', text)
if m:
return datetime.datetime(int(m.group(1)), int(m.group(2)),
int(m.group(3)), int(m.group(4)),
int(m.group(5)))
m = re.match('(\d+) (\w+) (\d+) (\d+):(\d+)', text)
if m:
return parse_french_date(text)
self.logger.warning('Unable to parse "%s"' % text)
return text
def parse_transaction(self, transaction, account):
t = FrenchTransaction(transaction['id'])
if not transaction['isPrimaryCurrency']:
cc = self.browser.convert_amount(account, transaction,
transaction['detailsLink'])
if not cc:
return []
t.original_amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction["isCredit"])
t.original_currency = u'' + transaction['amounts']['txnCurrency']
t.amount = self.format_amount(cc, transaction['isCredit'])
else:
t.amount = self.format_amount(
transaction['amounts']['net']['value'],
transaction['isCredit'])
date = parse_french_date(transaction['date']['formattedDate'] + ' ' +
transaction['date']['year'])
raw = transaction.get('counterparty', transaction['displayType'])
t.parse(date=date, raw=raw)
return [t]
def text2issue(self, issue, m):
# XXX HACK to support real incoming emails
if 'Subject' in m:
m['Title'] = m['Subject']
for key, (list_name, is_list_object) in self.ISSUE_FIELDS:
value = m.get(key)
if value is None:
continue
new_value = u''
for part in decode_header(value):
if part[1]:
new_value += unicode(part[0], part[1])
else:
new_value += part[0].decode('utf-8')
value = new_value
if is_list_object:
objects_list = getattr(issue.project, list_name)
value = self.get_list_item(objects_list, value)
# FIXME: autodetect
if key in ['start', 'due']:
if len(value) > 0:
#value = datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
value = parse_french_date(value)
else:
value = None
setattr(issue, key, value)
for key in issue.fields.keys():
value = m.get(self.sanitize_key(key))
if value is not None:
issue.fields[key] = value.decode('utf-8')
content = u''
for part in m.walk():
if part.get_content_type() == 'text/plain':
s = part.get_payload(decode=True)
charsets = part.get_charsets() + m.get_charsets()
for charset in charsets:
try:
if charset is not None:
content += unicode(s, charset)
else:
content += unicode(s, encoding='utf-8')
except UnicodeError as e:
self.logger.warning('Unicode error: %s' % e)
continue
except Exception as e:
self.logger.exception(e)
continue
else:
break
issue.body = content
m = re.search('([^< ]+@[^ >]+)', m['From'] or '')
if m:
return m.group(1)
