def iter_bills(self, sub): table = self.document.xpath('//table[@id="tableauDecompte"]')[0].xpath( './/tr') for tr in table: list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue date_str = list_tds[0].text month_str = date_str.split()[0] date = datetime.strptime( re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() amount = list_tds[1].text if amount is None: continue amount = re.sub(' euros', '', amount) bil = Bill() bil.id = sub._id + "." + date.strftime("%Y%m") bil.date = date bil.label = u'' + amount.strip() bil.format = u'pdf' filedate = date.strftime("%m%Y") bil._url = '/PortailAS/PDFServletReleveMensuel.dopdf' bil._args = {'PDF.moisRecherche': filedate} yield bil
def iter_documents(self, sub): try: table = self.doc.xpath('//table[@id="relevesMensuels"]')[0].xpath( './/tr') # When no operations was done in the last month, there is no table. That is fine. except IndexError: return for tr in table: list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue date_str = list_tds[0].text month_str = date_str.split()[0] date = datetime.strptime( re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() amount = list_tds[1].text if amount is None: continue amount = re.sub('[^\d,-]+', '', amount) bil = Bill() bil.id = sub._id + "." + date.strftime("%Y%m") bil.date = date bil.price = Decimal('-' + amount.strip().replace(',', '.')) bil.format = u'pdf' bil.type = u'bill' bil.label = date.strftime("%Y%m%d") bil._url = '/PortailAS/PDFServletReleveMensuel.dopdf?PDF.moisRecherche=' + date.strftime( "%m%Y") yield bil
def iter_bills(self, sub): #pdb.set_trace() years = [None] + self.document.xpath('//ul[@class="years"]/li/a') for year in years: #pdb.set_trace() if year is not None and year.attrib['href']: self.browser.location(year.attrib['href']) tables = self.browser.page.document.xpath('//table[contains(@summary, "factures")]') for table in tables: for tr in table.xpath('.//tr'): list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue url = re.sub('[\r\n\t]', '', list_tds[0].xpath('.//a')[0].attrib['href']) date_search = re.search('dateFactureQE=(\d+/\d+/\d+)', url) if not date_search: continue date = datetime.strptime(date_search.group(1), "%d/%m/%Y").date() amount = self.parser.tocleanstring(list_tds[2]) if amount is None: continue bill = Bill() bill.id = sub._id + "." + date.strftime("%Y%m%d") bill.price = Decimal(FrenchTransaction.clean_amount(amount)) bill.currency = bill.get_currency(amount) bill.date = date bill.label = self.parser.tocleanstring(list_tds[0]) bill.format = u'pdf' bill._url = url yield bill
def iter_bills(self, sub): try: table = self.doc.xpath('//table[@id="relevesMensuels"]')[0].xpath('.//tr') # When no operations was done in the last month, there is no table. That is fine. except IndexError: return for tr in table: list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue date_str = list_tds[0].text month_str = date_str.split()[0] date = datetime.strptime(re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() amount = list_tds[1].text if amount is None: continue amount = re.sub('[^\d,-]+', '', amount) bil = Bill() bil.id = sub._id + "." + date.strftime("%Y%m") bil.date = date bil.price = Decimal('-'+amount.strip().replace(',','.')) bil.format = u'pdf' bil.label = date.strftime("%Y%m%d") bil._url = '/PortailAS/PDFServletReleveMensuel.dopdf?PDF.moisRecherche='+date.strftime("%m%Y") yield bil
def iter_documents(self): table = self.doc.xpath('//table[@id="releveCompteMensuel"]')[0].xpath( './/tr') for tr in table: list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue date_str = tr.xpath('.//td[@class="cAlignGauche"]')[0].text month_str = date_str.split()[0] date = datetime.strptime( re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() amount = tr.xpath('.//td[@class="cAlignDroite"]')[0].text amount = re.sub('[^\d,-]+', '', amount) for format in ('CSV', 'PDF'): bil = Bill() bil.id = date.strftime("%Y%m") + format bil.date = date clean_amount = amount.strip().replace(',', '.') if clean_amount != '': bil.price = Decimal('-' + clean_amount) else: bil.price = 0 bil.label = u'' + date.strftime("%Y%m%d") bil.format = u'' + format bil.type = DocumentTypes.BILL filedate = date.strftime("%m%Y") bil.url = u'/PortailPS/fichier.do' bil._data = { 'FICHIER.type': format.lower() + '.releveCompteMensuel', 'dateReleve': filedate, 'FICHIER.titre': 'Releve' + filedate } yield bil
def iter_bills(self): table = self.doc.xpath('//table[@id="releveCompteMensuel"]')[0].xpath('.//tr') for tr in table: list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue date_str = tr.xpath('.//td[@class="cAlignGauche"]')[0].text month_str = date_str.split()[0] date = datetime.strptime(re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() amount = tr.xpath('.//td[@class="cAlignDroite"]')[0].text amount = re.sub('[^\d,-]+', '', amount) for format in ('CSV', 'PDF'): bil = Bill() bil.id = date.strftime("%Y%m") + format bil.date = date clean_amount = amount.strip().replace(',','.') if clean_amount != '': bil.price = Decimal('-'+clean_amount) else: bil.price = 0 bil.label = u''+date.strftime("%Y%m%d") bil.format = u''+format filedate = date.strftime("%m%Y") bil._url = '/PortailPS/fichier.do' bil._data = {'FICHIER.type': format.lower()+'.releveCompteMensuel', 'dateReleve': filedate, 'FICHIER.titre': 'Releve' + filedate } yield bil
def iter_bills(self): table = self.document.xpath( '//table[@id="releveCompteMensuel"]')[0].xpath('.//tr') for tr in table: list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue date_str = tr.xpath('.//td[@class="cAlignGauche"]')[0].text month_str = date_str.split()[0] date = datetime.strptime( re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() amount = tr.xpath('.//td[@class="cAlignDroite"]')[0].text for format in ('CSV', 'PDF'): bil = Bill() bil.id = date.strftime("%Y%m") + format bil.date = date bil.label = u'' + amount.strip() bil.format = u'' + format filedate = date.strftime("%m%Y") bil._url = '/PortailPS/fichier.do' bil._args = { 'FICHIER.type': format.lower() + '.releveCompteMensuel', 'dateReleve': filedate, 'FICHIER.titre': '', } yield bil
def _create_bill(self, date, price, link): bill = Bill() bill.id = date.__str__().replace('-', '') bill.date = date bill._price = price bill._url = link bill.format = u'pdf' bill.label = unicode(price) return bill
def on_loaded(self): self.details = {} self.datebills = {} for div in self.document.xpath('//div[@class="infosLigne pointer"]'): phonenumber = div.text phonenumber = phonenumber.split("-")[-1].strip() virtualnumber = div.attrib['onclick'].split('(')[1][1] self.details['num' + str(phonenumber)] = virtualnumber for div in self.document.xpath('//div[@class="infosConso"]'): num = div.attrib['id'].split('_')[1][0] self.details[num] = [] # National parsing divnat = div.xpath('div[@class="national"]')[0] self.parse_div(divnat, "National : %s | International : %s", num, False) # International parsing divint = div.xpath('div[@class="international hide"]')[0] if divint.xpath('div[@class="detail"]'): self.parse_div(divint, u"Appels émis : %s | Appels reçus : %s", num, True) for divbills in self.document.xpath('//div[@id="factContainer"]'): for divbill in divbills.xpath('.//div[@class="factLigne hide "]'): alink = divbill.xpath('.//div[@class="pdf"]/a')[0] localid = re.search('&l=(?P<id>\d*)&id', alink.attrib.get('href')).group('id') mydate_str = re.search('&date=(?P<date>\d*)$', alink.attrib.get('href')).group('date') mydate = datetime.strptime(mydate_str, "%Y%m%d").date() bill = Bill() bill.label = unicode(mydate_str) bill.id = unicode(mydate_str) bill.date = mydate bill.format = u"pdf" bill._url = alink.attrib.get('href') if "pdfrecap" in alink.attrib.get('href'): bill.id = "recap-" + bill.id if localid not in self.datebills: self.datebills[localid] = [] self.datebills[localid].append(bill)
def iter_bills(self, subscription): orders = self.iter_orders() for o in orders: b = Bill() b._url = o._bill['url'] b.id = '%s.%s' % (subscription.id, o.id) b.date = o.date b.price = o.total b.format = o._bill['format'] b.currency = b.get_currency(self.get_currency()) b.vat = o.tax yield b
def date_bills(self, parentid): max = 1 while len(self.document.xpath('//li[@id="liMois%s"]' % max)) > 0: li = self.document.xpath('//li[@id="liMois%s"]' % max)[0] max += 1 link = li.xpath('a')[0] bill = Bill() bill._url = link.attrib['href'] bill.label = unicode(link.text) bill.format = u"pdf" bill.id = parentid + bill.label.replace(' ', '') yield bill
def date_bills(self, parentid): max = 1 while len(self.document.xpath('//li[@id="liMois%s"]' % max)) > 0: li = self.document.xpath('//li[@id="liMois%s"]' % max)[0] max += 1 link = li.xpath('a')[0] bill = Bill() bill._url = link.attrib['href'] bill.label = link.text bill.format = u"pdf" bill.id = parentid + bill.label.replace(' ', '') yield bill
def iter_documents(self, subscription): orders = self.iter_orders() for o in orders: b = Bill() b.url = unicode(o._bill['url']) b.id = '%s.%s' % (subscription.id, o.id) b.date = o.date b.price = o.total b.format = o._bill['format'] b.type = u'bill' b.currency = b.get_currency(self.get_currency()) b.label = '%s %s' % (subscription.label, o.date) b.vat = o.tax yield b
def iter_documents(self, subscription): docs, docs_len, check, month_back, date = list(), -1, 0, 6, None # First request is known bills = self.request('pnrs') while check < month_back: # If not first if docs_len > -1 and date: if check > 0: # If nothing, we try 4 weeks back date = (datetime.strptime(date, '%Y-%m-%d') - relativedelta(weeks=4)).strftime('%Y-%m-%d') else: # Add 8 weeks to last date to be sure to get all date = (datetime.combine(date, datetime.min.time()) + relativedelta(weeks=8)).strftime('%Y-%m-%d') bills = self.request('pnrs?date=%s' % date) docs_len = len(docs) for proof, pnr, trip in zip(bills['proofs'], bills['pnrs'], bills['trips']): # Check if not already in docs list for doc in docs: if vars(doc)['id'].split('_', 1)[1] == pnr['id']: break else: b = Bill() b.id = '%s_%s' % (subscription.id, pnr['id']) b._url = proof['url'] b.date = Date().filter(proof['created_at']) b.format = u"pdf" b.label = u'Trajet du %s' % Date().filter( trip['departure_date']) b.type = DocumentTypes.BILL b.vat = CleanDecimal().filter('0') if pnr['cents']: b.price = CleanDecimal().filter( format(pnr['cents'] / float(100), '.2f')) b.currency = pnr['currency'] docs.append(b) check += 1 # If a new bill is found, we reset check if docs_len < len(docs): date = b.date check = 0 return iter(docs)
def iter_bills(self, subscriptionid): ul = self.document.xpath('//ul[@id="statements_form:statementsel"]') lis = ul[0].xpath('li') lis.pop(0) # Select alls for li in lis: acheck = li.xpath('a')[0] adirect = li.xpath('a')[1] label = unicode(acheck.text_content()) id = subscriptionid + '-' + label.replace(' ', '-') bill = Bill() bill.id = id bill.label = label bill.format = u"pdf" onmouse = adirect.attrib['onmouseover'] bill._localid = onmouse.split("'")[5] bill._url = adirect.attrib['href'] yield bill
def iter_documents(self, sub): elts = self.doc.xpath('//li[@class="rowdate"]') for elt in elts: try: elt.xpath('.//a[contains(@id,"lienPDFReleve")]')[0] except IndexError: continue date_str = elt.xpath('.//span[contains(@id,"moisEnCours")]')[0].text month_str = date_str.split()[0] date = datetime.strptime(re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() bil = Bill() bil.id = sub._id + "." + date.strftime("%Y%m") bil.date = date bil.format = u'pdf' bil.type = u'bill' bil.label = u'' + date.strftime("%Y%m%d") bil.url = urljoin(self.url, '/PortailAS/PDFServletReleveMensuel.dopdf?PDF.moisRecherche=%s' % date.strftime("%m%Y")) yield bil
def get_documents(self): documents = [] for document in self.doc: doc = Bill() doc.id = document['numFactureLabel'] doc.date = date.fromtimestamp(int(document['dateEmission'] / 1000)) doc.format = 'PDF' doc.label = 'Facture %s' % document['numFactureLabel'] doc.type = DocumentTypes.BILL doc.price = CleanDecimal().filter(document['montantTTC']) doc.currency = '€' doc._account_billing = document['compteFacturation'] doc._bill_number = document['numFacture'] documents.append(doc) return documents
def get_documents(self): documents = [] for document in self.doc: doc = Bill() doc.id = document['numFactureLabel'] doc.date = date.fromtimestamp(int(document['dateEmission'] / 1000)) doc.format = u'PDF' doc.label = 'Facture %s' % document['numFactureLabel'] doc.type = u'bill' doc.price = CleanDecimal().filter(document['montantTTC']) doc.currency = u'€' doc._account_billing = document['compteFacturation'] doc._bill_number = document['numFacture'] documents.append(doc) return documents
def iter_documents(self, subscription): docs, docs_len, check, month_back, date = list(), -1, 0, 6, None # First request is known bills = self.request('pnrs') while check < month_back: # If not first if docs_len > -1 and date: if check > 0: # If nothing, we try 4 weeks back date = (datetime.strptime(date, '%Y-%m-%d') - relativedelta(weeks=4)).strftime('%Y-%m-%d') else: # Add 8 weeks to last date to be sure to get all date = (datetime.combine(date, datetime.min.time()) + relativedelta(weeks=8)).strftime('%Y-%m-%d') bills = self.request('pnrs?date=%s' % date) docs_len = len(docs) for proof, pnr, trip in zip(bills['proofs'], bills['pnrs'], bills['trips']): # Check if not already in docs list for doc in docs: if vars(doc)['id'].split('_', 1)[1] == pnr['id']: break else: b = Bill() b.id = '%s_%s' % (subscription.id, pnr['id']) b._url = proof['url'] b.date = Date().filter(proof['created_at']) b.format = u"pdf" b.label = u'Trajet du %s' % Date().filter(trip['departure_date']) b.type = u"bill" b.vat = CleanDecimal().filter('0') if pnr['cents']: b.price = CleanDecimal().filter(format(pnr['cents']/float(100), '.2f')) b.currency = pnr['currency'] docs.append(b) check += 1 # If a new bill is found, we reset check if docs_len < len(docs): date = b.date check = 0 return iter(docs)
def on_loaded(self): self.details = {} self.datebills = [] for div in self.document.xpath('//div[@class="infosLigne pointer"]'): phonenumber = div.text phonenumber = phonenumber.split("-")[-1].strip() virtualnumber = div.attrib['onclick'].split('(')[1][1] self.details['num' + str(phonenumber)] = virtualnumber for div in self.document.xpath('//div[@class="infosConso"]'): num = div.attrib['id'].split('_')[1][0] self.details[num] = [] # National parsing divnat = div.xpath('div[@class="national"]')[0] self.parse_div(divnat, "National : %s | International : %s", num, False) # International parsing divint = div.xpath('div[@class="international hide"]')[0] self.parse_div(divint, u"Appels émis : %s | Appels reçus : %s", num, True) divbill = self.document.xpath('//div[@class="facture"]')[0] for trbill in divbill.xpath('table/tr'): mydate = unicode(trbill.find('td').text.split(":")[1].strip()) bill = Bill() bill.label = unicode(mydate) billid = mydate.replace('-', '') billid = billid[4:8] + billid[2:4] + billid[0:2] bill.id = phonenumber + "." + billid bill.date = date(*reversed([int(x) for x in mydate.split("-")])) alink = trbill.find('td/a') if alink.attrib.get("class") == "linkModal tips": bill.format = u'html' bill._url = alink.attrib.get('data-link') else: bill.format = u"pdf" bill._url = alink.attrib.get('href') self.datebills.append(bill)
def iter_documents(self, sub): elts = self.doc.xpath('//li[@class="rowdate"]') for elt in elts: try: elt.xpath('.//a[contains(@id,"lienPDFReleve")]')[0] except IndexError: continue date_str = elt.xpath( './/span[contains(@id,"moisEnCours")]')[0].text month_str = date_str.split()[0] date = datetime.strptime( re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() bil = Bill() bil.id = sub._id + "." + date.strftime("%Y%m") bil.date = date bil.format = 'pdf' bil.type = DocumentTypes.BILL bil.label = date.strftime("%Y%m%d") bil.url = '/PortailAS/PDFServletReleveMensuel.dopdf?PDF.moisRecherche=' + date.strftime( "%m%Y") yield bil
def iter_bills(self, sub): table = self.document.xpath('//table[@id="tableauDecompte"]')[0].xpath('.//tr') for tr in table: list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue date_str = list_tds[0].text month_str = date_str.split()[0] date = datetime.strptime(re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() amount = list_tds[1].text if amount is None: continue amount = re.sub(' euros', '', amount) bil = Bill() bil.id = sub._id + "." + date.strftime("%Y%m") bil.date = date bil.label = u''+amount.strip() bil.format = u'pdf' filedate = date.strftime("%m%Y") bil._url = '/PortailAS/PDFServletReleveMensuel.dopdf' bil._args = {'PDF.moisRecherche': filedate} yield bil
def iter_bills(self): table = self.document.xpath('//table[@id="releveCompteMensuel"]')[0].xpath('.//tr') for tr in table: list_tds = tr.xpath('.//td') if len(list_tds) == 0: continue date_str = tr.xpath('.//td[@class="cAlignGauche"]')[0].text month_str = date_str.split()[0] date = datetime.strptime(re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() amount = tr.xpath('.//td[@class="cAlignDroite"]')[0].text for format in ('CSV', 'PDF'): bil = Bill() bil.id = date.strftime("%Y%m") + format bil.date = date bil.label = u''+amount.strip() bil.format = u''+format filedate = date.strftime("%m%Y") bil._url = '/PortailPS/fichier.do' bil._args = {'FICHIER.type': format.lower() + '.releveCompteMensuel', 'dateReleve': filedate, 'FICHIER.titre': '', } yield bil