def run(self): sess = None try: sess = Session() batch = Batch.get_by_id(sess, self.batch_id) bill_types = keydefaultdict( lambda k: BillType.get_by_code(sess, k)) tprs = keydefaultdict(lambda k: None if k is None else Tpr.get_by_code(sess, k)) read_types = keydefaultdict( lambda k: ReadType.get_by_code(sess, k)) for bf in (sess.query(BatchFile).filter( BatchFile.batch == batch).order_by( BatchFile.upload_timestamp)): self.parser = _process_batch_file(sess, bf, self._log) for self.bill_num, raw_bill in enumerate( self.parser.make_raw_bills()): if "error" in raw_bill: self.failed_bills.append(raw_bill) else: try: mpan_core = raw_bill["mpan_core"] supply = Supply.get_by_mpan_core(sess, mpan_core) with sess.begin_nested(): bill = batch.insert_bill( sess, raw_bill["account"], raw_bill["reference"], raw_bill["issue_date"], raw_bill["start_date"], raw_bill["finish_date"], raw_bill["kwh"], raw_bill["net"], raw_bill["vat"], raw_bill["gross"], bill_types[raw_bill["bill_type_code"]], raw_bill["breakdown"], supply, ) for raw_read in raw_bill["reads"]: bill.insert_read( sess, tprs[raw_read["tpr_code"]], raw_read["coefficient"], raw_read["units"], raw_read["msn"], raw_read["mpan"], raw_read["prev_date"], raw_read["prev_value"], read_types[raw_read["prev_type_code"]], raw_read["pres_date"], raw_read["pres_value"], read_types[raw_read["pres_type_code"]], ) self.successful_bills.append(raw_bill) except KeyError as e: err = raw_bill.get("error", "") raw_bill["error"] = err + " " + str(e) self.failed_bills.append(raw_bill) except BadRequest as e: raw_bill["error"] = str(e.description) self.failed_bills.append(raw_bill) if len(self.failed_bills) == 0: sess.commit() self._log( "All the bills have been successfully loaded and attached " "to the batch.") else: sess.rollback() self._log(f"The import has finished, but there were " f"{len(self.failed_bills)} failures, and so the " f"whole import has been rolled back.") except BadRequest as e: sess.rollback() self._log(f"Problem: {e.description}") except BaseException: sess.rollback() self._log(f"I've encountered a problem: {traceback.format_exc()}") finally: if sess is not None: sess.close()
def run(self): sess = None try: sess = Session() self._log("Starting to parse the file with '" + self.parser_name + "'.") bill_types = keydefaultdict( lambda k: BillType.get_by_code(sess, k)) tprs = keydefaultdict(lambda k: None if k is None else Tpr.get_by_code(sess, k)) read_types = keydefaultdict( lambda k: ReadType.get_by_code(sess, k)) batch = Batch.get_by_id(sess, self.batch_id) contract = batch.contract raw_bills = self.parser.make_raw_bills() self._log("Successfully parsed the file, and now I'm starting to " "insert the raw bills.") for self.bill_num, raw_bill in enumerate(raw_bills): try: account = raw_bill['account'] supply = sess.query(Supply).join(Era).filter( or_( and_(Era.imp_supplier_contract == contract, Era.imp_supplier_account == account), and_(Era.exp_supplier_contract == contract, Era.exp_supplier_account == account), and_(Era.mop_contract == contract, Era.mop_account == account), and_(Era.dc_contract == contract, Era.dc_account == account))).distinct().order_by( Supply.id).first() if supply is None: raise BadRequest("Can't find an era with contract '" + contract.name + "' and account '" + account + "'.") with sess.begin_nested(): bill = batch.insert_bill( sess, account, raw_bill['reference'], raw_bill['issue_date'], raw_bill['start_date'], raw_bill['finish_date'], raw_bill['kwh'], raw_bill['net'], raw_bill['vat'], raw_bill['gross'], bill_types[raw_bill['bill_type_code']], raw_bill['breakdown'], supply) for raw_read in raw_bill['reads']: bill.insert_read( sess, tprs[raw_read['tpr_code']], raw_read['coefficient'], raw_read['units'], raw_read['msn'], raw_read['mpan'], raw_read['prev_date'], raw_read['prev_value'], read_types[raw_read['prev_type_code']], raw_read['pres_date'], raw_read['pres_value'], read_types[raw_read['pres_type_code']]) self.successful_bills.append(raw_bill) except BadRequest as e: raw_bill['error'] = str(e.description) self.failed_bills.append(raw_bill) if len(self.failed_bills) == 0: sess.commit() self._log( "All the bills have been successfully loaded and attached " "to the batch.") else: sess.rollback() self._log("The import has finished, but there were " + str(len(self.failed_bills)) + " failures, and so the " "whole import has been rolled back.") except BadRequest as e: sess.rollback() self._log("Problem: " + e.description) except BaseException: sess.rollback() self._log("I've encountered a problem: " + traceback.format_exc()) finally: if sess is not None: sess.close()
def run(self): sess = None try: sess = Session() self._log( "Starting to parse the file with '" + self.parser_name + "'.") set_read_write(sess) batch = Batch.get_by_id(sess, self.batch_id) raw_bills = self.parser.make_raw_bills() self._log( "Successfully parsed the file, and now I'm starting to " "insert the raw bills.") for self.bill_num, raw_bill in enumerate(raw_bills): try: with sess.begin_nested(): sess.execute( "set transaction isolation level serializable " "read write") bill_type = BillType.get_by_code( sess, raw_bill['bill_type_code']) bill = batch.insert_bill( sess, raw_bill['account'], raw_bill['reference'], raw_bill['issue_date'], raw_bill['start_date'], raw_bill['finish_date'], raw_bill['kwh'], raw_bill['net'], raw_bill['vat'], raw_bill['gross'], bill_type, raw_bill['breakdown']) sess.flush() for raw_read in raw_bill['reads']: tpr_code = raw_read['tpr_code'] if tpr_code is None: tpr = None else: tpr = Tpr.get_by_code(sess, tpr_code) prev_type = ReadType.get_by_code( sess, raw_read['prev_type_code']) pres_type = ReadType.get_by_code( sess, raw_read['pres_type_code']) bill.insert_read( sess, tpr, raw_read['coefficient'], raw_read['units'], raw_read['msn'], raw_read['mpan'], raw_read['prev_date'], raw_read['prev_value'], prev_type, raw_read['pres_date'], raw_read['pres_value'], pres_type) self.successful_bills.append(raw_bill) except BadRequest as e: raw_bill['error'] = str(e.description) self.failed_bills.append(raw_bill) if len(self.failed_bills) == 0: sess.commit() self._log( "All the bills have been successfully loaded and attached " "to the batch.") else: sess.rollback() self._log( "The import has finished, but there were " + str(len(self.failed_bills)) + " failures, and so the " "whole import has been rolled back.") except: sess.rollback() self._log("I've encountered a problem: " + traceback.format_exc()) finally: if sess is not None: sess.close()
def run(self): sess = None try: sess = Session() self._log( "Starting to parse the file with '" + self.parser_name + "'.") bill_types = keydefaultdict( lambda k: BillType.get_by_code(sess, k)) tprs = keydefaultdict( lambda k: None if k is None else Tpr.get_by_code(sess, k)) read_types = keydefaultdict( lambda k: ReadType.get_by_code(sess, k)) batch = Batch.get_by_id(sess, self.batch_id) contract = batch.contract raw_bills = self.parser.make_raw_bills() self._log( "Successfully parsed the file, and now I'm starting to " "insert the raw bills.") for self.bill_num, raw_bill in enumerate(raw_bills): try: account = raw_bill['account'] supply = sess.query(Supply).join(Era).filter( or_( and_( Era.imp_supplier_contract == contract, Era.imp_supplier_account == account), and_( Era.exp_supplier_contract == contract, Era.exp_supplier_account == account), and_( Era.mop_contract == contract, Era.mop_account == account), and_( Era.hhdc_contract == contract, Era.hhdc_account == account)) ).distinct().order_by(Supply.id).first() if supply is None: raise BadRequest( "Can't find an era with contract '" + contract.name + "' and account '" + account + "'.") with sess.begin_nested(): bill = batch.insert_bill( sess, account, raw_bill['reference'], raw_bill['issue_date'], raw_bill['start_date'], raw_bill['finish_date'], raw_bill['kwh'], raw_bill['net'], raw_bill['vat'], raw_bill['gross'], bill_types[raw_bill['bill_type_code']], raw_bill['breakdown'], supply) for raw_read in raw_bill['reads']: bill.insert_read( sess, tprs[raw_read['tpr_code']], raw_read['coefficient'], raw_read['units'], raw_read['msn'], raw_read['mpan'], raw_read['prev_date'], raw_read['prev_value'], read_types[raw_read['prev_type_code']], raw_read['pres_date'], raw_read['pres_value'], read_types[raw_read['pres_type_code']]) self.successful_bills.append(raw_bill) except BadRequest as e: raw_bill['error'] = str(e.description) self.failed_bills.append(raw_bill) if len(self.failed_bills) == 0: sess.commit() self._log( "All the bills have been successfully loaded and attached " "to the batch.") else: sess.rollback() self._log( "The import has finished, but there were " + str(len(self.failed_bills)) + " failures, and so the " "whole import has been rolled back.") except: sess.rollback() self._log("I've encountered a problem: " + traceback.format_exc()) finally: if sess is not None: sess.close()