def sub_batch_run(self, lines, batch_number, sub_batch_number, total_line_nb, check=False): success = False st = time() try: success = self._send_rpc(lines, batch_number, sub_batch_number, check=check) except Fault as e: log_error("Line %s %s failed" % (batch_number, sub_batch_number)) log_error(e.faultString) except ValueError as e: log_error("Line %s %s failed value error" % (batch_number, sub_batch_number)) except Exception as e: log_info("Unknown Problem") exc_type, exc_value, _ = sys.exc_info() #traceback.print_tb(exc_traceback, file=sys.stdout) log_error(exc_type) log_error(exc_value) if not success: self.writer.writerows(lines) log_info("time for batch %s - %s of %s : %s" % (batch_number, (sub_batch_number + 1) * self.batch_size, total_line_nb, time() - st))
def _send_rpc(self, lines, batch_number, sub_batch_number, check=False): res = self.model.load(self.header, lines, context=self.context) if res['messages']: for msg in res['messages']: log_error('batch %s, %s' % (batch_number, sub_batch_number)) log_error(msg) log_error(lines[msg['record']]) return False if len(res['ids']) != len(lines) and check: log_error("number of record import is different from the record to import, probably duplicate xml_id") return False return True
def launch_batch_fun(data_ids, batch_number, check=False): st = time() try: self.result[batch_number] = self.model.export_data( data_ids, self.header, context=self.context)['datas'] except Fault as e: log_error("export %s failed" % batch_number) log_error(e.faultString) except Exception as e: log_info("Unknown Problem") exc_type, exc_value, _ = sys.exc_info() #traceback.print_tb(exc_traceback, file=sys.stdout) log_error(exc_type) log_error(exc_value) log_info("time for batch %s: %s" % (batch_number, time() - st))
def check_id_column(header): try: header.index('id') except ValueError as ve: log_error("No External Id (id) column defined, please add one") raise ve