def sub_batch_run(self,
lines,
batch_number,
sub_batch_number,
total_line_nb,
check=False):
success = False
st = time()
try:
success = self._send_rpc(lines,
batch_number,
sub_batch_number,
check=check)
except Fault as e:
log_error("Line %s %s failed" % (batch_number, sub_batch_number))
log_error(e.faultString)
except ValueError as e:
log_error("Line %s %s failed value error" %
(batch_number, sub_batch_number))
except Exception as e:
log_info("Unknown Problem")
exc_type, exc_value, _ = sys.exc_info()
#traceback.print_tb(exc_traceback, file=sys.stdout)
log_error(exc_type)
log_error(exc_value)
if not success:
self.writer.writerows(lines)
log_info("time for batch %s - %s of %s : %s" %
(batch_number, (sub_batch_number + 1) * self.batch_size,
total_line_nb, time() - st))
def _send_rpc(self, lines, batch_number, sub_batch_number, check=False):
res = self.model.load(self.header, lines, context=self.context)
if res['messages']:
for msg in res['messages']:
log_error('batch %s, %s' % (batch_number, sub_batch_number))
log_error(msg)
log_error(lines[msg['record']])
return False
if len(res['ids']) != len(lines) and check:
log_error("number of record import is different from the record to import, probably duplicate xml_id")
return False
return True
def launch_batch_fun(data_ids, batch_number, check=False):
st = time()
try:
self.result[batch_number] = self.model.export_data(
data_ids, self.header, context=self.context)['datas']
except Fault as e:
log_error("export %s failed" % batch_number)
log_error(e.faultString)
except Exception as e:
log_info("Unknown Problem")
exc_type, exc_value, _ = sys.exc_info()
#traceback.print_tb(exc_traceback, file=sys.stdout)
log_error(exc_type)
log_error(exc_value)
log_info("time for batch %s: %s" % (batch_number, time() - st))
def check_id_column(header):
try:
header.index('id')
except ValueError as ve:
log_error("No External Id (id) column defined, please add one")
raise ve