def sub_batch_run(self,
lines,
batch_number,
sub_batch_number,
total_line_nb,
check=False):
success = False
st = time()
try:
success = self._send_rpc(lines,
batch_number,
sub_batch_number,
check=check)
except Fault as e:
log_error("Line %s %s failed" % (batch_number, sub_batch_number))
log_error(e.faultString)
except ValueError as e:
log_error("Line %s %s failed value error" %
(batch_number, sub_batch_number))
except Exception as e:
log_info("Unknown Problem")
exc_type, exc_value, _ = sys.exc_info()
#traceback.print_tb(exc_traceback, file=sys.stdout)
log_error(exc_type)
log_error(exc_value)
if not success:
self.writer.writerows(lines)
log_info("time for batch %s - %s of %s : %s" %
(batch_number, (sub_batch_number + 1) * self.batch_size,
total_line_nb, time() - st))
def import_data(config_file, model, header=None, data=None, file_csv=None, context=None, fail_file=False, encoding='utf-8-sig', separator=";", ignore=False, split=False, check=True, max_connection=1, batch_size=10, skip=0):
"""
header and data mandatory in file_csv is not provided
"""
ignore = ignore or []
context = context or {}
if file_csv:
header, data = read_file(file_csv, delimiter=separator, encoding=encoding, skip=skip)
fail_file = fail_file or file_csv + ".fail"
file_result = open(fail_file, "wb")
if not header or data == None:
raise ValueError("Please provide either a data file or a header and data")
object_registry = conf_lib.get_server_connection(config_file).get_model(model)
if file_csv:
writer = UnicodeWriter(file_result, delimiter=separator, encoding=encoding, quoting=csv.QUOTE_ALL)
else:
writer = ListWriter()
writer.writerow(filter_header_ignore(ignore, header))
if file_csv:
file_result.flush()
rpc_thread = RPCThreadImport(int(max_connection), object_registry, filter_header_ignore(ignore, header), writer, batch_size, context)
st = time()
data, split_index = split_sort(split, header, data)
i = 0
previous_split_value = False
while i < len(data):
lines = []
j = 0
while i < len(data) and (j < batch_size or do_not_split(split, previous_split_value, split_index, data[i])):
line = data[i][:len(header)]
lines.append(filter_line_ignore(ignore, header, line))
previous_split_value = line[split_index]
j += 1
i += 1
batch_number = split and "[%s] - [%s]" % (rpc_thread.thread_number(), previous_split_value) or "[%s]" % rpc_thread.thread_number()
rpc_thread.launch_batch(lines, batch_number, check)
rpc_thread.wait()
if file_csv:
file_result.close()
log_info("%s %s imported, total time %s second(s)" % (len(data), model, (time() - st)))
if file_csv:
return False, False
else:
return writer.header, writer.data
def launch_batch_fun(data_ids, batch_number, check=False):
st = time()
try:
self.result[batch_number] = self.model.export_data(
data_ids, self.header, context=self.context)['datas']
except Fault as e:
log_error("export %s failed" % batch_number)
log_error(e.faultString)
except Exception as e:
log_info("Unknown Problem")
exc_type, exc_value, _ = sys.exc_info()
#traceback.print_tb(exc_traceback, file=sys.stdout)
log_error(exc_type)
log_error(exc_value)
log_info("time for batch %s: %s" % (batch_number, time() - st))
def export_data(config_file,
model,
domain,
header,
context=None,
output=None,
max_connection=1,
batch_size=100,
separator=';',
encoding='utf-8-sig'):
object_registry = conf_lib.get_server_connection(config_file).get_model(
model)
if output:
file_result = open(output, "wb")
writer = UnicodeWriter(file_result,
delimiter=separator,
encoding=encoding,
quoting=csv.QUOTE_ALL)
else:
writer = ListWriter()
rpc_thread = RPCThreadExport(int(max_connection), object_registry, header,
writer, batch_size, context)
st = time()
ids = object_registry.search(domain, context=context)
i = 0
for b in batch(ids, batch_size):
batch_ids = [l for l in b]
rpc_thread.launch_batch(batch_ids, i)
i += 1
rpc_thread.wait()
log_info("%s %s exported, total time %s second(s)" % (len(ids), model,
(time() - st)))
log_info("Writing file")
rpc_thread.write_file(writer)
if output:
file_result.close()
return False, False
else:
return writer.header, writer.data
def skip_line(reader):
log_info("Skipping until line %s excluded" % skip)
for _ in xrange(1, skip):
reader.next()