def test_insert_load(mock_docall, m):
# mock setup
sf_bulk_connector.JOB_CHECK_TIMER = 0
mockers.mock_insert_load_delete(mock_docall, m)
conn = create_bulk_connector(setup(), 'test')
####### INSERT #####
csv_data = TEST_CSV_INSERT
bulk_result_ids = conn.bulk_insert('Account', csv_data, 1000, False)
# Retrieve ids of inserted results
result_ids = parse_batch_res_data(bulk_result_ids)
print "result_ids", result_ids
###### SELECT #####
selected = fetch_records_by_returned_ids(
conn, result_ids, "Name,Account_Birthday__c,Billing_Address__c,type")
# check if inserted data is equal to retrieved
expected = get_bulk_data_from_csv_stream(StringIO(TEST_CSV_INSERT))
try:
assert_equality(expected, selected)
except:
print "selected", selected
print "expected", expected
raise
###### DELETE #####
delid = result_ids.rows[0][0]
csv_delete_data = '"Id"\n"%s"' % (delid)
del_result_ids = conn.bulk_delete('Account', csv_delete_data, 100, False)
# Retrieve ids of inserted results
del_ids = parse_batch_res_data(del_result_ids)
expected_del_res = BulkData(fields=['Id', 'Success', 'Created', 'Error'],
rows=[(delid, u'true', u'false', u'')])
assert_equality(expected_del_res, del_ids)
def handle_transmitter_op(self, job_syntax_item, endpoint):
opname = job_syntax_item[OP_KEY]
# run batches sequentially / parallel
if BATCH_TYPE_KEY in job_syntax_item:
if job_syntax_item[BATCH_TYPE_KEY] == BATCH_TYPE_PARALLEL_KEY:
batch_seq = False
elif job_syntax_item[BATCH_TYPE_KEY] == BATCH_TYPE_SEQUENTIAL_KEY:
batch_seq = True
else:
getLogger(STDERR).error('Unknown batch type: %s',
job_syntax_item[BATCH_TYPE_KEY])
exit(1)
else:
batch_seq = False # parallel by default
csv_key_val = var_replaced(self.variables, job_syntax_item, CSV_KEY)
csv_filename = SqlExecutor.csv_name(csv_key_val)
csv_data = self.csvdata(csv_filename)
num_lines = len(csv_data)
# do nothing for empty data set
if num_lines <= 1:
getLogger(LOG).info('skip empty csv')
stub = ['"Id","Success","Created","Error"\n']
result_ids = parse_batch_res_data(stub)
else:
objname = job_syntax_item[endpoint]
conn = self.endpoints.endpoint(endpoint)
max_batch_size = int(job_syntax_item[BATCH_SIZE_KEY])
getLogger(STDOUT).info('EXECUTE: %s %s, lines count=%d', opname,
objname, num_lines - 1)
t_before = time.time()
if len(csv_data):
if opname == OP_UPDATE:
res = conn.bulk_update(objname, csv_data, max_batch_size,
batch_seq)
elif opname == OP_DELETE:
res = conn.bulk_delete(objname, csv_data, max_batch_size,
batch_seq)
elif opname == OP_INSERT:
res = conn.bulk_insert(objname, csv_data, max_batch_size,
batch_seq)
else:
getLogger(STDERR).error("Operation '%s' isn't supported" %
opname)
exit(1)
result_ids = parse_batch_res_data(res)
t_after = time.time()
getLogger(STDOUT).info('SF %s Took time: %.2f' \
% (opname, t_after-t_before))
if NEW_IDS_TABLE in job_syntax_item:
results_file_name = \
SqlExecutor.csv_name(job_syntax_item[NEW_IDS_TABLE])
with open(results_file_name, 'w') as result_ids_file:
csv_data = csv_from_bulk_data(result_ids)
result_ids_file.write(csv_data)
getLogger(LOG).info('Saved result ids: %s', results_file_name)
getLogger(LOG).info('Done: %s operation', opname)
def test_insert_update(mock_docall, m):
# mock setup
sf_bulk_connector.JOB_CHECK_TIMER = 0
mockers.mock_insert_update(mock_docall, m)
config = setup()
# test case when sessions file doesn't exist
sessions_file_name = config[DEFAULT_SETTINGS_SECTION][SESSIONS_SETTING]
remove(sessions_file_name)
conn = create_bulk_connector(config, 'test')
####### INSERT #####
csv_data = TEST_CSV_INSERT2
bulk_result_ids = conn.bulk_insert('Account', csv_data, 1000, False)
# Retrieve ids of inserted results
result_ids = parse_batch_res_data(bulk_result_ids)
print "ok insert"
###### UPDATE #####
selected_with_ids = fetch_records_by_returned_ids(
conn, result_ids, "Id,Name,Account_Birthday__c,Billing_Address__c,type")
# check if inserted data is equal to retrieved
name_idx = selected_with_ids.fields.index('Name')
rows = []
for item in selected_with_ids.rows:
item_update = list(item)
item_update[name_idx] = 'BLA-BLA-BLA'
rows.append(tuple(item_update))
update_data = BulkData(selected_with_ids.fields, rows)
csv_update = csv_from_bulk_data(update_data)
print "csv_update", csv_update
update_bulk_result_ids = conn.bulk_update('Account', csv_update, 1000, False)
# Retrieve ids of inserted results
update_result_ids = parse_batch_res_data(update_bulk_result_ids)
print "update_result_ids", update_result_ids
###### SELECT #####
selected_update = fetch_records_by_returned_ids(
conn, update_result_ids,
"Id,Name,Account_Birthday__c,Billing_Address__c,type")
# check if inserted data is equal to retrieved
try:
assert_equality(selected_update, update_data)
except:
print "update_data", update_data
print "selected_update", selected_update
raise
def fetch_records_by_returned_ids(conn, result_ids, columns):
id_idx = result_ids.fields.index('Id')
success_idx = result_ids.fields.index('Success')
where = ' OR '.join(["Id='%s'"%x[id_idx] for x in result_ids.rows if x[success_idx] == 'true'])
query = "SELECT %s FROM Account WHERE %s" % (columns, where)
print query
csv_rows = conn.bulk_load('Account', query)
selected = parse_batch_res_data(csv_rows)
return selected
def handle_op_returning_ids(self, opname, objname, res, merge=False):
if not merge:
result_ids = bulk_data.parse_batch_res_data(res)
else:
result_ids = res
id_idx = result_ids.fields.index('Id')
success_idx = result_ids.fields.index('Success')
if not merge:
error_idx = result_ids.fields.index('Error')
else:
error_idx = result_ids.fields.index('Message')
max_err_count_output = 100
for item in result_ids.rows:
if item[success_idx] != 'true':
max_err_count_output = max_err_count_output - 1
if max_err_count_output > 0:
getLogger(STDERR).error('Batch %s-%s: Id=%s, error:%s',
opname, objname, item[id_idx],
item[error_idx])
return result_ids
def handle_result(self, bulk_res):
# handle empty result - fix it by adding column names
if bulk_res and bulk_res[0] == EMPTY_SF_RESPONSE:
cols = SqlExecutor.get_query_columns(self.get_query())
header = ','.join(cols)
bulk_res = [header]
if len(bulk_res) > 1:
#ignore last empty results
bulk_res = bulk_res[:-1]
# handle result
if CSV_KEY in self.job_syntax_item:
csv_key_val = var_replaced(self.variables, self.job_syntax_item,
CSV_KEY)
csvfname = SqlExecutor.csv_name(csv_key_val)
bulk_data.save_escape_csv_lines_as_csv_file(csvfname, bulk_res)
elif VAR_KEY in self.job_syntax_item:
res = bulk_data.parse_batch_res_data(bulk_res)
if res.rows:
self.save_var(self.job_syntax_item[VAR_KEY], res.rows[0][0])