def test_insert_load(mock_docall, m): # mock setup sf_bulk_connector.JOB_CHECK_TIMER = 0 mockers.mock_insert_load_delete(mock_docall, m) conn = create_bulk_connector(setup(), 'test') ####### INSERT ##### csv_data = TEST_CSV_INSERT bulk_result_ids = conn.bulk_insert('Account', csv_data, 1000, False) # Retrieve ids of inserted results result_ids = parse_batch_res_data(bulk_result_ids) print "result_ids", result_ids ###### SELECT ##### selected = fetch_records_by_returned_ids( conn, result_ids, "Name,Account_Birthday__c,Billing_Address__c,type") # check if inserted data is equal to retrieved expected = get_bulk_data_from_csv_stream(StringIO(TEST_CSV_INSERT)) try: assert_equality(expected, selected) except: print "selected", selected print "expected", expected raise ###### DELETE ##### delid = result_ids.rows[0][0] csv_delete_data = '"Id"\n"%s"' % (delid) del_result_ids = conn.bulk_delete('Account', csv_delete_data, 100, False) # Retrieve ids of inserted results del_ids = parse_batch_res_data(del_result_ids) expected_del_res = BulkData(fields=['Id', 'Success', 'Created', 'Error'], rows=[(delid, u'true', u'false', u'')]) assert_equality(expected_del_res, del_ids)
def handle_transmitter_op(self, job_syntax_item, endpoint): opname = job_syntax_item[OP_KEY] # run batches sequentially / parallel if BATCH_TYPE_KEY in job_syntax_item: if job_syntax_item[BATCH_TYPE_KEY] == BATCH_TYPE_PARALLEL_KEY: batch_seq = False elif job_syntax_item[BATCH_TYPE_KEY] == BATCH_TYPE_SEQUENTIAL_KEY: batch_seq = True else: getLogger(STDERR).error('Unknown batch type: %s', job_syntax_item[BATCH_TYPE_KEY]) exit(1) else: batch_seq = False # parallel by default csv_key_val = var_replaced(self.variables, job_syntax_item, CSV_KEY) csv_filename = SqlExecutor.csv_name(csv_key_val) csv_data = self.csvdata(csv_filename) num_lines = len(csv_data) # do nothing for empty data set if num_lines <= 1: getLogger(LOG).info('skip empty csv') stub = ['"Id","Success","Created","Error"\n'] result_ids = parse_batch_res_data(stub) else: objname = job_syntax_item[endpoint] conn = self.endpoints.endpoint(endpoint) max_batch_size = int(job_syntax_item[BATCH_SIZE_KEY]) getLogger(STDOUT).info('EXECUTE: %s %s, lines count=%d', opname, objname, num_lines - 1) t_before = time.time() if len(csv_data): if opname == OP_UPDATE: res = conn.bulk_update(objname, csv_data, max_batch_size, batch_seq) elif opname == OP_DELETE: res = conn.bulk_delete(objname, csv_data, max_batch_size, batch_seq) elif opname == OP_INSERT: res = conn.bulk_insert(objname, csv_data, max_batch_size, batch_seq) else: getLogger(STDERR).error("Operation '%s' isn't supported" % opname) exit(1) result_ids = parse_batch_res_data(res) t_after = time.time() getLogger(STDOUT).info('SF %s Took time: %.2f' \ % (opname, t_after-t_before)) if NEW_IDS_TABLE in job_syntax_item: results_file_name = \ SqlExecutor.csv_name(job_syntax_item[NEW_IDS_TABLE]) with open(results_file_name, 'w') as result_ids_file: csv_data = csv_from_bulk_data(result_ids) result_ids_file.write(csv_data) getLogger(LOG).info('Saved result ids: %s', results_file_name) getLogger(LOG).info('Done: %s operation', opname)
def test_insert_update(mock_docall, m): # mock setup sf_bulk_connector.JOB_CHECK_TIMER = 0 mockers.mock_insert_update(mock_docall, m) config = setup() # test case when sessions file doesn't exist sessions_file_name = config[DEFAULT_SETTINGS_SECTION][SESSIONS_SETTING] remove(sessions_file_name) conn = create_bulk_connector(config, 'test') ####### INSERT ##### csv_data = TEST_CSV_INSERT2 bulk_result_ids = conn.bulk_insert('Account', csv_data, 1000, False) # Retrieve ids of inserted results result_ids = parse_batch_res_data(bulk_result_ids) print "ok insert" ###### UPDATE ##### selected_with_ids = fetch_records_by_returned_ids( conn, result_ids, "Id,Name,Account_Birthday__c,Billing_Address__c,type") # check if inserted data is equal to retrieved name_idx = selected_with_ids.fields.index('Name') rows = [] for item in selected_with_ids.rows: item_update = list(item) item_update[name_idx] = 'BLA-BLA-BLA' rows.append(tuple(item_update)) update_data = BulkData(selected_with_ids.fields, rows) csv_update = csv_from_bulk_data(update_data) print "csv_update", csv_update update_bulk_result_ids = conn.bulk_update('Account', csv_update, 1000, False) # Retrieve ids of inserted results update_result_ids = parse_batch_res_data(update_bulk_result_ids) print "update_result_ids", update_result_ids ###### SELECT ##### selected_update = fetch_records_by_returned_ids( conn, update_result_ids, "Id,Name,Account_Birthday__c,Billing_Address__c,type") # check if inserted data is equal to retrieved try: assert_equality(selected_update, update_data) except: print "update_data", update_data print "selected_update", selected_update raise
def fetch_records_by_returned_ids(conn, result_ids, columns): id_idx = result_ids.fields.index('Id') success_idx = result_ids.fields.index('Success') where = ' OR '.join(["Id='%s'"%x[id_idx] for x in result_ids.rows if x[success_idx] == 'true']) query = "SELECT %s FROM Account WHERE %s" % (columns, where) print query csv_rows = conn.bulk_load('Account', query) selected = parse_batch_res_data(csv_rows) return selected
def handle_op_returning_ids(self, opname, objname, res, merge=False): if not merge: result_ids = bulk_data.parse_batch_res_data(res) else: result_ids = res id_idx = result_ids.fields.index('Id') success_idx = result_ids.fields.index('Success') if not merge: error_idx = result_ids.fields.index('Error') else: error_idx = result_ids.fields.index('Message') max_err_count_output = 100 for item in result_ids.rows: if item[success_idx] != 'true': max_err_count_output = max_err_count_output - 1 if max_err_count_output > 0: getLogger(STDERR).error('Batch %s-%s: Id=%s, error:%s', opname, objname, item[id_idx], item[error_idx]) return result_ids
def handle_result(self, bulk_res): # handle empty result - fix it by adding column names if bulk_res and bulk_res[0] == EMPTY_SF_RESPONSE: cols = SqlExecutor.get_query_columns(self.get_query()) header = ','.join(cols) bulk_res = [header] if len(bulk_res) > 1: #ignore last empty results bulk_res = bulk_res[:-1] # handle result if CSV_KEY in self.job_syntax_item: csv_key_val = var_replaced(self.variables, self.job_syntax_item, CSV_KEY) csvfname = SqlExecutor.csv_name(csv_key_val) bulk_data.save_escape_csv_lines_as_csv_file(csvfname, bulk_res) elif VAR_KEY in self.job_syntax_item: res = bulk_data.parse_batch_res_data(bulk_res) if res.rows: self.save_var(self.job_syntax_item[VAR_KEY], res.rows[0][0])