def execute_python(src_table_path, rb_script_path, encoding, input_delim, input_policy, out_delim, out_policy, dst_table_path):
query = codecs.open(rb_script_path, encoding=encoding).read()
warnings = []
try:
rbql.query_csv(query, src_table_path, input_delim, input_policy, dst_table_path, out_delim, out_policy, encoding, warnings)
warning_report = '\n'.join(warnings)
vim_interface.set_vim_variable('psv_warning_report', warning_report)
vim_interface.set_vim_variable('psv_query_status', 'OK')
except Exception as e:
error_type, error_msg = rbql.exception_to_error_info(e)
vim_interface.report_error_to_vim(error_type, error_msg)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('query', help='Query string')
parser.add_argument('input_table_path', metavar='FILE', help='input path')
parser.add_argument('delim', help='Delimiter')
parser.add_argument('policy', help='csv split policy')
parser.add_argument('output_table_path',
metavar='FILE',
help='output path')
parser.add_argument('output_delim', help='Out Delimiter')
parser.add_argument('output_policy', help='Out csv policy')
parser.add_argument('encoding', help='encoding')
parser.add_argument('--skip_headers',
action='store_true',
help='skip headers')
args = parser.parse_args()
delim = args.delim
policy = args.policy
output_delim = args.output_delim
output_policy = args.output_policy
query = base64.standard_b64decode(args.query).decode("utf-8")
input_path = args.input_table_path
csv_encoding = args.encoding
output_path = args.output_table_path
skip_headers = args.skip_headers
try:
warnings = []
rbql.query_csv(query, input_path, delim, policy, output_path,
output_delim, output_policy, csv_encoding, warnings,
skip_headers)
sys.stdout.write(json.dumps({'warnings': warnings}))
except Exception as e:
error_type, error_msg = rbql.exception_to_error_info(e)
sys.stdout.write(
json.dumps({
'error_type': error_type,
'error_msg': error_msg
}))
def process_test_case(self, tmp_tests_dir, test_case):
test_name = test_case['test_name']
minimal_python_version = float(
test_case.get('minimal_python_version', 2.7))
if python_version < minimal_python_version:
print(
'Skipping {}: python version must be at least {}. Interpreter version is {}'
.format(test_name, minimal_python_version, python_version))
return
query = test_case.get('query_python', None)
if query is None:
return
debug_mode = test_case.get('debug_mode', False)
randomly_replace_var_names = test_case.get(
'randomly_replace_var_names', True)
with_headers = test_case.get('with_headers', False)
input_table_path = test_case['input_table_path']
query = query.replace('###UT_TESTS_DIR###', script_dir)
if randomly_replace_var_names:
query = randomly_replace_columns_dictionary_style(query)
input_table_path = os.path.join(script_dir, input_table_path)
expected_output_table_path = test_case.get(
'expected_output_table_path', None)
if expected_output_table_path is not None:
expected_output_table_path = os.path.join(
script_dir, expected_output_table_path)
expected_md5 = calc_file_md5(expected_output_table_path)
output_file_name = os.path.basename(expected_output_table_path)
actual_output_table_path = os.path.join(tmp_tests_dir,
output_file_name)
else:
actual_output_table_path = os.path.join(tmp_tests_dir,
'expected_empty_file')
expected_error = test_case.get('expected_error', None)
expected_warnings = test_case.get('expected_warnings', [])
delim = test_case['csv_separator']
policy = test_case['csv_policy']
encoding = test_case['csv_encoding']
comment_prefix = test_case.get('comment_prefix', None)
output_format = test_case.get('output_format', 'input')
out_delim, out_policy = (
delim, policy
) if output_format == 'input' else rbql_csv.interpret_named_csv_format(
output_format)
if debug_mode:
rbql_csv.set_debug_mode()
warnings = []
error_type, error_msg = None, None
try:
rbql_csv.query_csv(query, input_table_path, delim, policy,
actual_output_table_path, out_delim, out_policy,
encoding, warnings, with_headers,
comment_prefix)
except Exception as e:
if debug_mode:
raise
error_type, error_msg = rbql.exception_to_error_info(e)
self.assertTrue((expected_error is not None) == (
error_type is not None
), 'Inside json test: "{}". Expected error: {}, error_type, error_msg: {}'
.format(test_name, expected_error, error_type,
error_msg))
if expected_error is not None:
self.assertTrue(
error_msg.find(expected_error) != -1,
'Inside json test: "{}", Expected error: "{}", Actual error: "{}"'
.format(test_name, expected_error, error_msg))
else:
actual_md5 = calc_file_md5(actual_output_table_path)
self.assertTrue(
expected_md5 == actual_md5,
'md5 missmatch in test "{}". Expected table: {}, Actual table: {}'
.format(test_name, expected_output_table_path,
actual_output_table_path))
warnings = sorted(normalize_warnings(warnings))
expected_warnings = sorted(expected_warnings)
self.assertEqual(
expected_warnings, warnings,
'Inside json test: "{}". Expected warnings: {}, Actual warnings: {}'
.format(test_name, expected_warnings, warnings))
def process_test_case(self, test_case):
test_name = test_case['test_name']
query = test_case.get('query_python', None)
if query is None:
if python_version >= 3:
query = test_case.get('query_python_3', None)
else:
query = test_case.get('query_python_2', None)
debug_mode = test_case.get('debug_mode', False)
minimal_python_version = float(
test_case.get('minimal_python_version', 2.7))
if python_version < minimal_python_version:
print(
'Skipping {}: python version must be at least {}. Interpreter version is {}'
.format(test_name, minimal_python_version, python_version))
return
randomly_replace_var_names = test_case.get(
'randomly_replace_var_names', True)
if query is None:
self.assertTrue(test_case.get('query_js', None) is not None)
return # Skip this test
if randomly_replace_var_names:
query = randomly_replace_column_variable_style(query)
input_table = test_case['input_table']
join_table = test_case.get('join_table', None)
input_column_names = test_case.get('input_column_names', None)
join_column_names = test_case.get('join_column_names', None)
normalize_column_names = test_case.get('normalize_column_names', True)
user_init_code = test_case.get('python_init_code', '')
expected_output_table = test_case.get('expected_output_table', None)
expected_error_type = test_case.get('expected_error_type', None)
expected_error = test_case.get('expected_error', None)
if expected_error is None:
expected_error = test_case.get('expected_error_py', None)
if expected_error is None:
if python_version >= 3:
expected_error = test_case.get('expected_error_py_3', None)
else:
expected_error = test_case.get('expected_error_py_2', None)
expected_error_exact = test_case.get('expected_error_exact', False)
expected_warnings = test_case.get('expected_warnings', [])
output_table = []
if debug_mode:
rbql_engine.set_debug_mode()
warnings = []
error_type, error_msg = None, None
try:
rbql.query_table(query, input_table, output_table, warnings,
join_table, input_column_names, join_column_names,
normalize_column_names, user_init_code)
except Exception as e:
if debug_mode:
raise
error_type, error_msg = rbql.exception_to_error_info(e)
self.assertTrue((expected_error is not None) == (
error_type is not None
), 'Inside json test: "{}". Expected error: {}, error_type: {}, error_msg: {}'
.format(test_name, expected_error, error_type,
error_msg))
if expected_error_type is not None:
self.assertTrue(error_type == expected_error_type,
'Inside json test: {}'.format(test_name))
if expected_error is not None:
if expected_error_exact:
self.assertEqual(
expected_error, error_msg,
'Inside json test: {}. Expected error: {}, Actual error: {}'
.format(test_name, expected_error, error_msg))
else:
self.assertTrue(
error_msg.find(expected_error) != -1,
'Inside json test: {}'.format(test_name))
else:
round_floats(expected_output_table)
round_floats(output_table)
self.assertEqual(expected_output_table, output_table,
'Inside json test: {}'.format(test_name))
warnings = sorted(normalize_warnings(warnings))
expected_warnings = sorted(expected_warnings)
self.assertEqual(
expected_warnings, warnings,
'Inside json test: {}. Expected warnings: {}; Actual warnings: {}'
.format(test_name, ','.join(expected_warnings),
','.join(warnings)))
def process_test_case(self, test_case):
test_name = test_case['test_name']
query = test_case.get('query_python', None)
if query is None:
if python_version >= 3:
query = test_case.get('query_python_3', None)
else:
query = test_case.get('query_python_2', None)
debug_mode = test_case.get('debug_mode', False)
minimal_python_version = float(test_case.get('minimal_python_version', 2.7))
if python_version < minimal_python_version:
print('Skipping {}: python version must be at least {}. Interpreter version is {}'.format(test_name, minimal_python_version, python_version))
return
randomly_replace_var_names = test_case.get('randomly_replace_var_names', True)
if query is None:
self.assertTrue(test_case.get('query_js', None) is not None)
return # Skip this test
if randomly_replace_var_names:
query = randomly_replace_column_variable_style(query)
input_table = test_case['input_table']
join_table = test_case.get('join_table', None)
input_column_names = test_case.get('input_column_names', None)
join_column_names = test_case.get('join_column_names', None)
input_df = pandas.DataFrame(input_table, columns=input_column_names)
join_df = None if join_table is None else pandas.DataFrame(join_table, columns=join_column_names)
normalize_column_names = test_case.get('normalize_column_names', True)
user_init_code = test_case.get('python_init_code', '')
expected_output_header = test_case.get('expected_output_header', None)
expected_output_table = test_case.get('expected_output_table', None)
expected_output_df = None if expected_output_table is None else pandas.DataFrame(expected_output_table, columns=expected_output_header)
expected_error_type = test_case.get('expected_error_type', None)
expected_error = test_case.get('expected_error', None)
if expected_error is None:
expected_error = test_case.get('expected_error_py', None)
if expected_error is None:
if python_version >= 3:
expected_error = test_case.get('expected_error_py_3', None)
else:
expected_error = test_case.get('expected_error_py_2', None)
expected_error_exact = test_case.get('expected_error_exact', False)
expected_warnings = test_case.get('expected_warnings', [])
rbql_engine.set_debug_mode(debug_mode)
warnings = []
error_type, error_msg = None, None
try:
output_df = rbql_pandas.query_dataframe(query, input_df, warnings, join_df, normalize_column_names, user_init_code)
except Exception as e:
if debug_mode:
raise
error_type, error_msg = rbql.exception_to_error_info(e)
self.assertTrue((expected_error is not None) == (error_type is not None), 'Inside json test: "{}". Expected error: {}, error_type: {}, error_msg: {}'.format(test_name, expected_error, error_type, error_msg))
if expected_error_type is not None:
self.assertTrue(error_type == expected_error_type, 'Inside json test: {}'.format(test_name))
if expected_error is not None:
if expected_error_exact:
self.assertEqual(expected_error, error_msg, 'Inside json test: {}. Expected error: {}, Actual error: {}'.format(test_name, expected_error, error_msg))
else:
self.assertTrue(error_msg.find(expected_error) != -1, 'Inside json test: {}'.format(test_name))
else:
try:
if python_version >= 3:
assert_frame_equal(expected_output_df, output_df)
else:
# For some reason dataframe.column.inferred_type are different for expected and actual output in python2, let's disregard this.
assert_frame_equal(expected_output_df, output_df, check_column_type=False)
except Exception as e:
print('\nFailed inside json test: "{}"'.format(test_name))
raise
warnings = sorted(normalize_warnings(warnings))
if 'inconsistent input records' in expected_warnings:
# Since pandas dataframes are always squares we don't need this warning.
expected_warnings.remove('inconsistent input records')
expected_warnings = sorted(expected_warnings)
self.assertEqual(expected_warnings, warnings, 'Inside json test: {}. Expected warnings: {}; Actual warnings: {}'.format(test_name, ','.join(expected_warnings), ','.join(warnings)))