def execute_js(src_table_path, rb_script_path, meta_script_path,
dst_table_path, delim, csv_encoding):
if not rbql.system_has_node_js():
report_to_vim('Execution Error',
'Node.js is not found, test command: "node --version"')
return
try:
rbql_lines = codecs.open(rb_script_path, encoding='utf-8').readlines()
rbql.parse_to_js(src_table_path, dst_table_path, rbql_lines,
meta_script_path, delim, csv_encoding)
except rbql.RBParsingError as e:
report_to_vim('Parsing Error', str(e))
return
cmd = ['node', meta_script_path]
pobj = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_data, err_data = pobj.communicate()
error_code = pobj.returncode
if len(err_data) or len(out_data) or error_code != 0:
if len(err_data):
err_data = err_data.decode('latin-1')
else:
err_data = out_data.decode('latin-1')
if not len(err_data):
err_data = 'Unknown Error'
report_to_vim('Execution Error', err_data)
return
report_to_vim('OK')
def execute_js(src_table_path, rb_script_path, input_delim, input_policy, out_delim, out_policy, dst_table_path):
csv_encoding = rbql.default_csv_encoding
tmp_dir = tempfile.gettempdir()
meta_script_name = 'vim_rb_convert_{}.js'.format(get_random_suffix())
meta_script_path = os.path.join(tmp_dir, meta_script_name)
if not rbql.system_has_node_js():
vim_interface.report_error_to_vim('Execution Error', 'Node.js is not found, test command: "node --version"')
return
try:
rbql_lines = codecs.open(rb_script_path, encoding='utf-8').readlines()
rbql.parse_to_js(src_table_path, dst_table_path, rbql_lines, meta_script_path, input_delim, input_policy, out_delim, out_policy, csv_encoding, None)
except rbql.RBParsingError as e:
vim_interface.report_error_to_vim('Parsing Error', str(e))
return
cmd = ['node', meta_script_path]
pobj = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out_data, err_data = pobj.communicate()
error_code = pobj.returncode
operation_report = rbql.parse_json_report(error_code, err_data)
operation_error = operation_report.get('error')
if operation_error is not None:
vim_interface.report_error_to_vim('Execution Error', operation_error)
return
warnings = operation_report.get('warnings')
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
warning_report = '\n'.join(hr_warnings)
vim_interface.set_vim_variable('psv_warning_report', warning_report)
rbql.remove_if_possible(meta_script_path)
vim_interface.set_vim_variable('psv_query_status', 'OK')
def run_with_js(input_path, delim, policy, csv_encoding, query, output_delim,
output_policy, output_path):
if not rbql.system_has_node_js():
report_error_and_exit(
'User', 'Node.js is not found, test command: "node --version"')
tmp_dir = tempfile.gettempdir()
script_filename = 'rbconvert_{}'.format(time.time()).replace('.',
'_') + '.js'
tmp_path = os.path.join(tmp_dir, script_filename)
rbql.parse_to_js(input_path, output_path, [query], tmp_path, delim, policy,
output_delim, output_policy, csv_encoding, None)
cmd = ['node', tmp_path]
pobj = subprocess.Popen(cmd, stderr=subprocess.PIPE)
err_data = pobj.communicate()[1]
exit_code = pobj.returncode
report = {'result_path': output_path}
operation_report = rbql.parse_json_report(exit_code, err_data)
operation_error = operation_report.get('error')
if operation_error is not None:
report_error_and_exit('RBQL_backend', operation_error)
warnings = operation_report.get('warnings')
if warnings is not None:
warnings = rbql.make_warnings_human_readable(warnings)
report['warnings'] = warnings
rbql.remove_if_possible(tmp_path)
report_success_and_exit(report)
def run_file_query_test_js(query,
input_path,
testname,
import_modules=None,
csv_encoding=default_csv_encoding,
delim='\t'):
tmp_dir = tempfile.gettempdir()
rnd_string = '{}{}_{}_{}'.format(rainbow_ut_prefix, time.time(), testname,
random.randint(1, 100000000)).replace(
'.', '_')
script_filename = '{}.js'.format(rnd_string)
tmp_path = os.path.join(tmp_dir, script_filename)
dst_table_filename = '{}.tsv'.format(rnd_string)
output_path = os.path.join(tmp_dir, dst_table_filename)
rbql.parse_to_js(input_path, output_path, [query], tmp_path, delim,
csv_encoding, import_modules)
cmd = ['node', tmp_path]
pobj = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_data, err_data = pobj.communicate()
error_code = pobj.returncode
if len(err_data) or error_code != 0:
if len(err_data):
err_data = err_data.decode('latin-1')
raise RuntimeError(
"Error in file test: {}.\nError text:\n{}\n\nScript location: {}".
format(testname, err_data, tmp_path))
return output_path
def run_conversion_test_js(query,
input_table,
testname,
import_modules=None,
csv_encoding=default_csv_encoding,
delim='\t'):
tmp_dir = tempfile.gettempdir()
script_name = '{}{}_{}_{}.js'.format(rainbow_ut_prefix,
time.time(), testname,
random.randint(1, 100000000)).replace(
'.', '_')
tmp_path = os.path.join(tmp_dir, script_name)
rbql.parse_to_js(None, None, [query], tmp_path, delim, csv_encoding, None)
src = table_to_string(input_table, delim)
cmd = ['node', tmp_path]
pobj = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
out_data, err_data = pobj.communicate(src.encode(csv_encoding))
out_data = out_data.decode(csv_encoding)
error_code = pobj.returncode
if len(err_data) or error_code != 0:
if len(err_data):
err_data = err_data.decode('latin-1')
raise RuntimeError(
"Error in file test: {}.\nError text:\n{}\n\nScript location: {}".
format(testname, err_data, tmp_path))
if len(out_data):
out_lines = out_data[:-1].split('\n')
out_table = [ln.split(delim) for ln in out_lines]
else:
out_table = []
return out_table
def run_with_js(args):
if not rbql.system_has_node_js():
print_error_and_exit(
'Error: Node.js is not found, test command: "node --version"')
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit(
'Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit(
'Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
tmp_dir = tempfile.gettempdir()
script_filename = 'rbconvert_{}'.format(time.time()).replace('.',
'_') + '.js'
tmp_path = os.path.join(tmp_dir, script_filename)
rbql.parse_to_js(input_path, output_path, rbql_lines, tmp_path, delim,
policy, output_delim, output_policy, csv_encoding,
import_modules)
cmd = ['node', tmp_path]
pobj = subprocess.Popen(cmd, stderr=subprocess.PIPE)
err_data = pobj.communicate()[1]
exit_code = pobj.returncode
operation_report = rbql.parse_json_report(exit_code, err_data)
operation_error = operation_report.get('error')
if operation_error is not None:
error_msg = 'An error occured during js script execution:\n\n{}\n'.format(
operation_error)
error_msg += '\n================================================\n'
error_msg += 'Generated script location: {}'.format(tmp_path)
print_error_and_exit(error_msg)
warnings = operation_report.get('warnings')
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
rbql.remove_if_possible(tmp_path)
def run_with_js(args):
if not rbql.system_has_node_js():
print_error_and_exit('Error: Node.js is not found, test command: "node --version"')
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit('Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit('Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
tmp_dir = tempfile.gettempdir()
script_filename = 'rbconvert_{}'.format(time.time()).replace('.', '_') + '.js'
tmp_path = os.path.join(tmp_dir, script_filename)
rbql.parse_to_js(input_path, output_path, rbql_lines, tmp_path, delim, policy, output_delim, output_policy, csv_encoding, import_modules)
cmd = ['node', tmp_path]
pobj = subprocess.Popen(cmd, stderr=subprocess.PIPE)
err_data = pobj.communicate()[1]
exit_code = pobj.returncode
operation_report = rbql.parse_json_report(exit_code, err_data)
operation_error = operation_report.get('error')
if operation_error is not None:
error_msg = 'An error occured during js script execution:\n\n{}\n'.format(operation_error)
error_msg += '\n================================================\n'
error_msg += 'Generated script location: {}'.format(tmp_path)
print_error_and_exit(error_msg)
warnings = operation_report.get('warnings')
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
rbql.remove_if_possible(tmp_path)
def execute_js(src_table_path, rb_script_path, input_delim, input_policy,
out_delim, out_policy, dst_table_path):
csv_encoding = rbql.default_csv_encoding
tmp_dir = tempfile.gettempdir()
meta_script_name = 'vim_rb_convert_{}.js'.format(get_random_suffix())
meta_script_path = os.path.join(tmp_dir, meta_script_name)
if not rbql.system_has_node_js():
vim_interface.report_error_to_vim(
'Execution Error',
'Node.js is not found, test command: "node --version"')
return
try:
rbql_lines = codecs.open(rb_script_path, encoding='utf-8').readlines()
rbql.parse_to_js(src_table_path, dst_table_path, rbql_lines,
meta_script_path, input_delim, input_policy,
out_delim, out_policy, csv_encoding, None)
except rbql.RBParsingError as e:
vim_interface.report_error_to_vim('Parsing Error', str(e))
return
cmd = ['node', meta_script_path]
pobj = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out_data, err_data = pobj.communicate()
error_code = pobj.returncode
operation_report = rbql.parse_json_report(error_code, err_data)
operation_error = operation_report.get('error')
if operation_error is not None:
vim_interface.report_error_to_vim('Execution Error', operation_error)
return
warnings = operation_report.get('warnings')
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
warning_report = '\n'.join(hr_warnings)
vim_interface.set_vim_variable('psv_warning_report', warning_report)
rbql.remove_if_possible(meta_script_path)
vim_interface.set_vim_variable('psv_query_status', 'OK')