def execute_python(src_table_path, rb_script_path, input_delim, input_policy, out_delim, out_policy, dst_table_path):
csv_encoding = rbql.default_csv_encoding
with rbql.RbqlPyEnv() as worker_env:
try:
tmp_path = worker_env.module_path
rbql_lines = codecs.open(rb_script_path, encoding='utf-8').readlines()
rbql.parse_to_py(rbql_lines, tmp_path, input_delim, input_policy, out_delim, out_policy, csv_encoding, None)
except rbql.RBParsingError as e:
vim_interface.report_error_to_vim('Parsing Error', str(e))
return
try:
rbconvert = worker_env.import_worker()
warnings = None
with codecs.open(src_table_path, encoding=csv_encoding) as src, codecs.open(dst_table_path, 'w', encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
warning_report = '\n'.join(hr_warnings)
vim_interface.set_vim_variable('psv_warning_report', warning_report)
worker_env.remove_env_dir()
vim_interface.set_vim_variable('psv_query_status', 'OK')
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Original python exception:\n{}\n'.format(str(e))
vim_interface.report_error_to_vim('Execution Error', error_msg)
def execute_python(src_table_path, rb_script_path, input_delim, input_policy, out_delim, out_policy, dst_table_path):
csv_encoding = rbql.default_csv_encoding
tmp_dir = tempfile.gettempdir()
module_name = 'vim_rb_convert_{}'.format(get_random_suffix())
meta_script_name = '{}.py'.format(module_name)
meta_script_path = os.path.join(tmp_dir, meta_script_name)
try:
rbql_lines = codecs.open(rb_script_path, encoding='utf-8').readlines()
rbql.parse_to_py(rbql_lines, meta_script_path, input_delim, input_policy, out_delim, out_policy, csv_encoding, None)
except rbql.RBParsingError as e:
rbql.remove_if_possible(meta_script_path)
vim_interface.report_error_to_vim('Parsing Error', str(e))
return
sys.path.insert(0, tmp_dir)
try:
rbconvert = rbql.dynamic_import(module_name)
warnings = None
with codecs.open(src_table_path, encoding=csv_encoding) as src, codecs.open(dst_table_path, 'w', encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
warning_report = '\n'.join(hr_warnings)
vim_interface.set_vim_variable('psv_warning_report', warning_report)
rbql.remove_if_possible(meta_script_path)
vim_interface.set_vim_variable('psv_query_status', 'OK')
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Original python exception:\n{}\n'.format(str(e))
vim_interface.report_error_to_vim('Execution Error', error_msg)
with open(os.path.join(tmp_dir, 'last_rbql_exception'), 'w') as exc_dst:
traceback.print_exc(file=exc_dst)
def execute_python(src_table_path, rb_script_path, meta_script_path,
dst_table_path, delim, csv_encoding):
try:
rbql_lines = codecs.open(rb_script_path, encoding='utf-8').readlines()
rbql.parse_to_py(rbql_lines, meta_script_path, delim, csv_encoding)
except rbql.RBParsingError as e:
report_to_vim('Parsing Error', str(e))
return
module_name = os.path.basename(meta_script_path)
assert module_name.endswith('.py')
module_name = module_name[:-3]
module_dir = os.path.dirname(meta_script_path)
sys.path.insert(0, module_dir)
try:
rbconvert = rbql.dynamic_import(module_name)
with codecs.open(src_table_path,
encoding=csv_encoding) as src, codecs.open(
dst_table_path, 'w',
encoding=csv_encoding) as dst:
rbconvert.rb_transform(src, dst)
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Original python exception:\n{}\n'.format(str(e))
report_to_vim('Execution Error', error_msg)
tmp_dir = tempfile.gettempdir()
with open(os.path.join(tmp_dir, 'last_exception'), 'w') as exc_dst:
traceback.print_exc(file=exc_dst)
return
report_to_vim('OK')
def run_with_python(input_path, delim, policy, csv_encoding, query,
output_delim, output_policy, output_path):
with rbql.RbqlPyEnv() as worker_env:
tmp_path = worker_env.module_path
try:
rbql.parse_to_py([query], tmp_path, delim, policy, output_delim,
output_policy, csv_encoding, None)
except rbql.RBParsingError as e:
report_error_and_exit('RBQL_Parsing', str(e))
try:
report = {'result_path': output_path}
rbconvert = worker_env.import_worker()
src = None
if input_path:
src = codecs.open(input_path, encoding=csv_encoding)
else:
src = rbql.get_encoded_stdin(csv_encoding)
warnings = None
with codecs.open(output_path, 'w', encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
warnings = rbql.make_warnings_human_readable(warnings)
report['warnings'] = warnings
worker_env.remove_env_dir()
report_success_and_exit(report)
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Location of the generated module: {}\n'.format(
tmp_path)
error_msg += 'Original python exception:\n{}'.format(str(e))
report_error_and_exit('Wrapper', error_msg)
def run_conversion_test_py(query,
input_table,
testname,
import_modules=None,
join_csv_encoding=default_csv_encoding,
delim='\t'):
tmp_dir = tempfile.gettempdir()
if not len(sys.path) or sys.path[0] != tmp_dir:
sys.path.insert(0, tmp_dir)
module_name = '{}{}_{}_{}'.format(rainbow_ut_prefix, time.time(), testname,
random.randint(1, 100000000)).replace(
'.', '_')
module_filename = '{}.py'.format(module_name)
tmp_path = os.path.join(tmp_dir, module_filename)
src = table_to_stream(input_table, delim)
dst = io.StringIO()
rbql.parse_to_py([query], tmp_path, delim, join_csv_encoding,
import_modules)
assert os.path.isfile(tmp_path) and os.access(tmp_path, os.R_OK)
rbconvert = rbql.dynamic_import(module_name)
rbconvert.rb_transform(src, dst)
out_data = dst.getvalue()
if len(out_data):
out_lines = out_data[:-1].split('\n')
out_table = [ln.split(delim) for ln in out_lines]
else:
out_table = []
return out_table
def execute_python(src_table_path, rb_script_path, input_delim, input_policy,
out_delim, out_policy, dst_table_path):
csv_encoding = rbql.default_csv_encoding
tmp_dir = tempfile.gettempdir()
module_name = 'vim_rb_convert_{}'.format(get_random_suffix())
meta_script_name = '{}.py'.format(module_name)
meta_script_path = os.path.join(tmp_dir, meta_script_name)
try:
rbql_lines = codecs.open(rb_script_path, encoding='utf-8').readlines()
rbql.parse_to_py(rbql_lines, meta_script_path, input_delim,
input_policy, out_delim, out_policy, csv_encoding,
None)
except rbql.RBParsingError as e:
rbql.remove_if_possible(meta_script_path)
vim_interface.report_error_to_vim('Parsing Error', str(e))
return
sys.path.insert(0, tmp_dir)
try:
rbconvert = rbql.dynamic_import(module_name)
warnings = None
with codecs.open(src_table_path,
encoding=csv_encoding) as src, codecs.open(
dst_table_path, 'w',
encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
warning_report = '\n'.join(hr_warnings)
vim_interface.set_vim_variable('psv_warning_report',
warning_report)
rbql.remove_if_possible(meta_script_path)
vim_interface.set_vim_variable('psv_query_status', 'OK')
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Original python exception:\n{}\n'.format(str(e))
vim_interface.report_error_to_vim('Execution Error', error_msg)
with open(os.path.join(tmp_dir, 'last_rbql_exception'),
'w') as exc_dst:
traceback.print_exc(file=exc_dst)
def run_file_query_test_py(query,
input_path,
testname,
import_modules=None,
csv_encoding=default_csv_encoding,
delim='\t'):
tmp_dir = tempfile.gettempdir()
if not len(sys.path) or sys.path[0] != tmp_dir:
sys.path.insert(0, tmp_dir)
module_name = '{}{}_{}_{}'.format(rainbow_ut_prefix, time.time(), testname,
random.randint(1, 100000000)).replace(
'.', '_')
module_filename = '{}.py'.format(module_name)
tmp_path = os.path.join(tmp_dir, module_filename)
dst_table_filename = '{}.tsv'.format(module_name)
output_path = os.path.join(tmp_dir, dst_table_filename)
rbql.parse_to_py([query], tmp_path, delim, csv_encoding, import_modules)
rbconvert = rbql.dynamic_import(module_name)
with codecs.open(input_path, encoding=csv_encoding) as src, codecs.open(
output_path, 'w', encoding=csv_encoding) as dst:
rbconvert.rb_transform(src, dst)
return output_path
def run_with_python(args):
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit('Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit('Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
tmp_dir = tempfile.gettempdir()
module_name = 'rbconvert_{}'.format(time.time()).replace('.', '_')
module_filename = '{}.py'.format(module_name)
tmp_path = os.path.join(tmp_dir, module_filename)
sys.path.insert(0, tmp_dir)
try:
rbql.parse_to_py(rbql_lines, tmp_path, delim, policy, output_delim, output_policy, csv_encoding, import_modules)
except rbql.RBParsingError as e:
print_error_and_exit('RBQL Parsing Error: \t{}'.format(e))
if not os.path.isfile(tmp_path) or not os.access(tmp_path, os.R_OK):
print_error_and_exit('Error: Unable to find generated python module at {}.'.format(tmp_path))
try:
rbconvert = rbql.dynamic_import(module_name)
src = None
if input_path:
src = codecs.open(input_path, encoding=csv_encoding)
else:
src = rbql.get_encoded_stdin(csv_encoding)
warnings = None
if output_path:
with codecs.open(output_path, 'w', encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
else:
dst = rbql.get_encoded_stdout(csv_encoding)
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
rbql.remove_if_possible(tmp_path)
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Location of the generated module: {}\n\n'.format(tmp_path)
error_msg += 'Original python exception:\n{}\n'.format(str(e))
print_error_and_exit(error_msg)
def run_with_python(args):
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit(
'Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit(
'Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
tmp_dir = tempfile.gettempdir()
module_name = 'rbconvert_{}'.format(time.time()).replace('.', '_')
module_filename = '{}.py'.format(module_name)
tmp_path = os.path.join(tmp_dir, module_filename)
sys.path.insert(0, tmp_dir)
try:
rbql.parse_to_py(rbql_lines, tmp_path, delim, policy, output_delim,
output_policy, csv_encoding, import_modules)
except rbql.RBParsingError as e:
print_error_and_exit('RBQL Parsing Error: \t{}'.format(e))
if not os.path.isfile(tmp_path) or not os.access(tmp_path, os.R_OK):
print_error_and_exit(
'Error: Unable to find generated python module at {}.'.format(
tmp_path))
try:
rbconvert = rbql.dynamic_import(module_name)
src = None
if input_path:
src = codecs.open(input_path, encoding=csv_encoding)
else:
src = rbql.get_encoded_stdin(csv_encoding)
warnings = None
if output_path:
with codecs.open(output_path, 'w', encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
else:
dst = rbql.get_encoded_stdout(csv_encoding)
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
rbql.remove_if_possible(tmp_path)
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Location of the generated module: {}\n\n'.format(
tmp_path)
error_msg += 'Original python exception:\n{}\n'.format(str(e))
print_error_and_exit(error_msg)
def run_with_python(args):
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit(
'Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit(
'Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
with rbql.RbqlPyEnv() as worker_env:
tmp_path = worker_env.module_path
try:
rbql.parse_to_py(rbql_lines, tmp_path, delim, policy, output_delim,
output_policy, csv_encoding, import_modules)
except rbql.RBParsingError as e:
print_error_and_exit('RBQL Parsing Error: \t{}'.format(e))
try:
rbconvert = worker_env.import_worker()
src = None
if input_path:
src = codecs.open(input_path, encoding=csv_encoding)
else:
src = rbql.get_encoded_stdin(csv_encoding)
warnings = None
if output_path:
with codecs.open(output_path, 'w',
encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
else:
dst = rbql.get_encoded_stdout(csv_encoding)
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
worker_env.remove_env_dir()
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Location of the generated module: {}\n\n'.format(
tmp_path)
error_msg += 'Original python exception:\n{}\n'.format(str(e))
print_error_and_exit(error_msg)