def main():
parser = argparse.ArgumentParser()
parser.add_argument('delim', help='Delimiter')
parser.add_argument('policy', help='csv split policy')
parser.add_argument('query', help='Query string in rbql')
parser.add_argument('input_table_path',
metavar='FILE',
help='Read csv table from FILE instead of stdin')
parser.add_argument('encoding', help='Manually set csv table encoding')
parser.add_argument('output_delim', help='Out Delimiter')
parser.add_argument('output_policy', help='Out csv policy')
args = parser.parse_args()
delim = rbql.normalize_delim(args.delim)
policy = args.policy
output_delim = rbql.normalize_delim(args.output_delim)
output_policy = args.output_policy
query = base64.standard_b64decode(args.query).decode("utf-8")
input_path = args.input_table_path
csv_encoding = args.encoding
output_path = get_dst_table_path(input_path, output_delim)
run_with_python(input_path, delim, policy, csv_encoding, query,
output_delim, output_policy, output_path)
def converged_execute(src_table_path, rb_script_path, input_delim, input_policy, out_delim, out_policy):
try:
input_delim = rbql.normalize_delim(input_delim)
out_delim = rbql.normalize_delim(out_delim)
tmp_dir = tempfile.gettempdir()
table_name = os.path.basename(src_table_path)
dst_table_name = '{}.txt'.format(table_name)
dst_table_path = os.path.join(tmp_dir, dst_table_name)
vim_interface.set_vim_variable('psv_dst_table_path', dst_table_path)
execute_python(src_table_path, rb_script_path, input_delim, input_policy, out_delim, out_policy, dst_table_path)
except Exception as e:
vim_interface.report_error_to_vim('Execution Error', str(e))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('backend_language',
metavar='LANG',
help='script language to use in query',
choices=['python', 'js'])
parser.add_argument('delim', help='Delimiter')
parser.add_argument('policy',
help='csv split policy',
choices=['simple', 'quoted', 'monocolumn'])
parser.add_argument('query', help='Query string in rbql')
parser.add_argument('input_table_path',
metavar='FILE',
help='Read csv table from FILE instead of stdin')
args = parser.parse_args()
delim = rbql.normalize_delim(args.delim)
policy = args.policy
query = args.query
input_path = args.input_table_path
csv_encoding = rbql.default_csv_encoding
output_delim, output_policy = delim, policy
output_path = get_dst_table_path(input_path, output_delim)
if args.backend_language == 'python':
run_with_python(input_path, delim, policy, csv_encoding, query,
output_delim, output_policy, output_path)
else:
run_with_js(input_path, delim, policy, csv_encoding, query,
output_delim, output_policy, output_path)
def run_with_js(args):
if not rbql.system_has_node_js():
print_error_and_exit(
'Error: Node.js is not found, test command: "node --version"')
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit(
'Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit(
'Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
tmp_dir = tempfile.gettempdir()
script_filename = 'rbconvert_{}'.format(time.time()).replace('.',
'_') + '.js'
tmp_path = os.path.join(tmp_dir, script_filename)
rbql.parse_to_js(input_path, output_path, rbql_lines, tmp_path, delim,
policy, output_delim, output_policy, csv_encoding,
import_modules)
cmd = ['node', tmp_path]
pobj = subprocess.Popen(cmd, stderr=subprocess.PIPE)
err_data = pobj.communicate()[1]
exit_code = pobj.returncode
operation_report = rbql.parse_json_report(exit_code, err_data)
operation_error = operation_report.get('error')
if operation_error is not None:
error_msg = 'An error occured during js script execution:\n\n{}\n'.format(
operation_error)
error_msg += '\n================================================\n'
error_msg += 'Generated script location: {}'.format(tmp_path)
print_error_and_exit(error_msg)
warnings = operation_report.get('warnings')
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
rbql.remove_if_possible(tmp_path)
def converged_execute(meta_language, src_table_path, rb_script_path,
input_delim, input_policy, out_delim, out_policy):
try:
input_delim = rbql.normalize_delim(input_delim)
out_delim = rbql.normalize_delim(out_delim)
tmp_dir = tempfile.gettempdir()
table_name = os.path.basename(src_table_path)
dst_table_name = '{}.txt'.format(table_name)
dst_table_path = os.path.join(tmp_dir, dst_table_name)
vim_interface.set_vim_variable('psv_dst_table_path', dst_table_path)
assert meta_language in ['python', 'js']
if meta_language == 'python':
execute_python(src_table_path, rb_script_path, input_delim,
input_policy, out_delim, out_policy, dst_table_path)
else:
execute_js(src_table_path, rb_script_path, input_delim,
input_policy, out_delim, out_policy, dst_table_path)
except Exception as e:
vim_interface.report_error_to_vim('Execution Error', str(e))
def run_with_js(args):
if not rbql.system_has_node_js():
print_error_and_exit('Error: Node.js is not found, test command: "node --version"')
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit('Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit('Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
tmp_dir = tempfile.gettempdir()
script_filename = 'rbconvert_{}'.format(time.time()).replace('.', '_') + '.js'
tmp_path = os.path.join(tmp_dir, script_filename)
rbql.parse_to_js(input_path, output_path, rbql_lines, tmp_path, delim, policy, output_delim, output_policy, csv_encoding, import_modules)
cmd = ['node', tmp_path]
pobj = subprocess.Popen(cmd, stderr=subprocess.PIPE)
err_data = pobj.communicate()[1]
exit_code = pobj.returncode
operation_report = rbql.parse_json_report(exit_code, err_data)
operation_error = operation_report.get('error')
if operation_error is not None:
error_msg = 'An error occured during js script execution:\n\n{}\n'.format(operation_error)
error_msg += '\n================================================\n'
error_msg += 'Generated script location: {}'.format(tmp_path)
print_error_and_exit(error_msg)
warnings = operation_report.get('warnings')
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
rbql.remove_if_possible(tmp_path)
def run_with_python(args):
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit('Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit('Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
tmp_dir = tempfile.gettempdir()
module_name = 'rbconvert_{}'.format(time.time()).replace('.', '_')
module_filename = '{}.py'.format(module_name)
tmp_path = os.path.join(tmp_dir, module_filename)
sys.path.insert(0, tmp_dir)
try:
rbql.parse_to_py(rbql_lines, tmp_path, delim, policy, output_delim, output_policy, csv_encoding, import_modules)
except rbql.RBParsingError as e:
print_error_and_exit('RBQL Parsing Error: \t{}'.format(e))
if not os.path.isfile(tmp_path) or not os.access(tmp_path, os.R_OK):
print_error_and_exit('Error: Unable to find generated python module at {}.'.format(tmp_path))
try:
rbconvert = rbql.dynamic_import(module_name)
src = None
if input_path:
src = codecs.open(input_path, encoding=csv_encoding)
else:
src = rbql.get_encoded_stdin(csv_encoding)
warnings = None
if output_path:
with codecs.open(output_path, 'w', encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
else:
dst = rbql.get_encoded_stdout(csv_encoding)
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
rbql.remove_if_possible(tmp_path)
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Location of the generated module: {}\n\n'.format(tmp_path)
error_msg += 'Original python exception:\n{}\n'.format(str(e))
print_error_and_exit(error_msg)
def run_with_python(args):
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit(
'Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit(
'Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
tmp_dir = tempfile.gettempdir()
module_name = 'rbconvert_{}'.format(time.time()).replace('.', '_')
module_filename = '{}.py'.format(module_name)
tmp_path = os.path.join(tmp_dir, module_filename)
sys.path.insert(0, tmp_dir)
try:
rbql.parse_to_py(rbql_lines, tmp_path, delim, policy, output_delim,
output_policy, csv_encoding, import_modules)
except rbql.RBParsingError as e:
print_error_and_exit('RBQL Parsing Error: \t{}'.format(e))
if not os.path.isfile(tmp_path) or not os.access(tmp_path, os.R_OK):
print_error_and_exit(
'Error: Unable to find generated python module at {}.'.format(
tmp_path))
try:
rbconvert = rbql.dynamic_import(module_name)
src = None
if input_path:
src = codecs.open(input_path, encoding=csv_encoding)
else:
src = rbql.get_encoded_stdin(csv_encoding)
warnings = None
if output_path:
with codecs.open(output_path, 'w', encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
else:
dst = rbql.get_encoded_stdout(csv_encoding)
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
rbql.remove_if_possible(tmp_path)
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Location of the generated module: {}\n\n'.format(
tmp_path)
error_msg += 'Original python exception:\n{}\n'.format(str(e))
print_error_and_exit(error_msg)
def run_with_python(args):
delim = rbql.normalize_delim(args.delim)
policy = args.policy
if policy is None:
policy = 'quoted' if delim in [';', ','] else 'simple'
query = args.query
query_path = args.query_file
#convert_only = args.convert_only
input_path = args.input_table_path
output_path = args.output_table_path
import_modules = args.libs
csv_encoding = args.csv_encoding
output_delim, output_policy = interpret_format(args.out_format)
rbql_lines = None
if query is None and query_path is None:
print_error_and_exit(
'Error: provide either "--query" or "--query_path" option')
if query is not None and query_path is not None:
print_error_and_exit(
'Error: unable to use both "--query" and "--query_path" options')
if query_path is not None:
assert query is None
rbql_lines = codecs.open(query_path, encoding='utf-8').readlines()
else:
assert query_path is None
rbql_lines = [query]
with rbql.RbqlPyEnv() as worker_env:
tmp_path = worker_env.module_path
try:
rbql.parse_to_py(rbql_lines, tmp_path, delim, policy, output_delim,
output_policy, csv_encoding, import_modules)
except rbql.RBParsingError as e:
print_error_and_exit('RBQL Parsing Error: \t{}'.format(e))
try:
rbconvert = worker_env.import_worker()
src = None
if input_path:
src = codecs.open(input_path, encoding=csv_encoding)
else:
src = rbql.get_encoded_stdin(csv_encoding)
warnings = None
if output_path:
with codecs.open(output_path, 'w',
encoding=csv_encoding) as dst:
warnings = rbconvert.rb_transform(src, dst)
else:
dst = rbql.get_encoded_stdout(csv_encoding)
warnings = rbconvert.rb_transform(src, dst)
if warnings is not None:
hr_warnings = rbql.make_warnings_human_readable(warnings)
for warning in hr_warnings:
eprint('Warning: {}'.format(warning))
worker_env.remove_env_dir()
except Exception as e:
error_msg = 'Error: Unable to use generated python module.\n'
error_msg += 'Location of the generated module: {}\n\n'.format(
tmp_path)
error_msg += 'Original python exception:\n{}\n'.format(str(e))
print_error_and_exit(error_msg)