def create_symbolic_link(name):
cmd = 'ln -s /dev/stdin %s' % name
executor = Executor(silent_exit=True)
executor.execute('test_dmt_yes_no',
cmd,
input_data=None,
output_pipe=False)
def execute(self):
t_before = time.time()
executor = Executor()
cmd = 'sqlite3 -batch'
script = self._create_script(self.variables)
getLogger(LOG).debug('Sqlite script:\n%s', script)
executor.execute('refname', cmd, input_data=script, output_pipe=True)
res = executor.poll_for_complete(observer)
del executor
t_after = time.time()
csvname = ''
if CSV_KEY in self.job_syntax_item:
csvname = var_replaced(self.variables, self.job_syntax_item,
CSV_KEY)
self.fix_empty_res_table(csvname)
if ismoreinfo():
getLogger(MOREINFO).info('%s.csv - %.2fs' %
(csvname, t_after - t_before))
else:
getLogger(STDOUT).info('.')
res = res['refname']
if res[0] != 0:
raise Exception("Sqlite query error", self.get_query())
else:
self._handle_var_create(res)
retcode = res[1]
return retcode
def test_assert_type_error():
executor = Executor(silent_exit=True)
stdin_data = "SELECT 0 =>var:ASSERT=>assert:unsupported_assert"
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_assert_type_error',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
def test_bad_operation_error():
executor = Executor(silent_exit=True)
stdin_data = "SELECT Id FROM src.Acount => csv:forsubmit => src:badop:Account__c:200:resids"
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_bad_operation_error',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_assert_nonzero():
executor = Executor(silent_exit=True)
stdin_data = "SELECT 0 =>var:ASSERT_SHOULD_FAIL=>assert:nonzero"
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_assert_nonzero',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_batch_type_error():
executor = Executor(silent_exit=True)
stdin_data = "SELECT '777' as Id => csv:tmp => dst:insert:Bobject:1:ids => type:puquential"
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_batch_type_error',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_empty_query():
executor = Executor(silent_exit=True)
stdin_data = "=> csv:test"
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_empty_query',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_transmitter_value_error():
executor = Executor(silent_exit=True)
stdin_data = "SELECT 'csv.' => var"
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_transmitter_value_error',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_empty_batch():
executor = Executor(silent_exit=True)
stdin_data = """
SELECT i as Id FROM csv.ints10000 WHERE i<0 => csv:tmp => dst:insert:Bobject:1:ids"""
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_empty_batch',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_batch_param_error():
executor = Executor(silent_exit=True)
stdin_data = """
-- required param ids result table is omited, will raise an error on parsing stage
SELECT Id FROM src.Acount => csv:forsubmit => src:insert:Account__c:200"""
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_batch_param_error',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_macro_param_error():
executor = Executor(silent_exit=True)
stdin_data = """
-- QUERY value itself is ommited (actually query should not be passed ar macro param)
=> macro:macro_test:QUERY"""
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_bad_operation_error',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def run_test_graph(datadir, sqlfpath):
"""it's will be used by another test test_job_reader.test_job_controller
As all the data was prepared by mocked sources and mocks will not work in a subprocess"""
# run graph
executor = Executor()
graphpath = 'tests/test_graph'
graphdir = os.path.dirname(graphpath)
relative_path = os.path.relpath(tempfile.mkdtemp(), graphdir)
cmd = "python graph_dmt.py --conf-file test-config.ini --job-file %s --job-file tests/sql/test2.sql --save-graph %s --csvdir %s" % (
sqlfpath, graphpath, relative_path)
print relative_path
executor.execute('test_graph', cmd, input_data=None, output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_macro():
create_symbolic_link('tests/sql/dev_stdin')
executor = Executor(silent_exit=True)
stdin_data = """=> macro:macro_test:QUERY:SELECT 1:RES_TABLE_NAME:test
=> macro:macro_no_params
"""
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file tests/sql/dev_stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_macro',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def create_table_get_csv_data(tablename, script):
executor = Executor()
cmd = 'sqlite3 -batch'
executor.execute(tablename, cmd, input_data=script, output_pipe=True)
res = executor.poll_for_complete(observer)
del executor
try:
assert cmp_csv_lines(res[tablename], CSV_DATA)
#assert res[tablename] == CSV_DATA
except:
print '-------'
print res[tablename]
print CSV_DATA
raise
return res[tablename]
def test_unsupported_csv_prefix():
executor = Executor(silent_exit=True)
stdin_data = """
SELECT 'csv.' => var:CSV_PREFIX
-- such prefix doesn't supported
SELECT * FROM {CSV_PREFIX}ints10000 LIMIT 10 => csv:newints10
"""
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file /dev/stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_unsupported_csv_prefix',
cmd,
input_data=stdin_data,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_stepbystep_yes_maybe_no():
answers = """y
y
maybe_yes_maybe_no_maybe_xxx_idontknow
n
"""
executor = Executor()
cmd = "python mriya_dmt.py --conf-file test-config.ini --step-by-step --src-name 'foo1' --dst-name 'foo2' --job-file tests/sql/test.sql --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_dmt_yes_no',
cmd,
input_data=answers,
output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_dmt():
create_symbolic_link('tests/sql/dev_stdin')
executor = Executor()
stdin_data = """
SELECT "SELECT 'pen' as field1, 'pineapple' as field2, 'apple' as field3, 'pen' as field4" => var:QUERY => const:
=> macro:macro_test \
:RES_TABLE_NAME:test_fields_table
SELECT * from csv.ints10000 => csv:aggregation_test
-- put following line just to test some branches in graph.py
SELECT fake FROM src.fake => csv:aggregation_test:cache
SELECT fake FROM dst.fake2 => csv:aggregation_test:cache
SELECT * FROM csv.test_fields_table => var:info:publish
"""
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file tests/sql/dev_stdin --datadir %s" % (
tempfile.mkdtemp())
executor.execute('test_dmt', cmd, input_data=stdin_data, output_pipe=True)
res = executor.poll_for_complete(observer)
print res
def test_dmt_bad_param():
executor = Executor(silent_exit=True)
cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2'"
executor.execute('test_dmt_bad_param',
cmd,
input_data=None,
output_pipe=True)
executor.poll_for_complete(observer)
def async_engine_assert(stdin_pipe_data):
expected_order = ['test4.2', 'test4.1']
actual_order = []
def observer(refname, retcode, output):
stdout_data = None
if output:
stdout_data = output.read()
print refname, retcode, stdout_data
actual_order.append(refname)
executor = Executor()
cmd = 'echo "test1" && sleep 1 && echo first done'
executor.execute('test4.1', cmd, stdin_pipe_data)
cmd = 'echo "test2" && echo second done'
executor.execute('test4.2', cmd, stdin_pipe_data)
print "waiting..."
executor.poll_for_complete(observer)
print expected_order
print actual_order
assert expected_order == actual_order
del executor
def run_mriya(datadir):
executor = Executor()
jobs = []
for script in SQL_SCRIPTS:
jobs.append('--job-file')
jobs.append(script)
cmdl = [
'python', 'mriya_dmt.py', '--conf-file', 'test-config.ini',
'--src-name', 'foo1', '--dst-name', 'foo2', '--datadir', datadir
]
cmdl.extend(jobs)
cmd = ' '.join(cmdl)
print cmd
executor.execute('test_dmt', cmd, input_data=None, output_pipe=True)
executor.wait_for_complete()
def test_3():
executor = Executor()
executor.execute('t1', 'echo "test1" && sleep 1 && echo first done')
executor.execute('t2', 'echo "test2" && echo first done')
executor.wait_for_complete()