def test_dmt_bad_param(): executor = Executor(silent_exit=True) cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2'" executor.execute('test_dmt_bad_param', cmd, input_data=None, output_pipe=True) executor.poll_for_complete(observer)
def execute(self): t_before = time.time() executor = Executor() cmd = 'sqlite3 -batch' script = self._create_script(self.variables) getLogger(LOG).debug('Sqlite script:\n%s', script) executor.execute('refname', cmd, input_data=script, output_pipe=True) res = executor.poll_for_complete(observer) del executor t_after = time.time() csvname = '' if CSV_KEY in self.job_syntax_item: csvname = var_replaced(self.variables, self.job_syntax_item, CSV_KEY) self.fix_empty_res_table(csvname) if ismoreinfo(): getLogger(MOREINFO).info('%s.csv - %.2fs' % (csvname, t_after - t_before)) else: getLogger(STDOUT).info('.') res = res['refname'] if res[0] != 0: raise Exception("Sqlite query error", self.get_query()) else: self._handle_var_create(res) retcode = res[1] return retcode
def test_assert_type_error(): executor = Executor(silent_exit=True) stdin_data = "SELECT 0 =>var:ASSERT=>assert:unsupported_assert" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_assert_type_error', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer)
def test_bad_operation_error(): executor = Executor(silent_exit=True) stdin_data = "SELECT Id FROM src.Acount => csv:forsubmit => src:badop:Account__c:200:resids" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_bad_operation_error', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_transmitter_value_error(): executor = Executor(silent_exit=True) stdin_data = "SELECT 'csv.' => var" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_transmitter_value_error', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_batch_type_error(): executor = Executor(silent_exit=True) stdin_data = "SELECT '777' as Id => csv:tmp => dst:insert:Bobject:1:ids => type:puquential" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_batch_type_error', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_assert_nonzero(): executor = Executor(silent_exit=True) stdin_data = "SELECT 0 =>var:ASSERT_SHOULD_FAIL=>assert:nonzero" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_assert_nonzero', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_empty_query(): executor = Executor(silent_exit=True) stdin_data = "=> csv:test" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_empty_query', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_empty_batch(): executor = Executor(silent_exit=True) stdin_data = """ SELECT i as Id FROM csv.ints10000 WHERE i<0 => csv:tmp => dst:insert:Bobject:1:ids""" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_empty_batch', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def async_engine_assert(stdin_pipe_data): expected_order = ['test4.2', 'test4.1'] actual_order = [] def observer(refname, retcode, output): stdout_data = None if output: stdout_data = output.read() print refname, retcode, stdout_data actual_order.append(refname) executor = Executor() cmd = 'echo "test1" && sleep 1 && echo first done' executor.execute('test4.1', cmd, stdin_pipe_data) cmd = 'echo "test2" && echo second done' executor.execute('test4.2', cmd, stdin_pipe_data) print "waiting..." executor.poll_for_complete(observer) print expected_order print actual_order assert expected_order == actual_order del executor
def test_macro_param_error(): executor = Executor(silent_exit=True) stdin_data = """ -- QUERY value itself is ommited (actually query should not be passed ar macro param) => macro:macro_test:QUERY""" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_bad_operation_error', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_batch_param_error(): executor = Executor(silent_exit=True) stdin_data = """ -- required param ids result table is omited, will raise an error on parsing stage SELECT Id FROM src.Acount => csv:forsubmit => src:insert:Account__c:200""" cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_batch_param_error', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def run_test_graph(datadir, sqlfpath): """it's will be used by another test test_job_reader.test_job_controller As all the data was prepared by mocked sources and mocks will not work in a subprocess""" # run graph executor = Executor() graphpath = 'tests/test_graph' graphdir = os.path.dirname(graphpath) relative_path = os.path.relpath(tempfile.mkdtemp(), graphdir) cmd = "python graph_dmt.py --conf-file test-config.ini --job-file %s --job-file tests/sql/test2.sql --save-graph %s --csvdir %s" % ( sqlfpath, graphpath, relative_path) print relative_path executor.execute('test_graph', cmd, input_data=None, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_macro(): create_symbolic_link('tests/sql/dev_stdin') executor = Executor(silent_exit=True) stdin_data = """=> macro:macro_test:QUERY:SELECT 1:RES_TABLE_NAME:test => macro:macro_no_params """ cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name test --dst-name test --job-file tests/sql/dev_stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_macro', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def create_table_get_csv_data(tablename, script): executor = Executor() cmd = 'sqlite3 -batch' executor.execute(tablename, cmd, input_data=script, output_pipe=True) res = executor.poll_for_complete(observer) del executor try: assert cmp_csv_lines(res[tablename], CSV_DATA) #assert res[tablename] == CSV_DATA except: print '-------' print res[tablename] print CSV_DATA raise return res[tablename]
def test_unsupported_csv_prefix(): executor = Executor(silent_exit=True) stdin_data = """ SELECT 'csv.' => var:CSV_PREFIX -- such prefix doesn't supported SELECT * FROM {CSV_PREFIX}ints10000 LIMIT 10 => csv:newints10 """ cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file /dev/stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_unsupported_csv_prefix', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_stepbystep_yes_maybe_no(): answers = """y y maybe_yes_maybe_no_maybe_xxx_idontknow n """ executor = Executor() cmd = "python mriya_dmt.py --conf-file test-config.ini --step-by-step --src-name 'foo1' --dst-name 'foo2' --job-file tests/sql/test.sql --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_dmt_yes_no', cmd, input_data=answers, output_pipe=True) res = executor.poll_for_complete(observer) print res
def test_dmt(): create_symbolic_link('tests/sql/dev_stdin') executor = Executor() stdin_data = """ SELECT "SELECT 'pen' as field1, 'pineapple' as field2, 'apple' as field3, 'pen' as field4" => var:QUERY => const: => macro:macro_test \ :RES_TABLE_NAME:test_fields_table SELECT * from csv.ints10000 => csv:aggregation_test -- put following line just to test some branches in graph.py SELECT fake FROM src.fake => csv:aggregation_test:cache SELECT fake FROM dst.fake2 => csv:aggregation_test:cache SELECT * FROM csv.test_fields_table => var:info:publish """ cmd = "python mriya_dmt.py --conf-file test-config.ini --src-name 'foo1' --dst-name 'foo2' --job-file tests/sql/dev_stdin --datadir %s" % ( tempfile.mkdtemp()) executor.execute('test_dmt', cmd, input_data=stdin_data, output_pipe=True) res = executor.poll_for_complete(observer) print res