def test_mocked_merge_unsupported_object(m): loginit(STDERR) fake_bulk_connector = mock.Mock() fake_bulk_connector.bulk.sessionid = 'fake-sessionid' fake_bulk_connector.bulk.instance_url = 'https://fake-localhost' m.post(url=requests_mock.ANY, text='') sm = SfSoapMergeWrapper(fake_bulk_connector, 'UnsupportedObject', BULK_DATA_IN, 99) assert None == sm.validate()
def setup(): loginit(__name__) config = ConfigParser() with open(config_file, 'r') as conf_file: config.read_file(conf_file) # test case when sessions file exists but has no token sessions_file_name = config[DEFAULT_SETTINGS_SECTION][SESSIONS_SETTING] with open(sessions_file_name, 'w') as sessions_f: sessions_f.write('{"someuser": "******"}') return config
def test_read(): loginit(__name__) text = 'SELECT \\\n\ 1 => csv:const1\n\ SELECT 1 => var:MIN => dst:foo' test_stream = StringIO(text) lines = JobSyntax.prepare_lines(test_stream.readlines()) assert lines == [ 'SELECT 1 => csv:const1', 'SELECT 1 => var:MIN => dst:foo' ]
def test_job_controller(mock_docall, m): # mock setup sf_bulk_connector.JOB_CHECK_TIMER = 0 mockers.mock_job_controller(mock_docall, m) # test itself setdatadir(tempfile.mkdtemp()) # test debug coverage mriya.log.INITIALIZED_LOGGERS = {} mriya.log.LOGGING_LEVEL = logging.DEBUG loginit(__name__) print "test_job_controller" # prepare test_csv.csv test_csv = ['"Alexa__c"', '"hello\n\n2"'] with open(SqlExecutor.csv_name('test_csv'), "w") as test_csv_f: test_csv_f.write('"Alexa__c"\n"hello<N CR><N CR>2"\n') notch = 'test1234567' with open('tests/sql/complicated.sql') as sql_f: lines = sql_f.readlines() job_syntax = JobSyntaxExtended(lines) with open(config_filename) as conf_file: job_controller = JobController(conf_file.name, endpoint_names, job_syntax, {}, False) job_controller.run_job() del job_controller # check resulted data with open(SqlExecutor.csv_name('some_data_staging')) as resulted_file: csv_data = get_bulk_data_from_csv_stream(resulted_file) name_idx = csv_data.fields.index('Name') assert 1 == len(csv_data.rows) assert csv_data.rows[0][name_idx] == notch with open(SqlExecutor.csv_name('newids')) as newids_file: csv_data = get_bulk_data_from_csv_stream(newids_file) id_idx = csv_data.fields.index('Id') try: assert 1 == len(csv_data.rows) except: print "len(csv_data.rows)", len(csv_data.rows) raise for row in csv_data.rows: assert len(row[id_idx]) >= 15 assert open(SqlExecutor.csv_name('test_csv')).read() \ == open(SqlExecutor.csv_name('test_csv_2')).read() # run another test using created data run_test_graph(datadir(), "tests/sql/complicated.sql")
def fast_merge_mock(m, response_list): datadir = tempfile.mkdtemp() setdatadir(datadir) mock_oauth(m) mock_login(m) # test smaller part of merge flow but checking output # Also test how merge splits payload to sequence of chunks loginit(STDERR) fake_bulk_connector = mock.Mock() m.register_uri(method='POST', url='https://fake-localhost/services/Soap/c/37.0', response_list=response_list) sm = SfSoapMergeWrapper(fake_bulk_connector, 'Account', BULK_DATA_IN, 2) sm.sf_bulk_connector.bulk.sessionid = 'fake-sessionid' sm.sf_bulk_connector.instance_url = 'https://fake-localhost' res = sm.validate() assert res != None print res bulk_data = sm.run_merge() print bulk_data assert BULK_DATA_OUT == bulk_data
def test_columns(): loginit(__name__) query = "SELECT foo, (SELECT 1 WHERE 0=1) as foo2, foo as foo3, \ a.foo4 FROM " res = SqlExecutor.get_query_columns(query) try: assert (res == ['foo', 'foo2', 'foo3', 'foo4']) except: print res raise query = " SELECT Id as id2, \ b.bomba, (SELECT a.Id FROM foo a WHERE a.nope = b.Id) as id__c, \ (SELECT a.Id from foo2 a WHERE a.ggg = b.nnn) as super \ FROM sometable b WHERE ..." res = SqlExecutor.get_query_columns(query) try: assert (res == ['id2', 'bomba', 'id__c', 'super']) except: print res raise
def test_batch_splitter(): loginit(__name__) batch_ranges = SfBulkConnector.batch_ranges(10, 3) print batch_ranges assert (batch_ranges == [(0, 2), (3, 5), (6, 8), (9, 9)])
def test_var_csv(mock_docall, m): # mock setup sf_bulk_connector.JOB_CHECK_TIMER = 0 mockers.mock_var_csv(mock_docall, m) # test itself setdatadir(tempfile.mkdtemp()) loginit(__name__) print "test_var_csv" macro_lines = [ 'SELECT i from csv.ints10000 WHERE i>=CAST(10 as INTEGER) \ LIMIT 2; => batch_begin:i:NESTED', 'SELECT {NESTED}; => var:foo2', '=> batch_end:NESTED', "SELECT '{STATIC_VAR}'; => var:static_var" ] lines = [ 'SELECT 1; => var:one', "SELECT 'csv.ints10000'; => var:CSV_INTS10000 => const:", "SELECT * FROM {CSV_INTS10000} LIMIT 1; => var:VAR_0", 'SELECT "9+0"; => var:nine', 'SELECT Id FROM src.Account LIMIT 1 => csv:sfvar', 'SELECT * FROM csv.sfvar => var:sfvar', 'SELECT {one} as f1, {nine}+1 as f2; => csv:one_ten', "SELECT 'one_nine_ten' => var:ONENINETEN", 'SELECT f1, {nine} as f9, (SELECT f2 FROM csv.one_ten) as f10 \ FROM csv.one_ten; => csv:{ONENINETEN}', 'SELECT i from csv.ints10000 WHERE i>=2 LIMIT 2; \ => batch_begin:i:PARAM', 'SELECT {PARAM}; => var:foo', '=> macro:macro_test_batch:STATIC_VAR:something', '=> batch_end:PARAM', 'SELECT {PARAM}; => var:final_test' ] expected = [{ 'from': 'csv', 'query': 'SELECT 1;', 'var': 'one' }, { 'from': 'csv', 'query': "SELECT 'csv.ints10000';", 'var': 'CSV_INTS10000', 'const': '', 'from': 'csv' }, { 'from': 'csv', 'query': "SELECT * FROM {CSV_INTS10000} LIMIT 1;", 'var': 'VAR_0' }, { 'from': 'csv', 'query': 'SELECT "9+0";', 'var': 'nine' }, { 'objname': 'Account', 'query': 'SELECT Id FROM Account LIMIT 1', 'csv': 'sfvar', 'from': 'src', 'objname': 'Account' }, { 'query': 'SELECT * FROM sfvar', 'var': 'sfvar', 'csvlist': ['sfvar'], 'from': 'csv' }, { 'query': 'SELECT {one} as f1, {nine}+1 as f2;', 'csv': 'one_ten' }, { 'from': 'csv', 'query': "SELECT 'one_nine_ten'", 'var': 'ONENINETEN' }, { 'query': 'SELECT f1, {nine} as f9, (SELECT f2 FROM \ one_ten) as f10 FROM one_ten;', 'csvlist': ['one_ten'], 'csv': '{ONENINETEN}', 'from': 'csv' }, { 'batch': [{ 'line': 'SELECT {PARAM}; => var:foo', 'query': 'SELECT {PARAM};', 'var': 'foo', 'from': 'csv' }, { 'batch': [{ 'line': 'SELECT {NESTED}; => var:foo2', 'query': 'SELECT {NESTED};', 'var': 'foo2', 'from': 'csv' }], 'batch_begin': ('i', 'NESTED'), 'csvlist': ['ints10000'], 'from': 'csv', 'line': 'SELECT i from csv.ints10000 WHERE i>=CAST(10 as INTEGER) LIMIT 2; => batch_begin:i:NESTED', 'query': 'SELECT i from ints10000 WHERE i>=CAST(10 as INTEGER) LIMIT 2;' }, { 'line': "SELECT 'something'; => var:static_var", 'query': "SELECT 'something';", 'var': 'static_var', 'from': 'csv' }], 'batch_begin': ('i', 'PARAM'), 'csvlist': ['ints10000'], 'from': 'csv', 'query': 'SELECT i from ints10000 WHERE i>=2 LIMIT 2;' }, { 'from': 'csv', 'query': 'SELECT {PARAM};', 'var': 'final_test' }] job_syntax_extended = JobSyntaxExtended(lines, {'macro_test_batch': macro_lines}) assert_job_syntax_lines(job_syntax_extended.items(), expected) try: os.remove(SqlExecutor.csv_name('one_nine_ten')) except: pass with open(config_filename) as conf_file: job_controller = JobController(conf_file.name, endpoint_names, job_syntax_extended, {}, False) job_controller.run_job() res_batch_params = job_controller.variables[BATCH_PARAMS_KEY] assert res_batch_params == ['2', '3'] sfvar = job_controller.variables['sfvar'] assert len(sfvar) >= 15 final_param = job_controller.variables['final_test'] assert final_param == '3' del job_controller with open(SqlExecutor.csv_name('one_nine_ten')) as resulted_file: assert resulted_file.read() == 'f1,f9,f10\n1,9,10\n'
def test_job_syntax(): setdatadir(tempfile.mkdtemp()) loginit(__name__) lines = [ '--something', #this is a comment line, will not be added to parsed values 'SELECT 1 => csv:const1', 'SELECT 1 => var:MIN', 'SELECT f1, (SELECT f2 FROM csv.one_ten) as f10 FROM \ csv.one_ten, 9; => csv:final:cache => dst:insert:foo:1:res => type:sequential', 'SELECT 1 as bacth1 from csv.some_csv; \ => batch_begin:batch1:BATCH', 'SELECT 1 from dst.some_object WHERE b=a \ => csv:some_csv => batch_end:BATCH', '=> batch_end:BATCH', 'SELECT 1 as test, 2 as test2; => csv:foo:cache \ => dst:insert:test_table:1:new_ids', 'SELECT 1 as test, 2 as test2; => csv:foo \ => dst:insert:test_table:1:res' ] expected = [{ 'query': 'SELECT 1', 'csv': 'const1' }, { 'from': 'csv', 'query': 'SELECT 1', 'var': 'MIN' }, { 'query': 'SELECT f1, (SELECT f2 FROM one_ten) as f10 FROM one_ten, 9;', 'csv': 'final', 'from': 'csv', 'dst': 'foo', 'op': 'insert', 'type': 'sequential', 'cache': '', 'csvlist': ['one_ten'], 'batch_size': '1', 'new_ids_table': 'res' }, { 'query': 'SELECT 1 as bacth1 from some_csv;', 'batch_begin': ('batch1', 'BATCH'), 'from': 'csv', 'csvlist': ['some_csv'] }, { 'query': 'SELECT 1 from some_object WHERE b=a', 'csv': 'some_csv', 'from': 'dst', 'objname': 'some_object', 'batch_end': 'BATCH' }, { 'query': '', 'batch_end': 'BATCH' }, { 'query': 'SELECT 1 as test, 2 as test2;', 'op': 'insert', 'dst': 'test_table', 'csv': 'foo', 'cache': '', 'new_ids_table': 'new_ids', 'batch_size': '1' }, { 'query': 'SELECT 1 as test, 2 as test2;', 'csv': 'foo', 'op': 'insert', 'dst': 'test_table', 'batch_size': '1', 'new_ids_table': 'res' }] job_syntax = JobSyntax(lines) assert_job_syntax_lines(job_syntax.items(), expected)
id_idx = csv_data.fields.index('Id') try: assert 1 == len(csv_data.rows) except: print "len(csv_data.rows)", len(csv_data.rows) raise for row in csv_data.rows: assert len(row[id_idx]) >= 15 assert open(SqlExecutor.csv_name('test_csv')).read() \ == open(SqlExecutor.csv_name('test_csv_2')).read() # run another test using created data run_test_graph(datadir(), "tests/sql/complicated.sql") def test_batch_splitter(): loginit(__name__) batch_ranges = SfBulkConnector.batch_ranges(10, 3) print batch_ranges assert (batch_ranges == [(0, 2), (3, 5), (6, 8), (9, 9)]) if __name__ == '__main__': loginit(__name__) # test_batch_splitter() # test_columns() # test_read() # test_job_syntax() test_var_csv() # test_job_controller()
for input_param in args.job_file: for input_file in input_param: input_files.append(input_file) if not input_files: print "Arguments error: Job file/s not specified" parser.print_help() exit(1) # Get logfilenae config = ConfigParser() config.read_file(args.conf_file) if args.logdir: logdirname = args.logdir else: logdirname = config[DEFAULT_SETTINGS_SECTION][LOGDIR_SETTING] try: os.makedirs(logdirname) except OSError, e: if e.errno != errno.EEXIST: raise loginit(STDOUT) loginit(STDERR) loginit(LOG, os.path.join(logdirname, LOG)) getLogger(STDOUT).info("Started logging into %s", LOG) getLogger(STDOUT).info('Prepare graph for %s' % [x.name for x in input_files]) print_graph(args.conf_file, input_files, variables, args.save_graph, args.format, args.csvdir)
logdirname = args.logdir else: logdirname = config[DEFAULT_SETTINGS_SECTION][LOGDIR_SETTING] if args.datadir: datadirname = args.datadir else: datadirname = config[DEFAULT_SETTINGS_SECTION][DATADIR_SETTING] # update data path sql_executor.setdatadir(datadirname) try: os.makedirs(datadirname) except OSError, e: if e.errno != errno.EEXIST: raise loginit(STDOUT) loginit(STDERR) for input_file in jobs: getLogger(STDOUT).info('Starting %s' % input_file.name) # prepare log path logpath = os.path.join(logdirname, os.path.basename(input_file.name).split('.')[0]) try: os.makedirs(logdirname) except OSError, e: if e.errno != errno.EEXIST: raise loginit(LOG, logpath + '.log') if args.moreinfo: