def test_mixed_images_when_multiple_output(self): path1 = test_utils.create_file('test123.png') path2 = test_utils.create_file('images/test.png') path3 = test_utils.create_file('a.b.c.png') path4 = test_utils.create_file('test456.png') path5 = test_utils.create_file('some/long/path/me.jpg') config = create_config_model( 'my_script', output_files=[ inline_image(test_utils.temp_folder + os_utils.path_sep() + '#test\d+.png#'), inline_image(path2), inline_image(path3), inline_image('##any_path/path/\w+#.jpg') ]) execution_id = self.start_execution(config) paths = [ normalize_path(p) for p in (path1, path2, path3, path4, path5) ] for index, path in enumerate(paths): self.write_output(execution_id, '__ ' + path + ' __\n') self.wait_output_chunks(execution_id, chunks_count=index + 1) self.write_output(execution_id, '__ ' + path2 + ' __\n') self.wait_output_chunks(execution_id, chunks_count=len(paths) + 1) self.assert_images(*paths)
def test_single_asterisk_1_match(self): test_utils.create_file('test.txt') files = file_download_feature.find_matching_files('*/test.txt', None) self.assertEqual(files, [os.path.join(test_utils.temp_folder, 'test.txt')])
def test_server_file_when_wrong(self): test_utils.create_file('file1.txt') parameter = create_parameter_model('param', type=PARAM_TYPE_SERVER_FILE, file_dir=test_utils.temp_folder) error = parameter.validate_value('my.dat') self.assert_error(error)
def test_server_file_when_valid(self): filename = 'file1.txt' test_utils.create_file(filename) parameter = create_parameter_model('param', type=PARAM_TYPE_SERVER_FILE, file_dir=test_utils.temp_folder) error = parameter.validate_value(filename) self.assertIsNone(error)
def test_server_file_when_valid(self): filename = 'file1.txt' test_utils.create_file(filename) parameter = create_parameter_model('param', type=PARAM_TYPE_SERVER_FILE, file_dir=test_utils.temp_folder) error = parameter.validate_value(filename) self.assertIsNone(error)
def test_file_extensions(self): for extension in ['exe', 'dat', 'txt', 'sh', 'pdf', 'docx']: for file in ['file1', 'file2']: test_utils.create_file(file + '.' + extension) test_utils.create_dir('my_dir' + '.' + extension) files = model_helper.list_files(test_utils.temp_folder, file_extensions=['exe', 'pdf']) self.assertEqual(['file1.exe', 'file1.pdf', 'file2.exe', 'file2.pdf'], files)
def test_1_regex_and_asterisk(self): test_utils.create_file(os.path.join('some_folder', 'file.txt')) files = file_download_feature.find_matching_files('*/#1#folder=(\w+)#/*.txt', 'username=some_name\n ' 'folder=some_folder\n ' 'time=153514\n ' 'age=18, size=256Mb') self.assertEqual(files, [os.path.join(test_utils.temp_folder, 'some_folder', 'file.txt')])
def test_full_path_command(self): test_utils.create_file('test.sh') command_split = process_utils.split_command('test.sh', test_utils.temp_folder) self.assertEqual( command_split, [os.path.abspath(os.path.join(test_utils.temp_folder, 'test.sh'))])
def setUp(self): self.tmp_root = tempfile.mkdtemp() self.mappings_file = os.path.join(self.tmp_root, "targets.txt") test_utils.create_file(self.mappings_file, get_default_mapping_content(3)) self.expected = [ differential_expr.Mapping("P1234_101", "smpl1_1", "smpl1"), differential_expr.Mapping("P1234_102", "smpl1_2", "smpl1"), differential_expr.Mapping("P1234_103", "smpl1_3", "smpl1"), ]
def test_1_regex_and_asterisk(self): test_utils.create_file(os.path.join('some_folder', 'file.txt')) files = file_download_feature.find_matching_files('*/#1#folder=(\w+)#/*.txt', 'username=some_name\n ' 'folder=some_folder\n ' 'time=153514\n ' 'age=18, size=256Mb') self.assertEqual(files, [os.path.join(test_utils.temp_folder, 'some_folder', 'file.txt')])
def test_multiple_files_non_recursive(self): for dir in [None, 'documents', 'smth']: for file in ['my.txt', 'file.dat']: if dir: test_utils.create_file(os.path.join(dir, dir + '_' + file)) else: test_utils.create_file(file) self.assertEqual(['documents', 'file.dat', 'my.txt', 'smth'], self.read_provider_values())
def test_multiple_files_non_recursive(self): for dir in [None, 'documents', 'smth']: for file in ['my.txt', 'file.dat']: if dir: test_utils.create_file(os.path.join(dir, dir + '_' + file)) else: test_utils.create_file(file) files = model_helper.list_files(test_utils.temp_folder) self.assertEqual(['documents', 'file.dat', 'my.txt', 'smth'], files)
def test_single_asterisk_2_matches(self): test_utils.create_file('test1.txt') test_utils.create_file('test2.txt') files = file_download_feature.find_matching_files('*/test*.txt', None) self.assertCountEqual(files, [ os.path.join(test_utils.temp_folder, 'test1.txt'), os.path.join(test_utils.temp_folder, 'test2.txt') ])
def test_multiple_files_non_recursive(self): for dir in [None, 'documents', 'smth']: for file in ['my.txt', 'file.dat']: if dir: test_utils.create_file(os.path.join(dir, dir + '_' + file)) else: test_utils.create_file(file) files = model_helper.list_files(test_utils.temp_folder) self.assertEqual(['documents', 'file.dat', 'my.txt', 'smth'], files)
def test_output_files_with_parameter_substitution(self): file1 = test_utils.create_file('file1.txt', text='hello world') file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123') downloadable_files = self.perform_execution([ os.path.join(test_utils.temp_folder, '${p1}.txt'), os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')], parameter_values={'p1': 'file1', 'p2': 'child'}) self.assert_downloadable_files(downloadable_files, [file1, file2])
def test_double_asterisk_match(self): test_utils.create_file(os.path.join('test', 'test.txt')) files = set(file_download_feature.find_matching_files(test_utils.temp_folder + '/**', None)) self.assertCountEqual(files, { os.path.join(test_utils.temp_folder, ''), os.path.join(test_utils.temp_folder, 'test'), os.path.join(test_utils.temp_folder, 'test', 'test.txt') })
def test_double_asterisk_match(self): test_utils.create_file(os.path.join('test', 'test.txt')) files = set(file_download_feature.find_matching_files(test_utils.temp_folder + '/**', None)) self.assertCountEqual(files, { os.path.join(test_utils.temp_folder, ''), os.path.join(test_utils.temp_folder, 'test'), os.path.join(test_utils.temp_folder, 'test', 'test.txt') })
def test_output_files_with_parameter_substitution(self): file1 = test_utils.create_file('file1.txt', text='hello world') file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123') downloadable_files = self.perform_execution([ os.path.join(test_utils.temp_folder, '${p1}.txt'), os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')], parameter_values={'p1': 'file1', 'p2': 'child'}) self.assert_downloadable_files(downloadable_files, [file1, file2])
def test_file_extensions(self): for extension in ['exe', 'dat', 'txt', 'sh', 'pdf', 'docx']: for file in ['file1', 'file2']: test_utils.create_file(file + '.' + extension) test_utils.create_dir('my_dir' + '.' + extension) self.assertEqual( ['file1.exe', 'file1.pdf', 'file2.exe', 'file2.pdf'], self.read_provider_values(file_extensions=['exe', '.pdf']))
def test_file_extensions(self): for extension in ['exe', 'dat', 'txt', 'sh', 'pdf', 'docx']: for file in ['file1', 'file2']: test_utils.create_file(file + '.' + extension) test_utils.create_dir('my_dir' + '.' + extension) files = model_helper.list_files(test_utils.temp_folder, file_extensions=['exe', 'pdf']) self.assertEqual(['file1.exe', 'file1.pdf', 'file2.exe', 'file2.pdf'], files)
def test_single_dynamic_image_when_unnormalized(self): test_utils.create_file('sub/test.png') config = create_config_model('my_script', output_files=[inline_image('#([\.\w]+/)+\w+.png#')]) execution_id = self.start_execution(config) unnormalized_path = os.path.join(test_utils.temp_folder, '.', 'sub', '..', 'sub', 'test.png') self.write_output(execution_id, '_ ' + unnormalized_path + ' _\n') self.wait_output_chunks(execution_id, chunks_count=1) image_keys = [img[0] for img in self.images] self.assertEqual([unnormalized_path], image_keys)
def setUp(self): super().setUp() self.socket = None application = tornado.web.Application( [(r'/scripts/([^/]*)', ScriptConfigSocket)], login_url='/login.html', cookie_secret='12345') application.auth = TornadoAuth(None) application.authorizer = Authorizer(ANY_USER, [], [], EmptyGroupProvider()) application.identification = IpBasedIdentification( TrustedIpValidator(['127.0.0.1']), None) application.config_service = ConfigService(application.authorizer, test_utils.temp_folder) server = httpserver.HTTPServer(application) socket, self.port = testing.bind_unused_port() server.add_socket(socket) test_utils.setup() for dir in ['x', 'y', 'z']: for file in range(1, 4): filename = dir + str(file) + '.txt' test_utils.create_file( os.path.join('test1_files', dir, filename)) test1_files_path = os.path.join(test_utils.temp_folder, 'test1_files') test_utils.write_script_config( { 'name': 'Test script 1', 'script_path': 'ls', 'parameters': [ test_utils.create_script_param_config('text 1', required=True), test_utils.create_script_param_config( 'list 1', type='list', allowed_values=['A', 'B', 'C']), test_utils.create_script_param_config( 'file 1', type='server_file', file_dir=test1_files_path), test_utils.create_script_param_config( 'list 2', type='list', values_script='ls ' + test1_files_path + '/${file 1}') ] }, 'test_script_1')
def test_output_files_with_secure_parameters(self): test_utils.create_file('file1.txt', text='hello world') file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123') param1 = create_script_param_config('p1', secure=True) param2 = create_script_param_config('p2') downloadable_files = self.perform_execution([ os.path.join(test_utils.temp_folder, '${p1}.txt'), os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')], parameter_values={'p1': 'file1', 'p2': 'child'}, parameters=[param1, param2]) self.assert_downloadable_files(downloadable_files, [file2])
def test_multiple_static_images(self): path1 = test_utils.create_file('test1.png') path2 = test_utils.create_file('test2.png') path3 = test_utils.create_file('test3.png') config = create_config_model('my_script', output_files=[inline_image(path1), inline_image(path2), inline_image(path3)]) execution_id = self.start_execution(config) self.write_output(execution_id, '123\n' + '456') self.wait_output_chunks(execution_id, chunks_count=1) self.assert_images(path1, path2, path3)
def test_output_files_with_secure_parameters(self): test_utils.create_file('file1.txt', text='hello world') file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123') param1 = create_script_param_config('p1', secure=True) param2 = create_script_param_config('p2') downloadable_files = self.perform_execution([ os.path.join(test_utils.temp_folder, '${p1}.txt'), os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')], parameter_values={'p1': 'file1', 'p2': 'child'}, parameters=[param1, param2]) self.assert_downloadable_files(downloadable_files, [file2])
def test_allow_not_quoted_file_with_whitespaces_win(self): test_utils.set_win() file = test_utils.create_file('my script.py') command_split = process_utils.split_command('my script.py', test_utils.temp_folder) self.assertEqual(command_split, [os.path.abspath(file)])
def test_restore_dump_state_when_multiple_users(self): dump_file = test_utils.create_file('dump.json', text=json.dumps([{ 'username': '******', 'groups': ['group1', 'group2'], 'last_auth_update': 123 }, { 'groups': ['group3'], 'last_auth_update': 999 }, { 'username': '******', 'last_visit': 456 }])) authenticator = create_test_authenticator(dump_file=dump_file) self.assertEqual({'user_X', 'user_Y'}, authenticator._users.keys()) user_x_state = authenticator._users['user_X'] self.assertEqual('user_X', user_x_state.username) self.assertEqual(['group1', 'group2'], user_x_state.groups) self.assertEqual(123, user_x_state.last_auth_update) self.assertEqual(None, user_x_state.last_visit) user_y_state = authenticator._users['user_Y'] self.assertEqual('user_Y', user_y_state.username) self.assertEqual([], user_y_state.groups) self.assertEqual(None, user_y_state.last_auth_update) self.assertEqual(456, user_y_state.last_visit)
def test_find_multiple_images_by_same_pattern(self): path1 = test_utils.create_file('test123.png') test_utils.create_file('images/test.png') path3 = test_utils.create_file('a.b.c.png') path4 = test_utils.create_file('some/sub/folder/test456.png') config = create_config_model( 'my_script', output_files=[inline_image('##any_path.png#')]) execution_id = self.start_execution(config) paths = [normalize_path(p) for p in (path1, path3, path4)] for index, path in enumerate(paths): self.write_output(execution_id, '__ ' + path + ' __\n') self.wait_output_chunks(execution_id, chunks_count=index + 1) self.assert_images(*paths)
def test_file_upload_parameter_when_not_exists(self): parameter = self.create_parameter('param') parameter.type = 'file_upload' uploaded_file = test_utils.create_file('test.xml') error = model_helper.validate_parameter(parameter, {'param': uploaded_file + '_'}) self.assert_error(error)
def test_many_unicode_characters(self): long_unicode_text = ('ΩΨΔ\n' * 100000) test_utils.create_file('test.txt', text=long_unicode_text) process_wrapper = PtyProcessWrapper(['cat', 'test.txt'], test_utils.temp_folder, {}) process_wrapper.start() thread = threading.Thread(target=process_wrapper.wait_finish, daemon=True) thread.start() thread.join(timeout=0.1) output = ''.join(read_until_closed(process_wrapper.output_stream)) self.assertEqual(long_unicode_text, output) self.assertEqual(0, process_wrapper.get_return_code())
def test_file_upload_parameter_when_valid(self): parameter = self.create_parameter('param') parameter.type = 'file_upload' uploaded_file = test_utils.create_file('test.xml') error = model_helper.validate_parameter(parameter, {'param': uploaded_file}) self.assertIsNone(error)
def test_htpasswd_auth(self): file = test_utils.create_file('some-path', text='user1:1yL79Q78yczsM') config = _from_json({'auth': {'type': 'htpasswd', 'htpasswd_path': file}}) self.assertIsInstance(config.authenticator, HtpasswdAuthenticator) authenticated = config.authenticator.verifier.verify('user1', 'aaa') self.assertTrue(authenticated)
def test_file_upload_parameter_when_not_exists(self): script_config = script_configs.Config() parameter = self.add_parameter('param', script_config) parameter.type = 'file_upload' uploaded_file = test_utils.create_file('test.xml') valid = model_helper.validate_parameters({'param': uploaded_file + '_'}, script_config) self.assertFalse(valid)
def test_single_static_image(self): path = test_utils.create_file('test.png') config = create_config_model('my_script', output_files=[inline_image(path)]) execution_id = self.start_execution(config) self.write_output(execution_id, '123\n456') self.wait_output_chunks(execution_id, chunks_count=1) self.assert_images(path)
def test_single_dynamic_image(self): path = test_utils.create_file('test.png') config = create_config_model('my_script', output_files=[inline_image('##any_path.png#')]) execution_id = self.start_execution(config) full_path = file_utils.normalize_path(path) self.write_output(execution_id, '123\n' + full_path + '\n456') self.wait_output_chunks(execution_id, chunks_count=1) self.assert_images(full_path)
def setUp(self): self.tmp_root = tempfile.mkdtemp() dir1 = os.path.join(self.tmp_root, "1") dir2 = os.path.join(self.tmp_root, "2") os.makedirs(dir1) os.makedirs(dir2) content = get_count_file_content() test_utils.create_file(os.path.join(dir1, "counts_P1234_101.txt"), content) test_utils.create_file(os.path.join(dir2, "counts_P1234_102.txt"), content) self.expected = { 'ABC123X': { 'P1234_101': 4, 'P1234_102': 4, 'length': 3 }, 'ABC123Y': { 'P1234_101': 5, 'P1234_102': 5, 'length': 4 } }
def test_image_path_split_in_chunks(self): path = test_utils.create_file('test123.png') config = create_config_model('my_script', output_files=[inline_image('##any_path.png#')]) execution_id = self.start_execution(config) normalized = normalize_path(path) self.write_output(execution_id, normalized[:4]) self.wait_output_chunks(execution_id, chunks_count=1) self.write_output(execution_id, normalized[4:] + '\n') self.wait_output_chunks(execution_id, chunks_count=2) self.assert_images(path)
def test_update_script_config(self): self.start_server(12345, '127.0.0.1') xsrf_token = self.get_xsrf_token(self._admin_session) script_path = test_utils.create_file('my_script.py') test_utils.write_script_config( { 'name': 's1', 'script_path': script_path }, 's1', self.runners_folder) response = self._admin_session.put( 'http://127.0.0.1:12345/admin/scripts', data={ 'filename': 's1.json', 'config': json.dumps({ 'name': 'new name', 'script': { 'mode': 'new_code', 'path': script_path, 'code': 'abcdef' } }) }, headers={'X-XSRFToken': xsrf_token}, ) self.assertEqual(200, response.status_code) conf_response = self.request( 'get', 'http://127.0.0.1:12345/admin/scripts/new%20name', self._admin_session) self.assertEqual( { 'config': { 'name': 'new name', 'script_path': script_path }, 'filename': 's1.json' }, conf_response) script_content = file_utils.read_file(script_path) self.assertEqual('abcdef', script_content)
def test_get_script_code(self): self.start_server(12345, '127.0.0.1') script_path = test_utils.create_file('my_script.py') test_utils.write_script_config( { 'name': 's1', 'script_path': script_path }, 's1', self.runners_folder) response = self.request( 'get', 'http://127.0.0.1:12345/admin/scripts/s1/code', self._admin_session) self.assertEqual( { 'code': 'test text', 'file_path': os.path.abspath(script_path) }, response)
def test_image_path_split_in_chunks_and_no_newlines(self): path = test_utils.create_file('test123.png') config = create_config_model('my_script', output_files=[inline_image('##any_path.png#')]) execution_id = self.start_execution(config) normalized = normalize_path(path) self.write_output(execution_id, normalized[:4]) self.wait_output_chunks(execution_id, chunks_count=1) self.write_output(execution_id, normalized[4:]) self.wait_output_chunks(execution_id, chunks_count=2) self.executor_service.get_active_executor(execution_id, DEFAULT_USER).process_wrapper.stop() self.wait_close(execution_id) self.assert_images(path)
def test_prepare_exe(self): script_path = test_utils.create_file('test.exe') (command, shell) = prepare_cmd_for_win([script_path]) self.assertEqual([script_path], command) self.assertFalse(shell)
def test_full_path_command(self): file = test_utils.create_file('test.sh') command_split = process_utils.split_command('test.sh', test_utils.temp_folder) self.assertEqual(command_split, [os.path.abspath(file)])
def test_prepare_simple_py(self): script_path = test_utils.create_file('test.py') (command, shell) = prepare_cmd_for_win([script_path]) self.assertEqual([script_path], command) self.assertTrue(shell)
def test_prepare_py_with_ampersand_short_command(self): script_path = test_utils.create_file('test.py') (command, shell) = prepare_cmd_for_win([script_path, '&ping&']) self.assertEqual([script_path, '^&ping^&'], command) self.assertTrue(shell)
def test_prepare_py_with_ampersand_another_command(self): script_path = test_utils.create_file('test.py') (command, shell) = prepare_cmd_for_win([script_path, '"&& dir c:"']) self.assertEqual([script_path, '"^&^& dir c:"'], command) self.assertTrue(shell)
def test_single_file(self): test_utils.create_file('my.txt') files = model_helper.list_files(test_utils.temp_folder) self.assertEqual(['my.txt'], files)
def test_file_upload_parameter_when_valid(self): parameter = create_parameter_model('param', type='file_upload') uploaded_file = test_utils.create_file('test.xml') error = parameter.validate_value(uploaded_file) self.assertIsNone(error)
def test_server_file_when_wrong(self): test_utils.create_file('file1.txt') parameter = create_parameter_model('param', type=PARAM_TYPE_SERVER_FILE, file_dir=test_utils.temp_folder) error = parameter.validate_value('my.dat') self.assert_error(error)
def test_no_output_files_in_config(self): test_utils.create_file('file1.txt') downloadable_files = self.perform_execution(None) self.assertEqual([], downloadable_files)
def test_prepare_file_on_finish(self): file1_path = test_utils.create_file('file1.txt') downloadable_files = self.perform_execution([file1_path]) self.assert_downloadable_files(downloadable_files, [file1_path])
def test_path_when_exists_and_working_dir(self): file = test_utils.create_file('./scriptX.sh') command_split = process_utils.split_command('./scriptX.sh 123', test_utils.temp_folder) self.assertEqual([os.path.abspath(file), '123'], command_split)
def test_path_when_working_dir_and_abs_path(self): file = test_utils.create_file('scriptX.sh') abs_file = os.path.abspath(file) command_split = process_utils.split_command(abs_file + ' 123', 'my_dir') self.assertEqual([abs_file, '123'], command_split)
def test_double_asterisk_match_multiple_files_when_complex(self): test_utils.create_file(os.path.join('f1', 'test1.txt')) test_utils.create_file(os.path.join('f1', 'test2.txt')) test_utils.create_file(os.path.join('d2', 'test3.txt')) test_utils.create_file(os.path.join('d2', 'd3', 'test4.txt')) test_utils.create_file(os.path.join('d3', 'd4', 'd5', 'test5.png')) test_utils.create_file(os.path.join('d3', 'd6', 'd7', 'test6.txt')) temp_folder = file_utils.normalize_path(test_utils.temp_folder) files = set(file_download_feature.find_matching_files(temp_folder + '/d*/**/*.txt', None)) self.assertCountEqual(files, { os.path.join(temp_folder, 'd2', 'test3.txt'), os.path.join(temp_folder, 'd2', 'd3', 'test4.txt'), os.path.join(temp_folder, 'd3', 'd6', 'd7', 'test6.txt') })
def setUp(self): self.tmp_root = tempfile.mkdtemp() self.mappings_file = os.path.join(self.tmp_root, "targets.txt") content = get_default_mapping_content(1) + get_default_mapping_content(1) print(content) test_utils.create_file(self.mappings_file, content)
def test_file_upload_parameter_when_not_exists(self): parameter = create_parameter_model('param', type='file_upload') uploaded_file = test_utils.create_file('test.xml') error = parameter.validate_value(uploaded_file + '_') self.assert_error(error)