def test_mixed_images_when_multiple_output(self):
path1 = test_utils.create_file('test123.png')
path2 = test_utils.create_file('images/test.png')
path3 = test_utils.create_file('a.b.c.png')
path4 = test_utils.create_file('test456.png')
path5 = test_utils.create_file('some/long/path/me.jpg')
config = create_config_model(
'my_script',
output_files=[
inline_image(test_utils.temp_folder + os_utils.path_sep() +
'#test\d+.png#'),
inline_image(path2),
inline_image(path3),
inline_image('##any_path/path/\w+#.jpg')
])
execution_id = self.start_execution(config)
paths = [
normalize_path(p) for p in (path1, path2, path3, path4, path5)
]
for index, path in enumerate(paths):
self.write_output(execution_id, '__ ' + path + ' __\n')
self.wait_output_chunks(execution_id, chunks_count=index + 1)
self.write_output(execution_id, '__ ' + path2 + ' __\n')
self.wait_output_chunks(execution_id, chunks_count=len(paths) + 1)
self.assert_images(*paths)
def test_single_asterisk_1_match(self):
test_utils.create_file('test.txt')
files = file_download_feature.find_matching_files('*/test.txt', None)
self.assertEqual(files,
[os.path.join(test_utils.temp_folder, 'test.txt')])
def test_server_file_when_wrong(self):
test_utils.create_file('file1.txt')
parameter = create_parameter_model('param',
type=PARAM_TYPE_SERVER_FILE,
file_dir=test_utils.temp_folder)
error = parameter.validate_value('my.dat')
self.assert_error(error)
def test_server_file_when_valid(self):
filename = 'file1.txt'
test_utils.create_file(filename)
parameter = create_parameter_model('param', type=PARAM_TYPE_SERVER_FILE, file_dir=test_utils.temp_folder)
error = parameter.validate_value(filename)
self.assertIsNone(error)
def test_server_file_when_valid(self):
filename = 'file1.txt'
test_utils.create_file(filename)
parameter = create_parameter_model('param', type=PARAM_TYPE_SERVER_FILE, file_dir=test_utils.temp_folder)
error = parameter.validate_value(filename)
self.assertIsNone(error)
def test_file_extensions(self):
for extension in ['exe', 'dat', 'txt', 'sh', 'pdf', 'docx']:
for file in ['file1', 'file2']:
test_utils.create_file(file + '.' + extension)
test_utils.create_dir('my_dir' + '.' + extension)
files = model_helper.list_files(test_utils.temp_folder, file_extensions=['exe', 'pdf'])
self.assertEqual(['file1.exe', 'file1.pdf', 'file2.exe', 'file2.pdf'], files)
def test_1_regex_and_asterisk(self):
test_utils.create_file(os.path.join('some_folder', 'file.txt'))
files = file_download_feature.find_matching_files('*/#1#folder=(\w+)#/*.txt', 'username=some_name\n '
'folder=some_folder\n '
'time=153514\n '
'age=18, size=256Mb')
self.assertEqual(files, [os.path.join(test_utils.temp_folder, 'some_folder', 'file.txt')])
def test_full_path_command(self):
test_utils.create_file('test.sh')
command_split = process_utils.split_command('test.sh',
test_utils.temp_folder)
self.assertEqual(
command_split,
[os.path.abspath(os.path.join(test_utils.temp_folder, 'test.sh'))])
def setUp(self):
self.tmp_root = tempfile.mkdtemp()
self.mappings_file = os.path.join(self.tmp_root, "targets.txt")
test_utils.create_file(self.mappings_file, get_default_mapping_content(3))
self.expected = [
differential_expr.Mapping("P1234_101", "smpl1_1", "smpl1"),
differential_expr.Mapping("P1234_102", "smpl1_2", "smpl1"),
differential_expr.Mapping("P1234_103", "smpl1_3", "smpl1"),
]
def test_1_regex_and_asterisk(self):
test_utils.create_file(os.path.join('some_folder', 'file.txt'))
files = file_download_feature.find_matching_files('*/#1#folder=(\w+)#/*.txt', 'username=some_name\n '
'folder=some_folder\n '
'time=153514\n '
'age=18, size=256Mb')
self.assertEqual(files, [os.path.join(test_utils.temp_folder, 'some_folder', 'file.txt')])
def test_multiple_files_non_recursive(self):
for dir in [None, 'documents', 'smth']:
for file in ['my.txt', 'file.dat']:
if dir:
test_utils.create_file(os.path.join(dir, dir + '_' + file))
else:
test_utils.create_file(file)
self.assertEqual(['documents', 'file.dat', 'my.txt', 'smth'],
self.read_provider_values())
def test_multiple_files_non_recursive(self):
for dir in [None, 'documents', 'smth']:
for file in ['my.txt', 'file.dat']:
if dir:
test_utils.create_file(os.path.join(dir, dir + '_' + file))
else:
test_utils.create_file(file)
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['documents', 'file.dat', 'my.txt', 'smth'], files)
def test_single_asterisk_2_matches(self):
test_utils.create_file('test1.txt')
test_utils.create_file('test2.txt')
files = file_download_feature.find_matching_files('*/test*.txt', None)
self.assertCountEqual(files, [
os.path.join(test_utils.temp_folder, 'test1.txt'),
os.path.join(test_utils.temp_folder, 'test2.txt')
])
def test_multiple_files_non_recursive(self):
for dir in [None, 'documents', 'smth']:
for file in ['my.txt', 'file.dat']:
if dir:
test_utils.create_file(os.path.join(dir, dir + '_' + file))
else:
test_utils.create_file(file)
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['documents', 'file.dat', 'my.txt', 'smth'], files)
def test_output_files_with_parameter_substitution(self):
file1 = test_utils.create_file('file1.txt', text='hello world')
file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123')
downloadable_files = self.perform_execution([
os.path.join(test_utils.temp_folder, '${p1}.txt'),
os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')],
parameter_values={'p1': 'file1', 'p2': 'child'})
self.assert_downloadable_files(downloadable_files, [file1, file2])
def test_double_asterisk_match(self):
test_utils.create_file(os.path.join('test', 'test.txt'))
files = set(file_download_feature.find_matching_files(test_utils.temp_folder + '/**', None))
self.assertCountEqual(files, {
os.path.join(test_utils.temp_folder, ''),
os.path.join(test_utils.temp_folder, 'test'),
os.path.join(test_utils.temp_folder, 'test', 'test.txt')
})
def test_double_asterisk_match(self):
test_utils.create_file(os.path.join('test', 'test.txt'))
files = set(file_download_feature.find_matching_files(test_utils.temp_folder + '/**', None))
self.assertCountEqual(files, {
os.path.join(test_utils.temp_folder, ''),
os.path.join(test_utils.temp_folder, 'test'),
os.path.join(test_utils.temp_folder, 'test', 'test.txt')
})
def test_output_files_with_parameter_substitution(self):
file1 = test_utils.create_file('file1.txt', text='hello world')
file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123')
downloadable_files = self.perform_execution([
os.path.join(test_utils.temp_folder, '${p1}.txt'),
os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')],
parameter_values={'p1': 'file1', 'p2': 'child'})
self.assert_downloadable_files(downloadable_files, [file1, file2])
def test_file_extensions(self):
for extension in ['exe', 'dat', 'txt', 'sh', 'pdf', 'docx']:
for file in ['file1', 'file2']:
test_utils.create_file(file + '.' + extension)
test_utils.create_dir('my_dir' + '.' + extension)
self.assertEqual(
['file1.exe', 'file1.pdf', 'file2.exe', 'file2.pdf'],
self.read_provider_values(file_extensions=['exe', '.pdf']))
def test_file_extensions(self):
for extension in ['exe', 'dat', 'txt', 'sh', 'pdf', 'docx']:
for file in ['file1', 'file2']:
test_utils.create_file(file + '.' + extension)
test_utils.create_dir('my_dir' + '.' + extension)
files = model_helper.list_files(test_utils.temp_folder,
file_extensions=['exe', 'pdf'])
self.assertEqual(['file1.exe', 'file1.pdf', 'file2.exe', 'file2.pdf'],
files)
def test_single_dynamic_image_when_unnormalized(self):
test_utils.create_file('sub/test.png')
config = create_config_model('my_script', output_files=[inline_image('#([\.\w]+/)+\w+.png#')])
execution_id = self.start_execution(config)
unnormalized_path = os.path.join(test_utils.temp_folder, '.', 'sub', '..', 'sub', 'test.png')
self.write_output(execution_id, '_ ' + unnormalized_path + ' _\n')
self.wait_output_chunks(execution_id, chunks_count=1)
image_keys = [img[0] for img in self.images]
self.assertEqual([unnormalized_path], image_keys)
def setUp(self):
super().setUp()
self.socket = None
application = tornado.web.Application(
[(r'/scripts/([^/]*)', ScriptConfigSocket)],
login_url='/login.html',
cookie_secret='12345')
application.auth = TornadoAuth(None)
application.authorizer = Authorizer(ANY_USER, [], [],
EmptyGroupProvider())
application.identification = IpBasedIdentification(
TrustedIpValidator(['127.0.0.1']), None)
application.config_service = ConfigService(application.authorizer,
test_utils.temp_folder)
server = httpserver.HTTPServer(application)
socket, self.port = testing.bind_unused_port()
server.add_socket(socket)
test_utils.setup()
for dir in ['x', 'y', 'z']:
for file in range(1, 4):
filename = dir + str(file) + '.txt'
test_utils.create_file(
os.path.join('test1_files', dir, filename))
test1_files_path = os.path.join(test_utils.temp_folder, 'test1_files')
test_utils.write_script_config(
{
'name':
'Test script 1',
'script_path':
'ls',
'parameters': [
test_utils.create_script_param_config('text 1',
required=True),
test_utils.create_script_param_config(
'list 1', type='list', allowed_values=['A', 'B', 'C']),
test_utils.create_script_param_config(
'file 1',
type='server_file',
file_dir=test1_files_path),
test_utils.create_script_param_config(
'list 2',
type='list',
values_script='ls ' + test1_files_path + '/${file 1}')
]
}, 'test_script_1')
def test_output_files_with_secure_parameters(self):
test_utils.create_file('file1.txt', text='hello world')
file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123')
param1 = create_script_param_config('p1', secure=True)
param2 = create_script_param_config('p2')
downloadable_files = self.perform_execution([
os.path.join(test_utils.temp_folder, '${p1}.txt'),
os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')],
parameter_values={'p1': 'file1', 'p2': 'child'},
parameters=[param1, param2])
self.assert_downloadable_files(downloadable_files, [file2])
def test_multiple_static_images(self):
path1 = test_utils.create_file('test1.png')
path2 = test_utils.create_file('test2.png')
path3 = test_utils.create_file('test3.png')
config = create_config_model('my_script',
output_files=[inline_image(path1), inline_image(path2), inline_image(path3)])
execution_id = self.start_execution(config)
self.write_output(execution_id, '123\n' + '456')
self.wait_output_chunks(execution_id, chunks_count=1)
self.assert_images(path1, path2, path3)
def test_output_files_with_secure_parameters(self):
test_utils.create_file('file1.txt', text='hello world')
file2 = test_utils.create_file(os.path.join('sub', 'child', 'admin.log'), text='password=123')
param1 = create_script_param_config('p1', secure=True)
param2 = create_script_param_config('p2')
downloadable_files = self.perform_execution([
os.path.join(test_utils.temp_folder, '${p1}.txt'),
os.path.join(test_utils.temp_folder, 'sub', '${p2}', 'admin.log')],
parameter_values={'p1': 'file1', 'p2': 'child'},
parameters=[param1, param2])
self.assert_downloadable_files(downloadable_files, [file2])
def test_allow_not_quoted_file_with_whitespaces_win(self):
test_utils.set_win()
file = test_utils.create_file('my script.py')
command_split = process_utils.split_command('my script.py', test_utils.temp_folder)
self.assertEqual(command_split, [os.path.abspath(file)])
def test_restore_dump_state_when_multiple_users(self):
dump_file = test_utils.create_file('dump.json',
text=json.dumps([{
'username':
'******',
'groups': ['group1', 'group2'],
'last_auth_update':
123
}, {
'groups': ['group3'],
'last_auth_update':
999
}, {
'username': '******',
'last_visit': 456
}]))
authenticator = create_test_authenticator(dump_file=dump_file)
self.assertEqual({'user_X', 'user_Y'}, authenticator._users.keys())
user_x_state = authenticator._users['user_X']
self.assertEqual('user_X', user_x_state.username)
self.assertEqual(['group1', 'group2'], user_x_state.groups)
self.assertEqual(123, user_x_state.last_auth_update)
self.assertEqual(None, user_x_state.last_visit)
user_y_state = authenticator._users['user_Y']
self.assertEqual('user_Y', user_y_state.username)
self.assertEqual([], user_y_state.groups)
self.assertEqual(None, user_y_state.last_auth_update)
self.assertEqual(456, user_y_state.last_visit)
def test_find_multiple_images_by_same_pattern(self):
path1 = test_utils.create_file('test123.png')
test_utils.create_file('images/test.png')
path3 = test_utils.create_file('a.b.c.png')
path4 = test_utils.create_file('some/sub/folder/test456.png')
config = create_config_model(
'my_script', output_files=[inline_image('##any_path.png#')])
execution_id = self.start_execution(config)
paths = [normalize_path(p) for p in (path1, path3, path4)]
for index, path in enumerate(paths):
self.write_output(execution_id, '__ ' + path + ' __\n')
self.wait_output_chunks(execution_id, chunks_count=index + 1)
self.assert_images(*paths)
def test_file_upload_parameter_when_not_exists(self):
parameter = self.create_parameter('param')
parameter.type = 'file_upload'
uploaded_file = test_utils.create_file('test.xml')
error = model_helper.validate_parameter(parameter,
{'param': uploaded_file + '_'})
self.assert_error(error)
def test_many_unicode_characters(self):
long_unicode_text = ('ΩΨΔ\n' * 100000)
test_utils.create_file('test.txt', text=long_unicode_text)
process_wrapper = PtyProcessWrapper(['cat', 'test.txt'],
test_utils.temp_folder, {})
process_wrapper.start()
thread = threading.Thread(target=process_wrapper.wait_finish,
daemon=True)
thread.start()
thread.join(timeout=0.1)
output = ''.join(read_until_closed(process_wrapper.output_stream))
self.assertEqual(long_unicode_text, output)
self.assertEqual(0, process_wrapper.get_return_code())
def test_file_upload_parameter_when_valid(self):
parameter = self.create_parameter('param')
parameter.type = 'file_upload'
uploaded_file = test_utils.create_file('test.xml')
error = model_helper.validate_parameter(parameter,
{'param': uploaded_file})
self.assertIsNone(error)
def test_htpasswd_auth(self):
file = test_utils.create_file('some-path', text='user1:1yL79Q78yczsM')
config = _from_json({'auth': {'type': 'htpasswd',
'htpasswd_path': file}})
self.assertIsInstance(config.authenticator, HtpasswdAuthenticator)
authenticated = config.authenticator.verifier.verify('user1', 'aaa')
self.assertTrue(authenticated)
def test_file_upload_parameter_when_not_exists(self):
script_config = script_configs.Config()
parameter = self.add_parameter('param', script_config)
parameter.type = 'file_upload'
uploaded_file = test_utils.create_file('test.xml')
valid = model_helper.validate_parameters({'param': uploaded_file + '_'}, script_config)
self.assertFalse(valid)
def test_single_static_image(self):
path = test_utils.create_file('test.png')
config = create_config_model('my_script', output_files=[inline_image(path)])
execution_id = self.start_execution(config)
self.write_output(execution_id, '123\n456')
self.wait_output_chunks(execution_id, chunks_count=1)
self.assert_images(path)
def test_single_dynamic_image(self):
path = test_utils.create_file('test.png')
config = create_config_model('my_script', output_files=[inline_image('##any_path.png#')])
execution_id = self.start_execution(config)
full_path = file_utils.normalize_path(path)
self.write_output(execution_id, '123\n' + full_path + '\n456')
self.wait_output_chunks(execution_id, chunks_count=1)
self.assert_images(full_path)
def setUp(self):
self.tmp_root = tempfile.mkdtemp()
dir1 = os.path.join(self.tmp_root, "1")
dir2 = os.path.join(self.tmp_root, "2")
os.makedirs(dir1)
os.makedirs(dir2)
content = get_count_file_content()
test_utils.create_file(os.path.join(dir1, "counts_P1234_101.txt"),
content)
test_utils.create_file(os.path.join(dir2, "counts_P1234_102.txt"),
content)
self.expected = {
'ABC123X': {
'P1234_101': 4,
'P1234_102': 4,
'length': 3
},
'ABC123Y': {
'P1234_101': 5,
'P1234_102': 5,
'length': 4
}
}
def test_image_path_split_in_chunks(self):
path = test_utils.create_file('test123.png')
config = create_config_model('my_script', output_files=[inline_image('##any_path.png#')])
execution_id = self.start_execution(config)
normalized = normalize_path(path)
self.write_output(execution_id, normalized[:4])
self.wait_output_chunks(execution_id, chunks_count=1)
self.write_output(execution_id, normalized[4:] + '\n')
self.wait_output_chunks(execution_id, chunks_count=2)
self.assert_images(path)
def test_update_script_config(self):
self.start_server(12345, '127.0.0.1')
xsrf_token = self.get_xsrf_token(self._admin_session)
script_path = test_utils.create_file('my_script.py')
test_utils.write_script_config(
{
'name': 's1',
'script_path': script_path
}, 's1', self.runners_folder)
response = self._admin_session.put(
'http://127.0.0.1:12345/admin/scripts',
data={
'filename':
's1.json',
'config':
json.dumps({
'name': 'new name',
'script': {
'mode': 'new_code',
'path': script_path,
'code': 'abcdef'
}
})
},
headers={'X-XSRFToken': xsrf_token},
)
self.assertEqual(200, response.status_code)
conf_response = self.request(
'get', 'http://127.0.0.1:12345/admin/scripts/new%20name',
self._admin_session)
self.assertEqual(
{
'config': {
'name': 'new name',
'script_path': script_path
},
'filename': 's1.json'
}, conf_response)
script_content = file_utils.read_file(script_path)
self.assertEqual('abcdef', script_content)
def test_get_script_code(self):
self.start_server(12345, '127.0.0.1')
script_path = test_utils.create_file('my_script.py')
test_utils.write_script_config(
{
'name': 's1',
'script_path': script_path
}, 's1', self.runners_folder)
response = self.request(
'get', 'http://127.0.0.1:12345/admin/scripts/s1/code',
self._admin_session)
self.assertEqual(
{
'code': 'test text',
'file_path': os.path.abspath(script_path)
}, response)
def test_image_path_split_in_chunks_and_no_newlines(self):
path = test_utils.create_file('test123.png')
config = create_config_model('my_script', output_files=[inline_image('##any_path.png#')])
execution_id = self.start_execution(config)
normalized = normalize_path(path)
self.write_output(execution_id, normalized[:4])
self.wait_output_chunks(execution_id, chunks_count=1)
self.write_output(execution_id, normalized[4:])
self.wait_output_chunks(execution_id, chunks_count=2)
self.executor_service.get_active_executor(execution_id, DEFAULT_USER).process_wrapper.stop()
self.wait_close(execution_id)
self.assert_images(path)
def test_prepare_exe(self):
script_path = test_utils.create_file('test.exe')
(command, shell) = prepare_cmd_for_win([script_path])
self.assertEqual([script_path], command)
self.assertFalse(shell)
def test_full_path_command(self):
file = test_utils.create_file('test.sh')
command_split = process_utils.split_command('test.sh', test_utils.temp_folder)
self.assertEqual(command_split, [os.path.abspath(file)])
def test_prepare_simple_py(self):
script_path = test_utils.create_file('test.py')
(command, shell) = prepare_cmd_for_win([script_path])
self.assertEqual([script_path], command)
self.assertTrue(shell)
def test_prepare_py_with_ampersand_short_command(self):
script_path = test_utils.create_file('test.py')
(command, shell) = prepare_cmd_for_win([script_path, '&ping&'])
self.assertEqual([script_path, '^&ping^&'], command)
self.assertTrue(shell)
def test_prepare_py_with_ampersand_another_command(self):
script_path = test_utils.create_file('test.py')
(command, shell) = prepare_cmd_for_win([script_path, '"&& dir c:"'])
self.assertEqual([script_path, '"^&^& dir c:"'], command)
self.assertTrue(shell)
def test_single_file(self):
test_utils.create_file('my.txt')
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['my.txt'], files)
def test_file_upload_parameter_when_valid(self):
parameter = create_parameter_model('param', type='file_upload')
uploaded_file = test_utils.create_file('test.xml')
error = parameter.validate_value(uploaded_file)
self.assertIsNone(error)
def test_server_file_when_wrong(self):
test_utils.create_file('file1.txt')
parameter = create_parameter_model('param', type=PARAM_TYPE_SERVER_FILE, file_dir=test_utils.temp_folder)
error = parameter.validate_value('my.dat')
self.assert_error(error)
def test_no_output_files_in_config(self):
test_utils.create_file('file1.txt')
downloadable_files = self.perform_execution(None)
self.assertEqual([], downloadable_files)
def test_prepare_file_on_finish(self):
file1_path = test_utils.create_file('file1.txt')
downloadable_files = self.perform_execution([file1_path])
self.assert_downloadable_files(downloadable_files, [file1_path])
def test_path_when_exists_and_working_dir(self):
file = test_utils.create_file('./scriptX.sh')
command_split = process_utils.split_command('./scriptX.sh 123', test_utils.temp_folder)
self.assertEqual([os.path.abspath(file), '123'], command_split)
def test_path_when_working_dir_and_abs_path(self):
file = test_utils.create_file('scriptX.sh')
abs_file = os.path.abspath(file)
command_split = process_utils.split_command(abs_file + ' 123', 'my_dir')
self.assertEqual([abs_file, '123'], command_split)
def test_double_asterisk_match_multiple_files_when_complex(self):
test_utils.create_file(os.path.join('f1', 'test1.txt'))
test_utils.create_file(os.path.join('f1', 'test2.txt'))
test_utils.create_file(os.path.join('d2', 'test3.txt'))
test_utils.create_file(os.path.join('d2', 'd3', 'test4.txt'))
test_utils.create_file(os.path.join('d3', 'd4', 'd5', 'test5.png'))
test_utils.create_file(os.path.join('d3', 'd6', 'd7', 'test6.txt'))
temp_folder = file_utils.normalize_path(test_utils.temp_folder)
files = set(file_download_feature.find_matching_files(temp_folder + '/d*/**/*.txt', None))
self.assertCountEqual(files, {
os.path.join(temp_folder, 'd2', 'test3.txt'),
os.path.join(temp_folder, 'd2', 'd3', 'test4.txt'),
os.path.join(temp_folder, 'd3', 'd6', 'd7', 'test6.txt')
})
def setUp(self):
self.tmp_root = tempfile.mkdtemp()
self.mappings_file = os.path.join(self.tmp_root, "targets.txt")
content = get_default_mapping_content(1) + get_default_mapping_content(1)
print(content)
test_utils.create_file(self.mappings_file, content)
def test_file_upload_parameter_when_not_exists(self):
parameter = create_parameter_model('param', type='file_upload')
uploaded_file = test_utils.create_file('test.xml')
error = parameter.validate_value(uploaded_file + '_')
self.assert_error(error)
