def test_execute_python_prohibited_data_types_fail():
"""
'byte_array' and 'memory_view' are prohibited
"""
arguments = {
'parameters': {'code': dedent("""
byte_array = bytearray(5)
memory_view = memoryview(bytes(5))
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': None,
'input data 2': None
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
with pytest.raises(ValueError) as val_err:
util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'emit_event': _emit_event(output)})
assert "name 'bytearray' is not defined." \
" Many Python commands are not available in Lemonade" in str(
val_err.value)
def test_execute_python_prohibited_python_keywords_fail():
"""
'class', 'nonlocal', 'import', 'from' and 'as' are prohibited
"""
arguments = {
'parameters': {'code': dedent("""
from math import inf
class FailClass:
def failfunc():
nonlocal x
x = 10
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': None,
'input data 2': None
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
with pytest.raises(SyntaxError) as syn_err:
util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'emit_event': _emit_event(output)})
assert "Nonlocal statements are not allowed." in str(syn_err.value)
def test_execute_python_pandas_success():
"""the user can use pretty much every method from pandas, this may cause
problems because of the quantity of methods and future methods that will
be added"""
df1 = util.iris(['class', 'petalwidth'], size=10)
df2 = util.iris(['class'], size=10)
test_df = df1.copy()
arguments = {
'parameters': {'code': dedent("""
out1 = in1.drop(columns=['class'])
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': 'df1',
'input data 2': 'df2'
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
result = util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'df1': df1,
'df2': df2,
'emit_event': _emit_event(output)})
assert result['out1'].equals(test_df.drop(columns=['class']))
def test_execute_python_big_or_infinite_loops_success():
"""
The user can create big or infinite loops
Uncomment the code in dedent() method to test
"""
arguments = {
'parameters': {'code': dedent("""
# Example 1:
# for i in range(100000000000000000):
# pass
# Example 2:
# while True:
# pass
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': None,
'input data 2': None
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'emit_event': _emit_event(output)})
def test_execute_python_dangerous_zfill_method_success():
# The zfill() can cause a crash
arguments = {
'parameters': {'code': dedent("""
str_ing = ''
str_ing = str_ing.zfill(100)
# Example on how it can crash/overflow
# str_ing = str_ing.zfill(10000000000)
print(str_ing)
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': None,
'input data 2': None
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'emit_event': _emit_event(output)})
assert output[0]['message'] == ''.zfill(100) + '\n'
def test_execute_python_data_types_success():
"""
'range()' is allowed, it can cause problems...
"""
arguments = {
'parameters': {'code': dedent("""
str = 'tst'
int = 1
float = 0.1
complex_number = 1j
list = ['t', 'e', 's', 't']
tuple = ('t', 'e', 's', 't')
range = range(6)
dict = {'tst': 'success'}
set = {'t', 'e', 's', 't'}
frozenset= ({'t', 'e', 's', 't'})
bool = True
bytes = b"Hello"
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': None,
'input data 2': None
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
result = util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'emit_event': _emit_event(output)})
assert result['user_code'] == dedent("""
str = 'tst'
int = 1
float = 0.1
complex_number = 1j
list = ['t', 'e', 's', 't']
tuple = ('t', 'e', 's', 't')
range = range(6)
dict = {'tst': 'success'}
set = {'t', 'e', 's', 't'}
frozenset= ({'t', 'e', 's', 't'})
bool = True
bytes = b"Hello"
""")
def test_execute_python_success():
slice_size = 10
df = [
'df',
util.iris(['sepallength', 'sepalwidth', 'petalwidth', 'petallength'],
slice_size)
]
arguments = {
'parameters': {},
'named_inputs': {
'input data': df[0],
},
'named_outputs': {
'output data': 'out'
}
}
instance = ExecutePythonOperation(**arguments)
result = util.execute(instance.generate_code(), dict([df]))
assert result['out'].equals(util.iris(size=slice_size))
def test_execute_python_missing_parameters_fail():
arguments = {
'parameters': {},
'named_inputs': {
'input data 1': 'df1',
'input data 2': 'df2'
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
with pytest.raises(ValueError) as val_err:
ExecutePythonOperation(**arguments)
assert "Required parameter code must be informed for task" in str(val_err.value)
def test_execute_python_print_success():
arguments = {
'parameters': {'code': dedent("""
x = 'hello_world'
print(x)
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': None,
'input data 2': None
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'emit_event': _emit_event(output)})
assert output[0]['message'] == 'hello_world\n'
def test_pythoncode_missing_parameter_failure():
params = {'task': {'id': 1}}
with pytest.raises(ValueError):
n_in = {'input data 1': 'input_1'}
n_out = {'output data': 'output_1'}
ExecutePythonOperation(params, named_inputs=n_in, named_outputs=n_out)
def test_pythoncode_minimum_params_success():
params = {
ExecutePythonOperation.PYTHON_CODE_PARAM:
"df1['col3'] = df1['col1'] + df1['col2']",
'task': {
'id': 1
}
}
n_in = {'input data 1': 'input_1'}
n_out = {'output data': 'output_1'}
instance = ExecutePythonOperation(params,
named_inputs=n_in,
named_outputs=n_out)
code = instance.generate_code()
expected_code = dedent("""
import json
from RestrictedPython.Guards import safe_builtins
from RestrictedPython.RCompile import compile_restricted
from RestrictedPython.PrintCollector import PrintCollector
results = [r[1].result() for r in task_futures.items() if r[1].done()]
results = dict([(r['task_name'], r) for r in results])
# Input data
in1 = input_1
in2 = None
# Output data, initialized as None
out1 = None
out2 = None
# Variables and language supported
ctx = {
'wf_results': results,
'in1': in1,
'in2': in2,
'out1': out1,
'out2': out2,
# Restrictions in Python language
'_write_': lambda v: v,
'_getattr_': getattr,
'_getitem_': lambda ob, index: ob[index],
'_getiter_': lambda it: it,
'_print_': PrintCollector,
'json': json,
}
user_code = "df1['col3'] = df1['col1'] + df1['col2']"
ctx['__builtins__'] = safe_builtins
compiled_code = compile_restricted(user_code,
str('python_execute_1'), str('exec'))
try:
exec compiled_code in ctx
# Retrieve values changed in the context
out1 = ctx['out1']
out2 = ctx['out2']
if '_print' in ctx:
emit_event(name='update task',
message=ctx['_print'](),
status='RUNNING',
identifier='1')
except NameError as ne:
raise ValueError(_('Invalid name: {}. '
'Many Python commands are not available in Lemonade').format(ne))
except ImportError as ie:
raise ValueError(_('Command import is not supported'))
out_1_1 = out1
out_2_1 = out2
""")
result, msg = compare_ast(ast.parse(code), ast.parse(expected_code))
assert result, msg + format_code_comparison(code, expected_code)
def test_execute_python_keywords_success():
"""
'for' and 'while' are allowed, they can cause problems...
"""
arguments = {
'parameters': {'code': dedent("""
pass
a = True and False
assert not a
def testing():
global b
b = 'hello'
return a
testing()
del testing
try:
0/0
except(ZeroDivisionError):
pass
finally:
pass
for i in range(10):
if i == 5:
continue
if i == 8:
break
c = lambda x: x + 10
c(10)
d = 0
e = []
while d not in e:
e.append(d)
f = True or False
assert f
def creategen():
lis = range(3)
for i in lis:
yield i*i
creategen()
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': None,
'input data 2': None
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
result = util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'emit_event': _emit_event(output)})
assert result['user_code'] == dedent("""
pass
a = True and False
assert not a
def testing():
global b
b = 'hello'
return a
testing()
del testing
try:
0/0
except(ZeroDivisionError):
pass
finally:
pass
for i in range(10):
if i == 5:
continue
if i == 8:
break
c = lambda x: x + 10
c(10)
d = 0
e = []
while d not in e:
e.append(d)
f = True or False
assert f
def creategen():
lis = range(3)
for i in lis:
yield i*i
creategen()
""")
def test_execute_python_operators_success():
"""
Assignment Operators are prohibited (e.g., +=, -=, *=, /=)
"""
arguments = {
'parameters': {'code': dedent("""
1 + 1
1 - 1
1 * 1
1 / 1
1 & 1
1 ** 1
1 // 1
1 == 1
1 != 1
1 > 1
1 < 1
1 >= 1
1 <= 1
1 & 1
1 | 1
1 ^ 1
1 << 1
1 >> 1
"""),
'task': {'id': 0}},
'named_inputs': {
'input data 1': None,
'input data 2': None
},
'named_outputs': {
'output data 1': 'out1',
'output data 2': 'out2'
}
}
output = []
instance = ExecutePythonOperation(**arguments)
result = util.execute(instance.generate_code(),
{'task_futures': {'items': TestingBypass},
'emit_event': _emit_event(output)})
assert result['user_code'] == dedent("""
1 + 1
1 - 1
1 * 1
1 / 1
1 & 1
1 ** 1
1 // 1
1 == 1
1 != 1
1 > 1
1 < 1
1 >= 1
1 <= 1
1 & 1
1 | 1
1 ^ 1
1 << 1
1 >> 1
""")