class PythonexpTransform(BaseTransform):
"""
It applies python expressions to items.
- python_expression (str)
Valid python expression
"""
# List of options to set up the transform module
supported_options = {
'python_expressions': {'type': str_list}
}
def __init__(self, *args, **kwargs):
super(PythonexpTransform, self).__init__(*args, **kwargs)
self.python_expressions = self.read_option('python_expressions')
if not self.is_valid_python_expression(self.python_expressions):
raise ValueError('Python expression is not valid')
self.interpreter = Interpreter()
self.logger.info('PythonexpTransform has been initiated. Expressions: {!r}'.format(
self.python_expressions)
)
def transform_batch(self, batch):
for item in batch:
context = create_context(item=item)
for expression in self.python_expressions:
self.interpreter.eval(expression, context=context)
yield item
self.logger.debug('Transformed items')
# TODO: Make a expression validator
def is_valid_python_expression(self, python_expressions):
return True
def __init__(self, *args, **kwargs):
super(PythonexpTransform, self).__init__(*args, **kwargs)
self.python_expressions = self.read_option('python_expressions')
if not self.is_valid_python_expression(self.python_expressions):
raise ValueError('Python expression is not valid')
self.interpreter = Interpreter()
self.logger.info(
'PythonexpTransform has been initiated. Expressions: {!r}'.format(
self.python_expressions))
def __init__(self, *args, **kwargs):
super(PythonexpFilter, self).__init__(*args, **kwargs)
self.logger.warning('PythonexpFilter can import insecure code'
' -- only use it in contained environments')
self.expression = self.read_option('python_expression')
self.imports = load_imports(self.read_option('imports'))
self.interpreter = Interpreter()
self.logger.info('PythonexpFilter has been initiated.'
' Expression: {!r}'.format(self.expression))
class PythonExpGrouper(BaseGrouper):
"""
Groups items depending on python expressions. It adds the group membership information to items.
- python_expressions (list)
A list of python expressions to group by
"""
supported_options = {
'python_expressions': {'type': str_list}
}
def __init__(self, *args, **kwargs):
super(PythonExpGrouper, self).__init__(*args, **kwargs)
self.expressions = self.read_option('python_expressions', [])
self.interpreter = Interpreter()
def _get_membership(self, item):
try:
context = create_context(item=item)
return [
self.interpreter.eval(expression, context=context)
for expression in self.expressions
]
except Exception as ex:
self.logger.error(str(ex))
raise
def group_batch(self, batch):
for item in batch:
item.group_membership = tuple(self._get_membership(item))
yield item
class PythonexpFilter(BaseFilter):
"""
Filter items depending on python expression. This is NOT sure, so make sure you only use
it in contained environments
- python_expression (str)
Python expression to filter by
- imports(dict)
An object with neede imports for expressions
"""
# List of options
supported_options = {
'python_expression': {'type': six.string_types},
'imports': {'type': dict, 'default': {}},
}
def __init__(self, *args, **kwargs):
super(PythonexpFilter, self).__init__(*args, **kwargs)
self.logger.warning('PythonexpFilter can import insecure code'
' -- only use it in contained environments')
self.expression = self.read_option('python_expression')
self.imports = load_imports(self.read_option('imports'))
self.interpreter = Interpreter()
self.logger.info('PythonexpFilter has been initiated.'
' Expression: {!r}'.format(self.expression))
def filter(self, item):
try:
context = create_context(item=item, **self.imports)
return self.interpreter.eval(self.expression, context=context)
except Exception as ex:
self.logger.error(str(ex))
raise
class PythonExpGrouper(BaseGrouper):
"""
Groups items depending on python expressions. It adds the group membership information to items.
- python_expressions (list)
A list of python expressions to group by
"""
supported_options = {'python_expressions': {'type': str_list}}
def __init__(self, *args, **kwargs):
super(PythonExpGrouper, self).__init__(*args, **kwargs)
self.expressions = self.read_option('python_expressions', [])
self.interpreter = Interpreter()
def _get_membership(self, item):
try:
context = create_context(item=item)
return [
self.interpreter.eval(expression, context=context)
for expression in self.expressions
]
except Exception as ex:
self.logger.error(str(ex))
raise
def group_batch(self, batch):
for item in batch:
item.group_membership = tuple(self._get_membership(item))
yield item
class PythonMapTransform(BaseTransform):
"""Transform implementation that maps items using Python expressions
"""
supported_options = {"map": {"type": six.string_types}}
def __init__(self, *args, **kwargs):
super(PythonMapTransform, self).__init__(*args, **kwargs)
self.map_expression = self.read_option("map")
self.interpreter = Interpreter()
self.interpreter.check(self.map_expression)
def _map_item(self, it):
context = create_context(item=it)
return self.interpreter.eval(expression=self.map_expression, context=context)
def transform_batch(self, batch):
return (self._map_item(it) for it in batch)
def __init__(self, *args, **kwargs):
super(PythonexpFilter, self).__init__(*args, **kwargs)
self.logger.warning('PythonexpFilter can import insecure code'
' -- only use it in contained environments')
self.expression = self.read_option('python_expression')
self.imports = load_imports(self.read_option('imports'))
self.interpreter = Interpreter()
self.logger.info('PythonexpFilter has been initiated.'
' Expression: {!r}'.format(self.expression))
def __init__(self, *args, **kwargs):
super(PythonexpTransform, self).__init__(*args, **kwargs)
self.python_expressions = self.read_option('python_expressions')
if not self.is_valid_python_expression(self.python_expressions):
raise ValueError('Python expression is not valid')
self.interpreter = Interpreter()
self.logger.info('PythonexpTransform has been initiated. Expressions: {!r}'.format(
self.python_expressions)
)
class PythonInterpreterTest(unittest.TestCase):
def setUp(self):
self.interpreter = Interpreter()
def test_check(self):
with self.assertRaises(InvalidExpression):
self.interpreter.check(5)
with self.assertRaises(InvalidExpression):
self.interpreter.check('')
with self.assertRaises(SyntaxError):
self.interpreter.check('This is not a valid expression')
with self.assertRaises(InvalidExpression):
self.interpreter.check('2+2; 5+6')
class PythonInterpreterTest(unittest.TestCase):
def setUp(self):
self.interpreter = Interpreter()
def test_check(self):
with self.assertRaises(InvalidExpression):
self.interpreter.check(5)
with self.assertRaises(InvalidExpression):
self.interpreter.check('')
with self.assertRaises(SyntaxError):
self.interpreter.check('This is not a valid expression')
with self.assertRaises(InvalidExpression):
self.interpreter.check('2+2; 5+6')
class PythonMapTransform(BaseTransform):
"""Transform implementation that maps items using Python expressions
"""
supported_options = {
"map": {
'type': six.string_types
},
}
def __init__(self, *args, **kwargs):
super(PythonMapTransform, self).__init__(*args, **kwargs)
self.map_expression = self.read_option('map')
self.interpreter = Interpreter()
self.interpreter.check(self.map_expression)
def _map_item(self, it):
context = create_context(item=it)
return self.interpreter.eval(expression=self.map_expression,
context=context)
def transform_batch(self, batch):
return (self._map_item(it) for it in batch)
class PythonexpFilter(BaseFilter):
"""
Filter items depending on python expression. This is NOT sure, so make sure you only use
it in contained environments
- python_expression (str)
Python expression to filter by
- imports(dict)
An object with neede imports for expressions
"""
# List of options
supported_options = {
'python_expression': {
'type': six.string_types
},
'imports': {
'type': dict,
'default': {}
},
}
def __init__(self, *args, **kwargs):
super(PythonexpFilter, self).__init__(*args, **kwargs)
self.logger.warning('PythonexpFilter can import insecure code'
' -- only use it in contained environments')
self.expression = self.read_option('python_expression')
self.imports = load_imports(self.read_option('imports'))
self.interpreter = Interpreter()
self.logger.info('PythonexpFilter has been initiated.'
' Expression: {!r}'.format(self.expression))
def filter(self, item):
try:
context = create_context(item=item, **self.imports)
return self.interpreter.eval(self.expression, context=context)
except Exception as ex:
self.logger.error(str(ex))
raise
def __init__(self, *args, **kwargs):
super(PythonExpGrouper, self).__init__(*args, **kwargs)
self.expressions = self.read_option('python_expressions', [])
self.interpreter = Interpreter()
def __init__(self, *args, **kwargs):
super(PythonMapTransform, self).__init__(*args, **kwargs)
self.map_expression = self.read_option("map")
self.interpreter = Interpreter()
self.interpreter.check(self.map_expression)
def __init__(self, *args, **kwargs):
super(PythonMapTransform, self).__init__(*args, **kwargs)
self.map_expression = self.read_option('map')
self.interpreter = Interpreter()
self.interpreter.check(self.map_expression)
def __init__(self, *args, **kwargs):
super(PythonExpGrouper, self).__init__(*args, **kwargs)
self.expressions = self.read_option('python_expressions', [])
self.interpreter = Interpreter()
def setUp(self):
self.interpreter = Interpreter()
def setUp(self):
self.interpreter = Interpreter()
