class PythonexpTransform(BaseTransform): """ It applies python expressions to items. - python_expression (str) Valid python expression """ # List of options to set up the transform module supported_options = { 'python_expressions': {'type': str_list} } def __init__(self, *args, **kwargs): super(PythonexpTransform, self).__init__(*args, **kwargs) self.python_expressions = self.read_option('python_expressions') if not self.is_valid_python_expression(self.python_expressions): raise ValueError('Python expression is not valid') self.interpreter = Interpreter() self.logger.info('PythonexpTransform has been initiated. Expressions: {!r}'.format( self.python_expressions) ) def transform_batch(self, batch): for item in batch: context = create_context(item=item) for expression in self.python_expressions: self.interpreter.eval(expression, context=context) yield item self.logger.debug('Transformed items') # TODO: Make a expression validator def is_valid_python_expression(self, python_expressions): return True
class PythonexpTransform(BaseTransform): """ It applies python expressions to items. - python_expression (str) Valid python expression """ # List of options to set up the transform module supported_options = {'python_expressions': {'type': str_list}} def __init__(self, *args, **kwargs): super(PythonexpTransform, self).__init__(*args, **kwargs) self.python_expressions = self.read_option('python_expressions') if not self.is_valid_python_expression(self.python_expressions): raise ValueError('Python expression is not valid') self.interpreter = Interpreter() self.logger.info( 'PythonexpTransform has been initiated. Expressions: {!r}'.format( self.python_expressions)) def transform_batch(self, batch): for item in batch: context = create_context(item=item) for expression in self.python_expressions: self.interpreter.eval(expression, context=context) yield item self.logger.debug('Transformed items') # TODO: Make a expression validator def is_valid_python_expression(self, python_expressions): return True
class PythonexpFilter(BaseFilter): """ Filter items depending on python expression. This is NOT sure, so make sure you only use it in contained environments - python_expression (str) Python expression to filter by - imports(dict) An object with neede imports for expressions """ # List of options supported_options = { 'python_expression': {'type': six.string_types}, 'imports': {'type': dict, 'default': {}}, } def __init__(self, *args, **kwargs): super(PythonexpFilter, self).__init__(*args, **kwargs) self.logger.warning('PythonexpFilter can import insecure code' ' -- only use it in contained environments') self.expression = self.read_option('python_expression') self.imports = load_imports(self.read_option('imports')) self.interpreter = Interpreter() self.logger.info('PythonexpFilter has been initiated.' ' Expression: {!r}'.format(self.expression)) def filter(self, item): try: context = create_context(item=item, **self.imports) return self.interpreter.eval(self.expression, context=context) except Exception as ex: self.logger.error(str(ex)) raise
class PythonExpGrouper(BaseGrouper): """ Groups items depending on python expressions. It adds the group membership information to items. - python_expressions (list) A list of python expressions to group by """ supported_options = { 'python_expressions': {'type': str_list} } def __init__(self, *args, **kwargs): super(PythonExpGrouper, self).__init__(*args, **kwargs) self.expressions = self.read_option('python_expressions', []) self.interpreter = Interpreter() def _get_membership(self, item): try: context = create_context(item=item) return [ self.interpreter.eval(expression, context=context) for expression in self.expressions ] except Exception as ex: self.logger.error(str(ex)) raise def group_batch(self, batch): for item in batch: item.group_membership = tuple(self._get_membership(item)) yield item
class PythonExpGrouper(BaseGrouper): """ Groups items depending on python expressions. It adds the group membership information to items. - python_expressions (list) A list of python expressions to group by """ supported_options = {'python_expressions': {'type': str_list}} def __init__(self, *args, **kwargs): super(PythonExpGrouper, self).__init__(*args, **kwargs) self.expressions = self.read_option('python_expressions', []) self.interpreter = Interpreter() def _get_membership(self, item): try: context = create_context(item=item) return [ self.interpreter.eval(expression, context=context) for expression in self.expressions ] except Exception as ex: self.logger.error(str(ex)) raise def group_batch(self, batch): for item in batch: item.group_membership = tuple(self._get_membership(item)) yield item
class PythonMapTransform(BaseTransform): """Transform implementation that maps items using Python expressions """ supported_options = {"map": {"type": six.string_types}} def __init__(self, *args, **kwargs): super(PythonMapTransform, self).__init__(*args, **kwargs) self.map_expression = self.read_option("map") self.interpreter = Interpreter() self.interpreter.check(self.map_expression) def _map_item(self, it): context = create_context(item=it) return self.interpreter.eval(expression=self.map_expression, context=context) def transform_batch(self, batch): return (self._map_item(it) for it in batch)
class PythonexpFilter(BaseFilter): """ Filter items depending on python expression. This is NOT sure, so make sure you only use it in contained environments - python_expression (str) Python expression to filter by - imports(dict) An object with neede imports for expressions """ # List of options supported_options = { 'python_expression': { 'type': six.string_types }, 'imports': { 'type': dict, 'default': {} }, } def __init__(self, *args, **kwargs): super(PythonexpFilter, self).__init__(*args, **kwargs) self.logger.warning('PythonexpFilter can import insecure code' ' -- only use it in contained environments') self.expression = self.read_option('python_expression') self.imports = load_imports(self.read_option('imports')) self.interpreter = Interpreter() self.logger.info('PythonexpFilter has been initiated.' ' Expression: {!r}'.format(self.expression)) def filter(self, item): try: context = create_context(item=item, **self.imports) return self.interpreter.eval(self.expression, context=context) except Exception as ex: self.logger.error(str(ex)) raise
class PythonMapTransform(BaseTransform): """Transform implementation that maps items using Python expressions """ supported_options = { "map": { 'type': six.string_types }, } def __init__(self, *args, **kwargs): super(PythonMapTransform, self).__init__(*args, **kwargs) self.map_expression = self.read_option('map') self.interpreter = Interpreter() self.interpreter.check(self.map_expression) def _map_item(self, it): context = create_context(item=it) return self.interpreter.eval(expression=self.map_expression, context=context) def transform_batch(self, batch): return (self._map_item(it) for it in batch)