class MultiplyOperation(Numeric):
a = SpecField(0)
b = SpecField(1)
def apply(self, runner):
super(MultiplyOperation, self).apply(runner)
return runner.execute(self.a) * runner.execute(self.b)
def __repr__(self):
return '{} * {}'.format(self.a, self.b)
class Operation(Spec):
out_data_store = SpecField(default=None, serialize=False)
default_data_store = None
def execute(self, force=False):
return OperationRunner().execute(self, force=force)
def apply(self, runner):
raise NotImplementedError()
def get_out_data_store(self):
if self.out_data_store is not None:
return self.out_data_store
else:
return type(self).default_data_store
class as_operation(GenericDecorator):
"""
Creates an operation from a callable
:param out_type: Base class of the operation to be built. Defaults to `Operation`
:param out_name: Name of the class to be built, deafults to the decorated function name.
"""
out_type = PrimitiveField(default=Operation)
out_name = PrimitiveField(default=None)
cache_on = SpecField(default=None)
args_specifications = KwargsField()
def create_decorated(self,
to_wrap,
func_to_execute,
f_spec=None,
first_arg=None):
f_spec = f_spec or inspect.getargspec(to_wrap)
OperationClass = operation_from_func(
to_wrap=to_wrap,
func_to_execute=func_to_execute,
out_type=self.out_type,
out_name=self.out_name,
args_specifications=self.args_specifications,
f_spec=f_spec,
method_type=self.method_type,
first_arg=first_arg,
cache_on=self.cache_on)
return OperationClass
@staticmethod
def get_current_operation():
"""
Should be called inside a function decorated with as_operation
"""
# f_back brings you to the calling function, f_back brings you to the apply method of the
# dynamically created operation
frame = inspect.currentframe()
try:
res = frame.f_back.f_back.f_locals['self']
if not isinstance(res, Operation):
raise RuntimeError(
"This function should be called inside an operation created with the as_operation decorator"
)
return res
finally:
# Avoid reference cycle
del frame
class StorageRefactor(Operation):
doc = UnboundPrimitiveField(0, serialize=False)
storage_refactor = SpecField(default=None)
def add_field(self, field_type, field_name, default_value=None):
return AddField(field_type,
field_name,
default_value,
storage_refactor=self)
def rename_field(self, field_type, source, target):
return RenameField(field_type, source, target, storage_refactor=self)
def remove_field(self, field_type, field_name):
return RemoveField(field_type, field_name, storage_refactor=self)
def change_type(self, field_type, new_type):
return ChangeType(field_type, new_type, storage_refactor=self)
def _bind(self, meth_name, *args, **kwargs):
res = getattr(super(StorageRefactor, self), meth_name)(*args, **kwargs)
if self.storage_refactor is not None:
res.storage_refactor = res.storage_refactor.bind(*args, **kwargs)
return res
def bind(self, *args, **kwargs):
return self._bind('bind', *args, **kwargs)
def inplace_bind(self, *args, **kwargs):
return self._bind('inplace_bind', *args, **kwargs)
def apply(self, runner):
assert isinstance(self.doc, dict)
# if 'operation_1' in self.doc['type']: import ipdb;ipdb.set_trace()
doc = self.chain_transformations(self.doc)
return recursive_map(doc, self.chain_transformations)
def transformation(self, doc):
return doc
def chain_transformations(self, doc):
doc = self.transformation(doc)
if self.storage_refactor is not None:
doc = self.storage_refactor.chain_transformations(doc)
return doc
def operation_from_func(to_wrap,
func_to_execute,
out_type,
out_name,
args_specifications,
f_spec=None,
method_type=None,
first_arg=None,
cache_on=None):
"""
In the case of methods, to_wrap is not the same to func_to_execute
:param to_wrap: See `GenericDecorator.create_decorated` for an explanation
:param func_to_execute: See `GenericDecorator.create_decorated` for an explanation
:param cache_on: A data store onto which the operation should be cached
:return:
"""
f_spec = f_spec or inspect.getargspec(to_wrap)
out_name = out_name or to_wrap.__name__
# TODO: find the first_arg where the method was defined
if method_type == 'instance' and not isinstance(first_arg, Spec):
# Only when it's an instance of Spec we can identify
out_name = '{}@{}'.format(out_name, id(first_arg))
default_values = get_default_values(f_spec)
attrs = {}
binded_pos = 0
unbinded_pos = 0
for arg in f_spec.args:
if method_type == 'instance' and arg == 'self': continue
if method_type == 'class' and arg == 'cls': continue
if arg in args_specifications:
spec = args_specifications[arg]
if inspect.isclass(spec) and issubclass(spec, Spec):
spec = SpecField(base_type=spec)
# It can be either a class, or the instance itself
if inspect.isclass(spec) or inspect.isfunction(spec): spec = spec()
if isinstance(spec, UnboundField):
spec.pos = unbinded_pos
unbinded_pos += 1
else:
spec.pos = binded_pos
binded_pos += 1
else:
spec = PrimitiveField(binded_pos)
binded_pos += 1
if arg in default_values: spec.default = default_values[arg]
attrs[arg] = spec
def get_this_args(self, runner=None):
this_args = {}
for k, v in attrs.iteritems():
value = getattr(self, k)
if isinstance(v,
BaseSpecField) and runner is not None and isinstance(
value, Operation):
value = runner.execute(value)
this_args[k] = value
return this_args
def to_dict(self, include_all=False):
res = super(out_type, self).to_dict(include_all=include_all)
if method_type is not None:
res['type'] = get_import_path(first_arg, func_to_execute.__name__)
else:
res['type'] = get_import_path(func_to_execute)
return res
@property
def self(self):
if method_type is None:
raise RuntimeError(
'Can only be called with an operation created from a method')
return first_arg
def apply(self, runner):
this_args = self.get_this_args(runner)
return func_to_execute(**this_args)
cls_attrs = attrs.copy()
cls_attrs['func'] = staticmethod(func_to_execute)
cls_attrs['apply'] = apply
cls_attrs['get_this_args'] = get_this_args
cls_attrs['to_dict'] = to_dict
cls_attrs['self'] = self
cls = Operation.type2spec_class(out_name)
if cls is None:
# if the class does not exist, create it
cls = type(out_name, (out_type, ), cls_attrs)
else:
# otherwise update it
for k, v in cls_attrs.iteritems():
setattr(cls, k, v)
if cache_on is not None:
cls.default_data_store = cache_on
else:
cls.default_data_store = None
cls.__module__ = to_wrap.__module__
return cls
class StorageRefactor(Operation):
doc = UnboundPrimitiveField(0, serialize=False)
storage_refactor = SpecField(default=None)
def change_field(self, spec_type, field_name, old_value, new_value):
return ChangeField(spec_type, field_name, old_value, new_value, storage_refactor=self)
def change_type(self, spec_type, new_type):
return self.change_field(spec_type, 'type', get_import_path(spec_type), get_import_path(new_type))
def add_field(self, spec_type, field_name, default_value=None):
return AddField(spec_type, field_name, default_value, storage_refactor=self)
def rename_field(self, spec_type, source, target):
return RenameField(spec_type, source, target, storage_refactor=self)
def remove_field(self, spec_type, field_name):
return RemoveField(spec_type, field_name, storage_refactor=self)
def chain_refactor(self, refactor):
return ChainedRefactor(refactor, storage_refactor=self)
def project(self, field):
return ProjectedRefactor(field, storage_refactor=self)
def _bind(self, meth_name, *args, **kwargs):
res = getattr(super(StorageRefactor, self), meth_name)(*args, **kwargs)
if self.storage_refactor is not None:
res.storage_refactor = res.storage_refactor.bind(*args, **kwargs)
return res
def bind(self, *args, **kwargs):
return self._bind('bind', *args, **kwargs)
def inplace_bind(self, *args, **kwargs):
return self._bind('inplace_bind', *args, **kwargs)
def apply(self, runner):
assert isinstance(self.doc, dict)
doc = self.chain_transformations(self.doc)
doc = recursive_map(doc, self.chain_transformations)
return doc
def transformation(self, doc):
return doc
def chain_transformations(self, doc):
if not self.recurse_first:
doc = self.transformation(doc)
if self.storage_refactor is not None:
doc = self.storage_refactor.chain_transformations(doc)
if self.recurse_first:
doc = self.transformation(doc)
return doc
@property
def recurse_first(self):
# By default we always call self.transformation(doc) first
return False
@property
def empty(self):
return self.storage_refactor is None
def OperationField(pos=None, default=_no_default, base_type=None):
return SpecField(pos=pos,
default=default,
base_type=base_type or Operation)