Ejemplo n.º 1
0
class GenericBlock(BaseBlock):
    # block fields
    is_abstract = True

    _uuid = BlockField("uuid", FieldType.STR, None, is_immutable=True)
    _name = BlockField("name", FieldType.STR, None)
    name = "Generic block"

    _base_name = BlockField("base_name", FieldType.STR, "", is_immutable=True)

    _block_group = BlockField("block_group",
                              FieldType.STR,
                              "",
                              is_immutable=True)
    block_group = None

    _exp_id = BlockField("exp_id", FieldType.STR, None, is_immutable=True)

    _scope_name = BlockField("scope_name",
                             FieldType.STR,
                             "root",
                             is_immutable=True)
    _sub_scope_name = BlockField("sub_scope_name",
                                 FieldType.STR,
                                 None,
                                 is_immutable=True)
    _visible_scopes_list = BlockField("visible_scopes_list",
                                      FieldType.SIMPLE_LIST,
                                      is_immutable=True)

    _state = BlockField("state", FieldType.STR, "created")

    _ui_folded = BlockField("ui_folded", FieldType.BOOLEAN, init_val=False)
    _ui_internal_folded = BlockField("ui_internal_folded",
                                     FieldType.BOOLEAN,
                                     init_val=False)
    _show_collector_editor = BlockField("show_collector_editor",
                                        FieldType.BOOLEAN,
                                        init_val=False)

    _has_custom_layout = BlockField("has_custom_layout", FieldType.BOOLEAN)
    _custom_layout_name = BlockField("custom_layout_name", FieldType.STR)

    _create_new_scope = BlockField("create_new_scope", FieldType.BOOLEAN)
    create_new_scope = False

    is_block_supports_auto_execution = False

    _errors = BlockField("errors", FieldType.SIMPLE_LIST, list())
    _warnings = BlockField("warnings", FieldType.SIMPLE_LIST, list())
    _bound_inputs = BlockField("bound_inputs", FieldType.SIMPLE_DICT,
                               defaultdict())

    def move_to_exp(self, exp_id):
        pass

    def duplicate(self, exp_id, mapping):
        import copy
        old_uuid = self.uuid
        new_obj = copy.deepcopy(self)
        new_obj.uuid = "B" + uuid1().hex[:8]
        new_obj.exp_id = exp_id
        if new_obj.scope_name:
            # little hack, it uses the fact that a scope name has a structure root_uuid1_uuid2....
            parent_uuids = new_obj.scope_name.split('_')
            for parent_uuid in parent_uuids:
                new_obj.scope_name = new_obj.scope_name.replace(
                    parent_uuid, mapping[parent_uuid])
        scope = new_obj.get_scope()
        scope.load()

        for f_name, f in new_obj._block_serializer.outputs.iteritems():
            scope.register_variable(
                ScopeVar(new_obj.uuid, f_name, f.provided_data_type))
            # log.debug("Registering normal outputs: %s", f_name)
            # new_obj.register_provided_objects(scope, ScopeVar(new_obj.uuid, f_name, f.provided_data_type))
        scope.store()
        return new_obj

    def remap_inputs(self, mapping):
        for var in self.bound_inputs.itervalues():
            var.change_block(mapping)

    def __init__(self, exp_id=None, scope_name=None):
        """
            Building block for workflow
        """
        # TODO: due to dynamic inputs, find better solution
        self._block_serializer = BlockSerializer.clone(
            self.__class__._block_serializer)

        self.state = "created"
        self.uuid = "B" + uuid1().hex[:8]

        self.exp_id = exp_id
        exp = None
        if exp_id:
            exp = Experiment.get_exp_by_id(exp_id)
        self.scope_name = scope_name
        self.base_name = ""

        # Used only be meta-blocks
        self.children_blocks = []
        # End

        self._out_data = dict()
        self.out_manager = OutManager()

        self.input_manager = InputManager()

        # Automatic execution status map
        self.auto_exec_status_ready = set(["ready"])
        self.auto_exec_status_done = set(["done"])
        self.auto_exec_status_working = set(["working"])
        self.auto_exec_status_error = set(["execution_error"])

        # Init block fields
        for f_name, f in itertools.chain(
                self._block_serializer.fields.iteritems(),
                self._block_serializer.params.iteritems()):

            #if f_name not in self.__dict__ and not f.is_a_property:
            if not f.is_a_property and not hasattr(self, f_name):
                try:
                    setattr(self, f_name, f.init_val)
                except:
                    import ipdb
                    ipdb.set_trace()

        for f_name, f in self._block_serializer.inputs.iteritems():
            if f.multiply_extensible:
                setattr(self, f_name, [])  # Names of dynamically added ports

        # TODO: Hmm maybe more metaclass magic can be applied here
        scope = self.get_scope()
        scope.load()
        for f_name, f in self._block_serializer.outputs.iteritems():
            if exp:
                exp.log(self.uuid, "Registering normal outputs: %s" % f_name)
            log.debug("Registering normal outputs: %s", f_name)
            self.register_provided_objects(
                scope, ScopeVar(self.uuid, f_name, f.provided_data_type))
            # TODO: Use factories for init values
            #if f.init_val is not None:
            #    setattr(self, f.name, f.init_val)

        scope.store()

        for f_name, f in self._block_serializer.fields.items():
            if f.init_val is not None:
                #setattr(self, f.name, f.init_val)
                pass

        for f_name, f in self._block_serializer.inputs.iteritems():
            self.input_manager.register(f)

    def on_remove(self, *args, **kwargs):
        """
            Cleanup all created files
            TODO: github:#61
        """
        pass

    def get_exec_status(self):
        if self.state in self.auto_exec_status_done:
            return "done"
        if self.state in self.auto_exec_status_error:
            return "error"
        if self.state in self.auto_exec_status_ready:
            return "ready"

        return "not_ready"

    def bind_input_var(self, input_name, bound_var):
        if self.exp_id:
            exp = Experiment.get_exp_by_id(self.exp_id)
            exp.log(
                self.uuid, "bound input %s to %s in block: %s, exp: %s" %
                (input_name, bound_var, self.base_name, self.exp_id))

        log.debug("bound input %s to %s in block: %s, exp: %s", input_name,
                  bound_var, self.base_name, self.exp_id)
        self.bound_inputs[input_name] = bound_var

    def get_input_var(self, name):
        try:
            exp = Experiment.get_exp_by_id(self.exp_id)
            scope_var = self.bound_inputs[name]
            return exp.get_scope_var_value(scope_var)
        except:
            return None

    def get_out_var(self, name):
        if self.out_manager.contains(name):
            return self._out_data.get(name)
        elif self.create_new_scope:
            return self.get_inner_out_var(name)
        else:
            return None

    # def get_inner_out_var(self, name):
    #     raise NotImplementedError("Not implemented in the base class")

    # def set_inner_out_var(self, name, value):
    #     raise NotImplementedError("Not implemented in the base class")

    def set_out_var(self, name, value):
        self._out_data[name] = value

    def get_scope(self):
        exp = Experiment.get_exp_by_id(self.exp_id)
        return Scope(exp, self.scope_name)

    @property
    def sub_scope_name(self):
        if hasattr(self, "create_new_scope") and self.create_new_scope:
            return "%s_%s" % (self.scope_name, self.uuid)
        else:
            return ""

    @property
    def visible_scopes_list(self):
        scope = self.get_scope()
        scope_names_list = scope.get_parent_scope_list()
        scope_names_list.append(self.scope_name)
        return scope_names_list

    def get_sub_scope(self):
        exp = Experiment.get_exp_by_id(self.exp_id)
        return Scope(exp, self.sub_scope_name)

    def reset_execution_for_sub_blocks(self):
        exp = Experiment.get_exp_by_id(self.exp_id)
        for block_uuid, block in exp.get_blocks(self.children_blocks):
            block.do_action("reset_execution", exp)

    def get_input_blocks(self):
        required_blocks = []
        for f in self.input_manager.input_fields:
            if f.multiply_extensible:
                continue
            if self.bound_inputs.get(f.name) is None and f.required:
                raise RuntimeError("Not all required inputs are bound")
            elif self.bound_inputs.get(f.name):
                required_blocks.append(self.bound_inputs[f.name].block_uuid)
        return required_blocks

    def get_user_actions(self):
        """
            @rtype: list of workflow.blocks.fields.ActionRecord
        """
        return self._trans.user_visible(self.state)

    @log_timing
    def to_dict(self):
        result = self._block_serializer.to_dict(self)
        # import ipdb; ipdb.set_trace()
        return result

    def register_provided_objects(self, scope, scope_var):
        self.out_manager.register(scope_var.var_name, scope_var.data_type)
        scope.register_variable(scope_var)

    @log_timing
    def apply_action_from_js(self, action_name, *args, **kwargs):
        if self._trans.is_action_available(self.state, action_name):
            self.do_action(action_name, *args, **kwargs)
        elif hasattr(self, action_name) and hasattr(getattr(self, action_name),
                                                    "__call__"):
            return getattr(self, action_name)(*args, **kwargs)
        else:
            raise RuntimeError("Block %s doesn't have action: %s" %
                               (self.name, action_name))

    def do_action(self, action_name, exp, *args, **kwargs):
        # if action_name == "success" and self.block_base_name == "CROSS_VALID":
        #     from celery.contrib import rdb; rdb.set_trace()
        ar = self._trans.action_records_by_name[action_name]
        old_exec_state = self.get_exec_status()
        next_state = self._trans.next_state(self.state, action_name)

        if next_state is not None:
            log.debug("Do action: %s in block %s from state %s -> %s",
                      action_name, self.base_name, self.state, next_state)
            exp.log(
                self.uuid, "Do action: %s in block %s from state %s -> %s" %
                (action_name, self.base_name, self.state, next_state))
            self.state = next_state

            if old_exec_state != "done" and self.get_exec_status() == "done":
                if self.is_block_supports_auto_execution:
                    BlockUpdated(self.exp_id,
                                 block_uuid=self.uuid,
                                 block_alias=self.base_name,
                                 silent=True).send()
            exp.store_block(self)
            getattr(self, action_name)(exp, *args, **kwargs)

            if ar.reload_block_in_client:
                BlockUpdated(self.exp_id, self.uuid, self.base_name).send()

            # TODO: Check if self.scope_name is actually set to auto execution
            #
            if old_exec_state != "done" and self.get_exec_status() == "done" \
                    and ar.propagate_auto_execution \
                    and self.is_block_supports_auto_execution:
                exp.log(self.uuid, "Propagate execution: %s " % self.base_name)
                log.debug("Propagate execution: %s ", self.base_name)
                auto_exec_task.s(exp, self.scope_name).apply_async()
            elif self.state in self.auto_exec_status_error \
                    and self.is_block_supports_auto_execution:
                exp.log(self.uuid,
                        "Detected error during automated workflow execution")
                log.debug("Detected error during automated workflow execution")
                halt_execution_task.s(exp, self.scope_name).apply_async()

        else:
            raise RuntimeError(
                "Action %s isn't available for block %s in state %s" %
                (action_name, self.base_name, self.state))

    def change_base_name(self, exp, received_block, *args, **kwargs):
        # TODO: check if the name is correct
        new_name = received_block.get("base_name")

        if new_name:
            exp.change_block_alias(self, new_name)

    def toggle_ui_folded(self, exp, received_block, *args, **kwargs):
        self.ui_folded = received_block["ui_folded"]
        exp.store_block(self)

    def save_params(self, exp, received_block=None, *args, **kwargs):
        self._block_serializer.save_params(self, received_block)
        exp.store_block(self)
        self.validate_params(exp)

    def save_file_input(self,
                        exp,
                        field_name,
                        file_obj,
                        multiple=False,
                        upload_meta=None):
        if upload_meta is None:
            upload_meta = {}

        if not hasattr(self, field_name):
            raise Exception("Block doesn't have field: %s" % field_name)

        orig_name = file_obj.name
        local_filename = "%s_%s_%s" % (self.uuid[:8], field_name,
                                       file_obj.name)

        if not multiple:
            exp.log(self.uuid,
                    "Storing single upload to field: %s" % field_name)
            log.debug("Storing single upload to field: %s", field_name)
            ud, is_created = UploadedData.objects.get_or_create(
                exp=exp, block_uuid=self.uuid, var_name=field_name)

            file_obj.name = local_filename
            ud.data = file_obj
            ud.save()

            ufw = UploadedFileWrapper(ud.pk)
            ufw.orig_name = orig_name
            setattr(self, field_name, ufw)
            exp.store_block(self)
        else:
            exp.log(self.uuid, "Adding upload to field: %s" % field_name)
            log.debug("Adding upload to field: %s", field_name)

            ud, is_created = UploadedData.objects.get_or_create(
                exp=exp,
                block_uuid=self.uuid,
                var_name=field_name,
                filename=orig_name)

            file_obj.name = local_filename
            ud.data = file_obj
            ud.filename = orig_name
            ud.save()

            ufw = UploadedFileWrapper(ud.pk)
            ufw.orig_name = orig_name

            r = get_redis_instance()
            with redis_lock.Lock(
                    r,
                    ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)):
                exp.log(self.uuid, "Enter lock, file: %s" % orig_name)
                log.debug("Enter lock, file: %s", orig_name)
                block = exp.get_block(self.uuid)
                attr = getattr(block, field_name)

                attr[orig_name] = ufw
                exp.log(
                    self.uuid, "Added upload `%s` to collection: %s" %
                    (orig_name, attr.keys()))
                log.debug("Added upload `%s` to collection: %s", orig_name,
                          attr.keys())
                exp.store_block(block)
                exp.log(self.uuid, "Exit lock, file: %s" % orig_name)
                log.debug("Exit lock, file: %s", orig_name)

    def erase_file_input(self, exp, data):
        field_name = json.loads(data)["field_name"]
        field = self._block_serializer.params.get(field_name)

        if not field.options.get("multiple", False):
            #  single stored value
            ufw = getattr(self, field_name)
            ud = ufw.ud
            ud.delete()
            setattr(self, field_name, None)
        else:
            # multiple
            ufw_dict = getattr(self, field_name)
            for name, ufw in ufw_dict.items():
                ufw.ud.delete()
            setattr(self, field_name, MultiUploadField())

        exp.store_block(self)

    def add_dyn_input_hook(self, exp, dyn_port, new_port):
        """ to override later
        """
        pass

    def add_input_port(self, new_port):
        self._block_serializer.register(new_port)
        self.input_manager.register(new_port)

    def add_dyn_input(self, exp, received_block, *args, **kwargs):
        spec = received_block.get("_add_dyn_port")
        if not spec:
            return

        if not spec['new_port'] or not spec['input']:
            return

        dyn_port_name = spec['input']
        dyn_port = self._block_serializer.inputs.get(dyn_port_name)
        if not dyn_port:
            return

        order_num = 1000 + abs(dyn_port.order_num) * 10
        dp = getattr(self, dyn_port_name)
        if dp:
            order_num += len(dp)

        new_port = InputBlockField(
            name=spec['new_port'],
            required_data_type=dyn_port.required_data_type,
            order_num=order_num)

        self.add_input_port(new_port)
        getattr(self, dyn_port_name).append(spec["new_port"])

        self.add_dyn_input_hook(exp, dyn_port, new_port)
        exp.store_block(self)

    def validate_params_hook(self, exp, *args, **kwargs):
        return True

    def validate_params(self, exp, *args, **kwargs):
        is_valid = True

        # check required inputs
        if not self.input_manager.validate_inputs(self, self.bound_inputs,
                                                  self.errors, self.warnings):
            is_valid = False
        # check user provided values
        if not self._block_serializer.validate_params(self, exp):
            is_valid = False

        if not self.validate_params_hook(exp, *args, **kwargs):
            is_valid = False

        if is_valid:
            self.errors = []
            self.do_action("on_params_is_valid", exp)
        else:
            self.do_action("on_params_not_valid", exp)

    def on_params_is_valid(self, exp, *args, **kwargs):
        self.errors = []
        exp.store_block(self)

    def on_params_not_valid(self, exp, *args, **kwargs):
        pass

    def clean_errors(self):
        self.errors = []

    def error(self, exp, new_errors=None):
        if isinstance(new_errors, collections.Iterable):
            self.errors.extend(new_errors)
        elif new_errors:
            self.errors.append(new_errors)

        exp.store_block(self)

    def reset_execution(self, exp, *args, **kwargs):
        self.clean_errors()
        exp.store_block(self)
Ejemplo n.º 2
0
class GenericBlock(BaseBlock):
    # block fields
    is_abstract = True

    _uuid = BlockField("uuid", FieldType.STR, None, is_immutable=True)
    _name = BlockField("name", FieldType.STR, None)
    name = "Generic block"

    _base_name = BlockField("base_name", FieldType.STR, "", is_immutable=True)

    _block_group = BlockField("block_group", FieldType.STR, "", is_immutable=True)
    block_group = None

    _exp_id = BlockField("exp_id", FieldType.STR, None, is_immutable=True)

    _scope_name = BlockField("scope_name", FieldType.STR, "root", is_immutable=True)
    _sub_scope_name = BlockField("sub_scope_name", FieldType.STR, None, is_immutable=True)
    _visible_scopes_list = BlockField("visible_scopes_list",
                                      FieldType.SIMPLE_LIST, is_immutable=True)

    _state = BlockField("state", FieldType.STR, "created")

    _ui_folded = BlockField("ui_folded", FieldType.BOOLEAN, init_val=False)
    _ui_internal_folded = BlockField("ui_internal_folded", FieldType.BOOLEAN, init_val=False)
    _show_collector_editor = BlockField("show_collector_editor", FieldType.BOOLEAN, init_val=False)

    _has_custom_layout = BlockField("has_custom_layout", FieldType.BOOLEAN)
    _custom_layout_name = BlockField("custom_layout_name", FieldType.STR)

    _create_new_scope = BlockField("create_new_scope", FieldType.BOOLEAN)
    create_new_scope = False

    is_block_supports_auto_execution = False

    _errors = BlockField("errors", FieldType.SIMPLE_LIST, list())
    _warnings = BlockField("warnings", FieldType.SIMPLE_LIST, list())
    _bound_inputs = BlockField("bound_inputs", FieldType.SIMPLE_DICT, defaultdict())

    def move_to_exp(self, exp_id):
        pass

    def duplicate(self, exp_id, mapping):
        import copy
        old_uuid = self.uuid
        new_obj = copy.deepcopy(self)
        new_obj.uuid = "B" + uuid1().hex[:8]
        new_obj.exp_id = exp_id
        if new_obj.scope_name:
            # little hack, it uses the fact that a scope name has a structure root_uuid1_uuid2....
            parent_uuids = new_obj.scope_name.split('_')
            for parent_uuid in parent_uuids:
                new_obj.scope_name = new_obj.scope_name.replace(parent_uuid, mapping[parent_uuid])
        scope = new_obj.get_scope()
        scope.load()

        for f_name, f in new_obj._block_serializer.outputs.iteritems():
            scope.register_variable(ScopeVar(new_obj.uuid, f_name, f.provided_data_type))
            # log.debug("Registering normal outputs: %s", f_name)
            # new_obj.register_provided_objects(scope, ScopeVar(new_obj.uuid, f_name, f.provided_data_type))
        scope.store()
        return new_obj

    def remap_inputs(self, mapping):
        for var in self.bound_inputs.itervalues():
            var.change_block(mapping)

    def __init__(self, exp_id=None, scope_name=None):
        """
            Building block for workflow
        """
        # TODO: due to dynamic inputs, find better solution
        self._block_serializer = BlockSerializer.clone(self.__class__._block_serializer)

        self.state = "created"
        self.uuid = "B" + uuid1().hex[:8]

        self.exp_id = exp_id
        exp = None
        if exp_id:
            exp = Experiment.get_exp_by_id(exp_id)
        self.scope_name = scope_name
        self.base_name = ""

        # Used only be meta-blocks
        self.children_blocks = []
        # End

        self._out_data = dict()
        self.out_manager = OutManager()

        self.input_manager = InputManager()

        # Automatic execution status map
        self.auto_exec_status_ready = set(["ready"])
        self.auto_exec_status_done = set(["done"])
        self.auto_exec_status_working = set(["working"])
        self.auto_exec_status_error = set(["execution_error"])

        # Init block fields
        for f_name, f in itertools.chain(
                self._block_serializer.fields.iteritems(),
                self._block_serializer.params.iteritems()):

            #if f_name not in self.__dict__ and not f.is_a_property:
            if not f.is_a_property and not hasattr(self, f_name):
                try:
                    setattr(self, f_name, f.init_val)
                except:
                    import ipdb; ipdb.set_trace()

        for f_name, f in self._block_serializer.inputs.iteritems():
            if f.multiply_extensible:
                setattr(self, f_name, [])  # Names of dynamically added ports

        # TODO: Hmm maybe more metaclass magic can be applied here
        scope = self.get_scope()
        scope.load()
        for f_name, f in self._block_serializer.outputs.iteritems():
            if exp:
                exp.log(self.uuid, "Registering normal outputs: %s" % f_name)
            log.debug("Registering normal outputs: %s", f_name)
            self.register_provided_objects(scope, ScopeVar(self.uuid, f_name, f.provided_data_type))
            # TODO: Use factories for init values
            #if f.init_val is not None:
            #    setattr(self, f.name, f.init_val)

        scope.store()

        for f_name, f in self._block_serializer.fields.items():
            if f.init_val is not None:
                #setattr(self, f.name, f.init_val)
                pass

        for f_name, f in self._block_serializer.inputs.iteritems():
            self.input_manager.register(f)

    def on_remove(self, *args, **kwargs):
        """
            Cleanup all created files
            TODO: github:#61
        """
        pass


    def get_exec_status(self):
        if self.state in self.auto_exec_status_done:
            return "done"
        if self.state in self.auto_exec_status_error:
            return "error"
        if self.state in self.auto_exec_status_ready:
            return "ready"

        return "not_ready"

    def bind_input_var(self, input_name, bound_var):
        if self.exp_id:
            exp = Experiment.get_exp_by_id(self.exp_id)
            exp.log(self.uuid, "bound input %s to %s in block: %s, exp: %s" %
                    (input_name, bound_var, self.base_name, self.exp_id))

        log.debug("bound input %s to %s in block: %s, exp: %s",
                  input_name, bound_var, self.base_name, self.exp_id)
        self.bound_inputs[input_name] = bound_var

    def get_input_var(self, name):
        try:
            exp = Experiment.get_exp_by_id(self.exp_id)
            scope_var = self.bound_inputs[name]
            return exp.get_scope_var_value(scope_var)
        except:
            return None

    def get_out_var(self, name):
        if self.out_manager.contains(name):
            return self._out_data.get(name)
        elif self.create_new_scope:
            return self.get_inner_out_var(name)
        else:
            return None

    # def get_inner_out_var(self, name):
    #     raise NotImplementedError("Not implemented in the base class")

    # def set_inner_out_var(self, name, value):
    #     raise NotImplementedError("Not implemented in the base class")

    def set_out_var(self, name, value):
        self._out_data[name] = value

    def get_scope(self):
        exp = Experiment.get_exp_by_id(self.exp_id)
        return Scope(exp, self.scope_name)

    @property
    def sub_scope_name(self):
        if hasattr(self, "create_new_scope") and self.create_new_scope:
            return "%s_%s" % (self.scope_name, self.uuid)
        else:
            return ""

    @property
    def visible_scopes_list(self):
        scope = self.get_scope()
        scope_names_list = scope.get_parent_scope_list()
        scope_names_list.append(self.scope_name)
        return scope_names_list

    def get_sub_scope(self):
        exp = Experiment.get_exp_by_id(self.exp_id)
        return Scope(exp, self.sub_scope_name)

    def reset_execution_for_sub_blocks(self):
        exp = Experiment.get_exp_by_id(self.exp_id)
        for block_uuid, block in exp.get_blocks(self.children_blocks):
            block.do_action("reset_execution", exp)

    def get_input_blocks(self):
        required_blocks = []
        for f in self.input_manager.input_fields:
            if f.multiply_extensible:
                continue
            if self.bound_inputs.get(f.name) is None and f.required:
                raise RuntimeError("Not all required inputs are bound")
            elif self.bound_inputs.get(f.name):
                required_blocks.append(self.bound_inputs[f.name].block_uuid)
        return required_blocks

    def get_user_actions(self):
        """
            @rtype: list of workflow.blocks.fields.ActionRecord
        """
        return self._trans.user_visible(self.state)

    @log_timing
    def to_dict(self):
        result = self._block_serializer.to_dict(self)
        # import ipdb; ipdb.set_trace()
        return result

    def register_provided_objects(self, scope, scope_var):
        self.out_manager.register(scope_var.var_name, scope_var.data_type)
        scope.register_variable(scope_var)

    @log_timing
    def apply_action_from_js(self, action_name, *args, **kwargs):
        if self._trans.is_action_available(self.state, action_name):
            self.do_action(action_name, *args, **kwargs)
        elif hasattr(self, action_name) and hasattr(getattr(self, action_name), "__call__"):
            return getattr(self, action_name)(*args, **kwargs)
        else:
            raise RuntimeError("Block %s doesn't have action: %s" % (self.name, action_name))

    def do_action(self, action_name, exp, *args, **kwargs):
        # if action_name == "success" and self.block_base_name == "CROSS_VALID":
        #     from celery.contrib import rdb; rdb.set_trace()
        ar = self._trans.action_records_by_name[action_name]
        old_exec_state = self.get_exec_status()
        next_state = self._trans.next_state(self.state, action_name)

        if next_state is not None:
            log.debug("Do action: %s in block %s from state %s -> %s",
                      action_name, self.base_name, self.state, next_state)
            exp.log(self.uuid, "Do action: %s in block %s from state %s -> %s" %
                    (action_name, self.base_name, self.state, next_state))
            self.state = next_state

            if old_exec_state != "done" and self.get_exec_status() == "done":
                if self.is_block_supports_auto_execution:
                    BlockUpdated(self.exp_id,
                                 block_uuid=self.uuid, block_alias=self.base_name,
                                 silent=True).send()
            exp.store_block(self)
            getattr(self, action_name)(exp, *args, **kwargs)

            if ar.reload_block_in_client:
                BlockUpdated(self.exp_id, self.uuid, self.base_name).send()

            # TODO: Check if self.scope_name is actually set to auto execution
            #
            if old_exec_state != "done" and self.get_exec_status() == "done" \
                    and ar.propagate_auto_execution \
                    and self.is_block_supports_auto_execution:
                exp.log(self.uuid, "Propagate execution: %s " % self.base_name)
                log.debug("Propagate execution: %s ", self.base_name)
                auto_exec_task.s(exp, self.scope_name).apply_async()
            elif self.state in self.auto_exec_status_error \
                    and self.is_block_supports_auto_execution:
                exp.log(self.uuid, "Detected error during automated workflow execution")
                log.debug("Detected error during automated workflow execution")
                halt_execution_task.s(exp, self.scope_name).apply_async()

        else:
            raise RuntimeError("Action %s isn't available for block %s in state %s" %
                               (action_name, self.base_name, self.state))

    def change_base_name(self, exp, received_block, *args, **kwargs):
        # TODO: check if the name is correct
        new_name = received_block.get("base_name")

        if new_name:
            exp.change_block_alias(self, new_name)

    def toggle_ui_folded(self, exp, received_block, *args, **kwargs):
        self.ui_folded = received_block["ui_folded"]
        exp.store_block(self)

    def save_params(self, exp, received_block=None, *args, **kwargs):
        self._block_serializer.save_params(self, received_block)
        exp.store_block(self)
        self.validate_params(exp)

    def save_file_input(self, exp, field_name, file_obj, multiple=False, upload_meta=None):
        if upload_meta is None:
            upload_meta = {}

        if not hasattr(self, field_name):
            raise Exception("Block doesn't have field: %s" % field_name)

        orig_name = file_obj.name
        local_filename = "%s_%s_%s" % (self.uuid[:8], field_name, file_obj.name)

        if not multiple:
            exp.log(self.uuid, "Storing single upload to field: %s" % field_name)
            log.debug("Storing single upload to field: %s", field_name)
            ud, is_created = UploadedData.objects.get_or_create(
                exp=exp, block_uuid=self.uuid, var_name=field_name)

            file_obj.name = local_filename
            ud.data = file_obj
            ud.save()

            ufw = UploadedFileWrapper(ud.pk)
            ufw.orig_name = orig_name
            setattr(self, field_name, ufw)
            exp.store_block(self)
        else:
            exp.log(self.uuid, "Adding upload to field: %s" % field_name)
            log.debug("Adding upload to field: %s", field_name)

            ud, is_created = UploadedData.objects.get_or_create(
                exp=exp, block_uuid=self.uuid, var_name=field_name, filename=orig_name)

            file_obj.name = local_filename
            ud.data = file_obj
            ud.filename = orig_name
            ud.save()

            ufw = UploadedFileWrapper(ud.pk)
            ufw.orig_name = orig_name

            r = get_redis_instance()
            with redis_lock.Lock(r, ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)):
                exp.log(self.uuid, "Enter lock, file: %s" % orig_name)
                log.debug("Enter lock, file: %s", orig_name)
                block = exp.get_block(self.uuid)
                attr = getattr(block, field_name)

                attr[orig_name] = ufw
                exp.log(self.uuid, "Added upload `%s` to collection: %s" % (orig_name, attr.keys()))
                log.debug("Added upload `%s` to collection: %s", orig_name, attr.keys())
                exp.store_block(block)
                exp.log(self.uuid, "Exit lock, file: %s" % orig_name)
                log.debug("Exit lock, file: %s", orig_name)

    def erase_file_input(self, exp, data):
        field_name = json.loads(data)["field_name"]
        field = self._block_serializer.params.get(field_name)

        if not field.options.get("multiple", False):
            #  single stored value
            ufw = getattr(self, field_name)
            ud = ufw.ud
            ud.delete()
            setattr(self, field_name, None)
        else:
            # multiple
            ufw_dict = getattr(self, field_name)
            for name, ufw in ufw_dict.items():
                ufw.ud.delete()
            setattr(self, field_name, MultiUploadField())

        exp.store_block(self)

    def add_dyn_input_hook(self, exp, dyn_port, new_port):
        """ to override later
        """
        pass

    def add_input_port(self, new_port):
        self._block_serializer.register(new_port)
        self.input_manager.register(new_port)

    def add_dyn_input(self, exp, received_block, *args, **kwargs):
        spec = received_block.get("_add_dyn_port")
        if not spec:
            return

        if not spec['new_port'] or not spec['input']:
            return

        dyn_port_name = spec['input']
        dyn_port = self._block_serializer.inputs.get(dyn_port_name)
        if not dyn_port:
            return

        order_num = 1000 + abs(dyn_port.order_num) * 10
        dp = getattr(self, dyn_port_name)
        if dp:
            order_num += len(dp)

        new_port = InputBlockField(
            name=spec['new_port'],
            required_data_type=dyn_port.required_data_type,
            order_num=order_num
        )

        self.add_input_port(new_port)
        getattr(self, dyn_port_name).append(spec["new_port"])

        self.add_dyn_input_hook(exp, dyn_port, new_port)
        exp.store_block(self)

    def validate_params_hook(self, exp, *args, **kwargs):
        return True

    def validate_params(self, exp, *args, **kwargs):
        is_valid = True

        # check required inputs
        if not self.input_manager.validate_inputs(
                self, self.bound_inputs, self.errors, self.warnings):
            is_valid = False
        # check user provided values
        if not self._block_serializer.validate_params(self, exp):
            is_valid = False

        if not self.validate_params_hook(exp, *args, **kwargs):
            is_valid = False

        if is_valid:
            self.errors = []
            self.do_action("on_params_is_valid", exp)
        else:
            self.do_action("on_params_not_valid", exp)

    def on_params_is_valid(self, exp, *args, **kwargs):
        self.errors = []
        exp.store_block(self)

    def on_params_not_valid(self, exp, *args, **kwargs):
        pass

    def clean_errors(self):
        self.errors = []

    def error(self, exp, new_errors=None):
        if isinstance(new_errors, collections.Iterable):
            self.errors.extend(new_errors)
        elif new_errors:
            self.errors.append(new_errors)

        exp.store_block(self)

    def reset_execution(self, exp, *args, **kwargs):
        self.clean_errors()
        exp.store_block(self)
Ejemplo n.º 3
0
    def __init__(self, exp_id=None, scope_name=None):
        """
            Building block for workflow
        """
        # TODO: due to dynamic inputs, find better solution
        self._block_serializer = BlockSerializer.clone(
            self.__class__._block_serializer)

        self.state = "created"
        self.uuid = "B" + uuid1().hex[:8]

        self.exp_id = exp_id
        exp = None
        if exp_id:
            exp = Experiment.get_exp_by_id(exp_id)
        self.scope_name = scope_name
        self.base_name = ""

        # Used only be meta-blocks
        self.children_blocks = []
        # End

        self._out_data = dict()
        self.out_manager = OutManager()

        self.input_manager = InputManager()

        # Automatic execution status map
        self.auto_exec_status_ready = set(["ready"])
        self.auto_exec_status_done = set(["done"])
        self.auto_exec_status_working = set(["working"])
        self.auto_exec_status_error = set(["execution_error"])

        # Init block fields
        for f_name, f in itertools.chain(
                self._block_serializer.fields.iteritems(),
                self._block_serializer.params.iteritems()):

            #if f_name not in self.__dict__ and not f.is_a_property:
            if not f.is_a_property and not hasattr(self, f_name):
                try:
                    setattr(self, f_name, f.init_val)
                except:
                    import ipdb
                    ipdb.set_trace()

        for f_name, f in self._block_serializer.inputs.iteritems():
            if f.multiply_extensible:
                setattr(self, f_name, [])  # Names of dynamically added ports

        # TODO: Hmm maybe more metaclass magic can be applied here
        scope = self.get_scope()
        scope.load()
        for f_name, f in self._block_serializer.outputs.iteritems():
            if exp:
                exp.log(self.uuid, "Registering normal outputs: %s" % f_name)
            log.debug("Registering normal outputs: %s", f_name)
            self.register_provided_objects(
                scope, ScopeVar(self.uuid, f_name, f.provided_data_type))
            # TODO: Use factories for init values
            #if f.init_val is not None:
            #    setattr(self, f.name, f.init_val)

        scope.store()

        for f_name, f in self._block_serializer.fields.items():
            if f.init_val is not None:
                #setattr(self, f.name, f.init_val)
                pass

        for f_name, f in self._block_serializer.inputs.iteritems():
            self.input_manager.register(f)
Ejemplo n.º 4
0
    def __init__(self, exp_id=None, scope_name=None):
        """
            Building block for workflow
        """
        # TODO: due to dynamic inputs, find better solution
        self._block_serializer = BlockSerializer.clone(self.__class__._block_serializer)

        self.state = "created"
        self.uuid = "B" + uuid1().hex[:8]

        self.exp_id = exp_id
        exp = None
        if exp_id:
            exp = Experiment.get_exp_by_id(exp_id)
        self.scope_name = scope_name
        self.base_name = ""

        # Used only be meta-blocks
        self.children_blocks = []
        # End

        self._out_data = dict()
        self.out_manager = OutManager()

        self.input_manager = InputManager()

        # Automatic execution status map
        self.auto_exec_status_ready = set(["ready"])
        self.auto_exec_status_done = set(["done"])
        self.auto_exec_status_working = set(["working"])
        self.auto_exec_status_error = set(["execution_error"])

        # Init block fields
        for f_name, f in itertools.chain(
                self._block_serializer.fields.iteritems(),
                self._block_serializer.params.iteritems()):

            #if f_name not in self.__dict__ and not f.is_a_property:
            if not f.is_a_property and not hasattr(self, f_name):
                try:
                    setattr(self, f_name, f.init_val)
                except:
                    import ipdb; ipdb.set_trace()

        for f_name, f in self._block_serializer.inputs.iteritems():
            if f.multiply_extensible:
                setattr(self, f_name, [])  # Names of dynamically added ports

        # TODO: Hmm maybe more metaclass magic can be applied here
        scope = self.get_scope()
        scope.load()
        for f_name, f in self._block_serializer.outputs.iteritems():
            if exp:
                exp.log(self.uuid, "Registering normal outputs: %s" % f_name)
            log.debug("Registering normal outputs: %s", f_name)
            self.register_provided_objects(scope, ScopeVar(self.uuid, f_name, f.provided_data_type))
            # TODO: Use factories for init values
            #if f.init_val is not None:
            #    setattr(self, f.name, f.init_val)

        scope.store()

        for f_name, f in self._block_serializer.fields.items():
            if f.init_val is not None:
                #setattr(self, f.name, f.init_val)
                pass

        for f_name, f in self._block_serializer.inputs.iteritems():
            self.input_manager.register(f)