class GenericBlock(BaseBlock): # block fields is_abstract = True _uuid = BlockField("uuid", FieldType.STR, None, is_immutable=True) _name = BlockField("name", FieldType.STR, None) name = "Generic block" _base_name = BlockField("base_name", FieldType.STR, "", is_immutable=True) _block_group = BlockField("block_group", FieldType.STR, "", is_immutable=True) block_group = None _exp_id = BlockField("exp_id", FieldType.STR, None, is_immutable=True) _scope_name = BlockField("scope_name", FieldType.STR, "root", is_immutable=True) _sub_scope_name = BlockField("sub_scope_name", FieldType.STR, None, is_immutable=True) _visible_scopes_list = BlockField("visible_scopes_list", FieldType.SIMPLE_LIST, is_immutable=True) _state = BlockField("state", FieldType.STR, "created") _ui_folded = BlockField("ui_folded", FieldType.BOOLEAN, init_val=False) _ui_internal_folded = BlockField("ui_internal_folded", FieldType.BOOLEAN, init_val=False) _show_collector_editor = BlockField("show_collector_editor", FieldType.BOOLEAN, init_val=False) _has_custom_layout = BlockField("has_custom_layout", FieldType.BOOLEAN) _custom_layout_name = BlockField("custom_layout_name", FieldType.STR) _create_new_scope = BlockField("create_new_scope", FieldType.BOOLEAN) create_new_scope = False is_block_supports_auto_execution = False _errors = BlockField("errors", FieldType.SIMPLE_LIST, list()) _warnings = BlockField("warnings", FieldType.SIMPLE_LIST, list()) _bound_inputs = BlockField("bound_inputs", FieldType.SIMPLE_DICT, defaultdict()) def move_to_exp(self, exp_id): pass def duplicate(self, exp_id, mapping): import copy old_uuid = self.uuid new_obj = copy.deepcopy(self) new_obj.uuid = "B" + uuid1().hex[:8] new_obj.exp_id = exp_id if new_obj.scope_name: # little hack, it uses the fact that a scope name has a structure root_uuid1_uuid2.... parent_uuids = new_obj.scope_name.split('_') for parent_uuid in parent_uuids: new_obj.scope_name = new_obj.scope_name.replace( parent_uuid, mapping[parent_uuid]) scope = new_obj.get_scope() scope.load() for f_name, f in new_obj._block_serializer.outputs.iteritems(): scope.register_variable( ScopeVar(new_obj.uuid, f_name, f.provided_data_type)) # log.debug("Registering normal outputs: %s", f_name) # new_obj.register_provided_objects(scope, ScopeVar(new_obj.uuid, f_name, f.provided_data_type)) scope.store() return new_obj def remap_inputs(self, mapping): for var in self.bound_inputs.itervalues(): var.change_block(mapping) def __init__(self, exp_id=None, scope_name=None): """ Building block for workflow """ # TODO: due to dynamic inputs, find better solution self._block_serializer = BlockSerializer.clone( self.__class__._block_serializer) self.state = "created" self.uuid = "B" + uuid1().hex[:8] self.exp_id = exp_id exp = None if exp_id: exp = Experiment.get_exp_by_id(exp_id) self.scope_name = scope_name self.base_name = "" # Used only be meta-blocks self.children_blocks = [] # End self._out_data = dict() self.out_manager = OutManager() self.input_manager = InputManager() # Automatic execution status map self.auto_exec_status_ready = set(["ready"]) self.auto_exec_status_done = set(["done"]) self.auto_exec_status_working = set(["working"]) self.auto_exec_status_error = set(["execution_error"]) # Init block fields for f_name, f in itertools.chain( self._block_serializer.fields.iteritems(), self._block_serializer.params.iteritems()): #if f_name not in self.__dict__ and not f.is_a_property: if not f.is_a_property and not hasattr(self, f_name): try: setattr(self, f_name, f.init_val) except: import ipdb ipdb.set_trace() for f_name, f in self._block_serializer.inputs.iteritems(): if f.multiply_extensible: setattr(self, f_name, []) # Names of dynamically added ports # TODO: Hmm maybe more metaclass magic can be applied here scope = self.get_scope() scope.load() for f_name, f in self._block_serializer.outputs.iteritems(): if exp: exp.log(self.uuid, "Registering normal outputs: %s" % f_name) log.debug("Registering normal outputs: %s", f_name) self.register_provided_objects( scope, ScopeVar(self.uuid, f_name, f.provided_data_type)) # TODO: Use factories for init values #if f.init_val is not None: # setattr(self, f.name, f.init_val) scope.store() for f_name, f in self._block_serializer.fields.items(): if f.init_val is not None: #setattr(self, f.name, f.init_val) pass for f_name, f in self._block_serializer.inputs.iteritems(): self.input_manager.register(f) def on_remove(self, *args, **kwargs): """ Cleanup all created files TODO: github:#61 """ pass def get_exec_status(self): if self.state in self.auto_exec_status_done: return "done" if self.state in self.auto_exec_status_error: return "error" if self.state in self.auto_exec_status_ready: return "ready" return "not_ready" def bind_input_var(self, input_name, bound_var): if self.exp_id: exp = Experiment.get_exp_by_id(self.exp_id) exp.log( self.uuid, "bound input %s to %s in block: %s, exp: %s" % (input_name, bound_var, self.base_name, self.exp_id)) log.debug("bound input %s to %s in block: %s, exp: %s", input_name, bound_var, self.base_name, self.exp_id) self.bound_inputs[input_name] = bound_var def get_input_var(self, name): try: exp = Experiment.get_exp_by_id(self.exp_id) scope_var = self.bound_inputs[name] return exp.get_scope_var_value(scope_var) except: return None def get_out_var(self, name): if self.out_manager.contains(name): return self._out_data.get(name) elif self.create_new_scope: return self.get_inner_out_var(name) else: return None # def get_inner_out_var(self, name): # raise NotImplementedError("Not implemented in the base class") # def set_inner_out_var(self, name, value): # raise NotImplementedError("Not implemented in the base class") def set_out_var(self, name, value): self._out_data[name] = value def get_scope(self): exp = Experiment.get_exp_by_id(self.exp_id) return Scope(exp, self.scope_name) @property def sub_scope_name(self): if hasattr(self, "create_new_scope") and self.create_new_scope: return "%s_%s" % (self.scope_name, self.uuid) else: return "" @property def visible_scopes_list(self): scope = self.get_scope() scope_names_list = scope.get_parent_scope_list() scope_names_list.append(self.scope_name) return scope_names_list def get_sub_scope(self): exp = Experiment.get_exp_by_id(self.exp_id) return Scope(exp, self.sub_scope_name) def reset_execution_for_sub_blocks(self): exp = Experiment.get_exp_by_id(self.exp_id) for block_uuid, block in exp.get_blocks(self.children_blocks): block.do_action("reset_execution", exp) def get_input_blocks(self): required_blocks = [] for f in self.input_manager.input_fields: if f.multiply_extensible: continue if self.bound_inputs.get(f.name) is None and f.required: raise RuntimeError("Not all required inputs are bound") elif self.bound_inputs.get(f.name): required_blocks.append(self.bound_inputs[f.name].block_uuid) return required_blocks def get_user_actions(self): """ @rtype: list of workflow.blocks.fields.ActionRecord """ return self._trans.user_visible(self.state) @log_timing def to_dict(self): result = self._block_serializer.to_dict(self) # import ipdb; ipdb.set_trace() return result def register_provided_objects(self, scope, scope_var): self.out_manager.register(scope_var.var_name, scope_var.data_type) scope.register_variable(scope_var) @log_timing def apply_action_from_js(self, action_name, *args, **kwargs): if self._trans.is_action_available(self.state, action_name): self.do_action(action_name, *args, **kwargs) elif hasattr(self, action_name) and hasattr(getattr(self, action_name), "__call__"): return getattr(self, action_name)(*args, **kwargs) else: raise RuntimeError("Block %s doesn't have action: %s" % (self.name, action_name)) def do_action(self, action_name, exp, *args, **kwargs): # if action_name == "success" and self.block_base_name == "CROSS_VALID": # from celery.contrib import rdb; rdb.set_trace() ar = self._trans.action_records_by_name[action_name] old_exec_state = self.get_exec_status() next_state = self._trans.next_state(self.state, action_name) if next_state is not None: log.debug("Do action: %s in block %s from state %s -> %s", action_name, self.base_name, self.state, next_state) exp.log( self.uuid, "Do action: %s in block %s from state %s -> %s" % (action_name, self.base_name, self.state, next_state)) self.state = next_state if old_exec_state != "done" and self.get_exec_status() == "done": if self.is_block_supports_auto_execution: BlockUpdated(self.exp_id, block_uuid=self.uuid, block_alias=self.base_name, silent=True).send() exp.store_block(self) getattr(self, action_name)(exp, *args, **kwargs) if ar.reload_block_in_client: BlockUpdated(self.exp_id, self.uuid, self.base_name).send() # TODO: Check if self.scope_name is actually set to auto execution # if old_exec_state != "done" and self.get_exec_status() == "done" \ and ar.propagate_auto_execution \ and self.is_block_supports_auto_execution: exp.log(self.uuid, "Propagate execution: %s " % self.base_name) log.debug("Propagate execution: %s ", self.base_name) auto_exec_task.s(exp, self.scope_name).apply_async() elif self.state in self.auto_exec_status_error \ and self.is_block_supports_auto_execution: exp.log(self.uuid, "Detected error during automated workflow execution") log.debug("Detected error during automated workflow execution") halt_execution_task.s(exp, self.scope_name).apply_async() else: raise RuntimeError( "Action %s isn't available for block %s in state %s" % (action_name, self.base_name, self.state)) def change_base_name(self, exp, received_block, *args, **kwargs): # TODO: check if the name is correct new_name = received_block.get("base_name") if new_name: exp.change_block_alias(self, new_name) def toggle_ui_folded(self, exp, received_block, *args, **kwargs): self.ui_folded = received_block["ui_folded"] exp.store_block(self) def save_params(self, exp, received_block=None, *args, **kwargs): self._block_serializer.save_params(self, received_block) exp.store_block(self) self.validate_params(exp) def save_file_input(self, exp, field_name, file_obj, multiple=False, upload_meta=None): if upload_meta is None: upload_meta = {} if not hasattr(self, field_name): raise Exception("Block doesn't have field: %s" % field_name) orig_name = file_obj.name local_filename = "%s_%s_%s" % (self.uuid[:8], field_name, file_obj.name) if not multiple: exp.log(self.uuid, "Storing single upload to field: %s" % field_name) log.debug("Storing single upload to field: %s", field_name) ud, is_created = UploadedData.objects.get_or_create( exp=exp, block_uuid=self.uuid, var_name=field_name) file_obj.name = local_filename ud.data = file_obj ud.save() ufw = UploadedFileWrapper(ud.pk) ufw.orig_name = orig_name setattr(self, field_name, ufw) exp.store_block(self) else: exp.log(self.uuid, "Adding upload to field: %s" % field_name) log.debug("Adding upload to field: %s", field_name) ud, is_created = UploadedData.objects.get_or_create( exp=exp, block_uuid=self.uuid, var_name=field_name, filename=orig_name) file_obj.name = local_filename ud.data = file_obj ud.filename = orig_name ud.save() ufw = UploadedFileWrapper(ud.pk) ufw.orig_name = orig_name r = get_redis_instance() with redis_lock.Lock( r, ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)): exp.log(self.uuid, "Enter lock, file: %s" % orig_name) log.debug("Enter lock, file: %s", orig_name) block = exp.get_block(self.uuid) attr = getattr(block, field_name) attr[orig_name] = ufw exp.log( self.uuid, "Added upload `%s` to collection: %s" % (orig_name, attr.keys())) log.debug("Added upload `%s` to collection: %s", orig_name, attr.keys()) exp.store_block(block) exp.log(self.uuid, "Exit lock, file: %s" % orig_name) log.debug("Exit lock, file: %s", orig_name) def erase_file_input(self, exp, data): field_name = json.loads(data)["field_name"] field = self._block_serializer.params.get(field_name) if not field.options.get("multiple", False): # single stored value ufw = getattr(self, field_name) ud = ufw.ud ud.delete() setattr(self, field_name, None) else: # multiple ufw_dict = getattr(self, field_name) for name, ufw in ufw_dict.items(): ufw.ud.delete() setattr(self, field_name, MultiUploadField()) exp.store_block(self) def add_dyn_input_hook(self, exp, dyn_port, new_port): """ to override later """ pass def add_input_port(self, new_port): self._block_serializer.register(new_port) self.input_manager.register(new_port) def add_dyn_input(self, exp, received_block, *args, **kwargs): spec = received_block.get("_add_dyn_port") if not spec: return if not spec['new_port'] or not spec['input']: return dyn_port_name = spec['input'] dyn_port = self._block_serializer.inputs.get(dyn_port_name) if not dyn_port: return order_num = 1000 + abs(dyn_port.order_num) * 10 dp = getattr(self, dyn_port_name) if dp: order_num += len(dp) new_port = InputBlockField( name=spec['new_port'], required_data_type=dyn_port.required_data_type, order_num=order_num) self.add_input_port(new_port) getattr(self, dyn_port_name).append(spec["new_port"]) self.add_dyn_input_hook(exp, dyn_port, new_port) exp.store_block(self) def validate_params_hook(self, exp, *args, **kwargs): return True def validate_params(self, exp, *args, **kwargs): is_valid = True # check required inputs if not self.input_manager.validate_inputs(self, self.bound_inputs, self.errors, self.warnings): is_valid = False # check user provided values if not self._block_serializer.validate_params(self, exp): is_valid = False if not self.validate_params_hook(exp, *args, **kwargs): is_valid = False if is_valid: self.errors = [] self.do_action("on_params_is_valid", exp) else: self.do_action("on_params_not_valid", exp) def on_params_is_valid(self, exp, *args, **kwargs): self.errors = [] exp.store_block(self) def on_params_not_valid(self, exp, *args, **kwargs): pass def clean_errors(self): self.errors = [] def error(self, exp, new_errors=None): if isinstance(new_errors, collections.Iterable): self.errors.extend(new_errors) elif new_errors: self.errors.append(new_errors) exp.store_block(self) def reset_execution(self, exp, *args, **kwargs): self.clean_errors() exp.store_block(self)
class GenericBlock(BaseBlock): # block fields is_abstract = True _uuid = BlockField("uuid", FieldType.STR, None, is_immutable=True) _name = BlockField("name", FieldType.STR, None) name = "Generic block" _base_name = BlockField("base_name", FieldType.STR, "", is_immutable=True) _block_group = BlockField("block_group", FieldType.STR, "", is_immutable=True) block_group = None _exp_id = BlockField("exp_id", FieldType.STR, None, is_immutable=True) _scope_name = BlockField("scope_name", FieldType.STR, "root", is_immutable=True) _sub_scope_name = BlockField("sub_scope_name", FieldType.STR, None, is_immutable=True) _visible_scopes_list = BlockField("visible_scopes_list", FieldType.SIMPLE_LIST, is_immutable=True) _state = BlockField("state", FieldType.STR, "created") _ui_folded = BlockField("ui_folded", FieldType.BOOLEAN, init_val=False) _ui_internal_folded = BlockField("ui_internal_folded", FieldType.BOOLEAN, init_val=False) _show_collector_editor = BlockField("show_collector_editor", FieldType.BOOLEAN, init_val=False) _has_custom_layout = BlockField("has_custom_layout", FieldType.BOOLEAN) _custom_layout_name = BlockField("custom_layout_name", FieldType.STR) _create_new_scope = BlockField("create_new_scope", FieldType.BOOLEAN) create_new_scope = False is_block_supports_auto_execution = False _errors = BlockField("errors", FieldType.SIMPLE_LIST, list()) _warnings = BlockField("warnings", FieldType.SIMPLE_LIST, list()) _bound_inputs = BlockField("bound_inputs", FieldType.SIMPLE_DICT, defaultdict()) def move_to_exp(self, exp_id): pass def duplicate(self, exp_id, mapping): import copy old_uuid = self.uuid new_obj = copy.deepcopy(self) new_obj.uuid = "B" + uuid1().hex[:8] new_obj.exp_id = exp_id if new_obj.scope_name: # little hack, it uses the fact that a scope name has a structure root_uuid1_uuid2.... parent_uuids = new_obj.scope_name.split('_') for parent_uuid in parent_uuids: new_obj.scope_name = new_obj.scope_name.replace(parent_uuid, mapping[parent_uuid]) scope = new_obj.get_scope() scope.load() for f_name, f in new_obj._block_serializer.outputs.iteritems(): scope.register_variable(ScopeVar(new_obj.uuid, f_name, f.provided_data_type)) # log.debug("Registering normal outputs: %s", f_name) # new_obj.register_provided_objects(scope, ScopeVar(new_obj.uuid, f_name, f.provided_data_type)) scope.store() return new_obj def remap_inputs(self, mapping): for var in self.bound_inputs.itervalues(): var.change_block(mapping) def __init__(self, exp_id=None, scope_name=None): """ Building block for workflow """ # TODO: due to dynamic inputs, find better solution self._block_serializer = BlockSerializer.clone(self.__class__._block_serializer) self.state = "created" self.uuid = "B" + uuid1().hex[:8] self.exp_id = exp_id exp = None if exp_id: exp = Experiment.get_exp_by_id(exp_id) self.scope_name = scope_name self.base_name = "" # Used only be meta-blocks self.children_blocks = [] # End self._out_data = dict() self.out_manager = OutManager() self.input_manager = InputManager() # Automatic execution status map self.auto_exec_status_ready = set(["ready"]) self.auto_exec_status_done = set(["done"]) self.auto_exec_status_working = set(["working"]) self.auto_exec_status_error = set(["execution_error"]) # Init block fields for f_name, f in itertools.chain( self._block_serializer.fields.iteritems(), self._block_serializer.params.iteritems()): #if f_name not in self.__dict__ and not f.is_a_property: if not f.is_a_property and not hasattr(self, f_name): try: setattr(self, f_name, f.init_val) except: import ipdb; ipdb.set_trace() for f_name, f in self._block_serializer.inputs.iteritems(): if f.multiply_extensible: setattr(self, f_name, []) # Names of dynamically added ports # TODO: Hmm maybe more metaclass magic can be applied here scope = self.get_scope() scope.load() for f_name, f in self._block_serializer.outputs.iteritems(): if exp: exp.log(self.uuid, "Registering normal outputs: %s" % f_name) log.debug("Registering normal outputs: %s", f_name) self.register_provided_objects(scope, ScopeVar(self.uuid, f_name, f.provided_data_type)) # TODO: Use factories for init values #if f.init_val is not None: # setattr(self, f.name, f.init_val) scope.store() for f_name, f in self._block_serializer.fields.items(): if f.init_val is not None: #setattr(self, f.name, f.init_val) pass for f_name, f in self._block_serializer.inputs.iteritems(): self.input_manager.register(f) def on_remove(self, *args, **kwargs): """ Cleanup all created files TODO: github:#61 """ pass def get_exec_status(self): if self.state in self.auto_exec_status_done: return "done" if self.state in self.auto_exec_status_error: return "error" if self.state in self.auto_exec_status_ready: return "ready" return "not_ready" def bind_input_var(self, input_name, bound_var): if self.exp_id: exp = Experiment.get_exp_by_id(self.exp_id) exp.log(self.uuid, "bound input %s to %s in block: %s, exp: %s" % (input_name, bound_var, self.base_name, self.exp_id)) log.debug("bound input %s to %s in block: %s, exp: %s", input_name, bound_var, self.base_name, self.exp_id) self.bound_inputs[input_name] = bound_var def get_input_var(self, name): try: exp = Experiment.get_exp_by_id(self.exp_id) scope_var = self.bound_inputs[name] return exp.get_scope_var_value(scope_var) except: return None def get_out_var(self, name): if self.out_manager.contains(name): return self._out_data.get(name) elif self.create_new_scope: return self.get_inner_out_var(name) else: return None # def get_inner_out_var(self, name): # raise NotImplementedError("Not implemented in the base class") # def set_inner_out_var(self, name, value): # raise NotImplementedError("Not implemented in the base class") def set_out_var(self, name, value): self._out_data[name] = value def get_scope(self): exp = Experiment.get_exp_by_id(self.exp_id) return Scope(exp, self.scope_name) @property def sub_scope_name(self): if hasattr(self, "create_new_scope") and self.create_new_scope: return "%s_%s" % (self.scope_name, self.uuid) else: return "" @property def visible_scopes_list(self): scope = self.get_scope() scope_names_list = scope.get_parent_scope_list() scope_names_list.append(self.scope_name) return scope_names_list def get_sub_scope(self): exp = Experiment.get_exp_by_id(self.exp_id) return Scope(exp, self.sub_scope_name) def reset_execution_for_sub_blocks(self): exp = Experiment.get_exp_by_id(self.exp_id) for block_uuid, block in exp.get_blocks(self.children_blocks): block.do_action("reset_execution", exp) def get_input_blocks(self): required_blocks = [] for f in self.input_manager.input_fields: if f.multiply_extensible: continue if self.bound_inputs.get(f.name) is None and f.required: raise RuntimeError("Not all required inputs are bound") elif self.bound_inputs.get(f.name): required_blocks.append(self.bound_inputs[f.name].block_uuid) return required_blocks def get_user_actions(self): """ @rtype: list of workflow.blocks.fields.ActionRecord """ return self._trans.user_visible(self.state) @log_timing def to_dict(self): result = self._block_serializer.to_dict(self) # import ipdb; ipdb.set_trace() return result def register_provided_objects(self, scope, scope_var): self.out_manager.register(scope_var.var_name, scope_var.data_type) scope.register_variable(scope_var) @log_timing def apply_action_from_js(self, action_name, *args, **kwargs): if self._trans.is_action_available(self.state, action_name): self.do_action(action_name, *args, **kwargs) elif hasattr(self, action_name) and hasattr(getattr(self, action_name), "__call__"): return getattr(self, action_name)(*args, **kwargs) else: raise RuntimeError("Block %s doesn't have action: %s" % (self.name, action_name)) def do_action(self, action_name, exp, *args, **kwargs): # if action_name == "success" and self.block_base_name == "CROSS_VALID": # from celery.contrib import rdb; rdb.set_trace() ar = self._trans.action_records_by_name[action_name] old_exec_state = self.get_exec_status() next_state = self._trans.next_state(self.state, action_name) if next_state is not None: log.debug("Do action: %s in block %s from state %s -> %s", action_name, self.base_name, self.state, next_state) exp.log(self.uuid, "Do action: %s in block %s from state %s -> %s" % (action_name, self.base_name, self.state, next_state)) self.state = next_state if old_exec_state != "done" and self.get_exec_status() == "done": if self.is_block_supports_auto_execution: BlockUpdated(self.exp_id, block_uuid=self.uuid, block_alias=self.base_name, silent=True).send() exp.store_block(self) getattr(self, action_name)(exp, *args, **kwargs) if ar.reload_block_in_client: BlockUpdated(self.exp_id, self.uuid, self.base_name).send() # TODO: Check if self.scope_name is actually set to auto execution # if old_exec_state != "done" and self.get_exec_status() == "done" \ and ar.propagate_auto_execution \ and self.is_block_supports_auto_execution: exp.log(self.uuid, "Propagate execution: %s " % self.base_name) log.debug("Propagate execution: %s ", self.base_name) auto_exec_task.s(exp, self.scope_name).apply_async() elif self.state in self.auto_exec_status_error \ and self.is_block_supports_auto_execution: exp.log(self.uuid, "Detected error during automated workflow execution") log.debug("Detected error during automated workflow execution") halt_execution_task.s(exp, self.scope_name).apply_async() else: raise RuntimeError("Action %s isn't available for block %s in state %s" % (action_name, self.base_name, self.state)) def change_base_name(self, exp, received_block, *args, **kwargs): # TODO: check if the name is correct new_name = received_block.get("base_name") if new_name: exp.change_block_alias(self, new_name) def toggle_ui_folded(self, exp, received_block, *args, **kwargs): self.ui_folded = received_block["ui_folded"] exp.store_block(self) def save_params(self, exp, received_block=None, *args, **kwargs): self._block_serializer.save_params(self, received_block) exp.store_block(self) self.validate_params(exp) def save_file_input(self, exp, field_name, file_obj, multiple=False, upload_meta=None): if upload_meta is None: upload_meta = {} if not hasattr(self, field_name): raise Exception("Block doesn't have field: %s" % field_name) orig_name = file_obj.name local_filename = "%s_%s_%s" % (self.uuid[:8], field_name, file_obj.name) if not multiple: exp.log(self.uuid, "Storing single upload to field: %s" % field_name) log.debug("Storing single upload to field: %s", field_name) ud, is_created = UploadedData.objects.get_or_create( exp=exp, block_uuid=self.uuid, var_name=field_name) file_obj.name = local_filename ud.data = file_obj ud.save() ufw = UploadedFileWrapper(ud.pk) ufw.orig_name = orig_name setattr(self, field_name, ufw) exp.store_block(self) else: exp.log(self.uuid, "Adding upload to field: %s" % field_name) log.debug("Adding upload to field: %s", field_name) ud, is_created = UploadedData.objects.get_or_create( exp=exp, block_uuid=self.uuid, var_name=field_name, filename=orig_name) file_obj.name = local_filename ud.data = file_obj ud.filename = orig_name ud.save() ufw = UploadedFileWrapper(ud.pk) ufw.orig_name = orig_name r = get_redis_instance() with redis_lock.Lock(r, ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)): exp.log(self.uuid, "Enter lock, file: %s" % orig_name) log.debug("Enter lock, file: %s", orig_name) block = exp.get_block(self.uuid) attr = getattr(block, field_name) attr[orig_name] = ufw exp.log(self.uuid, "Added upload `%s` to collection: %s" % (orig_name, attr.keys())) log.debug("Added upload `%s` to collection: %s", orig_name, attr.keys()) exp.store_block(block) exp.log(self.uuid, "Exit lock, file: %s" % orig_name) log.debug("Exit lock, file: %s", orig_name) def erase_file_input(self, exp, data): field_name = json.loads(data)["field_name"] field = self._block_serializer.params.get(field_name) if not field.options.get("multiple", False): # single stored value ufw = getattr(self, field_name) ud = ufw.ud ud.delete() setattr(self, field_name, None) else: # multiple ufw_dict = getattr(self, field_name) for name, ufw in ufw_dict.items(): ufw.ud.delete() setattr(self, field_name, MultiUploadField()) exp.store_block(self) def add_dyn_input_hook(self, exp, dyn_port, new_port): """ to override later """ pass def add_input_port(self, new_port): self._block_serializer.register(new_port) self.input_manager.register(new_port) def add_dyn_input(self, exp, received_block, *args, **kwargs): spec = received_block.get("_add_dyn_port") if not spec: return if not spec['new_port'] or not spec['input']: return dyn_port_name = spec['input'] dyn_port = self._block_serializer.inputs.get(dyn_port_name) if not dyn_port: return order_num = 1000 + abs(dyn_port.order_num) * 10 dp = getattr(self, dyn_port_name) if dp: order_num += len(dp) new_port = InputBlockField( name=spec['new_port'], required_data_type=dyn_port.required_data_type, order_num=order_num ) self.add_input_port(new_port) getattr(self, dyn_port_name).append(spec["new_port"]) self.add_dyn_input_hook(exp, dyn_port, new_port) exp.store_block(self) def validate_params_hook(self, exp, *args, **kwargs): return True def validate_params(self, exp, *args, **kwargs): is_valid = True # check required inputs if not self.input_manager.validate_inputs( self, self.bound_inputs, self.errors, self.warnings): is_valid = False # check user provided values if not self._block_serializer.validate_params(self, exp): is_valid = False if not self.validate_params_hook(exp, *args, **kwargs): is_valid = False if is_valid: self.errors = [] self.do_action("on_params_is_valid", exp) else: self.do_action("on_params_not_valid", exp) def on_params_is_valid(self, exp, *args, **kwargs): self.errors = [] exp.store_block(self) def on_params_not_valid(self, exp, *args, **kwargs): pass def clean_errors(self): self.errors = [] def error(self, exp, new_errors=None): if isinstance(new_errors, collections.Iterable): self.errors.extend(new_errors) elif new_errors: self.errors.append(new_errors) exp.store_block(self) def reset_execution(self, exp, *args, **kwargs): self.clean_errors() exp.store_block(self)
def __init__(self, exp_id=None, scope_name=None): """ Building block for workflow """ # TODO: due to dynamic inputs, find better solution self._block_serializer = BlockSerializer.clone( self.__class__._block_serializer) self.state = "created" self.uuid = "B" + uuid1().hex[:8] self.exp_id = exp_id exp = None if exp_id: exp = Experiment.get_exp_by_id(exp_id) self.scope_name = scope_name self.base_name = "" # Used only be meta-blocks self.children_blocks = [] # End self._out_data = dict() self.out_manager = OutManager() self.input_manager = InputManager() # Automatic execution status map self.auto_exec_status_ready = set(["ready"]) self.auto_exec_status_done = set(["done"]) self.auto_exec_status_working = set(["working"]) self.auto_exec_status_error = set(["execution_error"]) # Init block fields for f_name, f in itertools.chain( self._block_serializer.fields.iteritems(), self._block_serializer.params.iteritems()): #if f_name not in self.__dict__ and not f.is_a_property: if not f.is_a_property and not hasattr(self, f_name): try: setattr(self, f_name, f.init_val) except: import ipdb ipdb.set_trace() for f_name, f in self._block_serializer.inputs.iteritems(): if f.multiply_extensible: setattr(self, f_name, []) # Names of dynamically added ports # TODO: Hmm maybe more metaclass magic can be applied here scope = self.get_scope() scope.load() for f_name, f in self._block_serializer.outputs.iteritems(): if exp: exp.log(self.uuid, "Registering normal outputs: %s" % f_name) log.debug("Registering normal outputs: %s", f_name) self.register_provided_objects( scope, ScopeVar(self.uuid, f_name, f.provided_data_type)) # TODO: Use factories for init values #if f.init_val is not None: # setattr(self, f.name, f.init_val) scope.store() for f_name, f in self._block_serializer.fields.items(): if f.init_val is not None: #setattr(self, f.name, f.init_val) pass for f_name, f in self._block_serializer.inputs.iteritems(): self.input_manager.register(f)
def __init__(self, exp_id=None, scope_name=None): """ Building block for workflow """ # TODO: due to dynamic inputs, find better solution self._block_serializer = BlockSerializer.clone(self.__class__._block_serializer) self.state = "created" self.uuid = "B" + uuid1().hex[:8] self.exp_id = exp_id exp = None if exp_id: exp = Experiment.get_exp_by_id(exp_id) self.scope_name = scope_name self.base_name = "" # Used only be meta-blocks self.children_blocks = [] # End self._out_data = dict() self.out_manager = OutManager() self.input_manager = InputManager() # Automatic execution status map self.auto_exec_status_ready = set(["ready"]) self.auto_exec_status_done = set(["done"]) self.auto_exec_status_working = set(["working"]) self.auto_exec_status_error = set(["execution_error"]) # Init block fields for f_name, f in itertools.chain( self._block_serializer.fields.iteritems(), self._block_serializer.params.iteritems()): #if f_name not in self.__dict__ and not f.is_a_property: if not f.is_a_property and not hasattr(self, f_name): try: setattr(self, f_name, f.init_val) except: import ipdb; ipdb.set_trace() for f_name, f in self._block_serializer.inputs.iteritems(): if f.multiply_extensible: setattr(self, f_name, []) # Names of dynamically added ports # TODO: Hmm maybe more metaclass magic can be applied here scope = self.get_scope() scope.load() for f_name, f in self._block_serializer.outputs.iteritems(): if exp: exp.log(self.uuid, "Registering normal outputs: %s" % f_name) log.debug("Registering normal outputs: %s", f_name) self.register_provided_objects(scope, ScopeVar(self.uuid, f_name, f.provided_data_type)) # TODO: Use factories for init values #if f.init_val is not None: # setattr(self, f.name, f.init_val) scope.store() for f_name, f in self._block_serializer.fields.items(): if f.init_val is not None: #setattr(self, f.name, f.init_val) pass for f_name, f in self._block_serializer.inputs.iteritems(): self.input_manager.register(f)