def table(self): rc = self.rc to = TableObj() if rc: rc.load() header_axis = self.table_config.get("header_axis") index_axis_list = [] for axis, flag in self.table_config.get("multi_index_axis_dict", {}).iteritems(): if flag: index_axis_list.append(axis) if header_axis and index_axis_list and hasattr(self, "metric"): # log.debug("Can build table slice") df = rc.get_pandas_slice(header_axis, index_axis_list, metric_name=self.metric) # log.debug(df) to.html = df.to_html(float_format=pd_float_format_func) to.df = df else: if self.exp_id: exp = Experiment.get_exp_by_id(self.exp_id) exp.log( self.uuid, "Can't build table slice, header axis `%s`, index axis_list `%s`" % (header_axis, index_axis_list)) log.debug( "Can't build table slice, header axis `%s`, index axis_list `%s`", header_axis, index_axis_list) # log.debug("Table: %s", to.to_dict()) return to
def table(self): rc = self.rc to = TableObj() if rc: rc.load() header_axis = self.table_config.get("header_axis") index_axis_list = [] for axis, flag in self.table_config.get("multi_index_axis_dict", {}).iteritems(): if flag: index_axis_list.append(axis) if header_axis and index_axis_list and hasattr(self, "metric"): # log.debug("Can build table slice") df = rc.get_pandas_slice(header_axis, index_axis_list, metric_name=self.metric) # log.debug(df) to.html = df.to_html(float_format=pd_float_format_func) to.df = df else: if self.exp_id: exp = Experiment.get_exp_by_id(self.exp_id) exp.log(self.uuid, "Can't build table slice, header axis `%s`, index axis_list `%s`" % (header_axis, index_axis_list)) log.debug("Can't build table slice, header axis `%s`, index axis_list `%s`", header_axis, index_axis_list) # log.debug("Table: %s", to.to_dict()) return to
def get_input_var(self, name): try: exp = Experiment.get_exp_by_id(self.exp_id) scope_var = self.bound_inputs[name] return exp.get_scope_var_value(scope_var) except: return None
def bind_input_var(self, input_name, bound_var): if self.exp_id: exp = Experiment.get_exp_by_id(self.exp_id) exp.log(self.uuid, "bound input %s to %s in block: %s, exp: %s" % (input_name, bound_var, self.base_name, self.exp_id)) log.debug("bound input %s to %s in block: %s, exp: %s", input_name, bound_var, self.base_name, self.exp_id) self.bound_inputs[input_name] = bound_var
def __init__(self, *args, **kwargs): super(GlobalTest, self).__init__(*args, **kwargs) self.celery_task = None exp = Experiment.get_exp_by_id(self.exp_id) self.result = TableResult( base_dir=exp.get_data_folder(), base_filename="%s_gt_result" % self.uuid, ) self.result.headers = ['p-value', 'Statistic', 'Expected', 'Std.dev', '#Cov']
def bind_input_var(self, input_name, bound_var): if self.exp_id: exp = Experiment.get_exp_by_id(self.exp_id) exp.log( self.uuid, "bound input %s to %s in block: %s, exp: %s" % (input_name, bound_var, self.base_name, self.exp_id)) log.debug("bound input %s to %s in block: %s, exp: %s", input_name, bound_var, self.base_name, self.exp_id) self.bound_inputs[input_name] = bound_var
def __init__(self, *args, **kwargs): super(GenericRankingBlock, self).__init__(*args, **kwargs) self.ranking_name = None self.ranking_options = {} self.celery_task = None exp = Experiment.get_exp_by_id(self.exp_id) self.result = TableResult( base_dir=exp.get_data_folder(), base_filename="%s_gt_result" % self.uuid, ) self.set_out_var("result", self.result)
def available_metrics(self): try: return [ {"pk": metric_name, "str": metric.title} for metric_name, metric in metrics_dict.iteritems() if metric.produce_single_number ] except Exception, e: if self.exp_id: exp = Experiment.get_exp_by_id(self.exp_id) exp.log(self.uuid, e, severity="CRITICAL") log.exception(e) return []
def upload_data(request): if request.method == "POST": form = UploadForm(request.POST, request.FILES) if form.is_valid(): exp_id = form.cleaned_data['exp_id'] block_uuid = form.cleaned_data['block_uuid'] field_name = form.cleaned_data['field_name'] file_obj = form.cleaned_data['file'] multiple = form.cleaned_data['multiple'] log.debug("Multiple: %s", multiple) exp = Experiment.get_exp_by_id(exp_id) block = exp.get_block(block_uuid) block.save_file_input( exp, field_name, file_obj, multiple=multiple, upload_meta=request.POST["upload_meta"] ) return HttpResponse(status=204)
def upload_data(request): if request.method == "POST": form = UploadForm(request.POST, request.FILES) if form.is_valid(): exp_id = form.cleaned_data['exp_id'] block_uuid = form.cleaned_data['block_uuid'] field_name = form.cleaned_data['field_name'] file_obj = form.cleaned_data['file'] multiple = form.cleaned_data['multiple'] log.debug("Multiple: %s", multiple) exp = Experiment.get_exp_by_id(exp_id) block = exp.get_block(block_uuid) block.save_file_input(exp, field_name, file_obj, multiple=multiple, upload_meta=request.POST["upload_meta"]) return HttpResponse(status=204)
def __init__(self, exp_id=None, scope_name=None): """ Building block for workflow """ # TODO: due to dynamic inputs, find better solution self._block_serializer = BlockSerializer.clone( self.__class__._block_serializer) self.state = "created" self.uuid = "B" + uuid1().hex[:8] self.exp_id = exp_id exp = None if exp_id: exp = Experiment.get_exp_by_id(exp_id) self.scope_name = scope_name self.base_name = "" # Used only be meta-blocks self.children_blocks = [] # End self._out_data = dict() self.out_manager = OutManager() self.input_manager = InputManager() # Automatic execution status map self.auto_exec_status_ready = set(["ready"]) self.auto_exec_status_done = set(["done"]) self.auto_exec_status_working = set(["working"]) self.auto_exec_status_error = set(["execution_error"]) # Init block fields for f_name, f in itertools.chain( self._block_serializer.fields.iteritems(), self._block_serializer.params.iteritems()): #if f_name not in self.__dict__ and not f.is_a_property: if not f.is_a_property and not hasattr(self, f_name): try: setattr(self, f_name, f.init_val) except: import ipdb ipdb.set_trace() for f_name, f in self._block_serializer.inputs.iteritems(): if f.multiply_extensible: setattr(self, f_name, []) # Names of dynamically added ports # TODO: Hmm maybe more metaclass magic can be applied here scope = self.get_scope() scope.load() for f_name, f in self._block_serializer.outputs.iteritems(): if exp: exp.log(self.uuid, "Registering normal outputs: %s" % f_name) log.debug("Registering normal outputs: %s", f_name) self.register_provided_objects( scope, ScopeVar(self.uuid, f_name, f.provided_data_type)) # TODO: Use factories for init values #if f.init_val is not None: # setattr(self, f.name, f.init_val) scope.store() for f_name, f in self._block_serializer.fields.items(): if f.init_val is not None: #setattr(self, f.name, f.init_val) pass for f_name, f in self._block_serializer.inputs.iteritems(): self.input_manager.register(f)
def reset_execution_for_sub_blocks(self): exp = Experiment.get_exp_by_id(self.exp_id) for block_uuid, block in exp.get_blocks(self.children_blocks): block.do_action("reset_execution", exp)
def get_sub_scope(self): exp = Experiment.get_exp_by_id(self.exp_id) return Scope(exp, self.sub_scope_name)
def __init__(self, exp_id=None, scope_name=None): """ Building block for workflow """ # TODO: due to dynamic inputs, find better solution self._block_serializer = BlockSerializer.clone(self.__class__._block_serializer) self.state = "created" self.uuid = "B" + uuid1().hex[:8] self.exp_id = exp_id exp = None if exp_id: exp = Experiment.get_exp_by_id(exp_id) self.scope_name = scope_name self.base_name = "" # Used only be meta-blocks self.children_blocks = [] # End self._out_data = dict() self.out_manager = OutManager() self.input_manager = InputManager() # Automatic execution status map self.auto_exec_status_ready = set(["ready"]) self.auto_exec_status_done = set(["done"]) self.auto_exec_status_working = set(["working"]) self.auto_exec_status_error = set(["execution_error"]) # Init block fields for f_name, f in itertools.chain( self._block_serializer.fields.iteritems(), self._block_serializer.params.iteritems()): #if f_name not in self.__dict__ and not f.is_a_property: if not f.is_a_property and not hasattr(self, f_name): try: setattr(self, f_name, f.init_val) except: import ipdb; ipdb.set_trace() for f_name, f in self._block_serializer.inputs.iteritems(): if f.multiply_extensible: setattr(self, f_name, []) # Names of dynamically added ports # TODO: Hmm maybe more metaclass magic can be applied here scope = self.get_scope() scope.load() for f_name, f in self._block_serializer.outputs.iteritems(): if exp: exp.log(self.uuid, "Registering normal outputs: %s" % f_name) log.debug("Registering normal outputs: %s", f_name) self.register_provided_objects(scope, ScopeVar(self.uuid, f_name, f.provided_data_type)) # TODO: Use factories for init values #if f.init_val is not None: # setattr(self, f.name, f.init_val) scope.store() for f_name, f in self._block_serializer.fields.items(): if f.init_val is not None: #setattr(self, f.name, f.init_val) pass for f_name, f in self._block_serializer.inputs.iteritems(): self.input_manager.register(f)