def table(self):
rc = self.rc
to = TableObj()
if rc:
rc.load()
header_axis = self.table_config.get("header_axis")
index_axis_list = []
for axis, flag in self.table_config.get("multi_index_axis_dict",
{}).iteritems():
if flag:
index_axis_list.append(axis)
if header_axis and index_axis_list and hasattr(self, "metric"):
# log.debug("Can build table slice")
df = rc.get_pandas_slice(header_axis,
index_axis_list,
metric_name=self.metric)
# log.debug(df)
to.html = df.to_html(float_format=pd_float_format_func)
to.df = df
else:
if self.exp_id:
exp = Experiment.get_exp_by_id(self.exp_id)
exp.log(
self.uuid,
"Can't build table slice, header axis `%s`, index axis_list `%s`"
% (header_axis, index_axis_list))
log.debug(
"Can't build table slice, header axis `%s`, index axis_list `%s`",
header_axis, index_axis_list)
# log.debug("Table: %s", to.to_dict())
return to
def table(self):
rc = self.rc
to = TableObj()
if rc:
rc.load()
header_axis = self.table_config.get("header_axis")
index_axis_list = []
for axis, flag in self.table_config.get("multi_index_axis_dict", {}).iteritems():
if flag:
index_axis_list.append(axis)
if header_axis and index_axis_list and hasattr(self, "metric"):
# log.debug("Can build table slice")
df = rc.get_pandas_slice(header_axis, index_axis_list,
metric_name=self.metric)
# log.debug(df)
to.html = df.to_html(float_format=pd_float_format_func)
to.df = df
else:
if self.exp_id:
exp = Experiment.get_exp_by_id(self.exp_id)
exp.log(self.uuid, "Can't build table slice, header axis `%s`, index axis_list `%s`" %
(header_axis, index_axis_list))
log.debug("Can't build table slice, header axis `%s`, index axis_list `%s`",
header_axis, index_axis_list)
# log.debug("Table: %s", to.to_dict())
return to
def get_input_var(self, name):
try:
exp = Experiment.get_exp_by_id(self.exp_id)
scope_var = self.bound_inputs[name]
return exp.get_scope_var_value(scope_var)
except:
return None
def get_input_var(self, name):
try:
exp = Experiment.get_exp_by_id(self.exp_id)
scope_var = self.bound_inputs[name]
return exp.get_scope_var_value(scope_var)
except:
return None
def bind_input_var(self, input_name, bound_var):
if self.exp_id:
exp = Experiment.get_exp_by_id(self.exp_id)
exp.log(self.uuid, "bound input %s to %s in block: %s, exp: %s" %
(input_name, bound_var, self.base_name, self.exp_id))
log.debug("bound input %s to %s in block: %s, exp: %s",
input_name, bound_var, self.base_name, self.exp_id)
self.bound_inputs[input_name] = bound_var
def __init__(self, *args, **kwargs):
super(GlobalTest, self).__init__(*args, **kwargs)
self.celery_task = None
exp = Experiment.get_exp_by_id(self.exp_id)
self.result = TableResult(
base_dir=exp.get_data_folder(),
base_filename="%s_gt_result" % self.uuid,
)
self.result.headers = ['p-value', 'Statistic', 'Expected', 'Std.dev', '#Cov']
def bind_input_var(self, input_name, bound_var):
if self.exp_id:
exp = Experiment.get_exp_by_id(self.exp_id)
exp.log(
self.uuid, "bound input %s to %s in block: %s, exp: %s" %
(input_name, bound_var, self.base_name, self.exp_id))
log.debug("bound input %s to %s in block: %s, exp: %s", input_name,
bound_var, self.base_name, self.exp_id)
self.bound_inputs[input_name] = bound_var
def __init__(self, *args, **kwargs):
super(GenericRankingBlock, self).__init__(*args, **kwargs)
self.ranking_name = None
self.ranking_options = {}
self.celery_task = None
exp = Experiment.get_exp_by_id(self.exp_id)
self.result = TableResult(
base_dir=exp.get_data_folder(),
base_filename="%s_gt_result" % self.uuid,
)
self.set_out_var("result", self.result)
def __init__(self, *args, **kwargs):
super(GenericRankingBlock, self).__init__(*args, **kwargs)
self.ranking_name = None
self.ranking_options = {}
self.celery_task = None
exp = Experiment.get_exp_by_id(self.exp_id)
self.result = TableResult(
base_dir=exp.get_data_folder(),
base_filename="%s_gt_result" % self.uuid,
)
self.set_out_var("result", self.result)
def available_metrics(self):
try:
return [
{"pk": metric_name, "str": metric.title}
for metric_name, metric in metrics_dict.iteritems()
if metric.produce_single_number
]
except Exception, e:
if self.exp_id:
exp = Experiment.get_exp_by_id(self.exp_id)
exp.log(self.uuid, e, severity="CRITICAL")
log.exception(e)
return []
def upload_data(request):
if request.method == "POST":
form = UploadForm(request.POST, request.FILES)
if form.is_valid():
exp_id = form.cleaned_data['exp_id']
block_uuid = form.cleaned_data['block_uuid']
field_name = form.cleaned_data['field_name']
file_obj = form.cleaned_data['file']
multiple = form.cleaned_data['multiple']
log.debug("Multiple: %s", multiple)
exp = Experiment.get_exp_by_id(exp_id)
block = exp.get_block(block_uuid)
block.save_file_input(
exp, field_name, file_obj,
multiple=multiple,
upload_meta=request.POST["upload_meta"]
)
return HttpResponse(status=204)
def upload_data(request):
if request.method == "POST":
form = UploadForm(request.POST, request.FILES)
if form.is_valid():
exp_id = form.cleaned_data['exp_id']
block_uuid = form.cleaned_data['block_uuid']
field_name = form.cleaned_data['field_name']
file_obj = form.cleaned_data['file']
multiple = form.cleaned_data['multiple']
log.debug("Multiple: %s", multiple)
exp = Experiment.get_exp_by_id(exp_id)
block = exp.get_block(block_uuid)
block.save_file_input(exp,
field_name,
file_obj,
multiple=multiple,
upload_meta=request.POST["upload_meta"])
return HttpResponse(status=204)
def __init__(self, exp_id=None, scope_name=None):
"""
Building block for workflow
"""
# TODO: due to dynamic inputs, find better solution
self._block_serializer = BlockSerializer.clone(
self.__class__._block_serializer)
self.state = "created"
self.uuid = "B" + uuid1().hex[:8]
self.exp_id = exp_id
exp = None
if exp_id:
exp = Experiment.get_exp_by_id(exp_id)
self.scope_name = scope_name
self.base_name = ""
# Used only be meta-blocks
self.children_blocks = []
# End
self._out_data = dict()
self.out_manager = OutManager()
self.input_manager = InputManager()
# Automatic execution status map
self.auto_exec_status_ready = set(["ready"])
self.auto_exec_status_done = set(["done"])
self.auto_exec_status_working = set(["working"])
self.auto_exec_status_error = set(["execution_error"])
# Init block fields
for f_name, f in itertools.chain(
self._block_serializer.fields.iteritems(),
self._block_serializer.params.iteritems()):
#if f_name not in self.__dict__ and not f.is_a_property:
if not f.is_a_property and not hasattr(self, f_name):
try:
setattr(self, f_name, f.init_val)
except:
import ipdb
ipdb.set_trace()
for f_name, f in self._block_serializer.inputs.iteritems():
if f.multiply_extensible:
setattr(self, f_name, []) # Names of dynamically added ports
# TODO: Hmm maybe more metaclass magic can be applied here
scope = self.get_scope()
scope.load()
for f_name, f in self._block_serializer.outputs.iteritems():
if exp:
exp.log(self.uuid, "Registering normal outputs: %s" % f_name)
log.debug("Registering normal outputs: %s", f_name)
self.register_provided_objects(
scope, ScopeVar(self.uuid, f_name, f.provided_data_type))
# TODO: Use factories for init values
#if f.init_val is not None:
# setattr(self, f.name, f.init_val)
scope.store()
for f_name, f in self._block_serializer.fields.items():
if f.init_val is not None:
#setattr(self, f.name, f.init_val)
pass
for f_name, f in self._block_serializer.inputs.iteritems():
self.input_manager.register(f)
def reset_execution_for_sub_blocks(self):
exp = Experiment.get_exp_by_id(self.exp_id)
for block_uuid, block in exp.get_blocks(self.children_blocks):
block.do_action("reset_execution", exp)
def get_sub_scope(self):
exp = Experiment.get_exp_by_id(self.exp_id)
return Scope(exp, self.sub_scope_name)
def reset_execution_for_sub_blocks(self):
exp = Experiment.get_exp_by_id(self.exp_id)
for block_uuid, block in exp.get_blocks(self.children_blocks):
block.do_action("reset_execution", exp)
def get_sub_scope(self):
exp = Experiment.get_exp_by_id(self.exp_id)
return Scope(exp, self.sub_scope_name)
def __init__(self, exp_id=None, scope_name=None):
"""
Building block for workflow
"""
# TODO: due to dynamic inputs, find better solution
self._block_serializer = BlockSerializer.clone(self.__class__._block_serializer)
self.state = "created"
self.uuid = "B" + uuid1().hex[:8]
self.exp_id = exp_id
exp = None
if exp_id:
exp = Experiment.get_exp_by_id(exp_id)
self.scope_name = scope_name
self.base_name = ""
# Used only be meta-blocks
self.children_blocks = []
# End
self._out_data = dict()
self.out_manager = OutManager()
self.input_manager = InputManager()
# Automatic execution status map
self.auto_exec_status_ready = set(["ready"])
self.auto_exec_status_done = set(["done"])
self.auto_exec_status_working = set(["working"])
self.auto_exec_status_error = set(["execution_error"])
# Init block fields
for f_name, f in itertools.chain(
self._block_serializer.fields.iteritems(),
self._block_serializer.params.iteritems()):
#if f_name not in self.__dict__ and not f.is_a_property:
if not f.is_a_property and not hasattr(self, f_name):
try:
setattr(self, f_name, f.init_val)
except:
import ipdb; ipdb.set_trace()
for f_name, f in self._block_serializer.inputs.iteritems():
if f.multiply_extensible:
setattr(self, f_name, []) # Names of dynamically added ports
# TODO: Hmm maybe more metaclass magic can be applied here
scope = self.get_scope()
scope.load()
for f_name, f in self._block_serializer.outputs.iteritems():
if exp:
exp.log(self.uuid, "Registering normal outputs: %s" % f_name)
log.debug("Registering normal outputs: %s", f_name)
self.register_provided_objects(scope, ScopeVar(self.uuid, f_name, f.provided_data_type))
# TODO: Use factories for init values
#if f.init_val is not None:
# setattr(self, f.name, f.init_val)
scope.store()
for f_name, f in self._block_serializer.fields.items():
if f.init_val is not None:
#setattr(self, f.name, f.init_val)
pass
for f_name, f in self._block_serializer.inputs.iteritems():
self.input_manager.register(f)
