def save_file_input(self,
exp,
field_name,
file_obj,
multiple=False,
upload_meta=None):
if upload_meta is None:
upload_meta = {}
if not hasattr(self, field_name):
raise Exception("Block doesn't have field: %s" % field_name)
orig_name = file_obj.name
local_filename = "%s_%s_%s" % (self.uuid[:8], field_name,
file_obj.name)
if not multiple:
log.debug("Storing single upload to field: %s", field_name)
ud, is_created = UploadedData.objects.get_or_create(
exp=exp, block_uuid=self.uuid, var_name=field_name)
file_obj.name = local_filename
ud.data = file_obj
ud.save()
ufw = UploadedFileWrapper(ud.pk)
ufw.orig_name = orig_name
setattr(self, field_name, ufw)
exp.store_block(self)
else:
log.debug("Adding upload to field: %s", field_name)
ud, is_created = UploadedData.objects.get_or_create(
exp=exp,
block_uuid=self.uuid,
var_name=field_name,
filename=orig_name)
file_obj.name = local_filename
ud.data = file_obj
ud.filename = orig_name
ud.save()
ufw = UploadedFileWrapper(ud.pk)
ufw.orig_name = orig_name
r = get_redis_instance()
with redis_lock.Lock(
r,
ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)):
log.debug("Enter lock, file: %s", orig_name)
block = exp.get_block(self.uuid)
attr = getattr(block, field_name)
attr[orig_name] = ufw
log.debug("Added upload `%s` to collection: %s", orig_name,
attr.keys())
exp.store_block(block)
log.debug("Exit lock, file: %s", orig_name)
def save_file_input(self, exp, field_name, file_obj, multiple=False, upload_meta=None):
if upload_meta is None:
upload_meta = {}
if not hasattr(self, field_name):
raise Exception("Block doesn't have field: %s" % field_name)
orig_name = file_obj.name
local_filename = "%s_%s_%s" % (self.uuid[:8], field_name, file_obj.name)
if not multiple:
exp.log(self.uuid, "Storing single upload to field: %s" % field_name)
log.debug("Storing single upload to field: %s", field_name)
ud, is_created = UploadedData.objects.get_or_create(
exp=exp, block_uuid=self.uuid, var_name=field_name)
file_obj.name = local_filename
ud.data = file_obj
ud.save()
ufw = UploadedFileWrapper(ud.pk)
ufw.orig_name = orig_name
setattr(self, field_name, ufw)
exp.store_block(self)
else:
exp.log(self.uuid, "Adding upload to field: %s" % field_name)
log.debug("Adding upload to field: %s", field_name)
ud, is_created = UploadedData.objects.get_or_create(
exp=exp, block_uuid=self.uuid, var_name=field_name, filename=orig_name)
file_obj.name = local_filename
ud.data = file_obj
ud.filename = orig_name
ud.save()
ufw = UploadedFileWrapper(ud.pk)
ufw.orig_name = orig_name
r = get_redis_instance()
with redis_lock.Lock(r, ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)):
exp.log(self.uuid, "Enter lock, file: %s" % orig_name)
log.debug("Enter lock, file: %s", orig_name)
block = exp.get_block(self.uuid)
attr = getattr(block, field_name)
attr[orig_name] = ufw
exp.log(self.uuid, "Added upload `%s` to collection: %s" % (orig_name, attr.keys()))
log.debug("Added upload `%s` to collection: %s", orig_name, attr.keys())
exp.store_block(block)
exp.log(self.uuid, "Exit lock, file: %s" % orig_name)
log.debug("Exit lock, file: %s", orig_name)
def on_sub_scope_done(self, exp, *args, **kwargs):
"""
@type exp: Experiment
This action should be called by ScopeRunner
when all blocks in sub-scope have exec status == done
"""
r = get_redis_instance()
with redis_lock.Lock(
r, ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)):
cell = self.res_seq.sequence[self.inner_output_manager.iterator]
for name, scope_var in self.collector_spec.bound.iteritems():
var = exp.get_scope_var_value(scope_var)
log.debug("Collected %s from %s", var, scope_var.title)
if var is not None:
if hasattr(var, "clone"):
cell[name] = var.clone(
"%s_%s" %
(self.uuid, self.inner_output_manager.iterator))
else:
cell[name] = deepcopy(var)
self.res_seq.sequence[self.inner_output_manager.iterator] = cell
exp.store_block(self)
if len(cell) < len(self.res_seq.fields):
self.do_action("continue_collecting_sub_scope", exp)
else:
try:
self.inner_output_manager.next()
self.do_action("run_sub_scope", exp)
except StopIteration, e:
# All folds was processed without errors
self.build_result_collection(exp)
self.do_action("success", exp)
def on_sub_scope_done(self, exp, *args, **kwargs):
"""
@type exp: Experiment
This action should be called by ScopeRunner
when all blocks in sub-scope have exec status == done
"""
r = get_redis_instance()
with redis_lock.Lock(r, ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)):
cell = self.res_seq.sequence[self.inner_output_manager.iterator]
for name, scope_var in self.collector_spec.bound.iteritems():
var = exp.get_scope_var_value(scope_var)
exp.log(self.uuid, "Collected %s from %s" % (var, scope_var.title), severity="CRITICAL")
log.debug("Collected %s from %s", var, scope_var.title)
if var is not None:
if hasattr(var, "clone"):
cell[name] = var.clone("%s_%s" %
(self.uuid, self.inner_output_manager.iterator))
else:
cell[name] = deepcopy(var)
self.res_seq.sequence[self.inner_output_manager.iterator] = cell
exp.store_block(self)
if len(cell) < len(self.res_seq.fields):
self.do_action("continue_collecting_sub_scope", exp)
else:
try:
self.inner_output_manager.next()
self.do_action("run_sub_scope", exp)
except StopIteration, e:
# All folds were processed without errors
self.build_result_collection(exp)
self.do_action("success", exp)
