def default(self, id):
safe_id = id
if cherrypy.request.method == 'GET':
filename = self.block_store.filename(safe_id)
try:
response_body = serve_file(filename)
return response_body
except cherrypy.HTTPError as he:
if he.status == 404:
response_body = serve_file(
self.block_store.producer_filename(safe_id))
return response_body
else:
raise
elif cherrypy.request.method == 'POST':
request_body = cherrypy.request.body.read()
new_ref = ref_from_string(request_body, safe_id)
if self.backup_sender is not None:
self.backup_sender.add_data(safe_id, request_body)
#if self.task_pool is not None:
# self.task_pool.publish_refs({safe_id : new_ref})
return simplejson.dumps(new_ref, cls=SWReferenceJSONEncoder)
elif cherrypy.request.method == 'HEAD':
if os.path.exists(self.block_store.filename(id)):
return
else:
raise cherrypy.HTTPError(404)
else:
raise cherrypy.HTTPError(405)
def build_task_descriptor(cls,
task_descriptor,
parent_task_record,
process_record_id=None,
is_fixed=False,
command=None,
proc_pargs=[],
proc_kwargs={},
force_n_outputs=None,
n_extra_outputs=0,
extra_dependencies=[],
is_tail_spawn=False,
accept_ref_list_for_single=False):
#if process_record_id is None and start_command is None:
# raise BlameUserException("ProcExecutor tasks must specify either process_record_id or start_command")
if process_record_id is not None:
task_descriptor["task_private"]["id"] = process_record_id
if command is not None:
task_descriptor["task_private"]["command"] = command
task_descriptor["task_private"]["proc_pargs"] = proc_pargs
task_descriptor["task_private"]["proc_kwargs"] = proc_kwargs
task_descriptor["dependencies"].extend(extra_dependencies)
task_private_id = ("%s:_private" % task_descriptor["task_id"])
if is_fixed:
task_private_ref = SW2_FixedReference(task_private_id,
get_own_netloc())
write_fixed_ref_string(
pickle.dumps(task_descriptor["task_private"]),
task_private_ref)
else:
task_private_ref = ref_from_string(
pickle.dumps(task_descriptor["task_private"]), task_private_id)
parent_task_record.publish_ref(task_private_ref)
task_descriptor["task_private"] = task_private_ref
task_descriptor["dependencies"].append(task_private_ref)
if force_n_outputs is not None:
if "expected_outputs" in task_descriptor and len(
task_descriptor["expected_outputs"]) > 0:
raise BlameUserException(
"Task already had outputs, but force_n_outputs is set")
task_descriptor["expected_outputs"] = [
"%s:out:%d" % (task_descriptor["task_id"], i)
for i in range(force_n_outputs)
]
if not is_tail_spawn:
if len(task_descriptor["expected_outputs"]
) == 1 and not accept_ref_list_for_single:
return SW2_FutureReference(
task_descriptor["expected_outputs"][0])
else:
return [
SW2_FutureReference(refid)
for refid in task_descriptor["expected_outputs"]
]
def default(self, id):
safe_id = id
if cherrypy.request.method == 'GET':
filename = self.block_store.filename(safe_id)
try:
response_body = serve_file(filename)
return response_body
except cherrypy.HTTPError as he:
if he.status == 404:
response_body = serve_file(self.block_store.producer_filename(safe_id))
return response_body
else:
raise
elif cherrypy.request.method == 'POST':
request_body = cherrypy.request.body.read()
new_ref = ref_from_string(request_body, safe_id)
if self.backup_sender is not None:
self.backup_sender.add_data(safe_id, request_body)
#if self.task_pool is not None:
# self.task_pool.publish_refs({safe_id : new_ref})
return simplejson.dumps(new_ref, cls=SWReferenceJSONEncoder)
elif cherrypy.request.method == 'HEAD':
if os.path.exists(self.block_store.filename(id)):
return
else:
raise cherrypy.HTTPError(404)
else:
raise cherrypy.HTTPError(405)
def ref_from_object(object, encoder, id):
"""Encodes an object, returning either a DataValue or ConcreteReference as appropriate"""
cache_object(object, encoder, id)
buffer = StringIO()
encoders[encoder](object, buffer)
ret = ref_from_string(buffer.getvalue(), id)
buffer.close()
return ret
def ref_from_object(object, encoder, id):
"""Encodes an object, returning either a DataValue or ConcreteReference as appropriate"""
cache_object(object, encoder, id)
buffer = StringIO()
encoders[encoder](object, buffer)
ret = ref_from_string(buffer.getvalue(), id)
buffer.close()
return ret
def index(self):
if cherrypy.request.method == 'POST':
id = self.block_store.allocate_new_id()
request_body = cherrypy.request.body.read()
new_ref = ref_from_string(request_body, id)
if self.backup_sender is not None:
self.backup_sender.add_data(id, request_body)
#if self.task_pool is not None:
# self.task_pool.publish_refs({id : new_ref})
return simplejson.dumps(new_ref, cls=SWReferenceJSONEncoder)
elif cherrypy.request.method == 'GET':
return serve_file(self.block_store.generate_block_list_file())
else:
raise cherrypy.HTTPError(405)
def index(self):
if cherrypy.request.method == 'POST':
id = self.block_store.allocate_new_id()
request_body = cherrypy.request.body.read()
new_ref = ref_from_string(request_body, id)
if self.backup_sender is not None:
self.backup_sender.add_data(id, request_body)
#if self.task_pool is not None:
# self.task_pool.publish_refs({id : new_ref})
return simplejson.dumps(new_ref, cls=SWReferenceJSONEncoder)
elif cherrypy.request.method == 'GET':
return serve_file(self.block_store.generate_block_list_file())
else:
raise cherrypy.HTTPError(405)
def build_task_descriptor(cls, task_descriptor, parent_task_record,
process_record_id=None, is_fixed=False, command=None, proc_pargs=[], proc_kwargs={}, force_n_outputs=None,
n_extra_outputs=0, extra_dependencies=[], is_tail_spawn=False, accept_ref_list_for_single=False):
#if process_record_id is None and start_command is None:
# raise BlameUserException("ProcExecutor tasks must specify either process_record_id or start_command")
if process_record_id is not None:
task_descriptor["task_private"]["id"] = process_record_id
if command is not None:
task_descriptor["task_private"]["command"] = command
task_descriptor["task_private"]["proc_pargs"] = proc_pargs
task_descriptor["task_private"]["proc_kwargs"] = proc_kwargs
task_descriptor["dependencies"].extend(extra_dependencies)
task_private_id = ("%s:_private" % task_descriptor["task_id"])
if is_fixed:
task_private_ref = SW2_FixedReference(task_private_id, get_own_netloc())
write_fixed_ref_string(pickle.dumps(task_descriptor["task_private"]), task_private_ref)
else:
task_private_ref = ref_from_string(pickle.dumps(task_descriptor["task_private"]), task_private_id)
parent_task_record.publish_ref(task_private_ref)
task_descriptor["task_private"] = task_private_ref
task_descriptor["dependencies"].append(task_private_ref)
if force_n_outputs is not None:
if "expected_outputs" in task_descriptor and len(task_descriptor["expected_outputs"]) > 0:
raise BlameUserException("Task already had outputs, but force_n_outputs is set")
task_descriptor["expected_outputs"] = ["%s:out:%d" % (task_descriptor["task_id"], i) for i in range(force_n_outputs)]
if not is_tail_spawn:
if len(task_descriptor["expected_outputs"]) == 1 and not accept_ref_list_for_single:
return SW2_FutureReference(task_descriptor["expected_outputs"][0])
else:
return [SW2_FutureReference(refid) for refid in task_descriptor["expected_outputs"]]
def to_ref(self, refid):
if self.str is not None:
ref = ref_from_string(self.str, refid)
else:
ref = ref_from_external_file(self.filename, refid)
return ref
def to_ref(self, refid):
if self.str is not None:
ref = ref_from_string(self.str, refid)
else:
ref = ref_from_external_file(self.filename, refid)
return ref
