def default(self, id): safe_id = id if cherrypy.request.method == 'GET': filename = self.block_store.filename(safe_id) try: response_body = serve_file(filename) return response_body except cherrypy.HTTPError as he: if he.status == 404: response_body = serve_file( self.block_store.producer_filename(safe_id)) return response_body else: raise elif cherrypy.request.method == 'POST': request_body = cherrypy.request.body.read() new_ref = ref_from_string(request_body, safe_id) if self.backup_sender is not None: self.backup_sender.add_data(safe_id, request_body) #if self.task_pool is not None: # self.task_pool.publish_refs({safe_id : new_ref}) return simplejson.dumps(new_ref, cls=SWReferenceJSONEncoder) elif cherrypy.request.method == 'HEAD': if os.path.exists(self.block_store.filename(id)): return else: raise cherrypy.HTTPError(404) else: raise cherrypy.HTTPError(405)
def build_task_descriptor(cls, task_descriptor, parent_task_record, process_record_id=None, is_fixed=False, command=None, proc_pargs=[], proc_kwargs={}, force_n_outputs=None, n_extra_outputs=0, extra_dependencies=[], is_tail_spawn=False, accept_ref_list_for_single=False): #if process_record_id is None and start_command is None: # raise BlameUserException("ProcExecutor tasks must specify either process_record_id or start_command") if process_record_id is not None: task_descriptor["task_private"]["id"] = process_record_id if command is not None: task_descriptor["task_private"]["command"] = command task_descriptor["task_private"]["proc_pargs"] = proc_pargs task_descriptor["task_private"]["proc_kwargs"] = proc_kwargs task_descriptor["dependencies"].extend(extra_dependencies) task_private_id = ("%s:_private" % task_descriptor["task_id"]) if is_fixed: task_private_ref = SW2_FixedReference(task_private_id, get_own_netloc()) write_fixed_ref_string( pickle.dumps(task_descriptor["task_private"]), task_private_ref) else: task_private_ref = ref_from_string( pickle.dumps(task_descriptor["task_private"]), task_private_id) parent_task_record.publish_ref(task_private_ref) task_descriptor["task_private"] = task_private_ref task_descriptor["dependencies"].append(task_private_ref) if force_n_outputs is not None: if "expected_outputs" in task_descriptor and len( task_descriptor["expected_outputs"]) > 0: raise BlameUserException( "Task already had outputs, but force_n_outputs is set") task_descriptor["expected_outputs"] = [ "%s:out:%d" % (task_descriptor["task_id"], i) for i in range(force_n_outputs) ] if not is_tail_spawn: if len(task_descriptor["expected_outputs"] ) == 1 and not accept_ref_list_for_single: return SW2_FutureReference( task_descriptor["expected_outputs"][0]) else: return [ SW2_FutureReference(refid) for refid in task_descriptor["expected_outputs"] ]
def default(self, id): safe_id = id if cherrypy.request.method == 'GET': filename = self.block_store.filename(safe_id) try: response_body = serve_file(filename) return response_body except cherrypy.HTTPError as he: if he.status == 404: response_body = serve_file(self.block_store.producer_filename(safe_id)) return response_body else: raise elif cherrypy.request.method == 'POST': request_body = cherrypy.request.body.read() new_ref = ref_from_string(request_body, safe_id) if self.backup_sender is not None: self.backup_sender.add_data(safe_id, request_body) #if self.task_pool is not None: # self.task_pool.publish_refs({safe_id : new_ref}) return simplejson.dumps(new_ref, cls=SWReferenceJSONEncoder) elif cherrypy.request.method == 'HEAD': if os.path.exists(self.block_store.filename(id)): return else: raise cherrypy.HTTPError(404) else: raise cherrypy.HTTPError(405)
def ref_from_object(object, encoder, id): """Encodes an object, returning either a DataValue or ConcreteReference as appropriate""" cache_object(object, encoder, id) buffer = StringIO() encoders[encoder](object, buffer) ret = ref_from_string(buffer.getvalue(), id) buffer.close() return ret
def index(self): if cherrypy.request.method == 'POST': id = self.block_store.allocate_new_id() request_body = cherrypy.request.body.read() new_ref = ref_from_string(request_body, id) if self.backup_sender is not None: self.backup_sender.add_data(id, request_body) #if self.task_pool is not None: # self.task_pool.publish_refs({id : new_ref}) return simplejson.dumps(new_ref, cls=SWReferenceJSONEncoder) elif cherrypy.request.method == 'GET': return serve_file(self.block_store.generate_block_list_file()) else: raise cherrypy.HTTPError(405)
def build_task_descriptor(cls, task_descriptor, parent_task_record, process_record_id=None, is_fixed=False, command=None, proc_pargs=[], proc_kwargs={}, force_n_outputs=None, n_extra_outputs=0, extra_dependencies=[], is_tail_spawn=False, accept_ref_list_for_single=False): #if process_record_id is None and start_command is None: # raise BlameUserException("ProcExecutor tasks must specify either process_record_id or start_command") if process_record_id is not None: task_descriptor["task_private"]["id"] = process_record_id if command is not None: task_descriptor["task_private"]["command"] = command task_descriptor["task_private"]["proc_pargs"] = proc_pargs task_descriptor["task_private"]["proc_kwargs"] = proc_kwargs task_descriptor["dependencies"].extend(extra_dependencies) task_private_id = ("%s:_private" % task_descriptor["task_id"]) if is_fixed: task_private_ref = SW2_FixedReference(task_private_id, get_own_netloc()) write_fixed_ref_string(pickle.dumps(task_descriptor["task_private"]), task_private_ref) else: task_private_ref = ref_from_string(pickle.dumps(task_descriptor["task_private"]), task_private_id) parent_task_record.publish_ref(task_private_ref) task_descriptor["task_private"] = task_private_ref task_descriptor["dependencies"].append(task_private_ref) if force_n_outputs is not None: if "expected_outputs" in task_descriptor and len(task_descriptor["expected_outputs"]) > 0: raise BlameUserException("Task already had outputs, but force_n_outputs is set") task_descriptor["expected_outputs"] = ["%s:out:%d" % (task_descriptor["task_id"], i) for i in range(force_n_outputs)] if not is_tail_spawn: if len(task_descriptor["expected_outputs"]) == 1 and not accept_ref_list_for_single: return SW2_FutureReference(task_descriptor["expected_outputs"][0]) else: return [SW2_FutureReference(refid) for refid in task_descriptor["expected_outputs"]]
def to_ref(self, refid): if self.str is not None: ref = ref_from_string(self.str, refid) else: ref = ref_from_external_file(self.filename, refid) return ref