def main(): parser = OptionParser() parser.add_option("-i", "--index", action="store", dest="index", help="Index SWBS URI", metavar="URI", default=None) parser.add_option("-b", "--block", action="store", dest="block", help="Block SWBS URI", metavar="URI", default=None) (options, _) = parser.parse_args() h = httplib2.Http() if options.block is not None: netloc = get_netloc_for_sw_url(options.block) id = get_id_for_sw_url(options.block) response, _ = h.request('http://%s/admin/pin/%s' % (netloc, id), 'POST', 'pin') assert response.status == 200 print >>sys.stderr, 'Pinned block %s to %s' % (id, netloc) if options.index is not None: index_url = sw_to_external_url(options.index) _, content = h.request(index_url, 'GET') index = simplejson.loads(content, object_hook=json_decode_object_hook) for chunk in index: assert isinstance(chunk, SW2_ConcreteReference) for netloc in chunk.location_hints: response, _ = h.request('http://%s/admin/pin/%s' % (netloc, chunk.id), 'POST', 'pin') assert response.status == 200 print >>sys.stderr, 'Pinned block %s to %s' % (chunk.id, netloc)
def submit_task_and_wait(self, task_stmt): # 1. Update the local continuation with the current task_stmt. self.local_continuation.task_stmt = task_stmt # 2. POST the updated local continuation. http = httplib2.Http() master_data_uri = urlparse.urljoin(self.master_uri, "/data/") (_, content) = http.request(master_data_uri, "POST", pickle.dumps(self.local_continuation)) continuation_uri, size_hint = simplejson.loads(content) # 3. Submit a new task with the updated local continuation. task_descriptor = {'dependencies': {'_cont' : SWURLReference([continuation_uri], size_hint)}, 'handler': 'swi', 'save_continuation': True} master_task_submit_uri = urlparse.urljoin(self.master_uri, "/task/") (_, content) = http.request(master_task_submit_uri, "POST", simplejson.dumps(task_descriptor, cls=SWReferenceJSONEncoder)) submit_result = simplejson.loads(content) # 4. Block to get the final result. expected_output_id = submit_result['outputs'][0] notify_url = urlparse.urljoin(self.master_uri, "/global_data/%d/completion" % expected_output_id) (_, result_content) = http.request(notify_url) completion_result = simplejson.loads(result_content, object_hook=json_decode_object_hook) if completion_result["exited"]: raise Exception("Server exited") # 5. Get updated local continuation. N.B. The originally-spawned task may have delegated, so we need to find the task from the actual producer of the expected output. task_for_output_url = urlparse.urljoin(self.master_uri, "/global_data/%d/task" % expected_output_id) (_, content) = http.request(task_for_output_url, "GET") end_task_descriptor = simplejson.loads(content, object_hook=json_decode_object_hook) saved_continuation_url = sw_to_external_url(end_task_descriptor['saved_continuation_uri']) (_, content) = http.request(saved_continuation_url) self.local_continuation = pickle.loads(content) # 6. Dereference result. return self.dereference_task_result(completion_result.refs[0])
def submit_task_and_wait(self, task_stmt): # 1. Update the local continuation with the current task_stmt. self.local_continuation.task_stmt = task_stmt # 2. POST the updated local continuation. http = httplib2.Http() master_data_uri = urlparse.urljoin(self.master_uri, "/data/") (_, content) = http.request(master_data_uri, "POST", pickle.dumps(self.local_continuation)) continuation_uri, size_hint = simplejson.loads(content) # 3. Submit a new task with the updated local continuation. task_descriptor = { 'dependencies': { '_cont': SWURLReference([continuation_uri], size_hint) }, 'handler': 'swi', 'save_continuation': True } master_task_submit_uri = urlparse.urljoin(self.master_uri, "/task/") (_, content) = http.request( master_task_submit_uri, "POST", simplejson.dumps(task_descriptor, cls=SWReferenceJSONEncoder)) submit_result = simplejson.loads(content) # 4. Block to get the final result. expected_output_id = submit_result['outputs'][0] notify_url = urlparse.urljoin( self.master_uri, "/global_data/%d/completion" % expected_output_id) (_, result_content) = http.request(notify_url) completion_result = simplejson.loads( result_content, object_hook=json_decode_object_hook) if completion_result["exited"]: raise Exception("Server exited") # 5. Get updated local continuation. N.B. The originally-spawned task may have delegated, so we need to find the task from the actual producer of the expected output. task_for_output_url = urlparse.urljoin( self.master_uri, "/global_data/%d/task" % expected_output_id) (_, content) = http.request(task_for_output_url, "GET") end_task_descriptor = simplejson.loads( content, object_hook=json_decode_object_hook) saved_continuation_url = sw_to_external_url( end_task_descriptor['saved_continuation_uri']) (_, content) = http.request(saved_continuation_url) self.local_continuation = pickle.loads(content) # 6. Dereference result. return self.dereference_task_result(completion_result.refs[0])
def main(): parser = OptionParser() parser.add_option("-i", "--index", action="store", dest="index", help="Index SWBS URI", metavar="URI", default=None) parser.add_option("-b", "--block", action="store", dest="block", help="Block SWBS URI", metavar="URI", default=None) (options, _) = parser.parse_args() h = httplib2.Http() if options.block is not None: netloc = get_netloc_for_sw_url(options.block) id = get_id_for_sw_url(options.block) response, _ = h.request('http://%s/admin/pin/%s' % (netloc, id), 'POST', 'pin') assert response.status == 200 print >> sys.stderr, 'Pinned block %s to %s' % (id, netloc) if options.index is not None: index_url = sw_to_external_url(options.index) _, content = h.request(index_url, 'GET') index = simplejson.loads(content, object_hook=json_decode_object_hook) for chunk in index: assert isinstance(chunk, SW2_ConcreteReference) for netloc in chunk.location_hints: response, _ = h.request( 'http://%s/admin/pin/%s' % (netloc, chunk.id), 'POST', 'pin') assert response.status == 200 print >> sys.stderr, 'Pinned block %s to %s' % (chunk.id, netloc)