def compute_best_worker_for_task(self, task): netlocs = {} for input in task.inputs.values(): if isinstance(input, SWURLReference): if input.size_hint is None: # XXX: We don't know the size of objects from outside the # cluster. So we make a guess # TODO: Do something sensible here; probably HTTP HEAD input.size_hint = 10000000 for url in input.urls: netloc = get_netloc_for_sw_url(url) try: current_saving_for_netloc = netlocs[netloc] except KeyError: current_saving_for_netloc = 0 netlocs[netloc] = current_saving_for_netloc + input.size_hint elif isinstance(input, SW2_ConcreteReference) and input.size_hint is not None: for netloc in input.location_hints: try: current_saving_for_netloc = netlocs[netloc] except KeyError: current_saving_for_netloc = 0 netlocs[netloc] = current_saving_for_netloc + input.size_hint ranked_netlocs = [(saving, netloc) for (netloc, saving) in netlocs.items()] filtered_ranked_netlocs = filter(lambda (saving, netloc) : self.worker_pool.get_worker_at_netloc(netloc) is not None, ranked_netlocs) if len(filtered_ranked_netlocs) > 0: ret = self.worker_pool.get_worker_at_netloc(max(filtered_ranked_netlocs)[1]) return ret else: return None
def main(): parser = OptionParser() parser.add_option("-i", "--index", action="store", dest="index", help="Index SWBS URI", metavar="URI", default=None) parser.add_option("-b", "--block", action="store", dest="block", help="Block SWBS URI", metavar="URI", default=None) (options, _) = parser.parse_args() h = httplib2.Http() if options.block is not None: netloc = get_netloc_for_sw_url(options.block) id = get_id_for_sw_url(options.block) response, _ = h.request('http://%s/admin/pin/%s' % (netloc, id), 'POST', 'pin') assert response.status == 200 print >>sys.stderr, 'Pinned block %s to %s' % (id, netloc) if options.index is not None: index_url = sw_to_external_url(options.index) _, content = h.request(index_url, 'GET') index = simplejson.loads(content, object_hook=json_decode_object_hook) for chunk in index: assert isinstance(chunk, SW2_ConcreteReference) for netloc in chunk.location_hints: response, _ = h.request('http://%s/admin/pin/%s' % (netloc, chunk.id), 'POST', 'pin') assert response.status == 200 print >>sys.stderr, 'Pinned block %s to %s' % (chunk.id, netloc)
def compute_good_workers_for_task(self, task): netlocs = {} for input in task.inputs.values(): if isinstance(input, SWURLReference): if input.size_hint is None: # XXX: We don't know the size of objects from outside the # cluster. So we make a guess # TODO: Do something sensible here; probably HTTP HEAD input.size_hint = 10000000 for url in input.urls: netloc = get_netloc_for_sw_url(url) try: current_saving_for_netloc = netlocs[netloc] except KeyError: current_saving_for_netloc = 0 netlocs[ netloc] = current_saving_for_netloc + input.size_hint elif isinstance( input, SW2_SweetheartReference) and input.size_hint is not None: try: current_saving_for_netloc = netlocs[ input.sweetheart_netloc] except KeyError: current_saving_for_netloc = 0 netlocs[ netloc] = current_saving_for_netloc + SWEETHEART_FACTOR * input.size_hint # Accord the unboosted saving to other locations. for netloc in input.location_hints: try: current_saving_for_netloc = netlocs[netloc] except KeyError: current_saving_for_netloc = 0 netlocs[ netloc] = current_saving_for_netloc + input.size_hint elif isinstance( input, SW2_ConcreteReference) and input.size_hint is not None: for netloc in input.location_hints: try: current_saving_for_netloc = netlocs[netloc] except KeyError: current_saving_for_netloc = 0 netlocs[ netloc] = current_saving_for_netloc + input.size_hint ranked_netlocs = [(saving, netloc) for (netloc, saving) in netlocs.items()] filtered_ranked_netlocs = filter( lambda (saving, netloc): self.worker_pool.get_worker_at_netloc( netloc) is not None, ranked_netlocs) if len(filtered_ranked_netlocs) > 0: max_saving = max(filtered_ranked_netlocs)[0] for saving, netloc in filtered_ranked_netlocs: if saving > (EQUALLY_LOCAL_MARGIN * max_saving): yield self.worker_pool.get_worker_at_netloc(netloc)
def main(): parser = OptionParser() parser.add_option("-i", "--index", action="store", dest="index", help="Index SWBS URI", metavar="URI", default=None) parser.add_option("-b", "--block", action="store", dest="block", help="Block SWBS URI", metavar="URI", default=None) (options, _) = parser.parse_args() h = httplib2.Http() if options.block is not None: netloc = get_netloc_for_sw_url(options.block) id = get_id_for_sw_url(options.block) response, _ = h.request('http://%s/admin/pin/%s' % (netloc, id), 'POST', 'pin') assert response.status == 200 print >> sys.stderr, 'Pinned block %s to %s' % (id, netloc) if options.index is not None: index_url = sw_to_external_url(options.index) _, content = h.request(index_url, 'GET') index = simplejson.loads(content, object_hook=json_decode_object_hook) for chunk in index: assert isinstance(chunk, SW2_ConcreteReference) for netloc in chunk.location_hints: response, _ = h.request( 'http://%s/admin/pin/%s' % (netloc, chunk.id), 'POST', 'pin') assert response.status == 200 print >> sys.stderr, 'Pinned block %s to %s' % (chunk.id, netloc)
def compute_good_workers_for_task(self, task): netlocs = {} for input in task.inputs.values(): if isinstance(input, SWURLReference): if input.size_hint is None: # XXX: We don't know the size of objects from outside the # cluster. So we make a guess # TODO: Do something sensible here; probably HTTP HEAD input.size_hint = 10000000 for url in input.urls: netloc = get_netloc_for_sw_url(url) try: current_saving_for_netloc = netlocs[netloc] except KeyError: current_saving_for_netloc = 0 netlocs[netloc] = current_saving_for_netloc + input.size_hint elif isinstance(input, SW2_SweetheartReference) and input.size_hint is not None: try: current_saving_for_netloc = netlocs[input.sweetheart_netloc] except KeyError: current_saving_for_netloc = 0 netlocs[netloc] = current_saving_for_netloc + SWEETHEART_FACTOR * input.size_hint # Accord the unboosted saving to other locations. for netloc in input.location_hints: try: current_saving_for_netloc = netlocs[netloc] except KeyError: current_saving_for_netloc = 0 netlocs[netloc] = current_saving_for_netloc + input.size_hint elif isinstance(input, SW2_ConcreteReference) and input.size_hint is not None: for netloc in input.location_hints: try: current_saving_for_netloc = netlocs[netloc] except KeyError: current_saving_for_netloc = 0 netlocs[netloc] = current_saving_for_netloc + input.size_hint ranked_netlocs = [(saving, netloc) for (netloc, saving) in netlocs.items()] filtered_ranked_netlocs = filter(lambda (saving, netloc) : self.worker_pool.get_worker_at_netloc(netloc) is not None, ranked_netlocs) if len(filtered_ranked_netlocs) > 0: max_saving = max(filtered_ranked_netlocs)[0] for saving, netloc in filtered_ranked_netlocs: if saving > (EQUALLY_LOCAL_MARGIN * max_saving): yield self.worker_pool.get_worker_at_netloc(netloc)