def compute_best_worker_for_task(self, task):
netlocs = {}
for input in task.inputs.values():
if isinstance(input, SWURLReference):
if input.size_hint is None:
# XXX: We don't know the size of objects from outside the
# cluster. So we make a guess
# TODO: Do something sensible here; probably HTTP HEAD
input.size_hint = 10000000
for url in input.urls:
netloc = get_netloc_for_sw_url(url)
try:
current_saving_for_netloc = netlocs[netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[netloc] = current_saving_for_netloc + input.size_hint
elif isinstance(input, SW2_ConcreteReference) and input.size_hint is not None:
for netloc in input.location_hints:
try:
current_saving_for_netloc = netlocs[netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[netloc] = current_saving_for_netloc + input.size_hint
ranked_netlocs = [(saving, netloc) for (netloc, saving) in netlocs.items()]
filtered_ranked_netlocs = filter(lambda (saving, netloc) : self.worker_pool.get_worker_at_netloc(netloc) is not None, ranked_netlocs)
if len(filtered_ranked_netlocs) > 0:
ret = self.worker_pool.get_worker_at_netloc(max(filtered_ranked_netlocs)[1])
return ret
else:
return None
def main():
parser = OptionParser()
parser.add_option("-i", "--index", action="store", dest="index", help="Index SWBS URI", metavar="URI", default=None)
parser.add_option("-b", "--block", action="store", dest="block", help="Block SWBS URI", metavar="URI", default=None)
(options, _) = parser.parse_args()
h = httplib2.Http()
if options.block is not None:
netloc = get_netloc_for_sw_url(options.block)
id = get_id_for_sw_url(options.block)
response, _ = h.request('http://%s/admin/pin/%s' % (netloc, id), 'POST', 'pin')
assert response.status == 200
print >>sys.stderr, 'Pinned block %s to %s' % (id, netloc)
if options.index is not None:
index_url = sw_to_external_url(options.index)
_, content = h.request(index_url, 'GET')
index = simplejson.loads(content, object_hook=json_decode_object_hook)
for chunk in index:
assert isinstance(chunk, SW2_ConcreteReference)
for netloc in chunk.location_hints:
response, _ = h.request('http://%s/admin/pin/%s' % (netloc, chunk.id), 'POST', 'pin')
assert response.status == 200
print >>sys.stderr, 'Pinned block %s to %s' % (chunk.id, netloc)
def compute_good_workers_for_task(self, task):
netlocs = {}
for input in task.inputs.values():
if isinstance(input, SWURLReference):
if input.size_hint is None:
# XXX: We don't know the size of objects from outside the
# cluster. So we make a guess
# TODO: Do something sensible here; probably HTTP HEAD
input.size_hint = 10000000
for url in input.urls:
netloc = get_netloc_for_sw_url(url)
try:
current_saving_for_netloc = netlocs[netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[
netloc] = current_saving_for_netloc + input.size_hint
elif isinstance(
input,
SW2_SweetheartReference) and input.size_hint is not None:
try:
current_saving_for_netloc = netlocs[
input.sweetheart_netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[
netloc] = current_saving_for_netloc + SWEETHEART_FACTOR * input.size_hint
# Accord the unboosted saving to other locations.
for netloc in input.location_hints:
try:
current_saving_for_netloc = netlocs[netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[
netloc] = current_saving_for_netloc + input.size_hint
elif isinstance(
input,
SW2_ConcreteReference) and input.size_hint is not None:
for netloc in input.location_hints:
try:
current_saving_for_netloc = netlocs[netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[
netloc] = current_saving_for_netloc + input.size_hint
ranked_netlocs = [(saving, netloc)
for (netloc, saving) in netlocs.items()]
filtered_ranked_netlocs = filter(
lambda (saving, netloc): self.worker_pool.get_worker_at_netloc(
netloc) is not None, ranked_netlocs)
if len(filtered_ranked_netlocs) > 0:
max_saving = max(filtered_ranked_netlocs)[0]
for saving, netloc in filtered_ranked_netlocs:
if saving > (EQUALLY_LOCAL_MARGIN * max_saving):
yield self.worker_pool.get_worker_at_netloc(netloc)
def main():
parser = OptionParser()
parser.add_option("-i",
"--index",
action="store",
dest="index",
help="Index SWBS URI",
metavar="URI",
default=None)
parser.add_option("-b",
"--block",
action="store",
dest="block",
help="Block SWBS URI",
metavar="URI",
default=None)
(options, _) = parser.parse_args()
h = httplib2.Http()
if options.block is not None:
netloc = get_netloc_for_sw_url(options.block)
id = get_id_for_sw_url(options.block)
response, _ = h.request('http://%s/admin/pin/%s' % (netloc, id),
'POST', 'pin')
assert response.status == 200
print >> sys.stderr, 'Pinned block %s to %s' % (id, netloc)
if options.index is not None:
index_url = sw_to_external_url(options.index)
_, content = h.request(index_url, 'GET')
index = simplejson.loads(content, object_hook=json_decode_object_hook)
for chunk in index:
assert isinstance(chunk, SW2_ConcreteReference)
for netloc in chunk.location_hints:
response, _ = h.request(
'http://%s/admin/pin/%s' % (netloc, chunk.id), 'POST',
'pin')
assert response.status == 200
print >> sys.stderr, 'Pinned block %s to %s' % (chunk.id,
netloc)
def compute_good_workers_for_task(self, task):
netlocs = {}
for input in task.inputs.values():
if isinstance(input, SWURLReference):
if input.size_hint is None:
# XXX: We don't know the size of objects from outside the
# cluster. So we make a guess
# TODO: Do something sensible here; probably HTTP HEAD
input.size_hint = 10000000
for url in input.urls:
netloc = get_netloc_for_sw_url(url)
try:
current_saving_for_netloc = netlocs[netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[netloc] = current_saving_for_netloc + input.size_hint
elif isinstance(input, SW2_SweetheartReference) and input.size_hint is not None:
try:
current_saving_for_netloc = netlocs[input.sweetheart_netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[netloc] = current_saving_for_netloc + SWEETHEART_FACTOR * input.size_hint
# Accord the unboosted saving to other locations.
for netloc in input.location_hints:
try:
current_saving_for_netloc = netlocs[netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[netloc] = current_saving_for_netloc + input.size_hint
elif isinstance(input, SW2_ConcreteReference) and input.size_hint is not None:
for netloc in input.location_hints:
try:
current_saving_for_netloc = netlocs[netloc]
except KeyError:
current_saving_for_netloc = 0
netlocs[netloc] = current_saving_for_netloc + input.size_hint
ranked_netlocs = [(saving, netloc) for (netloc, saving) in netlocs.items()]
filtered_ranked_netlocs = filter(lambda (saving, netloc) : self.worker_pool.get_worker_at_netloc(netloc) is not None, ranked_netlocs)
if len(filtered_ranked_netlocs) > 0:
max_saving = max(filtered_ranked_netlocs)[0]
for saving, netloc in filtered_ranked_netlocs:
if saving > (EQUALLY_LOCAL_MARGIN * max_saving):
yield self.worker_pool.get_worker_at_netloc(netloc)
