def main():
parser = OptionParser()
parser.add_option("-m",
"--master",
action="store",
dest="master",
help="Master URI",
metavar="MASTER",
default=os.getenv("SW_MASTER"))
parser.add_option("-f",
"--force",
action="store_true",
dest="force",
help="Set this flag to really flush the blocks",
default=False)
(options, _) = parser.parse_args()
h = httplib2.Http()
workers = get_worker_netlocs(options.master)
for netloc in workers:
if options.force:
response, content = h.request(
'http://%s/admin/flush/really' % (netloc), 'POST', 'flush')
assert response.status == 200
print >> sys.stderr, 'Flushed worker: %s' % netloc
else:
response, content = h.request('http://%s/admin/flush/' % (netloc),
'POST', 'flush')
assert response.status == 200
print >> sys.stderr, 'Worker: %s' % netloc
print '---', content
def main():
parser = OptionParser()
parser.add_option(
"-m",
"--master",
action="store",
dest="master",
help="Master URI",
metavar="MASTER",
default=os.getenv("SW_MASTER"),
)
parser.add_option(
"-f",
"--force",
action="store_true",
dest="force",
help="Set this flag to really flush the blocks",
default=False,
)
(options, _) = parser.parse_args()
h = httplib2.Http()
workers = get_worker_netlocs(options.master)
for netloc in workers:
if options.force:
response, content = h.request("http://%s/admin/flush/really" % (netloc), "POST", "flush")
assert response.status == 200
print >> sys.stderr, "Flushed worker: %s" % netloc
else:
response, content = h.request("http://%s/admin/flush/" % (netloc), "POST", "flush")
assert response.status == 200
print >> sys.stderr, "Worker: %s" % netloc
print "---", content
def main():
parser = OptionParser()
parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER"))
parser.add_option("-p", "--prefix", action="store", dest="prefix", help="Block name prefix", metavar="NAME", default=None)
parser.add_option("-l", "--list", action="store_true", dest="list", help="Lists the pinned blocks in the cluster", default=False)
parser.add_option("-r", "--replication", action="store", dest="replication", help="Copies of each block", type="int", metavar="N", default=1)
(options, _) = parser.parse_args()
if (options.prefix is None and not options.list) or options.master is None:
parser.usage()
sys.exit(-1)
h = httplib2.Http()
workers = get_worker_netlocs(options.master)
if options.list:
id_to_netloc = {}
for netloc in workers:
response, content = h.request('http://%s/control/admin/pin/' % netloc, 'GET')
assert response.status == 200
pin_set = simplejson.loads(content, object_hook=json_decode_object_hook)
for ref in pin_set:
try:
existing_set = id_to_netloc[ref.id]
except KeyError:
existing_set = set()
id_to_netloc[ref.id] = existing_set
existing_set.add(netloc)
for id in sorted(id_to_netloc.keys()):
print '%s\t%s' % (id, ", ".join(id_to_netloc[id]))
else:
id_to_ref = {}
for netloc in workers:
response, content = h.request('http://%s/control/admin/pin/' % netloc, 'GET')
assert response.status == 200
pin_set = simplejson.loads(content, object_hook=json_decode_object_hook)
for ref in pin_set:
if ref.id.startswith(options.prefix) and not ref.id.endswith('index'):
try:
existing_ref = id_to_ref[ref.id]
existing_ref.combine_with(ref)
except KeyError:
id_to_ref[ref.id] = ref
sorted_ids = sorted(id_to_ref.keys())
new_index = []
for id in sorted_ids:
new_index.append(id_to_ref[id])
index_name = '%s:recovered_index' % str(uuid.uuid4())
with open(index_name, 'w') as f:
simplejson.dump(new_index, f, cls=SWReferenceJSONEncoder)
print 'Wrote index to %s' % index_name
targets = select_targets(workers, options.replication)
upload_string_to_targets(simplejson.dumps(new_index, cls=SWReferenceJSONEncoder), index_name, targets)
for target in targets:
print 'swbs://%s/%s' % (target, index_name)
def main():
parser = OptionParser()
parser.add_option("-m",
"--master",
action="store",
dest="master",
help="Master URI",
metavar="MASTER",
default=os.getenv("SW_MASTER"))
parser.add_option("-p",
"--prefix",
action="store",
dest="prefix",
help="Block name prefix",
metavar="NAME",
default=None)
parser.add_option("-l",
"--list",
action="store_true",
dest="list",
help="Lists the pinned blocks in the cluster",
default=False)
parser.add_option("-r",
"--replication",
action="store",
dest="replication",
help="Copies of each block",
type="int",
metavar="N",
default=1)
(options, _) = parser.parse_args()
if (options.prefix is None and not options.list) or options.master is None:
parser.usage()
sys.exit(-1)
h = httplib2.Http()
workers = get_worker_netlocs(options.master)
if options.list:
id_to_netloc = {}
for netloc in workers:
response, content = h.request('http://%s/admin/pin/' % netloc,
'GET')
assert response.status == 200
pin_set = simplejson.loads(content,
object_hook=json_decode_object_hook)
for ref in pin_set:
try:
existing_set = id_to_netloc[ref.id]
except KeyError:
existing_set = set()
id_to_netloc[ref.id] = existing_set
existing_set.add(netloc)
for id in sorted(id_to_netloc.keys()):
print '%s\t%s' % (id, ", ".join(id_to_netloc[id]))
else:
id_to_ref = {}
for netloc in workers:
response, content = h.request('http://%s/admin/pin/' % netloc,
'GET')
assert response.status == 200
pin_set = simplejson.loads(content,
object_hook=json_decode_object_hook)
for ref in pin_set:
if ref.id.startswith(
options.prefix) and not ref.id.endswith('index'):
try:
existing_ref = id_to_ref[ref.id]
existing_ref.combine_with(ref)
except KeyError:
id_to_ref[ref.id] = ref
sorted_ids = sorted(id_to_ref.keys())
new_index = []
for id in sorted_ids:
new_index.append(id_to_ref[id])
index_name = '%s:recovered_index' % str(uuid.uuid4())
with open(index_name, 'w') as f:
simplejson.dump(new_index, f, cls=SWReferenceJSONEncoder)
print 'Wrote index to %s' % index_name
targets = select_targets(workers, options.replication)
upload_string_to_targets(
simplejson.dumps(new_index, cls=SWReferenceJSONEncoder),
index_name, targets)
for target in targets:
print 'swbs://%s/%s' % (target, index_name)