class Cluster():
def __init__(self, config, cluster='main'):
self.ring = ConsistentHashingRouter(config.replication_factor(cluster))
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def getDestinations(self, metric):
return self.ring.getDestinations(metric)
class Cluster():
def __init__(self, config, cluster='main'):
self.ring = ConsistentHashingRouter(config.replication_factor(cluster))
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def getDestinations(self, metric):
return self.ring.getDestinations(metric)
metric_key = arg
# Check required key
if not metric_key:
print('Usage: python graphite-router.py -k <metric key>')
sys.exit(2)
## Settings
# Absolute path to the Graphite Data Directory
DATA_DIR = join(ROOT_DIR, 'storage/whisper')
# Parse config
settings.readFrom(join(ROOT_DIR, 'conf/carbon.conf'), 'relay')
# Read in destinations from config
destinations = util.parseDestinations(settings.DESTINATIONS)
# Setup Router
router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
for destination in destinations:
router.addDestination(destination);
# Echo routes
print('routes for ' + metric_key)
routes = router.getDestinations(metric_key)
for route in routes:
print(route)
for destination in destinations:
if destination[0] == target_node:
local_destinations.append(destination)
router.addDestination(destination);
# Walk Data dir and process orphaned whisper files
for dirname, dirnames, filenames in os.walk(DATA_DIR):
#if dirname.startswith(join(DATA_DIR, settings.CARBON_METRIC_PREFIX)):
# continue
for filename in filenames:
pathname = os.path.join(dirname, filename)
basename, ext = os.path.splitext(filename)
if '.wsp' != ext:
print('skipping %s' % relpath(pathname, DATA_DIR))
continue
metric_dest = router.getDestinations(relpath(join(dirname, basename), DATA_DIR).replace('/', '.'))
orphaned = True
for mdest in metric_dest:
if mdest in local_destinations:
orphaned = False
break
if orphaned:
print('renaming %s' % pathname)
os.rename(pathname, join(dirname, basename + '.wsp_orph'))
# os.unlink(pathname)
print "Beginning scan of: %s" % WHISPER_PATH
for dirname, dirnames, filenames in os.walk(WHISPER_PATH):
for filename in filenames:
pathname = os.path.join(dirname, filename)
rel_pathname = os.path.relpath(pathname, WHISPER_PATH)
basename, ext = os.path.splitext(filename)
if '.wsp' != ext:
print('skipping %s' % pathname)
# convert filesystem path to metric style path. remove WHISPER_DIR prefix, then
# convert / to .
rel_path = os.path.relpath(os.path.join(dirname, basename), WHISPER_PATH)
metric_path = rel_path.replace('/', '.')
dests = [d for d in router.getDestinations(metric_path)]
for d in dests:
if d not in file_map:
file_map[d] = []
file_map[d].append(rel_pathname) # relative pathname for rsync --files-from
if DEBUG:
print "%s\t%s" % (dests, pathname)
print "Finished scan. Writing file lists ...\n"
for shard, files in file_map.items():
shard_id = shard[2] # ('vnode-30', '3130', '30')
file_name = '%s.files' % shard_id # 30.files
print "Saving file list: %s ..." % file_name
with open(file_name, 'w') as f: