class Cluster(): def __init__(self, config, cluster='main'): self.ring = ConsistentHashingRouter(config.replication_factor(cluster)) try: dest_list = config.destinations(cluster) self.destinations = util.parseDestinations(dest_list) except ValueError as e: raise SystemExit("Unable to parse destinations!" + str(e)) for d in self.destinations: self.ring.addDestination(d) def getDestinations(self, metric): return self.ring.getDestinations(metric)
metric_key = arg # Check required key if not metric_key: print('Usage: python graphite-router.py -k <metric key>') sys.exit(2) ## Settings # Absolute path to the Graphite Data Directory DATA_DIR = join(ROOT_DIR, 'storage/whisper') # Parse config settings.readFrom(join(ROOT_DIR, 'conf/carbon.conf'), 'relay') # Read in destinations from config destinations = util.parseDestinations(settings.DESTINATIONS) # Setup Router router = ConsistentHashingRouter(settings.REPLICATION_FACTOR) for destination in destinations: router.addDestination(destination); # Echo routes print('routes for ' + metric_key) routes = router.getDestinations(metric_key) for route in routes: print(route)
for destination in destinations: if destination[0] == target_node: local_destinations.append(destination) router.addDestination(destination); # Walk Data dir and process orphaned whisper files for dirname, dirnames, filenames in os.walk(DATA_DIR): #if dirname.startswith(join(DATA_DIR, settings.CARBON_METRIC_PREFIX)): # continue for filename in filenames: pathname = os.path.join(dirname, filename) basename, ext = os.path.splitext(filename) if '.wsp' != ext: print('skipping %s' % relpath(pathname, DATA_DIR)) continue metric_dest = router.getDestinations(relpath(join(dirname, basename), DATA_DIR).replace('/', '.')) orphaned = True for mdest in metric_dest: if mdest in local_destinations: orphaned = False break if orphaned: print('renaming %s' % pathname) os.rename(pathname, join(dirname, basename + '.wsp_orph')) # os.unlink(pathname)
print "Beginning scan of: %s" % WHISPER_PATH for dirname, dirnames, filenames in os.walk(WHISPER_PATH): for filename in filenames: pathname = os.path.join(dirname, filename) rel_pathname = os.path.relpath(pathname, WHISPER_PATH) basename, ext = os.path.splitext(filename) if '.wsp' != ext: print('skipping %s' % pathname) # convert filesystem path to metric style path. remove WHISPER_DIR prefix, then # convert / to . rel_path = os.path.relpath(os.path.join(dirname, basename), WHISPER_PATH) metric_path = rel_path.replace('/', '.') dests = [d for d in router.getDestinations(metric_path)] for d in dests: if d not in file_map: file_map[d] = [] file_map[d].append(rel_pathname) # relative pathname for rsync --files-from if DEBUG: print "%s\t%s" % (dests, pathname) print "Finished scan. Writing file lists ...\n" for shard, files in file_map.items(): shard_id = shard[2] # ('vnode-30', '3130', '30') file_name = '%s.files' % shard_id # 30.files print "Saving file list: %s ..." % file_name with open(file_name, 'w') as f: