def main():
log_format = (
'%(asctime)s.%(msecs)d %(levelname)s %(module)s - '
'%(funcName)s: %(message)s')
logging.basicConfig(
stream=sys.stdout, level=logging.DEBUG,
format=log_format,
datefmt="%Y-%m-%d %H:%M:%S")
args = _parse_args()
config = carbonate_config.Config(args.config_file)
local_cluster = args.cluster
local_user = config.ssh_user(local_cluster)
local_node = args.node
remote_cluster = args.remote_cluster
remote_nodes = _get_nodes(config, remote_cluster, node=local_node)
base_dir = args.temp_dir
if getpass.getuser() != local_user:
_abort('This program must be run as the "%s" user' % local_user)
info('- I am %s part of cluster %s and trying to sync with %s' %
(local_node, local_cluster, remote_cluster))
remote_nodes = list(remote_nodes)
random.shuffle(remote_nodes)
for n, node in enumerate(remote_nodes):
info('- Syncing node %s (%d/%d)' % (node, n + 1, len(remote_nodes)))
info('- %s: Listing metrics' % node)
metrics = _list_metrics(
config, local_cluster, local_node, node, args.exclude,
ssh_options=args.ssh_options)
staging_dir = _make_staging_dir(base_dir, node)
metrics_fs = [carbonate_util.metric_to_fs(m) for m in metrics]
batches = _Batch(
staging_dir=staging_dir,
metrics_fs=metrics_fs,
start_time=args.start_time if HAVE_HEAL_WITH_TIME_RANGE else 0,
end_time=args.end_time if HAVE_HEAL_WITH_TIME_RANGE else time.now(),
remote_user=config.ssh_user(remote_cluster),
remote_node=node,
rsync_options=args.rsync_options,
ssh_options=args.ssh_options,
overwrite=args.overwrite if HAVE_HEAL_WITH_OVERWRITE else False,
).split_chunks(args.batch_size)
info('- %s: Merging and fetching %s metrics' % (node, len(metrics)))
with progressbar.ProgressBar(redirect_stdout=True) as bar:
_fetch_merge(batches, bar.update)
info(' %s: Cleaning up' % node)
shutil.rmtree(staging_dir)
info(' %s: Done' % node)
info('')
def test_metric_to_fs_no_prepend(self):
in_ = "servers.s0123.foo.bar"
out = "servers/s0123/foo/bar.wsp"
assert metric_to_fs(in_) == out
def test_metric_to_fs_prepend(self):
in_ = "servers.s0123.foo.bar"
out = "/mnt/data/whisper/servers/s0123/foo/bar.wsp"
assert metric_to_fs(in_, prepend='/mnt/data/whisper') == out
def test_metric_to_fs_no_prepend(self):
in_ = "servers.s0123.foo.bar"
out = "servers/s0123/foo/bar.wsp"
assert metric_to_fs(in_) == out
def test_metric_to_fs_prepend(self):
in_ = "servers.s0123.foo.bar"
out = "/mnt/data/whisper/servers/s0123/foo/bar.wsp"
assert metric_to_fs(in_, prepend='/mnt/data/whisper') == out