def test_generate_daemon_config(self):
""" Test the generation of the daemon config file"""
res = "[zkrs-new]\npath = /tmp\nread only = no\nuid = root\ngid = root\n\n"
zkclient = RsyncDestination('dummy', rsyncpath='/tmp', session='new')
filen = zkclient.generate_daemon_config()
filec = open(filen, "r").read()
self.assertEqual(filec, res)
def test_generate_daemon_config(self):
""" Test the generation of the daemon config file"""
res = "[zkrs-new]\npath = /tmp\nread only = no\nuid = root\ngid = root\n\n"
zkclient = RsyncDestination('dummy', rsyncpath='/tmp', session='new')
filen = zkclient.generate_daemon_config()
filec = open(filen, "r").read()
self.assertEqual(filec, res)
def start_destination(options, kwargs):
"""Starts a destination: Start zookeeper connection and rsync daemon"""
kwargs['rsyncport'] = options.rsyncport
kwargs['startport'] = options.startport
kwargs['domain'] = options.domain
rsyncD = RsyncDestination(options.servers, **kwargs)
rsyncD.run()
logger.debug('%s Ready' % rsyncD.get_whoami())
rsyncD.exit()
sys.exit(0)
def test_activate_and_pausing_dests(self):
""" Test the pausing , disabling and activation of destinations """
zkclient = RsyncDestination('dummy', rsyncpath='/tmp', session='new')
zkclient.pause()
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, '')
zkclient.activate()
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, 'active')
zkclient.pause()
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, 'paused')
zkclient = RsyncSource('dummy',
session='new',
netcat=True,
rsyncpath='/path/dummy',
rsyncdepth=2)
self.assertFalse(zkclient.dest_is_sane('test'))
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, '')
zkclient.setstr('/admin/rsync/new/dests/test', 'active')
self.assertTrue(zkclient.dest_is_sane('test'))
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, 'active')
zkclient.setstr('/admin/rsync/new/dests/test', 'paused')
self.assertFalse(zkclient.dest_is_sane('test'))
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, 'disabled')
def test_activate_and_pausing_dests(self):
""" Test the pausing , disabling and activation of destinations """
zkclient = RsyncDestination('dummy', rsyncpath='/tmp', session='new')
zkclient.pause()
val, dum = zkclient.get('/admin/rsync/new/dests/test')
self.assertEqual(val, '')
zkclient.activate()
val, dum = zkclient.get('/admin/rsync/new/dests/test')
self.assertEqual(val, 'active')
zkclient.pause()
val, dum = zkclient.get('/admin/rsync/new/dests/test')
self.assertEqual(val, 'paused')
zkclient = RsyncSource('dummy', session='new', netcat=True, rsyncpath='/path/dummy', rsyncdepth=2)
self.assertFalse(zkclient.dest_is_sane('test'))
val, dum = zkclient.get('/admin/rsync/new/dests/test')
self.assertEqual(val, '')
zkclient.set('/admin/rsync/new/dests/test', 'active')
self.assertTrue(zkclient.dest_is_sane('test'))
val, dum = zkclient.get('/admin/rsync/new/dests/test')
self.assertEqual(val, 'active')
zkclient.set('/admin/rsync/new/dests/test', 'paused')
self.assertFalse(zkclient.dest_is_sane('test'))
val, dum = zkclient.get('/admin/rsync/new/dests/test')
self.assertEqual(val, 'disabled')
def main():
""" Start a new rsync client (destination or source) in a specified session """
options = {
# Zookeeper connection options:
'servers' : ('list of zk servers', 'strlist', 'store', None),
'user' : ('user with creation rights on zookeeper', None, 'store', 'root', 'u'),
'passwd' : ('password for user with creation rights', None, 'store', 'admin', 'p'),
# Role options, define exactly one of these:
'source' : ('rsync source', None, 'store_true', False, 'S'),
'destination' : ('rsync destination', None, 'store_true', False, 'D'),
'pathsonly' : ('Only do a test run of the pathlist building', None, 'store_true', False),
'state' : ('Only do the state', None, 'store_true', False),
# Session options; should be the same on all clients of the session!
'session' : ('session name', None, 'store', 'default', 'N'),
'netcat' : ('run netcat test instead of rsync', None, 'store_true', False),
'dryrun' : ('run rsync in dry run mode', None, 'store_true', False, 'n'),
'rsyncpath' : ('rsync basepath', None, 'store', None, 'r'), # May differ between sources and dests
# Pathbuilding (Source clients and pathsonly ) specific options:
'excludere' : ('Exclude from pathbuilding', None, 'regex', re.compile('/\.snapshots(/.*|$)')),
'depth' : ('queue depth', "int", 'store', 4),
# Source clients options; should be the same on all clients of the session!:
'delete' : ('run rsync with --delete', None, 'store_true', False),
# Individual client options
'domain' : ('substitute domain', None, 'store', None),
'logfile' : ('Output to logfile', None, 'store', '/tmp/zkrsync/%(session)s-%(rstype)s-%(pid)s.log'),
# Individual Destination client specific options
'rsyncport' : ('force port on which rsyncd binds', "int", 'store', None),
'startport' : ('offset to look for rsyncd ports', "int", 'store', 4444)
}
go = simple_option(options)
acreds, admin_acl, rstype = zkrsync_parse(go.options)
if go.options.logfile:
logfile = go.options.logfile % {
'session': go.options.session,
'rstype': rstype,
'pid': str(os.getpid())
}
logdir = os.path.dirname(logfile)
if logdir:
if not os.path.exists(logdir):
os.makedirs(logdir)
os.chmod(logdir, stat.S_IRWXU)
fancylogger.logToFile(logfile)
logger.debug('Logging to file %s:' % logfile)
kwargs = {
'session' : go.options.session,
'default_acl' : [admin_acl],
'auth_data' : acreds,
'rsyncpath' : go.options.rsyncpath,
'netcat' : go.options.netcat,
}
if go.options.state:
rsyncP = RsyncSource(go.options.servers, **kwargs)
logger.info('Progress: %s of %s paths remaining' % (rsyncP.len_paths(), rsyncP.paths_total))
rsyncP.exit()
sys.exit(0)
elif go.options.pathsonly:
kwargs['rsyncdepth'] = go.options.depth
kwargs['excludere'] = go.options.excludere
rsyncP = RsyncSource(go.options.servers, **kwargs)
locked = rsyncP.acq_lock()
if locked:
starttime = time.time()
rsyncP.build_pathqueue()
endtime = time.time()
timing = endtime - starttime
pathqueue = rsyncP.path_queue
logger.info('Building with depth %i took %f seconds walltime. there are %i paths in the Queue'
% (go.options.depth, timing, len(pathqueue)))
rsyncP.delete(pathqueue.path, recursive=True)
rsyncP.release_lock()
else:
logger.error('There is already a lock on the pathtree of this session')
rsyncP.exit()
sys.exit(0)
elif rstype == CL_DEST:
# Start zookeeper connection and rsync daemon
kwargs['rsyncport'] = go.options.rsyncport
kwargs['startport'] = go.options.startport
kwargs['domain'] = go.options.domain
rsyncD = RsyncDestination(go.options.servers, **kwargs)
rsyncD.run()
logger.debug('%s Ready' % rsyncD.get_whoami())
rsyncD.exit()
sys.exit(0)
elif rstype == CL_SOURCE:
# Start zookeeper connections
kwargs['rsyncdepth'] = go.options.depth
kwargs['dryrun'] = go.options.dryrun
kwargs['delete'] = go.options.delete
kwargs['excludere'] = go.options.excludere
rsyncS = RsyncSource(go.options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
paths_total = rsyncS.build_pathqueue()
todo_paths = paths_total
while not rsyncS.isempty_pathqueue():
if todo_paths != rsyncS.len_paths(): # Output progress state
todo_paths = rsyncS.len_paths()
logger.info('Progress: %s of %s paths remaining' % (todo_paths, paths_total))
time.sleep(SLEEP_TIME)
rsyncS.shutdown_all()
rsyncS.exit()
sys.exit(0)
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
