def test_get_state(self, mock_len):
zkclient = RsyncSource('dummy', session='new', netcat=True, rsyncpath='/path/dummy', rsyncdepth=2)
mock_len.return_value = 5
self.assertEqual(zkclient.get_state(), 0)
mock_len.return_value = 0
self.assertEqual(zkclient.get_state(), ZKRS_NO_SUCH_SESSION_EXIT_CODE)
def test_generate_config(self):
""" Test the generation of the config file"""
zkclient = RsyncSource('dummy', netcat=True, rsyncpath='/path/dummy', rsyncdepth=2)
filen = zkclient.generate_file('/path/dummy/some/path')
filec = open(filen, "r").read()
self.assertEqual(filec, 'some/path/')
self.assertRaises(Exception, zkclient.generate_file, '/path/wrong/path')
def get_state(servers, kwargs):
"""Get the state of a running session"""
kwargs['verifypath'] = False # Not needed for state info
rsyncP = RsyncSource(servers, rsyncdepth=0, **kwargs)
code = rsyncP.get_state()
rsyncP.exit()
sys.exit(code)
def test_wirte_donefile(self):
""" Test the writing of the values to a cache file when done"""
donefile = "/tmp/done"
values = {
'completed' : 50,
'failed' : 5,
'unfinished' : 0
}
zkclient = RsyncSource('dummy', session='new', netcat=True, rsyncpath='/path/dummy', rsyncdepth=2, done_file=donefile)
zkclient.write_donefile(values)
cache_file = FileCache(donefile)
(timestamp, stats) = cache_file.load('stats')
self.assertEqual(values, stats)
def test_activate_and_pausing_dests(self):
""" Test the pausing , disabling and activation of destinations """
zkclient = RsyncDestination('dummy', rsyncpath='/tmp', session='new')
zkclient.pause()
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, '')
zkclient.activate()
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, 'active')
zkclient.pause()
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, 'paused')
zkclient = RsyncSource('dummy',
session='new',
netcat=True,
rsyncpath='/path/dummy',
rsyncdepth=2)
self.assertFalse(zkclient.dest_is_sane('test'))
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, '')
zkclient.setstr('/admin/rsync/new/dests/test', 'active')
self.assertTrue(zkclient.dest_is_sane('test'))
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, 'active')
zkclient.setstr('/admin/rsync/new/dests/test', 'paused')
self.assertFalse(zkclient.dest_is_sane('test'))
val, dum = zkclient.getstr('/admin/rsync/new/dests/test')
self.assertEqual(val, 'disabled')
def test_generate_stats(self):
""" Test the correct working of stats gathering and outputting """
zkclient = RsyncSource('dummy', netcat=True, rsyncpath='/path/dummy', rsyncdepth=2, verbose=True)
zkclient.parse_output(rsync_output)
self.assertEqual(zkclient.output_stats(), json_output)
zkclient.parse_output(rsync_output)
self.assertEqual(zkclient.output_stats(), json_output2)
def test_generate_stats(self):
""" Test the correct working of stats gathering and outputting """
zkclient = RsyncSource('dummy',
netcat=True,
rsyncpath='/path/dummy',
rsyncdepth=2,
verbose=True)
zkclient.parse_output(rsync_output)
self.assertEqual(literal_eval(zkclient.output_stats()), json_output)
zkclient.parse_output(rsync_output)
self.assertEqual(literal_eval(zkclient.output_stats()), json_output2)
def start_source(options, kwargs):
""" Start a rsync source"""
kwargs['rsyncdepth'] = options.depth
kwargs['dryrun'] = options.dryrun
kwargs['delete'] = options.delete
kwargs['excludere'] = options.excludere
# Start zookeeper connections
rsyncS = RsyncSource(options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
paths_total = rsyncS.build_pathqueue()
rsyncS.wait_and_keep_progress(paths_total)
rsyncS.shutdown_all()
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
def do_pathsonly(options, kwargs):
"""Only build the pathqueue and return timings"""
kwargs['rsyncdepth'] = options.depth
kwargs['excludere'] = options.excludere
rsyncP = RsyncSource(options.servers, **kwargs)
locked = rsyncP.acq_lock()
if locked:
starttime = time.time()
rsyncP.build_pathqueue()
endtime = time.time()
timing = endtime - starttime
pathqueue = rsyncP.path_queue
logger.info('Building with depth %i took %f seconds walltime. there are %i paths in the Queue'
% (options.depth, timing, len(pathqueue)))
rsyncP.delete(pathqueue.path, recursive=True)
rsyncP.release_lock()
else:
logger.error('There is already a lock on the pathtree of this session')
rsyncP.exit()
sys.exit(0)
def get_state(servers, kwargs):
"""Get the state of a running session"""
rsyncP = RsyncSource(go.options.servers, **kwargs)
logger.info('Progress: %s of %s paths remaining' % (rsyncP.len_paths(), rsyncP.paths_total))
rsyncP.exit()
sys.exit(0)
def start_source(options, kwargs):
""" Start a rsync source"""
kwargs['rsyncdepth'] = options.depth
kwargs['rsubpaths'] = options.rsubpaths
kwargs['arbitopts'] = options.arbitopts
kwargs['checksum'] = options.checksum
kwargs['done_file'] = options.done_file
kwargs['dryrun'] = options.dryrun
kwargs['delete'] = options.delete
kwargs['excludere'] = options.excludere
kwargs['excl_usr'] = options.excl_usr
kwargs['hardlinks'] = options.hardlinks
kwargs['inplace'] = options.inplace
kwargs['timeout'] = options.timeout
kwargs['verbose'] = options.verbose
# Start zookeeper connections
rsyncS = RsyncSource(options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
rsyncS.build_pathqueue()
rsyncS.wait_and_keep_progress()
rsyncS.shutdown_all()
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
def start_source(options, kwargs):
""" Start a rsync source"""
kwargs['rsyncdepth'] = options.depth
kwargs['rsubpaths'] = options.rsubpaths
kwargs['arbitopts'] = options.arbitopts
kwargs['checksum'] = options.checksum
kwargs['done_file'] = options.done_file
kwargs['dryrun'] = options.dryrun
kwargs['delete'] = options.delete
kwargs['excludere'] = options.excludere
kwargs['excl_usr'] = options.excl_usr
kwargs['hardlinks'] = options.hardlinks
kwargs['timeout'] = options.timeout
kwargs['verbose'] = options.verbose
# Start zookeeper connections
rsyncS = RsyncSource(options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
rsyncS.build_pathqueue()
rsyncS.wait_and_keep_progress()
rsyncS.shutdown_all()
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
def main():
""" Start a new rsync client (destination or source) in a specified session """
options = {
# Zookeeper connection options:
'servers' : ('list of zk servers', 'strlist', 'store', None),
'user' : ('user with creation rights on zookeeper', None, 'store', 'root', 'u'),
'passwd' : ('password for user with creation rights', None, 'store', 'admin', 'p'),
# Role options, define exactly one of these:
'source' : ('rsync source', None, 'store_true', False, 'S'),
'destination' : ('rsync destination', None, 'store_true', False, 'D'),
'pathsonly' : ('Only do a test run of the pathlist building', None, 'store_true', False),
'state' : ('Only do the state', None, 'store_true', False),
# Session options; should be the same on all clients of the session!
'session' : ('session name', None, 'store', 'default', 'N'),
'netcat' : ('run netcat test instead of rsync', None, 'store_true', False),
'dryrun' : ('run rsync in dry run mode', None, 'store_true', False, 'n'),
'rsyncpath' : ('rsync basepath', None, 'store', None, 'r'), # May differ between sources and dests
# Pathbuilding (Source clients and pathsonly ) specific options:
'excludere' : ('Exclude from pathbuilding', None, 'regex', re.compile('/\.snapshots(/.*|$)')),
'depth' : ('queue depth', "int", 'store', 4),
# Source clients options; should be the same on all clients of the session!:
'delete' : ('run rsync with --delete', None, 'store_true', False),
# Individual client options
'domain' : ('substitute domain', None, 'store', None),
'logfile' : ('Output to logfile', None, 'store', '/tmp/zkrsync/%(session)s-%(rstype)s-%(pid)s.log'),
# Individual Destination client specific options
'rsyncport' : ('force port on which rsyncd binds', "int", 'store', None),
'startport' : ('offset to look for rsyncd ports', "int", 'store', 4444)
}
go = simple_option(options)
acreds, admin_acl, rstype = zkrsync_parse(go.options)
if go.options.logfile:
logfile = go.options.logfile % {
'session': go.options.session,
'rstype': rstype,
'pid': str(os.getpid())
}
logdir = os.path.dirname(logfile)
if logdir:
if not os.path.exists(logdir):
os.makedirs(logdir)
os.chmod(logdir, stat.S_IRWXU)
fancylogger.logToFile(logfile)
logger.debug('Logging to file %s:' % logfile)
kwargs = {
'session' : go.options.session,
'default_acl' : [admin_acl],
'auth_data' : acreds,
'rsyncpath' : go.options.rsyncpath,
'netcat' : go.options.netcat,
}
if go.options.state:
rsyncP = RsyncSource(go.options.servers, **kwargs)
logger.info('Progress: %s of %s paths remaining' % (rsyncP.len_paths(), rsyncP.paths_total))
rsyncP.exit()
sys.exit(0)
elif go.options.pathsonly:
kwargs['rsyncdepth'] = go.options.depth
kwargs['excludere'] = go.options.excludere
rsyncP = RsyncSource(go.options.servers, **kwargs)
locked = rsyncP.acq_lock()
if locked:
starttime = time.time()
rsyncP.build_pathqueue()
endtime = time.time()
timing = endtime - starttime
pathqueue = rsyncP.path_queue
logger.info('Building with depth %i took %f seconds walltime. there are %i paths in the Queue'
% (go.options.depth, timing, len(pathqueue)))
rsyncP.delete(pathqueue.path, recursive=True)
rsyncP.release_lock()
else:
logger.error('There is already a lock on the pathtree of this session')
rsyncP.exit()
sys.exit(0)
elif rstype == CL_DEST:
# Start zookeeper connection and rsync daemon
kwargs['rsyncport'] = go.options.rsyncport
kwargs['startport'] = go.options.startport
kwargs['domain'] = go.options.domain
rsyncD = RsyncDestination(go.options.servers, **kwargs)
rsyncD.run()
logger.debug('%s Ready' % rsyncD.get_whoami())
rsyncD.exit()
sys.exit(0)
elif rstype == CL_SOURCE:
# Start zookeeper connections
kwargs['rsyncdepth'] = go.options.depth
kwargs['dryrun'] = go.options.dryrun
kwargs['delete'] = go.options.delete
kwargs['excludere'] = go.options.excludere
rsyncS = RsyncSource(go.options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
paths_total = rsyncS.build_pathqueue()
todo_paths = paths_total
while not rsyncS.isempty_pathqueue():
if todo_paths != rsyncS.len_paths(): # Output progress state
todo_paths = rsyncS.len_paths()
logger.info('Progress: %s of %s paths remaining' % (todo_paths, paths_total))
time.sleep(SLEEP_TIME)
rsyncS.shutdown_all()
rsyncS.exit()
sys.exit(0)
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
def test_attempt_run(self):
""" Test the attempt run command"""
zkclient = RsyncSource('dummy', netcat=True, rsyncdepth=2)
dummyq = LockingQueue('foo', 'bar')
self.assertTupleEqual(zkclient.attempt_run('echo test', dummyq), (0, 'test\n'))