def do_pathsonly(options, kwargs):
"""Only build the pathqueue and return timings"""
kwargs['rsyncdepth'] = options.depth
kwargs['excludere'] = options.excludere
kwargs['excl_usr'] = options.excl_usr
kwargs['rsubpaths'] = options.rsubpaths
rsyncP = RsyncSource(options.servers, **kwargs)
locked = rsyncP.acq_lock()
if locked:
starttime = time.time()
rsyncP.build_pathqueue()
endtime = time.time()
timing = endtime - starttime
pathqueue = rsyncP.path_queue
logger.info(
'Building with depth %i took %f seconds walltime. there are %i paths in the Queue'
% (options.depth, timing, len(pathqueue)))
rsyncP.delete(pathqueue.path, recursive=True)
rsyncP.release_lock()
else:
logger.error('There is already a lock on the pathtree of this session')
rsyncP.exit()
sys.exit(0)
def start_source(options, kwargs):
""" Start a rsync source"""
kwargs['rsyncdepth'] = options.depth
kwargs['dryrun'] = options.dryrun
kwargs['delete'] = options.delete
kwargs['excludere'] = options.excludere
# Start zookeeper connections
rsyncS = RsyncSource(options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
paths_total = rsyncS.build_pathqueue()
rsyncS.wait_and_keep_progress(paths_total)
rsyncS.shutdown_all()
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
def do_pathsonly(options, kwargs):
"""Only build the pathqueue and return timings"""
kwargs['rsyncdepth'] = options.depth
kwargs['excludere'] = options.excludere
rsyncP = RsyncSource(options.servers, **kwargs)
locked = rsyncP.acq_lock()
if locked:
starttime = time.time()
rsyncP.build_pathqueue()
endtime = time.time()
timing = endtime - starttime
pathqueue = rsyncP.path_queue
logger.info('Building with depth %i took %f seconds walltime. there are %i paths in the Queue'
% (options.depth, timing, len(pathqueue)))
rsyncP.delete(pathqueue.path, recursive=True)
rsyncP.release_lock()
else:
logger.error('There is already a lock on the pathtree of this session')
rsyncP.exit()
sys.exit(0)
def start_source(options, kwargs):
""" Start a rsync source"""
kwargs['rsyncdepth'] = options.depth
kwargs['rsubpaths'] = options.rsubpaths
kwargs['arbitopts'] = options.arbitopts
kwargs['checksum'] = options.checksum
kwargs['done_file'] = options.done_file
kwargs['dryrun'] = options.dryrun
kwargs['delete'] = options.delete
kwargs['excludere'] = options.excludere
kwargs['excl_usr'] = options.excl_usr
kwargs['hardlinks'] = options.hardlinks
kwargs['inplace'] = options.inplace
kwargs['timeout'] = options.timeout
kwargs['verbose'] = options.verbose
# Start zookeeper connections
rsyncS = RsyncSource(options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
rsyncS.build_pathqueue()
rsyncS.wait_and_keep_progress()
rsyncS.shutdown_all()
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
def start_source(options, kwargs):
""" Start a rsync source"""
kwargs['rsyncdepth'] = options.depth
kwargs['rsubpaths'] = options.rsubpaths
kwargs['arbitopts'] = options.arbitopts
kwargs['checksum'] = options.checksum
kwargs['done_file'] = options.done_file
kwargs['dryrun'] = options.dryrun
kwargs['delete'] = options.delete
kwargs['excludere'] = options.excludere
kwargs['excl_usr'] = options.excl_usr
kwargs['hardlinks'] = options.hardlinks
kwargs['timeout'] = options.timeout
kwargs['verbose'] = options.verbose
# Start zookeeper connections
rsyncS = RsyncSource(options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
rsyncS.build_pathqueue()
rsyncS.wait_and_keep_progress()
rsyncS.shutdown_all()
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)
def main():
""" Start a new rsync client (destination or source) in a specified session """
options = {
# Zookeeper connection options:
'servers' : ('list of zk servers', 'strlist', 'store', None),
'user' : ('user with creation rights on zookeeper', None, 'store', 'root', 'u'),
'passwd' : ('password for user with creation rights', None, 'store', 'admin', 'p'),
# Role options, define exactly one of these:
'source' : ('rsync source', None, 'store_true', False, 'S'),
'destination' : ('rsync destination', None, 'store_true', False, 'D'),
'pathsonly' : ('Only do a test run of the pathlist building', None, 'store_true', False),
'state' : ('Only do the state', None, 'store_true', False),
# Session options; should be the same on all clients of the session!
'session' : ('session name', None, 'store', 'default', 'N'),
'netcat' : ('run netcat test instead of rsync', None, 'store_true', False),
'dryrun' : ('run rsync in dry run mode', None, 'store_true', False, 'n'),
'rsyncpath' : ('rsync basepath', None, 'store', None, 'r'), # May differ between sources and dests
# Pathbuilding (Source clients and pathsonly ) specific options:
'excludere' : ('Exclude from pathbuilding', None, 'regex', re.compile('/\.snapshots(/.*|$)')),
'depth' : ('queue depth', "int", 'store', 4),
# Source clients options; should be the same on all clients of the session!:
'delete' : ('run rsync with --delete', None, 'store_true', False),
# Individual client options
'domain' : ('substitute domain', None, 'store', None),
'logfile' : ('Output to logfile', None, 'store', '/tmp/zkrsync/%(session)s-%(rstype)s-%(pid)s.log'),
# Individual Destination client specific options
'rsyncport' : ('force port on which rsyncd binds', "int", 'store', None),
'startport' : ('offset to look for rsyncd ports', "int", 'store', 4444)
}
go = simple_option(options)
acreds, admin_acl, rstype = zkrsync_parse(go.options)
if go.options.logfile:
logfile = go.options.logfile % {
'session': go.options.session,
'rstype': rstype,
'pid': str(os.getpid())
}
logdir = os.path.dirname(logfile)
if logdir:
if not os.path.exists(logdir):
os.makedirs(logdir)
os.chmod(logdir, stat.S_IRWXU)
fancylogger.logToFile(logfile)
logger.debug('Logging to file %s:' % logfile)
kwargs = {
'session' : go.options.session,
'default_acl' : [admin_acl],
'auth_data' : acreds,
'rsyncpath' : go.options.rsyncpath,
'netcat' : go.options.netcat,
}
if go.options.state:
rsyncP = RsyncSource(go.options.servers, **kwargs)
logger.info('Progress: %s of %s paths remaining' % (rsyncP.len_paths(), rsyncP.paths_total))
rsyncP.exit()
sys.exit(0)
elif go.options.pathsonly:
kwargs['rsyncdepth'] = go.options.depth
kwargs['excludere'] = go.options.excludere
rsyncP = RsyncSource(go.options.servers, **kwargs)
locked = rsyncP.acq_lock()
if locked:
starttime = time.time()
rsyncP.build_pathqueue()
endtime = time.time()
timing = endtime - starttime
pathqueue = rsyncP.path_queue
logger.info('Building with depth %i took %f seconds walltime. there are %i paths in the Queue'
% (go.options.depth, timing, len(pathqueue)))
rsyncP.delete(pathqueue.path, recursive=True)
rsyncP.release_lock()
else:
logger.error('There is already a lock on the pathtree of this session')
rsyncP.exit()
sys.exit(0)
elif rstype == CL_DEST:
# Start zookeeper connection and rsync daemon
kwargs['rsyncport'] = go.options.rsyncport
kwargs['startport'] = go.options.startport
kwargs['domain'] = go.options.domain
rsyncD = RsyncDestination(go.options.servers, **kwargs)
rsyncD.run()
logger.debug('%s Ready' % rsyncD.get_whoami())
rsyncD.exit()
sys.exit(0)
elif rstype == CL_SOURCE:
# Start zookeeper connections
kwargs['rsyncdepth'] = go.options.depth
kwargs['dryrun'] = go.options.dryrun
kwargs['delete'] = go.options.delete
kwargs['excludere'] = go.options.excludere
rsyncS = RsyncSource(go.options.servers, **kwargs)
# Try to retrieve session lock
locked = rsyncS.acq_lock()
if locked:
logger.debug('lock acquired')
watchnode = rsyncS.start_ready_rwatch()
if not watchnode:
sys.exit(1)
paths_total = rsyncS.build_pathqueue()
todo_paths = paths_total
while not rsyncS.isempty_pathqueue():
if todo_paths != rsyncS.len_paths(): # Output progress state
todo_paths = rsyncS.len_paths()
logger.info('Progress: %s of %s paths remaining' % (todo_paths, paths_total))
time.sleep(SLEEP_TIME)
rsyncS.shutdown_all()
rsyncS.exit()
sys.exit(0)
else:
rsyncS.ready_with_stop_watch()
logger.debug('ready to process paths')
while not rsyncS.is_ready():
logger.debug('trying to get a path out of Queue')
rsyncS.rsync(TIME_OUT)
logger.debug('%s Ready' % rsyncS.get_whoami())
rsyncS.exit()
sys.exit(0)