def start_fd_client(self, i, o, **opts):
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {'proto': repce.repce_version, 'object': Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise RuntimeError(
"RePCe major version mismatch: local %s, remote %s" %
(exrv, rv))
class SlaveRemote(object):
"""mix-in class to implement an interface to a remote slave"""
def connect_remote(self, rargs=[], **opts):
"""connects to a remote slave
Invoke an auxiliary utility (slave gsyncd, possibly wrapped)
which sets up the connection and set up a RePCe client to
communicate throuh its stdio.
"""
slave = opts.get('slave', self.url)
extra_opts = []
so = getattr(gconf, 'session_owner', None)
if so:
extra_opts += ['--session-owner', so]
if boolify(gconf.use_rsync_xattrs):
extra_opts.append('--use-rsync-xattrs')
po = Popen(rargs + gconf.remote_gsyncd.split() + extra_opts + \
['-N', '--listen', '--timeout', str(gconf.timeout), slave],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
gconf.transport = po
return self.start_fd_client(po.stdout, po.stdin, **opts)
def start_fd_client(self, i, o, **opts):
"""set up RePCe client, handshake with server
It's cut out as a separate method to let
subclasses hook into client startup
"""
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {'proto': repce.repce_version, 'object': Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise GsyncdError(
"RePCe major version mismatch: local %s, remote %s" %
(exrv, rv))
def rsync(self, files, *args):
"""invoke rsync"""
if not files:
raise GsyncdError("no files to sync")
logging.debug("files: " + ", ".join(files))
argv = gconf.rsync_command.split() + \
['-avR0', '--inplace', '--files-from=-', '--super','--stats', '--numeric-ids', '--no-implied-dirs'] + \
gconf.rsync_options.split() + (boolify(gconf.use_rsync_xattrs) and ['--xattrs'] or []) + \
['.'] + list(args)
po = Popen(argv, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
for f in files:
po.stdin.write(f)
po.stdin.write('\0')
po.stdin.close()
po.wait()
po.terminate_geterr(fail_on_err=False)
return po
class SlaveRemote(object):
"""mix-in class to implement an interface to a remote slave"""
def connect_remote(self, rargs=[], **opts):
"""connects to a remote slave
Invoke an auxiliary utility (slave gsyncd, possibly wrapped)
which sets up the connection and set up a RePCe client to
communicate throuh its stdio.
"""
slave = opts.get('slave', self.url)
extra_opts = []
so = getattr(gconf, 'session_owner', None)
if so:
extra_opts += ['--session-owner', so]
if boolify(gconf.use_rsync_xattrs):
extra_opts.append('--use-rsync-xattrs')
po = Popen(rargs + gconf.remote_gsyncd.split() + extra_opts + \
['-N', '--listen', '--timeout', str(gconf.timeout), slave],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
gconf.transport = po
return self.start_fd_client(po.stdout, po.stdin, **opts)
def start_fd_client(self, i, o, **opts):
"""set up RePCe client, handshake with server
It's cut out as a separate method to let
subclasses hook into client startup
"""
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {'proto': repce.repce_version, 'object': Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise GsyncdError("RePCe major version mismatch: local %s, remote %s" % (exrv, rv))
def rsync(self, files, *args):
"""invoke rsync"""
if not files:
raise GsyncdError("no files to sync")
logging.debug("files: " + ", ".join(files))
argv = gconf.rsync_command.split() + \
['-aR0', '--files-from=-', '--super','--stats', '--numeric-ids', '--no-implied-dirs'] + \
gconf.rsync_options.split() + (boolify(gconf.use_rsync_xattrs) and ['--xattrs'] or []) + \
['.'] + list(args)
po = Popen(argv, stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
for f in files:
po.stdin.write(f)
po.stdin.write('\0')
po.stdin.close()
po.wait()
po.terminate_geterr(fail_on_err = False)
return po
class SlaveRemote(object):
"""mix-in class to implement an interface to a remote slave"""
def connect_remote(self, rargs=[], **opts):
"""connects to a remote slave
Invoke an auxiliary utility (slave gsyncd, possibly wrapped)
which sets up the connection and set up a RePCe client to
communicate throuh its stdio.
"""
slave = opts.get("slave", self.url)
so = getattr(gconf, "session_owner", None)
if so:
so_args = ["--session-owner", so]
else:
so_args = []
po = Popen(
rargs + gconf.remote_gsyncd.split() + so_args + ["-N", "--listen", "--timeout", str(gconf.timeout), slave],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
gconf.transport = po
return self.start_fd_client(po.stdout, po.stdin, **opts)
def start_fd_client(self, i, o, **opts):
"""set up RePCe client, handshake with server
It's cut out as a separate method to let
subclasses hook into client startup
"""
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {"proto": repce.repce_version, "object": Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise GsyncdError("RePCe major version mismatch: local %s, remote %s" % (exrv, rv))
def rsync(self, files, *args):
"""invoke rsync"""
if not files:
raise GsyncdError("no files to sync")
logging.debug("files: " + ", ".join(files))
argv = (
gconf.rsync_command.split()
+ ["-aR", "--super", "--numeric-ids", "--no-implied-dirs"]
+ gconf.rsync_options.split()
+ files
+ list(args)
)
po = Popen(argv, stderr=subprocess.PIPE)
po.wait()
po.terminate_geterr(fail_on_err=False)
return po
def start_fd_client(self, i, o, **opts):
"""set up RePCe client, handshake with server
It's cut out as a separate method to let
subclasses hook into client startup
"""
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {'proto': repce.repce_version, 'object': Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise GsyncdError(
"RePCe major version mismatch: local %s, remote %s" %
(exrv, rv))
def start_fd_client(self, i, o, **opts):
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {'proto': repce.repce_version, 'object': Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise RuntimeError("RePCe major version mismatch: local %s, remote %s" % (exrv, rv))
def start_fd_client(self, i, o, **opts):
"""set up RePCe client, handshake with server
It's cut out as a separate method to let
subclasses hook into client startup
"""
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {"proto": repce.repce_version, "object": Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise GsyncdError("RePCe major version mismatch: local %s, remote %s" % (exrv, rv))
class SlaveRemote(object):
def connect_remote(self, rargs=[], **opts):
slave = opts.get('slave', self.url)
ix, ox = os.pipe()
iy, oy = os.pipe()
pid = os.fork()
if not pid:
os.close(ox)
os.dup2(ix, sys.stdin.fileno())
os.close(iy)
os.dup2(oy, sys.stdout.fileno())
so = getattr(gconf, 'session_owner', None)
if so:
so_args = ['--session-owner', so]
else:
so_args = []
argv = rargs + gconf.remote_gsyncd.split() + so_args + \
['-N', '--listen', '--timeout', str(gconf.timeout), slave]
os.execvp(argv[0], argv)
os.close(ix)
os.close(oy)
return self.start_fd_client(iy, ox, **opts)
def start_fd_client(self, i, o, **opts):
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {'proto': repce.repce_version, 'object': Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise RuntimeError(
"RePCe major version mismatch: local %s, remote %s" %
(exrv, rv))
def rsync(self, files, *args):
if not files:
raise RuntimeError("no files to sync")
logging.debug("files: " + ", ".join(files))
argv = gconf.rsync_command.split() + gconf.rsync_extra.split() + [
'-aR'
] + files + list(args)
return os.spawnvp(os.P_WAIT, argv[0], argv) == 0
class SlaveRemote(object):
def connect_remote(self, rargs=[], **opts):
slave = opts.get('slave', self.url)
ix, ox = os.pipe()
iy, oy = os.pipe()
pid = os.fork()
if not pid:
os.close(ox)
os.dup2(ix, sys.stdin.fileno())
os.close(iy)
os.dup2(oy, sys.stdout.fileno())
so = getattr(gconf, 'session_owner', None)
if so:
so_args = ['--session-owner', so]
else:
so_args = []
argv = rargs + gconf.remote_gsyncd.split() + so_args + \
['-N', '--listen', '--timeout', str(gconf.timeout), slave]
os.execvp(argv[0], argv)
os.close(ix)
os.close(oy)
return self.start_fd_client(iy, ox, **opts)
def start_fd_client(self, i, o, **opts):
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {'proto': repce.repce_version, 'object': Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise GsyncdError("RePCe major version mismatch: local %s, remote %s" % (exrv, rv))
def rsync(self, files, *args):
if not files:
raise GsyncdError("no files to sync")
logging.debug("files: " + ", ".join(files))
argv = gconf.rsync_command.split() + gconf.rsync_extra.split() + ['-aR'] + files + list(args)
return os.spawnvp(os.P_WAIT, argv[0], argv) == 0
class SlaveRemote(object):
"""mix-in class to implement an interface to a remote slave"""
def connect_remote(self, rargs=[], **opts):
"""connects to a remote slave
Invoke an auxiliary utility (slave gsyncd, possibly wrapped)
which sets up the connection and set up a RePCe client to
communicate throuh its stdio.
"""
slave = opts.get('slave', self.url)
extra_opts = []
so = getattr(gconf, 'session_owner', None)
if so:
extra_opts += ['--session-owner', so]
if boolify(gconf.use_rsync_xattrs):
extra_opts.append('--use-rsync-xattrs')
po = Popen(rargs + gconf.remote_gsyncd.split() + extra_opts +
['-N', '--listen', '--timeout', str(gconf.timeout),
slave],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
gconf.transport = po
return self.start_fd_client(po.stdout, po.stdin, **opts)
def start_fd_client(self, i, o, **opts):
"""set up RePCe client, handshake with server
It's cut out as a separate method to let
subclasses hook into client startup
"""
self.server = RepceClient(i, o)
rv = self.server.__version__()
exrv = {'proto': repce.repce_version, 'object': Server.version()}
da0 = (rv, exrv)
da1 = ({}, {})
for i in range(2):
for k, v in da0[i].iteritems():
da1[i][k] = int(v)
if da1[0] != da1[1]:
raise GsyncdError(
"RePCe major version mismatch: local %s, remote %s" %
(exrv, rv))
def rsync(self, files, *args):
"""invoke rsync"""
if not files:
raise GsyncdError("no files to sync")
logging.debug("files: " + ", ".join(files))
argv = gconf.rsync_command.split() + \
['-avR0', '--inplace', '--files-from=-', '--super',
'--stats', '--numeric-ids', '--no-implied-dirs'] + \
gconf.rsync_options.split() + \
(boolify(gconf.use_rsync_xattrs) and ['--xattrs'] or []) + \
['.'] + list(args)
po = Popen(argv, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
for f in files:
po.stdin.write(f)
po.stdin.write('\0')
po.stdin.close()
po.wait()
po.terminate_geterr(fail_on_err=False)
return po
def tarssh(self, files, slaveurl):
"""invoke tar+ssh
-z (compress) can be use if needed, but ommitting it now
as it results in wierd error (tar+ssh errors out (errcode: 2)
"""
if not files:
raise GsyncdError("no files to sync")
logging.debug("files: " + ", ".join(files))
(host, rdir) = slaveurl.split(':')
tar_cmd = ["tar", "-cf", "-", "--files-from", "-"]
ssh_cmd = gconf.ssh_command_tar.split() + \
[host, "tar", "--overwrite", "-xf", "-", "-C", rdir]
p0 = Popen(tar_cmd, stdout=subprocess.PIPE,
stdin=subprocess.PIPE, stderr=subprocess.PIPE)
p1 = Popen(ssh_cmd, stdin=p0.stdout, stderr=subprocess.PIPE)
for f in files:
p0.stdin.write(f)
p0.stdin.write('\n')
p0.stdin.close()
# wait() for tar to terminate, collecting any errors, further
# waiting for transfer to complete
p0.wait()
p0.terminate_geterr(fail_on_err=False)
p1.wait()
p1.terminate_geterr(fail_on_err=False)
return p1
def service_loop(self, *args):
"""enter service loop
- if slave given, instantiate GMaster and
pass control to that instance, which implements
master behavior
- else do that's what's inherited
"""
if args:
slave = args[0]
if gconf.local_path:
class brickserver(FILE.FILEServer):
local_path = gconf.local_path
aggregated = self.server
@classmethod
def entries(cls, path):
e = super(brickserver, cls).entries(path)
# on the brick don't mess with /.glusterfs
if path == '.':
try:
e.remove('.glusterfs')
except ValueError:
pass
return e
@classmethod
def lstat(cls, e):
""" path based backend stat """
return super(brickserver, cls).lstat(e)
@classmethod
def gfid(cls, e):
""" path based backend gfid fetch """
return super(brickserver, cls).gfid(e)
@classmethod
def linkto_check(cls, e):
return super(brickserver, cls).linkto_check(e)
if gconf.slave_id:
# define {,set_}xtime in slave, thus preempting
# the call to remote, so that it takes data from
# the local brick
slave.server.xtime = types.MethodType(
lambda _self, path, uuid: (
brickserver.xtime(path,
uuid + '.' + gconf.slave_id)
),
slave.server)
slave.server.stime = types.MethodType(
lambda _self, path, uuid: (
brickserver.stime(path,
uuid + '.' + gconf.slave_id)
),
slave.server)
slave.server.set_stime = types.MethodType(
lambda _self, path, uuid, mark: (
brickserver.set_stime(path,
uuid + '.' + gconf.slave_id,
mark)
),
slave.server)
(g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
g1.master.server = brickserver
g2.master.server = brickserver
g3.master.server = brickserver
else:
(g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
g1.master.server.aggregated = gmaster.master.server
g2.master.server.aggregated = gmaster.master.server
g3.master.server.aggregated = gmaster.master.server
# bad bad bad: bad way to do things like this
# need to make this elegant
# register the crawlers and start crawling
# g1 ==> Xsync, g2 ==> config.change_detector(changelog by default)
# g3 ==> changelog History
(inf, ouf, ra, wa) = gconf.rpc_fd.split(',')
os.close(int(ra))
os.close(int(wa))
changelog_agent = RepceClient(int(inf), int(ouf))
rv = changelog_agent.version()
if int(rv) != CHANGELOG_AGENT_CLIENT_VERSION:
raise GsyncdError(
"RePCe major version mismatch(changelog agent): "
"local %s, remote %s" %
(CHANGELOG_AGENT_CLIENT_VERSION, rv))
g1.register()
try:
(workdir, logfile) = g2.setup_working_dir()
# register with the changelog library
# 9 == log level (DEBUG)
# 5 == connection retries
changelog_agent.register(gconf.local_path,
workdir, logfile, 9, 5)
g2.register(changelog_agent)
g3.register(changelog_agent)
except ChangelogException as e:
logging.debug("Changelog register failed: %s - %s" %
(e.errno, e.strerror))
# Check if gsyncd restarted in pause state. If
# yes, send SIGSTOP to negative of monitor pid
# to go back to pause state.
if gconf.pause_on_start:
os.kill(-os.getppid(), signal.SIGSTOP)
gconf.pause_on_start = False
# oneshot: Try to use changelog history api, if not
# available switch to FS crawl
# Note: if config.change_detector is xsync then
# it will not use changelog history api
try:
g3.crawlwrap(oneshot=True)
except (ChangelogException, NoPurgeTimeAvailable,
PartialHistoryAvailable) as e:
if isinstance(e, ChangelogException):
logging.debug('Changelog history crawl failed, failback '
'to xsync: %s - %s' % (e.errno, e.strerror))
elif isinstance(e, NoPurgeTimeAvailable):
logging.debug('Using xsync crawl since no purge time '
'available')
elif isinstance(e, PartialHistoryAvailable):
logging.debug('Using xsync crawl after consuming history '
'till %s' % str(e))
g1.crawlwrap(oneshot=True)
# crawl loop: Try changelog crawl, if failed
# switch to FS crawl
try:
g2.crawlwrap()
except ChangelogException as e:
logging.debug('Changelog crawl failed, failback to xsync: '
'%s - %s' % (e.errno, e.strerror))
g1.crawlwrap()
else:
sup(self, *args)
def service_loop(self, *args):
"""enter service loop
- if slave given, instantiate GMaster and
pass control to that instance, which implements
master behavior
- else do that's what's inherited
"""
if args:
slave = args[0]
if gconf.local_path:
class brickserver(FILE.FILEServer):
local_path = gconf.local_path
aggregated = self.server
@classmethod
def entries(cls, path):
e = super(brickserver, cls).entries(path)
# on the brick don't mess with /.glusterfs
if path == ".":
try:
e.remove(".glusterfs")
except ValueError:
pass
return e
@classmethod
def lstat(cls, e):
""" path based backend stat """
return super(brickserver, cls).lstat(e)
@classmethod
def gfid(cls, e):
""" path based backend gfid fetch """
return super(brickserver, cls).gfid(e)
@classmethod
def linkto_check(cls, e):
return super(brickserver, cls).linkto_check(e)
if gconf.slave_id:
# define {,set_}xtime in slave, thus preempting
# the call to remote, so that it takes data from
# the local brick
slave.server.xtime = types.MethodType(
lambda _self, path, uuid: (brickserver.xtime(path, uuid + "." + gconf.slave_id)), slave.server
)
slave.server.stime = types.MethodType(
lambda _self, path, uuid: (brickserver.stime(path, uuid + "." + gconf.slave_id)), slave.server
)
slave.server.set_stime = types.MethodType(
lambda _self, path, uuid, mark: (
brickserver.set_stime(path, uuid + "." + gconf.slave_id, mark)
),
slave.server,
)
(g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
g1.master.server = brickserver
g2.master.server = brickserver
g3.master.server = brickserver
else:
(g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
g1.master.server.aggregated = gmaster.master.server
g2.master.server.aggregated = gmaster.master.server
g3.master.server.aggregated = gmaster.master.server
# bad bad bad: bad way to do things like this
# need to make this elegant
# register the crawlers and start crawling
# g1 ==> Xsync, g2 ==> config.change_detector(changelog by default)
# g3 ==> changelog History
changelog_register_failed = False
(inf, ouf, ra, wa) = gconf.rpc_fd.split(",")
os.close(int(ra))
os.close(int(wa))
changelog_agent = RepceClient(int(inf), int(ouf))
rv = changelog_agent.version()
if int(rv) != CHANGELOG_AGENT_CLIENT_VERSION:
raise GsyncdError(
"RePCe major version mismatch(changelog agent): "
"local %s, remote %s" % (CHANGELOG_AGENT_CLIENT_VERSION, rv)
)
try:
workdir = g2.setup_working_dir()
# Register only when change_detector is not set to
# xsync, else agent will generate changelog files
# in .processing directory of working dir
if gconf.change_detector != "xsync":
# register with the changelog library
# 9 == log level (DEBUG)
# 5 == connection retries
changelog_agent.register(
gconf.local_path,
workdir,
gconf.changelog_log_file,
g2.CHANGELOG_LOG_LEVEL,
g2.CHANGELOG_CONN_RETRIES,
)
register_time = int(time.time())
g2.register(register_time, changelog_agent)
g3.register(register_time, changelog_agent)
except ChangelogException:
changelog_register_failed = True
register_time = None
logging.info("Changelog register failed, fallback to xsync")
g1.register()
logging.info("Register time: %s" % register_time)
# oneshot: Try to use changelog history api, if not
# available switch to FS crawl
# Note: if config.change_detector is xsync then
# it will not use changelog history api
try:
if not changelog_register_failed:
g3.crawlwrap(oneshot=True)
else:
g1.crawlwrap(oneshot=True)
except (ChangelogException, NoPurgeTimeAvailable, PartialHistoryAvailable) as e:
if isinstance(e, ChangelogException):
logging.info(
"Changelog history crawl failed, fallback " "to xsync: %s - %s" % (e.errno, e.strerror)
)
elif isinstance(e, PartialHistoryAvailable):
logging.info(
"Partial history available, using xsync crawl" " after consuming history " "till %s" % str(e)
)
g1.crawlwrap(oneshot=True, no_stime_update=True, register_time=register_time)
# crawl loop: Try changelog crawl, if failed
# switch to FS crawl
try:
if not changelog_register_failed:
g2.crawlwrap()
else:
g1.crawlwrap()
except ChangelogException as e:
logging.info("Changelog crawl failed, fallback to xsync")
g1.crawlwrap()
else:
sup(self, *args)
def service_loop(self, *args):
"""enter service loop
- if slave given, instantiate GMaster and
pass control to that instance, which implements
master behavior
- else do that's what's inherited
"""
if args:
slave = args[0]
if gconf.local_path:
class brickserver(FILE.FILEServer):
local_path = gconf.local_path
aggregated = self.server
@classmethod
def entries(cls, path):
e = super(brickserver, cls).entries(path)
# on the brick don't mess with /.glusterfs
if path == '.':
try:
e.remove('.glusterfs')
except ValueError:
pass
return e
@classmethod
def lstat(cls, e):
""" path based backend stat """
return super(brickserver, cls).lstat(e)
@classmethod
def gfid(cls, e):
""" path based backend gfid fetch """
return super(brickserver, cls).gfid(e)
@classmethod
def linkto_check(cls, e):
return super(brickserver, cls).linkto_check(e)
if gconf.slave_id:
# define {,set_}xtime in slave, thus preempting
# the call to remote, so that it takes data from
# the local brick
slave.server.xtime = types.MethodType(
lambda _self, path, uuid: (
brickserver.xtime(path,
uuid + '.' + gconf.slave_id)
),
slave.server)
slave.server.stime = types.MethodType(
lambda _self, path, uuid: (
brickserver.stime(path,
uuid + '.' + gconf.slave_id)
),
slave.server)
slave.server.set_stime = types.MethodType(
lambda _self, path, uuid, mark: (
brickserver.set_stime(path,
uuid + '.' + gconf.slave_id,
mark)
),
slave.server)
(g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
g1.master.server = brickserver
g2.master.server = brickserver
g3.master.server = brickserver
else:
(g1, g2, g3) = self.gmaster_instantiate_tuple(slave)
g1.master.server.aggregated = gmaster.master.server
g2.master.server.aggregated = gmaster.master.server
g3.master.server.aggregated = gmaster.master.server
# bad bad bad: bad way to do things like this
# need to make this elegant
# register the crawlers and start crawling
# g1 ==> Xsync, g2 ==> config.change_detector(changelog by default)
# g3 ==> changelog History
(inf, ouf, ra, wa) = gconf.rpc_fd.split(',')
os.close(int(ra))
os.close(int(wa))
changelog_agent = RepceClient(int(inf), int(ouf))
rv = changelog_agent.version()
if int(rv) != CHANGELOG_AGENT_CLIENT_VERSION:
raise GsyncdError(
"RePCe major version mismatch(changelog agent): "
"local %s, remote %s" %
(CHANGELOG_AGENT_CLIENT_VERSION, rv))
g1.register()
try:
(workdir, logfile) = g2.setup_working_dir()
# register with the changelog library
# 9 == log level (DEBUG)
# 5 == connection retries
changelog_agent.register(gconf.local_path,
workdir, logfile, 9, 5)
g2.register(changelog_agent)
g3.register(changelog_agent)
except ChangelogException as e:
logging.debug("Changelog register failed: %s - %s" %
(e.errno, e.strerror))
# oneshot: Try to use changelog history api, if not
# available switch to FS crawl
# Note: if config.change_detector is xsync then
# it will not use changelog history api
try:
g3.crawlwrap(oneshot=True)
except (ChangelogException, NoPurgeTimeAvailable,
PartialHistoryAvailable) as e:
if isinstance(e, ChangelogException):
logging.debug('Changelog history crawl failed, failback '
'to xsync: %s - %s' % (e.errno, e.strerror))
elif isinstance(e, NoPurgeTimeAvailable):
logging.debug('Using xsync crawl since no purge time '
'available')
elif isinstance(e, PartialHistoryAvailable):
logging.debug('Using xsync crawl after consuming history '
'till %s' % str(e))
g1.crawlwrap(oneshot=True)
# crawl loop: Try changelog crawl, if failed
# switch to FS crawl
try:
g2.crawlwrap()
except ChangelogException as e:
logging.debug('Changelog crawl failed, failback to xsync: '
'%s - %s' % (e.errno, e.strerror))
g1.crawlwrap()
else:
sup(self, *args)
