def makersc(aa, check=True):
if not aa:
return ([], None, None)
ra = [resource.parse_url(u) for u in aa]
local = ra[0]
remote = None
if len(ra) > 1:
remote = ra[1]
if check and not local.can_connect_to(remote):
raise GsyncdError("%s cannot work with %s" % (local.path, remote and remote.path))
return (ra, local, remote)
def makersc(aa, check=True):
if not aa:
return ([], None, None)
ra = [resource.parse_url(u) for u in aa]
local = ra[0]
remote = None
if len(ra) > 1:
remote = ra[1]
if check and not local.can_connect_to(remote):
raise GsyncdError("%s cannot work with %s" % (local.path, remote and remote.path))
return (ra, local, remote)
def main_i():
"""internal main routine
parse command line, decide what action will be taken;
we can either:
- query/manipulate configuration
- format gsyncd urls using gsyncd's url parsing engine
- start service in following modes, in given stages:
- monitor: startup(), monitor()
- master: startup(), connect_remote(), connect(), service_loop()
- slave: startup(), connect(), service_loop()
"""
rconf = {'go_daemon': 'should'}
def store_abs(opt, optstr, val, parser):
if val and val != '-':
val = os.path.abspath(val)
setattr(parser.values, opt.dest, val)
def store_local(opt, optstr, val, parser):
rconf[opt.dest] = val
def store_local_curry(val):
return lambda o, oo, vx, p: store_local(o, oo, val, p)
def store_local_obj(op, dmake):
return lambda o, oo, vx, p: store_local(o, oo, FreeObject(op=op, **dmake(vx)), p)
op = OptionParser(usage="%prog [options...] <master> <slave>", version="%prog 0.0.1")
op.add_option('--gluster-command-dir', metavar='DIR', default='')
op.add_option('--gluster-log-file', metavar='LOGF', default=os.devnull, type=str, action='callback', callback=store_abs)
op.add_option('--gluster-log-level', metavar='LVL')
op.add_option('--gluster-params', metavar='PRMS', default='')
op.add_option('--gluster-cli-options', metavar='OPTS', default='--log-file=-')
op.add_option('--mountbroker', metavar='LABEL')
op.add_option('-p', '--pid-file', metavar='PIDF', type=str, action='callback', callback=store_abs)
op.add_option('-l', '--log-file', metavar='LOGF', type=str, action='callback', callback=store_abs)
op.add_option('--state-file', metavar='STATF', type=str, action='callback', callback=store_abs)
op.add_option('--ignore-deletes', default=False, action='store_true')
op.add_option('--use-rsync-xattrs', default=False, action='store_true')
op.add_option('-L', '--log-level', metavar='LVL')
op.add_option('-r', '--remote-gsyncd', metavar='CMD', default=os.path.abspath(sys.argv[0]))
op.add_option('--volume-id', metavar='UUID')
op.add_option('--session-owner', metavar='ID')
op.add_option('-s', '--ssh-command', metavar='CMD', default='ssh')
op.add_option('--rsync-command', metavar='CMD', default='rsync')
op.add_option('--rsync-options', metavar='OPTS', default='--sparse')
op.add_option('--rsync-ssh-options', metavar='OPTS', default='--compress')
op.add_option('--timeout', metavar='SEC', type=int, default=120)
op.add_option('--connection-timeout', metavar='SEC', type=int, default=60, help=SUPPRESS_HELP)
op.add_option('--sync-jobs', metavar='N', type=int, default=3)
op.add_option('--turns', metavar='N', type=int, default=0, help=SUPPRESS_HELP)
op.add_option('--allow-network', metavar='IPS', default='')
op.add_option('--state-socket-unencoded', metavar='SOCKF', type=str, action='callback', callback=store_abs)
op.add_option('--checkpoint', metavar='LABEL', default='')
# tunables for failover/failback mechanism:
# None - gsyncd behaves as normal
# blind - gsyncd works with xtime pairs to identify
# candidates for synchronization
# wrapup - same as normal mode but does not assign
# xtimes to orphaned files
# see crawl() for usage of the above tunables
op.add_option('--special-sync-mode', type=str, help=SUPPRESS_HELP)
op.add_option('-c', '--config-file', metavar='CONF', type=str, action='callback', callback=store_local)
# duh. need to specify dest or value will be mapped to None :S
op.add_option('--monitor', dest='monitor', action='callback', callback=store_local_curry(True))
op.add_option('--feedback-fd', dest='feedback_fd', type=int, help=SUPPRESS_HELP, action='callback', callback=store_local)
op.add_option('--listen', dest='listen', help=SUPPRESS_HELP, action='callback', callback=store_local_curry(True))
op.add_option('-N', '--no-daemon', dest="go_daemon", action='callback', callback=store_local_curry('dont'))
op.add_option('--debug', dest="go_daemon", action='callback', callback=lambda *a: (store_local_curry('dont')(*a),
setattr(a[-1].values, 'log_file', '-'),
setattr(a[-1].values, 'log_level', 'DEBUG'))),
for a in ('check', 'get'):
op.add_option('--config-' + a, metavar='OPT', type=str, dest='config', action='callback',
callback=store_local_obj(a, lambda vx: {'opt': vx}))
op.add_option('--config-get-all', dest='config', action='callback', callback=store_local_obj('get', lambda vx: {'opt': None}))
for m in ('', '-rx', '-glob'):
# call this code 'Pythonic' eh?
# have to define a one-shot local function to be able to inject (a value depending on the)
# iteration variable into the inner lambda
def conf_mod_opt_regex_variant(rx):
op.add_option('--config-set' + m, metavar='OPT VAL', type=str, nargs=2, dest='config', action='callback',
callback=store_local_obj('set', lambda vx: {'opt': vx[0], 'val': vx[1], 'rx': rx}))
op.add_option('--config-del' + m, metavar='OPT', type=str, dest='config', action='callback',
callback=store_local_obj('del', lambda vx: {'opt': vx, 'rx': rx}))
conf_mod_opt_regex_variant(m and m[1:] or False)
op.add_option('--normalize-url', dest='url_print', action='callback', callback=store_local_curry('normal'))
op.add_option('--canonicalize-url', dest='url_print', action='callback', callback=store_local_curry('canon'))
op.add_option('--canonicalize-escape-url', dest='url_print', action='callback', callback=store_local_curry('canon_esc'))
tunables = [ norm(o.get_opt_string()[2:]) for o in op.option_list if o.callback in (store_abs, 'store_true', None) and o.get_opt_string() not in ('--version', '--help') ]
remote_tunables = [ 'listen', 'go_daemon', 'timeout', 'session_owner', 'config_file', 'use_rsync_xattrs' ]
rq_remote_tunables = { 'listen': True }
# precedence for sources of values: 1) commandline, 2) cfg file, 3) defaults
# -- for this to work out we need to tell apart defaults from explicitly set
# options... so churn out the defaults here and call the parser with virgin
# values container.
defaults = op.get_default_values()
opts, args = op.parse_args(values=optparse.Values())
confdata = rconf.get('config')
if not (len(args) == 2 or \
(len(args) == 1 and rconf.get('listen')) or \
(len(args) <= 2 and confdata) or \
rconf.get('url_print')):
sys.stderr.write("error: incorrect number of arguments\n\n")
sys.stderr.write(op.get_usage() + "\n")
sys.exit(1)
restricted = os.getenv('_GSYNCD_RESTRICTED_')
if restricted:
allopts = {}
allopts.update(opts.__dict__)
allopts.update(rconf)
bannedtuns = set(allopts.keys()) - set(remote_tunables)
if bannedtuns:
raise GsyncdError('following tunables cannot be set with restricted SSH invocaton: ' + \
', '.join(bannedtuns))
for k, v in rq_remote_tunables.items():
if not k in allopts or allopts[k] != v:
raise GsyncdError('tunable %s is not set to value %s required for restricted SSH invocaton' % \
(k, v))
confrx = getattr(confdata, 'rx', None)
if confrx:
# peers are regexen, don't try to parse them
if confrx == 'glob':
args = [ '\A' + fnmatch.translate(a) for a in args ]
canon_peers = args
namedict = {}
else:
rscs = [resource.parse_url(u) for u in args]
dc = rconf.get('url_print')
if dc:
for r in rscs:
print(r.get_url(**{'normal': {},
'canon': {'canonical': True},
'canon_esc': {'canonical': True, 'escaped': True}}[dc]))
return
local = remote = None
if rscs:
local = rscs[0]
if len(rscs) > 1:
remote = rscs[1]
if not local.can_connect_to(remote):
raise GsyncdError("%s cannot work with %s" % (local.path, remote and remote.path))
pa = ([], [], [])
urlprms = ({}, {'canonical': True}, {'canonical': True, 'escaped': True})
for x in rscs:
for i in range(len(pa)):
pa[i].append(x.get_url(**urlprms[i]))
peers, canon_peers, canon_esc_peers = pa
# creating the namedict, a dict representing various ways of referring to / repreenting
# peers to be fillable in config templates
mods = (lambda x: x, lambda x: x[0].upper() + x[1:], lambda x: 'e' + x[0].upper() + x[1:])
if remote:
rmap = { local: ('local', 'master'), remote: ('remote', 'slave') }
else:
rmap = { local: ('local', 'slave') }
namedict = {}
for i in range(len(rscs)):
x = rscs[i]
for name in rmap[x]:
for j in range(3):
namedict[mods[j](name)] = pa[j][i]
if x.scheme == 'gluster':
namedict[name + 'vol'] = x.volume
if not 'config_file' in rconf:
rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]), "conf/gsyncd.conf")
gcnf = GConffile(rconf['config_file'], canon_peers, defaults.__dict__, opts.__dict__, namedict)
checkpoint_change = False
if confdata:
opt_ok = norm(confdata.opt) in tunables + [None]
if confdata.op == 'check':
if opt_ok:
sys.exit(0)
else:
sys.exit(1)
elif not opt_ok:
raise GsyncdError("not a valid option: " + confdata.opt)
if confdata.op == 'get':
gcnf.get(confdata.opt)
elif confdata.op == 'set':
gcnf.set(confdata.opt, confdata.val, confdata.rx)
elif confdata.op == 'del':
gcnf.delete(confdata.opt, confdata.rx)
# when modifying checkpoint, it's important to make a log
# of that, so in that case we go on to set up logging even
# if its just config invocation
if confdata.opt == 'checkpoint' and confdata.op in ('set', 'del') and \
not confdata.rx:
checkpoint_change = True
if not checkpoint_change:
return
gconf.__dict__.update(defaults.__dict__)
gcnf.update_to(gconf.__dict__)
gconf.__dict__.update(opts.__dict__)
gconf.configinterface = gcnf
if restricted and gconf.allow_network:
ssh_conn = os.getenv('SSH_CONNECTION')
if not ssh_conn:
#legacy env var
ssh_conn = os.getenv('SSH_CLIENT')
if ssh_conn:
allowed_networks = [ IPNetwork(a) for a in gconf.allow_network.split(',') ]
client_ip = IPAddress(ssh_conn.split()[0])
allowed = False
for nw in allowed_networks:
if client_ip in nw:
allowed = True
break
if not allowed:
raise GsyncdError("client IP address is not allowed")
ffd = rconf.get('feedback_fd')
if ffd:
fcntl.fcntl(ffd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
#normalize loglevel
lvl0 = gconf.log_level
if isinstance(lvl0, str):
lvl1 = lvl0.upper()
lvl2 = logging.getLevelName(lvl1)
# I have _never_ _ever_ seen such an utterly braindead
# error condition
if lvl2 == "Level " + lvl1:
raise GsyncdError('cannot recognize log level "%s"' % lvl0)
gconf.log_level = lvl2
if checkpoint_change:
GLogger._gsyncd_loginit(log_file=gconf.log_file, label='conf')
if confdata.op == 'set':
logging.info('checkpoint %s set' % confdata.val)
elif confdata.op == 'del':
logging.info('checkpoint info was reset')
return
go_daemon = rconf['go_daemon']
be_monitor = rconf.get('monitor')
if not be_monitor and isinstance(remote, resource.SSH) and \
go_daemon == 'should':
go_daemon = 'postconn'
log_file = None
else:
log_file = gconf.log_file
if be_monitor:
label = 'monitor'
elif remote:
#master
label = ''
else:
label = 'slave'
startup(go_daemon=go_daemon, log_file=log_file, label=label)
if be_monitor:
return monitor()
logging.info("syncing: %s" % " -> ".join(peers))
resource.Popen.init_errhandler()
if remote:
go_daemon = remote.connect_remote(go_daemon=go_daemon)
if go_daemon:
startup(go_daemon=go_daemon, log_file=gconf.log_file)
# complete remote connection in child
remote.connect_remote(go_daemon='done')
local.connect()
if ffd:
os.close(ffd)
local.service_loop(*[r for r in [remote] if r])
def main_i():
rconf = {'go_daemon': 'should'}
def store_abs(opt, optstr, val, parser):
setattr(parser.values, opt.dest, os.path.abspath(val))
def store_local(opt, optstr, val, parser):
rconf[opt.dest] = val
def store_local_curry(val):
return lambda o, oo, vx, p: store_local(o, oo, val, p)
op = OptionParser(usage="%prog [options...] <master> <slave>", version="%prog 0.0.1")
op.add_option('--gluster-command', metavar='CMD', default='glusterfs')
op.add_option('--gluster-log-file', metavar='LOGF', default=os.devnull, type=str, action='callback', callback=store_abs)
op.add_option('--gluster-log-level', metavar='LVL')
op.add_option('-p', '--pid-file', metavar='PIDF', type=str, action='callback', callback=store_abs)
op.add_option('-l', '--log-file', metavar='LOGF', type=str, action='callback', callback=store_abs)
op.add_option('-L', '--log-level', metavar='LVL')
op.add_option('-r', '--remote-gsyncd', metavar='CMD', default=os.path.abspath(sys.argv[0]))
op.add_option('-s', '--ssh-command', metavar='CMD', default='ssh')
op.add_option('--rsync-command', metavar='CMD', default='rsync')
op.add_option('--rsync-extra', metavar='ARGS', default='-sS', help=SUPPRESS_HELP)
op.add_option('--timeout', metavar='SEC', type=int, default=120)
op.add_option('--sync-jobs', metavar='N', type=int, default=3)
op.add_option('--turns', metavar='N', type=int, default=0, help=SUPPRESS_HELP)
op.add_option('-c', '--config-file', metavar='CONF', type=str, action='callback', callback=store_local)
# duh. need to specify dest or value will be mapped to None :S
op.add_option('--listen', dest='listen', help=SUPPRESS_HELP, action='callback', callback=store_local_curry(True))
op.add_option('-N', '--no-daemon', dest="go_daemon", action='callback', callback=store_local_curry('dont'))
op.add_option('--debug', dest="go_daemon", action='callback', callback=lambda *a: (store_local_curry('dont')(*a),
a[-1].values.__dict__.get('log_level') or \
a[-1].values.__dict__.update(log_level='DEBUG')))
op.add_option('--config-get', metavar='OPT', type=str, dest='config', action='callback', callback=store_local)
op.add_option('--config-get-all', dest='config', action='callback', callback=store_local_curry(True))
op.add_option('--config-set', metavar='OPT VAL', type=str, nargs=2, dest='config', action='callback',
callback=lambda o, oo, vx, p: store_local(o, oo, (vx[0], vx[1], False), p))
op.add_option('--config-set-rx', metavar='OPT VAL', type=str, nargs=2, dest='config', action='callback',
callback=lambda o, oo, vx, p: store_local(o, oo, (vx[0], vx[1], True), p))
op.add_option('--config-del', metavar='OPT', type=str, dest='config', action='callback', callback=lambda o, oo, vx, p:
store_local(o, oo, (vx, False, False), p))
op.add_option('--config-del-rx', metavar='OPT', type=str, dest='config', action='callback', callback=lambda o, oo, vx, p:
store_local(o, oo, (vx, False, True), p))
op.add_option('--canonicalize-url', dest='do_canon', action='callback', callback=store_local_curry('raw'))
op.add_option('--canonicalize-escape-url', dest='do_canon', action='callback', callback=store_local_curry('escaped'))
# precedence for sources of values: 1) commandline, 2) cfg file, 3) defaults
# -- for this to work out we need to tell apart defaults from explicitly set
# options... so churn out the defaults here and call the parser with virgin
# values container.
defaults = op.get_default_values()
opts, args = op.parse_args(values=optparse.Values())
confdata = rconf.get('config')
if not (len(args) == 2 or \
(len(args) == 1 and rconf.get('listen')) or \
(len(args) <= 2 and confdata) or \
rconf.get('do_canon')):
sys.stderr.write("error: incorrect number of arguments\n\n")
sys.stderr.write(op.get_usage() + "\n")
sys.exit(1)
if confdata and isinstance(confdata, tuple) and confdata[2]:
# peers are regexen, don't try to parse them
canon_peers = args
else:
rscs = [resource.parse_url(u) for u in args]
dc = rconf.get('do_canon')
if dc:
for r in rscs:
print(r.get_url(canonical=True, escaped=(dc=='escaped')))
return
local = remote = None
if rscs:
local = rscs[0]
if len(rscs) > 1:
remote = rscs[1]
if not local.can_connect_to(remote):
raise RuntimeError("%s cannot work with %s" % (local.path, remote and remote.path))
pa = ([], [])
canon = [False, True]
for x in (local, remote):
if x:
for i in range(2):
pa[i].append(x.get_url(canonical=canon[i]))
peers, canon_peers = pa
if not 'config_file' in rconf:
rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]), "conf/gsyncd.conf")
gcnf = GConffile(rconf['config_file'], canon_peers)
if confdata:
if isinstance(confdata, tuple):
if confdata[1]:
gcnf.set(*confdata)
else:
gcnf.delete(confdata[0], confdata[1])
else:
if confdata == True:
confdata = None
gcnf.get(confdata)
return
gconf.__dict__.update(defaults.__dict__)
gcnf.update_to(gconf.__dict__)
gconf.__dict__.update(opts.__dict__)
#normalize loglevel
lvl0 = gconf.log_level
if isinstance(lvl0, str):
lvl1 = lvl0.upper()
lvl2 = logging.getLevelName(lvl1)
# I have _never_ _ever_ seen such an utterly braindead
# error condition
if lvl2 == "Level " + lvl1:
raise RuntimeError('cannot recognize log level "%s"' % lvl0)
gconf.log_level = lvl2
go_daemon = rconf['go_daemon']
if isinstance(remote, resource.SSH) and go_daemon == 'should':
go_daemon = 'postconn'
log_file = None
else:
log_file = gconf.log_file
startup(go_daemon=go_daemon, log_file=log_file, slave=(not remote))
logging.info("syncing: %s" % " -> ".join(peers))
if remote:
go_daemon = remote.connect_remote(go_daemon=go_daemon)
if go_daemon:
startup(go_daemon=go_daemon, log_file=gconf.log_file)
# complete remote connection in child
remote.connect_remote(go_daemon='done')
local.connect()
local.service_loop(*[r for r in [remote] if r])
logging.info("exiting.")
def main_i():
rconf = {'go_daemon': 'should'}
def store_abs(opt, optstr, val, parser):
if val and val != '-':
val = os.path.abspath(val)
setattr(parser.values, opt.dest, val)
def store_local(opt, optstr, val, parser):
rconf[opt.dest] = val
def store_local_curry(val):
return lambda o, oo, vx, p: store_local(o, oo, val, p)
def store_local_obj(op, dmake):
return lambda o, oo, vx, p: store_local(o, oo, FreeObject(op=op, **dmake(vx)), p)
op = OptionParser(usage="%prog [options...] <master> <slave>", version="%prog 0.0.1")
op.add_option('--gluster-command', metavar='CMD', default='glusterfs')
op.add_option('--gluster-log-file', metavar='LOGF', default=os.devnull, type=str, action='callback', callback=store_abs)
op.add_option('--gluster-log-level', metavar='LVL')
op.add_option('-p', '--pid-file', metavar='PIDF', type=str, action='callback', callback=store_abs)
op.add_option('-l', '--log-file', metavar='LOGF', type=str, action='callback', callback=store_abs)
op.add_option('--state-file', metavar='STATF', type=str, action='callback', callback=store_abs)
op.add_option('-L', '--log-level', metavar='LVL')
op.add_option('-r', '--remote-gsyncd', metavar='CMD', default=os.path.abspath(sys.argv[0]))
op.add_option('--volume-id', metavar='UUID')
op.add_option('--session-owner', metavar='ID')
op.add_option('-s', '--ssh-command', metavar='CMD', default='ssh')
op.add_option('--rsync-command', metavar='CMD', default='rsync')
op.add_option('--rsync-extra', metavar='ARGS', default='-sS', help=SUPPRESS_HELP)
op.add_option('--timeout', metavar='SEC', type=int, default=120)
op.add_option('--sync-jobs', metavar='N', type=int, default=3)
op.add_option('--turns', metavar='N', type=int, default=0, help=SUPPRESS_HELP)
op.add_option('-c', '--config-file', metavar='CONF', type=str, action='callback', callback=store_local)
# duh. need to specify dest or value will be mapped to None :S
op.add_option('--monitor', dest='monitor', action='callback', callback=store_local_curry(True))
op.add_option('--feedback-fd', dest='feedback_fd', type=int, help=SUPPRESS_HELP, action='callback', callback=store_local)
op.add_option('--listen', dest='listen', help=SUPPRESS_HELP, action='callback', callback=store_local_curry(True))
op.add_option('-N', '--no-daemon', dest="go_daemon", action='callback', callback=store_local_curry('dont'))
op.add_option('--debug', dest="go_daemon", action='callback', callback=lambda *a: (store_local_curry('dont')(*a),
setattr(a[-1].values, 'log_file', '-'),
setattr(a[-1].values, 'log_level', 'DEBUG'))),
for a in ('check', 'get'):
op.add_option('--config-' + a, metavar='OPT', type=str, dest='config', action='callback',
callback=store_local_obj(a, lambda vx: {'opt': vx}))
op.add_option('--config-get-all', dest='config', action='callback', callback=store_local_obj('get', lambda vx: {'opt': None}))
for m in ('', '-rx'):
# call this code 'Pythonic' eh?
# have to define a one-shot local function to be able to inject (a value depending on the)
# iteration variable into the inner lambda
def conf_mod_opt_regex_variant(rx):
op.add_option('--config-set' + m, metavar='OPT VAL', type=str, nargs=2, dest='config', action='callback',
callback=store_local_obj('set', lambda vx: {'opt': vx[0], 'val': vx[1], 'rx': rx}))
op.add_option('--config-del' + m, metavar='OPT', type=str, dest='config', action='callback',
callback=store_local_obj('del', lambda vx: {'opt': vx, 'rx': rx}))
conf_mod_opt_regex_variant(not not m)
op.add_option('--normalize-url', dest='url_print', action='callback', callback=store_local_curry('normal'))
op.add_option('--canonicalize-url', dest='url_print', action='callback', callback=store_local_curry('canon'))
op.add_option('--canonicalize-escape-url', dest='url_print', action='callback', callback=store_local_curry('canon_esc'))
tunables = [ norm(o.get_opt_string()[2:]) for o in op.option_list if o.callback in (store_abs, None) and o.get_opt_string() not in ('--version', '--help') ]
# precedence for sources of values: 1) commandline, 2) cfg file, 3) defaults
# -- for this to work out we need to tell apart defaults from explicitly set
# options... so churn out the defaults here and call the parser with virgin
# values container.
defaults = op.get_default_values()
opts, args = op.parse_args(values=optparse.Values())
confdata = rconf.get('config')
if not (len(args) == 2 or \
(len(args) == 1 and rconf.get('listen')) or \
(len(args) <= 2 and confdata) or \
rconf.get('url_print')):
sys.stderr.write("error: incorrect number of arguments\n\n")
sys.stderr.write(op.get_usage() + "\n")
sys.exit(1)
if getattr(confdata, 'rx', None):
# peers are regexen, don't try to parse them
canon_peers = args
namedict = {}
else:
rscs = [resource.parse_url(u) for u in args]
dc = rconf.get('url_print')
if dc:
for r in rscs:
print(r.get_url(**{'normal': {},
'canon': {'canonical': True},
'canon_esc': {'canonical': True, 'escaped': True}}[dc]))
return
local = remote = None
if rscs:
local = rscs[0]
if len(rscs) > 1:
remote = rscs[1]
if not local.can_connect_to(remote):
raise GsyncdError("%s cannot work with %s" % (local.path, remote and remote.path))
pa = ([], [], [])
urlprms = ({}, {'canonical': True}, {'canonical': True, 'escaped': True})
for x in rscs:
for i in range(len(pa)):
pa[i].append(x.get_url(**urlprms[i]))
peers, canon_peers, canon_esc_peers = pa
# creating the namedict, a dict representing various ways of referring to / repreenting
# peers to be fillable in config templates
mods = (lambda x: x, lambda x: x[0].upper() + x[1:], lambda x: 'e' + x[0].upper() + x[1:])
if remote:
rmap = { local: ('local', 'master'), remote: ('remote', 'slave') }
else:
rmap = { local: ('local', 'slave') }
namedict = {}
for i in range(len(rscs)):
x = rscs[i]
for name in rmap[x]:
for j in range(3):
namedict[mods[j](name)] = pa[j][i]
if x.scheme == 'gluster':
namedict[name + 'vol'] = x.volume
if not 'config_file' in rconf:
rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]), "conf/gsyncd.conf")
gcnf = GConffile(rconf['config_file'], canon_peers, defaults.__dict__, opts.__dict__, namedict)
if confdata:
opt_ok = norm(confdata.opt) in tunables + [None]
if confdata.op == 'check':
if opt_ok:
sys.exit(0)
else:
sys.exit(1)
elif not opt_ok:
raise GsyncdError("not a valid option: " + confdata.opt)
if confdata.op == 'get':
gcnf.get(confdata.opt)
elif confdata.op == 'set':
gcnf.set(confdata.opt, confdata.val, confdata.rx)
elif confdata.op == 'del':
gcnf.delete(confdata.opt, confdata.rx)
return
gconf.__dict__.update(defaults.__dict__)
gcnf.update_to(gconf.__dict__)
gconf.__dict__.update(opts.__dict__)
gconf.configinterface = gcnf
ffd = rconf.get('feedback_fd')
if ffd:
fcntl.fcntl(ffd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
#normalize loglevel
lvl0 = gconf.log_level
if isinstance(lvl0, str):
lvl1 = lvl0.upper()
lvl2 = logging.getLevelName(lvl1)
# I have _never_ _ever_ seen such an utterly braindead
# error condition
if lvl2 == "Level " + lvl1:
raise GsyncdError('cannot recognize log level "%s"' % lvl0)
gconf.log_level = lvl2
go_daemon = rconf['go_daemon']
be_monitor = rconf.get('monitor')
if not be_monitor and isinstance(remote, resource.SSH) and \
go_daemon == 'should':
go_daemon = 'postconn'
log_file = None
else:
log_file = gconf.log_file
if be_monitor:
label = 'monitor'
elif remote:
#master
label = ''
else:
label = 'slave'
startup(go_daemon=go_daemon, log_file=log_file, label=label)
if be_monitor:
return monitor()
logging.info("syncing: %s" % " -> ".join(peers))
if remote:
go_daemon = remote.connect_remote(go_daemon=go_daemon)
if go_daemon:
startup(go_daemon=go_daemon, log_file=gconf.log_file)
# complete remote connection in child
remote.connect_remote(go_daemon='done')
local.connect()
if ffd:
os.close(ffd)
local.service_loop(*[r for r in [remote] if r])
def main_i():
rconf = {'go_daemon': 'should'}
def store_abs(opt, optstr, val, parser):
setattr(parser.values, opt.dest, os.path.abspath(val))
def store_local(opt, optstr, val, parser):
rconf[opt.dest] = val
def store_local_curry(val):
return lambda o, oo, vx, p: store_local(o, oo, val, p)
op = OptionParser(usage="%prog [options...] <master> <slave>", version="%prog 0.0.1")
op.add_option('--gluster-command', metavar='CMD', default='glusterfs')
op.add_option('--gluster-log-file', metavar='LOGF', default=os.devnull, type=str, action='callback', callback=store_abs)
op.add_option('--gluster-log-level', metavar='LVL')
op.add_option('-p', '--pid-file', metavar='PIDF', type=str, action='callback', callback=store_abs)
op.add_option('-l', '--log-file', metavar='LOGF', type=str, action='callback', callback=store_abs)
op.add_option('-L', '--log-level', metavar='LVL')
op.add_option('-r', '--remote-gsyncd', metavar='CMD', default=os.path.abspath(sys.argv[0]))
op.add_option('-s', '--ssh-command', metavar='CMD', default='ssh')
op.add_option('--rsync-command', metavar='CMD', default='rsync')
op.add_option('--rsync-extra', metavar='ARGS', default='-sS', help=SUPPRESS_HELP)
op.add_option('--timeout', metavar='SEC', type=int, default=120)
op.add_option('--sync-jobs', metavar='N', type=int, default=3)
op.add_option('--turns', metavar='N', type=int, default=0, help=SUPPRESS_HELP)
op.add_option('-c', '--config-file', metavar='CONF', type=str, action='callback', callback=store_local)
# duh. need to specify dest or value will be mapped to None :S
op.add_option('--listen', dest='listen', help=SUPPRESS_HELP, action='callback', callback=store_local_curry(True))
op.add_option('-N', '--no-daemon', dest="go_daemon", action='callback', callback=store_local_curry('dont'))
op.add_option('--debug', dest="go_daemon", action='callback', callback=lambda *a: (store_local_curry('dont')(*a),
setattr(a[-1].values, 'log_file', '-'),
setattr(a[-1].values, 'log_level', 'DEBUG'))),
op.add_option('--config-get', metavar='OPT', type=str, dest='config', action='callback', callback=store_local)
op.add_option('--config-get-all', dest='config', action='callback', callback=store_local_curry(True))
op.add_option('--config-set', metavar='OPT VAL', type=str, nargs=2, dest='config', action='callback',
callback=lambda o, oo, vx, p: store_local(o, oo, (vx[0], vx[1], False), p))
op.add_option('--config-set-rx', metavar='OPT VAL', type=str, nargs=2, dest='config', action='callback',
callback=lambda o, oo, vx, p: store_local(o, oo, (vx[0], vx[1], True), p))
op.add_option('--config-del', metavar='OPT', type=str, dest='config', action='callback', callback=lambda o, oo, vx, p:
store_local(o, oo, (vx, False, False), p))
op.add_option('--config-del-rx', metavar='OPT', type=str, dest='config', action='callback', callback=lambda o, oo, vx, p:
store_local(o, oo, (vx, False, True), p))
op.add_option('--canonicalize-url', dest='do_canon', action='callback', callback=store_local_curry('raw'))
op.add_option('--canonicalize-escape-url', dest='do_canon', action='callback', callback=store_local_curry('escaped'))
# precedence for sources of values: 1) commandline, 2) cfg file, 3) defaults
# -- for this to work out we need to tell apart defaults from explicitly set
# options... so churn out the defaults here and call the parser with virgin
# values container.
defaults = op.get_default_values()
opts, args = op.parse_args(values=optparse.Values())
confdata = rconf.get('config')
if not (len(args) == 2 or \
(len(args) == 1 and rconf.get('listen')) or \
(len(args) <= 2 and confdata) or \
rconf.get('do_canon')):
sys.stderr.write("error: incorrect number of arguments\n\n")
sys.stderr.write(op.get_usage() + "\n")
sys.exit(1)
if confdata and isinstance(confdata, tuple) and confdata[2]:
# peers are regexen, don't try to parse them
canon_peers = args
else:
rscs = [resource.parse_url(u) for u in args]
dc = rconf.get('do_canon')
if dc:
for r in rscs:
print(r.get_url(canonical=True, escaped=(dc=='escaped')))
return
local = remote = None
if rscs:
local = rscs[0]
if len(rscs) > 1:
remote = rscs[1]
if not local.can_connect_to(remote):
raise RuntimeError("%s cannot work with %s" % (local.path, remote and remote.path))
pa = ([], [])
canon = [False, True]
for x in (local, remote):
if x:
for i in range(2):
pa[i].append(x.get_url(canonical=canon[i]))
peers, canon_peers = pa
if not 'config_file' in rconf:
rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]), "conf/gsyncd.conf")
gcnf = GConffile(rconf['config_file'], canon_peers)
if confdata:
if isinstance(confdata, tuple):
if confdata[1]:
gcnf.set(*confdata)
else:
gcnf.delete(confdata[0], confdata[1])
else:
if confdata == True:
confdata = None
gcnf.get(confdata)
return
gconf.__dict__.update(defaults.__dict__)
gcnf.update_to(gconf.__dict__)
gconf.__dict__.update(opts.__dict__)
#normalize loglevel
lvl0 = gconf.log_level
if isinstance(lvl0, str):
lvl1 = lvl0.upper()
lvl2 = logging.getLevelName(lvl1)
# I have _never_ _ever_ seen such an utterly braindead
# error condition
if lvl2 == "Level " + lvl1:
raise RuntimeError('cannot recognize log level "%s"' % lvl0)
gconf.log_level = lvl2
go_daemon = rconf['go_daemon']
if isinstance(remote, resource.SSH) and go_daemon == 'should':
go_daemon = 'postconn'
log_file = None
else:
log_file = gconf.log_file
startup(go_daemon=go_daemon, log_file=log_file, slave=(not remote))
logging.info("syncing: %s" % " -> ".join(peers))
if remote:
go_daemon = remote.connect_remote(go_daemon=go_daemon)
if go_daemon:
startup(go_daemon=go_daemon, log_file=gconf.log_file)
# complete remote connection in child
remote.connect_remote(go_daemon='done')
local.connect()
local.service_loop(*[r for r in [remote] if r])
logging.info("exiting.")
def main_i():
rconf = {'go_daemon': 'should'}
def store_abs(opt, optstr, val, parser):
if val and val != '-':
val = os.path.abspath(val)
setattr(parser.values, opt.dest, val)
def store_local(opt, optstr, val, parser):
rconf[opt.dest] = val
def store_local_curry(val):
return lambda o, oo, vx, p: store_local(o, oo, val, p)
def store_local_obj(op, dmake):
return lambda o, oo, vx, p: store_local(
o, oo, FreeObject(op=op, **dmake(vx)), p)
op = OptionParser(usage="%prog [options...] <master> <slave>",
version="%prog 0.0.1")
op.add_option('--gluster-command', metavar='CMD', default='glusterfs')
op.add_option('--gluster-log-file',
metavar='LOGF',
default=os.devnull,
type=str,
action='callback',
callback=store_abs)
op.add_option('--gluster-log-level', metavar='LVL')
op.add_option('-p',
'--pid-file',
metavar='PIDF',
type=str,
action='callback',
callback=store_abs)
op.add_option('-l',
'--log-file',
metavar='LOGF',
type=str,
action='callback',
callback=store_abs)
op.add_option('--state-file',
metavar='STATF',
type=str,
action='callback',
callback=store_abs)
op.add_option('-L', '--log-level', metavar='LVL')
op.add_option('-r',
'--remote-gsyncd',
metavar='CMD',
default=os.path.abspath(sys.argv[0]))
op.add_option('--volume-id', metavar='UUID')
op.add_option('--session-owner', metavar='ID')
op.add_option('-s', '--ssh-command', metavar='CMD', default='ssh')
op.add_option('--rsync-command', metavar='CMD', default='rsync')
op.add_option('--rsync-extra',
metavar='ARGS',
default='-sS',
help=SUPPRESS_HELP)
op.add_option('--timeout', metavar='SEC', type=int, default=120)
op.add_option('--sync-jobs', metavar='N', type=int, default=3)
op.add_option('--turns',
metavar='N',
type=int,
default=0,
help=SUPPRESS_HELP)
op.add_option('-c',
'--config-file',
metavar='CONF',
type=str,
action='callback',
callback=store_local)
# duh. need to specify dest or value will be mapped to None :S
op.add_option('--monitor',
dest='monitor',
action='callback',
callback=store_local_curry(True))
op.add_option('--feedback-fd',
dest='feedback_fd',
type=int,
help=SUPPRESS_HELP,
action='callback',
callback=store_local)
op.add_option('--listen',
dest='listen',
help=SUPPRESS_HELP,
action='callback',
callback=store_local_curry(True))
op.add_option('-N',
'--no-daemon',
dest="go_daemon",
action='callback',
callback=store_local_curry('dont'))
op.add_option('--debug',
dest="go_daemon",
action='callback',
callback=lambda *a:
(store_local_curry('dont')
(*a), setattr(a[-1].values, 'log_file', '-'),
setattr(a[-1].values, 'log_level', 'DEBUG'))),
for a in ('check', 'get'):
op.add_option('--config-' + a,
metavar='OPT',
type=str,
dest='config',
action='callback',
callback=store_local_obj(a, lambda vx: {'opt': vx}))
op.add_option('--config-get-all',
dest='config',
action='callback',
callback=store_local_obj('get', lambda vx: {'opt': None}))
for m in ('', '-rx'):
# call this code 'Pythonic' eh?
# have to define a one-shot local function to be able to inject (a value depending on the)
# iteration variable into the inner lambda
def conf_mod_opt_regex_variant(rx):
op.add_option('--config-set' + m,
metavar='OPT VAL',
type=str,
nargs=2,
dest='config',
action='callback',
callback=store_local_obj(
'set', lambda vx: {
'opt': vx[0],
'val': vx[1],
'rx': rx
}))
op.add_option('--config-del' + m,
metavar='OPT',
type=str,
dest='config',
action='callback',
callback=store_local_obj(
'del', lambda vx: {
'opt': vx,
'rx': rx
}))
conf_mod_opt_regex_variant(not not m)
op.add_option('--normalize-url',
dest='url_print',
action='callback',
callback=store_local_curry('normal'))
op.add_option('--canonicalize-url',
dest='url_print',
action='callback',
callback=store_local_curry('canon'))
op.add_option('--canonicalize-escape-url',
dest='url_print',
action='callback',
callback=store_local_curry('canon_esc'))
tunables = [
norm(o.get_opt_string()[2:]) for o in op.option_list
if o.callback in (store_abs,
None) and o.get_opt_string() not in ('--version',
'--help')
]
# precedence for sources of values: 1) commandline, 2) cfg file, 3) defaults
# -- for this to work out we need to tell apart defaults from explicitly set
# options... so churn out the defaults here and call the parser with virgin
# values container.
defaults = op.get_default_values()
opts, args = op.parse_args(values=optparse.Values())
confdata = rconf.get('config')
if not (len(args) == 2 or \
(len(args) == 1 and rconf.get('listen')) or \
(len(args) <= 2 and confdata) or \
rconf.get('url_print')):
sys.stderr.write("error: incorrect number of arguments\n\n")
sys.stderr.write(op.get_usage() + "\n")
sys.exit(1)
if getattr(confdata, 'rx', None):
# peers are regexen, don't try to parse them
canon_peers = args
namedict = {}
else:
rscs = [resource.parse_url(u) for u in args]
dc = rconf.get('url_print')
if dc:
for r in rscs:
print(
r.get_url(
**{
'normal': {},
'canon': {
'canonical': True
},
'canon_esc': {
'canonical': True,
'escaped': True
}
}[dc]))
return
local = remote = None
if rscs:
local = rscs[0]
if len(rscs) > 1:
remote = rscs[1]
if not local.can_connect_to(remote):
raise RuntimeError("%s cannot work with %s" %
(local.path, remote and remote.path))
pa = ([], [], [])
urlprms = ({}, {
'canonical': True
}, {
'canonical': True,
'escaped': True
})
for x in rscs:
for i in range(len(pa)):
pa[i].append(x.get_url(**urlprms[i]))
peers, canon_peers, canon_esc_peers = pa
# creating the namedict, a dict representing various ways of referring to / repreenting
# peers to be fillable in config templates
mods = (lambda x: x, lambda x: x[0].upper() + x[1:],
lambda x: 'e' + x[0].upper() + x[1:])
if remote:
rmap = {local: ('local', 'master'), remote: ('remote', 'slave')}
else:
rmap = {local: ('local', 'slave')}
namedict = {}
for i in range(len(rscs)):
x = rscs[i]
for name in rmap[x]:
for j in range(3):
namedict[mods[j](name)] = pa[j][i]
if x.scheme == 'gluster':
namedict[name + 'vol'] = x.volume
if not 'config_file' in rconf:
rconf['config_file'] = os.path.join(os.path.dirname(sys.argv[0]),
"conf/gsyncd.conf")
gcnf = GConffile(rconf['config_file'], canon_peers, defaults.__dict__,
opts.__dict__, namedict)
if confdata:
opt_ok = norm(confdata.opt) in tunables + [None]
if confdata.op == 'check':
if opt_ok:
sys.exit(0)
else:
sys.exit(1)
elif not opt_ok:
raise RuntimeError("not a valid option: " + confdata.opt)
if confdata.op == 'get':
gcnf.get(confdata.opt)
elif confdata.op == 'set':
gcnf.set(confdata.opt, confdata.val, confdata.rx)
elif confdata.op == 'del':
gcnf.delete(confdata.opt, confdata.rx)
return
gconf.__dict__.update(defaults.__dict__)
gcnf.update_to(gconf.__dict__)
gconf.__dict__.update(opts.__dict__)
gconf.configinterface = gcnf
ffd = rconf.get('feedback_fd')
if ffd:
fcntl.fcntl(ffd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
#normalize loglevel
lvl0 = gconf.log_level
if isinstance(lvl0, str):
lvl1 = lvl0.upper()
lvl2 = logging.getLevelName(lvl1)
# I have _never_ _ever_ seen such an utterly braindead
# error condition
if lvl2 == "Level " + lvl1:
raise RuntimeError('cannot recognize log level "%s"' % lvl0)
gconf.log_level = lvl2
go_daemon = rconf['go_daemon']
be_monitor = rconf.get('monitor')
if not be_monitor and isinstance(remote, resource.SSH) and \
go_daemon == 'should':
go_daemon = 'postconn'
log_file = None
else:
log_file = gconf.log_file
if be_monitor:
label = 'monitor'
elif remote:
#master
label = ''
else:
label = 'slave'
startup(go_daemon=go_daemon, log_file=log_file, label=label)
if be_monitor:
return monitor()
logging.info("syncing: %s" % " -> ".join(peers))
if remote:
go_daemon = remote.connect_remote(go_daemon=go_daemon)
if go_daemon:
startup(go_daemon=go_daemon, log_file=gconf.log_file)
# complete remote connection in child
remote.connect_remote(go_daemon='done')
local.connect()
if ffd:
os.close(ffd)
local.service_loop(*[r for r in [remote] if r])
logging.info("exiting.")
