def parse_route_ranges(route_file, route_addr):
import re
with open(route_file) as route_in:
group_prog = re.compile('Group #')
route_prog = re.compile('Route\(([0-9a-fA-F]*), ([0-9a-fA-F:]*) ')
ranges = []
key_range = elliptics.IteratorRange()
key_range.key_begin = elliptics.Id([0] * 64, 0)
want_start_key = False
our_group = False
for line in route_in:
if group_prog.search(line):
if our_group:
# our group has been processed, exit
break
continue
match = route_prog.search(line)
if match:
route = match.group(1)
addr = match.group(2)
if addr == route_addr:
# we only want to parse a single group where our address was found
our_group = True
if want_start_key:
# we have received a range start which belongs to us
# but we are inside of the range which still belongs to us
# so combine this new range with previous one
# i.e. just skip this new start
continue
if not want_start_key:
key_range.key_end = elliptics.Id(transf(route), 0)
ranges.append(key_range)
want_start_key = True
continue
if want_start_key:
key_range = elliptics.IteratorRange()
key_range.key_begin = elliptics.Id(transf(route), 0)
want_start_key = False
continue
if not want_start_key:
key_range.key_end = elliptics.Id([255] * 64, 0)
ranges.append(key_range)
for r in ranges:
print r.key_begin, r.key_end
return ranges
def range(key_begin, key_end):
ret = elliptics.IteratorRange()
ret.key_begin = elliptics.Id(key_begin, 0)
ret.key_end = elliptics.Id(key_end, 0)
return ret
def parse_args():
from optparse import OptionParser
ctx = Ctx()
parser = OptionParser()
parser.usage = "%prog type [options]"
parser.description = __doc__
parser.add_option("-g",
"--groups",
action="store",
dest="groups",
default=None,
help="Comma separated list of groups [default: all]")
parser.add_option(
"-l",
"--log",
dest="log",
default='/dev/stderr',
metavar="FILE",
help="Output log messages from library to file [default: %default]")
parser.add_option("-L",
"--log-level",
action="store",
dest="log_level",
default="1",
help="Elliptics client verbosity [default: %default]")
parser.add_option("-r",
"--remote",
action="append",
dest="remote",
help="Elliptics node address [default: %default]")
parser.add_option(
"-d",
"--data",
action="store_true",
dest="data",
default=False,
help="Requests object's data with other info [default: %default]")
parser.add_option("-k",
"--key-begin",
action="store",
dest="key_begin",
default=None,
help="Begin key of range for iterating")
parser.add_option("-K",
"--key-end",
action="store",
dest="key_end",
default=None,
help="End key of range for iterating")
parser.add_option("-t",
"--time-begin",
action="store",
dest="time_begin",
default=None,
help="Begin timestamp of time range for iterating")
parser.add_option("-T",
"--time-end",
action="store",
dest="time_end",
default=None,
help="End timestamp of time range for iterating")
parser.add_option(
"-M",
"--no-meta",
action="store_true",
dest="no_meta",
default=False,
help=
"Run iterator without metadata (timestamp and user flags). This option conflicts with --data and --time-*, if one of them is specified, --no-meta will have no effect."
)
parser.add_option(
"-A",
"--addr",
action="store",
dest="route_addr",
default=None,
help=
"Address to lookup in route file. This address will be used to determine iterator ranges - ranges which DO NOT belong to selected node."
)
parser.add_option(
"-R",
"--route-file",
action="store",
dest="route_file",
default=None,
help=
"Route file contains 'dnet_balance' tool's output - route table dump, which will be parsed to find out ranges which DO NOT belong to selected address."
)
(options, args) = parser.parse_args()
if len(args) > 1:
raise ValueError("Too many arguments passed: {0}, expected: 1".format(
len(args)))
elif len(args) == 0:
raise ValueError(
"Please specify one of following modes: {0}".format(ALLOWED_MODES))
if args[0].lower() not in ALLOWED_MODES:
raise ValueError("Unknown mode: '{0}', allowed: {1}".format(
args[0], ALLOWED_MODES))
ctx.iterate_mode = args[0].lower()
try:
if options.groups:
ctx.groups = map(int, options.groups.split(','))
else:
ctx.groups = []
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}".format(
options.groups, repr(e)))
print("Using group list: {0}".format(ctx.groups))
try:
ctx.log_file = options.log
ctx.log_level = int(options.log_level)
except Exception as e:
raise ValueError("Can't parse log_level: '{0}': {1}".format(
options.log_level, repr(e)))
print("Using elliptics client log level: {0}".format(ctx.log_level))
if not options.remote:
raise ValueError(
"Please specify at least one remote address (-r option)")
try:
ctx.remotes = []
for r in options.remote:
ctx.remotes.append(elliptics.Address.from_host_port_family(r))
print("Using remote host:port:family: {0}".format(ctx.remotes[-1]))
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}".format(
options.remote, repr(e)))
ctx.no_meta = options.no_meta
try:
if options.time_begin:
ctx.time_begin = Time.from_epoch(options.time_begin)
else:
ctx.time_begin = None
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}".format(
options.timestamp, repr(e)))
print("Using time_begin: {0}".format(ctx.time_begin))
try:
if options.time_end:
ctx.time_end = Time.from_epoch(options.time_end)
else:
ctx.time_end = None
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}".format(
options.timestamp, repr(e)))
print("Using time_end: {0}".format(ctx.time_end))
ctx.data = options.data
key_range = elliptics.IteratorRange()
try:
if options.key_begin == '-1':
key_range.key_begin = elliptics.Id([255] * 64, 0)
elif options.key_begin:
key_range.key_begin = elliptics.Id(transf(options.key_begin), 0)
except Exception as e:
raise ValueError("Can't parse key_begin: '{0}': {1}".format(
options.key_begin, repr(e)))
try:
if options.key_end == '-1':
key_range.key_end = elliptics.Id([255] * 64, 0)
elif options.key_end:
key_range.key_end = elliptics.Id(transf(options.key_end), 0)
except Exception as e:
raise ValueError("Can't parse key_end: '{0}': {1}".format(
options.key_end, repr(e)))
ctx.ranges = []
if options.key_begin or options.key_end:
ctx.ranges = [key_range]
try:
if options.route_file and options.route_addr:
ranges = parse_route_ranges(options.route_file, options.route_addr)
ctx.ranges += ranges
except Exception as e:
raise ValueError(
"Can't parse route_file '{0}' and route_addr '{1}' options: {2}".
format(options.route_file, options.route_addr, repr(e)))
return ctx
def make_range(begin, end):
range = elliptics.IteratorRange()
range.key_begin = begin
range.key_end = end
return range
def range(key_begin, key_end):
ret = elliptics.IteratorRange()
ret.key_begin = key_begin
ret.key_end = key_end
return ret
def parse_args():
from optparse import OptionParser
ctx = Ctx()
parser = OptionParser()
parser.usage = "%prog type [options]"
parser.description = __doc__
parser.add_option("-g", "--groups", action="store", dest="groups", default=None,
help="Comma separated list of groups [default: all]")
parser.add_option("-l", "--log", dest="log", default='/dev/stderr', metavar="FILE",
help="Output log messages from library to file [default: %default]")
parser.add_option("-L", "--log-level", action="store", dest="log_level", default="1",
help="Elliptics client verbosity [default: %default]")
parser.add_option("-r", "--remote", action="append", dest="remote",
help="Elliptics node address [default: %default]")
parser.add_option("-d", "--data", action="store_true", dest="data", default=False,
help="Requests object's data with other info [default: %default]")
parser.add_option("-k", "--key-begin", action="store", dest="key_begin", default="0",
help="Begin key of range for iterating")
parser.add_option("-K", "--key-end", action="store", dest="key_end", default="-1",
help="End key of range for iterating")
parser.add_option("-t", "--time-begin", action="store", dest="time_begin", default=None,
help="Begin timestamp of time range for iterating")
parser.add_option("-T", "--time-end", action="store", dest="time_end", default=None,
help="End timestamp of time range for iterating")
(options, args) = parser.parse_args()
if len(args) > 1:
raise ValueError("Too many arguments passed: {0}, expected: 1"
.format(len(args)))
elif len(args) == 0:
raise ValueError("Please specify one of following modes: {0}"
.format(ALLOWED_MODES))
if args[0].lower() not in ALLOWED_MODES:
raise ValueError("Unknown mode: '{0}', allowed: {1}"
.format(args[0], ALLOWED_MODES))
ctx.iterate_mode = args[0].lower()
try:
if options.groups:
ctx.groups = map(int, options.groups.split(','))
else:
ctx.groups = []
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}".format(
options.groups, repr(e)))
print("Using group list: {0}".format(ctx.groups))
try:
ctx.log_file = options.log
ctx.log_level = int(options.log_level)
except Exception as e:
raise ValueError("Can't parse log_level: '{0}': {1}"
.format(options.log_level, repr(e)))
print("Using elliptics client log level: {0}".format(ctx.log_level))
if not options.remote:
raise ValueError("Please specify at least one remote address (-r option)")
try:
ctx.remotes = []
for r in options.remote:
ctx.remotes.append(elliptics.Address.from_host_port_family(r))
print("Using remote host:port:family: {0}".format(ctx.remotes[-1]))
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}"
.format(options.remote, repr(e)))
try:
if options.time_begin:
ctx.time_begin = Time.from_epoch(options.time_begin)
else:
ctx.time_begin = None
except Exception:
raise ValueError("Can't parse timestamp: '{0}': {1}"
.format(options.timestamp, repr(e)))
print("Using time_begin: {0}".format(ctx.time_begin))
try:
if options.time_end:
ctx.time_end = Time.from_epoch(options.time_end)
else:
ctx.time_end = None
except Exception:
raise ValueError("Can't parse timestamp: '{0}': {1}"
.format(options.timestamp, repr(e)))
print("Using time_end: {0}".format(ctx.time_end))
ctx.data = options.data
key_range = elliptics.IteratorRange()
try:
if options.key_begin == '-1':
key_range.key_begin = elliptics.Id([255] * 64, 0)
elif options.key_begin:
key_range.key_begin = elliptics.Id(transf(options.key_begin), 0)
else:
key_range.key_begin = elliptics.Id([0] * 64, 0)
except Exception:
raise ValueError("Can't parse key_begin: '{0}': {1}"
.format(options.key_begin, repr(e)))
try:
if options.key_end == '-1':
key_range.key_end = elliptics.Id([255] * 64, 0)
elif options.key_end:
key_range.key_end = elliptics.Id(transf(options.key_end), 0)
else:
key_range.key_end = elliptics.Id([255] * 64, 0)
except Exception:
raise ValueError("Can't parse key_end: '{0}': {1}"
.format(options.key_end, repr(e)))
ctx.ranges = [key_range]
return ctx