def parse_route_ranges(route_file, route_addr): import re with open(route_file) as route_in: group_prog = re.compile('Group #') route_prog = re.compile('Route\(([0-9a-fA-F]*), ([0-9a-fA-F:]*) ') ranges = [] key_range = elliptics.IteratorRange() key_range.key_begin = elliptics.Id([0] * 64, 0) want_start_key = False our_group = False for line in route_in: if group_prog.search(line): if our_group: # our group has been processed, exit break continue match = route_prog.search(line) if match: route = match.group(1) addr = match.group(2) if addr == route_addr: # we only want to parse a single group where our address was found our_group = True if want_start_key: # we have received a range start which belongs to us # but we are inside of the range which still belongs to us # so combine this new range with previous one # i.e. just skip this new start continue if not want_start_key: key_range.key_end = elliptics.Id(transf(route), 0) ranges.append(key_range) want_start_key = True continue if want_start_key: key_range = elliptics.IteratorRange() key_range.key_begin = elliptics.Id(transf(route), 0) want_start_key = False continue if not want_start_key: key_range.key_end = elliptics.Id([255] * 64, 0) ranges.append(key_range) for r in ranges: print r.key_begin, r.key_end return ranges
def range(key_begin, key_end): ret = elliptics.IteratorRange() ret.key_begin = elliptics.Id(key_begin, 0) ret.key_end = elliptics.Id(key_end, 0) return ret
def parse_args(): from optparse import OptionParser ctx = Ctx() parser = OptionParser() parser.usage = "%prog type [options]" parser.description = __doc__ parser.add_option("-g", "--groups", action="store", dest="groups", default=None, help="Comma separated list of groups [default: all]") parser.add_option( "-l", "--log", dest="log", default='/dev/stderr', metavar="FILE", help="Output log messages from library to file [default: %default]") parser.add_option("-L", "--log-level", action="store", dest="log_level", default="1", help="Elliptics client verbosity [default: %default]") parser.add_option("-r", "--remote", action="append", dest="remote", help="Elliptics node address [default: %default]") parser.add_option( "-d", "--data", action="store_true", dest="data", default=False, help="Requests object's data with other info [default: %default]") parser.add_option("-k", "--key-begin", action="store", dest="key_begin", default=None, help="Begin key of range for iterating") parser.add_option("-K", "--key-end", action="store", dest="key_end", default=None, help="End key of range for iterating") parser.add_option("-t", "--time-begin", action="store", dest="time_begin", default=None, help="Begin timestamp of time range for iterating") parser.add_option("-T", "--time-end", action="store", dest="time_end", default=None, help="End timestamp of time range for iterating") parser.add_option( "-M", "--no-meta", action="store_true", dest="no_meta", default=False, help= "Run iterator without metadata (timestamp and user flags). This option conflicts with --data and --time-*, if one of them is specified, --no-meta will have no effect." ) parser.add_option( "-A", "--addr", action="store", dest="route_addr", default=None, help= "Address to lookup in route file. This address will be used to determine iterator ranges - ranges which DO NOT belong to selected node." ) parser.add_option( "-R", "--route-file", action="store", dest="route_file", default=None, help= "Route file contains 'dnet_balance' tool's output - route table dump, which will be parsed to find out ranges which DO NOT belong to selected address." ) (options, args) = parser.parse_args() if len(args) > 1: raise ValueError("Too many arguments passed: {0}, expected: 1".format( len(args))) elif len(args) == 0: raise ValueError( "Please specify one of following modes: {0}".format(ALLOWED_MODES)) if args[0].lower() not in ALLOWED_MODES: raise ValueError("Unknown mode: '{0}', allowed: {1}".format( args[0], ALLOWED_MODES)) ctx.iterate_mode = args[0].lower() try: if options.groups: ctx.groups = map(int, options.groups.split(',')) else: ctx.groups = [] except Exception as e: raise ValueError("Can't parse grouplist: '{0}': {1}".format( options.groups, repr(e))) print("Using group list: {0}".format(ctx.groups)) try: ctx.log_file = options.log ctx.log_level = int(options.log_level) except Exception as e: raise ValueError("Can't parse log_level: '{0}': {1}".format( options.log_level, repr(e))) print("Using elliptics client log level: {0}".format(ctx.log_level)) if not options.remote: raise ValueError( "Please specify at least one remote address (-r option)") try: ctx.remotes = [] for r in options.remote: ctx.remotes.append(elliptics.Address.from_host_port_family(r)) print("Using remote host:port:family: {0}".format(ctx.remotes[-1])) except Exception as e: raise ValueError("Can't parse host:port:family: '{0}': {1}".format( options.remote, repr(e))) ctx.no_meta = options.no_meta try: if options.time_begin: ctx.time_begin = Time.from_epoch(options.time_begin) else: ctx.time_begin = None except Exception as e: raise ValueError("Can't parse timestamp: '{0}': {1}".format( options.timestamp, repr(e))) print("Using time_begin: {0}".format(ctx.time_begin)) try: if options.time_end: ctx.time_end = Time.from_epoch(options.time_end) else: ctx.time_end = None except Exception as e: raise ValueError("Can't parse timestamp: '{0}': {1}".format( options.timestamp, repr(e))) print("Using time_end: {0}".format(ctx.time_end)) ctx.data = options.data key_range = elliptics.IteratorRange() try: if options.key_begin == '-1': key_range.key_begin = elliptics.Id([255] * 64, 0) elif options.key_begin: key_range.key_begin = elliptics.Id(transf(options.key_begin), 0) except Exception as e: raise ValueError("Can't parse key_begin: '{0}': {1}".format( options.key_begin, repr(e))) try: if options.key_end == '-1': key_range.key_end = elliptics.Id([255] * 64, 0) elif options.key_end: key_range.key_end = elliptics.Id(transf(options.key_end), 0) except Exception as e: raise ValueError("Can't parse key_end: '{0}': {1}".format( options.key_end, repr(e))) ctx.ranges = [] if options.key_begin or options.key_end: ctx.ranges = [key_range] try: if options.route_file and options.route_addr: ranges = parse_route_ranges(options.route_file, options.route_addr) ctx.ranges += ranges except Exception as e: raise ValueError( "Can't parse route_file '{0}' and route_addr '{1}' options: {2}". format(options.route_file, options.route_addr, repr(e))) return ctx
def make_range(begin, end): range = elliptics.IteratorRange() range.key_begin = begin range.key_end = end return range
def range(key_begin, key_end): ret = elliptics.IteratorRange() ret.key_begin = key_begin ret.key_end = key_end return ret
def parse_args(): from optparse import OptionParser ctx = Ctx() parser = OptionParser() parser.usage = "%prog type [options]" parser.description = __doc__ parser.add_option("-g", "--groups", action="store", dest="groups", default=None, help="Comma separated list of groups [default: all]") parser.add_option("-l", "--log", dest="log", default='/dev/stderr', metavar="FILE", help="Output log messages from library to file [default: %default]") parser.add_option("-L", "--log-level", action="store", dest="log_level", default="1", help="Elliptics client verbosity [default: %default]") parser.add_option("-r", "--remote", action="append", dest="remote", help="Elliptics node address [default: %default]") parser.add_option("-d", "--data", action="store_true", dest="data", default=False, help="Requests object's data with other info [default: %default]") parser.add_option("-k", "--key-begin", action="store", dest="key_begin", default="0", help="Begin key of range for iterating") parser.add_option("-K", "--key-end", action="store", dest="key_end", default="-1", help="End key of range for iterating") parser.add_option("-t", "--time-begin", action="store", dest="time_begin", default=None, help="Begin timestamp of time range for iterating") parser.add_option("-T", "--time-end", action="store", dest="time_end", default=None, help="End timestamp of time range for iterating") (options, args) = parser.parse_args() if len(args) > 1: raise ValueError("Too many arguments passed: {0}, expected: 1" .format(len(args))) elif len(args) == 0: raise ValueError("Please specify one of following modes: {0}" .format(ALLOWED_MODES)) if args[0].lower() not in ALLOWED_MODES: raise ValueError("Unknown mode: '{0}', allowed: {1}" .format(args[0], ALLOWED_MODES)) ctx.iterate_mode = args[0].lower() try: if options.groups: ctx.groups = map(int, options.groups.split(',')) else: ctx.groups = [] except Exception as e: raise ValueError("Can't parse grouplist: '{0}': {1}".format( options.groups, repr(e))) print("Using group list: {0}".format(ctx.groups)) try: ctx.log_file = options.log ctx.log_level = int(options.log_level) except Exception as e: raise ValueError("Can't parse log_level: '{0}': {1}" .format(options.log_level, repr(e))) print("Using elliptics client log level: {0}".format(ctx.log_level)) if not options.remote: raise ValueError("Please specify at least one remote address (-r option)") try: ctx.remotes = [] for r in options.remote: ctx.remotes.append(elliptics.Address.from_host_port_family(r)) print("Using remote host:port:family: {0}".format(ctx.remotes[-1])) except Exception as e: raise ValueError("Can't parse host:port:family: '{0}': {1}" .format(options.remote, repr(e))) try: if options.time_begin: ctx.time_begin = Time.from_epoch(options.time_begin) else: ctx.time_begin = None except Exception: raise ValueError("Can't parse timestamp: '{0}': {1}" .format(options.timestamp, repr(e))) print("Using time_begin: {0}".format(ctx.time_begin)) try: if options.time_end: ctx.time_end = Time.from_epoch(options.time_end) else: ctx.time_end = None except Exception: raise ValueError("Can't parse timestamp: '{0}': {1}" .format(options.timestamp, repr(e))) print("Using time_end: {0}".format(ctx.time_end)) ctx.data = options.data key_range = elliptics.IteratorRange() try: if options.key_begin == '-1': key_range.key_begin = elliptics.Id([255] * 64, 0) elif options.key_begin: key_range.key_begin = elliptics.Id(transf(options.key_begin), 0) else: key_range.key_begin = elliptics.Id([0] * 64, 0) except Exception: raise ValueError("Can't parse key_begin: '{0}': {1}" .format(options.key_begin, repr(e))) try: if options.key_end == '-1': key_range.key_end = elliptics.Id([255] * 64, 0) elif options.key_end: key_range.key_end = elliptics.Id(transf(options.key_end), 0) else: key_range.key_end = elliptics.Id([255] * 64, 0) except Exception: raise ValueError("Can't parse key_end: '{0}': {1}" .format(options.key_end, repr(e))) ctx.ranges = [key_range] return ctx