try:
if options.elliptics_groups:
ctx.groups = map(int, options.elliptics_groups.split(','))
else:
ctx.groups = None
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}, traceback: {2}"
.format(options.elliptics_groups, repr(e), traceback.format_exc()))
if not ctx.groups:
raise ValueError("No group was specified")
log.info("Using group list: {0}".format(ctx.groups))
try:
if options.timestamp is None:
options.timestamp = 0
ctx.timestamp = Time.from_epoch(options.timestamp)
except Exception:
try:
ctx.timestamp = Time.from_string(options.timestamp)
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}, traceback: {2}"
.format(options.timestamp, repr(e), traceback.format_exc()))
log.info("Using timestamp: {0}".format(ctx.timestamp))
try:
ctx.batch_size = int(options.batch_size)
if ctx.batch_size <= 0:
raise ValueError("Batch size should be positive: {0}".format(ctx.batch_size))
except Exception as e:
raise ValueError("Can't parse batchsize: '{0}': {1}, traceback: {2}"
.format(options.batch_size, repr(e), traceback.format_exc()))
if options.elliptics_groups:
ctx.groups = map(int, options.elliptics_groups.split(','))
else:
ctx.groups = None
except Exception as e:
raise ValueError(
"Can't parse grouplist: '{0}': {1}, traceback: {2}".format(
options.elliptics_groups, repr(e), traceback.format_exc()))
if not ctx.groups:
raise ValueError("No group was specified")
log.info("Using group list: {0}".format(ctx.groups))
try:
if options.timestamp is None:
options.timestamp = 0
ctx.timestamp = Time.from_epoch(options.timestamp)
except Exception:
try:
ctx.timestamp = Time.from_string(options.timestamp)
except Exception as e:
raise ValueError(
"Can't parse timestamp: '{0}': {1}, traceback: {2}".format(
options.timestamp, repr(e), traceback.format_exc()))
log.info("Using timestamp: {0}".format(ctx.timestamp))
try:
ctx.batch_size = int(options.batch_size)
if ctx.batch_size <= 0:
raise ValueError("Batch size should be positive: {0}".format(
ctx.batch_size))
except Exception as e:
def parse_args():
from optparse import OptionParser
ctx = Ctx()
parser = OptionParser()
parser.usage = "%prog type [options]"
parser.description = __doc__
parser.add_option("-g",
"--groups",
action="store",
dest="groups",
default=None,
help="Comma separated list of groups [default: all]")
parser.add_option(
"-l",
"--log",
dest="log",
default='/dev/stderr',
metavar="FILE",
help="Output log messages from library to file [default: %default]")
parser.add_option("-L",
"--log-level",
action="store",
dest="log_level",
default="1",
help="Elliptics client verbosity [default: %default]")
parser.add_option("-r",
"--remote",
action="append",
dest="remote",
help="Elliptics node address [default: %default]")
parser.add_option(
"-d",
"--data",
action="store_true",
dest="data",
default=False,
help="Requests object's data with other info [default: %default]")
parser.add_option("-k",
"--key-begin",
action="store",
dest="key_begin",
default=None,
help="Begin key of range for iterating")
parser.add_option("-K",
"--key-end",
action="store",
dest="key_end",
default=None,
help="End key of range for iterating")
parser.add_option("-t",
"--time-begin",
action="store",
dest="time_begin",
default=None,
help="Begin timestamp of time range for iterating")
parser.add_option("-T",
"--time-end",
action="store",
dest="time_end",
default=None,
help="End timestamp of time range for iterating")
parser.add_option(
"-M",
"--no-meta",
action="store_true",
dest="no_meta",
default=False,
help=
"Run iterator without metadata (timestamp and user flags). This option conflicts with --data and --time-*, if one of them is specified, --no-meta will have no effect."
)
parser.add_option(
"-A",
"--addr",
action="store",
dest="route_addr",
default=None,
help=
"Address to lookup in route file. This address will be used to determine iterator ranges - ranges which DO NOT belong to selected node."
)
parser.add_option(
"-R",
"--route-file",
action="store",
dest="route_file",
default=None,
help=
"Route file contains 'dnet_balance' tool's output - route table dump, which will be parsed to find out ranges which DO NOT belong to selected address."
)
(options, args) = parser.parse_args()
if len(args) > 1:
raise ValueError("Too many arguments passed: {0}, expected: 1".format(
len(args)))
elif len(args) == 0:
raise ValueError(
"Please specify one of following modes: {0}".format(ALLOWED_MODES))
if args[0].lower() not in ALLOWED_MODES:
raise ValueError("Unknown mode: '{0}', allowed: {1}".format(
args[0], ALLOWED_MODES))
ctx.iterate_mode = args[0].lower()
try:
if options.groups:
ctx.groups = map(int, options.groups.split(','))
else:
ctx.groups = []
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}".format(
options.groups, repr(e)))
print("Using group list: {0}".format(ctx.groups))
try:
ctx.log_file = options.log
ctx.log_level = int(options.log_level)
except Exception as e:
raise ValueError("Can't parse log_level: '{0}': {1}".format(
options.log_level, repr(e)))
print("Using elliptics client log level: {0}".format(ctx.log_level))
if not options.remote:
raise ValueError(
"Please specify at least one remote address (-r option)")
try:
ctx.remotes = []
for r in options.remote:
ctx.remotes.append(elliptics.Address.from_host_port_family(r))
print("Using remote host:port:family: {0}".format(ctx.remotes[-1]))
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}".format(
options.remote, repr(e)))
ctx.no_meta = options.no_meta
try:
if options.time_begin:
ctx.time_begin = Time.from_epoch(options.time_begin)
else:
ctx.time_begin = None
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}".format(
options.timestamp, repr(e)))
print("Using time_begin: {0}".format(ctx.time_begin))
try:
if options.time_end:
ctx.time_end = Time.from_epoch(options.time_end)
else:
ctx.time_end = None
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}".format(
options.timestamp, repr(e)))
print("Using time_end: {0}".format(ctx.time_end))
ctx.data = options.data
key_range = elliptics.IteratorRange()
try:
if options.key_begin == '-1':
key_range.key_begin = elliptics.Id([255] * 64, 0)
elif options.key_begin:
key_range.key_begin = elliptics.Id(transf(options.key_begin), 0)
except Exception as e:
raise ValueError("Can't parse key_begin: '{0}': {1}".format(
options.key_begin, repr(e)))
try:
if options.key_end == '-1':
key_range.key_end = elliptics.Id([255] * 64, 0)
elif options.key_end:
key_range.key_end = elliptics.Id(transf(options.key_end), 0)
except Exception as e:
raise ValueError("Can't parse key_end: '{0}': {1}".format(
options.key_end, repr(e)))
ctx.ranges = []
if options.key_begin or options.key_end:
ctx.ranges = [key_range]
try:
if options.route_file and options.route_addr:
ranges = parse_route_ranges(options.route_file, options.route_addr)
ctx.ranges += ranges
except Exception as e:
raise ValueError(
"Can't parse route_file '{0}' and route_addr '{1}' options: {2}".
format(options.route_file, options.route_addr, repr(e)))
return ctx
def recovery(one_node, remotes, backend_id, address, groups, session, rtype, log_file, tmp_dir):
'''
Imports dnet_recovery tools and executes merge recovery. Checks result of merge.
'''
from elliptics_recovery.ctx import Ctx
from elliptics_recovery.route import RouteList
from elliptics_recovery.monitor import Monitor
from elliptics_recovery.etime import Time
import os
if rtype == RECOVERY.MERGE:
from elliptics_recovery.types.merge import main
elif rtype == RECOVERY.DC:
from elliptics_recovery.types.dc import main
else:
assert 0
ctx = Ctx()
cur_dir = os.getcwd()
ctx.tmp_dir = os.path.join(cur_dir, tmp_dir)
try:
os.makedirs(ctx.tmp_dir, 0755)
except: pass
ctx.log_file = os.path.join(ctx.tmp_dir, 'recovery.log')
import logging
import logging.handlers
log = logging.getLogger()
log.setLevel(logging.DEBUG)
formatter = logging.Formatter(fmt='%(asctime)-15s %(thread)d/%(process)d %(processName)s %(levelname)s %(message)s',
datefmt='%d %b %y %H:%M:%S')
ch = logging.FileHandler(ctx.log_file)
ch.setFormatter(formatter)
ch.setLevel(logging.DEBUG)
log.addHandler(ch)
ctx.dry_run = False
ctx.safe = False
ctx.one_node = one_node
ctx.custom_recover = ''
ctx.dump_file = None
ctx.chunk_size = 1024
ctx.log_level = 4
ctx.remotes = remotes
ctx.backend_id = backend_id
ctx.address = address
ctx.groups = groups
ctx.batch_size = 100
ctx.nprocess = 3
ctx.attempts = 1
ctx.monitor_port = None
ctx.wait_timeout = 36000
ctx.elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
ctx.routes = RouteList.from_session(session)
ctx.monitor = Monitor(ctx, None)
ctx.timestamp = Time.from_epoch(0)
recovery_res = main(ctx)
assert recovery_res
ctx.monitor.shutdown()
def main(options, args):
if len(args) > 1:
raise ValueError("Too many arguments passed: {0}, expected: 1".format(len(args)))
elif len(args) == 0:
raise ValueError("Please specify one of following types: {0}".format(ALLOWED_TYPES))
if args[0].lower() not in ALLOWED_TYPES:
raise ValueError("Unknown type: '{0}', allowed: {1}".format(args[0], ALLOWED_TYPES))
recovery_type = args[0].lower()
log.info("Initializing context")
ctx = Ctx()
ctx.dry_run = options.dry_run
ctx.safe = options.safe
ctx.one_node = bool(options.one_node)
ctx.custom_recover = options.custom_recover
ctx.no_meta = options.no_meta and (options.timestamp is None)
ctx.no_server_send = options.no_server_send
ctx.user_flags_set = frozenset(int(user_flags) for user_flags in options.user_flags_set)
try:
ctx.trace_id = int(options.trace_id, 16)
except Exception as e:
raise ValueError("Can't parse -T/--trace-id: '{0}': {1}, traceback: {2}"
.format(options.trace_id, repr(e), traceback.format_exc()))
if ctx.custom_recover:
ctx.custom_recover = os.path.abspath(ctx.custom_recover)
if options.dump_file:
ctx.dump_file = os.path.abspath(options.dump_file)
if not os.path.exists(ctx.dump_file):
raise ValueError('Dump file:`{}` does not exists'.format(options.dump_file))
else:
ctx.dump_file = None
try:
ctx.data_flow_rate = int(options.data_flow_rate) * 1024 * 1024
except Exception as e:
raise ValueError("Can't parse data_flow_rate: '{0}': {1}"
.format(options.data_flow_rate), repr(e))
try:
ctx.chunk_size = int(options.chunk_size)
except Exception as e:
raise ValueError("Can't parse chunk_size: '{0}': {1}, traceback: {2}"
.format(options.chunk_size, repr(e), traceback.format_exc()))
ctx.tmp_dir = options.tmp_dir.replace('%TYPE%', recovery_type)
if not os.path.exists(ctx.tmp_dir):
try:
os.makedirs(ctx.tmp_dir, 0755)
log.warning("Created tmp directory: {0}".format(ctx.tmp_dir))
except Exception as e:
raise ValueError("Directory: {0} does not exist and could not be created: {1}, traceback: {2}"
.format(ctx.tmp_dir, repr(e), traceback.format_exc()))
init_dir = os.getcwd()
os.chdir(ctx.tmp_dir)
log.info("Using tmp directory: {0}".format(ctx.tmp_dir))
try:
ctx.log_file = os.path.join(ctx.tmp_dir, options.elliptics_log)
try:
ctx.log_level = int(options.elliptics_log_level)
except:
ctx.log_level = options.elliptics_log_level
if isinstance(ctx.log_level, int):
ctx.log_level = elliptics.log_level.values[ctx.log_level]
else:
ctx.log_level = elliptics.log_level.names[ctx.log_level]
ctx.dump_keys = options.dump_keys
# FIXME: It may be inappropriate to use one log for both
# elliptics library and python app, esp. in presence of auto-rotation
fh = logging.handlers.WatchedFileHandler(ctx.log_file)
fh.setFormatter(formatter)
fh.setLevel(convert_elliptics_log_level(ctx.log_level))
log.addHandler(fh)
log.setLevel(convert_elliptics_log_level(ctx.log_level))
if options.debug:
ch.setLevel(logging.DEBUG)
log.setLevel(logging.DEBUG)
except Exception as e:
raise ValueError("Can't parse log_level: '{0}': {1}, traceback: {2}"
.format(options.elliptics_log_level, repr(e), traceback.format_exc()))
log.info("Using elliptics client log level: {0}".format(ctx.log_level))
try:
if options.lock:
import fcntl
ctx.lockfd = os.open(os.path.join(ctx.tmp_dir, options.lock), os.O_TRUNC | os.O_CREAT | os.O_RDWR)
fcntl.flock(ctx.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
log.info("Using lock file: {0}".format(options.lock))
else:
log.info("Not using lock file")
except Exception as e:
raise RuntimeError("Can't grab lock on: '{0}': {1}, traceback: {2}"
.format(options.lock, repr(e), traceback.format_exc()))
if not options.remotes and not ctx.one_node:
raise ValueError("At least one Elliptics node address should be given (-r option or -o).")
ctx.remotes = []
for remote in options.remotes:
try:
ctx.remotes.append(elliptics.Address.from_host_port_family(remote))
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}, traceback: {2}"
.format(remote, repr(e), traceback.format_exc()))
ctx.backend_id = None
# if one_node mode is on then use his address as a main otherwise use first from remotes
if ctx.one_node:
try:
ctx.address = elliptics.Address.from_host_port_family(options.one_node)
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}, traceback: {2}"
.format(remote, repr(e), traceback.format_exc()))
try:
if options.backend_id is not None:
ctx.backend_id = int(options.backend_id)
except Exception as e:
raise ValueError("Can't parse backend_id: '{0}': {1}, traceback: {2}"
.format(options.backend_id, repr(e), traceback.format_exc()))
elif ctx.remotes:
ctx.address = ctx.remotes[0]
log.info("Using host:port:family: {0}".format(ctx.address))
try:
if options.elliptics_groups:
ctx.groups = map(int, options.elliptics_groups.split(','))
else:
ctx.groups = None
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}, traceback: {2}"
.format(options.elliptics_groups, repr(e), traceback.format_exc()))
if not ctx.groups:
raise ValueError("No group was specified")
log.info("Using group list: {0}".format(ctx.groups))
try:
if options.timestamp is None:
options.timestamp = 0
ctx.timestamp = Time.from_epoch(options.timestamp)
except Exception:
try:
ctx.timestamp = Time.from_string(options.timestamp)
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}, traceback: {2}"
.format(options.timestamp, repr(e), traceback.format_exc()))
log.info("Using timestamp: {0}".format(ctx.timestamp))
try:
ctx.batch_size = int(options.batch_size)
if ctx.batch_size <= 0:
raise ValueError("Batch size should be positive: {0}".format(ctx.batch_size))
except Exception as e:
raise ValueError("Can't parse batchsize: '{0}': {1}, traceback: {2}"
.format(options.batch_size, repr(e), traceback.format_exc()))
log.info("Using batch_size: {0}".format(ctx.batch_size))
try:
ctx.nprocess = int(options.nprocess)
if ctx.nprocess <= 0:
raise ValueError("Number of processes should be positive: {0}".format(ctx.nprocess))
except Exception as e:
raise ValueError("Can't parse nprocess: '{0}': {1}, traceback: {2}"
.format(options.nprocess, repr(e), traceback.format_exc()))
try:
ctx.attempts = int(options.attempts)
if ctx.attempts <= 0:
raise ValueError("Number of attempts should be positive: {0}".format(ctx.attempts))
except Exception as e:
raise ValueError("Can't parse attempts: '{0}': {1}, traceback: {2}"
.format(options.attempts, repr(e), traceback.format_exc()))
ctx.stat_format = options.stat
if ctx.stat_format not in ALLOWED_STAT_FORMATS:
raise ValueError("Unknown statistics output format: '{0}'. Available formats are: {1}"
.format(ctx.stat_format, ALLOWED_STAT_FORMATS))
try:
ctx.monitor_port = int(options.monitor_port)
if ctx.monitor_port:
try:
from socket import getfqdn
base_url = 'http://{0}:{1}'.format(getfqdn(), ctx.monitor_port)
log.warning("Stats can be monitored via: {0}/{1}".format(base_url, "stats.txt"))
log.warning("Log can be viewed via: {0}/{1}".format(base_url, options.elliptics_log))
except Exception:
pass
except Exception as e:
raise ValueError("Can't parse monitor_port: '{0}': {1}, traceback: {2}"
.format(options.monitor_port, repr(e), traceback.format_exc()))
try:
ctx.wait_timeout = int(options.wait_timeout)
except Exception as e:
raise ValueError("Can't parse wait_timeout: '{0}': {1}, traceback: {2}"
.format(options.wait_timeout, repr(e), traceback.format_exc()))
try:
ctx.prepare_timeout = Time.from_epoch(options.prepare_timeout)
except Exception as e:
try:
ctx.prepare_timeout = Time.from_string(options.prepare_timeout).time
except Exception as e:
raise ValueError("Can't parse prepare_timeout: '{0}': {1}, traceback: {2}"
.format(options.wait_timeout, repr(e), traceback.format_exc()))
log.info("Using timeout: {0} for uncommitted records".format(ctx.prepare_timeout))
try:
log.info("Starting cleanup...")
cleanup(ctx.tmp_dir)
except Exception as e:
log.error("Cleanup failed: {0}, traceback: {1}"
.format(repr(e), traceback.format_exc()))
ctx.corrupted_keys = open(os.path.join(ctx.tmp_dir, 'corrupted_keys'), 'w')
log.info("Initializing monitor")
monitor = Monitor(ctx, ctx.monitor_port)
ctx.stats = monitor.stats
log.debug("Using following context:\n{0}".format(ctx))
ctx.routes = get_routes(ctx)
log.debug("Parsed routing table:\n{0}".format(ctx.routes))
if not ctx.routes:
raise RuntimeError("No routes was parsed from session")
log.debug("Total routes: {0}".format(len(ctx.routes)))
if set(ctx.groups) != set(ctx.routes.groups()):
raise RuntimeError("Not all specified groups({}) are presented in route-list({})"
.format(ctx.groups, ctx.routes.groups()))
try:
log.info("Creating pool of processes: %d", ctx.nprocess)
ctx.pool = Pool(processes=ctx.nprocess, initializer=worker_init)
if recovery_type == TYPE_MERGE:
if ctx.dump_file:
from elliptics_recovery.types.merge import dump_main
result = dump_main(ctx)
else:
from elliptics_recovery.types.merge import main
result = main(ctx)
elif recovery_type == TYPE_DC:
if ctx.dump_file:
from elliptics_recovery.types.dc import dump_main
result = dump_main(ctx)
else:
from elliptics_recovery.types.dc import main
result = main(ctx)
ctx.pool.close()
ctx.pool.terminate()
ctx.pool.join()
except Exception as e:
log.error("Recovering failed: %s, traceback: %s", repr(e), traceback.format_exc())
ctx.pool.terminate()
ctx.pool.join()
result = False
rc = int(not result)
log.info("Finished with rc: {0}".format(rc))
ctx.stats.counter('result', rc)
if options.no_exit:
raw_input("Press Enter to exit!")
monitor.shutdown()
monitor.update()
monitor.print_stat()
ctx.corrupted_keys.close()
os.chdir(init_dir)
return rc
def parse_args():
from optparse import OptionParser
ctx = Ctx()
parser = OptionParser()
parser.usage = "%prog type [options]"
parser.description = __doc__
parser.add_option("-g", "--groups", action="store", dest="groups", default=None,
help="Comma separated list of groups [default: all]")
parser.add_option("-l", "--log", dest="log", default='/dev/stderr', metavar="FILE",
help="Output log messages from library to file [default: %default]")
parser.add_option("-L", "--log-level", action="store", dest="log_level", default="1",
help="Elliptics client verbosity [default: %default]")
parser.add_option("-r", "--remote", action="append", dest="remote",
help="Elliptics node address [default: %default]")
parser.add_option("-d", "--data", action="store_true", dest="data", default=False,
help="Requests object's data with other info [default: %default]")
parser.add_option("-k", "--key-begin", action="store", dest="key_begin", default="0",
help="Begin key of range for iterating")
parser.add_option("-K", "--key-end", action="store", dest="key_end", default="-1",
help="End key of range for iterating")
parser.add_option("-t", "--time-begin", action="store", dest="time_begin", default=None,
help="Begin timestamp of time range for iterating")
parser.add_option("-T", "--time-end", action="store", dest="time_end", default=None,
help="End timestamp of time range for iterating")
(options, args) = parser.parse_args()
if len(args) > 1:
raise ValueError("Too many arguments passed: {0}, expected: 1"
.format(len(args)))
elif len(args) == 0:
raise ValueError("Please specify one of following modes: {0}"
.format(ALLOWED_MODES))
if args[0].lower() not in ALLOWED_MODES:
raise ValueError("Unknown mode: '{0}', allowed: {1}"
.format(args[0], ALLOWED_MODES))
ctx.iterate_mode = args[0].lower()
try:
if options.groups:
ctx.groups = map(int, options.groups.split(','))
else:
ctx.groups = []
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}".format(
options.groups, repr(e)))
print("Using group list: {0}".format(ctx.groups))
try:
ctx.log_file = options.log
ctx.log_level = int(options.log_level)
except Exception as e:
raise ValueError("Can't parse log_level: '{0}': {1}"
.format(options.log_level, repr(e)))
print("Using elliptics client log level: {0}".format(ctx.log_level))
if not options.remote:
raise ValueError("Please specify at least one remote address (-r option)")
try:
ctx.remotes = []
for r in options.remote:
ctx.remotes.append(elliptics.Address.from_host_port_family(r))
print("Using remote host:port:family: {0}".format(ctx.remotes[-1]))
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}"
.format(options.remote, repr(e)))
try:
if options.time_begin:
ctx.time_begin = Time.from_epoch(options.time_begin)
else:
ctx.time_begin = None
except Exception:
raise ValueError("Can't parse timestamp: '{0}': {1}"
.format(options.timestamp, repr(e)))
print("Using time_begin: {0}".format(ctx.time_begin))
try:
if options.time_end:
ctx.time_end = Time.from_epoch(options.time_end)
else:
ctx.time_end = None
except Exception:
raise ValueError("Can't parse timestamp: '{0}': {1}"
.format(options.timestamp, repr(e)))
print("Using time_end: {0}".format(ctx.time_end))
ctx.data = options.data
key_range = elliptics.IteratorRange()
try:
if options.key_begin == '-1':
key_range.key_begin = elliptics.Id([255] * 64, 0)
elif options.key_begin:
key_range.key_begin = elliptics.Id(transf(options.key_begin), 0)
else:
key_range.key_begin = elliptics.Id([0] * 64, 0)
except Exception:
raise ValueError("Can't parse key_begin: '{0}': {1}"
.format(options.key_begin, repr(e)))
try:
if options.key_end == '-1':
key_range.key_end = elliptics.Id([255] * 64, 0)
elif options.key_end:
key_range.key_end = elliptics.Id(transf(options.key_end), 0)
else:
key_range.key_end = elliptics.Id([255] * 64, 0)
except Exception:
raise ValueError("Can't parse key_end: '{0}': {1}"
.format(options.key_end, repr(e)))
ctx.ranges = [key_range]
return ctx
address = Address.from_host_port_family(options.elliptics_remote)
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}".format(options.elliptics_remote,
repr(e)))
node.add_remote(addr=address.host, port=address.port, family=address.family)
session = elliptics.Session(node)
routes = RouteList.from_session(session)
eid = routes.get_address_eid(address)
session.groups = [eid.group_id]
ranges = [IdRange.elliptics_range(start, stop) for start, stop in (IdRange(IdRange.ID_MIN, IdRange.ID_MAX),)]
records = session.start_iterator(eid,
ranges,
elliptics.iterator_types.network,
elliptics.iterator_flags.key_range,
Time.time_min().to_etime(),
Time.time_max().to_etime())
total_records = 0
for record in records:
if record.status != 0:
raise RuntimeError("Iteration status check failed: {0}".format(record.status))
total_records += 1
all_ranges = routes.get_local_ranges_by_address(address)
local_ranges = next((r for r in all_ranges if r.address == address), None)
ranges = [IdRange.elliptics_range(start, stop) for start, stop in local_ranges.id_ranges]
records = session.start_iterator(routes.get_address_eid(address),
ranges,
elliptics.iterator_types.network,
if options.elliptics_groups:
ctx.groups = map(int, options.elliptics_groups.split(','))
else:
ctx.groups = None
except Exception as e:
raise ValueError(
"Can't parse grouplist: '{0}': {1}, traceback: {2}".format(
options.elliptics_groups, repr(e), traceback.format_exc()))
if not ctx.groups:
raise ValueError("No group was specified")
log.info("Using group list: {0}".format(ctx.groups))
try:
if options.timestamp is None:
options.timestamp = 0
ctx.timestamp = Time.from_epoch(options.timestamp)
except Exception:
try:
ctx.timestamp = Time.from_string(options.timestamp)
except Exception as e:
raise ValueError(
"Can't parse timestamp: '{0}': {1}, traceback: {2}".format(
options.timestamp, repr(e), traceback.format_exc()))
log.info("Using timestamp: {0}".format(ctx.timestamp))
try:
ctx.batch_size = int(options.batch_size)
if ctx.batch_size <= 0:
raise ValueError("Batch size should be positive: {0}".format(
ctx.batch_size))
except Exception as e:
try:
if options.elliptics_groups:
ctx.groups = map(int, options.elliptics_groups.split(','))
else:
ctx.groups = None
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}, traceback: {2}"
.format(options.elliptics_groups, repr(e), traceback.format_exc()))
if not ctx.groups:
raise ValueError("No group was specified")
log.info("Using group list: {0}".format(ctx.groups))
try:
if options.timestamp is None:
options.timestamp = 0
ctx.timestamp = Time.from_epoch(options.timestamp)
except Exception:
try:
ctx.timestamp = Time.from_string(options.timestamp)
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}, traceback: {2}"
.format(options.timestamp, repr(e), traceback.format_exc()))
log.info("Using timestamp: {0}".format(ctx.timestamp))
try:
ctx.batch_size = int(options.batch_size)
if ctx.batch_size <= 0:
raise ValueError("Batch size should be positive: {0}".format(ctx.batch_size))
except Exception as e:
raise ValueError("Can't parse batchsize: '{0}': {1}, traceback: {2}"
.format(options.batch_size, repr(e), traceback.format_exc()))
def parse_args():
from optparse import OptionParser
ctx = Ctx()
parser = OptionParser()
parser.usage = "%prog type [options]"
parser.description = __doc__
parser.add_option("-g", "--groups", action="store", dest="groups", default=None,
help="Comma separated list of groups [default: all]")
parser.add_option("-l", "--log", dest="log", default='/dev/stderr', metavar="FILE",
help="Output log messages from library to file [default: %default]")
parser.add_option("-L", "--log-level", action="store", dest="log_level", default="1",
help="Elliptics client verbosity [default: %default]")
parser.add_option("-r", "--remote", action="append", dest="remote",
help="Elliptics node address [default: %default]")
parser.add_option("-d", "--data", action="store_true", dest="data", default=False,
help="Requests object's data with other info [default: %default]")
parser.add_option("-k", "--key-begin", action="store", dest="key_begin", default=None,
help="Begin key of range for iterating")
parser.add_option("-K", "--key-end", action="store", dest="key_end", default=None,
help="End key of range for iterating")
parser.add_option("-t", "--time-begin", action="store", dest="time_begin", default=None,
help="Begin timestamp of time range for iterating")
parser.add_option("-T", "--time-end", action="store", dest="time_end", default=None,
help="End timestamp of time range for iterating")
parser.add_option("-M", "--no-meta", action="store_true", dest="no_meta", default=False,
help="Run iterator without metadata (timestamp and user flags). This option conflicts with --data and --time-*, if one of them is specified, --no-meta will have no effect.")
parser.add_option("-A", "--addr", action="store", dest="route_addr", default=None,
help="Address to lookup in route file. This address will be used to determine iterator ranges - ranges which DO NOT belong to selected node.")
parser.add_option("-R", "--route-file", action="store", dest="route_file", default=None,
help="Route file contains 'dnet_balance' tool's output - route table dump, which will be parsed to find out ranges which DO NOT belong to selected address.")
(options, args) = parser.parse_args()
if len(args) > 1:
raise ValueError("Too many arguments passed: {0}, expected: 1"
.format(len(args)))
elif len(args) == 0:
raise ValueError("Please specify one of following modes: {0}"
.format(ALLOWED_MODES))
if args[0].lower() not in ALLOWED_MODES:
raise ValueError("Unknown mode: '{0}', allowed: {1}"
.format(args[0], ALLOWED_MODES))
ctx.iterate_mode = args[0].lower()
try:
if options.groups:
ctx.groups = map(int, options.groups.split(','))
else:
ctx.groups = []
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}".format(
options.groups, repr(e)))
print("Using group list: {0}".format(ctx.groups))
try:
ctx.log_file = options.log
ctx.log_level = int(options.log_level)
except Exception as e:
raise ValueError("Can't parse log_level: '{0}': {1}"
.format(options.log_level, repr(e)))
print("Using elliptics client log level: {0}".format(ctx.log_level))
if not options.remote:
raise ValueError("Please specify at least one remote address (-r option)")
try:
ctx.remotes = []
for r in options.remote:
ctx.remotes.append(elliptics.Address.from_host_port_family(r))
print("Using remote host:port:family: {0}".format(ctx.remotes[-1]))
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}"
.format(options.remote, repr(e)))
ctx.no_meta = options.no_meta
try:
if options.time_begin:
ctx.time_begin = Time.from_epoch(options.time_begin)
else:
ctx.time_begin = None
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}"
.format(options.timestamp, repr(e)))
print("Using time_begin: {0}".format(ctx.time_begin))
try:
if options.time_end:
ctx.time_end = Time.from_epoch(options.time_end)
else:
ctx.time_end = None
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}"
.format(options.timestamp, repr(e)))
print("Using time_end: {0}".format(ctx.time_end))
ctx.data = options.data
key_range = elliptics.IteratorRange()
try:
if options.key_begin == '-1':
key_range.key_begin = elliptics.Id([255] * 64, 0)
elif options.key_begin:
key_range.key_begin = elliptics.Id(transf(options.key_begin), 0)
except Exception as e:
raise ValueError("Can't parse key_begin: '{0}': {1}"
.format(options.key_begin, repr(e)))
try:
if options.key_end == '-1':
key_range.key_end = elliptics.Id([255] * 64, 0)
elif options.key_end:
key_range.key_end = elliptics.Id(transf(options.key_end), 0)
except Exception as e:
raise ValueError("Can't parse key_end: '{0}': {1}"
.format(options.key_end, repr(e)))
ctx.ranges = []
if options.key_begin or options.key_end:
ctx.ranges = [key_range]
try:
if options.route_file and options.route_addr:
ranges = parse_route_ranges(options.route_file, options.route_addr)
ctx.ranges += ranges
except Exception as e:
raise ValueError("Can't parse route_file '{0}' and route_addr '{1}' options: {2}"
.format(options.route_file, options.route_addr, repr(e)))
return ctx
def main(options, args):
if len(args) > 1:
raise ValueError("Too many arguments passed: {0}, expected: 1".format(len(args)))
elif len(args) == 0:
raise ValueError("Please specify one of following types: {0}".format(ALLOWED_TYPES))
if args[0].lower() not in ALLOWED_TYPES:
raise ValueError("Unknown type: '{0}', allowed: {1}".format(args[0], ALLOWED_TYPES))
recovery_type = args[0].lower()
log.info("Initializing context")
ctx = Ctx()
ctx.dry_run = options.dry_run
ctx.safe = options.safe
ctx.one_node = bool(options.one_node)
ctx.custom_recover = options.custom_recover
ctx.no_meta = options.no_meta and (options.timestamp is None)
ctx.no_server_send = options.no_server_send
ctx.user_flags_set = frozenset(int(user_flags) for user_flags in options.user_flags_set)
try:
ctx.trace_id = int(options.trace_id, 16)
except Exception as e:
raise ValueError("Can't parse -T/--trace-id: '{0}': {1}, traceback: {2}"
.format(options.trace_id, repr(e), traceback.format_exc()))
if ctx.custom_recover:
ctx.custom_recover = os.path.abspath(ctx.custom_recover)
if options.dump_file:
ctx.dump_file = os.path.abspath(options.dump_file)
if not os.path.exists(ctx.dump_file):
raise ValueError('Dump file:`{}` does not exists'.format(options.dump_file))
else:
ctx.dump_file = None
try:
ctx.data_flow_rate = int(options.data_flow_rate) * 1024 * 1024
except Exception as e:
raise ValueError("Can't parse data_flow_rate: '{0}': {1}"
.format(options.data_flow_rate), repr(e))
try:
ctx.chunk_size = int(options.chunk_size)
except Exception as e:
raise ValueError("Can't parse chunk_size: '{0}': {1}, traceback: {2}"
.format(options.chunk_size, repr(e), traceback.format_exc()))
ctx.tmp_dir = options.tmp_dir.replace('%TYPE%', recovery_type)
if not os.path.exists(ctx.tmp_dir):
try:
os.makedirs(ctx.tmp_dir, 0755)
log.warning("Created tmp directory: {0}".format(ctx.tmp_dir))
except Exception as e:
raise ValueError("Directory: {0} does not exist and could not be created: {1}, traceback: {2}"
.format(ctx.tmp_dir, repr(e), traceback.format_exc()))
init_dir = os.getcwd()
os.chdir(ctx.tmp_dir)
log.info("Using tmp directory: {0}".format(ctx.tmp_dir))
try:
ctx.log_file = os.path.join(ctx.tmp_dir, options.elliptics_log)
try:
ctx.log_level = int(options.elliptics_log_level)
except:
ctx.log_level = options.elliptics_log_level
if isinstance(ctx.log_level, int):
ctx.log_level = elliptics.log_level.values[ctx.log_level]
else:
ctx.log_level = elliptics.log_level.names[ctx.log_level]
ctx.dump_keys = options.dump_keys
# FIXME: It may be inappropriate to use one log for both
# elliptics library and python app, esp. in presence of auto-rotation
fh = logging.FileHandler(ctx.log_file)
fh.setFormatter(formatter)
fh.setLevel(convert_elliptics_log_level(ctx.log_level))
log.addHandler(fh)
log.setLevel(convert_elliptics_log_level(ctx.log_level))
if options.debug:
ch.setLevel(logging.DEBUG)
log.setLevel(logging.DEBUG)
except Exception as e:
raise ValueError("Can't parse log_level: '{0}': {1}, traceback: {2}"
.format(options.elliptics_log_level, repr(e), traceback.format_exc()))
log.info("Using elliptics client log level: {0}".format(ctx.log_level))
try:
if options.lock:
import fcntl
ctx.lockfd = os.open(os.path.join(ctx.tmp_dir, options.lock), os.O_TRUNC | os.O_CREAT | os.O_RDWR)
fcntl.flock(ctx.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
log.info("Using lock file: {0}".format(options.lock))
else:
log.info("Not using lock file")
except Exception as e:
raise RuntimeError("Can't grab lock on: '{0}': {1}, traceback: {2}"
.format(options.lock, repr(e), traceback.format_exc()))
if not options.remotes and not ctx.one_node:
raise ValueError("At least one Elliptics node address should be given (-r option or -o).")
ctx.remotes = []
for remote in options.remotes:
try:
ctx.remotes.append(elliptics.Address.from_host_port_family(remote))
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}, traceback: {2}"
.format(remote, repr(e), traceback.format_exc()))
ctx.backend_id = None
# if one_node mode is on then use his address as a main otherwise use first from remotes
if ctx.one_node:
try:
ctx.address = elliptics.Address.from_host_port_family(options.one_node)
except Exception as e:
raise ValueError("Can't parse host:port:family: '{0}': {1}, traceback: {2}"
.format(remote, repr(e), traceback.format_exc()))
try:
if options.backend_id is not None:
ctx.backend_id = int(options.backend_id)
except Exception as e:
raise ValueError("Can't parse backend_id: '{0}': {1}, traceback: {2}"
.format(options.backend_id, repr(e), traceback.format_exc()))
elif ctx.remotes:
ctx.address = ctx.remotes[0]
log.info("Using host:port:family: {0}".format(ctx.address))
try:
if options.elliptics_groups:
ctx.groups = map(int, options.elliptics_groups.split(','))
else:
ctx.groups = None
except Exception as e:
raise ValueError("Can't parse grouplist: '{0}': {1}, traceback: {2}"
.format(options.elliptics_groups, repr(e), traceback.format_exc()))
if not ctx.groups:
raise ValueError("No group was specified")
log.info("Using group list: {0}".format(ctx.groups))
try:
if options.timestamp is None:
options.timestamp = 0
ctx.timestamp = Time.from_epoch(options.timestamp)
except Exception:
try:
ctx.timestamp = Time.from_string(options.timestamp)
except Exception as e:
raise ValueError("Can't parse timestamp: '{0}': {1}, traceback: {2}"
.format(options.timestamp, repr(e), traceback.format_exc()))
log.info("Using timestamp: {0}".format(ctx.timestamp))
try:
ctx.batch_size = int(options.batch_size)
if ctx.batch_size <= 0:
raise ValueError("Batch size should be positive: {0}".format(ctx.batch_size))
except Exception as e:
raise ValueError("Can't parse batchsize: '{0}': {1}, traceback: {2}"
.format(options.batch_size, repr(e), traceback.format_exc()))
log.info("Using batch_size: {0}".format(ctx.batch_size))
try:
ctx.nprocess = int(options.nprocess)
if ctx.nprocess <= 0:
raise ValueError("Number of processes should be positive: {0}".format(ctx.nprocess))
except Exception as e:
raise ValueError("Can't parse nprocess: '{0}': {1}, traceback: {2}"
.format(options.nprocess, repr(e), traceback.format_exc()))
try:
ctx.attempts = int(options.attempts)
if ctx.attempts <= 0:
raise ValueError("Number of attempts should be positive: {0}".format(ctx.attempts))
except Exception as e:
raise ValueError("Can't parse attempts: '{0}': {1}, traceback: {2}"
.format(options.attempts, repr(e), traceback.format_exc()))
ctx.stat_format = options.stat
if ctx.stat_format not in ALLOWED_STAT_FORMATS:
raise ValueError("Unknown statistics output format: '{0}'. Available formats are: {1}"
.format(ctx.stat_format, ALLOWED_STAT_FORMATS))
try:
ctx.monitor_port = int(options.monitor_port)
if ctx.monitor_port:
try:
from socket import getfqdn
base_url = 'http://{0}:{1}'.format(getfqdn(), ctx.monitor_port)
log.warning("Stats can be monitored via: {0}/{1}".format(base_url, "stats.txt"))
log.warning("Log can be viewed via: {0}/{1}".format(base_url, options.elliptics_log))
except Exception:
pass
except Exception as e:
raise ValueError("Can't parse monitor_port: '{0}': {1}, traceback: {2}"
.format(options.monitor_port, repr(e), traceback.format_exc()))
try:
ctx.wait_timeout = int(options.wait_timeout)
except Exception as e:
raise ValueError("Can't parse wait_timeout: '{0}': {1}, traceback: {2}"
.format(options.wait_timeout, repr(e), traceback.format_exc()))
try:
ctx.prepare_timeout = Time.from_epoch(options.prepare_timeout)
except Exception as e:
try:
ctx.prepare_timeout = Time.from_string(options.prepare_timeout).time
except Exception as e:
raise ValueError("Can't parse prepare_timeout: '{0}': {1}, traceback: {2}"
.format(options.wait_timeout, repr(e), traceback.format_exc()))
log.info("Using timeout: {0} for uncommitted records".format(ctx.prepare_timeout))
try:
log.info("Starting cleanup...")
cleanup(ctx.tmp_dir)
except Exception as e:
log.error("Cleanup failed: {0}, traceback: {1}"
.format(repr(e), traceback.format_exc()))
ctx.corrupted_keys = open(os.path.join(ctx.tmp_dir, 'corrupted_keys'), 'w')
log.info("Initializing monitor")
monitor = Monitor(ctx, ctx.monitor_port)
ctx.stats = monitor.stats
log.debug("Using following context:\n{0}".format(ctx))
ctx.routes = get_routes(ctx)
log.debug("Parsed routing table:\n{0}".format(ctx.routes))
if not ctx.routes:
raise RuntimeError("No routes was parsed from session")
log.debug("Total routes: {0}".format(len(ctx.routes)))
if len(ctx.groups) == 0:
ctx.groups = ctx.routes.groups()
log.info("No groups specified: using all available groups: {0}".format(ctx.groups))
try:
log.info("Creating pool of processes: %d", ctx.nprocess)
ctx.pool = Pool(processes=ctx.nprocess, initializer=worker_init)
if recovery_type == TYPE_MERGE:
if ctx.dump_file:
from elliptics_recovery.types.merge import dump_main
result = dump_main(ctx)
else:
from elliptics_recovery.types.merge import main
result = main(ctx)
elif recovery_type == TYPE_DC:
if ctx.dump_file:
from elliptics_recovery.types.dc import dump_main
result = dump_main(ctx)
else:
from elliptics_recovery.types.dc import main
result = main(ctx)
ctx.pool.close()
ctx.pool.terminate()
ctx.pool.join()
except Exception as e:
log.error("Recovering failed: %s, traceback: %s", repr(e), traceback.format_exc())
ctx.pool.terminate()
ctx.pool.join()
result = False
rc = int(not result)
log.info("Finished with rc: {0}".format(rc))
ctx.stats.counter('result', rc)
if options.no_exit:
raw_input("Press Enter to exit!")
monitor.shutdown()
monitor.update()
monitor.print_stat()
ctx.corrupted_keys.close()
os.chdir(init_dir)
return rc
def recovery(one_node, remotes, backend_id, address, groups, session, rtype,
log_file, tmp_dir):
'''
Imports dnet_recovery tools and executes merge recovery. Checks result of merge.
'''
from elliptics_recovery.ctx import Ctx
from elliptics_recovery.route import RouteList
from elliptics_recovery.monitor import Monitor
from elliptics_recovery.etime import Time
import os
if rtype == RECOVERY.MERGE:
from elliptics_recovery.types.merge import main
elif rtype == RECOVERY.DC:
from elliptics_recovery.types.dc import main
else:
assert 0
ctx = Ctx()
cur_dir = os.getcwd()
ctx.tmp_dir = os.path.join(cur_dir, tmp_dir)
try:
os.makedirs(ctx.tmp_dir, 0755)
except:
pass
ctx.log_file = os.path.join(ctx.tmp_dir, 'recovery.log')
import logging
import logging.handlers
log = logging.getLogger()
log.setLevel(logging.DEBUG)
formatter = logging.Formatter(
fmt=
'%(asctime)-15s %(thread)d/%(process)d %(processName)s %(levelname)s %(message)s',
datefmt='%d %b %y %H:%M:%S')
ch = logging.FileHandler(ctx.log_file)
ch.setFormatter(formatter)
ch.setLevel(logging.DEBUG)
log.addHandler(ch)
ctx.dry_run = False
ctx.safe = False
ctx.one_node = one_node
ctx.custom_recover = ''
ctx.dump_file = None
ctx.chunk_size = 1024
ctx.log_level = elliptics.log_level.debug
ctx.remotes = remotes
ctx.backend_id = backend_id
ctx.address = address
ctx.groups = groups
ctx.batch_size = 100
ctx.nprocess = 3
ctx.attempts = 1
ctx.monitor_port = None
ctx.wait_timeout = 36000
ctx.elog = elliptics.Logger(ctx.log_file, int(ctx.log_level))
ctx.routes = RouteList.from_session(session)
ctx.monitor = Monitor(ctx, None)
ctx.timestamp = Time.from_epoch(0)
recovery_res = main(ctx)
assert recovery_res
ctx.monitor.shutdown()
node.add_remote(addr=address.host,
port=address.port,
family=address.family)
session = elliptics.Session(node)
routes = RouteList.from_session(session)
eid = routes.get_address_eid(address)
session.groups = [eid.group_id]
ranges = [
IdRange.elliptics_range(start, stop)
for start, stop in (IdRange(IdRange.ID_MIN, IdRange.ID_MAX), )
]
records = session.start_iterator(eid, ranges,
elliptics.iterator_types.network,
elliptics.iterator_flags.key_range,
Time.time_min().to_etime(),
Time.time_max().to_etime())
total_records = 0
for record in records:
if record.status != 0:
raise RuntimeError("Iteration status check failed: {0}".format(
record.status))
total_records += 1
all_ranges = routes.get_local_ranges_by_address(address)
local_ranges = next((r for r in all_ranges if r.address == address), None)
ranges = [
IdRange.elliptics_range(start, stop)
for start, stop in local_ranges.id_ranges