def update_progress(progress_info, options):
rows_done = 0
total_rows = 1
for current, max_count in progress_info:
curr_val = current.value
max_val = max_count.value
if curr_val < 0:
# There is a table that hasn't finished counting yet, we can't report progress
rows_done = 0
break
else:
rows_done += curr_val
total_rows += max_val
if not options.quiet:
utils_common.print_progress(float(rows_done) / total_rows, indent=4)
def run_clients(options, workingDir, db_table_set):
# Spawn one client for each db.table, up to options.clients at a time
exit_event = multiprocessing.Event()
processes = []
if six.PY3:
ctx = multiprocessing.get_context(multiprocessing.get_start_method())
error_queue = SimpleQueue(ctx=ctx)
else:
error_queue = SimpleQueue()
interrupt_event = multiprocessing.Event()
sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0)
hook_counter = multiprocessing.Value(ctypes.c_longlong, 0)
signal.signal(signal.SIGINT,
lambda a, b: abort_export(a, b, exit_event, interrupt_event))
errors = []
try:
progress_info = []
arg_lists = []
for db, table in db_table_set:
tableSize = int(
options.retryQuery(
"count",
query.db(db).table(table).info()
['doc_count_estimates'].sum()))
progress_info.append(
(multiprocessing.Value(ctypes.c_longlong, 0),
multiprocessing.Value(ctypes.c_longlong, tableSize)))
arg_lists.append((
db,
table,
workingDir,
options,
error_queue,
progress_info[-1],
sindex_counter,
hook_counter,
exit_event,
))
# Wait for all tables to finish
while processes or arg_lists:
time.sleep(0.1)
while not error_queue.empty():
exit_event.set() # Stop immediately if an error occurs
errors.append(error_queue.get())
processes = [
process for process in processes if process.is_alive()
]
if len(processes) < options.clients and len(arg_lists) > 0:
new_process = multiprocessing.Process(target=export_table,
args=arg_lists.pop(0))
new_process.start()
processes.append(new_process)
update_progress(progress_info, options)
# If we were successful, make sure 100% progress is reported
# (rows could have been deleted which would result in being done at less than 100%)
if len(errors
) == 0 and not interrupt_event.is_set() and not options.quiet:
utils_common.print_progress(1.0, indent=4)
# Continue past the progress output line and print total rows processed
def plural(num, text, plural_text):
return "%d %s" % (num, text if num == 1 else plural_text)
if not options.quiet:
print(
"\n %s exported from %s, with %s, and %s" %
(plural(sum([max(0, info[0].value)
for info in progress_info]), "row",
"rows"), plural(len(db_table_set), "table", "tables"),
plural(sindex_counter.value, "secondary index",
"secondary indexes"),
plural(hook_counter.value, "hook function",
"hook functions")))
finally:
signal.signal(signal.SIGINT, signal.SIG_DFL)
if interrupt_event.is_set():
raise RuntimeError("Interrupted")
if len(errors) != 0:
# multiprocessing queues don't handle tracebacks, so they've already been stringified in the queue
for error in errors:
print("%s" % error[1], file=sys.stderr)
if options.debug:
print("%s traceback: %s" % (error[0].__name__, error[2]),
file=sys.stderr)
raise RuntimeError("Errors occurred during export")
def rebuild_indexes(options):
# flesh out options.db_table
if not options.db_table:
options.db_table = [
utils_common.DbTable(x["db"], x["name"])
for x in options.retryQuery(
"all tables",
query.db("rethinkdb").table("table_config").pluck(["db", "name"]),
)
]
else:
for db_table in options.db_table[:]: # work from a copy
if not db_table[1]:
options.db_table += [
utils_common.DbTable(db_table[0], x)
for x in options.retryQuery(
"table list of %s" % db_table[0],
query.db(db_table[0]).table_list(),
)
]
del options.db_table[db_table]
# wipe out any indexes with the TMP_INDEX_PREFIX
for db, table in options.db_table:
for index in options.retryQuery(
"list indexes on %s.%s" % (db, table),
query.db(db).table(table).index_list(),
):
if index.startswith(TMP_INDEX_PREFIX):
options.retryQuery(
"drop index: %s.%s:%s" % (db, table, index),
query.db(index["db"])
.table(index["table"])
.index_drop(index["name"]),
)
# get the list of indexes to rebuild
indexes_to_build = []
for db, table in options.db_table:
indexes = None
if not options.force:
indexes = options.retryQuery(
"get outdated indexes from %s.%s" % (db, table),
query.db(db)
.table(table)
.index_status()
.filter({"outdated": True})
.get_field("index"),
)
else:
indexes = options.retryQuery(
"get all indexes from %s.%s" % (db, table),
query.db(db).table(table).index_status().get_field("index"),
)
for index in indexes:
indexes_to_build.append({"db": db, "table": table, "name": index})
# rebuild selected indexes
total_indexes = len(indexes_to_build)
indexes_completed = 0
progress_ratio = 0.0
highest_progress = 0.0
indexes_in_progress = []
if not options.quiet:
print(
"Rebuilding %d index%s: %s"
% (
total_indexes,
"es" if total_indexes > 1 else "",
", ".join(
["`%(db)s.%(table)s:%(name)s`" % i for i in indexes_to_build]
),
)
)
while len(indexes_to_build) > 0 or len(indexes_in_progress) > 0:
# Make sure we're running the right number of concurrent index rebuilds
while (
len(indexes_to_build) > 0 and len(indexes_in_progress) < options.concurrent
):
index = indexes_to_build.pop()
indexes_in_progress.append(index)
index["temp_name"] = TMP_INDEX_PREFIX + index["name"]
index["progress"] = 0
index["ready"] = False
existing_indexes = dict(
(x["index"], x["function"])
for x in options.retryQuery(
"existing indexes",
query.db(index["db"])
.table(index["table"])
.index_status()
.pluck("index", "function"),
)
)
if index["name"] not in existing_indexes:
raise AssertionError(
"{index_name} is not part of existing indexes {indexes}".format(
index_name=index["name"], indexes=", ".join(existing_indexes)
)
)
if index["temp_name"] not in existing_indexes:
options.retryQuery(
"create temp index: %(db)s.%(table)s:%(name)s" % index,
query.db(index["db"])
.table(index["table"])
.index_create(index["temp_name"], existing_indexes[index["name"]]),
)
# Report progress
highest_progress = max(highest_progress, progress_ratio)
if not options.quiet:
utils_common.print_progress(highest_progress)
# Check the status of indexes in progress
progress_ratio = 0.0
for index in indexes_in_progress:
status = options.retryQuery(
"progress `%(db)s.%(table)s` index `%(name)s`" % index,
query.db(index["db"])
.table(index["table"])
.index_status(index["temp_name"])
.nth(0),
)
if status["ready"]:
index["ready"] = True
options.retryQuery(
"rename `%(db)s.%(table)s` index `%(name)s`" % index,
query.db(index["db"])
.table(index["table"])
.index_rename(index["temp_name"], index["name"], overwrite=True),
)
else:
progress_ratio += status.get("progress", 0) / total_indexes
indexes_in_progress = [
index for index in indexes_in_progress if not index["ready"]
]
indexes_completed = (
total_indexes - len(indexes_to_build) - len(indexes_in_progress)
)
progress_ratio += float(indexes_completed) / total_indexes
if len(indexes_in_progress) == options.concurrent or (
len(indexes_in_progress) > 0 and len(indexes_to_build) == 0
):
# Short sleep to keep from killing the CPU
time.sleep(0.1)
# Make sure the progress bar says we're done and get past the progress bar line
if not options.quiet:
utils_common.print_progress(1.0)
print("")
def rebuild_indexes(options):
# flesh out options.db_table
if not options.db_table:
options.db_table = [
utils_common.DbTable(x['db'], x['name'])
for x in options.retryQuery(
'all tables',
query.db('rethinkdb').table('table_config').pluck(
['db', 'name']))
]
else:
for db_table in options.db_table[:]: # work from a copy
if not db_table[1]:
options.db_table += [
utils_common.DbTable(db_table[0], x)
for x in options.retryQuery(
'table list of %s' % db_table[0],
query.db(db_table[0]).table_list())
]
del options.db_table[db_table]
# wipe out any indexes with the TMP_INDEX_PREFIX
for db, table in options.db_table:
for index in options.retryQuery(
'list indexes on %s.%s' % (db, table),
query.db(db).table(table).index_list()):
if index.startswith(TMP_INDEX_PREFIX):
options.retryQuery(
'drop index: %s.%s:%s' % (db, table, index),
query.db(index['db']).table(index['table']).index_drop(
index['name']))
# get the list of indexes to rebuild
indexes_to_build = []
for db, table in options.db_table:
indexes = None
if not options.force:
indexes = options.retryQuery(
'get outdated indexes from %s.%s' % (db, table),
query.db(db).table(table).index_status().filter({
'outdated':
True
}).get_field('index'))
else:
indexes = options.retryQuery(
'get all indexes from %s.%s' % (db, table),
query.db(db).table(table).index_status().get_field('index'))
for index in indexes:
indexes_to_build.append({'db': db, 'table': table, 'name': index})
# rebuild selected indexes
total_indexes = len(indexes_to_build)
indexes_completed = 0
progress_ratio = 0.0
highest_progress = 0.0
indexes_in_progress = []
if not options.quiet:
print(
"Rebuilding %d index%s: %s" %
(total_indexes, 'es' if total_indexes > 1 else '', ", ".join(
["`%(db)s.%(table)s:%(name)s`" % i
for i in indexes_to_build])))
while len(indexes_to_build) > 0 or len(indexes_in_progress) > 0:
# Make sure we're running the right number of concurrent index rebuilds
while len(indexes_to_build) > 0 and len(
indexes_in_progress) < options.concurrent:
index = indexes_to_build.pop()
indexes_in_progress.append(index)
index['temp_name'] = TMP_INDEX_PREFIX + index['name']
index['progress'] = 0
index['ready'] = False
existing_indexes = dict(
(x['index'], x['function']) for x in options.retryQuery(
'existing indexes',
query.db(index['db']).table(index['table']).index_status().
pluck('index', 'function')))
if index['name'] not in existing_indexes:
raise AssertionError(
'{index_name} is not part of existing indexes {indexes}'.
format(index_name=index['name'],
indexes=', '.join(existing_indexes)))
if index['temp_name'] not in existing_indexes:
options.retryQuery(
'create temp index: %(db)s.%(table)s:%(name)s' % index,
query.db(index['db']).table(index['table']).index_create(
index['temp_name'], existing_indexes[index['name']]))
# Report progress
highest_progress = max(highest_progress, progress_ratio)
if not options.quiet:
utils_common.print_progress(highest_progress)
# Check the status of indexes in progress
progress_ratio = 0.0
for index in indexes_in_progress:
status = options.retryQuery(
"progress `%(db)s.%(table)s` index `%(name)s`" % index,
query.db(index['db']).table(index['table']).index_status(
index['temp_name']).nth(0))
if status['ready']:
index['ready'] = True
options.retryQuery(
"rename `%(db)s.%(table)s` index `%(name)s`" % index,
query.db(index['db']).table(index['table']).index_rename(
index['temp_name'], index['name'], overwrite=True))
else:
progress_ratio += status.get('progress', 0) / total_indexes
indexes_in_progress = [
index for index in indexes_in_progress if not index['ready']
]
indexes_completed = total_indexes - len(indexes_to_build) - len(
indexes_in_progress)
progress_ratio += float(indexes_completed) / total_indexes
if len(indexes_in_progress) == options.concurrent or \
(len(indexes_in_progress) > 0 and len(indexes_to_build) == 0):
# Short sleep to keep from killing the CPU
time.sleep(0.1)
# Make sure the progress bar says we're done and get past the progress bar line
if not options.quiet:
utils_common.print_progress(1.0)
print("")