def update_progress(progress_info, options): rows_done = 0 total_rows = 1 for current, max_count in progress_info: curr_val = current.value max_val = max_count.value if curr_val < 0: # There is a table that hasn't finished counting yet, we can't report progress rows_done = 0 break else: rows_done += curr_val total_rows += max_val if not options.quiet: utils_common.print_progress(float(rows_done) / total_rows, indent=4)
def run_clients(options, workingDir, db_table_set): # Spawn one client for each db.table, up to options.clients at a time exit_event = multiprocessing.Event() processes = [] if six.PY3: ctx = multiprocessing.get_context(multiprocessing.get_start_method()) error_queue = SimpleQueue(ctx=ctx) else: error_queue = SimpleQueue() interrupt_event = multiprocessing.Event() sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0) hook_counter = multiprocessing.Value(ctypes.c_longlong, 0) signal.signal(signal.SIGINT, lambda a, b: abort_export(a, b, exit_event, interrupt_event)) errors = [] try: progress_info = [] arg_lists = [] for db, table in db_table_set: tableSize = int( options.retryQuery( "count", query.db(db).table(table).info() ['doc_count_estimates'].sum())) progress_info.append( (multiprocessing.Value(ctypes.c_longlong, 0), multiprocessing.Value(ctypes.c_longlong, tableSize))) arg_lists.append(( db, table, workingDir, options, error_queue, progress_info[-1], sindex_counter, hook_counter, exit_event, )) # Wait for all tables to finish while processes or arg_lists: time.sleep(0.1) while not error_queue.empty(): exit_event.set() # Stop immediately if an error occurs errors.append(error_queue.get()) processes = [ process for process in processes if process.is_alive() ] if len(processes) < options.clients and len(arg_lists) > 0: new_process = multiprocessing.Process(target=export_table, args=arg_lists.pop(0)) new_process.start() processes.append(new_process) update_progress(progress_info, options) # If we were successful, make sure 100% progress is reported # (rows could have been deleted which would result in being done at less than 100%) if len(errors ) == 0 and not interrupt_event.is_set() and not options.quiet: utils_common.print_progress(1.0, indent=4) # Continue past the progress output line and print total rows processed def plural(num, text, plural_text): return "%d %s" % (num, text if num == 1 else plural_text) if not options.quiet: print( "\n %s exported from %s, with %s, and %s" % (plural(sum([max(0, info[0].value) for info in progress_info]), "row", "rows"), plural(len(db_table_set), "table", "tables"), plural(sindex_counter.value, "secondary index", "secondary indexes"), plural(hook_counter.value, "hook function", "hook functions"))) finally: signal.signal(signal.SIGINT, signal.SIG_DFL) if interrupt_event.is_set(): raise RuntimeError("Interrupted") if len(errors) != 0: # multiprocessing queues don't handle tracebacks, so they've already been stringified in the queue for error in errors: print("%s" % error[1], file=sys.stderr) if options.debug: print("%s traceback: %s" % (error[0].__name__, error[2]), file=sys.stderr) raise RuntimeError("Errors occurred during export")
def rebuild_indexes(options): # flesh out options.db_table if not options.db_table: options.db_table = [ utils_common.DbTable(x["db"], x["name"]) for x in options.retryQuery( "all tables", query.db("rethinkdb").table("table_config").pluck(["db", "name"]), ) ] else: for db_table in options.db_table[:]: # work from a copy if not db_table[1]: options.db_table += [ utils_common.DbTable(db_table[0], x) for x in options.retryQuery( "table list of %s" % db_table[0], query.db(db_table[0]).table_list(), ) ] del options.db_table[db_table] # wipe out any indexes with the TMP_INDEX_PREFIX for db, table in options.db_table: for index in options.retryQuery( "list indexes on %s.%s" % (db, table), query.db(db).table(table).index_list(), ): if index.startswith(TMP_INDEX_PREFIX): options.retryQuery( "drop index: %s.%s:%s" % (db, table, index), query.db(index["db"]) .table(index["table"]) .index_drop(index["name"]), ) # get the list of indexes to rebuild indexes_to_build = [] for db, table in options.db_table: indexes = None if not options.force: indexes = options.retryQuery( "get outdated indexes from %s.%s" % (db, table), query.db(db) .table(table) .index_status() .filter({"outdated": True}) .get_field("index"), ) else: indexes = options.retryQuery( "get all indexes from %s.%s" % (db, table), query.db(db).table(table).index_status().get_field("index"), ) for index in indexes: indexes_to_build.append({"db": db, "table": table, "name": index}) # rebuild selected indexes total_indexes = len(indexes_to_build) indexes_completed = 0 progress_ratio = 0.0 highest_progress = 0.0 indexes_in_progress = [] if not options.quiet: print( "Rebuilding %d index%s: %s" % ( total_indexes, "es" if total_indexes > 1 else "", ", ".join( ["`%(db)s.%(table)s:%(name)s`" % i for i in indexes_to_build] ), ) ) while len(indexes_to_build) > 0 or len(indexes_in_progress) > 0: # Make sure we're running the right number of concurrent index rebuilds while ( len(indexes_to_build) > 0 and len(indexes_in_progress) < options.concurrent ): index = indexes_to_build.pop() indexes_in_progress.append(index) index["temp_name"] = TMP_INDEX_PREFIX + index["name"] index["progress"] = 0 index["ready"] = False existing_indexes = dict( (x["index"], x["function"]) for x in options.retryQuery( "existing indexes", query.db(index["db"]) .table(index["table"]) .index_status() .pluck("index", "function"), ) ) if index["name"] not in existing_indexes: raise AssertionError( "{index_name} is not part of existing indexes {indexes}".format( index_name=index["name"], indexes=", ".join(existing_indexes) ) ) if index["temp_name"] not in existing_indexes: options.retryQuery( "create temp index: %(db)s.%(table)s:%(name)s" % index, query.db(index["db"]) .table(index["table"]) .index_create(index["temp_name"], existing_indexes[index["name"]]), ) # Report progress highest_progress = max(highest_progress, progress_ratio) if not options.quiet: utils_common.print_progress(highest_progress) # Check the status of indexes in progress progress_ratio = 0.0 for index in indexes_in_progress: status = options.retryQuery( "progress `%(db)s.%(table)s` index `%(name)s`" % index, query.db(index["db"]) .table(index["table"]) .index_status(index["temp_name"]) .nth(0), ) if status["ready"]: index["ready"] = True options.retryQuery( "rename `%(db)s.%(table)s` index `%(name)s`" % index, query.db(index["db"]) .table(index["table"]) .index_rename(index["temp_name"], index["name"], overwrite=True), ) else: progress_ratio += status.get("progress", 0) / total_indexes indexes_in_progress = [ index for index in indexes_in_progress if not index["ready"] ] indexes_completed = ( total_indexes - len(indexes_to_build) - len(indexes_in_progress) ) progress_ratio += float(indexes_completed) / total_indexes if len(indexes_in_progress) == options.concurrent or ( len(indexes_in_progress) > 0 and len(indexes_to_build) == 0 ): # Short sleep to keep from killing the CPU time.sleep(0.1) # Make sure the progress bar says we're done and get past the progress bar line if not options.quiet: utils_common.print_progress(1.0) print("")
def rebuild_indexes(options): # flesh out options.db_table if not options.db_table: options.db_table = [ utils_common.DbTable(x['db'], x['name']) for x in options.retryQuery( 'all tables', query.db('rethinkdb').table('table_config').pluck( ['db', 'name'])) ] else: for db_table in options.db_table[:]: # work from a copy if not db_table[1]: options.db_table += [ utils_common.DbTable(db_table[0], x) for x in options.retryQuery( 'table list of %s' % db_table[0], query.db(db_table[0]).table_list()) ] del options.db_table[db_table] # wipe out any indexes with the TMP_INDEX_PREFIX for db, table in options.db_table: for index in options.retryQuery( 'list indexes on %s.%s' % (db, table), query.db(db).table(table).index_list()): if index.startswith(TMP_INDEX_PREFIX): options.retryQuery( 'drop index: %s.%s:%s' % (db, table, index), query.db(index['db']).table(index['table']).index_drop( index['name'])) # get the list of indexes to rebuild indexes_to_build = [] for db, table in options.db_table: indexes = None if not options.force: indexes = options.retryQuery( 'get outdated indexes from %s.%s' % (db, table), query.db(db).table(table).index_status().filter({ 'outdated': True }).get_field('index')) else: indexes = options.retryQuery( 'get all indexes from %s.%s' % (db, table), query.db(db).table(table).index_status().get_field('index')) for index in indexes: indexes_to_build.append({'db': db, 'table': table, 'name': index}) # rebuild selected indexes total_indexes = len(indexes_to_build) indexes_completed = 0 progress_ratio = 0.0 highest_progress = 0.0 indexes_in_progress = [] if not options.quiet: print( "Rebuilding %d index%s: %s" % (total_indexes, 'es' if total_indexes > 1 else '', ", ".join( ["`%(db)s.%(table)s:%(name)s`" % i for i in indexes_to_build]))) while len(indexes_to_build) > 0 or len(indexes_in_progress) > 0: # Make sure we're running the right number of concurrent index rebuilds while len(indexes_to_build) > 0 and len( indexes_in_progress) < options.concurrent: index = indexes_to_build.pop() indexes_in_progress.append(index) index['temp_name'] = TMP_INDEX_PREFIX + index['name'] index['progress'] = 0 index['ready'] = False existing_indexes = dict( (x['index'], x['function']) for x in options.retryQuery( 'existing indexes', query.db(index['db']).table(index['table']).index_status(). pluck('index', 'function'))) if index['name'] not in existing_indexes: raise AssertionError( '{index_name} is not part of existing indexes {indexes}'. format(index_name=index['name'], indexes=', '.join(existing_indexes))) if index['temp_name'] not in existing_indexes: options.retryQuery( 'create temp index: %(db)s.%(table)s:%(name)s' % index, query.db(index['db']).table(index['table']).index_create( index['temp_name'], existing_indexes[index['name']])) # Report progress highest_progress = max(highest_progress, progress_ratio) if not options.quiet: utils_common.print_progress(highest_progress) # Check the status of indexes in progress progress_ratio = 0.0 for index in indexes_in_progress: status = options.retryQuery( "progress `%(db)s.%(table)s` index `%(name)s`" % index, query.db(index['db']).table(index['table']).index_status( index['temp_name']).nth(0)) if status['ready']: index['ready'] = True options.retryQuery( "rename `%(db)s.%(table)s` index `%(name)s`" % index, query.db(index['db']).table(index['table']).index_rename( index['temp_name'], index['name'], overwrite=True)) else: progress_ratio += status.get('progress', 0) / total_indexes indexes_in_progress = [ index for index in indexes_in_progress if not index['ready'] ] indexes_completed = total_indexes - len(indexes_to_build) - len( indexes_in_progress) progress_ratio += float(indexes_completed) / total_indexes if len(indexes_in_progress) == options.concurrent or \ (len(indexes_in_progress) > 0 and len(indexes_to_build) == 0): # Short sleep to keep from killing the CPU time.sleep(0.1) # Make sure the progress bar says we're done and get past the progress bar line if not options.quiet: utils_common.print_progress(1.0) print("")