def _print_chunk_migrations(self, chunks, moved_from=False): """Prints the chunk migration statistics in a table depending on to/from flag""" verbose = self.mloginfo.args['verbose'] chunks.reverse() if verbose: chunk_groupings = Grouping(group_by=lambda x: x.time) else: chunk_groupings = Grouping(group_by=lambda x: (x.time.strftime( "%Y-%m-%dT%H"), x.movedFromTo, x.namespace)) for chunk_moved in chunks: time, chunk_range, moved_to_from, namespace, steps, status, error_message = chunk_moved moved_tuple = ChunksTuple(time=time, range=chunk_range, movedFromTo=moved_to_from, namespace=namespace, steps=steps, migrationStatus=status, errorMessage=error_message) chunk_groupings.add(moved_tuple) move_to_from_title = 'to shard' if moved_from else 'from shard' if verbose: titles = [ ' time', move_to_from_title, 'namespace', 'chunk migration status' ] else: titles = [ ' time (/hour)', move_to_from_title, 'namespace', '# chunks migrations attempted', 'successful chunk migrations', 'failed chunk migrations' ] if len(chunk_groupings) == 0: print(" no chunk migrations found.") else: table_rows = [] for group, chunks in chunk_groupings.items(): if verbose: time = group.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] chunk = chunks[0] else: time, moved_to_from, namespace = group successful_count = 0 total_time_spent = 0 failed = dict() succeeded_after = dict() for chunk in chunks: if chunk.migrationStatus == "success": successful_count += 1 succeeded_after[chunk.range] = (True, chunk.time) total_time_spent += sum( int(ms) for step, ms in chunk.steps) else: count, timestamps = failed.get( chunk.errorMessage, (0, list())) count += 1 successful_after, timestamp = succeeded_after.get( chunk.range, (False, None)) if successful_after: timestamp = timestamp.strftime( "%H:%M:%S.%f")[:-3] chunk_time = chunk.time.strftime( "%H:%M:%S.%f")[:-3] timestamps.append( chunk_time + f' BECAME SUCCESSFUL AT: {timestamp}') else: timestamps.append( chunk.time.strftime("%H:%M:%S.%f")[:-3]) failed[chunk.errorMessage] = (count, timestamps) moved_chunks = OrderedDict() moved_chunks['time'] = f" {time}" moved_chunks['movedFromTo'] = moved_to_from moved_chunks['namespace'] = namespace if verbose: if chunk.migrationStatus == "success": total_time_spent = sum( int(ms) for step, ms in chunk.steps) msg = f"Successful | Total time spent {total_time_spent}ms" step_breakdown = ', '.join(f"{step}: {ms}ms" for step, ms in chunk.steps) moved_chunks[ 'chunkMigrationStatus'] = msg + f" ({step_breakdown})" else: moved_chunks[ 'chunkMigrationStatus'] = f"Failed with {chunk.errorMessage}" else: moved_chunks['numberOfChunks'] = f'{len(chunks)} chunk(s)' msg = (f"{successful_count} chunk(s) moved " + f"| Total time spent: {total_time_spent}ms") moved_chunks['successChunkMigrations'] = msg failed_migrations = "" for error, info in failed.items(): count, timestamps = info failed_migrations += ( f'{count} chunk(s): {timestamps} ' f'failed with "{error}".') if len(failed_migrations): moved_chunks[ 'failedChunkMigrations'] = failed_migrations else: moved_chunks[ 'failedChunkMigrations'] = "no failed chunks." table_rows.append(moved_chunks) print_table(table_rows, titles) if not verbose: print( "\nto show individual chunk migration, run with --verbose." )
def _print_chunk_statistics(self): """Prints the chunk split statistics in a table""" self.mloginfo.logfile.chunk_splits.reverse() chunk_split_groupings = Grouping( group_by=lambda x: (x.time.strftime("%Y-%m-%dT%H"), x.namespace)) for chunk_split in self.mloginfo.logfile.chunk_splits: time, split_range, namespace, numSplits, success, timeTaken, error = chunk_split split_tuple = SplitTuple(time=time, range=split_range, namespace=namespace, numSplits=numSplits, success=success, timeTaken=timeTaken, error=error) chunk_split_groupings.add(split_tuple) titles = [ ' time (/hour)', 'namespace', '# split-vectors issued', 'successful chunk splits', 'failed chunk splits' ] if len(chunk_split_groupings) == 0: print(" no chunk splits found.") else: table_rows = [] for group, splits in chunk_split_groupings.items(): time, namespace = group successful_count = 0 total_number_vectors = 0 split_succeeded_after = dict() failed_splits = dict() total_time_taken = 0 for split in splits: total_number_vectors += int(split.numSplits) if (not split.success) and split.error: count, timestamps = failed_splits.get( split.error, (0, list())) count += 1 if split_succeeded_after.get(split.range, False): timestamps.append( split.time.strftime("%H:%M:%S.%f")[:-3] + ' **WAS SUCCESSFUL AFTER**') else: timestamps.append( split.time.strftime("%H:%M:%S.%f")[:-3]) failed_splits[split.error] = (count, timestamps) elif split.success: split_succeeded_after[split.range] = True successful_count += 1 total_time_taken += sum( int(ms) for ms in split.timeTaken) split_summary = OrderedDict() split_summary['time'] = f" {time}" split_summary['namespace'] = namespace split_summary[ 'numSplitVectors'] = f'{total_number_vectors} split vector(s)' msg = (f"{successful_count} chunk(s) splitted" + f" | Total time spent: {total_time_taken}ms") split_summary['successfulSplits'] = msg failed_split = "" for error, info in failed_splits.items(): count, timestamps = info if error == "Jumbo": failed_split += (f'{count} chunk(s): ' + f'{timestamps} marked as {error}.') else: failed_split += (f'{count} chunk(s): {timestamps} ' + f'failed with "{error}". ') if len(failed_split): split_summary['failedChunkSplits'] = failed_split else: split_summary[ 'failedChunkSplits'] = "no failed chunk splits." table_rows.append(split_summary) print_table(table_rows, titles)