def resnet_cifar10(train_data, test_data, mean_data, network_name, epoch_size, num_quantization_bits=32, block_size=None, warm_up=0, max_epochs=160, restore=True, log_to_file=None, num_mbs_per_log=None, gen_heartbeat=False, scale_up=False, profiling=False, fp16=False): set_computation_network_trace_level(0) # NOTE: scaling up minibatch_size increases sample throughput. In 8-GPU machine, # ResNet110 samples-per-second is ~7x of single GPU, comparing to ~3x without scaling # up. However, bigger minimatch size on the same number of samples means less updates, # thus leads to higher training error. This is a trade-off of speed and accuracy minibatch_size = 128 * (Communicator.num_workers() if scale_up else 1) progress_printer = ProgressPrinter(freq=num_mbs_per_log, tag='Training', log_to_file=log_to_file, rank=Communicator.rank(), gen_heartbeat=gen_heartbeat, num_epochs=max_epochs) network = create_resnet_network(network_name, fp16) trainer = create_trainer(network, minibatch_size, epoch_size, num_quantization_bits, block_size, warm_up, progress_printer) train_source = create_image_mb_source(train_data, mean_data, train=True, total_number_of_samples=max_epochs * epoch_size) test_source = create_image_mb_source( test_data, mean_data, train=False, total_number_of_samples=C.io.FULL_DATA_SWEEP) train_and_test(network, trainer, train_source, test_source, minibatch_size, epoch_size, restore, profiling)
def parse_arguments(cfg): parser = argparse.ArgumentParser() parser.add_argument('-datadir', '--datadir', help='Data directory where the ImageNet dataset is located', required=False, default=cfg["DATA"].MAP_FILE_PATH) parser.add_argument('-outputdir', '--outputdir', help='Output directory for checkpoints and models', required=False, default=None) parser.add_argument('-logdir', '--logdir', help='Log file', required=False, default=None) parser.add_argument('-n', '--num_epochs', help='Total number of epochs to train', type=int, required=False, default=cfg["CNTK"].MAX_EPOCHS) parser.add_argument('-m', '--minibatch_size', help='Minibatch size', type=int, required=False, default=cfg.MB_SIZE) parser.add_argument('-e', '--epoch_size', help='Epoch size', type=int, required=False, default=cfg["DATA"].NUM_TRAIN_IMAGES) parser.add_argument('-q', '--quantized_bits', help='Number of quantized bits used for gradient aggregation', type=int, required=False, default='32') parser.add_argument('-r', '--restart', help='Indicating whether to restart from scratch (instead of restart from checkpoint file by default)', action='store_true') parser.add_argument('-device', '--device', type=int, help="Force to run the script on a specified device", required=False, default=None) parser.add_argument('-lrFactor', '--lrFactor', type=float, help="Scale factor for the lr schedule", required=False) parser.add_argument('-momentumPerMb', '--momentumPerMb', type=float, help="momentum per minibatch", required=False) parser.add_argument('-rndSeed', '--rndSeed', type=int, help="the random seed", required=False) parser.add_argument('-trainConv', '--trainConv', type=int, help="whether to train conv layers", required=False) args = vars(parser.parse_args()) if args['lrFactor'] is not None: cfg["CNTK"].LR_FACTOR = args['lrrFactor'] if args['num_epochs'] is not None: cfg["CNTK"].MAX_EPOCHS = args['num_epochs'] if args['momentumPerMb'] is not None: cfg.MOMENTUM_PER_MB = args['momentumPerMb'] if args['rndSeed'] is not None: cfg.RND_SEED = args['rndSeed'] if args['trainConv'] is not None: cfg["CNTK"].TRAIN_CONV_LAYERS = True if args['trainConv'] == 1 else False if args['datadir'] is not None: cfg["DATA"].MAP_FILE_PATH = args['datadir'] if args['outputdir'] is not None: cfg.OUTPUT_PATH = args['outputdir'] if args['logdir'] is not None: log_dir = args['logdir'] if args['device'] is not None: # Setting one worker on GPU and one worker on CPU. Otherwise memory consumption is too high for a single GPU. if Communicator.rank() == 0: cntk.device.try_set_default_device(cntk.device.gpu(args['device'])) else: cntk.device.try_set_default_device(cntk.device.cpu())
def parse_arguments(cfg): parser = argparse.ArgumentParser() parser.add_argument('-datadir', '--datadir', help='Data directory where the ImageNet dataset is located', required=False, default=cfg["DATA"].MAP_FILE_PATH) parser.add_argument('-outputdir', '--outputdir', help='Output directory for checkpoints and models', required=False, default=None) parser.add_argument('-logdir', '--logdir', help='Log file', required=False, default=None) parser.add_argument('-n', '--num_epochs', help='Total number of epochs to train', type=int, required=False, default=cfg["CNTK"].MAX_EPOCHS) parser.add_argument('-m', '--minibatch_size', help='Minibatch size', type=int, required=False, default=cfg.MB_SIZE) parser.add_argument('-e', '--epoch_size', help='Epoch size', type=int, required=False, default=cfg["DATA"].NUM_TRAIN_IMAGES) parser.add_argument('-q', '--quantized_bits', help='Number of quantized bits used for gradient aggregation', type=int, required=False, default='32') parser.add_argument('-r', '--restart', help='Indicating whether to restart from scratch (instead of restart from checkpoint file by default)', action='store_true') parser.add_argument('-device', '--device', type=int, help="Force to run the script on a specified device", required=False, default=None) parser.add_argument('-lrFactor', '--lrFactor', type=float, help="Scale factor for the lr schedule", required=False) parser.add_argument('-momentumPerMb', '--momentumPerMb', type=float, help="momentum per minibatch", required=False) parser.add_argument('-rndSeed', '--rndSeed', type=int, help="the random seed", required=False) parser.add_argument('-trainConv', '--trainConv', type=int, help="whether to train conv layers", required=False) args = vars(parser.parse_args()) if args['lrFactor'] is not None: cfg["CNTK"].LR_FACTOR = args['lrrFactor'] if args['num_epochs'] is not None: cfg["CNTK"].MAX_EPOCHS = args['num_epochs'] if args['momentumPerMb'] is not None: cfg.MOMENTUM_PER_MB = args['momentumPerMb'] if args['rndSeed'] is not None: cfg.RND_SEED = args['rndSeed'] if args['trainConv'] is not None: cfg["CNTK"].TRAIN_CONV_LAYERS = True if args['trainConv'] == 1 else False if args['datadir'] is not None: cfg["DATA"].MAP_FILE_PATH = args['datadir'] if args['outputdir'] is not None: cfg.OUTPUT_PATH = args['outputdir'] if args['logdir'] is not None: log_dir = args['logdir'] if args['device'] is not None: # Setting one worker on GPU and one worker on CPU. Otherwise memory consumption is too high for a single GPU. if Communicator.rank() == 0: cntk.device.try_set_default_device(cntk.device.gpu(args['device'])) else: cntk.device.try_set_default_device(cntk.device.cpu())
def resnet_imagenet(train_data, test_data, mean_data, network_name, epoch_size, num_quantization_bits=32, block_size=None, warm_up=0, max_epochs=90, restore=True, log_to_file=None, num_mbs_per_log=100, gen_heartbeat=False, scale_up=False, profiling=False): set_computation_network_trace_level(0) # NOTE: scaling up minibatch_size increases sample throughput. In 8-GPU machine, # ResNet110 samples-per-second is ~7x of single GPU, comparing to ~3x without scaling # up. However, bigger minimatch size on the same number of samples means less updates, # thus leads to higher training error. This is a trade-off of speed and accuracy minibatch_size = 32 * (Communicator.num_workers() if scale_up else 1) progress_printer = ProgressPrinter( freq=num_mbs_per_log, tag='Training', log_to_file=log_to_file, rank=Communicator.rank(), gen_heartbeat=gen_heartbeat, num_epochs=max_epochs) network = create_resnet_network(network_name) trainer = create_trainer(network, minibatch_size, epoch_size, num_quantization_bits, block_size, warm_up, progress_printer) train_source = create_image_mb_source(train_data, mean_data, train=True, total_number_of_samples=max_epochs * epoch_size) test_source = create_image_mb_source(test_data, mean_data, train=False, total_number_of_samples=C.io.FULL_DATA_SWEEP) train_and_test(network, trainer, train_source, test_source, minibatch_size, epoch_size, restore, profiling)
if not os.path.isdir(data_path): raise RuntimeError("Directory %s does not exist" % data_path) mean_data = os.path.join(abs_path, 'ImageNet1K_mean.xml') train_data = os.path.join(data_path, 'train_map.txt') test_data = os.path.join(data_path, 'val_map.txt') num_quantization_bits = args['quantized_bits'] epochs = args['epochs'] warm_up = args['distributed_after'] network_name = args['network'] scale_up = bool(args['scale_up']) # Create distributed trainer factory print("Start training: quantize_bit = {}, epochs = {}, distributed_after = {}".format(num_quantization_bits, epochs, warm_up)) resnet_imagenet(train_data, test_data, mean_data, network_name, epoch_size, num_quantization_bits, block_size=args['block_samples'], warm_up=args['distributed_after'], max_epochs=epochs, restore=not args['restart'], scale_up=scale_up, log_to_file=args['logdir'], profiling=args['profile']) # Must call MPI finalize when process exit without exceptions Communicator.finalize()
test_data = os.path.join(data_path, 'test_map.txt') num_quantization_bits = args['quantized_bits'] epochs = args['epochs'] warm_up = args['distributed_after'] network_name = args['network'] scale_up = bool(args['scale_up']) # Create distributed trainer factory print( "Start training: quantize_bit = {}, epochs = {}, distributed_after = {}" .format(num_quantization_bits, epochs, warm_up)) resnet_cifar10(train_data, test_data, mean_data, network_name, epoch_size, num_quantization_bits, block_size=args['block_samples'], warm_up=args['distributed_after'], max_epochs=epochs, restore=not args['restart'], scale_up=scale_up, log_to_file=args['logdir'], profiling=args['profile'], fp16=args['fp16']) # Must call MPI finalize when process exit without exceptions Communicator.finalize()
def set_global_vars(use_arg_parser=True): data_path = map_file_path # set and overwrite learning parameters globalvars['rpn_lr_factor'] = cfg["CNTK"].RPN_LR_FACTOR globalvars['frcn_lr_factor'] = cfg["CNTK"].FRCN_LR_FACTOR globalvars['e2e_lr_factor'] = cfg["CNTK"].E2E_LR_FACTOR globalvars['momentum_per_mb'] = cfg["CNTK"].MOMENTUM_PER_MB globalvars['e2e_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg[ "CNTK"].E2E_MAX_EPOCHS globalvars[ 'rpn_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg["CNTK"].RPN_EPOCHS globalvars['frcn_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg[ "CNTK"].FRCN_EPOCHS globalvars['rnd_seed'] = cfg.RNG_SEED globalvars['train_conv'] = cfg["CNTK"].TRAIN_CONV_LAYERS globalvars['train_e2e'] = cfg["CNTK"].TRAIN_E2E globalvars['fea_map_dim'] = cfg["CNTK"].FEA_MAP_DIM if use_arg_parser: parser = argparse.ArgumentParser() parser.add_argument( '-datadir', '--datadir', help='Data directory where the ImageNet dataset is located', required=False, default=data_path) parser.add_argument('-outputdir', '--outputdir', help='Output directory for checkpoints and models', required=False, default=None) parser.add_argument('-logdir', '--logdir', help='Log file', required=False, default=None) parser.add_argument('-n', '--num_epochs', help='Total number of epochs to train', type=int, required=False, default=cfg["CNTK"].E2E_MAX_EPOCHS) parser.add_argument('-m', '--minibatch_size', help='Minibatch size', type=int, required=False, default=mb_size) parser.add_argument('-e', '--epoch_size', help='Epoch size', type=int, required=False, default=epoch_size) parser.add_argument( '-q', '--quantized_bits', help='Number of quantized bits used for gradient aggregation', type=int, required=False, default='32') parser.add_argument( '-r', '--restart', help= 'Indicating whether to restart from scratch (instead of restart from checkpoint file by default)', action='store_true') parser.add_argument( '-device', '--device', type=int, help="Force to run the script on a specified device", required=False, default=None) parser.add_argument('-rpnLrFactor', '--rpnLrFactor', type=float, help="Scale factor for rpn lr schedule", required=False) parser.add_argument('-frcnLrFactor', '--frcnLrFactor', type=float, help="Scale factor for frcn lr schedule", required=False) parser.add_argument('-e2eLrFactor', '--e2eLrFactor', type=float, help="Scale factor for e2e lr schedule", required=False) parser.add_argument('-momentumPerMb', '--momentumPerMb', type=float, help="momentum per minibatch", required=False) parser.add_argument('-e2eEpochs', '--e2eEpochs', type=int, help="number of epochs for e2e training", required=False) parser.add_argument('-rpnEpochs', '--rpnEpochs', type=int, help="number of epochs for rpn training", required=False) parser.add_argument('-frcnEpochs', '--frcnEpochs', type=int, help="number of epochs for frcn training", required=False) parser.add_argument('-rndSeed', '--rndSeed', type=int, help="the random seed", required=False) parser.add_argument('-trainConv', '--trainConv', type=int, help="whether to train conv layers", required=False) parser.add_argument('-trainE2E', '--trainE2E', type=int, help="whether to train e2e (otherwise 4 stage)", required=False) args = vars(parser.parse_args()) if args['rpnLrFactor'] is not None: globalvars['rpn_lr_factor'] = args['rpnLrFactor'] if args['frcnLrFactor'] is not None: globalvars['frcn_lr_factor'] = args['frcnLrFactor'] if args['e2eLrFactor'] is not None: globalvars['e2e_lr_factor'] = args['e2eLrFactor'] if args['momentumPerMb'] is not None: globalvars['momentum_per_mb'] = args['momentumPerMb'] if args['e2eEpochs'] is not None: globalvars['e2e_epochs'] = args['e2eEpochs'] if args['rpnEpochs'] is not None: globalvars['rpn_epochs'] = args['rpnEpochs'] if args['frcnEpochs'] is not None: globalvars['frcn_epochs'] = args['frcnEpochs'] if args['rndSeed'] is not None: globalvars['rnd_seed'] = args['rndSeed'] if args['trainConv'] is not None: globalvars[ 'train_conv'] = True if args['trainConv'] == 1 else False if args['trainE2E'] is not None: globalvars['train_e2e'] = True if args['trainE2E'] == 1 else False if args['outputdir'] is not None: globalvars['output_path'] = args['outputdir'] if args['logdir'] is not None: log_dir = args['logdir'] if args['device'] is not None: # Setting one worker on GPU and one worker on CPU. Otherwise memory consumption is too high for a single GPU. if Communicator.rank() == 0: cntk.device.try_set_default_device( cntk.device.gpu(args['device'])) else: cntk.device.try_set_default_device(cntk.device.cpu()) if args['datadir'] is not None: data_path = args['datadir'] if not os.path.isdir(data_path): raise RuntimeError("Directory %s does not exist" % data_path) globalvars['class_map_file'] = os.path.join(data_path, globalvars['class_map_file']) globalvars['train_map_file'] = os.path.join(data_path, globalvars['train_map_file']) globalvars['test_map_file'] = os.path.join(data_path, globalvars['test_map_file']) globalvars['train_roi_file'] = os.path.join(data_path, globalvars['train_roi_file']) globalvars['test_roi_file'] = os.path.join(data_path, globalvars['test_roi_file']) if cfg["CNTK"].FORCE_DETERMINISTIC: force_deterministic_algorithms() np.random.seed(seed=globalvars['rnd_seed']) globalvars['classes'] = parse_class_map_file(globalvars['class_map_file']) globalvars['num_classes'] = len(globalvars['classes']) if cfg["CNTK"].DEBUG_OUTPUT: # report args print("Using the following parameters:") print("Flip image : {}".format(cfg["TRAIN"].USE_FLIPPED)) print("Train conv layers: {}".format(globalvars['train_conv'])) print("Random seed : {}".format(globalvars['rnd_seed'])) print("Momentum per MB : {}".format(globalvars['momentum_per_mb'])) if globalvars['train_e2e']: print("E2E epochs : {}".format(globalvars['e2e_epochs'])) else: print("RPN lr factor : {}".format(globalvars['rpn_lr_factor'])) print("RPN epochs : {}".format(globalvars['rpn_epochs'])) print("FRCN lr factor : {}".format(globalvars['frcn_lr_factor'])) print("FRCN epochs : {}".format(globalvars['frcn_epochs']))
def set_global_vars(use_arg_parser = True): data_path = map_file_path # set and overwrite learning parameters globalvars['rpn_lr_factor'] = cfg["CNTK"].RPN_LR_FACTOR globalvars['frcn_lr_factor'] = cfg["CNTK"].FRCN_LR_FACTOR globalvars['e2e_lr_factor'] = cfg["CNTK"].E2E_LR_FACTOR globalvars['momentum_per_mb'] = cfg["CNTK"].MOMENTUM_PER_MB globalvars['e2e_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg["CNTK"].E2E_MAX_EPOCHS globalvars['rpn_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg["CNTK"].RPN_EPOCHS globalvars['frcn_epochs'] = 1 if cfg["CNTK"].FAST_MODE else cfg["CNTK"].FRCN_EPOCHS globalvars['rnd_seed'] = cfg.RNG_SEED globalvars['train_conv'] = cfg["CNTK"].TRAIN_CONV_LAYERS globalvars['train_e2e'] = cfg["CNTK"].TRAIN_E2E if use_arg_parser: parser = argparse.ArgumentParser() parser.add_argument('-datadir', '--datadir', help='Data directory where the ImageNet dataset is located', required=False, default=data_path) parser.add_argument('-outputdir', '--outputdir', help='Output directory for checkpoints and models', required=False, default=None) parser.add_argument('-logdir', '--logdir', help='Log file', required=False, default=None) parser.add_argument('-n', '--num_epochs', help='Total number of epochs to train', type=int, required=False, default=cfg["CNTK"].E2E_MAX_EPOCHS) parser.add_argument('-m', '--minibatch_size', help='Minibatch size', type=int, required=False, default=mb_size) parser.add_argument('-e', '--epoch_size', help='Epoch size', type=int, required=False, default=epoch_size) parser.add_argument('-q', '--quantized_bits', help='Number of quantized bits used for gradient aggregation', type=int, required=False, default='32') parser.add_argument('-r', '--restart', help='Indicating whether to restart from scratch (instead of restart from checkpoint file by default)', action='store_true') parser.add_argument('-device', '--device', type=int, help="Force to run the script on a specified device", required=False, default=None) parser.add_argument('-rpnLrFactor', '--rpnLrFactor', type=float, help="Scale factor for rpn lr schedule", required=False) parser.add_argument('-frcnLrFactor', '--frcnLrFactor', type=float, help="Scale factor for frcn lr schedule", required=False) parser.add_argument('-e2eLrFactor', '--e2eLrFactor', type=float, help="Scale factor for e2e lr schedule", required=False) parser.add_argument('-momentumPerMb', '--momentumPerMb', type=float, help="momentum per minibatch", required=False) parser.add_argument('-e2eEpochs', '--e2eEpochs', type=int, help="number of epochs for e2e training", required=False) parser.add_argument('-rpnEpochs', '--rpnEpochs', type=int, help="number of epochs for rpn training", required=False) parser.add_argument('-frcnEpochs', '--frcnEpochs', type=int, help="number of epochs for frcn training", required=False) parser.add_argument('-rndSeed', '--rndSeed', type=int, help="the random seed", required=False) parser.add_argument('-trainConv', '--trainConv', type=int, help="whether to train conv layers", required=False) parser.add_argument('-trainE2E', '--trainE2E', type=int, help="whether to train e2e (otherwise 4 stage)", required=False) args = vars(parser.parse_args()) if args['rpnLrFactor'] is not None: globalvars['rpn_lr_factor'] = args['rpnLrFactor'] if args['frcnLrFactor'] is not None: globalvars['frcn_lr_factor'] = args['frcnLrFactor'] if args['e2eLrFactor'] is not None: globalvars['e2e_lr_factor'] = args['e2eLrFactor'] if args['momentumPerMb'] is not None: globalvars['momentum_per_mb'] = args['momentumPerMb'] if args['e2eEpochs'] is not None: globalvars['e2e_epochs'] = args['e2eEpochs'] if args['rpnEpochs'] is not None: globalvars['rpn_epochs'] = args['rpnEpochs'] if args['frcnEpochs'] is not None: globalvars['frcn_epochs'] = args['frcnEpochs'] if args['rndSeed'] is not None: globalvars['rnd_seed'] = args['rndSeed'] if args['trainConv'] is not None: globalvars['train_conv'] = True if args['trainConv']==1 else False if args['trainE2E'] is not None: globalvars['train_e2e'] = True if args['trainE2E']==1 else False if args['outputdir'] is not None: globalvars['output_path'] = args['outputdir'] if args['logdir'] is not None: log_dir = args['logdir'] if args['device'] is not None: # Setting one worker on GPU and one worker on CPU. Otherwise memory consumption is too high for a single GPU. if Communicator.rank() == 0: cntk.device.try_set_default_device(cntk.device.gpu(args['device'])) else: cntk.device.try_set_default_device(cntk.device.cpu()) if args['datadir'] is not None: data_path = args['datadir'] if not os.path.isdir(data_path): raise RuntimeError("Directory %s does not exist" % data_path) globalvars['class_map_file'] = os.path.join(data_path, globalvars['class_map_file']) globalvars['train_map_file'] = os.path.join(data_path, globalvars['train_map_file']) globalvars['test_map_file'] = os.path.join(data_path, globalvars['test_map_file']) globalvars['train_roi_file'] = os.path.join(data_path, globalvars['train_roi_file']) globalvars['test_roi_file'] = os.path.join(data_path, globalvars['test_roi_file']) if cfg["CNTK"].FORCE_DETERMINISTIC: force_deterministic_algorithms() np.random.seed(seed=globalvars['rnd_seed']) globalvars['classes'] = parse_class_map_file(globalvars['class_map_file']) globalvars['num_classes'] = len(globalvars['classes']) if cfg["CNTK"].DEBUG_OUTPUT: # report args print("Using the following parameters:") print("Flip image : {}".format(cfg["TRAIN"].USE_FLIPPED)) print("Train conv layers: {}".format(globalvars['train_conv'])) print("Random seed : {}".format(globalvars['rnd_seed'])) print("Momentum per MB : {}".format(globalvars['momentum_per_mb'])) if globalvars['train_e2e']: print("E2E epochs : {}".format(globalvars['e2e_epochs'])) else: print("RPN lr factor : {}".format(globalvars['rpn_lr_factor'])) print("RPN epochs : {}".format(globalvars['rpn_epochs'])) print("FRCN lr factor : {}".format(globalvars['frcn_lr_factor'])) print("FRCN epochs : {}".format(globalvars['frcn_epochs']))