default=0.1,
help='Default learning rate')
parser.add_argument('--max-epochs',
type=int,
default=settings.MAX_EPOCHS,
help='Default maximum epochs to train')
parser.add_argument('--pretrain',
type=str,
default=None,
help='Specify the pretrain path')
parser.add_argument('--num-steps', type=int, default=35)
parser.add_argument(
'--compressor',
type=str,
default='sigmathresallgather',
choices=compressors.keys(),
help='Specify the compressors if \'compression\' is open')
parser.add_argument('--density',
type=float,
default=1,
help='Density for sparsification')
parser.add_argument('--threshold',
type=int,
default=524288000,
help='Specify the threshold for gradient merging')
args = parser.parse_args()
batch_size = args.batch_size * args.nsteps_update
prefix = settings.PREFIX
if args.density < 1:
prefix = 'comp-' + args.compressor + '-' + prefix
logdir = 'allreduce-%s-thres-%dkbytes/%s-n%d-bs%d-lr%.4f-ns%d-ds%s' % (
np.save(fn2, arr2)
np.save(fn3, arr3)
np.save(fn5, arr5)
# For comparison purpose <=== End
optimizer._allreducer._profiling_norms = []
optimizer.stop()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="AllReduce trainer")
parser.add_argument('--batch-size', type=int, default=32)
parser.add_argument('--nsteps-update', type=int, default=1)
parser.add_argument('--nworkers', type=int, default=1, help='Just for experiments, and it cannot be used in production')
parser.add_argument('--nwpernode', type=int, default=1, help='Number of workers per node')
parser.add_argument('--compression', dest='compression', action='store_true')
parser.add_argument('--compressor', type=str, default='topk', choices=compressors.keys(), help='Specify the compressors if \'compression\' is open')
parser.add_argument('--sigma-scale', type=float, default=2.5, help='Maximum sigma scaler for sparsification')
parser.add_argument('--density', type=float, default=0.01, help='Density for sparsification')
parser.add_argument('--dataset', type=str, default='imagenet', choices=_support_datasets, help='Specify the dataset for training')
parser.add_argument('--dnn', type=str, default='resnet50', choices=_support_dnns, help='Specify the neural network for training')
parser.add_argument('--data-dir', type=str, default='./data', help='Specify the data root path')
parser.add_argument('--lr', type=float, default=0.1, help='Default learning rate')
parser.add_argument('--max-epochs', type=int, default=90, help='Default maximum epochs to train')
parser.add_argument('--pretrain', type=str, default=None, help='Specify the pretrain path')
parser.set_defaults(compression=False)
args = parser.parse_args()
batch_size = args.batch_size * args.nsteps_update
prefix = settings.PREFIX
if args.compression:
prefix = 'comp-' + args.compressor + '-' + prefix
logdir = 'allreduce-%s/%s-n%d-bs%d-lr%.4f-ns%d-sg%.2f-ds%s' % (prefix, args.dnn, args.nworkers, batch_size, args.lr, args.nsteps_update, args.sigma_scale, str(args.density))
help='number of warm-up batches that don\'t count towards benchmark')
parser.add_argument('--num-batches-per-iter', type=int, default=10,
help='number of batches per benchmark iteration')
parser.add_argument('--num-iters', type=int, default=10,
help='number of benchmark iterations')
parser.add_argument('--no-cuda', action='store_true', default=False,
help='disables CUDA training')
parser.add_argument('--use-adasum', action='store_true', default=False,
help='use adasum algorithm to do reduction')
parser.add_argument('--mgwfbp', action='store_true', default=False, help='Use MG-WFBP')
parser.add_argument('--threshold', type=int, default=536870912, help='Set threshold if mgwfbp is False')
parser.add_argument('--rdma', action='store_true', default=False, help='Use RDMA')
parser.add_argument('--compressor', type=str, default='none', choices=compressors.keys(), help='Specify the compressors if density < 1.0')
parser.add_argument('--density', type=float, default=1, help='Density for sparsification')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
hvd.init()
if args.cuda:
# Horovod: pin GPU to local rank.
torch.cuda.set_device(hvd.local_rank())
cudnn.benchmark = True
# Set up standard model.
model = getattr(models, args.model)()
def main():
# Build argument parser
parser = argparse.ArgumentParser(
description="Do various data compression techniques")
parser.add_argument('-f',
'--file',
dest='infile',
nargs='?',
type=argparse.FileType('rb'),
default=sys.stdin,
help="The input data for data compression.")
parser.add_argument('-o',
'--output',
dest='outfile',
nargs='?',
type=argparse.FileType('wb'),
default=sys.stdout,
help="Path to the compression output.")
# Sub commands
subparsers = parser.add_subparsers()
encode_parser = subparsers.add_parser('encode')
encode_parser.add_argument('encoders',
choices=compressors.keys(),
nargs='*')
decode_parser = subparsers.add_parser('decode')
decode_parser.add_argument('decoders',
choices=compressors.keys(),
nargs='*')
args = parser.parse_args()
# Read input data
data = bytearray()
if args.infile is sys.stdin:
# From stdin
for line in args.infile:
data += line.encode()
else:
# From file, opened in binary mode
for line in args.infile:
data += line
sys.stderr.write(f"Consuming {len(data)} Bytes\n")
# Get workflow steps from CLI
if hasattr(args, 'encoders'):
workflow = [(encode, e, f"{e} encoding") for e in args.encoders]
if hasattr(args, 'decoders'):
workflow = [(decode, d, f"{d} decoding") for d in args.decoders]
# Process all workflow steps
for step in workflow:
bar = IncrementalBar(step[2],
max=len(data),
suffix='%(index)d/%(max)d Bytes [%(elapsed)ds]')
data = step[0](data, step[1], bar)
bar.finish()
# Write output data
sys.stderr.write(f"Output {len(data)} Bytes:\n")
if args.outfile is sys.stdout:
args.outfile.write(data.decode())
args.outfile.flush()
else:
args.outfile.write(data)
args.outfile.flush()