def run_image(model, sess_init, inputs):
pred_config = PredictConfig(
model=model,
session_init=sess_init,
input_names=['input'],
output_names=['output']
)
predictor = OfflinePredictor(pred_config)
meta = dataset.ILSVRCMeta()
words = meta.get_synset_words_1000()
transformers = imgaug.AugmentorList(fbresnet_augmentor(isTrain=False))
for f in inputs:
assert os.path.isfile(f), f
img = cv2.imread(f).astype('float32')
assert img is not None
img = transformers.augment(img)[np.newaxis, :, :, :]
outputs = predictor(img)[0]
prob = outputs[0]
ret = prob.argsort()[-10:][::-1]
names = [words[i] for i in ret]
print(f + ":")
print(list(zip(names, prob[ret])))
def run_image(model, sess_init, inputs):
pred_config = PredictConfig(
model=model,
session_init=sess_init,
input_names=['input'],
output_names=['output']
)
predictor = OfflinePredictor(pred_config)
meta = dataset.ILSVRCMeta()
words = meta.get_synset_words_1000()
transformers = imgaug.AugmentorList(fbresnet_augmentor(isTrain=False))
for f in inputs:
assert os.path.isfile(f), f
img = cv2.imread(f).astype('float32')
assert img is not None
img = transformers.augment(img)[np.newaxis, :, :, :]
outputs = predictor(img)[0]
prob = outputs[0]
ret = prob.argsort()[-10:][::-1]
names = [words[i] for i in ret]
print(f + ":")
print(list(zip(names, prob[ret])))
def get_data(name, batch, parallel=None):
isTrain = name == 'train'
augmentors = fbresnet_augmentor(isTrain)
return get_imagenet_dataflow(args.data,
name,
batch,
augmentors,
parallel=parallel)
def get_data(name, data_dir, batch, crop_method, color_augmentation = False,
CAM_dir_pkl = None, repeat_times = 1, strict_order = False, remainder_TR = False):
isTrain = name == 'train'
augmentors = fbresnet_augmentor(isTrain, crop_method, color_augmentation)
return get_AVA2012_dataflow(data_dir, name, batch, augmentors, \
CAM_dir_pkl = CAM_dir_pkl, CAMCropR = ((isTrain) and (crop_method == 'CAMCropR')), \
repeat_times = repeat_times, strict_order = strict_order, remainder_TR = remainder_TR)
def get_data(train_or_test):
isTrain = train_or_test == 'train'
augs = fbresnet_augmentor(isTrain)
meta = dataset.ILSVRCMeta()
pp_mean = meta.get_per_pixel_mean()
augs.append(imgaug.MapImage(lambda x: x - pp_mean[16:-16, 16:-16]))
ds = get_imagenet_dataflow(args.data, train_or_test, BATCH_SIZE, augs)
return ds
def get_data(train_or_test):
isTrain = train_or_test == 'train'
augs = fbresnet_augmentor(isTrain)
meta = dataset.ILSVRCMeta()
pp_mean = meta.get_per_pixel_mean()
augs.append(imgaug.MapImage(lambda x: x - pp_mean[16:-16, 16:-16]))
ds = get_imagenet_dataflow(args.data, train_or_test, BATCH_SIZE, augs)
return ds
def get_data(name, batch):
isTrain = name == 'train'
global args
augmentors = fbresnet_augmentor(isTrain)
if isTrain:
print("Training batch:", batch)
return get_imagenet_dataflow(args.data, name, batch, augmentors)
else:
imagenet1k = get_imagenet_dataflow(args.data, name, batch, augmentors)
return imagenet1k
def get_data(train_or_test):
isTrain = train_or_test == 'train'
augs = fbresnet_augmentor(isTrain)
meta = dataset.ILSVRCMeta()
pp_mean = meta.get_per_pixel_mean()
augs.append(imgaug.MapImage(lambda x: x - pp_mean[16:-16, 16:-16]))
data_path = TRAIN_LIST_PATH if train_or_test == 'train' else VALID_LIST_PATH
ds = get_imagenet_dataflow(data_path, train_or_test, BATCH_SIZE, augs)
return ds
def get_data(train_or_test):
# completely copied from imagenet-resnet.py example
isTrain = train_or_test == 'train'
datadir = args.data
ds = dataset.ILSVRC12(datadir, train_or_test, shuffle=isTrain)
augmentors = fbresnet_augmentor(isTrain)
augmentors.append(imgaug.ToUint8())
ds = AugmentImageComponent(ds, augmentors, copy=False)
if isTrain:
ds = MultiProcessRunnerZMQ(ds, min(25, multiprocessing.cpu_count()))
ds = BatchData(ds, BATCH_SIZE, remainder=not isTrain)
return ds
def get_data(train_or_test):
# completely copied from imagenet-resnet.py example
isTrain = train_or_test == 'train'
datadir = args.data
ds = dataset.ILSVRC12(datadir, train_or_test, shuffle=isTrain)
augmentors = fbresnet_augmentor(isTrain)
augmentors.append(imgaug.ToUint8())
ds = AugmentImageComponent(ds, augmentors, copy=False)
if isTrain:
ds = PrefetchDataZMQ(ds, min(25, multiprocessing.cpu_count()))
ds = BatchData(ds, BATCH_SIZE, remainder=not isTrain)
return ds
help='per-GPU batch size',
default=32,
type=int)
parser.add_argument('--benchmark', action='store_true')
parser.add_argument('--no-zmq-ops', action='store_true')
args = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = ''
if args.fake:
ds = FakeData([[args.batch, 224, 224, 3], [args.batch]],
1000,
random=False,
dtype=['uint8', 'int32'])
else:
augs = fbresnet_augmentor(
True) if args.aug == 'fbresnet' else small_augmentor()
ds = get_data(args.batch, augs)
logger.info("Serving data on {}".format(socket.gethostname()))
if args.benchmark:
ds = MapData(ds, dump_arrays)
TestDataSpeed(ds, warmup=300).start()
else:
format = None if args.no_zmq_ops else 'zmq_ops'
send_dataflow_zmq(ds,
'ipc://@imagenet-train-b{}'.format(args.batch),
hwm=150,
format=format,
bind=True)
def get_config(model, fake=False):
batch = args.batch
total_batch = batch * hvd.size()
if fake:
data = FakeData(
[[args.batch, 224, 224, 3], [args.batch]], 1000,
random=False, dtype=['uint8', 'int32'])
data = StagingInput(QueueInput(data))
callbacks = []
steps_per_epoch = 50
else:
logger.info("#Tower: {}; Batch size per tower: {}".format(hvd.size(), batch))
zmq_addr = 'ipc://@imagenet-train-b{}'.format(batch)
if args.no_zmq_ops:
dataflow = RemoteDataZMQ(zmq_addr, hwm=150, bind=False)
data = QueueInput(dataflow)
else:
data = ZMQInput(zmq_addr, 30, bind=False)
data = StagingInput(data, nr_stage=1)
steps_per_epoch = int(np.round(1281167 / total_batch))
"""
Sec 2.1: Linear Scaling Rule: When the minibatch size is multiplied by k, multiply the learning rate by k.
"""
BASE_LR = 0.1 * (total_batch // 256)
logger.info("Base LR: {}".format(BASE_LR))
"""
Sec 5.1:
We call this number (0.1 * kn / 256 ) the reference learning rate,
and reduce it by 1/10 at the 30-th, 60-th, and 80-th epoch
"""
callbacks = [
ModelSaver(max_to_keep=100),
EstimatedTimeLeft(),
ScheduledHyperParamSetter(
'learning_rate', [(0, BASE_LR), (30, BASE_LR * 1e-1), (60, BASE_LR * 1e-2),
(80, BASE_LR * 1e-3)]),
]
if BASE_LR > 0.1:
"""
Sec 2.2: In practice, with a large minibatch of size kn, we start from a learning rate of η and increment
it by a constant amount at each iteration such that it reachesη = kη after 5 epochs.
After the warmup phase, we go back to the original learning rate schedule.
"""
callbacks.append(
ScheduledHyperParamSetter(
'learning_rate', [(0, 0.1), (5 * steps_per_epoch, BASE_LR)],
interp='linear', step_based=True))
if args.validation is not None:
# TODO For distributed training, you probably don't want everyone to wait for master doing validation.
# Better to start a separate job, since the model is saved.
if args.validation == 'master' and hvd.rank() == 0:
# For reproducibility, do not use remote data for validation
dataset_val = get_val_dataflow(
args.data, 64, fbresnet_augmentor(False))
infs = [ClassificationError('wrong-top1', 'val-error-top1'),
ClassificationError('wrong-top5', 'val-error-top5')]
callbacks.append(InferenceRunner(QueueInput(dataset_val), infs))
# For simple validation tasks such as image classification, distributed validation is possible.
elif args.validation == 'distributed':
dataset_val = get_val_dataflow(
args.data, 64, fbresnet_augmentor(False),
num_splits=hvd.size(), split_index=hvd.rank())
infs = [HorovodClassificationError('wrong-top1', 'val-error-top1'),
HorovodClassificationError('wrong-top5', 'val-error-top5')]
callbacks.append(
InferenceRunner(QueueInput(dataset_val), infs).set_chief_only(False))
return TrainConfig(
model=model,
data=data,
callbacks=callbacks,
steps_per_epoch=steps_per_epoch,
max_epoch=35 if args.fake else 95,
)
action='store_true')
"""
Sec 2.3: We keep the per-worker sample size n constant when we change the number of workers k.
In this work, we use n = 32 which has performed well for a wide range of datasets and networks.
"""
parser.add_argument('--batch', help='per-GPU batch size', default=32, type=int)
args = parser.parse_args()
model = Model(args.depth, args.norm, args.use_ws)
model.accum_grad = args.accum_grad
if args.weight_decay_norm:
model.weight_decay_pattern = ".*/W|.*/gamma|.*/beta"
if args.eval:
batch = 128 # something that can run on one gpu
ds = get_val_dataflow(args.data, batch, fbresnet_augmentor(False))
eval_classification(model, get_model_loader(args.load), ds)
sys.exit()
logger.info("Training on {}".format(socket.gethostname()))
# Print some information for sanity check.
os.system("nvidia-smi")
assert args.load is None
hvd.init()
if args.logdir is None:
args.logdir = os.path.join('train_log', 'Horovod-{}GPUs-{}Batch'.format(hvd.size(), args.batch))
if hvd.rank() == 0:
logger.set_logger_dir(args.logdir, 'd')
def get_data(name, batch, target_shape):
isTrain = name == 'train'
augmentors = fbresnet_augmentor(isTrain, target_shape)
return get_imagenet_dataflow(args.data, name, batch, augmentors)
def get_data(dataset_name):
isTrain = dataset_name == 'train'
augmentors = fbresnet_augmentor(isTrain)
return get_imagenet_dataflow(args.data, dataset_name, BATCH_SIZE,
augmentors)
parser.add_argument('--view', action='store_true')
parser.add_argument('--log-filename', type=str, default='')
args = parser.parse_args()
logging.getLogger("requests").setLevel(logging.WARNING)
if not args.log_filename:
logging.basicConfig(level=logging.INFO,
format='[%(asctime)s %(levelname)s] %(message)s',
stream=sys.stderr)
else:
logging.basicConfig(level=logging.INFO,
format='[%(asctime)s %(levelname)s] %(message)s',
filename=args.log_filename)
if args.augment:
augmentors = fbresnet_augmentor(isTrain=True)
else:
augmentors = [
df.imgaug.Resize((128, 128)),
]
ds = dataflow.dataset.ILSVRC12(
args.service_code, 'train',
shuffle=True).parallel(num_threads=args.threads)
ds = df.AugmentImageComponent(ds, augmentors, copy=False)
ds = df.PrefetchDataZMQ(ds, nr_proc=args.process)
if args.view:
ds = dataflow.utils.image.Viewer(ds,
lambda x: x[1] == 4,
'label-4',
prob=1.0,
def get_inference_augmentor():
return fbresnet_augmentor(False)
default=32, type=int)
parser.add_argument('--warmup', help='prefetch buffer size',
default=150, type=int)
parser.add_argument('--port', help='server port',
default=1000, type=int)
parser.add_argument('--benchmark', action='store_true')
parser.add_argument('--no-zmq-ops', action='store_true')
args = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = ''
if args.fake:
ds = FakeData(
[[args.batch, args.image_size, args.image_size, 3], [args.batch]],
1000, random=False, dtype=['uint8', 'int32'])
else:
augs = fbresnet_augmentor(True, image_size=args.image_size)
ds = get_data(args.batch, augs, args.worker)
logger.info("Serving data on {}".format(socket.gethostname()))
if args.benchmark:
from zmq_ops import dump_arrays
ds = MapData(ds, dump_arrays)
TestDataSpeed(ds, warmup=300).start()
else:
format = None if args.no_zmq_ops else 'zmq_ops'
send_dataflow_zmq(
ds, 'ipc://@imagenet-train-b{}-p{}'.format(args.batch, args.port),
hwm=args.warmup, format=format, bind=True)
def get_data(name, batch, data_aug=True):
isTrain = name == 'train'
augmentors = fbresnet_augmentor(isTrain) if data_aug \
else normal_augmentor(isTrain)
return get_imagenet_dataflow(
args.data, name, batch, augmentors)
parser.add_argument('--data', help='ILSVRC dataset dir')
parser.add_argument('--fake', help='use fakedata to test or benchmark this model', action='store_true')
args = parser.parse_args()
logger.set_logger_dir(os.path.join("train_log", "imagenet-resnet-keras"))
tf.keras.backend.set_image_data_format('channels_first')
nr_gpu = get_nr_gpu()
if args.fake:
df_train = FakeData([[64, 224, 224, 3], [64, 1000]], 5000, random=False, dtype='uint8')
df_val = FakeData([[64, 224, 224, 3], [64, 1000]], 5000, random=False)
else:
batch_size = TOTAL_BATCH_SIZE // nr_gpu
assert args.data is not None
df_train = get_imagenet_dataflow(
args.data, 'train', batch_size, fbresnet_augmentor(True))
df_val = get_imagenet_dataflow(
args.data, 'val', batch_size, fbresnet_augmentor(False))
def one_hot(label):
return np.eye(1000)[label]
df_train = MapDataComponent(df_train, one_hot, 1)
df_val = MapDataComponent(df_val, one_hot, 1)
M = KerasModel(
resnet50,
inputs_desc=[InputDesc(tf.uint8, [None, 224, 224, 3], 'images')],
targets_desc=[InputDesc(tf.float32, [None, 1000], 'labels')],
input=df_train,
trainer=SyncMultiGPUTrainerReplicated(nr_gpu))
help='per-GPU batch size',
default=32,
type=int)
parser.add_argument('--benchmark', action='store_true')
parser.add_argument('--no-zmq-ops', action='store_true')
args = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = ''
if args.fake:
ds = FakeData([[args.batch, 224, 224, 3], [args.batch]],
1000,
random=False,
dtype=['uint8', 'int32'])
else:
augs = fbresnet_augmentor(True)
ds = get_data(args.batch, augs)
logger.info("Serving data on {}".format(socket.gethostname()))
if args.benchmark:
from zmq_ops import dump_arrays
ds = MapData(ds, dump_arrays)
TestDataSpeed(ds, warmup=300).start()
else:
format = None if args.no_zmq_ops else 'zmq_ops'
send_dataflow_zmq(ds,
'ipc://@imagenet-train-b{}'.format(args.batch),
hwm=150,
format=format,
bind=True)
def get_inference_augmentor():
return fbresnet_augmentor(False)
def get_data(name, batch):
isTrain = name == 'train'
# 根据是否是训练数据获得对应的augmentors
augmentors = fbresnet_augmentor(isTrain)
return get_imagenet_dataflow(args.data, name, batch,
augmentors) # 返回图像数据流？
def get_data(name, batch):
isTrain = name == 'train'
global args
augmentors = fbresnet_augmentor(isTrain)
return get_imagenet_dataflow(args.data, name, batch, augmentors)
logger.set_logger_dir(os.path.join("train_log", "imagenet-resnet-keras"))
tf.keras.backend.set_image_data_format('channels_first')
num_gpu = get_num_gpu()
if args.fake:
df_train = FakeData([[64, 224, 224, 3], [64, 1000]],
5000,
random=False,
dtype='uint8')
df_val = FakeData([[64, 224, 224, 3], [64, 1000]], 5000, random=False)
else:
batch_size = TOTAL_BATCH_SIZE // num_gpu
assert args.data is not None
df_train = get_imagenet_dataflow(args.data, 'train', batch_size,
fbresnet_augmentor(True))
df_val = get_imagenet_dataflow(args.data, 'val', batch_size,
fbresnet_augmentor(False))
def one_hot(label):
return np.eye(1000)[label]
df_train = MapDataComponent(df_train, one_hot, 1)
df_val = MapDataComponent(df_val, one_hot, 1)
M = KerasModel(
resnet50,
inputs_desc=[InputDesc(tf.uint8, [None, 224, 224, 3], 'images')],
targets_desc=[InputDesc(tf.float32, [None, 1000], 'labels')],
input=df_train,
trainer=SyncMultiGPUTrainerReplicated(num_gpu))
def get_data(name, batch, data_aug=True):
isTrain = name == 'train'
augmentors = fbresnet_augmentor(isTrain) if data_aug \
else normal_augmentor(isTrain)
return get_imagenet_dataflow(args.data, name, batch, augmentors)
parser.add_argument('--data', help='ILSVRC dataset dir')
parser.add_argument('--fake', help='use fakedata to test or benchmark this model', action='store_true')
args = parser.parse_args()
logger.set_logger_dir("train_log/imagenet-resnet-keras")
tf.keras.backend.set_image_data_format('channels_first')
nr_gpu = get_nr_gpu()
if args.fake:
df_train = FakeData([[64, 224, 224, 3], [64, 1000]], 5000, random=False, dtype='uint8')
df_val = FakeData([[64, 224, 224, 3], [64, 1000]], 5000, random=False)
else:
batch_size = TOTAL_BATCH_SIZE // nr_gpu
assert args.data is not None
df_train = get_imagenet_dataflow(
args.data, 'train', batch_size, fbresnet_augmentor(True))
df_val = get_imagenet_dataflow(
args.data, 'val', batch_size, fbresnet_augmentor(False))
def one_hot(label):
return np.eye(1000)[label]
df_train = MapDataComponent(df_train, one_hot, 1)
df_val = MapDataComponent(df_val, one_hot, 1)
M = KerasModel(
resnet50,
inputs_desc=[InputDesc(tf.uint8, [None, 224, 224, 3], 'images')],
targets_desc=[InputDesc(tf.float32, [None, 1000], 'labels')],
input=df_train,
trainer=SyncMultiGPUTrainerReplicated(nr_gpu))
def get_data(name, batch):
isTrain = name == 'train'
augmentors = fbresnet_augmentor(isTrain)
return get_imagenet_dataflow(
args.data, name, batch, augmentors)
def get_data(dataset_name):
isTrain = dataset_name == 'train'
augmentors = fbresnet_augmentor(isTrain)
return get_imagenet_dataflow(
args.data, dataset_name, BATCH_SIZE, augmentors)