def test_delta_mark_classification():
validation_size = 1/3
batch_size = 3
image_size = [256, 128]
all_data_num = 3
train_dataset = setup_dataset(Dummy,
subset="train",
validation_size=validation_size,
batch_size=batch_size,
pre_processor=Resize(image_size))
validation_dataset = setup_dataset(Dummy,
subset="validation",
validation_size=validation_size,
batch_size=batch_size,
pre_processor=Resize(image_size))
assert train_dataset.num_per_epoch == (1 - validation_size) * all_data_num
assert validation_dataset.num_per_epoch == validation_size * all_data_num
for _ in range(5):
images, labels = train_dataset.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == train_dataset.num_classes
def test_training():
"""Test only no error raised."""
config = EasyDict()
config.NETWORK_CLASS = Darknet
config.DATASET_CLASS = Dummy
config.IS_DEBUG = False
config.IMAGE_SIZE = [28, 14]
config.BATCH_SIZE = 2
config.TEST_STEPS = 1
config.MAX_STEPS = 2
config.SAVE_CHECKPOINT_STEPS = 1
config.KEEP_CHECKPOINT_MAX = 5
config.SUMMARISE_STEPS = 1
config.IS_PRETRAIN = False
config.TASK = Tasks.CLASSIFICATION
# network model config
config.NETWORK = EasyDict()
config.NETWORK.OPTIMIZER_CLASS = tf.train.AdamOptimizer
config.NETWORK.OPTIMIZER_KWARGS = {"learning_rate": 0.001}
config.NETWORK.IMAGE_SIZE = config.IMAGE_SIZE
config.NETWORK.BATCH_SIZE = config.BATCH_SIZE
# daasegt config
config.DATASET = EasyDict()
config.DATASET.PRE_PROCESSOR = Resize(config.IMAGE_SIZE)
config.DATASET.BATCH_SIZE = config.BATCH_SIZE
environment.init("test_darknet")
prepare_dirs(recreate=True)
start_training(config)
def test_training():
"""Verify only that no error raised."""
config = SmartDict()
config.NETWORK_CLASS = LMBiSeNet
config.DATASET_CLASS = DummyCamvid
config.IS_DEBUG = False
config.IMAGE_SIZE = [128, 160]
config.BATCH_SIZE = 2
config.TEST_STEPS = 1
config.MAX_STEPS = 2
config.SAVE_CHECKPOINT_STEPS = 1
config.KEEP_CHECKPOINT_MAX = 5
config.SUMMARISE_STEPS = 1
config.IS_PRETRAIN = False
config.TASK = Tasks.SEMANTIC_SEGMENTATION
# network model config
config.NETWORK = SmartDict()
config.NETWORK.OPTIMIZER_CLASS = tf.train.AdamOptimizer
config.NETWORK.OPTIMIZER_KWARGS = {"learning_rate": 0.001}
config.NETWORK.IMAGE_SIZE = config.IMAGE_SIZE
config.NETWORK.BATCH_SIZE = config.BATCH_SIZE
config.NETWORK.DATA_FORMAT = "NHWC"
# daasegt config
config.DATASET = SmartDict()
config.DATASET.PRE_PROCESSOR = Resize(config.IMAGE_SIZE)
config.DATASET.BATCH_SIZE = config.BATCH_SIZE
config.DATASET.DATA_FORMAT = "NHWC"
environment.init("test_lm_bisenet")
prepare_dirs(recreate=True)
start_training(config, profile_step=1)
def test_delta_mark_classification_has_validation_path():
batch_size = 3
image_size = [256, 128]
train_data_num = 3
validation_data_num = 2
train_dataset = setup_dataset(DummyHasValidation,
subset="train",
batch_size=batch_size,
pre_processor=Resize(image_size))
validation_dataset = setup_dataset(DummyHasValidation,
subset="validation",
batch_size=batch_size,
pre_processor=Resize(image_size))
assert train_dataset.num_per_epoch == train_data_num
assert validation_dataset.num_per_epoch == validation_data_num
for _ in range(5):
images, labels = train_dataset.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == train_dataset.num_classes
for _ in range(5):
images, labels = validation_dataset.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == validation_dataset.num_classes
def test_image_folder():
validation_size = 0.2
batch_size = 3
image_size = [256, 256]
train_dataset = setup_dataset(Dummy,
subset="train",
validation_size=validation_size,
batch_size=batch_size,
pre_processor=Resize(image_size))
validation_dataset = setup_dataset(Dummy,
subset="validation",
validation_size=validation_size,
batch_size=batch_size,
pre_processor=Resize(image_size))
expected_image_dir = os.path.join(environment.DATA_DIR, Dummy.extend_dir)
expected_paths = [
image_path
for image_path in glob(os.path.join(expected_image_dir, "**/*"))
if os.path.isfile(image_path)
and imghdr.what(image_path) in {"jpeg", "png"}
]
assert len(expected_paths) * (
1 - validation_size) == train_dataset.num_per_epoch
assert len(expected_paths) * (
validation_size) == validation_dataset.num_per_epoch
for _ in range(5):
images, labels = train_dataset.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == train_dataset.num_classes
def test_resize():
IMAGE_SIZE = [32, 32]
orig_image = np.zeros(shape=(1024, 512, 3), dtype=np.uint8)
orig_mask = np.zeros(shape=(1024, 512, 3), dtype=np.uint8)
pre_processor = Resize(IMAGE_SIZE)
resized = pre_processor(image=orig_image, mask=orig_mask)
resized_image = resized["image"]
resized_mask = resized["mask"]
assert isinstance(resized_image, np.ndarray)
assert isinstance(resized_mask, np.ndarray)
assert resized_image.shape[:2] == (32, 32)
assert resized_image.shape[2] == 3
assert resized_mask.shape[:2] == (32, 32)
assert resized_mask.shape[2] == 3
def test_open_images_v4_classification():
batch_size = 1
image_size = [256, 256]
dataset = OpenImagesV4Classification(batch_size=batch_size,
pre_processor=Resize(image_size))
dataset = DatasetIterator(dataset)
for _ in range(5):
images, labels = dataset.feed()
assert isinstance(images, np.ndarray)
assert images.shape[0] == batch_size
assert images.shape[1] == image_size[0]
assert images.shape[2] == image_size[1]
assert images.shape[3] == 3
assert isinstance(labels, np.ndarray)
assert labels.shape[0] == batch_size
assert labels.shape[1] == dataset.num_classes
def test_training():
"""Test only that no error raised."""
config = EasyDict()
config.NETWORK_CLASS = LmResnetQuantize
config.DATASET_CLASS = Dummy
config.IS_DEBUG = False
config.IMAGE_SIZE = [32, 32]
config.BATCH_SIZE = 2
config.TEST_STEPS = 1
config.MAX_STEPS = 2
config.SAVE_CHECKPOINT_STEPS = 1
config.KEEP_CHECKPOINT_MAX = 5
config.SUMMARISE_STEPS = 1
config.IS_PRETRAIN = False
config.TASK = Tasks.CLASSIFICATION
# network model config
config.NETWORK = EasyDict()
config.NETWORK.OPTIMIZER_CLASS = tf.train.AdamOptimizer
config.NETWORK.OPTIMIZER_KWARGS = {"learning_rate": 0.001}
config.NETWORK.IMAGE_SIZE = config.IMAGE_SIZE
config.NETWORK.BATCH_SIZE = config.BATCH_SIZE
config.NETWORK.ACTIVATION_QUANTIZER = linear_mid_tread_half_quantizer
config.NETWORK.ACTIVATION_QUANTIZER_KWARGS = {
'bit': 2,
'max_value': 2
}
config.NETWORK.WEIGHT_QUANTIZER = binary_mean_scaling_quantizer
config.NETWORK.WEIGHT_QUANTIZER_KWARGS = {}
# dataset config
config.DATASET = EasyDict()
config.DATASET.PRE_PROCESSOR = Resize(config.IMAGE_SIZE)
config.DATASET.BATCH_SIZE = config.BATCH_SIZE
environment.init("test_lm_resnet_quantize")
prepare_dirs(recreate=True)
start_training(config)
CLASSES = DATASET_CLASS.classes
MAX_STEPS = 100000
SAVE_CHECKPOINT_STEPS = 5000
KEEP_CHECKPOINT_MAX = 5
TEST_STEPS = 1000
SUMMARISE_STEPS = 100
# pretrain
IS_PRETRAIN = False
PRETRAIN_VARS = []
PRETRAIN_DIR = ""
PRETRAIN_FILE = ""
PRE_PROCESSOR = Sequence([
Resize(size=IMAGE_SIZE),
DivideBy255(),
])
POST_PROCESSOR = None
STEP_PER_EPOCH = 50000 // BATCH_SIZE
TUNE_SPEC = {
'run': 'tunable',
'resources_per_trial': {
"cpu": 2,
"gpu": 0.5
},
'stop': {
'mean_accuracy': 0.87,
'training_iteration': 200,
DATA_FORMAT = "NHWC"
TASK = Tasks.CLASSIFICATION
CLASSES = DATASET_CLASS.classes
MAX_STEPS = 100000
SAVE_CHECKPOINT_STEPS = 1000
KEEP_CHECKPOINT_MAX = 5
TEST_STEPS = 1000
SUMMARISE_STEPS = 100
# pretrain
IS_PRETRAIN = False
PRETRAIN_VARS = []
PRETRAIN_DIR = ""
PRETRAIN_FILE = ""
PRE_PROCESSOR = Sequence([Resize(size=IMAGE_SIZE), DivideBy255()])
POST_PROCESSOR = None
NETWORK = SmartDict()
NETWORK.OPTIMIZER_CLASS = tf.compat.v1.train.MomentumOptimizer
NETWORK.OPTIMIZER_KWARGS = {"momentum": 0.9}
NETWORK.LEARNING_RATE_FUNC = tf.compat.v1.train.cosine_decay
# Train data num is 28709
step_per_epoch = 28709 // BATCH_SIZE
NETWORK.LEARNING_RATE_KWARGS = {'learning_rate': 0.1, 'decay_steps': 100000}
NETWORK.IMAGE_SIZE = IMAGE_SIZE
NETWORK.BATCH_SIZE = BATCH_SIZE
NETWORK.DATA_FORMAT = DATA_FORMAT
NETWORK.WEIGHT_DECAY_RATE = 0.0005
NETWORK.ACTIVATION_QUANTIZER = linear_mid_tread_half_quantizer
NETWORK.ACTIVATION_QUANTIZER_KWARGS = {'bit': 2, 'max_value': 2}
TASK = Tasks.CLASSIFICATION
CLASSES = DATASET_CLASS(subset="train", batch_size=1).classes
MAX_STEPS = 2
SAVE_CHECKPOINT_STEPS = 1
KEEP_CHECKPOINT_MAX = 5
TEST_STEPS = 100
SUMMARISE_STEPS = 100
# pretrain
IS_PRETRAIN = False
PRETRAIN_VARS = []
PRETRAIN_DIR = ""
PRETRAIN_FILE = ""
PRE_PROCESSOR = Sequence([Resize(size=IMAGE_SIZE), PerImageStandardization()])
POST_PROCESSOR = None
NETWORK = SmartDict()
NETWORK.OPTIMIZER_CLASS = tf.train.AdamOptimizer
NETWORK.OPTIMIZER_KWARGS = {"learning_rate": 0.001}
NETWORK.IMAGE_SIZE = IMAGE_SIZE
NETWORK.BATCH_SIZE = BATCH_SIZE
NETWORK.DATA_FORMAT = DATA_FORMAT
NETWORK.WEIGHT_DECAY_RATE = 0.0005
NETWORK.ACTIVATION_QUANTIZER = linear_mid_tread_half_quantizer
NETWORK.ACTIVATION_QUANTIZER_KWARGS = {'bit': 2, 'max_value': 2}
NETWORK.WEIGHT_QUANTIZER = binary_mean_scaling_quantizer
NETWORK.WEIGHT_QUANTIZER_KWARGS = {}
# dataset
