def convert_segmentation_dataset(split_f,
output_dir,
image_dir,
label_dir,
image_format='jpg',
label_format='png',
num_shards=4,
image_stem_transform=lambda x: x,
label_stem_transform=lambda x: x):
split_f = fmt_path(split_f)
output_dir = fmt_path(output_dir)
image_dir = fmt_path(image_dir)
label_dir = fmt_path(label_dir)
output_dir.mkdir(parents=True, exist_ok=True)
split = split_f.stem
filenames = split_f.read_text().splitlines()
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(num_shards)))
print('Processing ' + split)
for shard_id in range(num_shards):
output_f = output_dir / ('%s-%05d-of-%05d.tfrecord' %
(split, shard_id, num_shards))
with tf.io.TFRecordWriter(str(output_f)) as writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
print('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id),
end='')
image_fp = image_dir / (image_stem_transform(filenames[i]) +
'.' + image_format)
image_data, (width, height, channels) = read_image(image_fp)
label_fp = label_dir / (label_stem_transform(filenames[i]) +
'.' + label_format)
label_data, (label_width,
label_height) = read_label(label_fp,
format=label_format)
if height != label_height or width != label_width:
raise RuntimeError(
'Shape mismatched between image and label.')
example = image_label_to_tfexample(image_data, filenames[i],
image_format, height, width,
channels, label_data,
label_format)
writer.write(example.SerializeToString())
print()
def find_most_recent(work_dir, pattern):
d = fmt_path(work_dir)
saves = list(d.glob(pattern))
if len(saves) == 0:
raise FileNotFoundError("No checkpoint to load in %s" % work_dir)
fp = max(saves, key=lambda f: f.stat().st_mtime)
return fp
def convert_numpy_dataset(X, y, split, output_dir, num_shards=4):
assert len(X) == len(y)
output_dir = fmt_path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
num_examples = len(X)
num_per_shard = int(math.ceil(num_examples / float(num_shards)))
for shard_id in range(num_shards):
output_f = output_dir / ('%s-%05d-of-%05d.tfrecord' %
(split, shard_id, num_shards))
with tf.io.TFRecordWriter(str(output_f)) as writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_examples)
for i in range(start_idx, end_idx):
print('\r>> Converting image %d/%d shard %d' %
(i + 1, num_examples, shard_id),
end='')
example = tf.train.Example(features=tf.train.Features(
feature={
'image': _bytes_feature(X[i].tobytes()),
'label': _int64_feature(y[i]),
}))
writer.write(example.SerializeToString())
print()
def __init__(self,
model,
criterion,
optimizers,
train_metrics: Mapping[str, Metric],
eval_metrics: Mapping[str, Metric],
work_dir,
grad_clip_norm=0.0,
multiple_steps=True,
xla_compile=False,
output_transform=default_metric_transform,
n_batches_per_step=None):
if not isinstance(optimizers, Sequence):
optimizers = [optimizers]
optimizers = list(optimizers)
self._strategy = parse_strategy('auto')
work_dir = fmt_path(work_dir)
self.model = model
self.criterion = criterion
self.optimizers = optimizers
self.train_metrics = train_metrics
self.eval_metrics = eval_metrics
self.work_dir = work_dir
self.dtype = tf.dtypes.as_dtype(
mixed_precision.global_policy().compute_dtype)
if self.dtype == tf.float16:
self.optimizers = [
mixed_precision.LossScaleOptimizer(optimizer, 'dynamic') if
not isinstance(optimizer, mixed_precision.LossScaleOptimizer)
else optimizer for optimizer in self.optimizers
]
self.grad_clip_norm = grad_clip_norm
self.multiple_steps = multiple_steps
self.xla_compile = xla_compile
self.output_transform = output_transform
self._log_dir = self.work_dir / "runs"
self._writer = None
self._verbose = True
self._state = {
"train": {},
"eval": {},
"test": {},
}
# epoch -> stage -> metric -> value
self.metric_history = MetricHistory(["train", "eval", "test"])
self._terminated = False
self.set_global_state("epoch", tf.Variable(-1, dtype=tf.int64))
if self.xla_compile:
self.xla_train_batch = tf.function(self.train_batch,
experimental_compile=True)
self.n_batches_per_step = n_batches_per_step
def create_split_file(image_dir, target):
image_ids = [
re.match("([a-z]+_[0-9]{6}_[0-9]{6})_leftImg8bit", f.stem).group(1)
for f in eglob(image_dir, "*/*.png")
]
split_f = fmt_path(target)
split_f.write_text('\n'.join(image_ids))
return split_f
def __init__(self,
model,
criterion,
optimizers,
lr_schedulers,
train_metrics,
eval_metrics,
work_dir,
fp16=False,
device='auto',
grad_clip_norm=0.0,
channels_last=False):
if not isinstance(optimizers, Sequence):
optimizers = [optimizers]
optimizers = list(optimizers)
if not isinstance(lr_schedulers, Sequence):
lr_schedulers = [lr_schedulers]
if device == 'auto':
device = 'cuda' if CUDA else 'cpu'
device = torch.device(device)
work_dir = fmt_path(work_dir)
model.to(device)
if isinstance(criterion, nn.Module):
criterion.to(device)
if fp16:
self.scaler = torch.cuda.amp.GradScaler()
self.model = model
self.criterion = criterion
self.optimizers = optimizers
self.lr_schedulers = lr_schedulers
self.train_metrics = train_metrics
self.eval_metrics = eval_metrics
self.work_dir = work_dir
self.fp16 = fp16
self.device = device
self.grad_clip_norm = grad_clip_norm
self.channels_last = channels_last
if self.channels_last:
self.model = self.model.to(memory_format=torch.channels_last)
self._log_dir = self.work_dir / "runs"
current_time = datetime.now().strftime('%b%d_%H-%M-%S')
self._writer = SummaryWriter(str(self._log_dir / current_time),
flush_secs=10)
self._verbose = True
self._state = {
"train": {},
"eval": {},
"test": {},
}
# epoch -> stage -> metric -> value
self.metric_history = MetricHistory(["train", "eval", "test"])
self._terminated = False
def find_most_recent(work_dir, pattern):
d = fmt_path(work_dir)
pattern = pattern
saves = list(d.glob(pattern))
if len(saves) == 0:
return None
else:
fp = max(saves, key=lambda f: f.stat().st_mtime)
return fp
def __init__(self,
model: nn.Module,
criterion: Callable,
optimizers: Union[Optimizer, Sequence[Optimizer]],
lr_schedulers: Union[_LRScheduler, Sequence[_LRScheduler]],
metrics: Dict[str, Metric],
test_metrics: Dict[str, Metric],
save_path: Union[Path, str] = ".",
fp16: bool = False,
lr_step_on_iter: bool = False,
device: Optional[str] = None,
**kwargs):
# Check Arguments
if not isinstance(optimizers, Sequence):
optimizers = [optimizers]
if not isinstance(lr_schedulers, Sequence):
lr_schedulers = [lr_schedulers]
if device is None:
device = 'cuda' if CUDA else 'cpu'
device = torch.device(device)
save_path = fmt_path(save_path)
model.to(device)
if fp16:
from apex import amp
model, optimizer = amp.initialize(model,
optimizers,
opt_level="O1",
verbosity=0)
# Set Arguments
self.model = model
self.criterion = criterion
self.optimizers = optimizers
self.lr_schedulers = lr_schedulers
self.metrics = metrics
self.test_metrics = test_metrics
self.save_path = save_path
self.fp16 = fp16
self.lr_step_on_iter = lr_step_on_iter
self.device = device
self.log_path = self.save_path / "runs"
current_time = datetime.now().strftime('%b%d_%H-%M-%S')
self.writer = SummaryWriter(str(self.log_path / current_time),
flush_secs=10)
self._train_engine_state = None
self._eval_engine_state = None
self._traier_state = TrainerState.INIT
self._epochs = 0
self._kwargs = kwargs
def read_image(fp):
fp = fmt_path(fp)
b = fp.read_bytes()
img = tf.image.decode_image(b)
height, width = img.shape[:2]
if len(img.shape) == 3:
channels = img.shape[2]
else:
channels = 1
return b, (width, height, channels)
def sample(ann_file, k):
ann_file = fmt_path(ann_file)
d = read_json(ann_file)
images = d['images']
n = len(images)
indices = list(range(n))
random.shuffle(indices)
sub_indices = indices[:k]
extract(ann_file, d, sub_indices, suffix="sub")
def save(self, save_dir=None, model_only=False):
if save_dir is None:
save_dir = self.work_dir
else:
save_dir = fmt_path(save_dir)
files = list(eglob(save_dir, "ckpt.*"))
if len(files) != 0:
for f in files:
f.write_bytes(b'')
rm(f)
save_path = str(save_dir / "ckpt")
ckpt, ckpt_options = self._make_ckpt(model_only=model_only)
path = ckpt.write(save_path, ckpt_options)
print('Save learner to %s' % path)
def train_test_split(ann_file, test_ratio, seed=0):
ann_file = fmt_path(ann_file)
d = read_json(ann_file)
n = len(d['images'])
n_test = int(n * test_ratio)
n_train = n - n_test
indices = list(range(n))
random.seed(seed)
random.shuffle(indices)
train_indices = indices[:n_train]
test_indices = indices[n_train:]
extract(ann_file, d, train_indices, "train")
extract(ann_file, d, test_indices, "test")
from tqdm import tqdm
from PIL import Image
import numpy as np
from hhutil.io import fmt_path, eglob
c = np.bincount([], minlength=256)
d = fmt_path(
"/Users/hrvvi/Downloads/datasets/Cityscapes/gtFine_trainvaltest/gtFine/val"
)
images = list(eglob(d, "*/*_gtFine_labelIds.png"))
for img in tqdm(images):
x = np.array(Image.open(img))
c += np.bincount(x.flat, minlength=256)
label_smoothing=cfg.get("label_smooth"))
epochs = cfg.epochs
optimizer = get_optimizer(cfg.Optimizer, model.parameters())
lr_scheduler = get_lr_scheduler(cfg.LRScheduler, optimizer, epochs)
train_metrics = {'loss': TrainLoss()}
if not use_mix:
train_metrics['acc'] = Accuracy()
test_metrics = {
'loss': Loss(CrossEntropyLoss()),
'acc': Accuracy(),
}
work_dir = fmt_path(cfg.get("work_dir"))
trainer = Trainer(model,
criterion,
optimizer,
lr_scheduler,
train_metrics,
test_metrics,
work_dir=work_dir,
fp16=fp16,
device=cfg.get("device", 'auto'))
if args.resume:
resume(trainer, args.resume)
eval_freq = cfg.get("eval_freq", 1)
save_freq = cfg.get("save_freq")
import re
import random
from hhutil.io import fmt_path, eglob, copy
sample_ratio = 0.01
min_sample_class = 2
src = fmt_path(f"/Users/hrvvi/Downloads/datasets/Cityscapes")
tgt = fmt_path(f"/Users/hrvvi/Downloads/datasets/Cityscapes_sub")
src_image, src_label = src / "leftImg8bit_trainvaltest", src / "gtFine_trainvaltest"
tgt_image, tgt_label = tgt / "leftImg8bit_trainvaltest", tgt / "gtFine_trainvaltest"
tgt_image.mkdir(parents=True, exist_ok=True)
tgt_label.mkdir(parents=True, exist_ok=True)
for f in ["license.txt", "README"]:
copy(src_image / f, tgt_image)
copy(src_label / f, tgt_label)
for split in ["train", "val", "test"]:
src_image_split, src_label_split = src_image / "leftImg8bit" / split, src_label / "gtFine" / split
tgt_image_split, tgt_label_split = tgt_image / "leftImg8bit" / split, tgt_label / "gtFine" / split
for class_d in eglob(src_image_split, "*"):
class_name = class_d.stem
print(split, class_name)
src_image_split_class, src_label_split_class = src_image_split / class_name, src_label_split / class_name
tgt_image_split_class, tgt_label_split_class = tgt_image_split / class_name, tgt_label_split / class_name
tgt_image_split_class.mkdir(parents=True, exist_ok=True)
tgt_label_split_class.mkdir(parents=True, exist_ok=True)
image_ids = [
re.match("([a-z]+_[0-9]{6}_[0-9]{6})_leftImg8bit", image_f.stem).group(1)
for image_f in eglob(src_image_split_class, "*.png")
]
})
return train_results, valid_results
def parse(self, log_file):
train_ends, train_losses, train_accs, valid_ends, valid_losses, valid_accs = self.extract(
log_file)
total_epochs = len(valid_ends)
epoch_train_time = (parse(valid_ends[-1]) -
parse(valid_ends[0])).seconds / (total_epochs - 1)
print(f"%.4f(%.4f) %.4f(%.4f) %.1f" %
(valid_accs[-1], valid_accs.max(), train_losses[-1],
train_losses[valid_accs.argmax() - len(valid_accs)],
epoch_train_time))
root = fmt_path("/Users/hrvvi/Code/Library/experiments/CIFAR100-TensorFlow")
parser = LogParser()
def get_logs(n):
return list(eglob(root / "log", "%d-*.log" % n))
def get_metrics(n, metrics):
train_res, valid_res = zip(*[parser.extract(log) for log in get_logs(n)])
results = []
for m in metrics:
stage, metric, i = m.split("_")
res = train_res if stage == 'train' else valid_res
if i == 'max':
results.append(np.array([df[metric].max() for df in res]))
import argparse
from hhutil.io import fmt_path
from hanser.datasets.segmentation.tfrecord import convert_segmentation_dataset
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Convert VOC Segmentation to TFRecord')
parser.add_argument('-r', '--root', help='VOC root')
parser.add_argument('-o', '--output_dir', help='output path')
parser.add_argument('-p', '--part', help='part')
parser.add_argument('-n', '--num_shards', help='number of shards')
args = parser.parse_args()
root = fmt_path(args.root)
output_dir = fmt_path(args.output_dir)
part = args.part
num_shards = int(args.num_shards)
split_f = root / f"ImageSets/Segmentation/{part}.txt"
image_dir = root / 'JPEGImages'
label_dir = root / root / 'SegmentationClassAug'
convert_segmentation_dataset(split_f, output_dir, image_dir, label_dir, num_shards=num_shards)
import h5py
import imagesize
from hhutil.io import fmt_path, save_json
root = fmt_path("/Users/hrvvi/Downloads/test")
f = h5py.File(root / "digitStruct.mat")
d = f['digitStruct']
def read_name(ref):
x = d[ref][:].tostring().decode('UTF-16LE')
return x
def read_bboxes(ref):
g = d[ref]
keys = list(g.keys())
n = g[keys[0]].shape[0]
results = []
if n == 1:
xs = []
for k in ['left', 'top', 'width', 'height', 'label']:
x = g[k][:][0, 0]
xs.append(x)
bboxes = [xs[:4]]
labels = [xs[4]]
return bboxes, labels
else:
for k in ['left', 'top', 'width', 'height', 'label']: