def move(self, src, dst, **kwds):
if self.getmeta("atomic.rename", False):
if kwds.get("overwrite", False) or not self.exists(dst):
try:
self.rename(src, dst)
return
except FSError:
pass
FS.move(self, src, dst, **kwds)
def move(self, src, dst, **kwds):
if self.getmeta("atomic.rename",False):
if kwds.get("overwrite",False) or not self.exists(dst):
try:
self.rename(src,dst)
return
except FSError:
pass
FS.move(self, src, dst, **kwds)
def _preprocess_images(base_fs, output_fs: FS, split):
fname = "{}-image-tmp".format(split)
result_path = output_fs.getsyspath(fname)
batch_size = 32
img_ds = RawImageDataset(base_fs, split)
dataloader = data.DataLoader(img_ds, batch_size=batch_size, num_workers=2)
result_size = (len(img_ds), 1024, 14, 14)
result = np.memmap(result_path, np.float32, "w+", shape=result_size)
resnet = get_resnet().to(config.torch_device())
with torch.no_grad():
progbar = tqdm(dataloader,
desc="Preprocessing images -- {}".format(split))
for ix, img in enumerate(progbar):
img = resnet(img.to(config.torch_device())).cpu().numpy()
result[ix * batch_size:(ix + 1) * batch_size] = img
output_fs.move(fname, "{}-image".format(split), True)
def move(self, src, dst, **kwds):
FS.move(self,src,dst,**kwds)
path = relpath(normpath(src))
with self._size_lock:
self._file_sizes.pop(path,None)
def move(self, src, dst, **kwds):
FS.move(self, src, dst, **kwds)
path = relpath(normpath(src))
with self._size_lock:
self._file_sizes.pop(path, None)
