예제 #1
0
 def move(self, src, dst, **kwds):
     if self.getmeta("atomic.rename", False):
         if kwds.get("overwrite", False) or not self.exists(dst):
             try:
                 self.rename(src, dst)
                 return
             except FSError:
                 pass
     FS.move(self, src, dst, **kwds)
예제 #2
0
 def move(self, src, dst, **kwds):
     if self.getmeta("atomic.rename",False):
         if kwds.get("overwrite",False) or not self.exists(dst):
             try:
                 self.rename(src,dst)
                 return
             except FSError:
                 pass
     FS.move(self, src, dst, **kwds)
예제 #3
0
def _preprocess_images(base_fs, output_fs: FS, split):
    fname = "{}-image-tmp".format(split)
    result_path = output_fs.getsyspath(fname)

    batch_size = 32
    img_ds = RawImageDataset(base_fs, split)
    dataloader = data.DataLoader(img_ds, batch_size=batch_size, num_workers=2)
    result_size = (len(img_ds), 1024, 14, 14)
    result = np.memmap(result_path, np.float32, "w+", shape=result_size)

    resnet = get_resnet().to(config.torch_device())

    with torch.no_grad():
        progbar = tqdm(dataloader,
                       desc="Preprocessing images -- {}".format(split))
        for ix, img in enumerate(progbar):
            img = resnet(img.to(config.torch_device())).cpu().numpy()
            result[ix * batch_size:(ix + 1) * batch_size] = img

    output_fs.move(fname, "{}-image".format(split), True)
예제 #4
0
 def move(self, src, dst, **kwds):
     FS.move(self,src,dst,**kwds)
     path = relpath(normpath(src))
     with self._size_lock:
         self._file_sizes.pop(path,None)
예제 #5
0
 def move(self, src, dst, **kwds):
     FS.move(self, src, dst, **kwds)
     path = relpath(normpath(src))
     with self._size_lock:
         self._file_sizes.pop(path, None)