def before_run(self, runner): if not self.out_dir: self.out_dir = runner.work_dir self.file_client = FileClient.infer_client(self.file_client_args, self.out_dir) # if `self.out_dir` is not equal to `runner.work_dir`, it means that # `self.out_dir` is set so the final `self.out_dir` is the # concatenation of `self.out_dir` and the last level directory of # `runner.work_dir` if self.out_dir != runner.work_dir: basename = osp.basename(runner.work_dir.rstrip(osp.sep)) self.out_dir = self.file_client.join_path(self.out_dir, basename) runner.logger.info( (f'The best checkpoint will be saved to {self.out_dir} by ' f'{self.file_client.name}')) if self.save_best is not None: if runner.meta is None: warnings.warn('runner.meta is None. Creating an empty one.') runner.meta = dict() runner.meta.setdefault('hook_msgs', dict()) self.best_ckpt_path = runner.meta['hook_msgs'].get( 'best_ckpt', None)
def before_run(self, runner): if not self.out_dir: self.out_dir = runner.work_dir self.file_client = FileClient.infer_client(self.file_client_args, self.out_dir) # if `self.out_dir` is not equal to `runner.work_dir`, it means that # `self.out_dir` is set so the final `self.out_dir` is the # concatenation of `self.out_dir` and the last level directory of # `runner.work_dir` if self.out_dir != runner.work_dir: basename = osp.basename(runner.work_dir.rstrip(osp.sep)) self.out_dir = self.file_client.join_path(self.out_dir, basename) runner.logger.info((f'Checkpoints will be saved to {self.out_dir} by ' f'{self.file_client.name}.')) # disable the create_symlink option because some file backends do not # allow to create a symlink if 'create_symlink' in self.args: if self.args[ 'create_symlink'] and not self.file_client.allow_symlink: self.args['create_symlink'] = False warnings.warn( ('create_symlink is set as True by the user but is changed' 'to be False because creating symbolic link is not ' f'allowed in {self.file_client.name}')) else: self.args['create_symlink'] = self.file_client.allow_symlink
def imwrite(img, file_path, params=None, auto_mkdir=None, file_client_args=None): """Write image to file. Note: In v1.4.1 and later, add `file_client_args` parameters. Warning: The parameter `auto_mkdir` will be deprecated in the future and every file clients will make directory automatically. Args: img (ndarray): Image array to be written. file_path (str): Image file path. params (None or list): Same as opencv :func:`imwrite` interface. auto_mkdir (bool): If the parent folder of `file_path` does not exist, whether to create it automatically. It will be deprecated. file_client_args (dict | None): Arguments to instantiate a FileClient. See :class:`mmcv.fileio.FileClient` for details. Default: None. Returns: bool: Successful or not. Examples: >>> # write to hard disk client >>> ret = mmcv.imwrite(img, '/path/to/img.jpg') >>> # infer the file backend by the prefix s3 >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg') >>> # manually set the file backend petrel >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', file_client_args={ ... 'backend': 'petrel'}) """ assert is_filepath(file_path) file_path = str(file_path) if auto_mkdir is not None: warnings.warn( 'The parameter `auto_mkdir` will be deprecated in the future and ' 'every file clients will make directory automatically.') file_client = FileClient.infer_client(file_client_args, file_path) img_ext = osp.splitext(file_path)[-1] # Encode image according to image suffix. # For example, if image path is '/path/your/img.jpg', the encode # format is '.jpg'. flag, img_buff = cv2.imencode(img_ext, img, params) file_client.put(img_buff.tobytes(), file_path) return flag
def imread(img_or_path, flag='color', channel_order='bgr', backend=None, file_client_args=None): """Read an image. Note: In v1.4.1 and later, add `file_client_args` parameters. Args: img_or_path (ndarray or str or Path): Either a numpy array or str or pathlib.Path. If it is a numpy array (loaded image), then it will be returned as is. flag (str): Flags specifying the color type of a loaded image, candidates are `color`, `grayscale`, `unchanged`, `color_ignore_orientation` and `grayscale_ignore_orientation`. By default, `cv2` and `pillow` backend would rotate the image according to its EXIF info unless called with `unchanged` or `*_ignore_orientation` flags. `turbojpeg` and `tifffile` backend always ignore image's EXIF info regardless of the flag. The `turbojpeg` backend only supports `color` and `grayscale`. channel_order (str): Order of channel, candidates are `bgr` and `rgb`. backend (str | None): The image decoding backend type. Options are `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`. If backend is None, the global imread_backend specified by ``mmcv.use_backend()`` will be used. Default: None. file_client_args (dict | None): Arguments to instantiate a FileClient. See :class:`mmcv.fileio.FileClient` for details. Default: None. Returns: ndarray: Loaded image array. Examples: >>> import mmcv >>> img_path = '/path/to/img.jpg' >>> img = mmcv.imread(img_path) >>> img = mmcv.imread(img_path, flag='color', channel_order='rgb', ... backend='cv2') >>> img = mmcv.imread(img_path, flag='color', channel_order='bgr', ... backend='pillow') >>> s3_img_path = 's3://bucket/img.jpg' >>> # infer the file backend by the prefix s3 >>> img = mmcv.imread(s3_img_path) >>> # manually set the file backend petrel >>> img = mmcv.imread(s3_img_path, file_client_args={ ... 'backend': 'petrel'}) >>> http_img_path = 'http://path/to/img.jpg' >>> img = mmcv.imread(http_img_path) >>> img = mmcv.imread(http_img_path, file_client_args={ ... 'backend': 'http'}) """ if isinstance(img_or_path, Path): img_or_path = str(img_or_path) if isinstance(img_or_path, np.ndarray): return img_or_path elif is_str(img_or_path): file_client = FileClient.infer_client(file_client_args, img_or_path) img_bytes = file_client.get(img_or_path) return imfrombytes(img_bytes, flag, channel_order, backend) else: raise TypeError('"img" must be a numpy array or a str or ' 'a pathlib.Path object')
def create_test_job_batch(commands, model_info, args, port, script_name): fname = model_info.name config = Path(model_info.config) assert config.exists(), f'{fname}: {config} not found.' http_prefix = 'https://download.openmmlab.com/mmclassification/' if 's3://' in args.checkpoint_root: from mmcv.fileio import FileClient from petrel_client.common.exception import AccessDeniedError file_client = FileClient.infer_client(uri=args.checkpoint_root) checkpoint = file_client.join_path( args.checkpoint_root, model_info.weights[len(http_prefix):]) try: exists = file_client.exists(checkpoint) except AccessDeniedError: exists = False else: checkpoint_root = Path(args.checkpoint_root) checkpoint = checkpoint_root / model_info.weights[len(http_prefix):] exists = checkpoint.exists() if not exists: print(f'WARNING: {fname}: {checkpoint} not found.') return None job_name = f'{args.job_name}_{fname}' work_dir = Path(args.work_dir) / fname work_dir.mkdir(parents=True, exist_ok=True) if args.mail is not None and 'NONE' not in args.mail_type: mail_cfg = (f'#SBATCH --mail {args.mail}\n' f'#SBATCH --mail-type {args.mail_type}\n') else: mail_cfg = '' if args.quotatype is not None: quota_cfg = f'#SBATCH --quotatype {args.quotatype}\n' else: quota_cfg = '' launcher = 'none' if args.local else 'slurm' runner = 'python' if args.local else 'srun python' job_script = (f'#!/bin/bash\n' f'#SBATCH --output {work_dir}/job.%j.out\n' f'#SBATCH --partition={args.partition}\n' f'#SBATCH --job-name {job_name}\n' f'#SBATCH --gres=gpu:8\n' f'{mail_cfg}{quota_cfg}' f'#SBATCH --ntasks-per-node=8\n' f'#SBATCH --ntasks=8\n' f'#SBATCH --cpus-per-task=5\n\n' f'{runner} -u {script_name} {config} {checkpoint} ' f'--out={work_dir / "result.pkl"} --metrics accuracy ' f'--out-items=none ' f'--cfg-option dist_params.port={port} ' f'--launcher={launcher}\n') with open(work_dir / 'job.sh', 'w') as f: f.write(job_script) commands.append(f'echo "{config}"') if args.local: commands.append(f'bash {work_dir}/job.sh') else: commands.append(f'sbatch {work_dir}/job.sh') return work_dir / 'job.sh'