def simple_default_setup(cfg): """ Perform some basic common setups at the beginning of a job, including: 1. Set up the detectron2 logger 2. Log basic information about environment, and config 3. Backup the config to the output directory Args: cfg (CfgNode): the full config to be used """ output_dir = cfg.OUTPUT_DIR if comm.is_main_process() and output_dir: PathManager.mkdirs(output_dir) rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank, name="fvcore") logger = setup_logger(output_dir, distributed_rank=rank) logger.info("Rank of current process: {}. World size: {}".format( rank, comm.get_world_size())) logger.info("Environment info:\n" + collect_env_info()) logger.info("Running with full config:\n{}".format(cfg)) if comm.is_main_process() and output_dir: # Note: some of our scripts may expect the existence of # config.yaml in output directory path = os.path.join(output_dir, "config.yaml") with PathManager.open(path, "w") as f: f.write(cfg.dump()) logger.info("Full config saved to {}".format(os.path.abspath(path))) # make sure each worker has a different, yet deterministic seed if specified seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank)
def default_setup(cfg, args): """ Perform some basic common setups at the beginning of a job, including: 1. Set up the detectron2 logger 2. Log basic information about environment, cmdline arguments, and config 3. Backup the config to the output directory Args: cfg (CfgNode or omegaconf.DictConfig): the full config to be used args (argparse.NameSpace): the command line arguments to be logged """ output_dir = _try_get_key(cfg, "OUTPUT_DIR", "output_dir", "train.output_dir") if comm.is_main_process() and output_dir: PathManager.mkdirs(output_dir) rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank, name="fvcore") logger = setup_logger(output_dir, distributed_rank=rank) logger.info("Rank of current process: {}. World size: {}".format( rank, comm.get_world_size())) logger.info("Environment info:\n" + collect_env_info()) logger.info("Command line arguments: " + str(args)) if hasattr(args, "config_file") and args.config_file != "": logger.info("Contents of args.config_file={}:\n{}".format( args.config_file, _highlight( PathManager.open(args.config_file, "r").read(), args.config_file), )) if comm.is_main_process() and output_dir: # Note: some of our scripts may expect the existence of # config.yaml in output directory path = os.path.join(output_dir, "config.yaml") if isinstance(cfg, CfgNode): logger.info("Running with full config:\n{}".format( _highlight(cfg.dump(), ".yaml"))) with PathManager.open(path, "w") as f: f.write(cfg.dump()) else: LazyConfig.save(cfg, path) logger.info("Full config saved to {}".format(path)) # make sure each worker has a different, yet deterministic seed if specified seed = _try_get_key(cfg, "SEED", "train.seed", default=-1) seed_all_rng(None if seed < 0 else seed + rank) # cudnn benchmark has large overhead. It shouldn't be used considering the small size of # typical validation set. if not (hasattr(args, "eval_only") and args.eval_only): torch.backends.cudnn.benchmark = _try_get_key(cfg, "CUDNN_BENCHMARK", "train.cudnn_benchmark", default=False)
def default_setup(cfg, args): """ Perform some basic common setups at the beginning of a job, including: 1. Set up the detectron2 logger 2. Log basic information about environment, cmdline arguments, and config 3. Backup the config to the output directory Args: cfg (CfgNode): the full config to be used args (argparse.NameSpace): the command line arguments to be logged """ output_dir = cfg.OUTPUT_DIR if comm.is_main_process() and output_dir: PathManager.mkdirs(output_dir) if cfg.OWOD.COMPUTE_ENERGY: PathManager.mkdirs(os.path.join(output_dir, cfg.OWOD.ENERGY_SAVE_PATH)) if cfg.OWOD.ENABLE_CLUSTERING: PathManager.mkdirs(os.path.join(output_dir, cfg.OWOD.FEATURE_STORE_SAVE_PATH)) rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank, name="fvcore") logger = setup_logger(output_dir, distributed_rank=rank) logger.info("Rank of current process: {}. World size: {}".format(rank, comm.get_world_size())) logger.info("Environment info:\n" + collect_env_info()) logger.info("Command line arguments: " + str(args)) if hasattr(args, "config_file") and args.config_file != "": logger.info( "Contents of args.config_file={}:\n{}".format( args.config_file, PathManager.open(args.config_file, "r").read() ) ) logger.info("Running with full config:\n{}".format(cfg)) if comm.is_main_process() and output_dir: # Note: some of our scripts may expect the existence of # config.yaml in output directory path = os.path.join(output_dir, "config.yaml") with PathManager.open(path, "w") as f: f.write(cfg.dump()) logger.info("Full config saved to {}".format(path)) # make sure each worker has a different, yet deterministic seed if specified seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank) # cudnn benchmark has large overhead. It shouldn't be used considering the small size of # typical validation set. if not (hasattr(args, "eval_only") and args.eval_only): torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
def my_default_setup(cfg, args): """Perform some basic common setups at the beginning of a job, including: 1. Set up the detectron2 logger 2. Log basic information about environment, cmdline arguments, and config 3. Backup the config to the output directory Args: cfg (CfgNode): the full config to be used args (argparse.NameSpace): the command line arguments to be logged """ output_dir = cfg.OUTPUT_DIR if comm.is_main_process() and output_dir: mmcv.mkdir_or_exist(output_dir) rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank) for _mod in ["PIL", "chardet"]: # disable DEBUG logs logging.getLogger(_mod).setLevel(logging.INFO) logger.info("Rank of current process: {}. World size: {}".format( rank, comm.get_world_size())) logger.info("Environment info:\n" + collect_env_info()) logger.info("Command line arguments: " + str(args)) if hasattr(args, "config_file") and args.config_file != "": logger.info("Contents of args.config_file={}:\n{}".format( args.config_file, PathManager.open(args.config_file, "r").read())) logger.info("Running with full config:\n{}".format(cfg)) if comm.is_main_process() and output_dir: # Note: some of our scripts may expect the existence of # config.yaml in output directory # path = os.path.join(output_dir, "config.yaml") # with PathManager.open(path, "w") as f: # f.write(cfg.dump()) path = osp.join(output_dir, osp.basename(args.config_file)) cfg.dump(path) logger.info("Full config saved to {}".format(path)) assert ( args.num_gpus <= torch.cuda.device_count() and args.num_gpus >= 1 ), f"args.num_gpus: {args.num_gpus}, available num gpus: {torch.cuda.device_count()}" # make sure each worker has a different, yet deterministic seed if specified seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank) # cudnn benchmark has large overhead. It shouldn't be used considering the small size of # typical validation set. if not (hasattr(args, "eval_only") and args.eval_only): torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
def test_training_sampler_seed(self): seed_all_rng(42) sampler = TrainingSampler(30) data = list(itertools.islice(sampler, 65)) seed_all_rng(42) sampler = TrainingSampler(30) seed_all_rng(999) # should be ineffective data2 = list(itertools.islice(sampler, 65)) self.assertEqual(data, data2)
def worker_init_reset_seed(worker_id): seed_all_rng(np.random.randint(2**31) + worker_id)
def worker_init_reset_seed(worker_id): initial_seed = torch.initial_seed() % 2**31 seed_all_rng(initial_seed + worker_id)
import os from collections import defaultdict import captioning.utils.opts as opts import captioning.models as models from captioning.data.pth_loader import CaptionDataset import captioning.utils.eval_utils as eval_utils import captioning.utils.misc as utils from captioning.utils.rewards import init_scorer, get_self_critical_reward from captioning.modules.loss_wrapper import LossWrapper import pytorch_lightning as pl import detectron2.utils.comm as d2comm from detectron2.utils.env import seed_all_rng seed_all_rng(1234) class LitModel(pl.LightningModule): def __init__(self, opt): super().__init__() self.opt = opt # Intilaize dataset self.dataset = CaptionDataset(opt) opt.vocab_size = self.dataset.vocab_size opt.seq_length = self.dataset.seq_length self.batch_size = opt.batch_size # Build model opt.vocab = self.dataset.get_vocab() model = models.setup(opt)
def default_setup(cfg, args): """ Perform some basic common setups at the beginning of a job, including: 1. Set up the detectron2 logger 2. Log basic information about environment, cmdline arguments, and config 3. Backup the config to the output directory Args: cfg (CfgNode): the full config to be used args (argparse.NameSpace): the command line arguments to be logged """ if args.aml: if args.itp: data_store = '/' + os.environ['AZUREML_DATAREFERENCE_{}'.format( args.aml_data_store)].split('/')[-1] else: data_store = os.environ['AZUREML_DATAREFERENCE_{}'.format( args.aml_data_store)] cfg.defrost() cfg.OUTPUT_DIR = os.path.join( data_store, args.aml_work_dir_prefix, os.path.splitext(os.path.basename(args.config_file))[0]) cfg.freeze() print('output directory: ', cfg.OUTPUT_DIR) else: cfg.defrost() cfg.OUTPUT_DIR = os.path.join( './work_dirs', os.path.splitext(os.path.basename(args.config_file))[0]) cfg.freeze() output_dir = cfg.OUTPUT_DIR if comm.is_main_process() and output_dir: PathManager.mkdirs(output_dir) rank = comm.get_rank() setup_logger(output_dir, distributed_rank=rank, name="fvcore") logger = setup_logger(output_dir, distributed_rank=rank) logger.info("Rank of current process: {}. World size: {}".format( rank, comm.get_world_size())) logger.info("Environment info:\n" + collect_env_info()) logger.info("Command line arguments: " + str(args)) if hasattr(args, "config_file") and args.config_file != "": logger.info("Contents of args.config_file={}:\n{}".format( args.config_file, PathManager.open(args.config_file, "r").read())) logger.info("Running with full config:\n{}".format(cfg)) if comm.is_main_process() and output_dir: # Note: some of our scripts may expect the existence of # config.yaml in output directory path = os.path.join(output_dir, "config.yaml") with PathManager.open(path, "w") as f: f.write(cfg.dump()) logger.info("Full config saved to {}".format(path)) # make sure each worker has a different, yet deterministic seed if specified seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank) # cudnn benchmark has large overhead. It shouldn't be used considering the small size of # typical validation set. if not (hasattr(args, "eval_only") and args.eval_only): torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
if __name__ == "__main__": """ Generate point-based supervision for COCO dataset. Usage: python tools/prepare_coco_point_annotations_without_masks.py \ NUM_POINTS_PER_INSTANCE NUM_VERSIONS_WITH_DIFFERENT_SEED Example to generate point-based COCO dataset with 10 points per instance: python tools/prepare_coco_point_annotations_without_masks.py 10 """ # Fix random seed seed_all_rng(12345) assert len(sys.argv) >= 2, "Please provide number of points to sample per instance" dataset_dir = os.path.join(os.getenv("DETECTRON2_DATASETS", "datasets"), "coco/annotations") num_points_per_instance = int(sys.argv[1]) if len(sys.argv) == 3: repeat = int(sys.argv[2]) else: repeat = 1 s = "instances_train2017" for version in range(repeat): print( "Start sampling {} points per instance for annotations {}.".format( num_points_per_instance, s ) )