def main(): """For local testing purposes. Note: VNDK snapshot must be already installed under prebuilts/vndk/v{version}. """ ANDROID_BUILD_TOP = utils.get_android_build_top() PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') args = get_args() vndk_version = args.vndk_version install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) if not os.path.isdir(install_dir): raise ValueError( 'Please provide valid VNDK version. {} does not exist.' .format(install_dir)) utils.set_logging_config(args.verbose) buildfile_generator = GenBuildFile(install_dir, vndk_version) buildfile_generator.generate_root_android_bp() buildfile_generator.generate_common_android_bp() buildfile_generator.generate_android_bp() logging.info('Done.')
def main(): """For local testing purposes. Note: VNDK snapshot must be already installed under prebuilts/vndk/v{version}. """ ANDROID_BUILD_TOP = utils.get_android_build_top() PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') args = get_args() vndk_version = args.vndk_version install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) if not os.path.isdir(install_dir): raise ValueError( 'Please provide valid VNDK version. {} does not exist.'.format( install_dir)) utils.set_logging_config(args.verbose) buildfile_generator = GenBuildFile(install_dir, vndk_version) buildfile_generator.generate_root_android_bp() buildfile_generator.generate_common_android_bp() buildfile_generator.generate_android_bp() logging.info('Done.')
def main(): """For local testing purposes. Note: VNDK snapshot must be already installed under prebuilts/vndk/v{version}. """ ANDROID_BUILD_TOP = utils.get_android_build_top() PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') args = get_args() vndk_version = args.vndk_version install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) remote = args.remote if not os.path.isdir(install_dir): raise ValueError( 'Please provide valid VNDK version. {} does not exist.' .format(install_dir)) utils.set_logging_config(args.verbose) temp_artifact_dir = tempfile.mkdtemp() os.chdir(temp_artifact_dir) manifest_pattern = 'manifest_{}.xml'.format(args.build) manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME) logging.info('Fetching {file} from {branch} (bid: {build})'.format( file=manifest_pattern, branch=args.branch, build=args.build)) utils.fetch_artifact(args.branch, args.build, manifest_pattern, manifest_dest) license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, temp_artifact_dir, remote) try: license_checker.check_gpl_projects() except ValueError as error: logging.error('Error: {}'.format(error)) raise finally: logging.info( 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) shutil.rmtree(temp_artifact_dir) logging.info('Done.')
def main(): """For local testing purposes. Note: VNDK snapshot must be already installed under prebuilts/vndk/v{version}. """ ANDROID_BUILD_TOP = utils.get_android_build_top() PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk') args = get_args() vndk_version = args.vndk_version install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) remote = args.remote if not os.path.isdir(install_dir): raise ValueError( 'Please provide valid VNDK version. {} does not exist.' .format(install_dir)) utils.set_logging_config(args.verbose) temp_artifact_dir = tempfile.mkdtemp() os.chdir(temp_artifact_dir) manifest_pattern = 'manifest_{}.xml'.format(args.build) manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME) logging.info('Fetching {file} from {branch} (bid: {build})'.format( file=manifest_pattern, branch=args.branch, build=args.build)) utils.fetch_artifact(args.branch, args.build, manifest_pattern, manifest_dest) license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, temp_artifact_dir, remote) try: license_checker.check_gpl_projects() except ValueError as error: logging.error('Error: {}'.format(error)) raise finally: logging.info( 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) shutil.rmtree(temp_artifact_dir) logging.info('Done.')
def main(): """Program entry point.""" args = get_args() if args.local: if args.build or args.branch: raise ValueError( 'When --local option is set, --branch or --build cannot be ' 'specified.') elif not os.path.isdir(args.local): raise RuntimeError( 'The specified local directory, {}, does not exist.'.format( args.local)) else: if not (args.build and args.branch): raise ValueError( 'Please provide both --branch and --build or set --local ' 'option.') vndk_version = str(args.vndk_version) install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) if not os.path.isdir(install_dir): raise RuntimeError( 'The directory for VNDK snapshot version {ver} does not exist.\n' 'Please request a new git project for prebuilts/vndk/v{ver} ' 'before installing new snapshot.'.format(ver=vndk_version)) utils.set_logging_config(args.verbose) os.chdir(install_dir) if not args.use_current_branch: start_branch(args.build) remove_old_snapshot(install_dir) os.makedirs(utils.COMMON_DIR_PATH) temp_artifact_dir = None if not args.local: temp_artifact_dir = tempfile.mkdtemp() try: install_snapshot(args.branch, args.build, args.local, install_dir, temp_artifact_dir) gather_notice_files(install_dir) revise_ld_config_txt_if_needed(vndk_version) buildfile_generator = GenBuildFile(install_dir, vndk_version) update_buildfiles(buildfile_generator) if not args.local: license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, temp_artifact_dir) check_gpl_license(license_checker) logger.info( 'Successfully updated VNDK snapshot v{}'.format(vndk_version)) except Exception as error: logger.error('FAILED TO INSTALL SNAPSHOT: {}'.format(error)) raise finally: if temp_artifact_dir: logger.info( 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) shutil.rmtree(temp_artifact_dir) if not args.local: commit(args.branch, args.build, vndk_version) logger.info('Successfully created commit for VNDK snapshot v{}'.format( vndk_version)) logger.info('Done.')
commands.py hello [--name=<TXT>] [-q | -d] commands.py -h commands.py -v Options: -h, --help display this message and exit -v, --version display version -n, --name=TXT greeting name [default: You] -q, --quiet set log level to WARNING [default: INFO] -d, --debug set log level to DEBUG [default: INFO] """ import logging from docopt import docopt from docopt_dispatch import dispatch from utils import set_logging_config from settings import VERSION @dispatch.on('hello') def do_hello(name=None, **kwargs): print("Hello, {}!". format(name)) if __name__ == '__main__': kwargs = docopt(__doc__) set_logging_config(kwargs) logging.debug(kwargs) dispatch(__doc__, version=VERSION)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--device', type=str, default='cuda:0', help='gpu device number of using') parser.add_argument( '--config', type=str, default=os.path.join('.', 'config', '5way_1shot_resnet12_mini-imagenet.py'), help='config file with parameters of the experiment. ' 'It is assumed that the config file is placed under the directory ./config' ) parser.add_argument( '--checkpoint_dir', type=str, default=os.path.join('.', 'checkpoints'), help='path that checkpoint will be saved and loaded. ' 'It is assumed that the checkpoint file is placed under the directory ./checkpoints' ) parser.add_argument('--num_gpu', type=int, default=1, help='number of gpu') parser.add_argument('--display_step', type=int, default=100, help='display training information in how many step') parser.add_argument('--log_step', type=int, default=100, help='log information in how many steps') parser.add_argument( '--log_dir', type=str, default=os.path.join('.', 'logs'), help='path that log will be saved. ' 'It is assumed that the checkpoint file is placed under the directory ./logs' ) parser.add_argument('--dataset_root', type=str, default='./data', help='root directory of dataset') parser.add_argument('--seed', type=int, default=222, help='random seed') parser.add_argument('--mode', type=str, default='train', help='train or eval') args_opt = parser.parse_args() config_file = args_opt.config # Set train and test datasets and the corresponding data loaders config = imp.load_source("", config_file).config train_opt = config['train_config'] eval_opt = config['eval_config'] args_opt.exp_name = '{}way_{}shot_{}_{}'.format(train_opt['num_ways'], train_opt['num_shots'], config['backbone'], config['dataset_name']) train_opt['num_queries'] = 1 eval_opt['num_queries'] = 1 set_logging_config(os.path.join(args_opt.log_dir, args_opt.exp_name)) logger = logging.getLogger('main') # Load the configuration params of the experiment logger.info('Launching experiment from: {}'.format(config_file)) logger.info('Generated logs will be saved to: {}'.format(args_opt.log_dir)) logger.info('Generated checkpoints will be saved to: {}'.format( args_opt.checkpoint_dir)) print() logger.info('-------------command line arguments-------------') logger.info(args_opt) print() logger.info('-------------configs-------------') logger.info(config) # set random seed np.random.seed(args_opt.seed) torch.manual_seed(args_opt.seed) torch.cuda.manual_seed_all(args_opt.seed) random.seed(args_opt.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if config['dataset_name'] == 'mini-imagenet': dataset = MiniImagenet print('Dataset: MiniImagenet') elif config['dataset_name'] == 'tiered-imagenet': dataset = TieredImagenet print('Dataset: TieredImagenet') elif config['dataset_name'] == 'cifar-fs': dataset = Cifar print('Dataset: Cifar') elif config['dataset_name'] == 'cub-200-2011': dataset = CUB200 print('Dataset: CUB200') else: logger.info( 'Invalid dataset: {}, please specify a dataset from ' 'mini-imagenet, tiered-imagenet, cifar-fs and cub-200-2011.'. format(config['dataset_name'])) exit() cifar_flag = True if args_opt.exp_name.__contains__('cifar') else False if config['backbone'] == 'resnet12': enc_module = ResNet12(emb_size=config['emb_size'], cifar_flag=cifar_flag) print('Backbone: ResNet12') elif config['backbone'] == 'resnet18': enc_module = ResNet18(emb_size=config['emb_size']) print('Backbone: ResNet18') elif config['backbone'] == 'wrn': enc_module = WRN(emb_size=config['emb_size']) print('Backbone: WRN') elif config['backbone'] == 'convnet': enc_module = ConvNet(emb_size=config['emb_size'], cifar_flag=cifar_flag) print('Backbone: ConvNet') else: logger.info( 'Invalid backbone: {}, please specify a backbone model from ' 'convnet, resnet12, resnet18 and wrn.'.format(config['backbone'])) exit() gnn_module = DPGN( config['num_generation'], train_opt['dropout'], train_opt['num_ways'] * train_opt['num_shots'], train_opt['num_ways'] * train_opt['num_shots'] + train_opt['num_ways'] * train_opt['num_queries'], train_opt['loss_indicator'], config['point_distance_metric'], config['distribution_distance_metric']) # multi-gpu configuration [ print('GPU: {} Spec: {}'.format(i, torch.cuda.get_device_name(i))) for i in range(args_opt.num_gpu) ] if args_opt.num_gpu > 1: print('Construct multi-gpu model ...') enc_module = nn.DataParallel(enc_module, device_ids=range(args_opt.num_gpu), dim=0) gnn_module = nn.DataParallel(gnn_module, device_ids=range(args_opt.num_gpu), dim=0) print('done!\n') if not os.path.exists( os.path.join(args_opt.checkpoint_dir, args_opt.exp_name)): os.makedirs(os.path.join(args_opt.checkpoint_dir, args_opt.exp_name)) logger.info('no checkpoint for model: {}, make a new one at {}'.format( args_opt.exp_name, os.path.join(args_opt.checkpoint_dir, args_opt.exp_name))) best_step = 0 else: if not os.path.exists( os.path.join(args_opt.checkpoint_dir, args_opt.exp_name, 'model_best.pth.tar')): best_step = 0 else: logger.info('find a checkpoint, loading checkpoint from {}'.format( os.path.join(args_opt.checkpoint_dir, args_opt.exp_name))) best_checkpoint = torch.load( os.path.join(args_opt.checkpoint_dir, args_opt.exp_name, 'model_best.pth.tar')) logger.info('best model pack loaded') best_step = best_checkpoint['iteration'] enc_module.load_state_dict( best_checkpoint['enc_module_state_dict']) gnn_module.load_state_dict( best_checkpoint['gnn_module_state_dict']) logger.info( 'current best test accuracy is: {}, at step: {}'.format( best_checkpoint['test_acc'], best_step)) dataset_train = dataset(root=args_opt.dataset_root, partition='train') dataset_valid = dataset(root=args_opt.dataset_root, partition='val') dataset_test = dataset(root=args_opt.dataset_root, partition='test') train_loader = DataLoader(dataset_train, num_tasks=train_opt['batch_size'], num_ways=train_opt['num_ways'], num_shots=train_opt['num_shots'], num_queries=train_opt['num_queries'], epoch_size=train_opt['iteration']) valid_loader = DataLoader(dataset_valid, num_tasks=eval_opt['batch_size'], num_ways=eval_opt['num_ways'], num_shots=eval_opt['num_shots'], num_queries=eval_opt['num_queries'], epoch_size=eval_opt['iteration']) test_loader = DataLoader(dataset_test, num_tasks=eval_opt['batch_size'], num_ways=eval_opt['num_ways'], num_shots=eval_opt['num_shots'], num_queries=eval_opt['num_queries'], epoch_size=eval_opt['iteration']) data_loader = { 'train': train_loader, 'val': valid_loader, 'test': test_loader } # create trainer trainer = DPGNTrainer(enc_module=enc_module, gnn_module=gnn_module, data_loader=data_loader, log=logger, arg=args_opt, config=config, best_step=best_step) if args_opt.mode == 'train': trainer.train() elif args_opt.mode == 'eval': trainer.eval() else: print('select a mode') exit()
help=""" NSML mode setting """, ) parser.add_argument( "--iteration", type=int, default=0, help=""" NSML default setting """, ) parser.add_argument( "--pause", type=int, default=0, help=""" NSML default setting """, ) args = parser.parse_args() with open(args.base_config, "r") as f: defined_config = json.load(f) config = NestedNamespace() config.load_from_json(defined_config) config.nsml = args set_logging_config() if args.mode == "train_and_evaluate": re_train_and_evaluate(config) elif args.mode == "test" or args.mode == "infer": test(config) else: raise ValueError(f"Unrecognized mode. {config.mode}")
def main(): """Program entry point.""" args = get_args() local = None if args.local: local = os.path.expanduser(args.local) if local: if args.build or args.branch: raise ValueError( 'When --local option is set, --branch or --build cannot be ' 'specified.') elif not os.path.isdir(local): raise RuntimeError( 'The specified local directory, {}, does not exist.'.format( local)) else: if not (args.build and args.branch): raise ValueError( 'Please provide both --branch and --build or set --local ' 'option.') vndk_version = args.vndk_version install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version)) if not os.path.isdir(install_dir): raise RuntimeError( 'The directory for VNDK snapshot version {ver} does not exist.\n' 'Please request a new git project for prebuilts/vndk/v{ver} ' 'before installing new snapshot.'.format(ver=vndk_version)) utils.set_logging_config(args.verbose) os.chdir(install_dir) if not args.use_current_branch: start_branch(args.build) remove_old_snapshot(install_dir) os.makedirs(utils.COMMON_DIR_PATH) temp_artifact_dir = None if not local: temp_artifact_dir = tempfile.mkdtemp() try: install_snapshot(args.branch, args.build, local, install_dir, temp_artifact_dir) gather_notice_files(install_dir) post_processe_files_if_needed(vndk_version) buildfile_generator = GenBuildFile(install_dir, vndk_version) update_buildfiles(buildfile_generator) if not local: license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, temp_artifact_dir, args.remote) check_gpl_license(license_checker) logging.info( 'Successfully updated VNDK snapshot v{}'.format(vndk_version)) except Exception as error: logging.error('FAILED TO INSTALL SNAPSHOT: {}'.format(error)) raise finally: if temp_artifact_dir: logging.info( 'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir)) shutil.rmtree(temp_artifact_dir) if not local: commit(args.branch, args.build, vndk_version) logging.info( 'Successfully created commit for VNDK snapshot v{}'.format( vndk_version)) logging.info('Done.')
from commands import AddFixturesCommand, MakeMigrationsCommand, MigrateCommand, RunCommand from settings import SERVICE_NAME, current_settings from utils import parse_args, set_logging_config COMMANDS = { "makemigrations": MakeMigrationsCommand, "migrate": MigrateCommand, "addfixtures": AddFixturesCommand, "run": RunCommand, } if __name__ == "__main__": args = parse_args(message=SERVICE_NAME, commands=COMMANDS.keys()) if args.config is None: current_settings.from_env(service=SERVICE_NAME) else: current_settings.from_ini(service=SERVICE_NAME, filename=args.config) if all((current_settings.logging.filetype, current_settings.logging.filename)): set_logging_config( level=current_settings.logging.level, filetype=current_settings.logging.filetype, filename=current_settings.logging.filename, ) command = COMMANDS[args.command](settings=current_settings) command.run()