def generate_probs(model_dir, inp_fp, weights_fp, topk, out_ext, cuda_device, overwrite, batch_size, extraction_ratio, out): import_submodules('imojie') args = argparse.Namespace() args.archive_file = model_dir args.cuda_device = cuda_device args.embedding_sources_mapping = {} args.extend_vocab = None args.batch_weight_key = '' args.output_file = '' args.overrides = "{'model': {'token_based_metric': null}, 'iterator': {'batch_size': " + str( batch_size) + ", \ 'instances_per_epoch': null}, 'trainer':{'num_epochs':1}, 'dataset_reader': {'max_tokens': 10000, \ 'gradients': false, 'max_extractions': 30, 'extraction_ratio': " + str( extraction_ratio) + ", 'probability': true \ }, 'validation_dataset_reader': null}" args.weights_file = weights_fp args.input_file = inp_fp probs = evaluate_from_args(args) probsD = dict() # For some reason the last batch results are repeated in the probs # Not an issue as they are just overwritten while forming the probsD for i in range(len(probs['example_ids'])): probsD[probs['example_ids'][i]] = probs['probs'][i] lines = open(inp_fp).readlines() all_fields = [] for line_number, line in enumerate(lines): line = line.strip('\n') fields = line.split('\t') if line_number not in probsD: # the example is too large and rejected by dataloader ('max_tokens' argument) continue # Removing appended extractions after reranking fields[0] = fields[0].split('[SEP]')[0].strip() fields[2] = str(probsD[line_number]) fields.append(str(line_number)) all_fields.append('\t'.join(fields)) if topk == None: return all_fields else: # sorting all_fields according to the confidences assigned by bert_encoder all_fields_sorted = [] prev_sent = None exts = [] for f in all_fields: sent = f.split('\t')[0] if sent != prev_sent: if prev_sent != None: exts = toolz.unique(exts, key=lambda x: x.split('\t')[1]) exts = sorted(exts, reverse=True, key=lambda x: float(x.split('\t')[2])) if topk != None: exts = exts[:topk] all_fields_sorted.extend(exts) prev_sent = sent exts = [f] else: exts.append(f) exts = sorted(exts, reverse=True, key=lambda x: float(x.split('\t')[2])) all_fields_sorted.extend(exts) open(out, 'w').write('\n'.join(all_fields_sorted)) print('Probabilities written to: ', out) return all_fields_sorted
import sys sys.path.insert(1, os.getcwd()) from gits_pull import gits_pull from mock import patch, Mock def parse_args(args): parser = argparse.ArgumentParser() return parser.parse_args(args) @patch("argparse.ArgumentParser.parse_args", return_value=argparse.Namespace(nocommit=True, rebase=False, branch="branch name")) @patch("subprocess.Popen") @patch("helper.get_current_branch", return_value="current branch") def test_gits_pull_happy_case_with_no_commit_and_given_branch( mock_current_branch, mock_var, mock_args): """ Function to test gits pull, success case with no commits, no rebase and given branch """ mocked_pipe = Mock() attrs = {'communicate.return_value': (b'', 'error'), 'returncode': 0} mocked_pipe.configure_mock(**attrs) mock_var.return_value = mocked_pipe mock_args = parse_args(mock_args) test_result = gits_pull(mock_args)
def test_get_sample_dict_sra_expand_from_xml_sample(): sample_elements = get_test_sample_elements() args = argparse.Namespace(expand_sra_details=True) expected_sample_dict = { "repository": "CDE", "submitted_sample_id": "NWD1", "biosample_id": "SAMN1", "submitted_subject_id": "ABC", "dbgap_sample_id": "1", "dbgap_subject_id": "1", "sra_sample_id": "SRS1", "consent_code": "1", "consent_short_name": "GRU-IRB", "sex": "male", "body_site": "Whole blood", "analyte_type": "DNA", "dbgap_status": "Loaded", "sample_use": ["Seq_DNA_SNP_CNV", "WGS"], "sra_data_details": { "status": "public", "experiments": "1", "runs": "3", "bases": "406977793500", "size_Gb": "74", "experiment_type": "WGS", "platform": "ILLUMINA", "center": "ABC Fast Track Services", }, "study_accession": "phs001234.v3.p1", "study_accession_with_consent": "phs001234.v3.p1.c1", "study_with_consent": "phs001234.c1", "study_subject_id": "phs001234.v3_ABC", } sample_dict = dbgap_extract.get_sample_dict_from_xml_sample( "phs001234.v3.p1", sample_elements[0], args) sample_dict["sample_use"] = json.loads(sample_dict["sample_use"]) sample_dict["sra_data_details"] = json.loads( sample_dict["sra_data_details"]) assert_dict_equality(sample_dict, expected_sample_dict) expected_sample_dict = { "repository": "FGH", "submitted_sample_id": "NWD1", "biosample_id": "SAMN2", "submitted_subject_id": "CDE", "dbgap_sample_id": "1", "dbgap_subject_id": "2", "sra_sample_id": "SRS2", "consent_code": "1", "consent_short_name": "GRU-IRB", "sex": "female", "body_site": "Whole blood", "analyte_type": "DNA", "dbgap_status": "Loaded", "sample_use": ["Seq_DNA_SNP_CWB", "GWS"], "sra_data_details": { "status": "public", "experiments": "1", "runs": "2", "bases": "250660703000", "size_Gb": "49", "experiment_type": "WGS", "platform": "ILLUMINA", "center": "CDE Fast Track Services", }, "study_accession": "phs001234.v3.p1", "study_accession_with_consent": "phs001234.v3.p1.c1", "study_with_consent": "phs001234.c1", "study_subject_id": "phs001234.v3_CDE", } sample_dict = dbgap_extract.get_sample_dict_from_xml_sample( "phs001234.v3.p1", sample_elements[1], args) sample_dict["sample_use"] = json.loads(sample_dict["sample_use"]) sample_dict["sra_data_details"] = json.loads( sample_dict["sra_data_details"]) assert_dict_equality(sample_dict, expected_sample_dict)
parser.add_argument('--test_atts', dest='test_atts', nargs='+', help='test_atts') parser.add_argument('--test_ints', dest='test_ints', type=float, nargs='+', help='test_ints') parser.add_argument('--num_test', dest='num_test', type=int) parser.add_argument('--load_epoch', dest='load_epoch', type=str, default='latest') parser.add_argument('--custom_img', action='store_true') parser.add_argument('--custom_data', type=str, default='./data/custom') parser.add_argument('--custom_attr', type=str, default='./data/list_attr_custom.txt') parser.add_argument('--gpu', action='store_true') parser.add_argument('--multi_gpu', action='store_true') return parser.parse_args(args) args_ = parse() print(args_) with open(join('output', args_.experiment_name, 'setting.txt'), 'r') as f: args = json.load(f, object_hook=lambda d: argparse.Namespace(**d)) args.test_atts = args_.test_atts args.test_ints = args_.test_ints args.num_test = args_.num_test args.load_epoch = args_.load_epoch args.custom_img = args_.custom_img args.custom_data = args_.custom_data args.custom_attr = args_.custom_attr args.gpu = args_.gpu args.multi_gpu = args_.multi_gpu print(args) assert args.test_atts is not None, 'test_atts should be chosen in %s' % (str(args.attrs)) for a in args.test_atts:
def get_basemodel(args): if args.model == 'none': return None if args.model == 'resnet50' and args.dataset == 'imagenet': model = models.resnet50(pretrained=True).eval() normalize = NormalizeLayer(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) else: path = glob.glob( os.path.join('./models', args.model, '**', 'checkpoint.pt.best')) path_tar = glob.glob( os.path.join('./models', args.model, '**', 'checkpoint.pth.tar')) if not (len(path) > 0 or (len(path_tar) > 0 and args.dataset in ['cifar', 'imagenet'])): print("Could not load model") sys.exit(1) if len(path_tar) > 0 and args.dataset in [ 'cifar', 'imagenet', 'restricted_imagenet' ]: sys.path.append('smoothing-adversarial/code') from architectures import get_architecture path = path_tar[0] print('Loading model from', path) checkpoint = torch.load(path, map_location='cpu') if args.dataset == 'cifar': model = get_architecture(checkpoint["arch"], 'cifar10') else: model = get_architecture(checkpoint["arch"], args.dataset) model.load_state_dict(checkpoint['state_dict']) model = model.to('cpu') for i, m in enumerate(model): if isinstance(m, torch.nn.DataParallel): model[i] = m.module normalize = None model = model[1:] print(model) else: path = path[0] print('Loading model from', path) if args.dataset in ['imagenet', 'restricted_imagenet', 'cifar']: ds_class = datasets.DATASETS[args.dataset] ds = ds_class("./ds/imagenet" if args.dataset != 'cifar' else './ds/cifar') model, _ = model_utils.make_and_restore_model( arch=('resnet18' if args.dataset == 'cifar' else 'resnet50'), dataset=ds, resume_path=path, parallel=False) normalize = model.normalizer model = model.model elif args.dataset in ['mnist', 'fashionmnist', 'GTSRB']: if 'mnist' in args.dataset: num_classes = 10 color_channels = 1 mean = torch.tensor([0.1307]) std = torch.tensor([0.3081]) if 'convnet' in path: print('convenet') model = MNISTConvNet() else: model = resnet18(num_classes=num_classes, color_channels=color_channels) elif args.dataset == 'GTSRB': num_classes = 43 color_channels = 3 mean = torch.tensor([0.3337, 0.3064, 0.3171]) std = torch.tensor([0.2672, 0.2564, 0.2629]) model = resnet18(num_classes=num_classes, color_channels=color_channels) model = RobustnessModelInnerWrapper(model) d = argparse.Namespace() d.mean = mean d.std = std model = AttackerModel(model, d) checkpoint = torch.load(path, pickle_module=dill) state_dict_path = 'model' if not ('model' in checkpoint): state_dict_path = 'state_dict' sd = checkpoint[state_dict_path] sd = {k[len('module.'):]: v for k, v in sd.items()} sd = {(k if 'model.net' in k else k.replace( 'model.', 'model.net.')): v for k, v in sd.items()} model.load_state_dict(sd) normalize = model.normalizer model = model.model else: assert (False) m = [] if normalize is not None: m.append(normalize.to(args.device)) if args.radiusDecrease >= 0: shape = {'rot': 'circ', 'trans': 'rect'}[args.transformation] size = { 'mnist': (1, 28, 28), 'fashionmnist': (1, 28, 28), 'cifar': (3, 32, 32), 'GTSRB': (3, np.inf, np.inf), 'imagenet': (3, np.inf, np.inf), 'restricted_imagenet': (3, np.inf, np.inf) }[args.dataset] if args.resize_post_transform > 0: size = (size[0], min(size[1], args.resize_post_transform), min(size[2], args.resize_post_transform)) if args.center_crop_post_transform > 0: size = (size[0], min(size[1], args.center_crop_post_transform), min(size[2], args.center_crop_post_transform)) V = VingetteModule(size, shape, args.radiusDecrease) m.append(V) m.append(model) model = torch.nn.Sequential(*m) if args.use_cuda: model = torch.nn.DataParallel(model.to(args.device)) model = model.eval().to(args.device) return model
def roll_settings(weights): ret = argparse.Namespace() ret.name = get_choice('name', weights) if ret.name: ret.name = handle_name(ret.name) glitches_required = get_choice('glitches_required', weights) if glitches_required not in ['none', 'no_logic', 'overworld_glitches']: logging.warning("Only NMG, OWG and No Logic supported") glitches_required = 'none' ret.logic = {None: 'noglitches', 'none': 'noglitches', 'no_logic': 'nologic', 'overworld_glitches': 'owglitches'}[glitches_required] # item_placement = get_choice('item_placement') # not supported in ER dungeon_items = get_choice('dungeon_items', weights) if dungeon_items == 'full' or dungeon_items == True: dungeon_items = 'mcsb' elif dungeon_items == 'standard': dungeon_items = "" elif not dungeon_items: dungeon_items = "" ret.mapshuffle = get_choice('map_shuffle', weights) if 'map_shuffle' in weights else 'm' in dungeon_items ret.compassshuffle = get_choice('compass_shuffle', weights) if 'compass_shuffle' in weights else 'c' in dungeon_items ret.keyshuffle = get_choice('smallkey_shuffle', weights) if 'smallkey_shuffle' in weights else 's' in dungeon_items ret.bigkeyshuffle = get_choice('bigkey_shuffle', weights) if 'bigkey_shuffle' in weights else 'b' in dungeon_items ret.accessibility = get_choice('accessibility', weights) entrance_shuffle = get_choice('entrance_shuffle', weights) ret.shuffle = entrance_shuffle if entrance_shuffle != 'none' else 'vanilla' goal = get_choice('goals', weights) ret.goal = {'ganon': 'ganon', 'fast_ganon': 'crystals', 'dungeons': 'dungeons', 'pedestal': 'pedestal', 'triforce-hunt': 'triforcehunt' }[goal] ret.openpyramid = goal == 'fast_ganon' ret.crystals_gt = get_choice('tower_open', weights) ret.crystals_ganon = get_choice('ganon_open', weights) ret.mode = get_choice('world_state', weights) if ret.mode == 'retro': ret.mode = 'open' ret.retro = True ret.hints = get_choice('hints', weights) ret.swords = {'randomized': 'random', 'assured': 'assured', 'vanilla': 'vanilla', 'swordless': 'swordless' }[get_choice('weapons', weights)] ret.difficulty = get_choice('item_pool', weights) ret.item_functionality = get_choice('item_functionality', weights) ret.shufflebosses = {'none': 'none', 'simple': 'basic', 'full': 'normal', 'random': 'chaos' }[get_choice('boss_shuffle', weights)] ret.shuffleenemies = {'none': 'none', 'shuffled': 'shuffled', 'random': 'chaos' }[get_choice('enemy_shuffle', weights)] ret.enemy_damage = {'default': 'default', 'shuffled': 'shuffled', 'random': 'chaos' }[get_choice('enemy_damage', weights)] ret.enemy_health = get_choice('enemy_health', weights) ret.shufflepots = get_choice('pot_shuffle', weights) ret.beemizer = int(get_choice('beemizer', weights)) if 'beemizer' in weights else 0 ret.timer = {'none': False, None: False, False: False, 'timed': 'timed', 'timed_ohko': 'timed-ohko', 'ohko': 'ohko', 'timed_countdown': 'timed-countdown', 'display': 'display'}[get_choice('timer', weights)] if 'timer' in weights.keys() else False ret.dungeon_counters = get_choice('dungeon_counters', weights) if 'dungeon_counters' in weights else 'default' ret.progressive = convert_to_on_off(get_choice('progressive', weights)) if "progressive" in weights else 'on' inventoryweights = weights.get('startinventory', {}) startitems = [] for item in inventoryweights.keys(): itemvalue = get_choice(item, inventoryweights) if item.startswith(('Progressive ', 'Small Key ', 'Rupee', 'Piece of Heart', 'Boss Heart Container', 'Sanctuary Heart Container', 'Arrow', 'Bombs ', 'Bomb ', 'Bottle')) and isinstance( itemvalue, int): for i in range(int(itemvalue)): startitems.append(item) elif itemvalue: startitems.append(item) ret.glitch_boots = get_choice('glitch_boots', weights) if 'glitch_boots' in weights else True ret.startinventory = ','.join(startitems) ret.remote_items = get_choice('remote_items', weights) if 'remote_items' in weights else False if 'rom' in weights: romweights = weights['rom'] ret.sprite = get_choice('sprite', romweights) ret.disablemusic = get_choice('disablemusic', romweights) ret.extendedmsu = get_choice('extendedmsu', romweights) ret.quickswap = get_choice('quickswap', romweights) ret.fastmenu = get_choice('menuspeed', romweights) ret.heartcolor = get_choice('heartcolor', romweights) ret.heartbeep = convert_to_on_off(get_choice('heartbeep', romweights)) ret.ow_palettes = get_choice('ow_palettes', romweights) ret.uw_palettes = get_choice('uw_palettes', romweights) return ret
class TestConfig(unittest.TestCase): """Class to test Config Class. Mock is used to mock external calls""" def setUp(self): """Set up for the unit tests""" self.config = Config(gac_ui=ui.cli, create_config=False) def tearDown(self): """Run Clean Up""" self.config.clean_up() @patch( "argparse.ArgumentParser.parse_args", return_value=argparse.Namespace( username="******", profile=None, insecure=False, resolve=None, mfa_code=None, remember_device=False, output_format=None, roles=None, action_register_device=False, action_configure=False, action_list_profiles=False, action_list_roles=False, action_store_json_creds=False, action_setup_fido_authenticator=False, ), ) def test_get_args_username(self, mock_arg): """Test to make sure username gets returned""" self.config.get_args() self.assertEqual(self.config.username, "ann") def test_read_config(self): """Test to make sure getting config works""" test_ui = MockUserInterface(argv=[ "--profile", "myprofile", ]) with open(test_ui.HOME + "/.okta_aws_login_config", "w") as config_file: config_file.write(""" [myprofile] client_id = foo """) config = Config(gac_ui=test_ui, create_config=False) config.conf_profile = "myprofile" profile_config = config.get_config_dict() self.assertEqual(profile_config, {"client_id": "foo"}) def test_read_config_inherited(self): """Test to make sure getting config works when inherited""" test_ui = MockUserInterface(argv=[ "--profile", "myprofile", ]) with open(test_ui.HOME + "/.okta_aws_login_config", "w") as config_file: config_file.write(""" [mybase] client_id = bar aws_appname = baz [myprofile] inherits = mybase client_id = foo aws_rolename = myrole """) config = Config(gac_ui=test_ui, create_config=False) config.conf_profile = "myprofile" profile_config = config.get_config_dict() self.assertEqual(profile_config, { "client_id": "foo", "aws_appname": "baz", "aws_rolename": "myrole", }) def test_read_nested_config_inherited(self): """Test to make sure getting config works when inherited""" test_ui = MockUserInterface(argv=[ "--profile", "myprofile", ]) with open(test_ui.HOME + "/.okta_aws_login_config", "w") as config_file: config_file.write(""" [mybase-level1] client_id = bar [mybase-level2] inherits = mybase-level1 aws_appname = baz [myprofile] inherits = mybase-level2 client_id = foo aws_rolename = myrole """) config = Config(gac_ui=test_ui, create_config=False) config.conf_profile = "myprofile" profile_config = config.get_config_dict() self.assertEqual(profile_config, { "client_id": "foo", "aws_appname": "baz", "aws_rolename": "myrole", }) def test_fail_if_profile_not_found(self): """Test to make sure missing Default fails properly""" test_ui = MockUserInterface(argv=[]) with open(test_ui.HOME + "/.okta_aws_login_config", "w") as config_file: config_file.write(""" [myprofile] client_id = foo """) config = Config(gac_ui=test_ui, create_config=False) config.conf_profile = "DEFAULT" with self.assertRaises(errors.GimmeAWSCredsError) as context: config.get_config_dict() self.assertTrue( 'DEFAULT profile is missing! This is profile is required when not using --profile' == context.exception.message)
def train(self, ep = 0): parser = argparse.ArgumentParser() parser.add_argument('--weights', type=str, default=config.weight, help='initial weights path') parser.add_argument('--cfg', type=str, default=config.config_model, help='models/yolov5s.yaml path') parser.add_argument('--data', type=str, default=config.config_data, help='data.yaml path') parser.add_argument('--hyp', type=str, default='data/hyp.scratch.yaml', help='hyperparameters path') parser.add_argument('--epochs', type=int, default=config.epochs) parser.add_argument('--batch-size', type=int, default=config.batch_size, help='total batch size for all GPUs') parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='[train, test] image sizes') parser.add_argument('--rect', action='store_true', help='rectangular training') parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') parser.add_argument('--notest', action='store_true', help='only test final epoch') parser.add_argument('--noautoanchor', action='store_true', help='disable autoanchor check') # parser.add_argument('--noautoanchor', type=bool, default=True, help='disable autoanchor check') parser.add_argument('--evolve', action='store_true', help='evolve hyperparameters') # parser.add_argument('--evolve', type = bool, default = False, help='evolve hyperparameters') parser.add_argument('--bucket', type=str, default='', help='gsutil bucket') parser.add_argument('--cache-images', action='store_true', help='cache images for faster training') parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training') parser.add_argument('--device', default=config.device, help='cuda device, i.e. 0 or 0,1,2,3 or cpu') # using GPU 0 parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%') parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') # parser.add_argument('--adam', action='store_true', help='use torch.optim.Adam() optimizer') parser.add_argument('--adam', type=int, default=config.adam, help='use torch.optim.Adam() optimizer') # using Adam optimizer parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') parser.add_argument('--local_rank', type=int, default=-1, help='DDP parameter, do not modify') parser.add_argument('--log-imgs', type=int, default=16, help='number of images for W&B logging, max 100') parser.add_argument('--log-artifacts', action='store_true', help='log artifacts, i.e. final trained model') parser.add_argument('--workers', type=int, default=8, help='maximum number of dataloader workers') parser.add_argument('--project', default=config.project_train, help='save to project/name') parser.add_argument('--name', default=config.name, help='save to project/name') # parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') parser.add_argument('--exist-ok', type=int, default=config.exist_ok, help='existing project/name ok, do not increment') opt = parser.parse_args() # Set DDP variables opt.total_batch_size = opt.batch_size opt.world_size = int(os.environ['WORLD_SIZE']) if 'WORLD_SIZE' in os.environ else 1 opt.global_rank = int(os.environ['RANK']) if 'RANK' in os.environ else -1 set_logging(opt.global_rank) if opt.global_rank in [-1, 0]: check_git_status() # Resume if opt.resume: # resume an interrupted run ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist' with open(Path(ckpt).parent.parent / 'opt.yaml') as f: opt = argparse.Namespace(**yaml.load(f, Loader=yaml.FullLoader)) # replace opt.cfg, opt.weights, opt.resume = '', ckpt, True logger.info('Resuming training from %s' % ckpt) else: # opt.hyp = opt.hyp or ('hyp.finetune.yaml' if opt.weights else 'hyp.scratch.yaml') opt.data, opt.cfg, opt.hyp = check_file(opt.data), check_file(opt.cfg), check_file(opt.hyp) # check files assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified' opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test) opt.name = 'evolve' if opt.evolve else opt.name opt.save_dir = increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok | opt.evolve) # increment run # DDP mode device = select_device(opt.device, batch_size=opt.batch_size) if opt.local_rank != -1: assert torch.cuda.device_count() > opt.local_rank torch.cuda.set_device(opt.local_rank) device = torch.device('cuda', opt.local_rank) dist.init_process_group(backend='nccl', init_method='env://') # distributed backend assert opt.batch_size % opt.world_size == 0, '--batch-size must be multiple of CUDA device count' opt.batch_size = opt.total_batch_size // opt.world_size # Hyperparameters with open(opt.hyp) as f: hyp = yaml.load(f, Loader=yaml.FullLoader) # load hyps if 'box' not in hyp: warn('Compatibility: %s missing "box" which was renamed from "giou" in %s' % (opt.hyp, 'https://github.com/ultralytics/yolov5/pull/1120')) hyp['box'] = hyp.pop('giou') # Train logger.info(opt) if not opt.evolve: tb_writer = None # init loggers if opt.global_rank in [-1, 0]: logger.info(f'Start Tensorboard with "tensorboard --logdir {opt.project}", view at http://localhost:6006/') tb_writer = SummaryWriter(opt.save_dir) # Tensorboard train(hyp, opt, device, tb_writer, wandb, ep)
# misc_args.add_argument( # "--snapshot_interval", # type=int, # default=50, # help="save checkpoint every '--snapshost_interval' steps", # ) args = parser.parse_args() arg_groups = {} for group in parser._action_groups: group_dict = { a.dest: getattr(args, a.dest, None) for a in group._group_actions if a.dest != "help" } arg_groups[group.title] = argparse.Namespace(**group_dict) config = {} config["trainer"] = vars(arg_groups["optional arguments"]) config["model"] = vars(arg_groups["model arguments"]) config["data"] = vars(arg_groups["data arguments"]) # config["misc"] = vars(arg_groups["misc arguments"]) config["prep"] = {"resize_size": 256, "crop_size": 224} if config["data"]["dset"] == "office-31": if ( ("amazon" in args.source_domain and "webcam" in args.target_domain) or ("webcam" in args.source_domain and "dslr" in args.target_domain) or ("webcam" in args.source_domain and "amazon" in args.target_domain) or ("dslr" in args.source_domain and "amazon" in args.target_domain) ):
def __init__(self, config_file=None, config=None, args=None, **kwargs): if args is None: args = dict() if isinstance(args, argparse.Namespace): args = args.__dict__ args.update(kwargs) args = dict([(k, v) for k, v in args.items() if v is not None]) self.client = argparse.Namespace() self.foreman = argparse.Namespace() self.worker = argparse.Namespace() self.config_file = config_file self.config = config raw_config = dict() if self.config_file is None: self.config_file = args.get('config_file', None) if self.config_file is None: conf_dirs = appdirs.AppDirs(CONFIG_APP_NAME, CONFIG_APP_AUTHOR, multipath=True) check_dirs = conf_dirs.user_config_dir.split(':') check_dirs += [os.path.join('/etc', CONFIG_APP_NAME)] check_dirs += conf_dirs.site_config_dir.split(':') logging.debug('Looking for configuration in {}'.format(check_dirs)) for path in check_dirs: check_path = os.path.join(path, CONFIG_FILE) if os.path.isfile(check_path): self.config_file = check_path break if self.config_file is not None: parser = configparser.ConfigParser() parser.read(self.config_file) raw_config = dict([(k, dict(v)) for (k, v) in parser.items()]) if self.config is None: self.config = args.get('config', None) default_config = raw_config.get('default', dict()) default_config.update(args) if self.config is None: self.config = default_config.get('config', 'client') logging.info('Using config {} in {} {}'.format(self.config, self.config_file, args)) if self.config is not None and self.config not in raw_config: logging.warning('Config {} not found in config file {}'.format( self.config, self.config_file)) client_config = raw_config.get(self.config, dict()) foreman_config = raw_config.get('foreman', dict()) worker_config = raw_config.get('worker', dict()) self.client.__setattr__( 'server', client_config.get( 'server', default_config.get('server', None) or CONFIG_SERVER)) self.client.__setattr__( 'username', client_config.get('username', default_config.get('username', None))) self.client.__setattr__( 'password', client_config.get('password', default_config.get('password', None))) self.client.__setattr__( 'cpus', parse_int( client_config.get('cpus', default_config.get('cpus', None)))) self.client.__setattr__( 'memory', parse_memory( client_config.get('memory', default_config.get('memory', None)))) self.client.__setattr__( 'swap', parse_memory( client_config.get('swap', default_config.get('swap', None)))) self.client.__setattr__( 'storage', parse_memory( client_config.get('storage', default_config.get('storage', None)))) self.client.__setattr__( 'image', parse_memory( client_config.get('image', default_config.get('image', None)))) self.client.__setattr__( 'workspace', parse_memory( client_config.get('workspace', default_config.get('workspace', None)))) self.client.__setattr__( 'pids', parse_int( client_config.get('pids', default_config.get('pids', None)))) self.client.__setattr__( 'time', parse_time( client_config.get('time', default_config.get('time', None)))) self.client.__setattr__( 'network', parse_bool( client_config.get('network', default_config.get('network', None)))) self.client.__setattr__( 'gpus', parse_int( client_config.get('gpus', default_config.get('gpus', None)))) self.foreman.__setattr__( 'temp_path', foreman_config.get('temp', default_config.get('temp', None))) self.foreman.__setattr__( 'interval', parse_float( foreman_config.get( 'interval', default_config.get('interval', None) or FOREMAN_INTERVAL))) self.foreman.__setattr__( 'concurency', parse_int( foreman_config.get( 'concurency', default_config.get('concurency', None) or FOREMAN_CONCURENCY))) self.foreman.__setattr__( 'pull', parse_bool( foreman_config.get('pull', default_config.get('pull', None) or False))) self.foreman.__setattr__( 'cpus', parse_int( foreman_config.get('cpus', default_config.get('cpus', None)))) self.foreman.__setattr__( 'memory', parse_memory( foreman_config.get('memory', default_config.get('memory', None)))) self.foreman.__setattr__( 'swap', parse_memory( foreman_config.get('swap', default_config.get('swap', None)))) self.foreman.__setattr__( 'storage', parse_memory( foreman_config.get('storage', default_config.get('storage', None)))) self.foreman.__setattr__( 'image', parse_memory( foreman_config.get('image', default_config.get('image', None)))) self.foreman.__setattr__( 'workspace', parse_memory( foreman_config.get('workspace', default_config.get('workspace', None)))) self.foreman.__setattr__( 'pids', parse_int( foreman_config.get('pids', default_config.get('pids', None)))) self.foreman.__setattr__( 'time', parse_time( foreman_config.get('time', default_config.get('time', None)))) self.foreman.__setattr__( 'network', parse_bool( foreman_config.get('network', default_config.get('network', None)))) self.foreman.__setattr__( 'auto_tags', parse_bool( foreman_config.get('auto_tags', default_config.get('auto_tags', [])))) tags = parse_str_list( foreman_config.get('tags', default_config.get('tags', None) or [])) if self.foreman.auto_tags: tags = set(tags) tags.update(foreman_auto_tags()) tags = list(tags) self.foreman.__setattr__('tags', tags) self.foreman.__setattr__( 'gpus', parse_int( foreman_config.get('gpus', default_config.get('gpus', None)))) self.worker.__setattr__( 'debug', parse_bool( worker_config.get('debug', default_config.get('debug', None) or False))) self.worker.__setattr__( 'verbose', parse_bool( worker_config.get('verbose', default_config.get('verbose', None) or False))) self.worker.__setattr__( 'temp_path', worker_config.get('temp', default_config.get('temp', None))) self.worker.__setattr__( 'pull', parse_bool( worker_config.get('pull', default_config.get('pull', None) or False))) self.worker.__setattr__( 'cpus', parse_int( worker_config.get('cpus', default_config.get('cpus', None)))) self.worker.__setattr__( 'memory', parse_memory( worker_config.get('memory', default_config.get('memory', None)))) self.worker.__setattr__( 'swap', parse_memory( worker_config.get('swap', default_config.get('swap', None)))) self.worker.__setattr__( 'storage', parse_memory( worker_config.get('storage', default_config.get('storage', None)))) self.worker.__setattr__( 'image', parse_memory( worker_config.get('image', default_config.get('image', None)))) self.worker.__setattr__( 'workspace', parse_memory( worker_config.get('workspace', default_config.get('workspace', None)))) self.worker.__setattr__( 'pids', parse_int( worker_config.get('pids', default_config.get('pids', None)))) self.worker.__setattr__( 'time', parse_time( worker_config.get('time', default_config.get('time', None)))) self.worker.__setattr__( 'network', parse_bool( worker_config.get('network', default_config.get('network', None)))) self.worker.__setattr__( 'gpus', parse_int( worker_config.get('gpus', default_config.get('gpus', None))))
def task(ctx, config): """ Deploy ceph cluster using cephadm For example, teuthology.yaml can contain the 'defaults' section: defaults: cephadm: containers: image: 'quay.io/ceph-ci/ceph' Using overrides makes it possible to customize it per run. The equivalent 'overrides' section looks like: overrides: cephadm: containers: image: 'quay.io/ceph-ci/ceph' :param ctx: the argparse.Namespace object :param config: the config dict """ if config is None: config = {} assert isinstance(config, dict), \ "task only supports a dictionary for configuration" overrides = ctx.config.get('overrides', {}) teuthology.deep_merge(config, overrides.get('ceph', {})) teuthology.deep_merge(config, overrides.get('cephadm', {})) log.info('Config: ' + str(config)) # set up cluster context if not hasattr(ctx, 'ceph'): ctx.ceph = {} ctx.managers = {} if 'cluster' not in config: config['cluster'] = 'ceph' cluster_name = config['cluster'] if cluster_name not in ctx.ceph: ctx.ceph[cluster_name] = argparse.Namespace() ctx.ceph[cluster_name].bootstrapped = False # image teuth_defaults = teuth_config.get('defaults', {}) cephadm_defaults = teuth_defaults.get('cephadm', {}) containers_defaults = cephadm_defaults.get('containers', {}) container_image_name = containers_defaults.get('image', None) containers = config.get('containers', {}) container_image_name = containers.get('image', container_image_name) if not hasattr(ctx.ceph[cluster_name], 'image'): ctx.ceph[cluster_name].image = config.get('image') ref = None if not ctx.ceph[cluster_name].image: if not container_image_name: raise Exception( "Configuration error occurred. " "The 'image' value is undefined for 'cephadm' task. " "Please provide corresponding options in the task's " "config, task 'overrides', or teuthology 'defaults' " "section.") sha1 = config.get('sha1') flavor = config.get('flavor', 'default') if sha1: if flavor == "crimson": ctx.ceph[ cluster_name].image = container_image_name + ':' + sha1 + '-' + flavor else: ctx.ceph[ cluster_name].image = container_image_name + ':' + sha1 ref = sha1 else: # hmm, fall back to branch? branch = config.get('branch', 'master') ref = branch ctx.ceph[cluster_name].image = container_image_name + ':' + branch log.info('Cluster image is %s' % ctx.ceph[cluster_name].image) with contextutil.nested( #if the cluster is already bootstrapped bypass corresponding methods lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\ else initialize_config(ctx=ctx, config=config), lambda: ceph_initial(), lambda: normalize_hostnames(ctx=ctx), lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\ else download_cephadm(ctx=ctx, config=config, ref=ref), lambda: ceph_log(ctx=ctx, config=config), lambda: ceph_crash(ctx=ctx, config=config), lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\ else ceph_bootstrap(ctx, config), lambda: crush_setup(ctx=ctx, config=config), lambda: ceph_mons(ctx=ctx, config=config), lambda: distribute_config_and_admin_keyring(ctx=ctx, config=config), lambda: ceph_mgrs(ctx=ctx, config=config), lambda: ceph_osds(ctx=ctx, config=config), lambda: ceph_mdss(ctx=ctx, config=config), lambda: ceph_rgw(ctx=ctx, config=config), lambda: ceph_iscsi(ctx=ctx, config=config), lambda: ceph_monitoring('prometheus', ctx=ctx, config=config), lambda: ceph_monitoring('node-exporter', ctx=ctx, config=config), lambda: ceph_monitoring('alertmanager', ctx=ctx, config=config), lambda: ceph_monitoring('grafana', ctx=ctx, config=config), lambda: ceph_clients(ctx=ctx, config=config), lambda: create_rbd_pool(ctx=ctx, config=config), ): ctx.managers[cluster_name] = CephManager( ctx.ceph[cluster_name].bootstrap_remote, ctx=ctx, logger=log.getChild('ceph_manager.' + cluster_name), cluster=cluster_name, cephadm=True, ) try: if config.get('wait-for-healthy', True): healthy(ctx=ctx, config=config) log.info('Setup complete, yielding') yield finally: log.info('Teardown begin')
def main(pathActivations, pathOutput, nGroups=1, nClusters=50, MAX_ITER=100, batchSizeGPU=50, debug=False, extension='.pt', getDistanceEstimation=False, load=False, perIterSize=-1, recursionLevel=2, save=False, save_last=5, seqList=None): # Test the extension is valid if extension not in ['.txt', '.npy', '.pt']: raise ValueError(f'Activation file extension invalid ({extension})') torch.cuda.empty_cache() args = argparse.Namespace(**locals()) # Export absolute paths for later use pathActivations = os.path.abspath(pathActivations) pathOutput = os.path.abspath(pathOutput) if not load: assert os.path.exists(pathOutput) is False, \ f"The output file {pathOutput} already exists, please check the option --load !" assert os.path.exists(os.path.join(os.path.dirname(pathOutput), "checkpoint_last.pt")) is False, \ "Found last_checkpoint.pt in the output directory, please check the option --load !" print(args) seqNames, speakers = findAllSeqs(pathActivations, speaker_level=recursionLevel, extension=extension, loadCache=True) if seqList is not None: seqNames = filterSeqs(seqList, seqNames) if debug: nsamples = 1000 print(f"Debug mode activated, get only {nsamples} samples!") shuffle(seqNames) seqNames = seqNames[:nsamples] if getDistanceEstimation: shuffle(seqNames) seqNames = seqNames[:5000] print("") print(f'Loading activations at {pathActivations}') start_time = time.time() dataset = SequentialData(pathActivations, seqNames, None) print(f"Dataset loaded in {time.time()-start_time} seconds !") print("") nGPUs = torch.cuda.device_count() if nGPUs == 0: raise RuntimeError('No GPU found') batchSize = batchSizeGPU * nGPUs dataloader = dataset.getDataLoader(batchSize, numWorkers=0) print(f"Length of dataLoader: {len(dataloader)}") print("") # Check if dir exists if not os.path.exists(os.path.dirname(pathOutput)) and os.path.dirname(pathOutput): Path(os.path.dirname(pathOutput)).mkdir(parents=True, exist_ok=True) pathConfig = f"{os.path.splitext(pathOutput)[0]}_args.json" with open(pathConfig, 'w') as file: json.dump(vars(args), file, indent=2) out_state_dict = {} print("Starting the clustering...") start_time = time.time() # Using a dumb lambda function to skip feature extraction as we start from # the activations clusters = kMeanGPU(dataloader, lambda x: x, nClusters, nGroups, perIterSize=perIterSize, MAX_ITER=MAX_ITER, save=save, load=load, save_dir=os.path.dirname(pathOutput), save_last=save_last, ).cpu() print(f'Ran clustering ' f'in {time.time() - start_time:.2f} seconds') clusterModule = kMeanCluster(clusters) out_state_dict["state_dict"] = clusterModule.state_dict() out_state_dict["n_clusters"] = nClusters out_state_dict['dim'] = clusters.size(2) torch.save(out_state_dict, pathOutput) with open(pathConfig, 'w') as file: json.dump(vars(args), file, indent=2)
def __init__(self, tokenizer, **kwargs): super().__init__() args = argparse.Namespace(tokenizer=tokenizer, **kwargs) self.tokenizer = encoders.build_tokenizer(args) assert self.tokenizer is not None
def __init__(self, bpe, **kwargs): super().__init__() args = argparse.Namespace(bpe=bpe, **kwargs) self.bpe = encoders.build_bpe(args) assert self.bpe is not None
def main(**kwargs): args = argparse.Namespace(**kwargs) if 'save' in args: if os.path.exists(args.save): raise RuntimeError('Output file "{}" already exists.'.format( args.save)) if args.seed is not None: pyro.set_rng_seed(args.seed) X, true_counts = load_data() X_size = X.size(0) if args.cuda: X = X.cuda() # Build a function to compute z_pres prior probabilities. if args.z_pres_prior_raw: def base_z_pres_prior_p(t): return args.z_pres_prior else: base_z_pres_prior_p = make_prior(args.z_pres_prior) # Wrap with logic to apply any annealing. def z_pres_prior_p(opt_step, time_step): p = base_z_pres_prior_p(time_step) if args.anneal_prior == 'none': return p else: decay = dict(lin=lin_decay, exp=exp_decay)[args.anneal_prior] return decay(p, args.anneal_prior_to, args.anneal_prior_begin, args.anneal_prior_duration, opt_step) model_arg_keys = [ 'window_size', 'rnn_hidden_size', 'decoder_output_bias', 'decoder_output_use_sigmoid', 'baseline_scalar', 'encoder_net', 'decoder_net', 'predict_net', 'embed_net', 'bl_predict_net', 'non_linearity', 'pos_prior_mean', 'pos_prior_sd', 'scale_prior_mean', 'scale_prior_sd' ] model_args = { key: getattr(args, key) for key in model_arg_keys if key in args } air = AIR(num_steps=args.model_steps, x_size=50, use_masking=not args.no_masking, use_baselines=not args.no_baselines, z_what_size=args.encoder_latent_size, use_cuda=args.cuda, **model_args) if args.verbose: print(air) print(args) if 'load' in args: print('Loading parameters...') air.load_state_dict(torch.load(args.load)) vis = visdom.Visdom(env=args.visdom_env) # Viz sample from prior. if args.viz: z, x = air.prior(5, z_pres_prior_p=partial(z_pres_prior_p, 0)) vis.images(draw_many(x, tensor_to_objs(latents_to_tensor(z)))) def per_param_optim_args(module_name, param_name, tags): lr = args.baseline_learning_rate if 'baseline' in tags else args.learning_rate return {'lr': lr} svi = SVI(air.model, air.guide, optim.Adam(per_param_optim_args), loss='ELBO', trace_graph=True) # Do inference. t0 = time.time() examples_to_viz = X[5:10] for i in range(1, args.num_steps + 1): loss = svi.step(X, args.batch_size, z_pres_prior_p=partial(z_pres_prior_p, i)) if args.progress_every > 0 and i % args.progress_every == 0: print('i={}, epochs={:.2f}, elapsed={:.2f}, elbo={:.2f}'.format( i, (i * args.batch_size) / X_size, (time.time() - t0) / 3600, loss / X_size)) if args.viz and i % args.viz_every == 0: trace = poutine.trace(air.guide).get_trace(examples_to_viz, None) z, recons = poutine.replay(air.prior, trace)(examples_to_viz.size(0)) z_wheres = tensor_to_objs(latents_to_tensor(z)) # Show data with inferred objection positions. vis.images(draw_many(examples_to_viz, z_wheres)) # Show reconstructions of data. vis.images(draw_many(recons, z_wheres)) if args.eval_every > 0 and i % args.eval_every == 0: # Measure accuracy on subset of training data. acc, counts, error_z, error_ix = count_accuracy( X, true_counts, air, 1000) print('i={}, accuracy={}, counts={}'.format( i, acc, counts.numpy().tolist())) if args.viz and error_ix.size(0) > 0: vis.images(draw_many(X[error_ix[0:5]], tensor_to_objs(error_z[0:5])), opts=dict(caption='errors ({})'.format(i))) if 'save' in args and i % args.save_every == 0: print('Saving parameters...') torch.save(air.state_dict(), args.save)
def main(): """ Computational Genomics Lab, Genomics Institute, UC Santa Cruz Toil BWA pipeline Alignment of fastq reads via BWA-kit General usage: 1. Type "toil-bwa generate" to create an editable manifest and config in the current working directory. 2. Parameterize the pipeline by editing the config. 3. Fill in the manifest with information pertaining to your samples. 4. Type "toil-bwa run [jobStore]" to execute the pipeline. Please read the README.md located in the source directory or at: https://github.com/BD2KGenomics/toil-scripts/tree/master/src/toil_scripts/bwa_alignment Structure of the BWA pipeline (per sample) 0 --> 1 0 = Download sample 1 = Run BWA-kit =================================================================== :Dependencies: cURL: apt-get install curl Toil: pip install toil Docker: wget -qO- https://get.docker.com/ | sh Optional: S3AM: pip install --s3am (requires ~/.boto config file) Boto: pip install boto """ # Define Parser object and add to Toil parser = argparse.ArgumentParser( description=main.__doc__, formatter_class=argparse.RawTextHelpFormatter) subparsers = parser.add_subparsers(dest='command') # Generate subparsers subparsers.add_parser( 'generate-config', help='Generates an editable config in the current working directory.') subparsers.add_parser( 'generate-manifest', help='Generates an editable manifest in the current working directory.' ) subparsers.add_parser( 'generate', help='Generates a config and manifest in the current working directory.' ) # Run subparser parser_run = subparsers.add_parser('run', help='Runs the BWA alignment pipeline') group = parser_run.add_mutually_exclusive_group() parser_run.add_argument( '--config', default='config-toil-bwa.yaml', type=str, help= 'Path to the (filled in) config file, generated with "generate-config".' ) group.add_argument( '--manifest', default='manifest-toil-bwa.tsv', type=str, help= 'Path to the (filled in) manifest file, generated with "generate-manifest". ' '\nDefault value: "%(default)s".') group.add_argument( '--sample', nargs='+', action=required_length(2, 3), help= 'Space delimited sample UUID and fastq files in the format: uuid url1 [url2].' ) # Print docstring help if no arguments provided if len(sys.argv) == 1: parser.print_help() sys.exit(1) Job.Runner.addToilOptions(parser_run) args = parser.parse_args() # Parse subparsers related to generation of config and manifest cwd = os.getcwd() if args.command == 'generate-config' or args.command == 'generate': generate_file(os.path.join(cwd, 'config-toil-bwa.yaml'), generate_config) if args.command == 'generate-manifest' or args.command == 'generate': generate_file(os.path.join(cwd, 'manifest-toil-bwa.tsv'), generate_manifest) # Pipeline execution elif args.command == 'run': require(os.path.exists(args.config), '{} not found. Please run generate-config'.format(args.config)) if not args.sample: args.sample = None require( os.path.exists(args.manifest), '{} not found and no sample provided. ' 'Please run "generate-manifest"'.format(args.manifest)) # Parse config parsed_config = { x.replace('-', '_'): y for x, y in yaml.load(open(args.config).read()).iteritems() } config = argparse.Namespace(**parsed_config) config.maxCores = int(args.maxCores) if args.maxCores else sys.maxint samples = [args.sample[0], args.sample[1:] ] if args.sample else parse_manifest(args.manifest) # Sanity checks require(config.ref, 'Missing URL for reference file: {}'.format(config.ref)) require(config.output_dir, 'No output location specified: {}'.format(config.output_dir)) # Launch Pipeline Job.Runner.startToil( Job.wrapJobFn(download_reference_files, config, samples), args)
def main(): parser = argparse.ArgumentParser( description="Pull transactions information and format it to csv or ledger format.", add_help=False, ) subs = parser.add_subparsers(title="actions") # Parser for the verbosity, common to all subparsers commonp = argparse.ArgumentParser(add_help=False) commonp.add_argument("--verbose", "-v", action="count", default=0) commonp.add_argument( "--output", type=argparse.FileType("w"), default=sys.stdout, help="File to wite the transaction data to. Default to stdout.", ) # Parser for the init command initp = subs.add_parser( "init", description="init the current directory, populating the pledger config files.", parents=[commonp], ) initp.add_argument( "--force", "-f", nargs="?", default=False, const=True, help="If set, will overwrite the config files if some already exists", ) initp.set_defaults(func=xact.init_fn) # Parser for the fetch command fetchp = subs.add_parser( "fetch", description="fetch the transaction history from the specified account.", parents=[commonp], ) fetchp.add_argument( "bank", help="The institution to fetch the transaction history from." ) fetchp.add_argument( "--start", help="Starting date, should be formatted as YYYY-MM-DD.", type=date.fromisoformat, default=date.min, ) fetchp.add_argument( "--end", help="Ending date, should be formatted as YYYY-MM-DD.", type=date.fromisoformat, default=date.today(), ) # We can either pick one account or all fo them. If no option selected # picks all the account listed for that institution in the account file. group = fetchp.add_mutually_exclusive_group() group.add_argument( "--account", help="If only interested in one specific account.", ) group.add_argument( "--all", nargs="?", default=False, const=True, help="Fetches transactions for every account at the institution.", ) fetchp.set_defaults(func=xact.fetch_fn) # Parser for the convert command convertp = subs.add_parser( "convert", description="Converts transaction data to csv or ledger format.", parents=[commonp], ) convertp.add_argument( "file", help="The file containing the transaction history, in json format.", ) convertp.add_argument( "--format", choices=["csv", "ledger"], required=True, help="The format of the exported data.", ) convertp.add_argument( "--currency", "-c", nargs="?", default=False, const=True, help="If set, appends the currency iso code after the transaction amount", ) convertp.add_argument( "--fields", default=["date", "amount", "name"], type=field_list, help="The fields of the transaction data the are to be exported.", ) convertp.add_argument( "--negate", "-n", nargs="?", default=False, const=True, help="Flip the sign of the transaction amount.", ) convertp.set_defaults(func=xact.convert_fn) # Parser for the list command listp = subs.add_parser( "list", description="List account related information", parents=[commonp] ) listp.add_argument( "account", help="The account to fetch the transaction history from." ) listp.set_defaults(func=xact.list_fn) args = parser.parse_args() if args == argparse.Namespace(): parser.print_help() return # Set verbosity if args.verbose > 1: verbosity = logging.DEBUG elif args.verbose > 0: verbosity = logging.INFO else: verbosity = logging.WARNING logging.basicConfig(level=verbosity) args.func(args)
def __init__(self, parent: DetectVideoApp = None): super().__init__() self.parent = parent self.args = argparse.Namespace() self.args.is_classifier = False self.classifier_cfg = None self.classifier_weights = None self.setWindowIcon(QIcon('../images/Logo.png')) layout = QVBoxLayout() self.startDir = os.getcwd() # Create video selector self.useWebCam = QCheckBox("Use web cam") self.useWebCam.setChecked(False) self.useWebCamLabel = QLabel("Use web cam") self.useWebCam.stateChanged.connect(self.use_webcam_clicked) horizontal_layout = QHBoxLayout() horizontal_layout.addWidget(self.useWebCamLabel) horizontal_layout.addWidget(self.useWebCam, alignment=Qt.AlignLeft) horizontal_layout.addStretch(1) self.btnIm = QPushButton("Select video to detect") self.btnIm.clicked.connect(self.get_video) self.textIm = QTextEdit() layout.addLayout(horizontal_layout, 1) layout.addStretch(1) layout.addLayout(SelectQPushButton(self.btnIm), 1) layout.addLayout(SelectQText(self.textIm), 2) layout.addStretch(1) self.textIm.setReadOnly(True) # Select data set self.select_ds_label = QLabel("Select dataset") self.select_ds = DataSetsManager.get_data_set_combo() self.select_ds.setObjectName("SelectCombo") self.select_ds.currentTextChanged.connect(self.on_data_set_changed) layout.addLayout(SelectQCombo(self.select_ds_label, self.select_ds), 2) layout.addStretch(1) # Select weights file self.btnW = QPushButton("Select weights file") self.btnW.clicked.connect(self.get_weights) self.textW = QLineEdit() self.textW.setReadOnly(True) layout.addLayout(SelectQPushButton(self.btnW), 1) layout.addLayout(SelectQPushButton(self.textW), 1) layout.addStretch(1) # Select Config file self.btnConf = QPushButton("Select Config file") self.btnConf.clicked.connect(self.get_config) self.textConf = QLineEdit() self.textConf.setReadOnly(True) layout.addLayout(SelectQPushButton(self.btnConf), 1) layout.addLayout(SelectQText(self.textConf), 1) layout.addStretch(1) # Select Names file self.btnNames = QPushButton("Select Names file") self.btnNames.clicked.connect(self.get_names) self.textNames = QLineEdit() self.textNames.setReadOnly(True) layout.addLayout(SelectQPushButton(self.btnNames), 1) layout.addLayout(SelectQText(self.textNames), 1) layout.addStretch(1) bs_label = QLabel('Batch size') conf_label = QLabel('Confidence') nms_label = QLabel('Nms threshold') res_label = QLabel('Resolution') self.bsEdit = QLineEdit() self.confEdit = QLineEdit() self.nmsEdit = QLineEdit() self.resEdit = QLineEdit() self.bsEdit.setText("1") self.confEdit.setText("0.5") self.nmsEdit.setText("0.4") self.resEdit.setText("416") self.textIm.setText("../vid1_Driving_in_Gothenburg_Sweden.mp4") self.args.video = "../vid1_Driving_in_Gothenburg_Sweden.mp4" self.on_data_set_changed('Swedish') self.select_ds.setCurrentText('Swedish') grid = QGridLayout() grid.setSpacing(10) grid.addWidget(bs_label, 1, 0) grid.addWidget(conf_label, 2, 0) grid.addWidget(nms_label, 3, 0) grid.addWidget(res_label, 4, 0) grid.addWidget(self.bsEdit, 1, 1) grid.addWidget(self.confEdit, 2, 1) grid.addWidget(self.nmsEdit, 3, 1) grid.addWidget(self.resEdit, 4, 1) grid.setColumnStretch(0, 1) grid.setColumnStretch(1, 1) grid.setColumnStretch(2, 2) layout.addLayout(grid, 5) tracking_layout = QHBoxLayout() self.use_tracking = QCheckBox("Use tracking") self.use_tracking.setChecked(True) self.use_tracking_label = QLabel("Use tracking: ") self.use_tracking.stateChanged.connect(self.use_tracking_clicked) tracking_layout.addWidget(self.use_tracking_label) tracking_layout.addWidget(self.use_tracking) tracking_layout.addStretch(1) layout.addLayout(tracking_layout, 1) self.tracking = None self.select_tracking_label = QLabel("Select tracking") self.select_tracking = QComboBox() self.select_tracking.setItemDelegate(QStyledItemDelegate()) self.select_tracking.setObjectName("SelectCombo") self.select_tracking.addItems(["Sort", "Deep Sort"]) self.select_tracking.currentIndexChanged.connect(self.selection_tracking_change) self.select_tracking.setCurrentIndex(1) layout.addLayout(SelectQCombo(self.select_tracking_label, self.select_tracking), 2) count_layout = QHBoxLayout() self.count_enabled = False self.use_count = QCheckBox("Count performance") self.use_count.setChecked(False) self.use_count_label = QLabel("Count statistics: ") count_layout.addWidget(self.use_count_label) count_layout.addWidget(self.use_count) count_layout.addStretch(1) layout.addLayout(count_layout, 1) layout.addStretch(1) back_button = QPushButton("Back") ok_button = QPushButton("OK") cancel_button = QPushButton("Cancel") hor_box = QHBoxLayout() hor_box.addWidget(back_button, alignment=Qt.AlignLeft) hor_box.addStretch(2) hor_box.addWidget(ok_button) hor_box.addWidget(cancel_button) back_button.clicked.connect(self.back_detection) ok_button.clicked.connect(self.start_detection) cancel_button.clicked.connect(cancel_detection) layout.addLayout(hor_box, 2) self.setLayout(layout)
# Set DDP variables opt.total_batch_size = opt.batch_size opt.world_size = int(os.environ['WORLD_SIZE']) if 'WORLD_SIZE' in os.environ else 1 opt.global_rank = int(os.environ['RANK']) if 'RANK' in os.environ else -1 set_logging(opt.global_rank) if opt.global_rank in [-1, 0]: check_git_status() # Resume if opt.resume: # resume an interrupted run ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path log_dir = Path(ckpt).parent.parent # runs/exp0 assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist' with open(log_dir / 'opt.yaml') as f: opt = argparse.Namespace(**yaml.load(f, Loader=yaml.FullLoader)) # replace opt.cfg, opt.weights, opt.resume = '', ckpt, True logger.info('Resuming training from %s' % ckpt) else: # opt.hyp = opt.hyp or ('hyp.finetune.yaml' if opt.weights else 'hyp.scratch.yaml') opt.data, opt.cfg, opt.hyp = check_file(opt.data), check_file(opt.cfg), check_file(opt.hyp) # check files assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified' opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test) log_dir = increment_dir(Path(opt.logdir) / 'exp', opt.name) # runs/exp1 # DDP mode device = select_device(opt.device, batch_size=opt.batch_size) if opt.local_rank != -1: assert torch.cuda.device_count() > opt.local_rank torch.cuda.set_device(opt.local_rank)
def run(name, config, print_details=False): """ Runs the extension. Args: name: The name of the extension config: The test's config dictionary print_details: Whether to print the analysis details to stdout (default: False) Returns: A LIVVkit page element containing the LIVVkit elements to display on a webpage """ # FIXME: move into a config to NameSpace function test_args = OrderedDict([(k.replace('-', '_'), v) for k, v in config.items()]) test_args = argparse.Namespace(**test_args) test_args.img_dir = os.path.join(livvkit.output_dir, 'validation', 'imgs', name) fn.mkdir_p(test_args.img_dir) details, img_gal = main(test_args) if print_details: _print_details(details) domain_headers = ['Null hypothesis', 'T test (t, P)'] global_tbl_el = { 'Type': 'V-H Table', 'Title': 'Validation', 'TableTitle': 'Analyzed variables', 'Headers': domain_headers, 'Data': details['global'], } land_tbl_el = { 'Type': 'V-H Table', 'Title': 'Validation', 'TableTitle': 'Analyzed variables', 'Headers': domain_headers, 'Data': details['land'], } ocean_tbl_el = { 'Type': 'V-H Table', 'Title': 'Validation', 'TableTitle': 'Analyzed variables', 'Headers': domain_headers, 'Data': details['ocean'], } bib_html = bib2html(os.path.join(os.path.dirname(__file__), 'tsc.bib')) detail_tables = {"global": [], "land": [], "ocean": []} table_headers = ["Variable", *domain_headers] for level in detail_tables: for _time in details[level]: tbl_data = {_hdr: [] for _hdr in table_headers} for _var in details[level][_time]: tbl_data["Variable"].append(_var) for _hdr, val in details[level][_time][_var].items(): tbl_data[_hdr].append(val) detail_tables[level].append( el.Table(title=f"{level.capitalize()}, {_time}", data=tbl_data)) tabs = el.Tabs({ 'Figures': [img_gal], 'Global_details': detail_tables["global"], 'Land_details': detail_tables["land"], 'Ocean_details': detail_tables["ocean"], 'References': [el.RawHTML(bib_html)] }) results = el.Table(title="Results", data=OrderedDict({ 'Test status': [details['overall']], 'Global': [details['domains']['delta_l2_global']], 'Land': [details['domains']['delta_l2_land']], 'Ocean': [details['domains']['delta_l2_ocean']], 'Ensembles': [ 'statistically identical' if details['overall'] == 'Pass' else 'statistically different' ], })) # FIXME: Put into a ___ function doc_text = __doc__.format( (1 - test_args.p_threshold) * 100).replace('\n\n', '<br><br>') page = el.Page(name, doc_text, elements=[results, tabs]) return page
"critical": logging.CRITICAL } log_level_parameter = os.getenv("SENZING_LOG_LEVEL", "info").lower() log_level = log_level_map.get(log_level_parameter, logging.INFO) logging.basicConfig(format=log_format, level=log_level) # Parse the command line arguments. subcommand = os.getenv("SENZING_SUBCOMMAND", None) parser = get_parser() if len(sys.argv) > 1: args = parser.parse_args() subcommand = args.subcommand elif subcommand: args = argparse.Namespace(subcommand=subcommand) else: parser.print_help() exit_silently() # Transform subcommand from CLI parameter to function name string. subcommand_function_name = "do_{0}".format(subcommand.replace('-', '_')) # Test to see if function exists in the code. if subcommand_function_name not in globals(): logging.warn(message_warn(498, subcommand)) parser.print_help() exit_silently()
set_parser = getattr(import_module("common"), 'set_parser') parser = set_parser() args, unknown = parser.parse_known_args() # put arguments from (inference) config_file print(f'from_only_config: {args.from_only_config}') if args.from_only_config: # load (inference) config file # TODO 파일 안 열릴 때 종료하는 예외처리 with open(args.config_path, 'r') as f: config_dict = json.load(f) with open(config_dict['saved_inference_config_path'], 'r') as f: inference_config_dict = json.load(f) args = argparse.Namespace() d = vars(args) for key, value in inference_config_dict.items(): d[key] = value # add variable to argments for record d = vars(args) if args.submission_user_key != '': user_key = args.submission_user_key output_file_path = os.path.join(args.submission_dir, f'{args.saved_model_name}.csv') desc = f'model:{args.model}, backbone:{args.backbone}, neck:{args.neck} optimizer:{args.optimizer}, scheduler:{args.scheduler} lr:{args.lr}\n' \ f'epoch:{args.best_epoch}/{args.epochs}, best loss:{args.best_loss}, batch size:{args.batch_size}\n' \ f'train augmentation:{args.train_augmentation}, val augmentation:{args.val_augmentation}, test augmentation:{args.test_augmentation}' submit(user_key, output_file_path, desc)
def __init__(self, idim, odim, args): """Construct an E2E object. :param int idim: dimension of inputs :param int odim: dimension of outputs :param Namespace args: argument Namespace containing options """ super(E2E, self).__init__() torch.nn.Module.__init__(self) # fill missing arguments for compatibility args = fill_missing_args(args, self.add_arguments) self.etype = args.etype self.verbose = args.verbose # NOTE: for self.build method args.char_list = getattr(args, "char_list", None) self.char_list = args.char_list self.outdir = args.outdir self.space = args.sym_space self.blank = args.sym_blank self.reporter = Reporter() # below means the last number becomes eos/sos ID # note that sos/eos IDs are identical self.sos = odim - 1 self.eos = odim - 1 self.pad = 0 # NOTE: we reserve index:0 for <pad> although this is reserved for a blank class # in ASR. However, blank labels are not used in MT. # To keep the vocabulary size, # we use index:0 for padding instead of adding one more class. # subsample info self.subsample = get_subsample(args, mode="mt", arch="rnn") # label smoothing info if args.lsm_type and os.path.isfile(args.train_json): logging.info("Use label smoothing with " + args.lsm_type) labeldist = label_smoothing_dist( odim, args.lsm_type, transcript=args.train_json ) else: labeldist = None # multilingual related self.multilingual = getattr(args, "multilingual", False) self.replace_sos = getattr(args, "replace_sos", False) # encoder self.embed = torch.nn.Embedding(idim, args.eunits, padding_idx=self.pad) self.dropout = torch.nn.Dropout(p=args.dropout_rate) self.enc = encoder_for(args, args.eunits, self.subsample) # attention self.att = att_for(args) # decoder self.dec = decoder_for(args, odim, self.sos, self.eos, self.att, labeldist) # tie source and target emeddings if args.tie_src_tgt_embedding: if idim != odim: raise ValueError( "When using tie_src_tgt_embedding, idim and odim must be equal." ) if args.eunits != args.dunits: raise ValueError( "When using tie_src_tgt_embedding, eunits and dunits must be equal." ) self.embed.weight = self.dec.embed.weight # tie emeddings and the classfier if args.tie_classifier: if args.context_residual: raise ValueError( "When using tie_classifier, context_residual must be turned off." ) self.dec.output.weight = self.dec.embed.weight # weight initialization self.init_like_fairseq() # options for beam search if args.report_bleu: trans_args = { "beam_size": args.beam_size, "penalty": args.penalty, "ctc_weight": 0, "maxlenratio": args.maxlenratio, "minlenratio": args.minlenratio, "lm_weight": args.lm_weight, "rnnlm": args.rnnlm, "nbest": args.nbest, "space": args.sym_space, "blank": args.sym_blank, "tgt_lang": False, } self.trans_args = argparse.Namespace(**trans_args) self.report_bleu = args.report_bleu else: self.report_bleu = False self.rnnlm = None self.logzero = -10000000000.0 self.loss = None self.acc = None
def main(_run): # =============== # INTRO # =============== args = argparse.Namespace(**_run.config) torch.manual_seed(args.seed) np.random.seed(args.seed) if args.gpu > -1: torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if args.gpu < 0: device = torch.device("cpu") else: device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") data_transforms = { 'train': transforms.Compose([ transforms.Resize((224, 224)), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Lambda(lambda x: x.repeat(3, 1, 1)), ]), 'val': transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Lambda(lambda x: x.repeat(3, 1, 1)) ]), } train_dataset = ChexpertDataset(os.path.join(CONSTS.DATA_DIR, 'CheXpert', 'train_pleural.csv'), root_dir=CONSTS.DATA_DIR, transform=data_transforms['train']) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True) val_dataset = ChexpertDataset(os.path.join(CONSTS.DATA_DIR, 'CheXpert', 'test_pleural.csv'), root_dir=CONSTS.DATA_DIR, transform=data_transforms['val']) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False) model_path = os.path.join(CONSTS.RESULTS_DIR, 'models', args.exp_name) if not os.path.exists(model_path): os.makedirs(model_path) output_dir = os.path.join(CONSTS.RESULTS_DIR, 'outputs', args.exp_name) if not os.path.exists(output_dir): os.makedirs(output_dir) tensorboard_path = os.path.join(CONSTS.RESULTS_DIR, 'logs', 'tensorboard', args.exp_name) if not os.path.exists(tensorboard_path): os.makedirs(tensorboard_path) model = models.resnet50(pretrained=False) num_ftrs = model.fc.in_features model.fc = nn.Linear(num_ftrs, 2) model.load_state_dict( torch.load(os.path.join(model_path, 'best_model.pth'))) model.to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr) scheduler = ReduceLROnPlateau(optimizer, 'min', patience=3, verbose=True) writer = SummaryWriter(tensorboard_path, comment=args.exp_name) patience = 20 early_stopping = EarlyStopping(patience=patience, verbose=True, save=os.path.join(model_path, 'best_model.pth')) for epoch in range(1, args.epochs + 1): # train_epoch_loss, train_epoch_accuracy = train_epoch(epoch, model, train_loader, device, optimizer, args) # print('====> Epoch: {} Average training loss: {:.4f}'.format( # epoch, train_epoch_loss)) # print('====> Epoch: {} Average training accuracy: {:.4f} %'.format( # epoch, train_epoch_accuracy*100)) val_epoch_loss, val_epoch_accuracy = val(model, val_loader, device) print('====> val set loss: {:.4f}'.format(val_epoch_loss)) print('====> val set accuracy: {:.4f} %'.format(val_epoch_accuracy * 100)) # writer.add_scalar('Loss/train', train_epoch_loss, epoch) writer.add_scalar('Loss/val', val_epoch_loss, epoch) # writer.add_scalar('Accuracy/train', train_epoch_accuracy, epoch) writer.add_scalar('Accuracy/val', val_epoch_accuracy, epoch) # scheduler.step(train_epoch_loss) early_stopping(val_epoch_loss, model) writer.close()
def main(args): """ Execute a wrapper over log-analyze-parse, aka 'check'. """ logger.setup_logger(args.verbose if 'verbose' in args else None) def __load_module(name): """Loads the given subcommand's definition from the libs.""" try: module = libhandlers.load_module(name) except ImportError: LOG.error("Couldn't import subcommand '" + name + "'.") raise return module def __update_if_key_exists(source, target, key): """Append the source Namespace's element with 'key' to target with the same key, but only if it exists.""" if key in source: setattr(target, key, getattr(source, key)) # If no output directory is given then the checker results go to a # temporary directory. This way the subsequent "quick" checks don't pollute # the result list of a previous check. If the detection status function is # intended to be used (i.e. by keeping the .plist files) then the output # directory has to be provided explicitly. is_temp_output = False if 'output_dir' in args: output_dir = args.output_dir else: output_dir = tempfile.mkdtemp() is_temp_output = True output_dir = os.path.abspath(output_dir) if not os.path.exists(output_dir): os.makedirs(output_dir) logfile = None is_command = False try: # --- Step 1.: Perform logging if build command was specified. if 'command' in args: logfile = tempfile.NamedTemporaryFile().name is_command = True # Translate the argument list between check and log. log_args = argparse.Namespace(command=args.command, logfile=logfile) __update_if_key_exists(args, log_args, 'quiet') __update_if_key_exists(args, log_args, 'verbose') log_module = __load_module('log') LOG.debug("Calling LOG with args:") LOG.debug(log_args) log_module.main(log_args) elif 'logfile' in args: logfile = args.logfile # --- Step 2.: Perform the analysis. if not os.path.exists(logfile): raise OSError("The specified logfile '" + logfile + "' does not " "exist.") analyze_args = argparse.Namespace(logfile=[logfile], output_path=output_dir, output_format='plist', jobs=args.jobs) # Some arguments don't have default values. # We can't set these keys to None because it would result in an error # after the call. args_to_update = [ 'quiet', 'skipfile', 'analyzers', 'add_compiler_defaults', 'clangsa_args_cfg_file', 'tidy_args_cfg_file', 'capture_analysis_output', 'ctu_phases', 'stats_output', 'stats_dir', 'stats_enabled', 'enable_all', 'ordered_checkers', # --enable and --disable. 'timeout' ] for key in args_to_update: __update_if_key_exists(args, analyze_args, key) if 'force' in args or 'clean' in args: setattr(analyze_args, 'clean', True) __update_if_key_exists(args, analyze_args, 'verbose') analyze_module = __load_module('analyze') LOG.debug("Calling ANALYZE with args:") LOG.debug(analyze_args) analyze_module.main(analyze_args) # --- Step 3.: Print to stdout. parse_args = argparse.Namespace(input=[output_dir], input_format='plist') __update_if_key_exists(args, parse_args, 'print_steps') __update_if_key_exists(args, parse_args, 'verbose') __update_if_key_exists(args, parse_args, 'skipfile') parse_module = __load_module('parse') LOG.debug("Calling PARSE with args:") LOG.debug(parse_args) parse_module.main(parse_args) except ImportError: LOG.error("Check failed: couldn't import a library.") except Exception as ex: LOG.error("Running check failed. " + ex.message) import traceback traceback.print_exc() finally: if is_temp_output: shutil.rmtree(output_dir) if is_command: os.remove(logfile) LOG.debug("Check finished.")
import pytest import random args = argparse.Namespace( elayers=4, subsample="1_2_2_1_1", etype="vggblstmp", eunits=320, eprojs=320, dlayers=2, dunits=300, atype="location", aconv_chans=10, aconv_filts=100, mtlalpha=0.5, adim=320, dropout_rate=0.0, beam_size=3, penalty=0.5, maxlenratio=1.0, minlenratio=0.0, ctc_weight=0.2, verbose=True, char_list=[u"あ", u"い", u"う", u"え", u"お"], outdir=None, seed=1 ) def test_lecun_init_torch(): torch = pytest.importorskip("torch")
def populate_layer_additional_data_dict(self, missing_default_data_group_is_OK=False): self.progress.new('Layer additional data ops') self.progress.update('...') data_groups_common_to_all_profile_dbs = set([]) for p in self.input_profile_db_paths: if self.input_profile_db_paths.index(p) == 0: data_groups_common_to_all_profile_dbs = set(TableForLayerAdditionalData(argparse.Namespace(profile_db = p)).get_group_names()) else: data_groups_common_to_all_profile_dbs.intersection_update(set(TableForLayerAdditionalData(argparse.Namespace(profile_db = p)).get_group_names())) if 'default' not in data_groups_common_to_all_profile_dbs: if missing_default_data_group_is_OK: pass else: raise ConfigError("There is something wrong with your input databases. The group name 'default'\ should be common to all of them, but it doesn't seem to be the case :/ How did\ you end up with an anvi'o single profile database that doesn't have the 'default'\ group in its additional layer data table? It is very likely that your profiling\ step failed for some reason for one or more of your databases :(") taxonomic_data_groups = set(constants.levels_of_taxonomy).intersection(data_groups_common_to_all_profile_dbs) regular_data_groups = data_groups_common_to_all_profile_dbs.difference(taxonomic_data_groups) self.progress.end() self.__populate_layer_additional_data_dict_for_regular_data_groups(regular_data_groups) self.__populate_layer_additional_data_dict_for_taxonomic_data_groups(taxonomic_data_groups)
def main(): # Load configuration args, recog_params, dir_name = parse_args_eval(sys.argv[1:]) args = compute_subsampling_factor(args) # Setting for logging if os.path.isfile(os.path.join(args.recog_dir, 'plot.log')): os.remove(os.path.join(args.recog_dir, 'plot.log')) set_logger(os.path.join(args.recog_dir, 'plot.log'), stdout=args.recog_stdout) for i, s in enumerate(args.recog_sets): # Load dataloader dataloader = build_dataloader(args=args, tsv_path=s, batch_size=1, is_test=True) if i == 0: # Load the ASR model model = Speech2Text(args, dir_name) epoch = int(float(args.recog_model[0].split('-')[-1]) * 10) / 10 if args.recog_n_average > 1: # Model averaging for Transformer model = average_checkpoints(model, args.recog_model[0], n_average=args.recog_n_average) else: load_checkpoint(args.recog_model[0], model) # Ensemble (different models) ensemble_models = [model] if len(args.recog_model) > 1: for recog_model_e in args.recog_model[1:]: conf_e = load_config( os.path.join(os.path.dirname(recog_model_e), 'conf.yml')) args_e = copy.deepcopy(args) for k, v in conf_e.items(): if 'recog' not in k: setattr(args_e, k, v) model_e = Speech2Text(args_e) load_checkpoint(recog_model_e, model_e) if args.recog_n_gpus >= 1: model_e.cuda() ensemble_models += [model_e] # Load the LM for shallow fusion if not args.lm_fusion: # first path if args.recog_lm is not None and args.recog_lm_weight > 0: conf_lm = load_config( os.path.join(os.path.dirname(args.recog_lm), 'conf.yml')) args_lm = argparse.Namespace() for k, v in conf_lm.items(): setattr(args_lm, k, v) lm = build_lm(args_lm) load_checkpoint(args.recog_lm, lm) if args_lm.backward: model.lm_bwd = lm else: model.lm_fwd = lm # NOTE: only support for first path if not args.recog_unit: args.recog_unit = args.unit logger.info('recog unit: %s' % args.recog_unit) logger.info('recog oracle: %s' % args.recog_oracle) logger.info('epoch: %d' % epoch) logger.info('batch size: %d' % args.recog_batch_size) logger.info('beam width: %d' % args.recog_beam_width) logger.info('min length ratio: %.3f' % args.recog_min_len_ratio) logger.info('max length ratio: %.3f' % args.recog_max_len_ratio) logger.info('length penalty: %.3f' % args.recog_length_penalty) logger.info('length norm: %s' % args.recog_length_norm) logger.info('coverage penalty: %.3f' % args.recog_coverage_penalty) logger.info('coverage threshold: %.3f' % args.recog_coverage_threshold) logger.info('CTC weight: %.3f' % args.recog_ctc_weight) logger.info('fist LM path: %s' % args.recog_lm) logger.info('LM weight: %.3f' % args.recog_lm_weight) logger.info('GNMT: %s' % args.recog_gnmt_decoding) logger.info('forward-backward attention: %s' % args.recog_fwd_bwd_attention) logger.info('resolving UNK: %s' % args.recog_resolving_unk) logger.info('ensemble: %d' % (len(ensemble_models))) logger.info('ASR decoder state carry over: %s' % (args.recog_asr_state_carry_over)) logger.info('LM state carry over: %s' % (args.recog_lm_state_carry_over)) logger.info('model average (Transformer): %d' % (args.recog_n_average)) # GPU setting if args.recog_n_gpus >= 1: model.cudnn_setting(deterministic=True, benchmark=False) model.cuda() save_path = mkdir_join(args.recog_dir, 'att_weights') # Clean directory if save_path is not None and os.path.isdir(save_path): shutil.rmtree(save_path) os.mkdir(save_path) while True: batch, is_new_epoch = dataloader.next( recog_params['recog_batch_size']) nbest_hyps_id, aws = model.decode( batch['xs'], recog_params, dataloader.idx2token[0], exclude_eos=False, refs_id=batch['ys'], ensemble_models=ensemble_models[1:] if len(ensemble_models) > 1 else [], speakers=batch['sessions'] if dataloader.corpus == 'swbd' else batch['speakers']) best_hyps_id = [h[0] for h in nbest_hyps_id] # Get CTC probs ctc_probs, topk_ids = None, None if args.ctc_weight > 0: ctc_probs, topk_ids, xlens = model.get_ctc_probs( batch['xs'], task='ys', temperature=1, topk=min(100, model.vocab)) # NOTE: ctc_probs: '[B, T, topk]' ctc_probs_sub1, topk_ids_sub1 = None, None if args.ctc_weight_sub1 > 0: ctc_probs_sub1, topk_ids_sub1, xlens_sub1 = model.get_ctc_probs( batch['xs'], task='ys_sub1', temperature=1, topk=min(100, model.vocab_sub1)) if model.bwd_weight > 0.5: # Reverse the order best_hyps_id = [hyp[::-1] for hyp in best_hyps_id] aws = [[aw[0][:, ::-1]] for aw in aws] for b in range(len(batch['xs'])): tokens = dataloader.idx2token[0](best_hyps_id[b], return_list=True) spk = batch['speakers'][b] plot_attention_weights( aws[b][0][:, :len(tokens)], tokens, spectrogram=batch['xs'][b][:, :dataloader.input_dim] if args.input_type == 'speech' else None, factor=args.subsample_factor, ref=batch['text'][b].lower(), save_path=mkdir_join(save_path, spk, batch['utt_ids'][b] + '.png'), figsize=(20, 8), ctc_probs=ctc_probs[b, :xlens[b]] if ctc_probs is not None else None, ctc_topk_ids=topk_ids[b] if topk_ids is not None else None, ctc_probs_sub1=ctc_probs_sub1[b, :xlens_sub1[b]] if ctc_probs_sub1 is not None else None, ctc_topk_ids_sub1=topk_ids_sub1[b] if topk_ids_sub1 is not None else None) if model.bwd_weight > 0.5: hyp = ' '.join(tokens[::-1]) else: hyp = ' '.join(tokens) logger.info('utt-id: %s' % batch['utt_ids'][b]) logger.info('Ref: %s' % batch['text'][b].lower()) logger.info('Hyp: %s' % hyp) logger.info('-' * 50) if is_new_epoch: break
def main(opt, callbacks=Callbacks()): # Checks set_logging(RANK) if RANK in [-1, 0]: print_args(FILE.stem, opt) check_git_status() check_requirements(exclude=['thop']) # Resume if opt.resume and not check_wandb_resume( opt) and not opt.evolve: # resume an interrupted run ckpt = opt.resume if isinstance( opt.resume, str) else get_latest_run() # specified or most recent path assert os.path.isfile( ckpt), 'ERROR: --resume checkpoint does not exist' with open(Path(ckpt).parent.parent / 'opt.yaml') as f: opt = argparse.Namespace(**yaml.safe_load(f)) # replace opt.cfg, opt.weights, opt.resume = '', ckpt, True # reinstate LOGGER.info(f'Resuming training from {ckpt}') else: opt.data, opt.cfg, opt.hyp, opt.weights, opt.project = \ check_file(opt.data), check_yaml(opt.cfg), check_yaml(opt.hyp), str(opt.weights), str(opt.project) # checks assert len(opt.cfg) or len( opt.weights), 'either --cfg or --weights must be specified' if opt.evolve: opt.project = str(ROOT / 'runs/evolve') opt.exist_ok, opt.resume = opt.resume, False # pass resume to exist_ok and disable resume opt.save_dir = str( increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # DDP mode device = select_device(opt.device, batch_size=opt.batch_size) if LOCAL_RANK != -1: assert torch.cuda.device_count( ) > LOCAL_RANK, 'insufficient CUDA devices for DDP command' assert opt.batch_size % WORLD_SIZE == 0, '--batch-size must be multiple of CUDA device count' assert not opt.image_weights, '--image-weights argument is not compatible with DDP training' assert not opt.evolve, '--evolve argument is not compatible with DDP training' torch.cuda.set_device(LOCAL_RANK) device = torch.device('cuda', LOCAL_RANK) dist.init_process_group( backend="nccl" if dist.is_nccl_available() else "gloo") # Train if not opt.evolve: train(opt.hyp, opt, device, callbacks) if WORLD_SIZE > 1 and RANK == 0: LOGGER.info('Destroying process group... ') dist.destroy_process_group() # Evolve hyperparameters (optional) else: # Hyperparameter evolution metadata (mutation scale 0-1, lower_limit, upper_limit) meta = { 'lr0': (1, 1e-5, 1e-1), # initial learning rate (SGD=1E-2, Adam=1E-3) 'lrf': (1, 0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf) 'momentum': (0.3, 0.6, 0.98), # SGD momentum/Adam beta1 'weight_decay': (1, 0.0, 0.001), # optimizer weight decay 'warmup_epochs': (1, 0.0, 5.0), # warmup epochs (fractions ok) 'warmup_momentum': (1, 0.0, 0.95), # warmup initial momentum 'warmup_bias_lr': (1, 0.0, 0.2), # warmup initial bias lr 'box': (1, 0.02, 0.2), # box loss gain 'cls': (1, 0.2, 4.0), # cls loss gain 'cls_pw': (1, 0.5, 2.0), # cls BCELoss positive_weight 'obj': (1, 0.2, 4.0), # obj loss gain (scale with pixels) 'obj_pw': (1, 0.5, 2.0), # obj BCELoss positive_weight 'iou_t': (0, 0.1, 0.7), # IoU training threshold 'anchor_t': (1, 2.0, 8.0), # anchor-multiple threshold 'anchors': (2, 2.0, 10.0), # anchors per output grid (0 to ignore) 'fl_gamma': (0, 0.0, 2.0), # focal loss gamma (efficientDet default gamma=1.5) 'hsv_h': (1, 0.0, 0.1), # image HSV-Hue augmentation (fraction) 'hsv_s': (1, 0.0, 0.9), # image HSV-Saturation augmentation (fraction) 'hsv_v': (1, 0.0, 0.9), # image HSV-Value augmentation (fraction) 'degrees': (1, 0.0, 45.0), # image rotation (+/- deg) 'translate': (1, 0.0, 0.9), # image translation (+/- fraction) 'scale': (1, 0.0, 0.9), # image scale (+/- gain) 'shear': (1, 0.0, 10.0), # image shear (+/- deg) 'perspective': (0, 0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 'flipud': (1, 0.0, 1.0), # image flip up-down (probability) 'fliplr': (0, 0.0, 1.0), # image flip left-right (probability) 'mosaic': (1, 0.0, 1.0), # image mixup (probability) 'mixup': (1, 0.0, 1.0), # image mixup (probability) 'copy_paste': (1, 0.0, 1.0) } # segment copy-paste (probability) with open(opt.hyp) as f: hyp = yaml.safe_load(f) # load hyps dict if 'anchors' not in hyp: # anchors commented in hyp.yaml hyp['anchors'] = 3 opt.noval, opt.nosave, save_dir = True, True, Path( opt.save_dir) # only val/save final epoch # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices evolve_yaml, evolve_csv = save_dir / 'hyp_evolve.yaml', save_dir / 'evolve.csv' if opt.bucket: os.system(f'gsutil cp gs://{opt.bucket}/evolve.csv {save_dir}' ) # download evolve.csv if exists for _ in range(opt.evolve): # generations to evolve if evolve_csv.exists( ): # if evolve.csv exists: select best hyps and mutate # Select parent(s) parent = 'single' # parent selection method: 'single' or 'weighted' x = np.loadtxt(evolve_csv, ndmin=2, delimiter=',', skiprows=1) n = min(5, len(x)) # number of previous results to consider x = x[np.argsort(-fitness(x))][:n] # top n mutations w = fitness(x) - fitness(x).min() + 1E-6 # weights (sum > 0) if parent == 'single' or len(x) == 1: # x = x[random.randint(0, n - 1)] # random selection x = x[random.choices(range(n), weights=w)[0]] # weighted selection elif parent == 'weighted': x = (x * w.reshape( n, 1)).sum(0) / w.sum() # weighted combination # Mutate mp, s = 0.8, 0.2 # mutation probability, sigma npr = np.random npr.seed(int(time.time())) g = np.array([meta[k][0] for k in hyp.keys()]) # gains 0-1 ng = len(meta) v = np.ones(ng) while all( v == 1 ): # mutate until a change occurs (prevent duplicates) v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0) for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300) hyp[k] = float(x[i + 7] * v[i]) # mutate # Constrain to limits for k, v in meta.items(): hyp[k] = max(hyp[k], v[1]) # lower limit hyp[k] = min(hyp[k], v[2]) # upper limit hyp[k] = round(hyp[k], 5) # significant digits # Train mutation results = train(hyp.copy(), opt, device, callbacks) # Write mutation results print_mutation(results, hyp.copy(), save_dir, opt.bucket) # Plot results plot_evolve(evolve_csv) print( f'Hyperparameter evolution finished\n' f"Results saved to {colorstr('bold', save_dir)}\n" f'Use best hyperparameters example: $ python train.py --hyp {evolve_yaml}' )
matplotlib.use('Agg') np.random.seed(1003) # set args args = { 'ckptpath': './model.ckpt', 'dataset_dir': sys.argv[1], 'output_dir': sys.argv[2], 'noodles_pasta': [2431, 2352, 2379, 2466, 2383], 'rice': [2533, 2504, 2520, 2495, 2553], 'soup': [3292, 3159, 3094, 3161, 3278], 'vegetable_fruit': [523, 550, 507, 532, 561], 'num_features': 11, 'device': 'cuda' } args = argparse.Namespace(**args) # build model model = Model( make_layers([ 32, 32, 32, 'M', 64, 64, 64, 'M', 128, 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M' ]) ).to(args.device) # load checkpoint checkpoint = torch.load(args.ckptpath) model.load_state_dict(checkpoint['model_state_dict'])