def generate_probs(model_dir, inp_fp, weights_fp, topk, out_ext, cuda_device,
overwrite, batch_size, extraction_ratio, out):
import_submodules('imojie')
args = argparse.Namespace()
args.archive_file = model_dir
args.cuda_device = cuda_device
args.embedding_sources_mapping = {}
args.extend_vocab = None
args.batch_weight_key = ''
args.output_file = ''
args.overrides = "{'model': {'token_based_metric': null}, 'iterator': {'batch_size': " + str(
batch_size) + ", \
'instances_per_epoch': null}, 'trainer':{'num_epochs':1}, 'dataset_reader': {'max_tokens': 10000, \
'gradients': false, 'max_extractions': 30, 'extraction_ratio': " + str(
extraction_ratio) + ", 'probability': true \
}, 'validation_dataset_reader': null}"
args.weights_file = weights_fp
args.input_file = inp_fp
probs = evaluate_from_args(args)
probsD = dict()
# For some reason the last batch results are repeated in the probs
# Not an issue as they are just overwritten while forming the probsD
for i in range(len(probs['example_ids'])):
probsD[probs['example_ids'][i]] = probs['probs'][i]
lines = open(inp_fp).readlines()
all_fields = []
for line_number, line in enumerate(lines):
line = line.strip('\n')
fields = line.split('\t')
if line_number not in probsD: # the example is too large and rejected by dataloader ('max_tokens' argument)
continue
# Removing appended extractions after reranking
fields[0] = fields[0].split('[SEP]')[0].strip()
fields[2] = str(probsD[line_number])
fields.append(str(line_number))
all_fields.append('\t'.join(fields))
if topk == None:
return all_fields
else:
# sorting all_fields according to the confidences assigned by bert_encoder
all_fields_sorted = []
prev_sent = None
exts = []
for f in all_fields:
sent = f.split('\t')[0]
if sent != prev_sent:
if prev_sent != None:
exts = toolz.unique(exts, key=lambda x: x.split('\t')[1])
exts = sorted(exts,
reverse=True,
key=lambda x: float(x.split('\t')[2]))
if topk != None:
exts = exts[:topk]
all_fields_sorted.extend(exts)
prev_sent = sent
exts = [f]
else:
exts.append(f)
exts = sorted(exts,
reverse=True,
key=lambda x: float(x.split('\t')[2]))
all_fields_sorted.extend(exts)
open(out, 'w').write('\n'.join(all_fields_sorted))
print('Probabilities written to: ', out)
return all_fields_sorted
import sys
sys.path.insert(1, os.getcwd())
from gits_pull import gits_pull
from mock import patch, Mock
def parse_args(args):
parser = argparse.ArgumentParser()
return parser.parse_args(args)
@patch("argparse.ArgumentParser.parse_args",
return_value=argparse.Namespace(nocommit=True,
rebase=False,
branch="branch name"))
@patch("subprocess.Popen")
@patch("helper.get_current_branch", return_value="current branch")
def test_gits_pull_happy_case_with_no_commit_and_given_branch(
mock_current_branch, mock_var, mock_args):
"""
Function to test gits pull, success case with no commits, no rebase and given branch
"""
mocked_pipe = Mock()
attrs = {'communicate.return_value': (b'', 'error'), 'returncode': 0}
mocked_pipe.configure_mock(**attrs)
mock_var.return_value = mocked_pipe
mock_args = parse_args(mock_args)
test_result = gits_pull(mock_args)
def test_get_sample_dict_sra_expand_from_xml_sample():
sample_elements = get_test_sample_elements()
args = argparse.Namespace(expand_sra_details=True)
expected_sample_dict = {
"repository": "CDE",
"submitted_sample_id": "NWD1",
"biosample_id": "SAMN1",
"submitted_subject_id": "ABC",
"dbgap_sample_id": "1",
"dbgap_subject_id": "1",
"sra_sample_id": "SRS1",
"consent_code": "1",
"consent_short_name": "GRU-IRB",
"sex": "male",
"body_site": "Whole blood",
"analyte_type": "DNA",
"dbgap_status": "Loaded",
"sample_use": ["Seq_DNA_SNP_CNV", "WGS"],
"sra_data_details": {
"status": "public",
"experiments": "1",
"runs": "3",
"bases": "406977793500",
"size_Gb": "74",
"experiment_type": "WGS",
"platform": "ILLUMINA",
"center": "ABC Fast Track Services",
},
"study_accession": "phs001234.v3.p1",
"study_accession_with_consent": "phs001234.v3.p1.c1",
"study_with_consent": "phs001234.c1",
"study_subject_id": "phs001234.v3_ABC",
}
sample_dict = dbgap_extract.get_sample_dict_from_xml_sample(
"phs001234.v3.p1", sample_elements[0], args)
sample_dict["sample_use"] = json.loads(sample_dict["sample_use"])
sample_dict["sra_data_details"] = json.loads(
sample_dict["sra_data_details"])
assert_dict_equality(sample_dict, expected_sample_dict)
expected_sample_dict = {
"repository": "FGH",
"submitted_sample_id": "NWD1",
"biosample_id": "SAMN2",
"submitted_subject_id": "CDE",
"dbgap_sample_id": "1",
"dbgap_subject_id": "2",
"sra_sample_id": "SRS2",
"consent_code": "1",
"consent_short_name": "GRU-IRB",
"sex": "female",
"body_site": "Whole blood",
"analyte_type": "DNA",
"dbgap_status": "Loaded",
"sample_use": ["Seq_DNA_SNP_CWB", "GWS"],
"sra_data_details": {
"status": "public",
"experiments": "1",
"runs": "2",
"bases": "250660703000",
"size_Gb": "49",
"experiment_type": "WGS",
"platform": "ILLUMINA",
"center": "CDE Fast Track Services",
},
"study_accession": "phs001234.v3.p1",
"study_accession_with_consent": "phs001234.v3.p1.c1",
"study_with_consent": "phs001234.c1",
"study_subject_id": "phs001234.v3_CDE",
}
sample_dict = dbgap_extract.get_sample_dict_from_xml_sample(
"phs001234.v3.p1", sample_elements[1], args)
sample_dict["sample_use"] = json.loads(sample_dict["sample_use"])
sample_dict["sra_data_details"] = json.loads(
sample_dict["sra_data_details"])
assert_dict_equality(sample_dict, expected_sample_dict)
parser.add_argument('--test_atts', dest='test_atts', nargs='+', help='test_atts')
parser.add_argument('--test_ints', dest='test_ints', type=float, nargs='+', help='test_ints')
parser.add_argument('--num_test', dest='num_test', type=int)
parser.add_argument('--load_epoch', dest='load_epoch', type=str, default='latest')
parser.add_argument('--custom_img', action='store_true')
parser.add_argument('--custom_data', type=str, default='./data/custom')
parser.add_argument('--custom_attr', type=str, default='./data/list_attr_custom.txt')
parser.add_argument('--gpu', action='store_true')
parser.add_argument('--multi_gpu', action='store_true')
return parser.parse_args(args)
args_ = parse()
print(args_)
with open(join('output', args_.experiment_name, 'setting.txt'), 'r') as f:
args = json.load(f, object_hook=lambda d: argparse.Namespace(**d))
args.test_atts = args_.test_atts
args.test_ints = args_.test_ints
args.num_test = args_.num_test
args.load_epoch = args_.load_epoch
args.custom_img = args_.custom_img
args.custom_data = args_.custom_data
args.custom_attr = args_.custom_attr
args.gpu = args_.gpu
args.multi_gpu = args_.multi_gpu
print(args)
assert args.test_atts is not None, 'test_atts should be chosen in %s' % (str(args.attrs))
for a in args.test_atts:
def get_basemodel(args):
if args.model == 'none': return None
if args.model == 'resnet50' and args.dataset == 'imagenet':
model = models.resnet50(pretrained=True).eval()
normalize = NormalizeLayer(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
else:
path = glob.glob(
os.path.join('./models', args.model, '**', 'checkpoint.pt.best'))
path_tar = glob.glob(
os.path.join('./models', args.model, '**', 'checkpoint.pth.tar'))
if not (len(path) > 0 or
(len(path_tar) > 0 and args.dataset in ['cifar', 'imagenet'])):
print("Could not load model")
sys.exit(1)
if len(path_tar) > 0 and args.dataset in [
'cifar', 'imagenet', 'restricted_imagenet'
]:
sys.path.append('smoothing-adversarial/code')
from architectures import get_architecture
path = path_tar[0]
print('Loading model from', path)
checkpoint = torch.load(path, map_location='cpu')
if args.dataset == 'cifar':
model = get_architecture(checkpoint["arch"], 'cifar10')
else:
model = get_architecture(checkpoint["arch"], args.dataset)
model.load_state_dict(checkpoint['state_dict'])
model = model.to('cpu')
for i, m in enumerate(model):
if isinstance(m, torch.nn.DataParallel):
model[i] = m.module
normalize = None
model = model[1:]
print(model)
else:
path = path[0]
print('Loading model from', path)
if args.dataset in ['imagenet', 'restricted_imagenet', 'cifar']:
ds_class = datasets.DATASETS[args.dataset]
ds = ds_class("./ds/imagenet"
if args.dataset != 'cifar' else './ds/cifar')
model, _ = model_utils.make_and_restore_model(
arch=('resnet18'
if args.dataset == 'cifar' else 'resnet50'),
dataset=ds,
resume_path=path,
parallel=False)
normalize = model.normalizer
model = model.model
elif args.dataset in ['mnist', 'fashionmnist', 'GTSRB']:
if 'mnist' in args.dataset:
num_classes = 10
color_channels = 1
mean = torch.tensor([0.1307])
std = torch.tensor([0.3081])
if 'convnet' in path:
print('convenet')
model = MNISTConvNet()
else:
model = resnet18(num_classes=num_classes,
color_channels=color_channels)
elif args.dataset == 'GTSRB':
num_classes = 43
color_channels = 3
mean = torch.tensor([0.3337, 0.3064, 0.3171])
std = torch.tensor([0.2672, 0.2564, 0.2629])
model = resnet18(num_classes=num_classes,
color_channels=color_channels)
model = RobustnessModelInnerWrapper(model)
d = argparse.Namespace()
d.mean = mean
d.std = std
model = AttackerModel(model, d)
checkpoint = torch.load(path, pickle_module=dill)
state_dict_path = 'model'
if not ('model' in checkpoint):
state_dict_path = 'state_dict'
sd = checkpoint[state_dict_path]
sd = {k[len('module.'):]: v for k, v in sd.items()}
sd = {(k if 'model.net' in k else k.replace(
'model.', 'model.net.')): v
for k, v in sd.items()}
model.load_state_dict(sd)
normalize = model.normalizer
model = model.model
else:
assert (False)
m = []
if normalize is not None:
m.append(normalize.to(args.device))
if args.radiusDecrease >= 0:
shape = {'rot': 'circ', 'trans': 'rect'}[args.transformation]
size = {
'mnist': (1, 28, 28),
'fashionmnist': (1, 28, 28),
'cifar': (3, 32, 32),
'GTSRB': (3, np.inf, np.inf),
'imagenet': (3, np.inf, np.inf),
'restricted_imagenet': (3, np.inf, np.inf)
}[args.dataset]
if args.resize_post_transform > 0:
size = (size[0], min(size[1], args.resize_post_transform),
min(size[2], args.resize_post_transform))
if args.center_crop_post_transform > 0:
size = (size[0], min(size[1], args.center_crop_post_transform),
min(size[2], args.center_crop_post_transform))
V = VingetteModule(size, shape, args.radiusDecrease)
m.append(V)
m.append(model)
model = torch.nn.Sequential(*m)
if args.use_cuda:
model = torch.nn.DataParallel(model.to(args.device))
model = model.eval().to(args.device)
return model
def roll_settings(weights):
ret = argparse.Namespace()
ret.name = get_choice('name', weights)
if ret.name:
ret.name = handle_name(ret.name)
glitches_required = get_choice('glitches_required', weights)
if glitches_required not in ['none', 'no_logic', 'overworld_glitches']:
logging.warning("Only NMG, OWG and No Logic supported")
glitches_required = 'none'
ret.logic = {None: 'noglitches', 'none': 'noglitches', 'no_logic': 'nologic', 'overworld_glitches': 'owglitches'}[glitches_required]
# item_placement = get_choice('item_placement')
# not supported in ER
dungeon_items = get_choice('dungeon_items', weights)
if dungeon_items == 'full' or dungeon_items == True:
dungeon_items = 'mcsb'
elif dungeon_items == 'standard':
dungeon_items = ""
elif not dungeon_items:
dungeon_items = ""
ret.mapshuffle = get_choice('map_shuffle', weights) if 'map_shuffle' in weights else 'm' in dungeon_items
ret.compassshuffle = get_choice('compass_shuffle', weights) if 'compass_shuffle' in weights else 'c' in dungeon_items
ret.keyshuffle = get_choice('smallkey_shuffle', weights) if 'smallkey_shuffle' in weights else 's' in dungeon_items
ret.bigkeyshuffle = get_choice('bigkey_shuffle', weights) if 'bigkey_shuffle' in weights else 'b' in dungeon_items
ret.accessibility = get_choice('accessibility', weights)
entrance_shuffle = get_choice('entrance_shuffle', weights)
ret.shuffle = entrance_shuffle if entrance_shuffle != 'none' else 'vanilla'
goal = get_choice('goals', weights)
ret.goal = {'ganon': 'ganon',
'fast_ganon': 'crystals',
'dungeons': 'dungeons',
'pedestal': 'pedestal',
'triforce-hunt': 'triforcehunt'
}[goal]
ret.openpyramid = goal == 'fast_ganon'
ret.crystals_gt = get_choice('tower_open', weights)
ret.crystals_ganon = get_choice('ganon_open', weights)
ret.mode = get_choice('world_state', weights)
if ret.mode == 'retro':
ret.mode = 'open'
ret.retro = True
ret.hints = get_choice('hints', weights)
ret.swords = {'randomized': 'random',
'assured': 'assured',
'vanilla': 'vanilla',
'swordless': 'swordless'
}[get_choice('weapons', weights)]
ret.difficulty = get_choice('item_pool', weights)
ret.item_functionality = get_choice('item_functionality', weights)
ret.shufflebosses = {'none': 'none',
'simple': 'basic',
'full': 'normal',
'random': 'chaos'
}[get_choice('boss_shuffle', weights)]
ret.shuffleenemies = {'none': 'none',
'shuffled': 'shuffled',
'random': 'chaos'
}[get_choice('enemy_shuffle', weights)]
ret.enemy_damage = {'default': 'default',
'shuffled': 'shuffled',
'random': 'chaos'
}[get_choice('enemy_damage', weights)]
ret.enemy_health = get_choice('enemy_health', weights)
ret.shufflepots = get_choice('pot_shuffle', weights)
ret.beemizer = int(get_choice('beemizer', weights)) if 'beemizer' in weights else 0
ret.timer = {'none': False,
None: False,
False: False,
'timed': 'timed',
'timed_ohko': 'timed-ohko',
'ohko': 'ohko',
'timed_countdown': 'timed-countdown',
'display': 'display'}[get_choice('timer', weights)] if 'timer' in weights.keys() else False
ret.dungeon_counters = get_choice('dungeon_counters', weights) if 'dungeon_counters' in weights else 'default'
ret.progressive = convert_to_on_off(get_choice('progressive', weights)) if "progressive" in weights else 'on'
inventoryweights = weights.get('startinventory', {})
startitems = []
for item in inventoryweights.keys():
itemvalue = get_choice(item, inventoryweights)
if item.startswith(('Progressive ', 'Small Key ', 'Rupee', 'Piece of Heart', 'Boss Heart Container',
'Sanctuary Heart Container', 'Arrow', 'Bombs ', 'Bomb ', 'Bottle')) and isinstance(
itemvalue, int):
for i in range(int(itemvalue)):
startitems.append(item)
elif itemvalue:
startitems.append(item)
ret.glitch_boots = get_choice('glitch_boots', weights) if 'glitch_boots' in weights else True
ret.startinventory = ','.join(startitems)
ret.remote_items = get_choice('remote_items', weights) if 'remote_items' in weights else False
if 'rom' in weights:
romweights = weights['rom']
ret.sprite = get_choice('sprite', romweights)
ret.disablemusic = get_choice('disablemusic', romweights)
ret.extendedmsu = get_choice('extendedmsu', romweights)
ret.quickswap = get_choice('quickswap', romweights)
ret.fastmenu = get_choice('menuspeed', romweights)
ret.heartcolor = get_choice('heartcolor', romweights)
ret.heartbeep = convert_to_on_off(get_choice('heartbeep', romweights))
ret.ow_palettes = get_choice('ow_palettes', romweights)
ret.uw_palettes = get_choice('uw_palettes', romweights)
return ret
class TestConfig(unittest.TestCase):
"""Class to test Config Class.
Mock is used to mock external calls"""
def setUp(self):
"""Set up for the unit tests"""
self.config = Config(gac_ui=ui.cli, create_config=False)
def tearDown(self):
"""Run Clean Up"""
self.config.clean_up()
@patch(
"argparse.ArgumentParser.parse_args",
return_value=argparse.Namespace(
username="******",
profile=None,
insecure=False,
resolve=None,
mfa_code=None,
remember_device=False,
output_format=None,
roles=None,
action_register_device=False,
action_configure=False,
action_list_profiles=False,
action_list_roles=False,
action_store_json_creds=False,
action_setup_fido_authenticator=False,
),
)
def test_get_args_username(self, mock_arg):
"""Test to make sure username gets returned"""
self.config.get_args()
self.assertEqual(self.config.username, "ann")
def test_read_config(self):
"""Test to make sure getting config works"""
test_ui = MockUserInterface(argv=[
"--profile",
"myprofile",
])
with open(test_ui.HOME + "/.okta_aws_login_config",
"w") as config_file:
config_file.write("""
[myprofile]
client_id = foo
""")
config = Config(gac_ui=test_ui, create_config=False)
config.conf_profile = "myprofile"
profile_config = config.get_config_dict()
self.assertEqual(profile_config, {"client_id": "foo"})
def test_read_config_inherited(self):
"""Test to make sure getting config works when inherited"""
test_ui = MockUserInterface(argv=[
"--profile",
"myprofile",
])
with open(test_ui.HOME + "/.okta_aws_login_config",
"w") as config_file:
config_file.write("""
[mybase]
client_id = bar
aws_appname = baz
[myprofile]
inherits = mybase
client_id = foo
aws_rolename = myrole
""")
config = Config(gac_ui=test_ui, create_config=False)
config.conf_profile = "myprofile"
profile_config = config.get_config_dict()
self.assertEqual(profile_config, {
"client_id": "foo",
"aws_appname": "baz",
"aws_rolename": "myrole",
})
def test_read_nested_config_inherited(self):
"""Test to make sure getting config works when inherited"""
test_ui = MockUserInterface(argv=[
"--profile",
"myprofile",
])
with open(test_ui.HOME + "/.okta_aws_login_config",
"w") as config_file:
config_file.write("""
[mybase-level1]
client_id = bar
[mybase-level2]
inherits = mybase-level1
aws_appname = baz
[myprofile]
inherits = mybase-level2
client_id = foo
aws_rolename = myrole
""")
config = Config(gac_ui=test_ui, create_config=False)
config.conf_profile = "myprofile"
profile_config = config.get_config_dict()
self.assertEqual(profile_config, {
"client_id": "foo",
"aws_appname": "baz",
"aws_rolename": "myrole",
})
def test_fail_if_profile_not_found(self):
"""Test to make sure missing Default fails properly"""
test_ui = MockUserInterface(argv=[])
with open(test_ui.HOME + "/.okta_aws_login_config",
"w") as config_file:
config_file.write("""
[myprofile]
client_id = foo
""")
config = Config(gac_ui=test_ui, create_config=False)
config.conf_profile = "DEFAULT"
with self.assertRaises(errors.GimmeAWSCredsError) as context:
config.get_config_dict()
self.assertTrue(
'DEFAULT profile is missing! This is profile is required when not using --profile'
== context.exception.message)
def train(self, ep = 0):
parser = argparse.ArgumentParser()
parser.add_argument('--weights', type=str, default=config.weight, help='initial weights path')
parser.add_argument('--cfg', type=str, default=config.config_model, help='models/yolov5s.yaml path')
parser.add_argument('--data', type=str, default=config.config_data, help='data.yaml path')
parser.add_argument('--hyp', type=str, default='data/hyp.scratch.yaml', help='hyperparameters path')
parser.add_argument('--epochs', type=int, default=config.epochs)
parser.add_argument('--batch-size', type=int, default=config.batch_size, help='total batch size for all GPUs')
parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='[train, test] image sizes')
parser.add_argument('--rect', action='store_true', help='rectangular training')
parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training')
parser.add_argument('--nosave', action='store_true', help='only save final checkpoint')
parser.add_argument('--notest', action='store_true', help='only test final epoch')
parser.add_argument('--noautoanchor', action='store_true', help='disable autoanchor check')
# parser.add_argument('--noautoanchor', type=bool, default=True, help='disable autoanchor check')
parser.add_argument('--evolve', action='store_true', help='evolve hyperparameters')
# parser.add_argument('--evolve', type = bool, default = False, help='evolve hyperparameters')
parser.add_argument('--bucket', type=str, default='', help='gsutil bucket')
parser.add_argument('--cache-images', action='store_true', help='cache images for faster training')
parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training')
parser.add_argument('--device', default=config.device, help='cuda device, i.e. 0 or 0,1,2,3 or cpu') # using GPU 0
parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%')
parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class')
# parser.add_argument('--adam', action='store_true', help='use torch.optim.Adam() optimizer')
parser.add_argument('--adam', type=int, default=config.adam, help='use torch.optim.Adam() optimizer') # using Adam optimizer
parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode')
parser.add_argument('--local_rank', type=int, default=-1, help='DDP parameter, do not modify')
parser.add_argument('--log-imgs', type=int, default=16, help='number of images for W&B logging, max 100')
parser.add_argument('--log-artifacts', action='store_true', help='log artifacts, i.e. final trained model')
parser.add_argument('--workers', type=int, default=8, help='maximum number of dataloader workers')
parser.add_argument('--project', default=config.project_train, help='save to project/name')
parser.add_argument('--name', default=config.name, help='save to project/name')
# parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
parser.add_argument('--exist-ok', type=int, default=config.exist_ok, help='existing project/name ok, do not increment')
opt = parser.parse_args()
# Set DDP variables
opt.total_batch_size = opt.batch_size
opt.world_size = int(os.environ['WORLD_SIZE']) if 'WORLD_SIZE' in os.environ else 1
opt.global_rank = int(os.environ['RANK']) if 'RANK' in os.environ else -1
set_logging(opt.global_rank)
if opt.global_rank in [-1, 0]:
check_git_status()
# Resume
if opt.resume: # resume an interrupted run
ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path
assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist'
with open(Path(ckpt).parent.parent / 'opt.yaml') as f:
opt = argparse.Namespace(**yaml.load(f, Loader=yaml.FullLoader)) # replace
opt.cfg, opt.weights, opt.resume = '', ckpt, True
logger.info('Resuming training from %s' % ckpt)
else:
# opt.hyp = opt.hyp or ('hyp.finetune.yaml' if opt.weights else 'hyp.scratch.yaml')
opt.data, opt.cfg, opt.hyp = check_file(opt.data), check_file(opt.cfg), check_file(opt.hyp) # check files
assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified'
opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test)
opt.name = 'evolve' if opt.evolve else opt.name
opt.save_dir = increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok | opt.evolve) # increment run
# DDP mode
device = select_device(opt.device, batch_size=opt.batch_size)
if opt.local_rank != -1:
assert torch.cuda.device_count() > opt.local_rank
torch.cuda.set_device(opt.local_rank)
device = torch.device('cuda', opt.local_rank)
dist.init_process_group(backend='nccl', init_method='env://') # distributed backend
assert opt.batch_size % opt.world_size == 0, '--batch-size must be multiple of CUDA device count'
opt.batch_size = opt.total_batch_size // opt.world_size
# Hyperparameters
with open(opt.hyp) as f:
hyp = yaml.load(f, Loader=yaml.FullLoader) # load hyps
if 'box' not in hyp:
warn('Compatibility: %s missing "box" which was renamed from "giou" in %s' %
(opt.hyp, 'https://github.com/ultralytics/yolov5/pull/1120'))
hyp['box'] = hyp.pop('giou')
# Train
logger.info(opt)
if not opt.evolve:
tb_writer = None # init loggers
if opt.global_rank in [-1, 0]:
logger.info(f'Start Tensorboard with "tensorboard --logdir {opt.project}", view at http://localhost:6006/')
tb_writer = SummaryWriter(opt.save_dir) # Tensorboard
train(hyp, opt, device, tb_writer, wandb, ep)
# misc_args.add_argument(
# "--snapshot_interval",
# type=int,
# default=50,
# help="save checkpoint every '--snapshost_interval' steps",
# )
args = parser.parse_args()
arg_groups = {}
for group in parser._action_groups:
group_dict = {
a.dest: getattr(args, a.dest, None)
for a in group._group_actions
if a.dest != "help"
}
arg_groups[group.title] = argparse.Namespace(**group_dict)
config = {}
config["trainer"] = vars(arg_groups["optional arguments"])
config["model"] = vars(arg_groups["model arguments"])
config["data"] = vars(arg_groups["data arguments"])
# config["misc"] = vars(arg_groups["misc arguments"])
config["prep"] = {"resize_size": 256, "crop_size": 224}
if config["data"]["dset"] == "office-31":
if (
("amazon" in args.source_domain and "webcam" in args.target_domain)
or ("webcam" in args.source_domain and "dslr" in args.target_domain)
or ("webcam" in args.source_domain and "amazon" in args.target_domain)
or ("dslr" in args.source_domain and "amazon" in args.target_domain)
):
def __init__(self, config_file=None, config=None, args=None, **kwargs):
if args is None:
args = dict()
if isinstance(args, argparse.Namespace):
args = args.__dict__
args.update(kwargs)
args = dict([(k, v) for k, v in args.items() if v is not None])
self.client = argparse.Namespace()
self.foreman = argparse.Namespace()
self.worker = argparse.Namespace()
self.config_file = config_file
self.config = config
raw_config = dict()
if self.config_file is None:
self.config_file = args.get('config_file', None)
if self.config_file is None:
conf_dirs = appdirs.AppDirs(CONFIG_APP_NAME,
CONFIG_APP_AUTHOR,
multipath=True)
check_dirs = conf_dirs.user_config_dir.split(':')
check_dirs += [os.path.join('/etc', CONFIG_APP_NAME)]
check_dirs += conf_dirs.site_config_dir.split(':')
logging.debug('Looking for configuration in {}'.format(check_dirs))
for path in check_dirs:
check_path = os.path.join(path, CONFIG_FILE)
if os.path.isfile(check_path):
self.config_file = check_path
break
if self.config_file is not None:
parser = configparser.ConfigParser()
parser.read(self.config_file)
raw_config = dict([(k, dict(v)) for (k, v) in parser.items()])
if self.config is None:
self.config = args.get('config', None)
default_config = raw_config.get('default', dict())
default_config.update(args)
if self.config is None:
self.config = default_config.get('config', 'client')
logging.info('Using config {} in {} {}'.format(self.config,
self.config_file, args))
if self.config is not None and self.config not in raw_config:
logging.warning('Config {} not found in config file {}'.format(
self.config, self.config_file))
client_config = raw_config.get(self.config, dict())
foreman_config = raw_config.get('foreman', dict())
worker_config = raw_config.get('worker', dict())
self.client.__setattr__(
'server',
client_config.get(
'server',
default_config.get('server', None) or CONFIG_SERVER))
self.client.__setattr__(
'username',
client_config.get('username', default_config.get('username',
None)))
self.client.__setattr__(
'password',
client_config.get('password', default_config.get('password',
None)))
self.client.__setattr__(
'cpus',
parse_int(
client_config.get('cpus', default_config.get('cpus', None))))
self.client.__setattr__(
'memory',
parse_memory(
client_config.get('memory', default_config.get('memory',
None))))
self.client.__setattr__(
'swap',
parse_memory(
client_config.get('swap', default_config.get('swap', None))))
self.client.__setattr__(
'storage',
parse_memory(
client_config.get('storage',
default_config.get('storage', None))))
self.client.__setattr__(
'image',
parse_memory(
client_config.get('image', default_config.get('image', None))))
self.client.__setattr__(
'workspace',
parse_memory(
client_config.get('workspace',
default_config.get('workspace', None))))
self.client.__setattr__(
'pids',
parse_int(
client_config.get('pids', default_config.get('pids', None))))
self.client.__setattr__(
'time',
parse_time(
client_config.get('time', default_config.get('time', None))))
self.client.__setattr__(
'network',
parse_bool(
client_config.get('network',
default_config.get('network', None))))
self.client.__setattr__(
'gpus',
parse_int(
client_config.get('gpus', default_config.get('gpus', None))))
self.foreman.__setattr__(
'temp_path',
foreman_config.get('temp', default_config.get('temp', None)))
self.foreman.__setattr__(
'interval',
parse_float(
foreman_config.get(
'interval',
default_config.get('interval', None) or FOREMAN_INTERVAL)))
self.foreman.__setattr__(
'concurency',
parse_int(
foreman_config.get(
'concurency',
default_config.get('concurency', None)
or FOREMAN_CONCURENCY)))
self.foreman.__setattr__(
'pull',
parse_bool(
foreman_config.get('pull',
default_config.get('pull', None) or False)))
self.foreman.__setattr__(
'cpus',
parse_int(
foreman_config.get('cpus', default_config.get('cpus', None))))
self.foreman.__setattr__(
'memory',
parse_memory(
foreman_config.get('memory',
default_config.get('memory', None))))
self.foreman.__setattr__(
'swap',
parse_memory(
foreman_config.get('swap', default_config.get('swap', None))))
self.foreman.__setattr__(
'storage',
parse_memory(
foreman_config.get('storage',
default_config.get('storage', None))))
self.foreman.__setattr__(
'image',
parse_memory(
foreman_config.get('image', default_config.get('image',
None))))
self.foreman.__setattr__(
'workspace',
parse_memory(
foreman_config.get('workspace',
default_config.get('workspace', None))))
self.foreman.__setattr__(
'pids',
parse_int(
foreman_config.get('pids', default_config.get('pids', None))))
self.foreman.__setattr__(
'time',
parse_time(
foreman_config.get('time', default_config.get('time', None))))
self.foreman.__setattr__(
'network',
parse_bool(
foreman_config.get('network',
default_config.get('network', None))))
self.foreman.__setattr__(
'auto_tags',
parse_bool(
foreman_config.get('auto_tags',
default_config.get('auto_tags', []))))
tags = parse_str_list(
foreman_config.get('tags',
default_config.get('tags', None) or []))
if self.foreman.auto_tags:
tags = set(tags)
tags.update(foreman_auto_tags())
tags = list(tags)
self.foreman.__setattr__('tags', tags)
self.foreman.__setattr__(
'gpus',
parse_int(
foreman_config.get('gpus', default_config.get('gpus', None))))
self.worker.__setattr__(
'debug',
parse_bool(
worker_config.get('debug',
default_config.get('debug', None) or False)))
self.worker.__setattr__(
'verbose',
parse_bool(
worker_config.get('verbose',
default_config.get('verbose', None)
or False)))
self.worker.__setattr__(
'temp_path',
worker_config.get('temp', default_config.get('temp', None)))
self.worker.__setattr__(
'pull',
parse_bool(
worker_config.get('pull',
default_config.get('pull', None) or False)))
self.worker.__setattr__(
'cpus',
parse_int(
worker_config.get('cpus', default_config.get('cpus', None))))
self.worker.__setattr__(
'memory',
parse_memory(
worker_config.get('memory', default_config.get('memory',
None))))
self.worker.__setattr__(
'swap',
parse_memory(
worker_config.get('swap', default_config.get('swap', None))))
self.worker.__setattr__(
'storage',
parse_memory(
worker_config.get('storage',
default_config.get('storage', None))))
self.worker.__setattr__(
'image',
parse_memory(
worker_config.get('image', default_config.get('image', None))))
self.worker.__setattr__(
'workspace',
parse_memory(
worker_config.get('workspace',
default_config.get('workspace', None))))
self.worker.__setattr__(
'pids',
parse_int(
worker_config.get('pids', default_config.get('pids', None))))
self.worker.__setattr__(
'time',
parse_time(
worker_config.get('time', default_config.get('time', None))))
self.worker.__setattr__(
'network',
parse_bool(
worker_config.get('network',
default_config.get('network', None))))
self.worker.__setattr__(
'gpus',
parse_int(
worker_config.get('gpus', default_config.get('gpus', None))))
def task(ctx, config):
"""
Deploy ceph cluster using cephadm
For example, teuthology.yaml can contain the 'defaults' section:
defaults:
cephadm:
containers:
image: 'quay.io/ceph-ci/ceph'
Using overrides makes it possible to customize it per run.
The equivalent 'overrides' section looks like:
overrides:
cephadm:
containers:
image: 'quay.io/ceph-ci/ceph'
:param ctx: the argparse.Namespace object
:param config: the config dict
"""
if config is None:
config = {}
assert isinstance(config, dict), \
"task only supports a dictionary for configuration"
overrides = ctx.config.get('overrides', {})
teuthology.deep_merge(config, overrides.get('ceph', {}))
teuthology.deep_merge(config, overrides.get('cephadm', {}))
log.info('Config: ' + str(config))
# set up cluster context
if not hasattr(ctx, 'ceph'):
ctx.ceph = {}
ctx.managers = {}
if 'cluster' not in config:
config['cluster'] = 'ceph'
cluster_name = config['cluster']
if cluster_name not in ctx.ceph:
ctx.ceph[cluster_name] = argparse.Namespace()
ctx.ceph[cluster_name].bootstrapped = False
# image
teuth_defaults = teuth_config.get('defaults', {})
cephadm_defaults = teuth_defaults.get('cephadm', {})
containers_defaults = cephadm_defaults.get('containers', {})
container_image_name = containers_defaults.get('image', None)
containers = config.get('containers', {})
container_image_name = containers.get('image', container_image_name)
if not hasattr(ctx.ceph[cluster_name], 'image'):
ctx.ceph[cluster_name].image = config.get('image')
ref = None
if not ctx.ceph[cluster_name].image:
if not container_image_name:
raise Exception(
"Configuration error occurred. "
"The 'image' value is undefined for 'cephadm' task. "
"Please provide corresponding options in the task's "
"config, task 'overrides', or teuthology 'defaults' "
"section.")
sha1 = config.get('sha1')
flavor = config.get('flavor', 'default')
if sha1:
if flavor == "crimson":
ctx.ceph[
cluster_name].image = container_image_name + ':' + sha1 + '-' + flavor
else:
ctx.ceph[
cluster_name].image = container_image_name + ':' + sha1
ref = sha1
else:
# hmm, fall back to branch?
branch = config.get('branch', 'master')
ref = branch
ctx.ceph[cluster_name].image = container_image_name + ':' + branch
log.info('Cluster image is %s' % ctx.ceph[cluster_name].image)
with contextutil.nested(
#if the cluster is already bootstrapped bypass corresponding methods
lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\
else initialize_config(ctx=ctx, config=config),
lambda: ceph_initial(),
lambda: normalize_hostnames(ctx=ctx),
lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\
else download_cephadm(ctx=ctx, config=config, ref=ref),
lambda: ceph_log(ctx=ctx, config=config),
lambda: ceph_crash(ctx=ctx, config=config),
lambda: _bypass() if (ctx.ceph[cluster_name].bootstrapped)\
else ceph_bootstrap(ctx, config),
lambda: crush_setup(ctx=ctx, config=config),
lambda: ceph_mons(ctx=ctx, config=config),
lambda: distribute_config_and_admin_keyring(ctx=ctx, config=config),
lambda: ceph_mgrs(ctx=ctx, config=config),
lambda: ceph_osds(ctx=ctx, config=config),
lambda: ceph_mdss(ctx=ctx, config=config),
lambda: ceph_rgw(ctx=ctx, config=config),
lambda: ceph_iscsi(ctx=ctx, config=config),
lambda: ceph_monitoring('prometheus', ctx=ctx, config=config),
lambda: ceph_monitoring('node-exporter', ctx=ctx, config=config),
lambda: ceph_monitoring('alertmanager', ctx=ctx, config=config),
lambda: ceph_monitoring('grafana', ctx=ctx, config=config),
lambda: ceph_clients(ctx=ctx, config=config),
lambda: create_rbd_pool(ctx=ctx, config=config),
):
ctx.managers[cluster_name] = CephManager(
ctx.ceph[cluster_name].bootstrap_remote,
ctx=ctx,
logger=log.getChild('ceph_manager.' + cluster_name),
cluster=cluster_name,
cephadm=True,
)
try:
if config.get('wait-for-healthy', True):
healthy(ctx=ctx, config=config)
log.info('Setup complete, yielding')
yield
finally:
log.info('Teardown begin')
def main(pathActivations, pathOutput, nGroups=1, nClusters=50, MAX_ITER=100,
batchSizeGPU=50, debug=False, extension='.pt', getDistanceEstimation=False,
load=False, perIterSize=-1, recursionLevel=2, save=False, save_last=5,
seqList=None):
# Test the extension is valid
if extension not in ['.txt', '.npy', '.pt']:
raise ValueError(f'Activation file extension invalid ({extension})')
torch.cuda.empty_cache()
args = argparse.Namespace(**locals())
# Export absolute paths for later use
pathActivations = os.path.abspath(pathActivations)
pathOutput = os.path.abspath(pathOutput)
if not load:
assert os.path.exists(pathOutput) is False, \
f"The output file {pathOutput} already exists, please check the option --load !"
assert os.path.exists(os.path.join(os.path.dirname(pathOutput), "checkpoint_last.pt")) is False, \
"Found last_checkpoint.pt in the output directory, please check the option --load !"
print(args)
seqNames, speakers = findAllSeqs(pathActivations,
speaker_level=recursionLevel,
extension=extension,
loadCache=True)
if seqList is not None:
seqNames = filterSeqs(seqList, seqNames)
if debug:
nsamples = 1000
print(f"Debug mode activated, get only {nsamples} samples!")
shuffle(seqNames)
seqNames = seqNames[:nsamples]
if getDistanceEstimation:
shuffle(seqNames)
seqNames = seqNames[:5000]
print("")
print(f'Loading activations at {pathActivations}')
start_time = time.time()
dataset = SequentialData(pathActivations, seqNames, None)
print(f"Dataset loaded in {time.time()-start_time} seconds !")
print("")
nGPUs = torch.cuda.device_count()
if nGPUs == 0:
raise RuntimeError('No GPU found')
batchSize = batchSizeGPU * nGPUs
dataloader = dataset.getDataLoader(batchSize, numWorkers=0)
print(f"Length of dataLoader: {len(dataloader)}")
print("")
# Check if dir exists
if not os.path.exists(os.path.dirname(pathOutput)) and os.path.dirname(pathOutput):
Path(os.path.dirname(pathOutput)).mkdir(parents=True, exist_ok=True)
pathConfig = f"{os.path.splitext(pathOutput)[0]}_args.json"
with open(pathConfig, 'w') as file:
json.dump(vars(args), file, indent=2)
out_state_dict = {}
print("Starting the clustering...")
start_time = time.time()
# Using a dumb lambda function to skip feature extraction as we start from
# the activations
clusters = kMeanGPU(dataloader, lambda x: x, nClusters, nGroups,
perIterSize=perIterSize,
MAX_ITER=MAX_ITER,
save=save, load=load,
save_dir=os.path.dirname(pathOutput),
save_last=save_last,
).cpu()
print(f'Ran clustering '
f'in {time.time() - start_time:.2f} seconds')
clusterModule = kMeanCluster(clusters)
out_state_dict["state_dict"] = clusterModule.state_dict()
out_state_dict["n_clusters"] = nClusters
out_state_dict['dim'] = clusters.size(2)
torch.save(out_state_dict, pathOutput)
with open(pathConfig, 'w') as file:
json.dump(vars(args), file, indent=2)
def __init__(self, tokenizer, **kwargs):
super().__init__()
args = argparse.Namespace(tokenizer=tokenizer, **kwargs)
self.tokenizer = encoders.build_tokenizer(args)
assert self.tokenizer is not None
def __init__(self, bpe, **kwargs):
super().__init__()
args = argparse.Namespace(bpe=bpe, **kwargs)
self.bpe = encoders.build_bpe(args)
assert self.bpe is not None
def main(**kwargs):
args = argparse.Namespace(**kwargs)
if 'save' in args:
if os.path.exists(args.save):
raise RuntimeError('Output file "{}" already exists.'.format(
args.save))
if args.seed is not None:
pyro.set_rng_seed(args.seed)
X, true_counts = load_data()
X_size = X.size(0)
if args.cuda:
X = X.cuda()
# Build a function to compute z_pres prior probabilities.
if args.z_pres_prior_raw:
def base_z_pres_prior_p(t):
return args.z_pres_prior
else:
base_z_pres_prior_p = make_prior(args.z_pres_prior)
# Wrap with logic to apply any annealing.
def z_pres_prior_p(opt_step, time_step):
p = base_z_pres_prior_p(time_step)
if args.anneal_prior == 'none':
return p
else:
decay = dict(lin=lin_decay, exp=exp_decay)[args.anneal_prior]
return decay(p, args.anneal_prior_to, args.anneal_prior_begin,
args.anneal_prior_duration, opt_step)
model_arg_keys = [
'window_size', 'rnn_hidden_size', 'decoder_output_bias',
'decoder_output_use_sigmoid', 'baseline_scalar', 'encoder_net',
'decoder_net', 'predict_net', 'embed_net', 'bl_predict_net',
'non_linearity', 'pos_prior_mean', 'pos_prior_sd', 'scale_prior_mean',
'scale_prior_sd'
]
model_args = {
key: getattr(args, key)
for key in model_arg_keys if key in args
}
air = AIR(num_steps=args.model_steps,
x_size=50,
use_masking=not args.no_masking,
use_baselines=not args.no_baselines,
z_what_size=args.encoder_latent_size,
use_cuda=args.cuda,
**model_args)
if args.verbose:
print(air)
print(args)
if 'load' in args:
print('Loading parameters...')
air.load_state_dict(torch.load(args.load))
vis = visdom.Visdom(env=args.visdom_env)
# Viz sample from prior.
if args.viz:
z, x = air.prior(5, z_pres_prior_p=partial(z_pres_prior_p, 0))
vis.images(draw_many(x, tensor_to_objs(latents_to_tensor(z))))
def per_param_optim_args(module_name, param_name, tags):
lr = args.baseline_learning_rate if 'baseline' in tags else args.learning_rate
return {'lr': lr}
svi = SVI(air.model,
air.guide,
optim.Adam(per_param_optim_args),
loss='ELBO',
trace_graph=True)
# Do inference.
t0 = time.time()
examples_to_viz = X[5:10]
for i in range(1, args.num_steps + 1):
loss = svi.step(X,
args.batch_size,
z_pres_prior_p=partial(z_pres_prior_p, i))
if args.progress_every > 0 and i % args.progress_every == 0:
print('i={}, epochs={:.2f}, elapsed={:.2f}, elbo={:.2f}'.format(
i, (i * args.batch_size) / X_size, (time.time() - t0) / 3600,
loss / X_size))
if args.viz and i % args.viz_every == 0:
trace = poutine.trace(air.guide).get_trace(examples_to_viz, None)
z, recons = poutine.replay(air.prior,
trace)(examples_to_viz.size(0))
z_wheres = tensor_to_objs(latents_to_tensor(z))
# Show data with inferred objection positions.
vis.images(draw_many(examples_to_viz, z_wheres))
# Show reconstructions of data.
vis.images(draw_many(recons, z_wheres))
if args.eval_every > 0 and i % args.eval_every == 0:
# Measure accuracy on subset of training data.
acc, counts, error_z, error_ix = count_accuracy(
X, true_counts, air, 1000)
print('i={}, accuracy={}, counts={}'.format(
i, acc,
counts.numpy().tolist()))
if args.viz and error_ix.size(0) > 0:
vis.images(draw_many(X[error_ix[0:5]],
tensor_to_objs(error_z[0:5])),
opts=dict(caption='errors ({})'.format(i)))
if 'save' in args and i % args.save_every == 0:
print('Saving parameters...')
torch.save(air.state_dict(), args.save)
def main():
"""
Computational Genomics Lab, Genomics Institute, UC Santa Cruz
Toil BWA pipeline
Alignment of fastq reads via BWA-kit
General usage:
1. Type "toil-bwa generate" to create an editable manifest and config in the current working directory.
2. Parameterize the pipeline by editing the config.
3. Fill in the manifest with information pertaining to your samples.
4. Type "toil-bwa run [jobStore]" to execute the pipeline.
Please read the README.md located in the source directory or at:
https://github.com/BD2KGenomics/toil-scripts/tree/master/src/toil_scripts/bwa_alignment
Structure of the BWA pipeline (per sample)
0 --> 1
0 = Download sample
1 = Run BWA-kit
===================================================================
:Dependencies:
cURL: apt-get install curl
Toil: pip install toil
Docker: wget -qO- https://get.docker.com/ | sh
Optional:
S3AM: pip install --s3am (requires ~/.boto config file)
Boto: pip install boto
"""
# Define Parser object and add to Toil
parser = argparse.ArgumentParser(
description=main.__doc__,
formatter_class=argparse.RawTextHelpFormatter)
subparsers = parser.add_subparsers(dest='command')
# Generate subparsers
subparsers.add_parser(
'generate-config',
help='Generates an editable config in the current working directory.')
subparsers.add_parser(
'generate-manifest',
help='Generates an editable manifest in the current working directory.'
)
subparsers.add_parser(
'generate',
help='Generates a config and manifest in the current working directory.'
)
# Run subparser
parser_run = subparsers.add_parser('run',
help='Runs the BWA alignment pipeline')
group = parser_run.add_mutually_exclusive_group()
parser_run.add_argument(
'--config',
default='config-toil-bwa.yaml',
type=str,
help=
'Path to the (filled in) config file, generated with "generate-config".'
)
group.add_argument(
'--manifest',
default='manifest-toil-bwa.tsv',
type=str,
help=
'Path to the (filled in) manifest file, generated with "generate-manifest". '
'\nDefault value: "%(default)s".')
group.add_argument(
'--sample',
nargs='+',
action=required_length(2, 3),
help=
'Space delimited sample UUID and fastq files in the format: uuid url1 [url2].'
)
# Print docstring help if no arguments provided
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
Job.Runner.addToilOptions(parser_run)
args = parser.parse_args()
# Parse subparsers related to generation of config and manifest
cwd = os.getcwd()
if args.command == 'generate-config' or args.command == 'generate':
generate_file(os.path.join(cwd, 'config-toil-bwa.yaml'),
generate_config)
if args.command == 'generate-manifest' or args.command == 'generate':
generate_file(os.path.join(cwd, 'manifest-toil-bwa.tsv'),
generate_manifest)
# Pipeline execution
elif args.command == 'run':
require(os.path.exists(args.config),
'{} not found. Please run generate-config'.format(args.config))
if not args.sample:
args.sample = None
require(
os.path.exists(args.manifest),
'{} not found and no sample provided. '
'Please run "generate-manifest"'.format(args.manifest))
# Parse config
parsed_config = {
x.replace('-', '_'): y
for x, y in yaml.load(open(args.config).read()).iteritems()
}
config = argparse.Namespace(**parsed_config)
config.maxCores = int(args.maxCores) if args.maxCores else sys.maxint
samples = [args.sample[0], args.sample[1:]
] if args.sample else parse_manifest(args.manifest)
# Sanity checks
require(config.ref,
'Missing URL for reference file: {}'.format(config.ref))
require(config.output_dir,
'No output location specified: {}'.format(config.output_dir))
# Launch Pipeline
Job.Runner.startToil(
Job.wrapJobFn(download_reference_files, config, samples), args)
def main():
parser = argparse.ArgumentParser(
description="Pull transactions information and format it to csv or ledger format.",
add_help=False,
)
subs = parser.add_subparsers(title="actions")
# Parser for the verbosity, common to all subparsers
commonp = argparse.ArgumentParser(add_help=False)
commonp.add_argument("--verbose", "-v", action="count", default=0)
commonp.add_argument(
"--output",
type=argparse.FileType("w"),
default=sys.stdout,
help="File to wite the transaction data to. Default to stdout.",
)
# Parser for the init command
initp = subs.add_parser(
"init",
description="init the current directory, populating the pledger config files.",
parents=[commonp],
)
initp.add_argument(
"--force",
"-f",
nargs="?",
default=False,
const=True,
help="If set, will overwrite the config files if some already exists",
)
initp.set_defaults(func=xact.init_fn)
# Parser for the fetch command
fetchp = subs.add_parser(
"fetch",
description="fetch the transaction history from the specified account.",
parents=[commonp],
)
fetchp.add_argument(
"bank", help="The institution to fetch the transaction history from."
)
fetchp.add_argument(
"--start",
help="Starting date, should be formatted as YYYY-MM-DD.",
type=date.fromisoformat,
default=date.min,
)
fetchp.add_argument(
"--end",
help="Ending date, should be formatted as YYYY-MM-DD.",
type=date.fromisoformat,
default=date.today(),
)
# We can either pick one account or all fo them. If no option selected
# picks all the account listed for that institution in the account file.
group = fetchp.add_mutually_exclusive_group()
group.add_argument(
"--account", help="If only interested in one specific account.",
)
group.add_argument(
"--all",
nargs="?",
default=False,
const=True,
help="Fetches transactions for every account at the institution.",
)
fetchp.set_defaults(func=xact.fetch_fn)
# Parser for the convert command
convertp = subs.add_parser(
"convert",
description="Converts transaction data to csv or ledger format.",
parents=[commonp],
)
convertp.add_argument(
"file", help="The file containing the transaction history, in json format.",
)
convertp.add_argument(
"--format",
choices=["csv", "ledger"],
required=True,
help="The format of the exported data.",
)
convertp.add_argument(
"--currency",
"-c",
nargs="?",
default=False,
const=True,
help="If set, appends the currency iso code after the transaction amount",
)
convertp.add_argument(
"--fields",
default=["date", "amount", "name"],
type=field_list,
help="The fields of the transaction data the are to be exported.",
)
convertp.add_argument(
"--negate",
"-n",
nargs="?",
default=False,
const=True,
help="Flip the sign of the transaction amount.",
)
convertp.set_defaults(func=xact.convert_fn)
# Parser for the list command
listp = subs.add_parser(
"list", description="List account related information", parents=[commonp]
)
listp.add_argument(
"account", help="The account to fetch the transaction history from."
)
listp.set_defaults(func=xact.list_fn)
args = parser.parse_args()
if args == argparse.Namespace():
parser.print_help()
return
# Set verbosity
if args.verbose > 1:
verbosity = logging.DEBUG
elif args.verbose > 0:
verbosity = logging.INFO
else:
verbosity = logging.WARNING
logging.basicConfig(level=verbosity)
args.func(args)
def __init__(self, parent: DetectVideoApp = None):
super().__init__()
self.parent = parent
self.args = argparse.Namespace()
self.args.is_classifier = False
self.classifier_cfg = None
self.classifier_weights = None
self.setWindowIcon(QIcon('../images/Logo.png'))
layout = QVBoxLayout()
self.startDir = os.getcwd()
# Create video selector
self.useWebCam = QCheckBox("Use web cam")
self.useWebCam.setChecked(False)
self.useWebCamLabel = QLabel("Use web cam")
self.useWebCam.stateChanged.connect(self.use_webcam_clicked)
horizontal_layout = QHBoxLayout()
horizontal_layout.addWidget(self.useWebCamLabel)
horizontal_layout.addWidget(self.useWebCam, alignment=Qt.AlignLeft)
horizontal_layout.addStretch(1)
self.btnIm = QPushButton("Select video to detect")
self.btnIm.clicked.connect(self.get_video)
self.textIm = QTextEdit()
layout.addLayout(horizontal_layout, 1)
layout.addStretch(1)
layout.addLayout(SelectQPushButton(self.btnIm), 1)
layout.addLayout(SelectQText(self.textIm), 2)
layout.addStretch(1)
self.textIm.setReadOnly(True)
# Select data set
self.select_ds_label = QLabel("Select dataset")
self.select_ds = DataSetsManager.get_data_set_combo()
self.select_ds.setObjectName("SelectCombo")
self.select_ds.currentTextChanged.connect(self.on_data_set_changed)
layout.addLayout(SelectQCombo(self.select_ds_label, self.select_ds), 2)
layout.addStretch(1)
# Select weights file
self.btnW = QPushButton("Select weights file")
self.btnW.clicked.connect(self.get_weights)
self.textW = QLineEdit()
self.textW.setReadOnly(True)
layout.addLayout(SelectQPushButton(self.btnW), 1)
layout.addLayout(SelectQPushButton(self.textW), 1)
layout.addStretch(1)
# Select Config file
self.btnConf = QPushButton("Select Config file")
self.btnConf.clicked.connect(self.get_config)
self.textConf = QLineEdit()
self.textConf.setReadOnly(True)
layout.addLayout(SelectQPushButton(self.btnConf), 1)
layout.addLayout(SelectQText(self.textConf), 1)
layout.addStretch(1)
# Select Names file
self.btnNames = QPushButton("Select Names file")
self.btnNames.clicked.connect(self.get_names)
self.textNames = QLineEdit()
self.textNames.setReadOnly(True)
layout.addLayout(SelectQPushButton(self.btnNames), 1)
layout.addLayout(SelectQText(self.textNames), 1)
layout.addStretch(1)
bs_label = QLabel('Batch size')
conf_label = QLabel('Confidence')
nms_label = QLabel('Nms threshold')
res_label = QLabel('Resolution')
self.bsEdit = QLineEdit()
self.confEdit = QLineEdit()
self.nmsEdit = QLineEdit()
self.resEdit = QLineEdit()
self.bsEdit.setText("1")
self.confEdit.setText("0.5")
self.nmsEdit.setText("0.4")
self.resEdit.setText("416")
self.textIm.setText("../vid1_Driving_in_Gothenburg_Sweden.mp4")
self.args.video = "../vid1_Driving_in_Gothenburg_Sweden.mp4"
self.on_data_set_changed('Swedish')
self.select_ds.setCurrentText('Swedish')
grid = QGridLayout()
grid.setSpacing(10)
grid.addWidget(bs_label, 1, 0)
grid.addWidget(conf_label, 2, 0)
grid.addWidget(nms_label, 3, 0)
grid.addWidget(res_label, 4, 0)
grid.addWidget(self.bsEdit, 1, 1)
grid.addWidget(self.confEdit, 2, 1)
grid.addWidget(self.nmsEdit, 3, 1)
grid.addWidget(self.resEdit, 4, 1)
grid.setColumnStretch(0, 1)
grid.setColumnStretch(1, 1)
grid.setColumnStretch(2, 2)
layout.addLayout(grid, 5)
tracking_layout = QHBoxLayout()
self.use_tracking = QCheckBox("Use tracking")
self.use_tracking.setChecked(True)
self.use_tracking_label = QLabel("Use tracking: ")
self.use_tracking.stateChanged.connect(self.use_tracking_clicked)
tracking_layout.addWidget(self.use_tracking_label)
tracking_layout.addWidget(self.use_tracking)
tracking_layout.addStretch(1)
layout.addLayout(tracking_layout, 1)
self.tracking = None
self.select_tracking_label = QLabel("Select tracking")
self.select_tracking = QComboBox()
self.select_tracking.setItemDelegate(QStyledItemDelegate())
self.select_tracking.setObjectName("SelectCombo")
self.select_tracking.addItems(["Sort", "Deep Sort"])
self.select_tracking.currentIndexChanged.connect(self.selection_tracking_change)
self.select_tracking.setCurrentIndex(1)
layout.addLayout(SelectQCombo(self.select_tracking_label, self.select_tracking), 2)
count_layout = QHBoxLayout()
self.count_enabled = False
self.use_count = QCheckBox("Count performance")
self.use_count.setChecked(False)
self.use_count_label = QLabel("Count statistics: ")
count_layout.addWidget(self.use_count_label)
count_layout.addWidget(self.use_count)
count_layout.addStretch(1)
layout.addLayout(count_layout, 1)
layout.addStretch(1)
back_button = QPushButton("Back")
ok_button = QPushButton("OK")
cancel_button = QPushButton("Cancel")
hor_box = QHBoxLayout()
hor_box.addWidget(back_button, alignment=Qt.AlignLeft)
hor_box.addStretch(2)
hor_box.addWidget(ok_button)
hor_box.addWidget(cancel_button)
back_button.clicked.connect(self.back_detection)
ok_button.clicked.connect(self.start_detection)
cancel_button.clicked.connect(cancel_detection)
layout.addLayout(hor_box, 2)
self.setLayout(layout)
# Set DDP variables
opt.total_batch_size = opt.batch_size
opt.world_size = int(os.environ['WORLD_SIZE']) if 'WORLD_SIZE' in os.environ else 1
opt.global_rank = int(os.environ['RANK']) if 'RANK' in os.environ else -1
set_logging(opt.global_rank)
if opt.global_rank in [-1, 0]:
check_git_status()
# Resume
if opt.resume: # resume an interrupted run
ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path
log_dir = Path(ckpt).parent.parent # runs/exp0
assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist'
with open(log_dir / 'opt.yaml') as f:
opt = argparse.Namespace(**yaml.load(f, Loader=yaml.FullLoader)) # replace
opt.cfg, opt.weights, opt.resume = '', ckpt, True
logger.info('Resuming training from %s' % ckpt)
else:
# opt.hyp = opt.hyp or ('hyp.finetune.yaml' if opt.weights else 'hyp.scratch.yaml')
opt.data, opt.cfg, opt.hyp = check_file(opt.data), check_file(opt.cfg), check_file(opt.hyp) # check files
assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified'
opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test)
log_dir = increment_dir(Path(opt.logdir) / 'exp', opt.name) # runs/exp1
# DDP mode
device = select_device(opt.device, batch_size=opt.batch_size)
if opt.local_rank != -1:
assert torch.cuda.device_count() > opt.local_rank
torch.cuda.set_device(opt.local_rank)
def run(name, config, print_details=False):
"""
Runs the extension.
Args:
name: The name of the extension
config: The test's config dictionary
print_details: Whether to print the analysis details to stdout
(default: False)
Returns:
A LIVVkit page element containing the LIVVkit elements to display on a webpage
"""
# FIXME: move into a config to NameSpace function
test_args = OrderedDict([(k.replace('-', '_'), v)
for k, v in config.items()])
test_args = argparse.Namespace(**test_args)
test_args.img_dir = os.path.join(livvkit.output_dir, 'validation', 'imgs',
name)
fn.mkdir_p(test_args.img_dir)
details, img_gal = main(test_args)
if print_details:
_print_details(details)
domain_headers = ['Null hypothesis', 'T test (t, P)']
global_tbl_el = {
'Type': 'V-H Table',
'Title': 'Validation',
'TableTitle': 'Analyzed variables',
'Headers': domain_headers,
'Data': details['global'],
}
land_tbl_el = {
'Type': 'V-H Table',
'Title': 'Validation',
'TableTitle': 'Analyzed variables',
'Headers': domain_headers,
'Data': details['land'],
}
ocean_tbl_el = {
'Type': 'V-H Table',
'Title': 'Validation',
'TableTitle': 'Analyzed variables',
'Headers': domain_headers,
'Data': details['ocean'],
}
bib_html = bib2html(os.path.join(os.path.dirname(__file__), 'tsc.bib'))
detail_tables = {"global": [], "land": [], "ocean": []}
table_headers = ["Variable", *domain_headers]
for level in detail_tables:
for _time in details[level]:
tbl_data = {_hdr: [] for _hdr in table_headers}
for _var in details[level][_time]:
tbl_data["Variable"].append(_var)
for _hdr, val in details[level][_time][_var].items():
tbl_data[_hdr].append(val)
detail_tables[level].append(
el.Table(title=f"{level.capitalize()}, {_time}",
data=tbl_data))
tabs = el.Tabs({
'Figures': [img_gal],
'Global_details': detail_tables["global"],
'Land_details': detail_tables["land"],
'Ocean_details': detail_tables["ocean"],
'References': [el.RawHTML(bib_html)]
})
results = el.Table(title="Results",
data=OrderedDict({
'Test status': [details['overall']],
'Global': [details['domains']['delta_l2_global']],
'Land': [details['domains']['delta_l2_land']],
'Ocean': [details['domains']['delta_l2_ocean']],
'Ensembles': [
'statistically identical' if details['overall']
== 'Pass' else 'statistically different'
],
}))
# FIXME: Put into a ___ function
doc_text = __doc__.format(
(1 - test_args.p_threshold) * 100).replace('\n\n', '<br><br>')
page = el.Page(name, doc_text, elements=[results, tabs])
return page
"critical": logging.CRITICAL
}
log_level_parameter = os.getenv("SENZING_LOG_LEVEL", "info").lower()
log_level = log_level_map.get(log_level_parameter, logging.INFO)
logging.basicConfig(format=log_format, level=log_level)
# Parse the command line arguments.
subcommand = os.getenv("SENZING_SUBCOMMAND", None)
parser = get_parser()
if len(sys.argv) > 1:
args = parser.parse_args()
subcommand = args.subcommand
elif subcommand:
args = argparse.Namespace(subcommand=subcommand)
else:
parser.print_help()
exit_silently()
# Transform subcommand from CLI parameter to function name string.
subcommand_function_name = "do_{0}".format(subcommand.replace('-', '_'))
# Test to see if function exists in the code.
if subcommand_function_name not in globals():
logging.warn(message_warn(498, subcommand))
parser.print_help()
exit_silently()
set_parser = getattr(import_module("common"), 'set_parser')
parser = set_parser()
args, unknown = parser.parse_known_args()
# put arguments from (inference) config_file
print(f'from_only_config: {args.from_only_config}')
if args.from_only_config:
# load (inference) config file
# TODO 파일 안 열릴 때 종료하는 예외처리
with open(args.config_path, 'r') as f:
config_dict = json.load(f)
with open(config_dict['saved_inference_config_path'], 'r') as f:
inference_config_dict = json.load(f)
args = argparse.Namespace()
d = vars(args)
for key, value in inference_config_dict.items():
d[key] = value
# add variable to argments for record
d = vars(args)
if args.submission_user_key != '':
user_key = args.submission_user_key
output_file_path = os.path.join(args.submission_dir,
f'{args.saved_model_name}.csv')
desc = f'model:{args.model}, backbone:{args.backbone}, neck:{args.neck} optimizer:{args.optimizer}, scheduler:{args.scheduler} lr:{args.lr}\n' \
f'epoch:{args.best_epoch}/{args.epochs}, best loss:{args.best_loss}, batch size:{args.batch_size}\n' \
f'train augmentation:{args.train_augmentation}, val augmentation:{args.val_augmentation}, test augmentation:{args.test_augmentation}'
submit(user_key, output_file_path, desc)
def __init__(self, idim, odim, args):
"""Construct an E2E object.
:param int idim: dimension of inputs
:param int odim: dimension of outputs
:param Namespace args: argument Namespace containing options
"""
super(E2E, self).__init__()
torch.nn.Module.__init__(self)
# fill missing arguments for compatibility
args = fill_missing_args(args, self.add_arguments)
self.etype = args.etype
self.verbose = args.verbose
# NOTE: for self.build method
args.char_list = getattr(args, "char_list", None)
self.char_list = args.char_list
self.outdir = args.outdir
self.space = args.sym_space
self.blank = args.sym_blank
self.reporter = Reporter()
# below means the last number becomes eos/sos ID
# note that sos/eos IDs are identical
self.sos = odim - 1
self.eos = odim - 1
self.pad = 0
# NOTE: we reserve index:0 for <pad> although this is reserved for a blank class
# in ASR. However, blank labels are not used in MT.
# To keep the vocabulary size,
# we use index:0 for padding instead of adding one more class.
# subsample info
self.subsample = get_subsample(args, mode="mt", arch="rnn")
# label smoothing info
if args.lsm_type and os.path.isfile(args.train_json):
logging.info("Use label smoothing with " + args.lsm_type)
labeldist = label_smoothing_dist(
odim, args.lsm_type, transcript=args.train_json
)
else:
labeldist = None
# multilingual related
self.multilingual = getattr(args, "multilingual", False)
self.replace_sos = getattr(args, "replace_sos", False)
# encoder
self.embed = torch.nn.Embedding(idim, args.eunits, padding_idx=self.pad)
self.dropout = torch.nn.Dropout(p=args.dropout_rate)
self.enc = encoder_for(args, args.eunits, self.subsample)
# attention
self.att = att_for(args)
# decoder
self.dec = decoder_for(args, odim, self.sos, self.eos, self.att, labeldist)
# tie source and target emeddings
if args.tie_src_tgt_embedding:
if idim != odim:
raise ValueError(
"When using tie_src_tgt_embedding, idim and odim must be equal."
)
if args.eunits != args.dunits:
raise ValueError(
"When using tie_src_tgt_embedding, eunits and dunits must be equal."
)
self.embed.weight = self.dec.embed.weight
# tie emeddings and the classfier
if args.tie_classifier:
if args.context_residual:
raise ValueError(
"When using tie_classifier, context_residual must be turned off."
)
self.dec.output.weight = self.dec.embed.weight
# weight initialization
self.init_like_fairseq()
# options for beam search
if args.report_bleu:
trans_args = {
"beam_size": args.beam_size,
"penalty": args.penalty,
"ctc_weight": 0,
"maxlenratio": args.maxlenratio,
"minlenratio": args.minlenratio,
"lm_weight": args.lm_weight,
"rnnlm": args.rnnlm,
"nbest": args.nbest,
"space": args.sym_space,
"blank": args.sym_blank,
"tgt_lang": False,
}
self.trans_args = argparse.Namespace(**trans_args)
self.report_bleu = args.report_bleu
else:
self.report_bleu = False
self.rnnlm = None
self.logzero = -10000000000.0
self.loss = None
self.acc = None
def main(_run):
# ===============
# INTRO
# ===============
args = argparse.Namespace(**_run.config)
torch.manual_seed(args.seed)
np.random.seed(args.seed)
if args.gpu > -1:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
if args.gpu < 0:
device = torch.device("cpu")
else:
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
data_transforms = {
'train':
transforms.Compose([
transforms.Resize((224, 224)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Lambda(lambda x: x.repeat(3, 1, 1)),
]),
'val':
transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Lambda(lambda x: x.repeat(3, 1, 1))
]),
}
train_dataset = ChexpertDataset(os.path.join(CONSTS.DATA_DIR, 'CheXpert',
'train_pleural.csv'),
root_dir=CONSTS.DATA_DIR,
transform=data_transforms['train'])
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True)
val_dataset = ChexpertDataset(os.path.join(CONSTS.DATA_DIR, 'CheXpert',
'test_pleural.csv'),
root_dir=CONSTS.DATA_DIR,
transform=data_transforms['val'])
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False)
model_path = os.path.join(CONSTS.RESULTS_DIR, 'models', args.exp_name)
if not os.path.exists(model_path):
os.makedirs(model_path)
output_dir = os.path.join(CONSTS.RESULTS_DIR, 'outputs', args.exp_name)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
tensorboard_path = os.path.join(CONSTS.RESULTS_DIR, 'logs', 'tensorboard',
args.exp_name)
if not os.path.exists(tensorboard_path):
os.makedirs(tensorboard_path)
model = models.resnet50(pretrained=False)
num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, 2)
model.load_state_dict(
torch.load(os.path.join(model_path, 'best_model.pth')))
model.to(device)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
scheduler = ReduceLROnPlateau(optimizer, 'min', patience=3, verbose=True)
writer = SummaryWriter(tensorboard_path, comment=args.exp_name)
patience = 20
early_stopping = EarlyStopping(patience=patience,
verbose=True,
save=os.path.join(model_path,
'best_model.pth'))
for epoch in range(1, args.epochs + 1):
# train_epoch_loss, train_epoch_accuracy = train_epoch(epoch, model, train_loader, device, optimizer, args)
# print('====> Epoch: {} Average training loss: {:.4f}'.format(
# epoch, train_epoch_loss))
# print('====> Epoch: {} Average training accuracy: {:.4f} %'.format(
# epoch, train_epoch_accuracy*100))
val_epoch_loss, val_epoch_accuracy = val(model, val_loader, device)
print('====> val set loss: {:.4f}'.format(val_epoch_loss))
print('====> val set accuracy: {:.4f} %'.format(val_epoch_accuracy *
100))
# writer.add_scalar('Loss/train', train_epoch_loss, epoch)
writer.add_scalar('Loss/val', val_epoch_loss, epoch)
# writer.add_scalar('Accuracy/train', train_epoch_accuracy, epoch)
writer.add_scalar('Accuracy/val', val_epoch_accuracy, epoch)
# scheduler.step(train_epoch_loss)
early_stopping(val_epoch_loss, model)
writer.close()
def main(args):
"""
Execute a wrapper over log-analyze-parse, aka 'check'.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
def __load_module(name):
"""Loads the given subcommand's definition from the libs."""
try:
module = libhandlers.load_module(name)
except ImportError:
LOG.error("Couldn't import subcommand '" + name + "'.")
raise
return module
def __update_if_key_exists(source, target, key):
"""Append the source Namespace's element with 'key' to target with
the same key, but only if it exists."""
if key in source:
setattr(target, key, getattr(source, key))
# If no output directory is given then the checker results go to a
# temporary directory. This way the subsequent "quick" checks don't pollute
# the result list of a previous check. If the detection status function is
# intended to be used (i.e. by keeping the .plist files) then the output
# directory has to be provided explicitly.
is_temp_output = False
if 'output_dir' in args:
output_dir = args.output_dir
else:
output_dir = tempfile.mkdtemp()
is_temp_output = True
output_dir = os.path.abspath(output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
logfile = None
is_command = False
try:
# --- Step 1.: Perform logging if build command was specified.
if 'command' in args:
logfile = tempfile.NamedTemporaryFile().name
is_command = True
# Translate the argument list between check and log.
log_args = argparse.Namespace(command=args.command,
logfile=logfile)
__update_if_key_exists(args, log_args, 'quiet')
__update_if_key_exists(args, log_args, 'verbose')
log_module = __load_module('log')
LOG.debug("Calling LOG with args:")
LOG.debug(log_args)
log_module.main(log_args)
elif 'logfile' in args:
logfile = args.logfile
# --- Step 2.: Perform the analysis.
if not os.path.exists(logfile):
raise OSError("The specified logfile '" + logfile + "' does not "
"exist.")
analyze_args = argparse.Namespace(logfile=[logfile],
output_path=output_dir,
output_format='plist',
jobs=args.jobs)
# Some arguments don't have default values.
# We can't set these keys to None because it would result in an error
# after the call.
args_to_update = [
'quiet',
'skipfile',
'analyzers',
'add_compiler_defaults',
'clangsa_args_cfg_file',
'tidy_args_cfg_file',
'capture_analysis_output',
'ctu_phases',
'stats_output',
'stats_dir',
'stats_enabled',
'enable_all',
'ordered_checkers', # --enable and --disable.
'timeout'
]
for key in args_to_update:
__update_if_key_exists(args, analyze_args, key)
if 'force' in args or 'clean' in args:
setattr(analyze_args, 'clean', True)
__update_if_key_exists(args, analyze_args, 'verbose')
analyze_module = __load_module('analyze')
LOG.debug("Calling ANALYZE with args:")
LOG.debug(analyze_args)
analyze_module.main(analyze_args)
# --- Step 3.: Print to stdout.
parse_args = argparse.Namespace(input=[output_dir],
input_format='plist')
__update_if_key_exists(args, parse_args, 'print_steps')
__update_if_key_exists(args, parse_args, 'verbose')
__update_if_key_exists(args, parse_args, 'skipfile')
parse_module = __load_module('parse')
LOG.debug("Calling PARSE with args:")
LOG.debug(parse_args)
parse_module.main(parse_args)
except ImportError:
LOG.error("Check failed: couldn't import a library.")
except Exception as ex:
LOG.error("Running check failed. " + ex.message)
import traceback
traceback.print_exc()
finally:
if is_temp_output:
shutil.rmtree(output_dir)
if is_command:
os.remove(logfile)
LOG.debug("Check finished.")
import pytest
import random
args = argparse.Namespace(
elayers=4,
subsample="1_2_2_1_1",
etype="vggblstmp",
eunits=320,
eprojs=320,
dlayers=2,
dunits=300,
atype="location",
aconv_chans=10,
aconv_filts=100,
mtlalpha=0.5,
adim=320,
dropout_rate=0.0,
beam_size=3,
penalty=0.5,
maxlenratio=1.0,
minlenratio=0.0,
ctc_weight=0.2,
verbose=True,
char_list=[u"あ", u"い", u"う", u"え", u"お"],
outdir=None,
seed=1
)
def test_lecun_init_torch():
torch = pytest.importorskip("torch")
def populate_layer_additional_data_dict(self, missing_default_data_group_is_OK=False):
self.progress.new('Layer additional data ops')
self.progress.update('...')
data_groups_common_to_all_profile_dbs = set([])
for p in self.input_profile_db_paths:
if self.input_profile_db_paths.index(p) == 0:
data_groups_common_to_all_profile_dbs = set(TableForLayerAdditionalData(argparse.Namespace(profile_db = p)).get_group_names())
else:
data_groups_common_to_all_profile_dbs.intersection_update(set(TableForLayerAdditionalData(argparse.Namespace(profile_db = p)).get_group_names()))
if 'default' not in data_groups_common_to_all_profile_dbs:
if missing_default_data_group_is_OK:
pass
else:
raise ConfigError("There is something wrong with your input databases. The group name 'default'\
should be common to all of them, but it doesn't seem to be the case :/ How did\
you end up with an anvi'o single profile database that doesn't have the 'default'\
group in its additional layer data table? It is very likely that your profiling\
step failed for some reason for one or more of your databases :(")
taxonomic_data_groups = set(constants.levels_of_taxonomy).intersection(data_groups_common_to_all_profile_dbs)
regular_data_groups = data_groups_common_to_all_profile_dbs.difference(taxonomic_data_groups)
self.progress.end()
self.__populate_layer_additional_data_dict_for_regular_data_groups(regular_data_groups)
self.__populate_layer_additional_data_dict_for_taxonomic_data_groups(taxonomic_data_groups)
def main():
# Load configuration
args, recog_params, dir_name = parse_args_eval(sys.argv[1:])
args = compute_subsampling_factor(args)
# Setting for logging
if os.path.isfile(os.path.join(args.recog_dir, 'plot.log')):
os.remove(os.path.join(args.recog_dir, 'plot.log'))
set_logger(os.path.join(args.recog_dir, 'plot.log'),
stdout=args.recog_stdout)
for i, s in enumerate(args.recog_sets):
# Load dataloader
dataloader = build_dataloader(args=args,
tsv_path=s,
batch_size=1,
is_test=True)
if i == 0:
# Load the ASR model
model = Speech2Text(args, dir_name)
epoch = int(float(args.recog_model[0].split('-')[-1]) * 10) / 10
if args.recog_n_average > 1:
# Model averaging for Transformer
model = average_checkpoints(model,
args.recog_model[0],
n_average=args.recog_n_average)
else:
load_checkpoint(args.recog_model[0], model)
# Ensemble (different models)
ensemble_models = [model]
if len(args.recog_model) > 1:
for recog_model_e in args.recog_model[1:]:
conf_e = load_config(
os.path.join(os.path.dirname(recog_model_e),
'conf.yml'))
args_e = copy.deepcopy(args)
for k, v in conf_e.items():
if 'recog' not in k:
setattr(args_e, k, v)
model_e = Speech2Text(args_e)
load_checkpoint(recog_model_e, model_e)
if args.recog_n_gpus >= 1:
model_e.cuda()
ensemble_models += [model_e]
# Load the LM for shallow fusion
if not args.lm_fusion:
# first path
if args.recog_lm is not None and args.recog_lm_weight > 0:
conf_lm = load_config(
os.path.join(os.path.dirname(args.recog_lm),
'conf.yml'))
args_lm = argparse.Namespace()
for k, v in conf_lm.items():
setattr(args_lm, k, v)
lm = build_lm(args_lm)
load_checkpoint(args.recog_lm, lm)
if args_lm.backward:
model.lm_bwd = lm
else:
model.lm_fwd = lm
# NOTE: only support for first path
if not args.recog_unit:
args.recog_unit = args.unit
logger.info('recog unit: %s' % args.recog_unit)
logger.info('recog oracle: %s' % args.recog_oracle)
logger.info('epoch: %d' % epoch)
logger.info('batch size: %d' % args.recog_batch_size)
logger.info('beam width: %d' % args.recog_beam_width)
logger.info('min length ratio: %.3f' % args.recog_min_len_ratio)
logger.info('max length ratio: %.3f' % args.recog_max_len_ratio)
logger.info('length penalty: %.3f' % args.recog_length_penalty)
logger.info('length norm: %s' % args.recog_length_norm)
logger.info('coverage penalty: %.3f' % args.recog_coverage_penalty)
logger.info('coverage threshold: %.3f' %
args.recog_coverage_threshold)
logger.info('CTC weight: %.3f' % args.recog_ctc_weight)
logger.info('fist LM path: %s' % args.recog_lm)
logger.info('LM weight: %.3f' % args.recog_lm_weight)
logger.info('GNMT: %s' % args.recog_gnmt_decoding)
logger.info('forward-backward attention: %s' %
args.recog_fwd_bwd_attention)
logger.info('resolving UNK: %s' % args.recog_resolving_unk)
logger.info('ensemble: %d' % (len(ensemble_models)))
logger.info('ASR decoder state carry over: %s' %
(args.recog_asr_state_carry_over))
logger.info('LM state carry over: %s' %
(args.recog_lm_state_carry_over))
logger.info('model average (Transformer): %d' %
(args.recog_n_average))
# GPU setting
if args.recog_n_gpus >= 1:
model.cudnn_setting(deterministic=True, benchmark=False)
model.cuda()
save_path = mkdir_join(args.recog_dir, 'att_weights')
# Clean directory
if save_path is not None and os.path.isdir(save_path):
shutil.rmtree(save_path)
os.mkdir(save_path)
while True:
batch, is_new_epoch = dataloader.next(
recog_params['recog_batch_size'])
nbest_hyps_id, aws = model.decode(
batch['xs'],
recog_params,
dataloader.idx2token[0],
exclude_eos=False,
refs_id=batch['ys'],
ensemble_models=ensemble_models[1:]
if len(ensemble_models) > 1 else [],
speakers=batch['sessions']
if dataloader.corpus == 'swbd' else batch['speakers'])
best_hyps_id = [h[0] for h in nbest_hyps_id]
# Get CTC probs
ctc_probs, topk_ids = None, None
if args.ctc_weight > 0:
ctc_probs, topk_ids, xlens = model.get_ctc_probs(
batch['xs'],
task='ys',
temperature=1,
topk=min(100, model.vocab))
# NOTE: ctc_probs: '[B, T, topk]'
ctc_probs_sub1, topk_ids_sub1 = None, None
if args.ctc_weight_sub1 > 0:
ctc_probs_sub1, topk_ids_sub1, xlens_sub1 = model.get_ctc_probs(
batch['xs'],
task='ys_sub1',
temperature=1,
topk=min(100, model.vocab_sub1))
if model.bwd_weight > 0.5:
# Reverse the order
best_hyps_id = [hyp[::-1] for hyp in best_hyps_id]
aws = [[aw[0][:, ::-1]] for aw in aws]
for b in range(len(batch['xs'])):
tokens = dataloader.idx2token[0](best_hyps_id[b],
return_list=True)
spk = batch['speakers'][b]
plot_attention_weights(
aws[b][0][:, :len(tokens)],
tokens,
spectrogram=batch['xs'][b][:, :dataloader.input_dim]
if args.input_type == 'speech' else None,
factor=args.subsample_factor,
ref=batch['text'][b].lower(),
save_path=mkdir_join(save_path, spk,
batch['utt_ids'][b] + '.png'),
figsize=(20, 8),
ctc_probs=ctc_probs[b, :xlens[b]]
if ctc_probs is not None else None,
ctc_topk_ids=topk_ids[b] if topk_ids is not None else None,
ctc_probs_sub1=ctc_probs_sub1[b, :xlens_sub1[b]]
if ctc_probs_sub1 is not None else None,
ctc_topk_ids_sub1=topk_ids_sub1[b]
if topk_ids_sub1 is not None else None)
if model.bwd_weight > 0.5:
hyp = ' '.join(tokens[::-1])
else:
hyp = ' '.join(tokens)
logger.info('utt-id: %s' % batch['utt_ids'][b])
logger.info('Ref: %s' % batch['text'][b].lower())
logger.info('Hyp: %s' % hyp)
logger.info('-' * 50)
if is_new_epoch:
break
def main(opt, callbacks=Callbacks()):
# Checks
set_logging(RANK)
if RANK in [-1, 0]:
print_args(FILE.stem, opt)
check_git_status()
check_requirements(exclude=['thop'])
# Resume
if opt.resume and not check_wandb_resume(
opt) and not opt.evolve: # resume an interrupted run
ckpt = opt.resume if isinstance(
opt.resume,
str) else get_latest_run() # specified or most recent path
assert os.path.isfile(
ckpt), 'ERROR: --resume checkpoint does not exist'
with open(Path(ckpt).parent.parent / 'opt.yaml') as f:
opt = argparse.Namespace(**yaml.safe_load(f)) # replace
opt.cfg, opt.weights, opt.resume = '', ckpt, True # reinstate
LOGGER.info(f'Resuming training from {ckpt}')
else:
opt.data, opt.cfg, opt.hyp, opt.weights, opt.project = \
check_file(opt.data), check_yaml(opt.cfg), check_yaml(opt.hyp), str(opt.weights), str(opt.project) # checks
assert len(opt.cfg) or len(
opt.weights), 'either --cfg or --weights must be specified'
if opt.evolve:
opt.project = str(ROOT / 'runs/evolve')
opt.exist_ok, opt.resume = opt.resume, False # pass resume to exist_ok and disable resume
opt.save_dir = str(
increment_path(Path(opt.project) / opt.name,
exist_ok=opt.exist_ok))
# DDP mode
device = select_device(opt.device, batch_size=opt.batch_size)
if LOCAL_RANK != -1:
assert torch.cuda.device_count(
) > LOCAL_RANK, 'insufficient CUDA devices for DDP command'
assert opt.batch_size % WORLD_SIZE == 0, '--batch-size must be multiple of CUDA device count'
assert not opt.image_weights, '--image-weights argument is not compatible with DDP training'
assert not opt.evolve, '--evolve argument is not compatible with DDP training'
torch.cuda.set_device(LOCAL_RANK)
device = torch.device('cuda', LOCAL_RANK)
dist.init_process_group(
backend="nccl" if dist.is_nccl_available() else "gloo")
# Train
if not opt.evolve:
train(opt.hyp, opt, device, callbacks)
if WORLD_SIZE > 1 and RANK == 0:
LOGGER.info('Destroying process group... ')
dist.destroy_process_group()
# Evolve hyperparameters (optional)
else:
# Hyperparameter evolution metadata (mutation scale 0-1, lower_limit, upper_limit)
meta = {
'lr0':
(1, 1e-5, 1e-1), # initial learning rate (SGD=1E-2, Adam=1E-3)
'lrf':
(1, 0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf)
'momentum': (0.3, 0.6, 0.98), # SGD momentum/Adam beta1
'weight_decay': (1, 0.0, 0.001), # optimizer weight decay
'warmup_epochs': (1, 0.0, 5.0), # warmup epochs (fractions ok)
'warmup_momentum': (1, 0.0, 0.95), # warmup initial momentum
'warmup_bias_lr': (1, 0.0, 0.2), # warmup initial bias lr
'box': (1, 0.02, 0.2), # box loss gain
'cls': (1, 0.2, 4.0), # cls loss gain
'cls_pw': (1, 0.5, 2.0), # cls BCELoss positive_weight
'obj': (1, 0.2, 4.0), # obj loss gain (scale with pixels)
'obj_pw': (1, 0.5, 2.0), # obj BCELoss positive_weight
'iou_t': (0, 0.1, 0.7), # IoU training threshold
'anchor_t': (1, 2.0, 8.0), # anchor-multiple threshold
'anchors': (2, 2.0, 10.0), # anchors per output grid (0 to ignore)
'fl_gamma':
(0, 0.0, 2.0), # focal loss gamma (efficientDet default gamma=1.5)
'hsv_h': (1, 0.0, 0.1), # image HSV-Hue augmentation (fraction)
'hsv_s': (1, 0.0,
0.9), # image HSV-Saturation augmentation (fraction)
'hsv_v': (1, 0.0, 0.9), # image HSV-Value augmentation (fraction)
'degrees': (1, 0.0, 45.0), # image rotation (+/- deg)
'translate': (1, 0.0, 0.9), # image translation (+/- fraction)
'scale': (1, 0.0, 0.9), # image scale (+/- gain)
'shear': (1, 0.0, 10.0), # image shear (+/- deg)
'perspective':
(0, 0.0, 0.001), # image perspective (+/- fraction), range 0-0.001
'flipud': (1, 0.0, 1.0), # image flip up-down (probability)
'fliplr': (0, 0.0, 1.0), # image flip left-right (probability)
'mosaic': (1, 0.0, 1.0), # image mixup (probability)
'mixup': (1, 0.0, 1.0), # image mixup (probability)
'copy_paste': (1, 0.0, 1.0)
} # segment copy-paste (probability)
with open(opt.hyp) as f:
hyp = yaml.safe_load(f) # load hyps dict
if 'anchors' not in hyp: # anchors commented in hyp.yaml
hyp['anchors'] = 3
opt.noval, opt.nosave, save_dir = True, True, Path(
opt.save_dir) # only val/save final epoch
# ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices
evolve_yaml, evolve_csv = save_dir / 'hyp_evolve.yaml', save_dir / 'evolve.csv'
if opt.bucket:
os.system(f'gsutil cp gs://{opt.bucket}/evolve.csv {save_dir}'
) # download evolve.csv if exists
for _ in range(opt.evolve): # generations to evolve
if evolve_csv.exists(
): # if evolve.csv exists: select best hyps and mutate
# Select parent(s)
parent = 'single' # parent selection method: 'single' or 'weighted'
x = np.loadtxt(evolve_csv, ndmin=2, delimiter=',', skiprows=1)
n = min(5, len(x)) # number of previous results to consider
x = x[np.argsort(-fitness(x))][:n] # top n mutations
w = fitness(x) - fitness(x).min() + 1E-6 # weights (sum > 0)
if parent == 'single' or len(x) == 1:
# x = x[random.randint(0, n - 1)] # random selection
x = x[random.choices(range(n),
weights=w)[0]] # weighted selection
elif parent == 'weighted':
x = (x * w.reshape(
n, 1)).sum(0) / w.sum() # weighted combination
# Mutate
mp, s = 0.8, 0.2 # mutation probability, sigma
npr = np.random
npr.seed(int(time.time()))
g = np.array([meta[k][0] for k in hyp.keys()]) # gains 0-1
ng = len(meta)
v = np.ones(ng)
while all(
v == 1
): # mutate until a change occurs (prevent duplicates)
v = (g * (npr.random(ng) < mp) * npr.randn(ng) *
npr.random() * s + 1).clip(0.3, 3.0)
for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300)
hyp[k] = float(x[i + 7] * v[i]) # mutate
# Constrain to limits
for k, v in meta.items():
hyp[k] = max(hyp[k], v[1]) # lower limit
hyp[k] = min(hyp[k], v[2]) # upper limit
hyp[k] = round(hyp[k], 5) # significant digits
# Train mutation
results = train(hyp.copy(), opt, device, callbacks)
# Write mutation results
print_mutation(results, hyp.copy(), save_dir, opt.bucket)
# Plot results
plot_evolve(evolve_csv)
print(
f'Hyperparameter evolution finished\n'
f"Results saved to {colorstr('bold', save_dir)}\n"
f'Use best hyperparameters example: $ python train.py --hyp {evolve_yaml}'
)
matplotlib.use('Agg')
np.random.seed(1003)
# set args
args = {
'ckptpath': './model.ckpt',
'dataset_dir': sys.argv[1],
'output_dir': sys.argv[2],
'noodles_pasta': [2431, 2352, 2379, 2466, 2383],
'rice': [2533, 2504, 2520, 2495, 2553],
'soup': [3292, 3159, 3094, 3161, 3278],
'vegetable_fruit': [523, 550, 507, 532, 561],
'num_features': 11,
'device': 'cuda'
}
args = argparse.Namespace(**args)
# build model
model = Model(
make_layers([
32, 32, 32, 'M',
64, 64, 64, 'M',
128, 128, 128, 'M',
256, 256, 256, 256, 'M',
512, 512, 512, 512, 'M'
])
).to(args.device)
# load checkpoint
checkpoint = torch.load(args.ckptpath)
model.load_state_dict(checkpoint['model_state_dict'])
