def preloader():
current_path = os.path.dirname(os.path.abspath(__file__))
config_path = f'{current_path}/config/hrnet_plate.yaml'
seg_weights = f'{current_path}/weights/hrnetv2_hrnet_plate_199.pth'
output_dir = f'{current_path}/'
data_name = 'plate'
cfg = get_cfg_defaults()
cfg.defrost()
cfg.merge_from_file(config_path)
cfg.merge_from_list([
"train.config_path",
config_path,
"train.output_dir",
output_dir,
"dataset.data_name",
data_name,
])
print(
torch.load(os.path.join(f"{current_path}/data", data_name + ".pth"),
map_location=torch.device('cpu')))
model = create_model(cfg)
# if torch.cuda.is_available():
# model.cuda()
model = nn.DataParallel(model)
print(torch.load(seg_weights, map_location=torch.device('cpu')).keys())
model.load_state_dict(
torch.load(seg_weights,
map_location=torch.device('cpu'))["state_dict"])
model.eval()
return model, cfg,
def __init__(self):
self.cfg = get_cfg_defaults()
self.model = build_model(self.cfg)
self.device = torch.device(self.cfg.MODEL.DEVICE)
self.model.to(self.device)
self.model.train(False)
self.model.load_state_dict(torch.load(self.cfg.MODEL.WEIGHTS))
def __init__(self, width):
super(HRNet, self).__init__()
cfg = get_cfg_defaults()
cfg.merge_from_file("./experiments/hrnet.yaml")
self.hr0 = HighResolutionNet(cfg)
self.hr1 = HighResolutionNet(cfg)
self.head = Regressor(in_channel=540, width=int(width / 4), n_class=5)
def __init__(self, model_path):
self.cfg = get_cfg_defaults()
self.model = build_model(self.cfg)
self.device = torch.device(self.cfg.MODEL.DEVICE)
self.model.to(self.device)
self.model.train(False)
self.state_dict = torch.load(model_path)
def load_model(model_path, cfg_path):
import h5py
import json
cfg = get_cfg_defaults()
cfg.merge_from_file(cfg_path)
cfg.freeze()
logging.info(cfg)
with h5py.File(model_path, "r") as f:
mappings = json.loads(f.attrs["mappings"])
label_key = f.attrs["label_key"]
vocab_size = f.attrs["vocab_size"]
n_class_labels = f.attrs["n_class_labels"]
word_length = f.attrs["word_length"]
if cfg.MODEL.CLASSIFIER == ["crf"]:
from .keraslayers.ChainCRF import create_custom_objects
custom_objects = create_custom_objects()
model = keras.models.load_model(model_path, custom_objects=custom_objects)
bilstm = BiLSTM(cfg)
bilstm.set_vocab(vocab_size, n_class_labels, word_length, mappings)
bilstm.model = model
bilstm.label_key = label_key
return bilstm
def main():
args = arg_parse()
# ---- set configs, logger and device ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
tb_logger = TensorBoardLogger("tb_logs", name=cfg.DATASET.NAME)
device = "cuda" if torch.cuda.is_available() else "cpu"
# ---- set dataset ----
train_dataset = BindingDBDataset(name=cfg.DATASET.NAME, split="train", path=cfg.DATASET.PATH)
val_dataset = BindingDBDataset(name=cfg.DATASET.NAME, split="valid", path=cfg.DATASET.PATH)
test_dataset = BindingDBDataset(name=cfg.DATASET.NAME, split="test", path=cfg.DATASET.PATH)
train_loader = DataLoader(dataset=train_dataset, shuffle=True, batch_size=cfg.SOLVER.TRAIN_BATCH_SIZE)
val_loader = DataLoader(dataset=val_dataset, shuffle=True, batch_size=cfg.SOLVER.TEST_BATCH_SIZE)
test_loader = DataLoader(dataset=test_dataset, shuffle=True, batch_size=cfg.SOLVER.TEST_BATCH_SIZE)
# ---- set model ----
model = get_model(cfg)
# ---- training and evaluation ----
gpus = 1 if device == "cuda" else 0
checkpoint_callback = ModelCheckpoint(monitor='val_loss', mode="min")
trainer = pl.Trainer(max_epochs=cfg.SOLVER.MAX_EPOCHS, gpus=gpus, logger=tb_logger, callbacks=[checkpoint_callback])
trainer.fit(model, train_dataloader=train_loader, val_dataloaders=val_loader)
trainer.test(test_dataloaders=test_loader)
def main():
args = arg_parse()
# ---- setup device ----
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('==> Using device ' + device)
# ---- setup config ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
seed.set_seed(cfg.SOLVER.SEED)
# ---- setup logger ----
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
logger = logging.construct_logger('context_cnns', cfg.OUTPUT_DIR)
logger.info(f'Using {device}')
logger.info('\n' + cfg.dump())
# ---- setup dataset ----
train_loader, val_loader = get_cifar(cfg)
# ---- setup model ----
print('==> Building model..')
net = get_model(cfg)
net = net.to(device)
model_stats = summary(net, (3, 32, 32), depth=10)
logger.info('\n'+str(model_stats))
if device=='cuda':
net = torch.nn.DataParallel(net)
# ---- setup trainers ----
optim = torch.optim.SGD(net.parameters(), lr=cfg.SOLVER.BASE_LR,
momentum=cfg.SOLVER.MOMENTUM,
weight_decay=cfg.SOLVER.WEIGHT_DECAY)
trainer = Trainer(
device,
train_loader,
val_loader,
net,
optim,
logger,
cfg
)
if args.resume:
# Load checkpoint
print('==> Resuming from checkpoint..')
cp = torch.load(args.resume)
trainer.model.load_state_dict(cp['net'])
trainer.optim.load_state_dict(cp['optim'])
trainer.epochs = cp['epoch']
trainer.train_acc = cp['train_accuracy']
trainer.val_acc = cp['test_accuracy']
trainer.train()
def main():
"""The main for this domain adapation example, showing the workflow"""
args = arg_parse()
# ---- setup device ----
device = "cuda" if torch.cuda.is_available() else "cpu"
print("==> Using device " + device)
# ---- setup configs ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
set_seed(cfg.SOLVER.SEED)
# ---- setup logger and output ----
output_dir = os.path.join(cfg.OUTPUT_DIR, cfg.DATASET.NAME, args.output)
os.makedirs(output_dir, exist_ok=True)
logger = construct_logger("isonet", output_dir)
logger.info("Using " + device)
logger.info("\n" + cfg.dump())
# ---- setup dataset ----
train_loader, val_loader = get_cifar(cfg)
print("==> Building model..")
net = get_model(cfg)
# print(net)
net = net.to(device)
# model_stats = summary(net, (3, 32, 32))
# logger.info('\n'+str(model_stats))
# Needed even for single GPU https://discuss.pytorch.org/t/attributeerror-net-object-has-no-attribute-module/45652
if device == "cuda":
net = torch.nn.DataParallel(net)
optim = torch.optim.SGD(
net.parameters(),
lr=cfg.SOLVER.BASE_LR,
momentum=cfg.SOLVER.MOMENTUM,
weight_decay=cfg.SOLVER.WEIGHT_DECAY,
dampening=cfg.SOLVER.DAMPENING,
nesterov=cfg.SOLVER.NESTEROV,
)
trainer = Trainer(device, train_loader, val_loader, net, optim, logger,
output_dir, cfg)
if args.resume:
# Load checkpoint
print("==> Resuming from checkpoint..")
cp = torch.load(args.resume)
trainer.model.load_state_dict(cp["net"])
trainer.optim.load_state_dict(cp["optim"])
trainer.epochs = cp["epoch"]
trainer.train_acc = cp["train_accuracy"]
trainer.val_acc = cp["test_accuracy"]
trainer.train()
def main():
# For now hard coding the values for all methods below, change to read it using config file
print("Start..")
args = arg_parse()
# ---- set configs, logger and device ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
device = "cuda" if torch.cuda.is_available() else "cpu"
# Fetch the bindingDB dataset based on the name defined in config file
bindingdb_dataset = DTI(name=cfg.DATASET.NAME)
# Apply cluster based on the 3 types of cluster
# can define cluster_type -> k_means, agglomerative, dbscan
# bindingdb_dataset = apply_clustering(bindingdb_dataset, num_of_clusters=cfg.SOLVER.NUM_OF_CLUSTERS,
# cluster_type=cfg.MODEL.CLUSTER_TYPE)
bindingdb_dataset_cluster = apply_clustering(
bindingdb_dataset,
num_of_clusters=cfg.SOLVER.NUM_OF_CLUSTERS,
cluster_type=cfg.MODEL.CLUSTER_TYPE)
# Split the data based on the clusters formed by specifying the split in fraction
train_dataset, val_dataset, test_dataset = get_split_by_clusters(
bindingdb_dataset_cluster, num_of_clusters=cfg.SOLVER.NUM_OF_CLUSTERS)
train_dataset = DTADataset(ds=train_dataset)
val_dataset = DTADataset(ds=val_dataset)
test_dataset = DTADataset(ds=test_dataset)
train_loader = DataLoader(dataset=train_dataset,
shuffle=True,
batch_size=cfg.SOLVER.TRAIN_BATCH_SIZE)
val_loader = DataLoader(dataset=val_dataset,
shuffle=True,
batch_size=cfg.SOLVER.TEST_BATCH_SIZE)
test_loader = DataLoader(dataset=test_dataset,
shuffle=True,
batch_size=cfg.SOLVER.TEST_BATCH_SIZE)
# ---- set model ----
model = get_model(cfg)
# ---- training and evaluation ----
gpus = 1 if device == "cuda" else 0
checkpoint_callback = ModelCheckpoint(monitor='val_loss', mode="min")
trainer = pl.Trainer(max_epochs=cfg.SOLVER.MAX_EPOCHS,
gpus=gpus,
callbacks=[checkpoint_callback])
trainer.fit(model,
train_dataloader=train_loader,
val_dataloaders=val_loader)
trainer.test(test_dataloaders=test_loader)
print("Done!!!")
def main():
"""The main for this multi-source domain adaptation example, showing the workflow"""
args = arg_parse()
# ---- setup configs ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
print(cfg)
# ---- setup output ----
format_str = "@%(asctime)s %(name)s [%(levelname)s] - (%(message)s)"
logging.basicConfig(format=format_str)
# ---- setup dataset ----
if type(cfg.DATASET.SOURCE) == list:
sub_domain_set = cfg.DATASET.SOURCE + [cfg.DATASET.TARGET]
else:
sub_domain_set = None
num_channels = cfg.DATASET.NUM_CHANNELS
if cfg.DATASET.NAME.upper() == "DIGITS":
kwargs = {"return_domain_label": True}
else:
kwargs = {"download": True, "return_domain_label": True}
data_access = ImageAccess.get_multi_domain_images(
cfg.DATASET.NAME.upper(), cfg.DATASET.ROOT, sub_domain_set=sub_domain_set, **kwargs
)
# Repeat multiple times to get std
for i in range(0, cfg.DATASET.NUM_REPEAT):
seed = cfg.SOLVER.SEED + i * 10
dataset = MultiDomainAdapDataset(data_access, random_state=seed)
set_seed(seed) # seed_everything in pytorch_lightning did not set torch.backends.cudnn
print(f"==> Building model for seed {seed} ......")
# ---- setup model and logger ----
model, train_params = get_model(cfg, dataset, num_channels)
tb_logger = TensorBoardLogger(cfg.OUTPUT.TB_DIR, name="seed{}".format(seed))
checkpoint_callback = ModelCheckpoint(
filename="{epoch}-{step}-{valid_loss:.4f}", monitor="valid_loss", mode="min",
)
progress_bar = TQDMProgressBar(cfg.OUTPUT.PB_FRESH)
trainer = pl.Trainer(
min_epochs=cfg.SOLVER.MIN_EPOCHS,
max_epochs=cfg.SOLVER.MAX_EPOCHS,
callbacks=[checkpoint_callback, progress_bar],
gpus=args.gpus,
auto_select_gpus=True,
logger=tb_logger, # logger,
# weights_summary='full',
fast_dev_run=False, # True,
)
trainer.fit(model)
trainer.test()
def __init__(self, width):
super(HRNet, self).__init__()
cfg = get_cfg_defaults()
cfg.merge_from_file("./experiments/exp.yaml")
self.hr = HighResolutionNet(cfg)
self.add_coords = AddCoords(rank=2)
self.conv5 = nn.Conv2d(7, 7, 3, padding=1)
self.conv6 = nn.Conv2d(7, 7, 3, padding=1)
self.conv7 = nn.Conv2d(7, 4, 1)
self.pool = nn.MaxPool2d(width, stride=width)
def main():
N = 256
H = 32
W = 32
x = np.random.randn(N, H, W, 3)
y = np.random.randn(N, H, W, 3)
cfg = get_cfg_defaults()
dnet = GenericKernel(kernel_cfg=cfg.KERNEL)
start = timer()
K = generate_kernel(dnet, x, y)
end = timer()
print(f"{N} x {N} kernel took {end - start} seconds")
def main(config_file: str):
cfg = get_cfg_defaults()
cfg.merge_from_file(config_file)
cfg.TRAIN.N_PARALLEL = 1
cfg.freeze()
metrics = eval_model(cfg)
out_file = os.path.join(cfg.TEST.OUTDIR, "performance_score.json")
json.dump(metrics, open(out_file, "w"))
for k, v in metrics.items():
print(f"{k}: {v:.2}")
def __init__(self):
self.this_dir = osp.abspath(osp.dirname(__file__))
yaml_path = osp.join(self.this_dir, 'face_lib.yaml')
self.cfg = get_cfg_defaults()
self.cfg.merge_from_file(yaml_path)
self.cfg.freeze()
if self.cfg.PET_ENGINE.USE_ENGINE:
self.init_pet_engine()
self.cos_dis = torch.nn.CosineSimilarity()
self.face_db_dict = {}
def main():
"""The main for this domain adaptation example, showing the workflow"""
args = arg_parse()
# ---- setup configs ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
print(cfg)
# ---- setup output ----
format_str = "@%(asctime)s %(name)s [%(levelname)s] - (%(message)s)"
logging.basicConfig(format=format_str)
# ---- setup dataset ----
source, target, num_channels = DigitDataset.get_source_target(
DigitDataset(cfg.DATASET.SOURCE.upper()),
DigitDataset(cfg.DATASET.TARGET.upper()), cfg.DATASET.ROOT)
dataset = MultiDomainDatasets(
source,
target,
config_weight_type=cfg.DATASET.WEIGHT_TYPE,
config_size_type=cfg.DATASET.SIZE_TYPE,
valid_split_ratio=cfg.DATASET.VALID_SPLIT_RATIO,
)
# Repeat multiple times to get std
for i in range(0, cfg.DATASET.NUM_REPEAT):
seed = cfg.SOLVER.SEED + i * 10
# seed_everything in pytorch_lightning did not set torch.backends.cudnn
set_seed(seed)
print(f"==> Building model for seed {seed} ......")
# ---- setup model and logger ----
model, train_params = get_model(cfg, dataset, num_channels)
tb_logger = pl_loggers.TensorBoardLogger(cfg.OUTPUT.TB_DIR,
name="seed{}".format(seed))
checkpoint_callback = ModelCheckpoint(
filename="{epoch}-{step}-{valid_loss:.4f}",
monitor="valid_loss",
mode="min",
)
progress_bar = TQDMProgressBar(cfg.OUTPUT.PB_FRESH)
trainer = pl.Trainer(
min_epochs=cfg.SOLVER.MIN_EPOCHS,
max_epochs=cfg.SOLVER.MAX_EPOCHS,
callbacks=[checkpoint_callback, progress_bar],
logger=tb_logger,
gpus=args.gpus,
)
trainer.fit(model)
trainer.test()
def __init__(self, model_path):
config = self._get_tf_config()
args = get_cfg_defaults()
self.tf_graph = tf.Graph()
with self.tf_graph.as_default():
self.policy = CMPPolicy(args, is_training=False, batch_norm_is_training=False,
only_single_step_graph=True)
self.sess = tf.Session(config=config)
self.sess.run(self.policy.init_op)
self.policy.saver_op.restore(self.sess, model_path)
def main():
args = arg_parse()
# ---- setup device ----
device = "cuda" if torch.cuda.is_available() else "cpu"
print("==> Using device " + device)
# ---- setup config ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
seed.set_seed(cfg.SOLVER.SEED)
# ---- setup logger ----
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
logger = logging.construct_logger("context_cnns", cfg.OUTPUT_DIR)
logger.info(f"Using {device}")
logger.info("\n" + cfg.dump())
# ---- setup dataset ----
train_loader, valid_loader = get_cifar(cfg)
# ---- setup model ----
print("==> Building model..")
net = get_model(cfg)
net = net.to(device)
model_stats = summary(net, (3, 32, 32))
logger.info("\n" + str(model_stats))
if device == "cuda":
net = torch.nn.DataParallel(net)
# ---- setup trainers ----
optim = torch.optim.SGD(
net.parameters(), lr=cfg.SOLVER.BASE_LR, momentum=cfg.SOLVER.MOMENTUM, weight_decay=cfg.SOLVER.WEIGHT_DECAY
)
trainer = Trainer(device, train_loader, valid_loader, net, optim, logger, cfg)
if args.resume:
# Load checkpoint
print("==> Resuming from checkpoint..")
cp = torch.load(args.resume)
trainer.model.load_state_dict(cp["net"])
trainer.optim.load_state_dict(cp["optim"])
trainer.epochs = cp["epoch"]
trainer.train_acc = cp["train_accuracy"]
trainer.valid_acc = cp["test_accuracy"]
trainer.train()
def run():
cfg = get_cfg_defaults()
cfg.merge_from_file("config/base.yaml")
cfg.freeze()
log = ezlog.EZLOG(os.path.basename(__file__))
log.logger.info("=================restarting=========================")
word_counter = Counter()
############################################
#SCAN TXT
with open(cfg.SEGMENTED_TXT, 'rb') as f:
#f = codecs.open(cfg['subword']['output'], 'r', encoding='utf8')
for line in tqdm(f):
try:
line = line.decode('utf-8').strip()
except Exception as e:
print(line)
print(e)
continue
article_one = line.strip().split(' ')
if article_one == []:
continue
conter_one = Counter(article_one)
word_counter += conter_one
log.logger.info("process lines done")
#############################################
#GET TOP-K WORDS
N = len(word_counter.keys())
words_counter = word_counter.most_common(N)
words = []
for word, freq in words_counter:
if freq < cfg.MIN_WORD_FREQ:
break
words.append(word)
##############################################
##WORD2INT AND INT2WORD
word2int, int2word = {}, {}
words = list(set(words))
for index, word in enumerate(words):
word2int[word] = index
int2word[index] = word
with open(cfg.WORD_DICT, 'wb') as f:
pickle.dump((word2int, int2word, words_counter), f)
log.logger.info("create file :{}".format(cfg.WORD_DICT))
def main(config_file: str):
cfg = get_cfg_defaults()
cfg.merge_from_file(config_file)
cfg.freeze()
if not os.path.exists(cfg.LOG.OUTDIR):
os.makedirs(cfg.LOG.OUTDIR)
filename = os.path.join(cfg.LOG.OUTDIR, "config.yaml")
with open(filename, "w") as f:
f.write(cfg.dump())
if cfg.LOG.NEPTUNE:
setup_neptune(cfg)
train(cfg)
def main():
args = arg_parse()
# ---- setup device ----
device = "cuda" if torch.cuda.is_available() else "cpu"
print("==> Using device " + device)
# ---- setup configs ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
seed.set_seed(cfg.SOLVER.SEED)
# ---- setup logger and output ----
output_dir = os.path.join(cfg.OUTPUT_DIR, cfg.DATASET.NAME, args.output)
os.makedirs(output_dir, exist_ok=True)
logger = lu.construct_logger("gripnet", output_dir)
logger.info("Using " + device)
logger.info(cfg.dump())
# ---- setup dataset ----
download_file_by_url(cfg.DATASET.URL, cfg.DATASET.ROOT, "pose.pt", "pt")
data = torch.load(os.path.join(cfg.DATASET.ROOT, "pose.pt"))
device = torch.device(device)
data = data.to(device)
# ---- setup model ----
print("==> Building model..")
model = GripNet(
cfg.GRIPN.GG_LAYERS, cfg.GRIPN.GD_LAYERS, cfg.GRIPN.DD_LAYERS, data.n_d_node, data.n_g_node, data.n_dd_edge_type
).to(device)
# TODO Visualize model
# ---- setup trainers ----
optimizer = torch.optim.Adam(model.parameters(), lr=cfg.SOLVER.BASE_LR)
# TODO
trainer = Trainer(cfg, device, data, model, optimizer, logger, output_dir)
if args.resume:
# Load checkpoint
print("==> Resuming from checkpoint..")
cp = torch.load(args.resume)
trainer.model.load_state_dict(cp["net"])
trainer.optim.load_state_dict(cp["optim"])
trainer.epochs = cp["epoch"]
trainer.train_auprc = cp["train_auprc"]
trainer.valid_auprc = cp["valid_auprc"]
trainer.train_auroc = cp["train_auroc"]
trainer.valid_auroc = cp["valid_auroc"]
trainer.train_ap = cp["train_ap"]
trainer.valid_ap = cp["valid_ap"]
trainer.train()
def run():
custom_punc = '------------ ，。、【 】 “”：；（）《》‘’{}？！⑦()、%^>℃：.”“^-——=&#@￥… …\n\r '
all_punc = set(punctuation + custom_punc)
cfg = get_cfg_defaults()
cfg.merge_from_file("config/base.yaml")
cfg.freeze()
#working_folder = os.path.dirname(__file__)
#raw_output = os.path.join(working_folder, cfg.SEGMENTED_TXT)
raw_output = cfg.SEGMENTED_TXT
if not os.path.exists(os.path.dirname(raw_output)):
os.makedirs(os.path.dirname(raw_output))
program = os.path.basename(__file__) # 得到文件名
logger = logging.getLogger(program)
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
logging.root.setLevel(level=logging.INFO)
#################################################
#LOADING STOPWORDS
stopwords = []
f = codecs.open(cfg.STOPWORDS, 'r', encoding='utf8')
for line in f:
stopwords.append(line.strip())
stopwords = set(stopwords)
all_punc = all_punc | stopwords
bar = tqdm(os.listdir(os.path.join(cfg.RAW_TEXT_DIR)))
lines = []
for txt in bar:
bar.set_description(txt)
raw_input = os.path.join(cfg.RAW_TEXT_DIR, txt)
with open(raw_input, 'r', encoding='utf-8') as f:
for line in f:
words = jieba.lcut(line) #分词
words = filter(lambda x: x not in all_punc, words) #停用词和标点符号
words = filter(lambda x: len(x) >= cfg.MIN_WORD_LEN,
words) #单词最小长度
words = list(words)
if words == []:
continue
lines.append(' '.join(words))
with open(raw_output, 'w', encoding='utf-8') as f:
f.write('\n'.join(lines))
logger.info("Finished Saved articles.")
def main():
parser = argparse.ArgumentParser(description="PyTorch Self-driving Car Training and Inference.")
parser.add_argument(
"--config-file",
default="",
metavar="file",
help="path to config file",
type=str,
)
parser.add_argument(
"--mode",
default="test",
metavar="mode",
help="'train' or 'test'",
type=str,
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
# build the config
cfg = get_cfg_defaults()
# cfg.merge_from_file(args.config_file)
# cfg.merge_from_list(args.opts)
cfg.freeze()
# setup the logger
if not os.path.isdir(cfg.OUTPUT.DIR):
os.mkdir(cfg.OUTPUT.DIR)
logger = setup_logger("balad-mobile.train", cfg.OUTPUT.DIR,
'{0:%Y-%m-%d %H:%M:%S}_log'.format(datetime.now()))
logger.info(args)
logger.info("Running with config:\n{}".format(cfg))
# TRAIN
train(cfg)
# Visualize
visualization(cfg)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('expert_policy_file', type=str)
parser.add_argument('envname', type=str)
parser.add_argument('data_file', type=str)
parser.add_argument('--render', action='store_true')
parser.add_argument('--max_timesteps', type=int)
parser.add_argument('--num_rollouts', type=int, default=20,
help='Number of expert roll outs')
parser.add_argument('--test', action='store_true')
args = parser.parse_args()
# build the config
cfg = get_cfg_defaults()
cfg.freeze()
if args.test:
test(args, cfg)
else:
train(args, cfg)
def parse_args():
parser = argparse.ArgumentParser(description='Train network')
parser.add_argument('--cfg',
help='experiment configure file name',
required=True,
type=str)
parser.add_argument("--local_rank", type=int, default=-1)
parser.add_argument('opts',
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER)
args = parser.parse_args()
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.merge_from_list(args.opts)
cfg.freeze()
return args, cfg
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('envname', type=str)
parser.add_argument('--dagger', action='store_true')
parser.add_argument('--num_rollouts',
type=int,
default=20,
help='Number of expert roll outs in dagger mode')
args = parser.parse_args()
# build the config
cfg = get_cfg_defaults()
cfg.freeze()
data = load_data(cfg, args.envname)
X_train, X_test, y_train, y_test = split_data(cfg, data)
input_size = np.prod(X_train[0].shape)
output_size = np.prod(y_train[0].shape)
model = build_model(cfg, input_size, output_size)
if args.dagger:
model_file_path = get_model_file_path(cfg,
cfg.MODEL.NN.DAGGER_FILE_NAME,
args.envname)
rewards_mean, rewards_std = train_dagger(cfg, model, model_file_path,
args.envname, X_train,
y_train, args.num_rollouts)
print('mean of rewards per iterations is', rewards_mean)
print('std of rewards per iterations is', rewards_std)
else:
model_file_path = get_model_file_path(cfg, cfg.MODEL.NN.BC_FILE_NAME,
args.envname)
train(cfg, model, model_file_path, X_train, y_train)
test(model_file_path, X_test, y_test)
def main():
"""The main for this domain adaptation example, showing the workflow"""
args = arg_parse()
# ---- setup configs ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
print(cfg)
# ---- setup output ----
format_str = "@%(asctime)s %(name)s [%(levelname)s] - (%(message)s)"
logging.basicConfig(format=format_str)
# ---- setup dataset ----
seed = cfg.SOLVER.SEED
source, target, num_classes = VideoDataset.get_source_target(
VideoDataset(cfg.DATASET.SOURCE.upper()),
VideoDataset(cfg.DATASET.TARGET.upper()), seed, cfg)
dataset = VideoMultiDomainDatasets(
source,
target,
image_modality=cfg.DATASET.IMAGE_MODALITY,
seed=seed,
config_weight_type=cfg.DATASET.WEIGHT_TYPE,
config_size_type=cfg.DATASET.SIZE_TYPE,
)
# ---- setup model and logger ----
model, train_params = get_model(cfg, dataset, num_classes)
trainer = pl.Trainer(
logger=False,
resume_from_checkpoint=args.ckpt,
gpus=args.gpus,
)
model_test = weights_update(model=model, checkpoint=torch.load(args.ckpt))
# test scores
trainer.test(model=model_test)
def main():
parser = argparse.ArgumentParser("Train + evaluate kernel model")
parser.add_argument("--config", default=None)
parser.add_argument('--config-updates', default=[], nargs='*')
args = parser.parse_args()
cfg = get_cfg_defaults()
if args.config is not None:
cfg.merge_from_file(args.config)
cfg.merge_from_list(args.config_updates)
if cfg.SYSTEM.NUM_GPUS == 0:
cfg.SYSTEM.NUM_GPUS = torch.cuda.device_count()
cfg.freeze()
print(cfg)
# load dataset
random.seed(cfg.DATASET.RAND_SEED)
X_train, y_train, X_test, y_test = load_dataset(cfg)
print(X_train.shape)
print(X_test.shape)
K_train, K_test = generate_kernels(cfg, X_train, X_test)
best_model_result, best_train_result, best_test_result = solve_kernel(cfg, K_train, y_train, K_test, y_test)
def main():
parser = argparse.ArgumentParser(
description="Deep Neural Networks for 3D Anaglyph Image Generation")
parser.add_argument(
"--config-file",
default="",
metavar="file",
help="path to config file",
type=str,
)
parser.add_argument(
"--mode",
default="test",
metavar="mode",
help="'train' or 'test'",
type=str,
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
# build the config
cfg = get_cfg_defaults()
cfg.merge_from_file(args.config_file)
cfg.merge_from_list(args.opts)
cfg.freeze()
dataset = build_dataset(cfg)
model, optimizer = build_model(cfg)
train(cfg, optimizer, dataset)
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--cfg', required=True)
parser.add_argument(
'--load',
help='resume from a checkpoint with optimizer parameter attached')
parser.add_argument('--n_threads', default=4)
parser.add_argument('--save', required=True)
parser.add_argument('--max_memory', type=int, default=None)
parser.add_argument('--mem_every', type=int, default=5)
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('opts',
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER)
args = parser.parse_args()
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.merge_from_list(args.opts)
cfg.freeze()
return args, cfg
def main():
"""The main for this domain adaptation example, showing the workflow"""
args = arg_parse()
# ---- setup configs ----
cfg = get_cfg_defaults()
cfg.merge_from_file(args.cfg)
cfg.freeze()
print(cfg)
# ---- setup output ----
os.makedirs(cfg.OUTPUT.DIR, exist_ok=True)
format_str = "@%(asctime)s %(name)s [%(levelname)s] - (%(message)s)"
logging.basicConfig(format=format_str)
# ---- setup dataset ----
source, target, num_channels = DigitDataset.get_source_target(
DigitDataset(cfg.DATASET.SOURCE.upper()),
DigitDataset(cfg.DATASET.TARGET.upper()), cfg.DATASET.ROOT)
dataset = MultiDomainDatasets(source,
target,
config_weight_type=cfg.DATASET.WEIGHT_TYPE,
config_size_type=cfg.DATASET.SIZE_TYPE)
# Repeat multiple times to get std
for i in range(0, cfg.DATASET.NUM_REPEAT):
seed = cfg.SOLVER.SEED + i * 10
# seed_everything in pytorch_lightning did not set torch.backends.cudnn
set_seed(seed)
print(f"==> Building model for seed {seed} ......")
# ---- setup model and logger ----
model, train_params = get_model(cfg, dataset, num_channels)
logger, results, checkpoint_callback, test_csv_file = setup_logger(
train_params, cfg.OUTPUT.DIR, cfg.DAN.METHOD, seed)
trainer = pl.Trainer(
progress_bar_refresh_rate=cfg.OUTPUT.PB_FRESH, # in steps
min_epochs=cfg.SOLVER.MIN_EPOCHS,
max_epochs=cfg.SOLVER.MAX_EPOCHS,
checkpoint_callback=checkpoint_callback,
# resume_from_checkpoint=last_checkpoint_file,
gpus=args.gpus,
logger=False, # logger,
# weights_summary='full',
fast_dev_run=True, # True,
)
trainer.fit(model)
results.update(
is_validation=True,
method_name=cfg.DAN.METHOD,
seed=seed,
metric_values=trainer.callback_metrics,
)
# test scores
trainer.test()
results.update(
is_validation=False,
method_name=cfg.DAN.METHOD,
seed=seed,
metric_values=trainer.callback_metrics,
)
results.to_csv(test_csv_file)
results.print_scores(cfg.DAN.METHOD)
