def main():
"""
The main method, which checks environment vars and collects all of the required input options.
Returns 0 on success
"""
# Read in the command line options
args, work_dir, docleanup, cspp_afire_version, logfile = argument_parser()
# Check various paths and environment variables that are "must haves".
try:
_, afire_home = check_and_convert_env_var('CSPP_ACTIVE_FIRE_HOME')
_, afire_ancil_path = check_and_convert_env_var(
'CSPP_ACTIVE_FIRE_STATIC_DIR')
_ = check_and_convert_path(None,
os.path.join(afire_home,
'static_ancillary'),
check_write=False)
_ = check_and_convert_path(None, work_dir, check_write=False)
except CsppEnvironment as e:
LOG.error(e.value)
LOG.error(
'Installation error, Make sure all software components were installed.'
)
return 2
afire_options = {}
afire_options['inputs'] = args.inputs
afire_options['afire_home'] = os.path.abspath(afire_home)
afire_options['i_band'] = args.i_band
afire_options['work_dir'] = os.path.abspath(args.work_dir)
afire_options['ancil_dir'] = afire_ancil_path
afire_options['cache_dir'] = setup_cache_dir(args.cache_dir,
afire_options['work_dir'],
'CSPP_ACTIVE_FIRE_CACHE_DIR')
afire_options['ancillary_only'] = args.ancillary_only
afire_options['cache_window'] = args.cache_window
afire_options['preserve_cache'] = args.preserve_cache
afire_options['num_cpu'] = args.num_cpu
afire_options['docleanup'] = docleanup
afire_options['version'] = cspp_afire_version
rc = 0
try:
attempted_runs, successful_runs, crashed_runs, problem_runs = process_afire_inputs(
work_dir, afire_options)
LOG.info('attempted_runs {}'.format(attempted_runs))
LOG.info('successful_runs {}'.format(successful_runs))
LOG.info('problem_runs {}'.format(problem_runs))
LOG.info('crashed_runs {}'.format(crashed_runs))
except Exception:
LOG.error(traceback.format_exc())
rc = 1
return rc
def main():
"""
Load data and train a model on it.
"""
args = argument_parser().parse_args()
print(args)
random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.set_device(0)
train_set, val_set, test_set = read_dataset(DATA_DIR)
if not args.pretrained:
print('Training...')
train(train_set, test_set, args.checkpoint, **train_kwargs(args))
else:
print('Restoring from checkpoint...')
model_state, op_state, meta_iteration, cur_meta_step_size, accuracy_tracking = load_checkpoint(
args.checkpoint)
train(train_set, test_set, args.checkpoint, model_state, op_state,
**train_kwargs(args))
print('\nEvaluating...')
model_state, op_state, meta_iteration, cur_meta_step_size, accuracy_tracking = load_checkpoint(
args.checkpoint)
do_evaluation(model_state, op_state, args.checkpoint, val_set, test_set,
train_set)
def load_args(self):
if isinstance(self.params,list):
temp_argv = sys.argv
sys.argv[1:] = self.params
# global variables
parser = argument_parser()
self.args = parser.parse_args()
sys.argv = temp_argv
else:
self.args = Namespace(**self.params)
print(self.args)
def main():
args = argument_parser()
dataset_files = {
'seq2seq': {
'train': args.data_dir + '/formatted_movie_lines.txt',
'dev': args.data_dir + '/formatted_movie_lines.txt',
'bak': '/formatted_movie_lines_bak.txt'
},
}
save_dirs = {
'senti2class': 'senti2class_models',
'senti3class': 'senti3class_models',
'senti6class': 'senti6class_models'
}
args.dataset_files = dataset_files[args.name]
#args.tgt_size = tgt_sizes[args.name]
#args.save_dir = save_dirs[args.name]
logger = get_logger(args)
logger.info(args)
check_gpu(args, logger)
program = TorchProgram(args, logger)
if args.train:
program.train()
write_config(args, logger)
if args.test and args.load_dir:
program.test_seq2seq(program.vocab)
if args.export:
program.export(args.export_path)
from torchreid.data_manager import VideoDataManager
from torchreid import models
from torchreid.losses import CrossEntropyLoss, TripletLoss, DeepSupervision
from torchreid.utils.iotools import save_checkpoint, check_isfile
from torchreid.utils.avgmeter import AverageMeter
from torchreid.utils.loggers import Logger, RankLogger
from torchreid.utils.torchtools import count_num_param, open_all_layers, open_specified_layers, accuracy, load_pretrained_weights
from torchreid.utils.reidtools import visualize_ranked_results
from torchreid.utils.generaltools import set_random_seed
from torchreid.eval_metrics import evaluate
from torchreid.samplers import RandomIdentitySampler
from torchreid.optimizers import init_optimizer
from torchreid.lr_schedulers import init_lr_scheduler
# global variables
parser = argument_parser()
args = parser.parse_args()
def main():
global args
set_random_seed(args.seed)
if not args.use_avai_gpus:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
log_name = 'log_test.txt' if args.evaluate else 'log_train.txt'
sys.stdout = Logger(osp.join(args.save_dir, log_name))
print('==========\nArgs:{}\n=========='.format(args))
"""
Helpers for evaluating models.
"""
import os
import numpy as np
import tqdm
import pandas as pd
from reptile import Reptile
from args import argument_parser, evaluate_kwargs
from util import mean_confidence_interval
args = argument_parser().parse_args()
eval_kwargs = evaluate_kwargs(args)
# pylint: disable=R0913,R0914
def evaluate(dataset,
model_state,
op_state,
num_classes=5,
num_shots=5,
eval_inner_batch_size=5,
eval_inner_iters=50,
num_samples=10000,
transductive=False,
cuda=False,
pin_memory=False,
foml=False):
"""
Evaluate a model on a dataset.
spacy_en = spacy.load('en')
def tokenizer(text): # create a tokenizer function
# 返回 a list of <class 'spacy.tokens.token.Token'>
return [tok.text for tok in spacy_en.tokenizer(text)]
from torchtext import data
import numpy as np
from data import text_utils
if __name__ == '__main__':
args = argument_parser()
with open("seq2seq/bak/TEXT.Field", "rb") as f:
TEXT = dill.load(f)
LENGTH = data.Field(sequential=False, use_vocab=False)
embeddings = np.random.random((len(TEXT.vocab.itos), args.embed_size))
args.TEXT = TEXT
encoder = SN_MODELS["encoder"](embeddings, args)
# atten = SN_MODELS["attention"](args.hidden_size * 4, 300)
#decoder = SN_MODELS["decoder"](embeddings, args)
atten = SN_MODELS["attention"](args.hidden_size, "general")
decoder = SN_MODELS["decoder"](embeddings, args, atten)
model_class = SN_MODELS[args.model_name]
