def main():
args = parse_arguments()
config = get_config(args.config_file, is_test=args.test)
np.random.seed(config.seed)
# torch.manual_seed(config.seed)
# torch.cuda.manual_seed_all(config.seed)
config.use_gpu = config.use_gpu and torch.cuda.is_available()
# log info
log_file = os.path.join(config.save_dir,
"log_exp_{}.txt".format(config.run_id))
logger = setup_logging(args.log_level, log_file)
logger.info("Writing log file to {}".format(log_file))
logger.info("Exp instance id = {}".format(config.run_id))
logger.info("Exp comment = {}".format(args.comment))
logger.info("Config =")
print(">" * 80)
pprint(config)
print("<" * 80)
# Run the experiment
try:
runner = eval(config.runner)(config)
if not args.test:
runner.train()
else:
runner.test()
except:
logger.error(traceback.format_exc())
sys.exit(0)
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
from utils.logger import setup_logging
setup_logging(os.path.join("configs", "logging.json"))
def main():
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'prs_dev.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?") from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()
from dotenv import find_dotenv, load_dotenv
from utils.auth import jwt
app = Flask(__name__)
app.config.update({
'SECRET_KEY':
'FOICLIENTAPP',
'SQLALCHEMY_TRACK_MODIFICATIONS':
False,
'SQLALCHEMY_DATABASE_URI':
BaseConfig.SQLALCHEMY_DATABASE_URI,
})
setup_logging(os.path.join(BaseConfig.PROJECT_ROOT,
'logging.conf')) # important to do this first
def init_app(run_mode=os.getenv('FLASK_ENV', 'production')):
app.config.from_object(CONFIG[run_mode])
CORS(app)
from .requestapi import REQUESTAPI_BLUEPRINT
app.register_blueprint(REQUESTAPI_BLUEPRINT)
ExceptionHandler(app)
setup_jwt_manager(app, jwt)
db.init_app(app)
import sentry_sdk
from flask import Flask, current_app
from pay_api.models import CfsAccount as CfsAccountModel
from pay_api.models import PaymentAccount as PaymentAccountModel
from pay_api.models import db, ma
from pay_api.services.cfs_service import CFSService
from pay_api.services.oauth_service import OAuthService
from pay_api.utils.constants import DEFAULT_COUNTRY, DEFAULT_CURRENCY
from pay_api.utils.enums import AuthHeaderType, ContentType, PaymentMethod
from sentry_sdk.integrations.flask import FlaskIntegration
import config
from utils.logger import setup_logging
setup_logging(
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'logging.conf')) # important to do this first
def create_app(run_mode=os.getenv('FLASK_ENV', 'production')):
"""Return a configured Flask App using the Factory method."""
app = Flask(__name__)
app.config.from_object(config.CONFIGURATION[run_mode])
# Configure Sentry
if app.config.get('SENTRY_DSN', None): # pragma: no cover
sentry_sdk.init(dsn=app.config.get('SENTRY_DSN'),
integrations=[FlaskIntegration()])
db.init_app(app)
ma.init_app(app)
def main():
parser = argparse.ArgumentParser(description='Phoneme classification task')
parser.add_argument('--load',
type=str,
help='Path of model checkpoint to load',
required=True)
parser.add_argument('--config',
'-c',
type=str,
help='Name of config file to load',
required=True)
args = parser.parse_args()
setup_logging(args)
cfg = parse(args)
# use all gpus available
num_gpu = torch.cuda.device_count()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
normalize = get_normalizer(cfg.data_type)
# get the dataloaders
test_data = PhonemeDataset(cfg.num_frames,
cfg.phn_idx_map,
cfg.data_type,
mode='test',
transform=normalize)
test_loader = DataLoader(dataset=test_data,
batch_size=cfg.batch_size,
shuffle=True)
logging.info('Cmd: python {0}'.format(' '.join(sys.argv)))
# set up model and load checkpoint
if not cfg.model_arch:
raise ValueError('Model architecture not specified')
if not cfg.model_arch in ARCH_TO_MODEL:
raise ValueError('Model architecture {} does not exist.'.format(
cfg.model_arch))
model = ARCH_TO_MODEL[cfg.model_arch](cfg.num_channels, cfg.num_frames,
cfg.num_classes, cfg.num_dimensions)
with open(os.path.join(cfg.load), 'rb') as f:
model.load_state_dict(torch.load(f))
model = model.to(device)
model = torch.nn.DataParallel(model)
# test the model
test_err, predictions = predict(model, test_loader, device, cfg)
logging.info('Test PER: {:.2f}%'.format(test_err))
# save the output into the a pickle file in the log directory
output_dir = os.path.join(cfg.log_dir, 'predict.pkl')
logging.info('Prediction saved in {}'.format(output_dir))
with open(output_dir, 'w') as file:
pkl.dump(predictions, file)
# plot the confusion matrix
plot_phoneme_cm(predictions, cfg.phn_idx_map)
from api.server import run_server
from utils.logger import setup_logging
if __name__ == '__main__':
setup_logging()
run_server()
def main():
args = parse_arguments()
config = get_bo_config(args.config_file)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
config.use_gpu = config.use_gpu and torch.cuda.is_available()
device = torch.device('cuda' if config.use_gpu else 'cpu')
# log info
log_file = os.path.join(config.save_dir,
"log_exp_{}.txt".format(config.run_id))
logger = setup_logging(args.log_level, log_file)
logger.info("Writing log file to {}".format(log_file))
logger.info("Exp instance id = {}".format(config.run_id))
logger.info("Exp comment = {}".format(args.comment))
logger.info("Config =")
print(">" * 80)
pprint(config)
print("<" * 80)
#load model
model = eval(config.model.name)(config.model)
model_snapshot = torch.load(config.model.pretrained_model,
map_location=device)
model.load_state_dict(model_snapshot["model"], strict=True)
model.to(device)
if config.use_gpu:
model = nn.DataParallel(model, device_ids=config.gpus).cuda()
# Run the experiment
results_list = bq_loop(config.bq, model)
if config.bq.is_GPY:
if config.bq.is_sparseGP:
pickle.dump(
results_list,
open(
os.path.join(
config.bq.save_dir,
config.bq.name + '_sparseGP_init_p' +
str(config.bq.init_num_data) + '_inducing_p' +
str(config.bq.num_inducing_pts) + '_results.p'), 'wb'))
else:
pickle.dump(
results_list,
open(
os.path.join(
config.bq.save_dir, config.bq.name + '_fullGP_init_p' +
str(config.bq.init_num_data) + '_results.p'), 'wb'))
else:
if config.bq.is_ai:
pickle.dump(
results_list,
open(
os.path.join(
config.bq.save_dir, config.bq.name + '_ai_init_p' +
str(config.bq.init_num_data) + '_results.p'), 'wb'))
else:
pickle.dump(
results_list,
open(
os.path.join(
config.bq.save_dir, config.bq.name + '_opt_init_p' +
str(config.bq.init_num_data) + 'iter' +
str(config.bq.opt_iter) + 'lr' +
str(config.bq.opt_lr) + '_results.p'), 'wb'))
sys.exit(0)
def main():
parser = argparse.ArgumentParser(description='Phoneme classification task')
parser.add_argument('--config',
'-c',
type=str,
help='Name of config file to load',
required=True)
args = parser.parse_args()
setup_logging(args)
cfg = parse(args)
# use all gpus available
num_gpu = torch.cuda.device_count()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if device == 'gpu': logging.info('Look ma, I\'m using a GPU!')
# get the dataloaders
train_dataloader, valid_dataloader, test_dataloader = get_dataloaders(
cfg.batch_size, cfg.num_frames, cfg.phn_idx_map, cfg.data_type)
# setup hyperparameter sweeping
if not cfg.num_sweeps:
cfg.num_sweeps = 1
logging.info('Cmd: python {0}'.format(' '.join(sys.argv)))
logging.info('Running experiment <{0}> in train mode.'.format(
cfg.config, cfg.mode))
orig_cfg = cfg
for sweep_count in range(cfg.num_sweeps):
cfg = sample_cfg(orig_cfg)
# create a directory for the model to be saved
cfg.sweep_dir = os.path.join(cfg.log_dir, 's_{}'.format(sweep_count))
os.mkdir(cfg.sweep_dir)
logging.info('Sweep Count: {}'.format(sweep_count))
logging.info('Config:\n {0}'.format(cfg))
# set up model and load if checkpoint provided
if not cfg.model_arch:
raise ValueError('Model architecture not specified')
if not cfg.model_arch in ARCH_TO_MODEL:
raise ValueError('Model architecture {} does not exist.'.format(
cfg.model_arch))
model = ARCH_TO_MODEL[cfg.model_arch](cfg.num_channels, cfg.num_frames,
cfg.num_classes,
cfg.num_dimensions)
if cfg.load:
with open(os.path.join(cfg.load), 'rb') as f:
model.load_state_dict(torch.load(f))
model = model.to(device)
model = torch.nn.DataParallel(model)
# setup cross entropy loss for multi-class classification
loss = NLLLoss()
# setup the optimizer
optim = Adam(model.parameters(),
lr=cfg.learning_rate,
weight_decay=cfg.l2_regularizer)
# train the model
logging.info('Commencing training')
lowest_err = 100.
for epoch in range(1, cfg.epochs + 1):
train_loss, train_err = train(model, train_dataloader, loss,
device, cfg, optim)
val_loss, val_err = evaluate(model, valid_dataloader, loss, device,
cfg)
logging.info(
'Epoch: {}\tTrain Loss: {:.6f}\tTrain PER: {:.2f}\tVal Loss: {:.6f}\tVal PER: {:.2f}%'
.format(epoch, train_loss, train_err, val_loss, val_err))
if val_err < lowest_err:
logging.info(
'Lowest validation error achieved. Saving the model.')
save_model(model, cfg.sweep_dir, best=True)
lowest_err = val_err
# save the checkpoint of the latest version of the model
save_model(model, cfg.sweep_dir)
# test the model
_, test_err = evaluate(model, test_dataloader, loss, device, cfg)
logging.info('Test PER: {:.2f}%'.format(test_err))
for redundant in redundants:
use_redundant = (redundant == 'redundant')
store_data_path = os.path.join('./data/processed', data_process_type, cut, redundant)
if not os.path.exists(store_data_path):
os.makedirs(store_data_path)
processor = RawProcessor(file_paths=file_paths, store_path=store_data_path, use_cut=use_cut,
redundant=use_redundant)
# 处理数据
processor.process_raw_data()
# 写入数据
processor.prepare_data(data_split_dict={'train': 0.7, 'dev': 0.2, 'test': 0.1})
# 写入统计词频
write_counter(processor.words_counter, os.path.join(store_data_path, 'word_count.json'),
key=lambda kv: (kv[1], kv[0]), reverse=True)
write_counter(processor.length_counter, os.path.join(store_data_path, 'length_count.json'))
if __name__ == '__main__':
STOPWORDS = get_stopwords('./data/stopwords.txt')
if os.path.exists('./logs/data_info.log'):
os.remove('./logs/data_info.log')
setup_logging(default_path='./utils/logger_config.yaml')
logger = logging.getLogger("data_logger")
start()
# 清理已保存的数据
subprocess.call(r'find ./data/processed/data_all -name *_data -type f -print -exec rm {} \;', shell=True)