def main(args):
if not args or args[0] == '-h': usage()
tracefile = args[0]
with open(tracefile) as f:
mined_trees = json.load(f)
util.init_log('generalize_loop', '', mined_trees[0]['original'])
gmethod_trees = generalize_loop_trees(mined_trees)
print(json.dumps(gmethod_trees, indent=4))
def __init__(self,
params,
data_info,
fn_dict,
new_fn_dict,
log_level='INFO'):
""" Initialize EvalPopulation.
Args:
params: dictionary with parameters.
data_info: one of input.*Info objects.
fn_dict: dict with definitions of the functions (name and parameters);
format --> {'fn_name': ['FNClass', {'param1': value1, 'param2': value2}]}.
log_level: (str) one of "INFO", "DEBUG" or "NONE".
"""
self.train_params = params
self.data_info = data_info
self.fn_dict = fn_dict
self.fn_new_dict = new_fn_dict
self.timeout = 9000
self.logger = init_log(log_level, name=__name__)
self.comm = MPI.COMM_WORLD
self.size = self.comm.Get_size()
self.num_workers = self.size - 1
def master(args, comm):
""" Master function -> run the evolution and send parameters of evaluation task to workers.
Args:
args: dict with command-in-line parameters.
comm: MPI.COMM_WORLD.
"""
logger = init_log(args['log_level'], name=__name__)
if not os.path.exists(args['experiment_path']):
logger.info(f"Creating {args['experiment_path']} ...")
os.makedirs(args['experiment_path'])
# Evolution or continue previous evolution
if not args['continue_path']:
phase = 'evolution'
else:
phase = 'continue_evolution'
logger.info(
f"Continue evolution from: {args['continue_path']}. Checking files ..."
)
check_files(args['continue_path'])
logger.info(f"Getting parameters from {args['config_file']} ...")
config = cfg.ConfigParameters(args, phase=phase)
config.get_parameters()
logger.info(f"Saving parameters for {config.phase} phase ...")
config.save_params_logfile()
# Evaluation function for QNAS (train CNN and return validation accuracy)
eval_f = evaluation.EvalPopulation(
params=config.train_spec,
data_info=config.data_info,
fn_dict=config.fn_dict,
log_level=config.train_spec['log_level'],
new_fn_dict=config.fn_new_dict)
qnas_cnn = qnas.QNAS(eval_f,
config.train_spec['experiment_path'],
log_file=config.files_spec['log_file'],
log_level=config.train_spec['log_level'],
data_file=config.files_spec['data_file'])
qnas_cnn.initialize_qnas(**config.QNAS_spec)
# If continue previous evolution, load log file and read it at final generation
if phase == 'continue_evolution':
logger.info(
f"Loading {config.files_spec['previous_data_file']} file to get final "
f"generation ...")
qnas_cnn.load_qnas_data(
file_path=config.files_spec['previous_data_file'])
# Execute evolution
logger.info(f"Starting evolution ...")
qnas_cnn.evolve()
send_stop_signal(comm)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base_dir', default=os.path.expanduser('./'))
parser.add_argument('--input', default='training/train.txt')
parser.add_argument('--model', default='tacotron')
parser.add_argument(
'--name',
help='Name of the run. Used for logging. Defaults to model name.')
parser.add_argument(
'--hparams',
default='',
help=
'Hyperparameter overrides as a comma-separated list of name=value pairs'
)
parser.add_argument('--restore_step',
type=int,
help='Global step to restore from checkpoint.',
default=0)
parser.add_argument('--summary_interval',
type=int,
default=100,
help='Steps between running summary ops.')
parser.add_argument('--checkpoint_interval',
type=int,
default=1000,
help='Steps between writing checkpoints.')
parser.add_argument('--slack_url',
help='Slack webhook URL to get periodic reports.')
# parser.add_argument('--tf_log_level', type=int, default=1, help='Tensorflow C++ log level.')
parser.add_argument('--git',
action='store_true',
help='If set, verify that the client is clean.')
args = parser.parse_args()
# os.environ['TF_CPP_MIN_LOG_LEVEL'] = str(args.tf_log_level)
run_name = args.name or args.model
log_dir = os.path.join(args.base_dir, 'logs-%s' % run_name)
os.makedirs(log_dir, exist_ok=True)
init_log(os.path.join(log_dir, 'train.log'), run_name, args.slack_url)
# args还需要修改
# print(args.hparams)
# print(type(args.hparams))
# print(tacotron_hparams["decay_learning_rate"])
# print("========================================")
train(log_dir, args)
def __init__(self, gutenberg_files_root, prepared_training_data_root, corpus_root):
if not os.path.exists('../log'):
os.mkdir('../log')
self.logger = init_log('util', '../log/util.log')
self.gutenberg_files_root = gutenberg_files_root
self.prepared_training_data_root = prepared_training_data_root
self.corpus_root = corpus_root
self.bigram_frequency_dir = os.path.join(self.corpus_root, 'bigram_frequency')
def main():
conf_path = sys.argv[1]
conf = Config(conf_path)
global logger
logger = init_log(__file__, conf.log_path)
word_vectors = conf.word_vectors
adjust_input = True if conf.adjust_input else False
logger.info("loading data...")
if word_vectors not in ('rand', 'word2vec'):
raise ValueError('invalid parameter word_vectors %s' % word_vectors)
with open(conf.train_path, 'rb') as f:
x = pickle.load(f)
revs, W, W2, word_idx_map, vocab = x[0], x[1], x[2], x[3], x[4]
logger.info("data loaded!")
if adjust_input:
print "model architecture: CNN-non-static"
else:
print "model architecture: CNN-static"
if word_vectors == "rand":
print "using: random vectors"
U = W2
elif word_vectors == "word2vec":
print "using: word2vec vectors"
U = W
sys.stdout.flush()
results = []
datasets = make_idx_data_cv(revs, word_idx_map, conf.cv_index, max_l=15, k=300)
if os.path.exists(conf.model_path):
sc = SentConv.load(conf.model_path)
logger.info('Load existing model from %s' % conf.model_path)
else:
conf.log(logger)
sc = SentConv(filter_hs=conf.filter_hs, filter_num=conf.filter_num, n_hidden=conf.filter_num, n_out=2, word_idx_map=word_idx_map, wordvec=U, adjust_input=adjust_input)
logger.info('Initiate a model')
if conf.do_train:
try:
sc.fit(datasets, batch_size=conf.batch_size, n_epochs=conf.n_epochs)
except KeyboardInterrupt:
logger.warning('Got control C. Quit.')
return
finally:
sc.save(conf.model_path)
else:
logger.info('config says do not execute train process')
if conf.do_test:
test_result = sc.test_from_file(conf.test_path, encoding='gb18030', out_path=conf.test_out_path)
logger.info('test result of %s' % conf.test_path)
logger.info(test_result)
logger.info('test out path is %s' % conf.test_out_path)
else:
logger.info('config says do not execute test process')
return None
def run(webpages_dir, body_text_dir, log_path):
logger = util.init_log('TextExtract', log_path, console=False)
util.makedir(body_text_dir)
for name in os.listdir(webpages_dir):
logger.info('begin extract body text of %s' % name)
a_person_dir = os.path.join(webpages_dir, '%s/' % name)
for file_name in os.listdir(a_person_dir):
rank = file_name.split('.')[0]
html_path = os.path.join(a_person_dir, file_name)
content = text_extract(html_path, logger)
body_path = os.path.join(body_text_dir, name, '%s.txt' % rank)
write_body(content, body_path, logger)
return None
def __init__(self, eval_func, experiment_path, log_file, log_level,
data_file):
""" Initialize QNAS.
Args:
eval_func: function that will be used to evaluate individuals.
experiment_path: (str) path to the folder where logs and models will be saved.
log_file: (str) path to the file to keep logs.
log_level: (str) one of "INFO", "DEBUG" or "NONE".
"""
self.dtype = np.float64 # Type of all arrays excluding fitnesses
self.tolerance = 1.e-15 # Tolerance to compare floating point
self.best_so_far = 0.0 # Best fitness so far
self.best_so_far_id = [
0, 0
] # id = [generation, position in the population]
self.current_best_id = [0, 0]
self.current_gen = 0 # Current generation number
self.data_file = data_file
self.eval_func = eval_func
self.experiment_path = experiment_path
self.fitnesses = None # TF calculates accuracy with float32 precision
self.generations = None
self.update_quantum_gen = None
self.logger = init_log(log_level, name=__name__, file_path=log_file)
self.penalties = None
self.penalize_number = None
self.random = 0.0
self.raw_fitnesses = None
self.reducing_fns_list = []
self.replace_method = None
self.save_data_freq = np.Inf
self.total_eval = 0
self.qpop_params = None
self.qpop_net = None
parser.add_argument('--quantiles',
nargs='+',
help='quantiles',
type=float,
required=True)
parser.add_argument('--device',
default="cpu",
type=str,
required=False,
help='compute device')
parser.add_argument('--debug',
default=False,
dest='debug',
action='store_true')
args = parser.parse_args()
init_log(args)
torch.set_num_threads(hyperparams.default_threads)
my_print("Parameters:")
for k, v in sorted(vars(args).items()):
my_print("\t{0}: {1}".format(k, v))
# Set the random seed manually for reproducibility.
torch.manual_seed(hyperparams.random_seed)
###############################################################################
# Load data
###############################################################################
if torch.cuda.is_available():
if args.device != "cpu":
torch.cuda.set_device(int(args.device.split(":")[1]))
args.device = torch.device(args.device)
def testfetch():
fetcher = LogFetcher()
for ip in config.serverlist:
fetcher.add_hosthandler(
YesterdayDownloadHostHandler(
ip, '55666',
'http://%s:%s/archive/flashserver_%s.tar.gz',
config.archivedir,
'flashserver_%s.tar.gz'
)
)
fetcher.fetchall()
def testparse():
parser = YesterdayTarsLogParser(config.workbench, config.archivedir)
h_lost = ClassPacketLostHandler()
h_dis = ClassDisConnHandler()
parser.addloghandler(h_lost)
parser.addloghandler(h_dis)
parser.work()
if __name__ == '__main__':
util.init_log('mon_supervisor.%d.log' % os.getpid())
supervisor()
#testfetch()
#testparse()
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
import os
import io
import logging.config
# constants
ROOT_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
STORE_PATH = os.path.join(ROOT_PATH, 'stores')
INI_PATH = os.path.join(ROOT_PATH, 'ini')
LOG_INI_PATH = os.path.join(INI_PATH, 'log.ini')
MAIN_INI_PATH = os.path.join(INI_PATH, 'myhouse.ini')
import util
logger, root_logger = util.init_log(LOG_INI_PATH)
def main(tracefile):
with open(tracefile) as f:
mined_trees = json.load(f)
util.init_log('generalize_loop', '', mined_trees[0]['original'])
gmethod_trees = generalize_loop_trees(mined_trees)
print(json.dumps(gmethod_trees, indent=4))
def main(**args):
logger = init_log(args['log_level'], name=__name__)
# Check if *experiment_path* contains all the necessary files to retrain an evolved model
check_files(args['experiment_path'])
# Get all parameters
logger.info(f"Getting parameters from evolution ...")
config = cfg.ConfigParameters(args, phase='retrain')
config.get_parameters()
# Load log file and read it at the specified generation
s = f"last generation" if args['generation'] is None else f"generation {args['generation']}"
logger.info(f"Loading {config.files_spec['data_file']} file to get {s}, individual "
f"{args['individual']} ...")
config.load_evolved_data(generation=args['generation'],
individual=args['individual'])
if args['lr_schedule'] is not None:
special_params = train.train_schemes_map[args['lr_schedule']].get_params()
logger.info(f"Overriding train parameters to use special scheme "
f"'{args['lr_schedule']}' ...")
config.override_train_params(special_params)
# It is important to merge the dicts with the evolved_params first, as they need to be
# overwritten in case we are using one of the special train schemes.
train_params = {**config.evolved_params['params'], **config.train_spec}
best_ind_tese = ['conv_5_1_512', 'conv_3_1_128', 'conv_3_1_512',
'conv_5_1_256',
'avg_pool_2_2',
'conv_3_1_256',
'avg_pool_2_2',
'conv_5_1_128',
'avg_pool_2_2',
'max_pool_2_2']
# best_ind_tese = ['bv1p_3_1_128',
# 'bv1p_3_1_128',
# 'bv1p_3_1_256',
# 'avg_pool_2_2',
# 'no_op',
# 'bv1p_3_1_256',
# 'no_op',
# 'no_op',
# 'no_op',
# 'max_pool_2_2',
# 'max_pool_2_2',
# 'bv1_3_1_128',
# 'bv1_3_1_64',
# 'bv1p_3_1_256',
# 'no_op',
# 'bv1_3_1_256',
# 'max_pool_2_2',
# 'bv1_3_1_256',
# 'bv1p_3_1_64',
# 'no_op'
# ]
config.evolved_params['net'] = best_ind_tese
logger.info(f"Starting training of model {config.evolved_params['net']}")
valid_acc, test_info = train.train_and_eval(data_info=config.data_info,
params=train_params,
fn_dict=config.fn_dict,
net_list=config.evolved_params['net'],
lr_schedule=args['lr_schedule'],
run_train_eval=args['run_train_eval'])
logger.info(f"Saving parameters...")
config.save_params_logfile()
logger.info(f"Best accuracy in validation set: {valid_acc:.5f}")
logger.info(f"Final test accuracy: {test_info['accuracy']:.5f}")
logger.info(f"Final test confusion matrix:\n{test_info['confusion_matrix']}")
optimizer.zero_grad()
loss = criterion(feature_vector, label_batch)
if n_item % 50 == 1:
sw.add_scalar('loss_curve', loss, n_item)
loss.backward()
optimizer.step()
n_item = n_item + 1
if epoch % cycle_epoches_for_test == 1:
acc_now = evaluate(net, test_dataloader)
sw.add_scalar('acc_curve', acc_now, epoch)
logging.info("epoch:{}\tn_item:{}\tacc:{:.6f}".format(epoch, n_item, acc_now))
if acc_now > best_acc:
state = {
'net': net.state_dict(),
'acc': acc_now,
'epoch': epoch,
"optimizer": optimizer.state_dict(),
'n_item': n_item,
}
if not os.path.exists(pth_dir_path):
os.mkdir(pth_dir_path)
torch.save(state, pth_path)
sw.close()
if __name__ == '__main__':
output_files_dir = r"/data/cifar/10"
init_log(os.path.join(output_files_dir,'experiment_result.log'))
run(output_files_dir)
