예제 #1
0
def main(args):
    if not args or args[0] == '-h': usage()
    tracefile = args[0]
    with open(tracefile) as f:
        mined_trees = json.load(f)
    util.init_log('generalize_loop', '', mined_trees[0]['original'])
    gmethod_trees = generalize_loop_trees(mined_trees)
    print(json.dumps(gmethod_trees, indent=4))
예제 #2
0
    def __init__(self,
                 params,
                 data_info,
                 fn_dict,
                 new_fn_dict,
                 log_level='INFO'):
        """ Initialize EvalPopulation.

        Args:
            params: dictionary with parameters.
            data_info: one of input.*Info objects.
            fn_dict: dict with definitions of the functions (name and parameters);
                format --> {'fn_name': ['FNClass', {'param1': value1, 'param2': value2}]}.
            log_level: (str) one of "INFO", "DEBUG" or "NONE".
        """

        self.train_params = params
        self.data_info = data_info
        self.fn_dict = fn_dict
        self.fn_new_dict = new_fn_dict
        self.timeout = 9000
        self.logger = init_log(log_level, name=__name__)
        self.comm = MPI.COMM_WORLD
        self.size = self.comm.Get_size()
        self.num_workers = self.size - 1
예제 #3
0
def master(args, comm):
    """ Master function -> run the evolution and send parameters of evaluation task to workers.

    Args:
        args: dict with command-in-line parameters.
        comm: MPI.COMM_WORLD.
    """

    logger = init_log(args['log_level'], name=__name__)

    if not os.path.exists(args['experiment_path']):
        logger.info(f"Creating {args['experiment_path']} ...")
        os.makedirs(args['experiment_path'])

    # Evolution or continue previous evolution
    if not args['continue_path']:
        phase = 'evolution'
    else:
        phase = 'continue_evolution'
        logger.info(
            f"Continue evolution from: {args['continue_path']}. Checking files ..."
        )
        check_files(args['continue_path'])

    logger.info(f"Getting parameters from {args['config_file']} ...")
    config = cfg.ConfigParameters(args, phase=phase)
    config.get_parameters()
    logger.info(f"Saving parameters for {config.phase} phase ...")
    config.save_params_logfile()

    # Evaluation function for QNAS (train CNN and return validation accuracy)
    eval_f = evaluation.EvalPopulation(
        params=config.train_spec,
        data_info=config.data_info,
        fn_dict=config.fn_dict,
        log_level=config.train_spec['log_level'],
        new_fn_dict=config.fn_new_dict)

    qnas_cnn = qnas.QNAS(eval_f,
                         config.train_spec['experiment_path'],
                         log_file=config.files_spec['log_file'],
                         log_level=config.train_spec['log_level'],
                         data_file=config.files_spec['data_file'])

    qnas_cnn.initialize_qnas(**config.QNAS_spec)

    # If continue previous evolution, load log file and read it at final generation
    if phase == 'continue_evolution':
        logger.info(
            f"Loading {config.files_spec['previous_data_file']} file to get final "
            f"generation ...")
        qnas_cnn.load_qnas_data(
            file_path=config.files_spec['previous_data_file'])

    # Execute evolution
    logger.info(f"Starting evolution ...")
    qnas_cnn.evolve()

    send_stop_signal(comm)
예제 #4
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--base_dir', default=os.path.expanduser('./'))
    parser.add_argument('--input', default='training/train.txt')
    parser.add_argument('--model', default='tacotron')
    parser.add_argument(
        '--name',
        help='Name of the run. Used for logging. Defaults to model name.')
    parser.add_argument(
        '--hparams',
        default='',
        help=
        'Hyperparameter overrides as a comma-separated list of name=value pairs'
    )
    parser.add_argument('--restore_step',
                        type=int,
                        help='Global step to restore from checkpoint.',
                        default=0)
    parser.add_argument('--summary_interval',
                        type=int,
                        default=100,
                        help='Steps between running summary ops.')
    parser.add_argument('--checkpoint_interval',
                        type=int,
                        default=1000,
                        help='Steps between writing checkpoints.')
    parser.add_argument('--slack_url',
                        help='Slack webhook URL to get periodic reports.')
    # parser.add_argument('--tf_log_level', type=int, default=1, help='Tensorflow C++ log level.')
    parser.add_argument('--git',
                        action='store_true',
                        help='If set, verify that the client is clean.')
    args = parser.parse_args()
    # os.environ['TF_CPP_MIN_LOG_LEVEL'] = str(args.tf_log_level)
    run_name = args.name or args.model
    log_dir = os.path.join(args.base_dir, 'logs-%s' % run_name)
    os.makedirs(log_dir, exist_ok=True)
    init_log(os.path.join(log_dir, 'train.log'), run_name, args.slack_url)
    # args还需要修改
    # print(args.hparams)
    # print(type(args.hparams))

    # print(tacotron_hparams["decay_learning_rate"])
    # print("========================================")

    train(log_dir, args)
예제 #5
0
파일: prepare.py 프로젝트: Huarong/cloze
 def __init__(self, gutenberg_files_root, prepared_training_data_root, corpus_root):
     if not os.path.exists('../log'):
         os.mkdir('../log')
     self.logger = init_log('util', '../log/util.log')
     self.gutenberg_files_root = gutenberg_files_root
     self.prepared_training_data_root = prepared_training_data_root
     self.corpus_root = corpus_root
     self.bigram_frequency_dir = os.path.join(self.corpus_root, 'bigram_frequency')
예제 #6
0
def main():
    conf_path = sys.argv[1]
    conf = Config(conf_path)
    global logger
    logger = init_log(__file__, conf.log_path)
    word_vectors = conf.word_vectors
    adjust_input = True if conf.adjust_input else False
    logger.info("loading data...")
    if word_vectors not in ('rand', 'word2vec'):
        raise ValueError('invalid parameter word_vectors %s' % word_vectors)

    with open(conf.train_path, 'rb') as f:
        x = pickle.load(f)
    revs, W, W2, word_idx_map, vocab = x[0], x[1], x[2], x[3], x[4]
    logger.info("data loaded!")
    if adjust_input:
        print "model architecture: CNN-non-static"
    else:
        print "model architecture: CNN-static"
    if word_vectors == "rand":
        print "using: random vectors"
        U = W2
    elif word_vectors == "word2vec":
        print "using: word2vec vectors"
        U = W
    sys.stdout.flush()
    results = []
    datasets = make_idx_data_cv(revs, word_idx_map, conf.cv_index, max_l=15, k=300)
    if os.path.exists(conf.model_path):
        sc = SentConv.load(conf.model_path)
        logger.info('Load existing model from %s' % conf.model_path)
    else:
        conf.log(logger)
        sc = SentConv(filter_hs=conf.filter_hs, filter_num=conf.filter_num, n_hidden=conf.filter_num, n_out=2, word_idx_map=word_idx_map, wordvec=U, adjust_input=adjust_input)
        logger.info('Initiate a model')
    if conf.do_train:
        try:
            sc.fit(datasets, batch_size=conf.batch_size, n_epochs=conf.n_epochs)
        except KeyboardInterrupt:
            logger.warning('Got control C. Quit.')
            return
        finally:
            sc.save(conf.model_path)
    else:
        logger.info('config says do not execute train process')
    if conf.do_test:
        test_result = sc.test_from_file(conf.test_path, encoding='gb18030', out_path=conf.test_out_path)
        logger.info('test result of %s' % conf.test_path)
        logger.info(test_result)
        logger.info('test out path is %s' % conf.test_out_path)
    else:
        logger.info('config says do not execute test process')
    return None
예제 #7
0
def run(webpages_dir, body_text_dir, log_path):
    logger = util.init_log('TextExtract', log_path, console=False)
    util.makedir(body_text_dir)

    for name in os.listdir(webpages_dir):
        logger.info('begin extract body text of %s' % name)
        a_person_dir = os.path.join(webpages_dir, '%s/' % name)
        for file_name in os.listdir(a_person_dir):
            rank = file_name.split('.')[0]
            html_path = os.path.join(a_person_dir, file_name)
            content = text_extract(html_path, logger)
            body_path = os.path.join(body_text_dir, name, '%s.txt' % rank)
            write_body(content, body_path, logger)
    return None
예제 #8
0
    def __init__(self, eval_func, experiment_path, log_file, log_level,
                 data_file):
        """ Initialize QNAS.

        Args:
            eval_func: function that will be used to evaluate individuals.
            experiment_path: (str) path to the folder where logs and models will be saved.
            log_file: (str) path to the file to keep logs.
            log_level: (str) one of "INFO", "DEBUG" or "NONE".
        """

        self.dtype = np.float64  # Type of all arrays excluding fitnesses
        self.tolerance = 1.e-15  # Tolerance to compare floating point

        self.best_so_far = 0.0  # Best fitness so far
        self.best_so_far_id = [
            0, 0
        ]  # id = [generation, position in the population]
        self.current_best_id = [0, 0]
        self.current_gen = 0  # Current generation number
        self.data_file = data_file
        self.eval_func = eval_func
        self.experiment_path = experiment_path
        self.fitnesses = None  # TF calculates accuracy with float32 precision
        self.generations = None
        self.update_quantum_gen = None
        self.logger = init_log(log_level, name=__name__, file_path=log_file)
        self.penalties = None
        self.penalize_number = None
        self.random = 0.0
        self.raw_fitnesses = None
        self.reducing_fns_list = []
        self.replace_method = None
        self.save_data_freq = np.Inf
        self.total_eval = 0

        self.qpop_params = None
        self.qpop_net = None
예제 #9
0
parser.add_argument('--quantiles',
                    nargs='+',
                    help='quantiles',
                    type=float,
                    required=True)
parser.add_argument('--device',
                    default="cpu",
                    type=str,
                    required=False,
                    help='compute device')
parser.add_argument('--debug',
                    default=False,
                    dest='debug',
                    action='store_true')
args = parser.parse_args()
init_log(args)
torch.set_num_threads(hyperparams.default_threads)
my_print("Parameters:")
for k, v in sorted(vars(args).items()):
    my_print("\t{0}: {1}".format(k, v))

# Set the random seed manually for reproducibility.
torch.manual_seed(hyperparams.random_seed)

###############################################################################
# Load data
###############################################################################
if torch.cuda.is_available():
    if args.device != "cpu":
        torch.cuda.set_device(int(args.device.split(":")[1]))
        args.device = torch.device(args.device)
예제 #10
0
def testfetch():
    fetcher = LogFetcher()
    for ip in config.serverlist:
        fetcher.add_hosthandler(
            YesterdayDownloadHostHandler(
                ip, '55666',
                'http://%s:%s/archive/flashserver_%s.tar.gz',
                config.archivedir,
                'flashserver_%s.tar.gz'
            )
        )

    fetcher.fetchall()


def testparse():
    parser = YesterdayTarsLogParser(config.workbench, config.archivedir)
    h_lost = ClassPacketLostHandler()
    h_dis = ClassDisConnHandler()
    parser.addloghandler(h_lost)
    parser.addloghandler(h_dis)
    parser.work()


if __name__ == '__main__':
    util.init_log('mon_supervisor.%d.log' % os.getpid())
    supervisor()
    #testfetch()
    #testparse()
예제 #11
0
#!/usr/bin/env python
# -*- coding:utf-8 -*-

import sys
reload(sys)
sys.setdefaultencoding("utf-8")
import os
import io
import logging.config

# constants
ROOT_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
STORE_PATH = os.path.join(ROOT_PATH, 'stores')
INI_PATH = os.path.join(ROOT_PATH, 'ini')
LOG_INI_PATH = os.path.join(INI_PATH, 'log.ini')
MAIN_INI_PATH = os.path.join(INI_PATH, 'myhouse.ini')

import util
logger, root_logger = util.init_log(LOG_INI_PATH)
예제 #12
0
def main(tracefile):
    with open(tracefile) as f:
        mined_trees = json.load(f)
    util.init_log('generalize_loop', '', mined_trees[0]['original'])
    gmethod_trees = generalize_loop_trees(mined_trees)
    print(json.dumps(gmethod_trees, indent=4))
예제 #13
0
def main(**args):
    logger = init_log(args['log_level'], name=__name__)
    # Check if *experiment_path* contains all the necessary files to retrain an evolved model
    check_files(args['experiment_path'])

    # Get all parameters
    logger.info(f"Getting parameters from evolution ...")
    config = cfg.ConfigParameters(args, phase='retrain')
    config.get_parameters()

    # Load log file and read it at the specified generation
    s = f"last generation" if args['generation'] is None else f"generation {args['generation']}"
    logger.info(f"Loading {config.files_spec['data_file']} file to get {s}, individual "
                f"{args['individual']} ...")
    config.load_evolved_data(generation=args['generation'],
                             individual=args['individual'])

    if args['lr_schedule'] is not None:
        special_params = train.train_schemes_map[args['lr_schedule']].get_params()
        logger.info(f"Overriding train parameters to use special scheme "
                    f"'{args['lr_schedule']}' ...")
        config.override_train_params(special_params)

    # It is important to merge the dicts with the evolved_params first, as they need to be
    # overwritten in case we are using one of the special train schemes.
    train_params = {**config.evolved_params['params'], **config.train_spec}
    best_ind_tese = ['conv_5_1_512', 'conv_3_1_128', 'conv_3_1_512',
                     'conv_5_1_256',
                     'avg_pool_2_2',
                     'conv_3_1_256',
                     'avg_pool_2_2',
                     'conv_5_1_128',
                     'avg_pool_2_2',
                     'max_pool_2_2']

    # best_ind_tese = ['bv1p_3_1_128',
    #                  'bv1p_3_1_128',
    #                  'bv1p_3_1_256',
    #                  'avg_pool_2_2',
    #                  'no_op',
    #                  'bv1p_3_1_256',
    #                  'no_op',
    #                  'no_op',
    #                  'no_op',
    #                  'max_pool_2_2',
    #                  'max_pool_2_2',
    #                  'bv1_3_1_128',
    #                  'bv1_3_1_64',
    #                  'bv1p_3_1_256',
    #                  'no_op',
    #                  'bv1_3_1_256',
    #                  'max_pool_2_2',
    #                  'bv1_3_1_256',
    #                  'bv1p_3_1_64',
    #                  'no_op'
    #                  ]
    config.evolved_params['net'] = best_ind_tese
    logger.info(f"Starting training of model {config.evolved_params['net']}")
    valid_acc, test_info = train.train_and_eval(data_info=config.data_info,
                                                params=train_params,
                                                fn_dict=config.fn_dict,
                                                net_list=config.evolved_params['net'],
                                                lr_schedule=args['lr_schedule'],
                                                run_train_eval=args['run_train_eval'])
    logger.info(f"Saving parameters...")
    config.save_params_logfile()

    logger.info(f"Best accuracy in validation set: {valid_acc:.5f}")
    logger.info(f"Final test accuracy: {test_info['accuracy']:.5f}")
    logger.info(f"Final test confusion matrix:\n{test_info['confusion_matrix']}")
예제 #14
0
            optimizer.zero_grad()
            loss = criterion(feature_vector, label_batch)
            if n_item % 50 == 1:
                sw.add_scalar('loss_curve', loss, n_item)
            loss.backward()
            optimizer.step()
            n_item = n_item + 1
        if epoch % cycle_epoches_for_test == 1:
            acc_now = evaluate(net, test_dataloader)
            sw.add_scalar('acc_curve', acc_now, epoch)
            logging.info("epoch:{}\tn_item:{}\tacc:{:.6f}".format(epoch, n_item, acc_now))

            if acc_now > best_acc:
                state = {
                    'net': net.state_dict(),
                    'acc': acc_now,
                    'epoch': epoch,
                    "optimizer": optimizer.state_dict(),
                    'n_item': n_item,
                }
                if not os.path.exists(pth_dir_path):
                    os.mkdir(pth_dir_path)
                torch.save(state, pth_path)
    sw.close()


if __name__ == '__main__':
    output_files_dir = r"/data/cifar/10"
    init_log(os.path.join(output_files_dir,'experiment_result.log'))
    run(output_files_dir)