Esempio n. 1
0
def main():
    parse_args(config)
    set_logger(config, logger)
    if config['daemon']:
        daemon()
    pid_file = config['pidfile']
    mk_pid_file(pid_file)
    run_server(config)
    # Load the parameters from json file
    args = parser.parse_args()
    json_path = os.path.join(args.model_dir, 'params.json')
    assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
    params = utils.Params(json_path)

    # use GPU if available
    params.cuda = torch.cuda.is_available()

    # Set the random seed for reproducible experiments
    random.seed(230)
    torch.manual_seed(230)
    if params.cuda: torch.cuda.manual_seed(230)

    # Set the logger
    utils.set_logger(os.path.join(args.model_dir, 'train.log'))

    # Create the input data pipeline
    logging.info("Loading the datasets...")

    # fetch dataloaders, considering full-set vs. sub-set scenarios
    if params.subset_percent < 1.0:
        train_dl = data_loader.fetch_subset_dataloader('train', params)
    else:
        train_dl = data_loader.fetch_dataloader('train', params)
    
    dev_dl = data_loader.fetch_dataloader('dev', params)

    logging.info("- done.")

    """Based on the model_version, determine model/optimizer and KD training mode
Esempio n. 3
0
File: zomphp.py Progetto: wk8/ZomPhP
def main():
    # argument processing
    parser = argparse.ArgumentParser(description='Detect your PHP dead code')
    parser.add_argument('--dir', dest='dir', metavar='dir_path',
                        type=str, nargs=1, default=None,
                        help='Make ZomPHP process that directory')
    parser.add_argument('--ignore-sub-dirs', dest='ignore_sub_dirs',
                        metavar='dir_path', type=str, nargs='+', default=[],
                        help='A directory path (or list of those) that won\'t '
                        'be processed (only makes sense when used with the '
                        '--dir option)')
    parser.add_argument('--files', dest='files', metavar='file_path',
                        type=str, nargs='+', default=[],
                        help='A file or list of files (given as absolute paths'
                        ') to have processed by ZomPHP')
    parser.add_argument('--strict', dest='strict', action='store_const',
                        const=True, default=False, help='If set to true, will'
                        ' guarantee that any function NOT marked is indeed '
                        'used, but might also yield more false negatives (this'
                        ' option should only be used if you have files '
                        'containing functions with the same name)')
    parser.add_argument('--path-translation', dest='path_translation',
                        metavar='local_path path_in_db', type=str, nargs='+',
                        default=[], help='A list of couples of paths to '
                        'translate (useful if running the code in a different '
                        'location than the one running the PHP code)')
    parser.add_argument('--logging-level', dest='logging_level', metavar='level',
                        type=str, nargs=1, default=None, help='A logging '
                        'level to override the one set in the settings file')
    args = parser.parse_args()

    # start the logger
    utils.set_logger()

    # some sanity checks
    def check_abs_path(path, option_name):
        # helper function, checks the paths are absolute, and translates them to real paths
        if not path:
            return path
        if isinstance(path, (tuple, list)):
            return [check_abs_path(p, option_name) for p in path]
        if os.path.isabs(path):
            return os.path.realpath(path)
        logging.error('The --%s option requires using absolute paths (you entered %s) exiting' % (option_name, path))
        sys.exit(1)
    if bool(args.dir) == bool(args.files):
        logging.error('You must specify exactly one of the --dir or --files options, exiting')
        sys.exit(1)
    args.dir = check_abs_path(args.dir, 'dir')
    args.files = check_abs_path(args.files, 'files')
    if args.ignore_sub_dirs:
        if args.dir:
            args.ignore_sub_dirs = check_abs_path(args.ignore_sub_dirs, 'ignore-sub-dirs')
        else:
            logging.warning('Ignoring the --ignore-sub-dirs option, that option can only be used together with the --dir option')
    translator = utils.PathTranslator.build_translator(args.path_translation)

    # down to work!
    bckend = backend.get_new_backend()

    if args.dir:
        bckend.process_directory(args.dir[0], strict=args.strict, ignore_sub_dirs=args.ignore_sub_dirs, translator=translator)
    else:
        # then it must be --files
        for fle in args.files:
            bckend.process_file(fle, args.strict, translator=translator)

    logging.info(bckend.stats)
Esempio n. 4
0
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.cross_validation import train_test_split
from sklearn import linear_model, datasets
from sklearn import metrics
from sklearn import preprocessing
from sklearn.cross_validation import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from pandas import *

from settings import DATA_DIR, LOG_DIR
import utils

TOTAL_INDEX = 'djia gspc ixic'.split() # dow jones. snp500, nasdaq, vol
EXPID = utils.get_expid()
utils.set_logger('%s/%s.log' % (LOG_DIR, EXPID), 'DEBUG')
# TODO: log configurations (ex: parsing method etc) and/or commit id


def openfiles(filename, arg):

    data = pd.read_csv(filename, sep='\t', header = 0)
    data = data.where((pd.notnull(data)), '')   # Replace np.nan with ''
    if arg == 100: # X
        columns = ['id', 'text', 'closePrice', 'week', 'month', 'quater', 'year','djia', 'gspc', 'ixic', 'vix']
        data.columns = columns
        value = pd.DataFrame(data)
        value.index = data['id']
    else: # y
        columns = TOTAL_INDEX[arg]
        value = pd.DataFrame(data[TOTAL_INDEX[arg]])
    """
    # Load the parameters
    args = parser.parse_args()
    json_path = os.path.join(args.model_dir, 'params.json')
    assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
    params = utils.Params(json_path)

    # use GPU if available
    params.cuda = torch.cuda.is_available()     # use GPU is available

    # Set the random seed for reproducible experiments
    torch.manual_seed(230)
    if params.cuda: torch.cuda.manual_seed(230)
        
    # Get the logger
    utils.set_logger(os.path.join(args.model_dir, 'analysis.log'))

    # Create the input data pipeline
    logging.info("Loading the dataset...")

    # fetch dataloaders
    # train_dl = data_loader.fetch_dataloader('train', params)
    # dev_dl = data_loader.fetch_dataloader('dev', params)
    dataloader = data_loader.fetch_dataloader(args.dataset, params)

    logging.info("- done.")

    # Define the model graph
    model = resnet.ResNet18().cuda() if params.cuda else resnet.ResNet18()

    # fetch loss function and metrics
    # Launch training with this config
    cmd = "{python} train.py --model_dir={model_dir}".format(python=PYTHON,
                                                             model_dir=model_dir)
    print(cmd)
    check_call(cmd, shell=True)


if __name__ == "__main__":
    # Load the "reference" parameters from parent_dir json file
    args = parser.parse_args()
    json_path = os.path.join(args.parent_dir, 'params.json')
    assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
    params = utils.Params(json_path)

    # Set the logger
    utils.set_logger(os.path.join(args.parent_dir, 'search_hyperparameters.log'))

    '''
    Temperature and alpha search for KD on CNN (teacher model picked in params.json)
    Perform hypersearch (empirical grid): distilling 'temperature', loss weight 'alpha'
    '''

    # hyperparameters for KD
    alphas = [0.99, 0.95, 0.5, 0.1, 0.05]
    temperatures = [20., 10., 8., 6., 4.5, 3., 2., 1.5]

    logging.info("Searching hyperparameters...")
    logging.info("alphas: {}".format(alphas))
    logging.info("temperatures: {}".format(temperatures))

    for alpha in alphas: