def get_params(test_config):
"""get params and save them to root dir"""
prm = Parameters()
# get giles paths
prm.override(test_config)
test_parameter_file = os.path.join(prm.train.train_control.ROOT_DIR,
'test_parameters.ini')
log_file = os.path.join(prm.train.train_control.ROOT_DIR, 'test.log')
ret = True
if os.path.isfile(test_parameter_file):
warnings.warn('Test parameter file {} already exists'.format(
test_parameter_file))
ret = query_yes_no('Overwrite parameter file?')
if ret:
dir = os.path.dirname(test_parameter_file)
if not os.path.exists(dir):
os.makedirs(dir)
prm.save(test_parameter_file)
logging = logging_config(log_file)
logging.disable(logging.DEBUG)
return prm
def get_params(test_config):
"""get params and save them to root dir"""
prm = Parameters()
# get giles paths
prm.override(test_config) # just to get the LOG_DIR_LIST[0]
train_log_dir = prm.test.ensemble.LOG_DIR_LIST[0]
parameter_file = os.path.join(train_log_dir, 'parameters.ini')
test_parameter_file = os.path.join(prm.train.train_control.ROOT_DIR,
'test_parameters.ini')
all_parameter_file = os.path.join(prm.train.train_control.ROOT_DIR,
'all_parameters.ini')
log_file = os.path.join(prm.train.train_control.ROOT_DIR, 'test.log')
if not os.path.isfile(parameter_file):
raise AssertionError('Can not find file: {}'.format(parameter_file))
ret = True
if os.path.isfile(test_parameter_file):
warnings.warn('Test parameter file {} already exists'.format(
test_parameter_file))
ret = query_yes_no('Overwrite parameter file?')
if ret:
dir = os.path.dirname(test_parameter_file)
if not os.path.exists(dir):
os.makedirs(dir)
prm.save(test_parameter_file)
logging = logging_config(log_file)
logging.disable(logging.DEBUG)
# Done saving test parameters. Now doing the integration:
prm = Parameters()
prm.override(parameter_file)
prm.override(test_parameter_file)
ret = True
if os.path.isfile(all_parameter_file):
warnings.warn(
'All parameter file {} already exists'.format(all_parameter_file))
ret = query_yes_no('Overwrite parameter file?')
if ret:
dir = os.path.dirname(all_parameter_file)
if not os.path.exists(dir):
os.makedirs(dir)
prm.save(all_parameter_file)
return prm
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from lib.active_kmean import KMeansWrapper
from sklearn.datasets import make_blobs
logging = logging_config()
logging.disable(logging.DEBUG)
log = logger.get_logger('main')
prm_file = '/data/gilad/logs/log_2210_220817_wrn-fc2_kmeans_SGD_init_200_clusters_4_cap_204/parameters.ini'
prm = Parameters()
prm.override(prm_file)
dev = prm.network.DEVICE
factories = Factories(prm)
model = factories.get_model()
model.print_stats() # debug
preprocessor = factories.get_preprocessor()
preprocessor.print_stats() # debug
train_dataset = factories.get_train_dataset(preprocessor)
validation_dataset = factories.get_validation_dataset(preprocessor)
dataset_wrapper = DatasetWrapper(prm.dataset.DATASET_NAME + '_wrapper', prm, train_dataset, validation_dataset)
def get_params(test_config, parser_args=None):
"""get params and save them to root dir"""
# Just to get the ROOT_DIR and save prm test_config
prm = Parameters()
prm.override(test_config)
# get manual test parameters from config:
if parser_args is not None:
# overriding some parameters manually from parser:
prm.train.train_control.ROOT_DIR = parser_args.ROOT_DIR
prm.train.train_control.TEST_DIR = parser_args.ROOT_DIR + '/test'
prm.train.train_control.PREDICTION_DIR = parser_args.ROOT_DIR + '/prediction'
prm.train.train_control.CHECKPOINT_DIR = parser_args.ROOT_DIR + '/checkpoint'
prm.test.test_control.KNN_WEIGHTS = parser_args.KNN_WEIGHTS
prm.test.test_control.KNN_NORM = parser_args.KNN_NORM
prm.train.train_control.PCA_REDUCTION = (
parser_args.PCA_REDUCTION == 'True')
prm.train.train_control.PCA_EMBEDDING_DIMS = int(
parser_args.PCA_EMBEDDING_DIMS)
prm.test.test_control.KNN_NEIGHBORS = int(parser_args.KNN_NEIGHBORS)
prm.test.test_control.DUMP_NET = (parser_args.DUMP_NET == 'True')
prm.test.test_control.LOAD_FROM_DISK = (
parser_args.LOAD_FROM_DISK == 'True')
ROOT_DIR = prm.train.train_control.ROOT_DIR
# get time stamp
ts = get_timestamp()
# get files paths
parameter_file = os.path.join(ROOT_DIR, 'parameters.ini')
test_parameter_file = os.path.join(ROOT_DIR,
'test_parameters_' + ts + '.ini')
all_parameter_file = os.path.join(ROOT_DIR,
'all_parameters_' + ts + '.ini')
log_file = os.path.join(ROOT_DIR, 'test_' + ts + '.log')
logging = logging_config(log_file)
logging.disable(logging.DEBUG)
if not os.path.isfile(parameter_file):
raise AssertionError('Can not find file: {}'.format(parameter_file))
dir = os.path.dirname(test_parameter_file)
if not os.path.exists(dir):
os.makedirs(dir)
prm.save(test_parameter_file)
# Done saving test parameters. Now doing the integration:
prm = Parameters()
prm.override(parameter_file)
prm.override(test_parameter_file)
if parser_args is not None:
# overriding some parameters manually from parser:
prm.train.train_control.ROOT_DIR = parser_args.ROOT_DIR
prm.train.train_control.TEST_DIR = parser_args.ROOT_DIR + '/test'
prm.train.train_control.PREDICTION_DIR = parser_args.ROOT_DIR + '/prediction'
prm.train.train_control.CHECKPOINT_DIR = parser_args.ROOT_DIR + '/checkpoint'
prm.test.test_control.KNN_WEIGHTS = parser_args.KNN_WEIGHTS
prm.test.test_control.KNN_NORM = parser_args.KNN_NORM
prm.train.train_control.PCA_REDUCTION = (
parser_args.PCA_REDUCTION == 'True')
prm.train.train_control.PCA_EMBEDDING_DIMS = int(
parser_args.PCA_EMBEDDING_DIMS)
prm.test.test_control.KNN_NEIGHBORS = int(parser_args.KNN_NEIGHBORS)
prm.test.test_control.DUMP_NET = (parser_args.DUMP_NET == 'True')
prm.test.test_control.LOAD_FROM_DISK = (
parser_args.LOAD_FROM_DISK == 'True')
dir = os.path.dirname(all_parameter_file)
if not os.path.exists(dir):
os.makedirs(dir)
prm.save(all_parameter_file)
return prm