def sample_from_model(kwargs, model, size, x):
K.clear_session()
log = logging.getLogger("sample_from_model")
try:
model_folder = resolve_dir(kwargs['model_dir'])
log.info(
"sampling with parameters, conf: %s, kwargs: %s, model_folder: %s,"
"size: %d", conf, kwargs, model_folder, size)
device = get_device()
log.info("device to be used: %s", device)
log.info("model: %s, x.shape: %s", model, x.shape)
with tf.device(device):
model = rm_utils.load_model_and_params(
os.path.join(model_folder, "best_model"))
log.info("model loaded")
samples = model.sample(size, tf.convert_to_tensor(x))
log.info("model samples.shape: %s", samples.shape)
return samples
except:
error_msg = traceback.format_exc()
log.error("Sampling failed: %s", error_msg)
finally:
log.info("Sampling completed")
def generate_data(self):
data_train, data_validate, data_test = self.load_data_no_discrete_normalised_as_array(
resolve_dir(self.file))
self._train_size = data_train.shape[0]
self._validate_size = data_validate.shape[0]
self._test_size = data_test.shape[0]
return np.r_[data_train, data_validate, data_test]
def predict_from_model(y, x, kwargs, op_names, model):
K.clear_session()
log = logging.getLogger("predict_from_model")
try:
model_folder = resolve_dir(kwargs['model_dir'])
device = get_device()
log.info(
"using dev %s, kwargs: %s, conf: %s, model: %s, model_folder: %s, op_names: %s",
device, kwargs, conf, model, model_folder, op_names)
log.info("date received y.shape: %s, x.shape: %s", y.shape, x.shape)
with tf.device(device):
model = rm_utils.load_model_and_params(
os.path.join(model_folder, "best_model"))
marginals = kwargs['marginals']
log.info("preparing dataset start")
dataset = tf.data.Dataset.from_tensor_slices((y, x))
dataset = dataset.repeat(1)
dataset = dataset.prefetch(3 * conf.eval_batch_size)
dataset = dataset.batch(conf.eval_batch_size)
log.info("preparing dataset completed")
collector = rm_utils.InMemoryCollector()
log.info("Starting collecting prediction")
for i, (y, x) in enumerate(dataset):
results = {}
for op_name in op_names:
res = None
log.info(
"Running at %d, op_name: %s, y.shape: %s, x.shape: %s", i,
op_name, y.shape, x.shape)
if op_name == "log_likelihood":
ll = model.log_prob(y, x, marginals=marginals)
log.info(
"log_prob completed with %s",
"list of len: {}, shapes: {}".format(
len(ll), ", ".join([str(l.shape) for l in ll])) if
isinstance(ll, list) else "array of shape: {}".format(
str(ll.shape)))
if isinstance(ll, list):
ll = np.concatenate(ll, axis=1)
log.info("concatenate completed")
results["log_likelihood"] = ll
else:
raise ValueError(
"not recognized op name: {}".format(op_name))
collector.collect(results)
log.info("collect completed")
return collector.result()
except:
error_msg = traceback.format_exc()
log.error("Error in predict: %s", error_msg)
def load_estimator(self, model_dir, params, **kwargs):
estimator = TfEstimator(self.model,
resolve_dir(model_dir),
params,
x_size=kwargs["x_size"],
y_size=kwargs["y_size"])
estimator.restore(not_exists_ok=False if 'not_exists_ok' not in
kwargs else kwargs['not_exists_ok'])
return estimator
def generate_data(self):
f = h5py.File(resolve_dir(self.file), 'r')
data_train = f['train']
data_validate=f['validation']
data_test=f['test']
self._train_size = data_train.shape[0]
self._validate_size = data_validate.shape[0]
self._test_size = data_test.shape[0]
return np.r_[data_train, data_validate, data_test]
def __call__(self, *varg, **kwargs):
from models.tensorboard import Tensorboard
conf.visible_device_list = self.conf_override.visible_device_list
model_dir = self.model_dir(**kwargs)
resolved_dir = resolve_dir(model_dir)
self.log = logging.getLogger(self.__class__.__name__)
stats_model_dir = resolve_dir(os.path.join(model_dir, "stats"))
tb = Tensorboard(stats_model_dir)
try:
os.makedirs(stats_model_dir, exist_ok=True)
tb.log_dict_as_table("conf", conf.__dict__)
tb.log_dict_as_table("model_conf", conf.__dict__)
tb.log_dict_as_table("early_stopping",
self.early_stopping.__dict__)
model_dir, train_stats, validation_stats, test_stats = self.call_me(
model_dir, *varg, **kwargs)
tb.log_dict_as_table("train_stats", train_stats)
tb.log_dict_as_table("validation_stats", validation_stats)
tb.log_dict_as_table("test_stats", test_stats)
return model_dir, train_stats, validation_stats, test_stats
except Exception:
error_msg = traceback.format_exc()
self.log.error(error_msg)
tb.log_dict_as_table("message", {'error': error_msg})
self.log_error(kwargs, resolved_dir)
return None, None, None, None
finally:
tb.close()
def call_me(self, model_dir, *varg, **kwargs):
resolved_model_dir = resolve_dir(model_dir)
kwargs['model_dir'] = resolved_model_dir
self.data_loader.load_data()
self.log.info("About to train the model")
create_model_and_train(kwargs, resolved_model_dir, self.data_loader,
self.model.name(), self.early_stopping)
self.log.info("Train done")
train = create_model_and_validate(kwargs,
resolved_model_dir, self.data_loader,
self.model.name(), "train")
valid = create_model_and_validate(kwargs,
resolved_model_dir, self.data_loader,
self.model.name(), "valid")
test = create_model_and_validate(kwargs,
resolved_model_dir, self.data_loader,
self.model.name(), "test")
res_train = {}
if train is not None:
res_train['ll_mean'] = train[0]
res_train['ll_se'] = train[1]
else:
res_train['ll_mean'] = np.nan
res_train['ll_se'] = np.nan
res_valid = {}
if valid is not None:
res_valid['ll_mean'] = valid[0]
res_valid['ll_se'] = valid[1]
else:
res_valid['ll_mean'] = np.nan
res_valid['ll_se'] = np.nan
res_test = {}
if test is not None:
res_test['ll_mean'] = test[0]
res_test['ll_se'] = test[1]
else:
res_test['ll_mean'] = np.nan
res_test['ll_se'] = np.nan
self.log.info("Training and validation complete")
return model_dir, res_train, res_valid, res_test
def generate_data(self):
data = pd.read_pickle(resolve_dir(self.file))
data.drop("Meth", axis=1, inplace=True)
data.drop("Eth", axis=1, inplace=True)
data.drop("Time", axis=1, inplace=True)
B = self.get_correlation_numbers(data)
while np.any(B > 1):
col_to_remove = np.where(B > 1)[0][0]
col_name = data.columns[col_to_remove]
data.drop(col_name, axis=1, inplace=True)
B = self.get_correlation_numbers(data)
# print(data.corr())
data = (data - data.mean()) / data.std()
return data.as_matrix()
def save_best_model_exp(name, opt):
with open(
os.path.join(gen_utils.resolve_dir(conf.dir),
experiment_file(name)), 'w') as f:
data = OrderedDict()
data["model_factory"] = opt.model_factory.state
data["data_set_info"] = opt.data_set_info
data["best_model_valid_ll"] = opt.best_model_valid_ll
data["best_model_train_ll"] = opt.best_model_train_ll
data["best_model_test_ll"] = opt.best_model_test_ll
data["best_model_params"] = opt.best_model_params
data["best_model_dir"] = opt.best_model_dir
data["hyper_param_search_results"] = opt.hyper_param_search.results
data["hyper_param_search"] = opt.hyper_param_search.state
data["conf"] = conf.__dict__
json.dump(data, f, cls=NumpyEncoder, indent=4)
def __init__(self, experiment_name, persist_load_exp=True):
self._data_loader = None
self._model_factory = None
self._hyper_param_search = None
self._valid_batch_size = None
self._early_stopping = None
self._persist_load_exp = persist_load_exp
self.best_model_valid_ll = None
self.best_model_train_ll = None
self.best_model_test_ll = None
self.best_model_params = None
self.best_model_dir = None
self.data_set_info = None
conf.dir = os.path.join('{ROOT}', experiment_name)
resolved_dir = resolve_dir(conf.dir)
os.makedirs(resolved_dir, exist_ok=True)
self.log = None
def compute_mi(x, params, kwargs, y_size):
model_folder = resolve_dir(params['model_dir'])
all_y_vars = ["y%d" % i for i in range(y_size)]
mi = np.full((y_size, y_size), np.nan, dtype=np.float32)
def run_for_combination(y_i, y_j):
device = get_device()
def prob(**kwargs):
y_list = [kwargs[y_var] for y_var in all_y_vars]
y = tf.concat(y_list, axis=-1)
x_broadcasted = tf.broadcast_to(
x, [tf.shape(y)[0], tf.shape(x)[-1]])
return model.prob(y, x=x_broadcasted, training=False)
with tf.device(device):
model = utils.load_model_and_params(
os.path.join(model_folder, "best_model"))
integrate_out = [
"y%d" % i for i in set(range(y_size)) - set([y_i, y_j])
]
return tf_utils.mi(prob,
var1="y%d" % y_i,
var2="y%d" % y_j,
integrate_out=integrate_out,
**kwargs)
all_combinations = [(y_i, y_j) for y_i in range(y_size)
for y_j in range(y_i + 1, y_size)]
with multiprocessing.Pool(maxtasksperchild=1) as pool:
mis = pool.starmap(run_for_combination, all_combinations)
for i, (y_i, y_j) in enumerate(all_combinations):
mi[y_i][y_j] = mis[i]
return mi
def generate_data(self):
rng = np.random.RandomState(42)
data = np.load(resolve_dir(self.file))
rng.shuffle(data)
N = data.shape[0]
data = np.delete(data, 3, axis=1)
data = np.delete(data, 1, axis=1)
############################
# Add noise
############################
# global_intensity_noise = 0.1*rng.rand(N, 1)
voltage_noise = 0.01 * rng.rand(N, 1)
# grp_noise = 0.001*rng.rand(N, 1)
gap_noise = 0.001 * rng.rand(N, 1)
sm_noise = rng.rand(N, 3)
time_noise = np.zeros((N, 1))
# noise = np.hstack((gap_noise, grp_noise, voltage_noise, global_intensity_noise, sm_noise, time_noise))
# noise = np.hstack((gap_noise, grp_noise, voltage_noise, sm_noise, time_noise))
noise = np.hstack((gap_noise, voltage_noise, sm_noise, time_noise))
data = data + noise
return data
def generate_data(self):
data = np.load(resolve_dir(self.file))
return data
def data_file(self, type, ext):
return os.path.join(resolve_dir(conf.dir), self._file_name(type, ext))
def load(self, complain_on_diff=False, model_name=None):
if not self._persist_load_exp:
raise IOError()
name = model_name if model_name is not None else self.model_factory.model_name
with open(os.path.join(resolve_dir(conf.dir), experiment_file(name)), 'r') as f:
data = json.load(f)
self.best_model_valid_ll = data["best_model_valid_ll"]
self.best_model_train_ll = data["best_model_train_ll"]
self.best_model_test_ll = data["best_model_test_ll"]
self.best_model_params = data["best_model_params"]
self.best_model_dir = data["best_model_dir"]
self.data_set_info = data["data_set_info"]
hyper_param_search_results = data["hyper_param_search_results"]
hyper_param_search_state = data["hyper_param_search"]
model_factory_state = data["model_factory"]
for i, v in enumerate(hyper_param_search_state["space"]):
if 'bounds' in v and isinstance(v['bounds'][0], list):
elements = v['bounds']
new_elements = []
for element in elements:
new_elements.append(tuple(element))
v['bounds'] = new_elements
if complain_on_diff:
affecting_experiment = conf.values_affecting_experiment()
if affecting_experiment != {key: value for key, value in data["conf"].items() if
key in affecting_experiment}:
raise ValueError("Running model factory conf different than loaded one")
# num workers can change
affecting_experiment = conf.values_affecting_experiment()
if affecting_experiment != {key: value for key, value in data["conf"].items() if
key in affecting_experiment}:
raise ValueError("Running conf different than loaded one")
if self.hyper_param_search.state != hyper_param_search_state:
raise ValueError("hyper_param_search")
if self.model_factory.state != model_factory_state:
raise ValueError("model_factory")
if self.data_loader.state != self.data_set_info:
raise ValueError("data_loader")
if self.hyper_param_search is None:
self.hyper_param_search = HyperParamSearch.from_state(hyper_param_search_state)
if self.model_factory is None:
self.model_factory = TrainEvalModelFactory.from_state(model_factory_state)
if self.data_loader is None:
data_loader = DataLoader.from_state(self.data_set_info)
data_loader.load_from_file()
self.data_loader = data_loader
for hyper_param_step in hyper_param_search_results:
self.hyper_param_search.tell(
tuple([tuple(hyper_param_step['x'][dim.name]) if isinstance(hyper_param_step['x'][dim.name], list)
else hyper_param_step['x'][dim.name] for dim in self.hyper_param_search.space]),
hyper_param_step['train'],
hyper_param_step['validation'],
hyper_param_step['test'],
hyper_param_step['model_dir'] if 'model_dir' in hyper_param_step else "")
def run(self):
init_logging(os.path.join(resolve_dir(conf.dir), "output_{}.log".format(self.model_factory.model.name())))
self.log = logging.getLogger("experiment")
try:
self.load(complain_on_diff=True)
print("Loaded %s" % os.path.join(resolve_dir(conf.dir), experiment_file(self.model_factory.model_name)))
except IOError:
print("Clean run")
pass
futures = []
if conf.print_progress:
progress_mon = ProgressMonitor(self.hyper_param_search.num_samples, "{model}/{data_set}".
format(model=self.model_factory.model_name, data_set=self.data_set_name()))
else:
progress_mon = NoOpProgressMonitor()
done = self.hyper_param_search.done
progress_mon.progress(done)
device_assignment = {device: 0 for device in conf.visible_device_list}
self.data_loader.free()
tasks = []
try:
while True:
x = self.hyper_param_search.ask()
objective_fun = self.model_factory.create_train_eval(self.data_loader,
self.hyper_param_search.space_names,
self.early_stopping)
objective_fun.conf_override = deepcopy(conf)
args_named = self.hyper_param_search.to_named_params(x)
tasks.append(WorkItem(objective_fun, x, args_named, None))
except StopIteration:
pass
with (SameProcessExecutor() if conf.num_workers <= 0 else concurrent.futures.ProcessPoolExecutor(
conf.num_workers)) as executor:
task_id = 0
submitted_during_round = False
while len(futures) > 0 or len(tasks) > 0:
if task_id == 0:
submitted_during_round = False
#bring tasks that are restricted wrt device to the front of the queue
for i in range(len(tasks)):
task = tasks[i]
if np.any([re.search(pattern, task.name) for pattern, dev in conf.device_placement_mapping]):
del tasks[i]
tasks.insert(0, task)
made_round = task_id == len(tasks)
submit = False
if (len(futures) < conf.num_workers or conf.num_workers <= 0) and len(tasks) > 0:
task_id = task_id % len(tasks)
next_wi = tasks[task_id] # x is a list of n_points points
allowed_devices = list(device_assignment.keys())
if len(device_assignment) > 0:
for pattern, dev in conf.device_placement_mapping:
if re.search(pattern, next_wi.name):
allowed_devices = [dev]
break
allowed_device_assignment = {k: v for k, v in device_assignment.items() if
k in allowed_devices}
if len(allowed_device_assignment) > 0:
device = sorted([(t[1], t[0]) for t in list(allowed_device_assignment.items())])[0][1]
device_assignment[device] += 1
next_wi.objective_fun.conf_override.visible_device_list = [device]
submit = True
else:
next_wi.objective_fun.conf_override.visible_device_list = []
submit = True
if submit:
time.sleep(10)
submitted_during_round = True
next_wi.future = executor.submit(next_wi.objective_fun, **next_wi.args_named)
futures.append(next_wi)
del tasks[task_id]
else:
task_id += 1
for wi in list(futures):
try:
model_dir, train_eval, validation_eval, test_eval = wi.future.result(0)
if len(device_assignment) > 0:
device_assignment[wi.objective_fun.conf_override.visible_device_list[0]] -= 1
if abs(device_assignment[wi.objective_fun.conf_override.visible_device_list[0]]) < 1e-2:
device_assignment[wi.objective_fun.conf_override.visible_device_list[0]] = 0
self.train_eval_task_finished(
futures, wi, model_dir, train_eval, validation_eval, test_eval)
done += 1
progress_mon.progress(done)
except concurrent.futures.TimeoutError:
pass
if (len(futures) != 0 and len(futures) == conf.num_workers) or (
made_round and not submitted_during_round):
time.sleep(5)
for wi in list(futures):
model_dir, train_eval, validation_eval, test_eval = wi.future.result()
self.train_eval_task_finished(futures, wi, model_dir, train_eval, validation_eval, test_eval)
done += 1
progress_mon.progress(done)
self.save()
def load_data(self, generator_fn=None):
try:
self.load_from_file()
print("loaded data: %s" % self._file_name("data", "memmap"))
except IOError:
os.makedirs(resolve_dir(conf.dir), exist_ok=True)
data = self.generate_data(
) if generator_fn is None else generator_fn()
data = data.astype(getattr(np, "float%s" % conf.precision))
if self.uniqueness_threshold is not None:
cat_columns = []
for i in range(data.shape[1]):
uniqueness = len(np.unique(data[:, i])) / len(data[:, i])
if uniqueness < self.uniqueness_threshold:
cat_columns.append(i)
print("cat column: %d" % i)
non_cat_columns = [
i for i in range(data.shape[1]) if i not in cat_columns
]
data = data[:, non_cat_columns]
if self.y_transforms is not None:
for y_transform in self.y_transforms:
y_transform.transform(data)
self._data = np.memmap(self.data_file('data', "memmap"),
dtype=getattr(np, "float" + conf.precision),
mode="w+",
shape=data.shape)
self._data[:] = data[:]
with open(self.data_file('data', "json"), 'w') as out:
out.write(
json.dumps(
{
'dtype': "float" + conf.precision,
'shape': data.shape
},
indent=4))
del data
train, validation, test = self.sample_cross_validation()
self._train_data = np.memmap(self.data_file('train', "memmap"),
dtype=getattr(
np, "float" + conf.precision),
mode="w+",
shape=train.shape)
self._train_data[:] = train[:]
with open(self.data_file('train', "json"), 'w') as out:
out.write(
json.dumps(
{
'dtype': "float" + conf.precision,
'shape': train.shape
},
indent=4))
del train
self._validation_data = np.memmap(
self.data_file('validation', "memmap"),
dtype=getattr(np, "float" + conf.precision),
mode="w+",
shape=validation.shape)
self._validation_data[:] = validation[:]
with open(self.data_file('validation', "json"), 'w') as out:
out.write(
json.dumps(
{
'dtype': "float" + conf.precision,
'shape': validation.shape
},
indent=4))
del validation
self._test_data = np.memmap(self.data_file('test', "memmap"),
dtype=getattr(np, "float" +
conf.precision),
mode="w+",
shape=test.shape)
self._test_data[:] = test[:]
with open(self.data_file('test', "json"), 'w') as out:
out.write(
json.dumps(
{
'dtype': "float" + conf.precision,
'shape': test.shape
},
indent=4))
del test
print("generated and saved data: %s" %
self._file_name("data", "memap"))