def test_xenon_42_multi():
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine(
scheduler_adaptor='slurm',
location='ssh://fs0.das5.cs.vu.nl/home/jhidding',
credential=xenon.CertificateCredential(
username='******', certfile='/home/johannes/.ssh/id_rsa'),
jobs_properties={
'xenon.adaptors.schedulers.ssh.strictHostKeyChecking': 'false'
})
worker_config = XenonJobConfig(
prefix=Path('/home/jhidding/.local/share/workon/mcfly'),
working_dir='/home/jhidding/',
time_out=1000000000000,
verbose=False) # , options=['-C', 'TitanX', '--gres=gpu:1'])
result = run_xenon(C,
machine=machine,
worker_config=worker_config,
n_processes=2)
print("The answer is:", result)
def test_xenon_42_simple(xenon_server):
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine()
worker_config = XenonJobConfig(verbose=True)
result = run_xenon_simple(C, machine=machine, worker_config=worker_config)
assert (result == 42)
def test_get_intermediate_results():
r1 = add(1, 1)
r2 = sub(3, r1)
def foo(a, b, c):
return mul(add(a, b), c)
r3 = [foo(i, r2, r1) for i in range(6)]
r4 = accumulate(noodles.gather_all(r3))
run_single(r4)
assert noodles.result(r1) == 2
assert noodles.result(r2) == 1
assert [noodles.result(r) for r in r3] == [2, 4, 6, 8, 10, 12]
assert noodles.result(r4) == 42
def test_xenon_42_multi(xenon_server):
A = add(1, 1)
B = sub(3, A)
multiples = [mul(add(i, B), A) for i in range(6)]
C = schedule(sum)(gather_all(multiples))
machine = Machine()
worker_config = XenonJobConfig(queue_name='multi', verbose=True)
result = run_xenon(C,
machine=machine,
worker_config=worker_config,
n_processes=2)
assert (result == 42)
def map(f, lst):
return noodles.gather_all(f(x) for x in lst)
def test_gather_all():
return noodles.gather_all(add(x, 2 * x) for x in range(10))
def test_gather_all():
d = noodles.gather_all(add(x, 2*x) for x in range(10))
result = run_single(d)
assert result == list(range(0, 30, 3))
def train_models_on_samples(X_train,
y_train,
X_val,
y_val,
models,
nr_epochs=5,
subset_size=100,
verbose=True,
outputfile=None,
model_path=None,
early_stopping=False,
batch_size=20,
metric='accuracy',
use_noodles=None):
"""
Given a list of compiled models, this function trains
them all on a subset of the train data. If the given size of the subset is
smaller then the size of the data, the complete data set is used.
Parameters
----------
X_train : numpy array of shape (num_samples, num_timesteps, num_channels)
The input dataset for training
y_train : numpy array of shape (num_samples, num_classes)
The output classes for the train data, in binary format
X_val : numpy array of shape (num_samples_val, num_timesteps, num_channels)
The input dataset for validation
y_val : numpy array of shape (num_samples_val, num_classes)
The output classes for the validation data, in binary format
models : list of model, params, modeltypes
List of keras models to train
nr_epochs : int, optional
nr of epochs to use for training one model
subset_size :
The number of samples used from the complete train set
verbose : bool, optional
flag for displaying verbose output
outputfile: str, optional
Filename to store the model training results
model_path : str, optional
Directory to store the models as HDF5 files
early_stopping: bool
Stop when validation loss does not decrease
batch_size : int
nr of samples per batch
metric : str
metric to store in the history object
Returns
----------
histories : list of Keras History objects
train histories for all models
val_metrics : list of floats
validation accuraracies of the models
val_losses : list of floats
validation losses of the models
"""
# if subset_size is smaller then X_train, this will work fine
X_train_sub = X_train[:subset_size, :, :]
y_train_sub = y_train[:subset_size, :]
metric_name = get_metric_name(metric)
val_metrics = []
val_losses = []
def make_history(model, i=None):
model_metrics = [get_metric_name(name) for name in model.metrics]
if metric_name not in model_metrics:
raise ValueError(
'Invalid metric. The model was not compiled with {} as metric'.
format(metric_name))
if early_stopping:
callbacks = [
EarlyStopping(monitor='val_loss',
patience=0,
verbose=verbose,
mode='auto')
]
else:
callbacks = []
args = (model, X_train_sub, y_train_sub)
kwargs = {
'epochs': nr_epochs,
'batch_size': batch_size,
'validation_data': (X_val, y_val),
'verbose': verbose,
'callbacks': callbacks
}
if use_noodles is None:
# if not using noodles, save every nugget when it comes
trained_model = train_model(*args, **kwargs)
if outputfile is not None:
store_train_hist_as_json(models[i][1], models[i][2],
trained_model.history, outputfile)
if model_path is not None:
trained_model.save(
os.path.join(model_path, 'model_{}.h5'.format(i)))
return trained_model
else:
assert has_noodles, "Noodles is not installed, or could not be imported."
return noodles.schedule_hint(call_by_ref=['model']) \
(train_model)(*args, **kwargs)
if use_noodles is None:
trained_models = [
make_history(model[0], i) for i, model in enumerate(models)
]
else:
assert has_noodles, "Noodles is not installed, or could not be imported."
# in case of noodles, first run everything
training_wf = noodles.gather_all(
[make_history(model[0]) for model in models])
trained_models = use_noodles(training_wf)
# then save everything
for i, (history, model) in enumerate(trained_models):
if outputfile is not None:
store_train_hist_as_json(models[i][1], models[i][2], history,
outputfile)
if model_path is not None:
model.save(os.path.join(model_path, 'model_{}.h5'.format(i)))
# accumulate results
val_metrics = [tm.history['val_' + metric_name] for tm in trained_models]
val_losses = [tm.history['val_loss'] for tm in trained_models]
return [tm.history for tm in trained_models], val_metrics, val_losses