def create_simple(id, last_weights=False):
job_backend = JobBackend()
job_backend.load(id)
job_model = job_backend.get_job_model()
if last_weights:
weights_path = job_model.get_weights_filepath_latest()
else:
weights_path = job_model.get_weights_filepath_best()
if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0:
weight_url = job_backend.get_best_weight_url(id)
if not weight_url:
raise Exception("No weights available for this job.")
print(("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
model.job_prepare(job_model)
general_logger = GeneralLogger(job_backend=job_backend)
trainer = Trainer(job_backend, general_logger)
job_model.set_input_shape(trainer)
model = job_model.get_built_model(trainer)
job_model.load_weights(model, weights_path)
return job_model, model
def setup(self, x=None, nb_epoch=1, batch_size=16):
graph = self.model_to_graph(self.model)
from keras.preprocessing.image import Iterator
if isinstance(x, Iterator):
batch_size = x.batch_size
settings = {
'epochs': nb_epoch,
'batchSize': batch_size,
'optimizer': type(self.model.optimizer).__name__ if hasattr(self.model, 'optimizer') else ''
}
self.job_backend.ensure_model(self.job_backend.model_id, settings=settings, type=self.model_type)
self.trainer = Trainer(self.job_backend, self.job_backend.general_logger_stdout)
self.trainer.model = self.model
self.trainer.data_train = {'x': x}
self.job_backend.set_graph(graph)
self.callback = KerasLogger(self.trainer, self.job_backend, self.job_backend.general_logger_stdout, force_insights=self.insights)
self.callback.log_epoch = False
self.callback.model = self.model
self.callback.insight_layer = self.insight_layer
self.callback.confusion_matrix = self.confusion_matrix
return self.callback
def start_model(self, parsed_args):
from aetros import keras_model_utils
from aetros.backend import JobBackend
from aetros.logger import GeneralLogger
from aetros.Trainer import Trainer
from aetros.keras_model_utils import ensure_dir
if not parsed_args.id:
print("No job id given.")
sys.exit(1)
print("...")
self.lock.acquire()
job_backend = JobBackend(parsed_args.id)
job_backend.load_light_job()
self.job_model = job_backend.get_job_model()
if parsed_args.weights:
weights_path = parsed_args.weights
elif parsed_args.latest:
weights_path = self.job_model.get_weights_filepath_latest()
else:
weights_path = self.job_model.get_weights_filepath_best()
print("Check weights ...")
if not os.path.exists(weights_path) or os.path.getsize(
weights_path) == 0:
weight_url = job_backend.get_best_weight_url(parsed_args.id)
if not weight_url:
print("No weights available for this job.")
exit(1)
print(
("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
keras_model_utils.job_prepare(job_backend)
trainer = Trainer(job_backend)
self.job_model.set_input_shape(trainer)
print("Loading model ...")
model = self.job_model.get_built_model(trainer)
print(("Load weights %s ..." % (weights_path, )))
self.job_model.load_weights(model, weights_path)
print("Locked and loaded.")
self.lock.release()
return model
def start_model(self, parsed_args):
from aetros import network
from aetros.AetrosBackend import AetrosBackend
from aetros.GeneralLogger import GeneralLogger
from aetros.JobModel import JobModel
from aetros.Trainer import Trainer
from aetros.network import ensure_dir
print("...")
self.lock.acquire()
aetros_backend = AetrosBackend(parsed_args.id)
job = aetros_backend.get_light_job()
self.job_model = JobModel(aetros_backend, job)
if parsed_args.weights:
weights_path = parsed_args.weights
elif parsed_args.latest:
weights_path = self.job_model.get_weights_filepath_latest()
else:
weights_path = self.job_model.get_weights_filepath_best()
print("Check weights ...")
if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0:
weight_url = aetros_backend.get_best_weight_url(parsed_args.id)
if not weight_url:
print("No weights available for this job.")
exit(1)
print(("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
network.job_prepare(self.job_model.job)
general_logger = GeneralLogger(job)
trainer = Trainer(aetros_backend, self.job_model, general_logger)
self.job_model.set_input_shape(trainer)
print("Loading model ...")
model = self.job_model.get_built_model(trainer)
print(("Load weights %s ..." % (weights_path,)))
self.job_model.load_weights(model, weights_path)
print("Locked and loaded.")
self.lock.release()
return model
def setup(self, x=None, nb_epoch=1, batch_size=16):
graph = self.model_to_graph(self.model)
from keras.preprocessing.image import Iterator
if isinstance(x, Iterator):
batch_size = x.batch_size
settings = {
'epochs':
nb_epoch,
'batchSize':
batch_size,
'optimizer':
type(self.model.optimizer).__name__ if hasattr(
self.model, 'optimizer') else ''
}
self.job_backend.ensure_model(self.id,
self.model.to_json(),
settings=settings,
type=self.model_type,
graph=graph)
job_id = self.job_backend.create(self.id, insights=self.insights)
self.job_backend.start()
print(
"AETROS job '%s' created and started. Open http://%s/trainer/app#/job=%s to monitor the training."
% (job_id, self.job_backend.host, job_id))
job = self.job_backend.load_light_job()
general_logger = GeneralLogger(job, job_backend=self.job_backend)
self.trainer = Trainer(self.job_backend, general_logger)
self.monitoringThread = MonitoringThread(self.job_backend,
self.trainer)
self.monitoringThread.daemon = True
self.monitoringThread.start()
self.trainer.model = self.model
self.trainer.data_train = {'x': x}
self.callback = KerasLogger(self.trainer, self.job_backend,
general_logger)
self.callback.log_epoch = False
self.callback.model = self.model
self.callback.confusion_matrix = self.confusion_matrix
return self.callback
def setup(self, x=None, nb_epoch=1, batch_size=16):
graph = self.model_to_graph(self.model)
from keras.preprocessing.image import Iterator
if isinstance(x, Iterator):
batch_size = x.batch_size
settings = {
'epochs':
nb_epoch,
'batchSize':
batch_size,
'optimizer':
type(self.model.optimizer).__name__ if hasattr(
self.model, 'optimizer') else ''
}
self.aetros_backend.ensure_network(self.network_name,
self.model.to_json(),
settings=settings,
network_type=self.network_type,
graph=graph)
self.aetros_backend.create_job(self.network_name,
insights=self.insights)
job = self.aetros_backend.get_light_job()
self.job_model = JobModel(self.aetros_backend, job)
general_logger = GeneralLogger(job, aetros_backend=self.aetros_backend)
self.trainer = Trainer(self.aetros_backend, self.job_model,
general_logger)
self.monitoringThread = MonitoringThread(self.aetros_backend,
self.trainer)
self.monitoringThread.daemon = True
self.monitoringThread.start()
network.collect_system_information(self.trainer)
self.trainer.model = self.model
self.trainer.data_train = {'x': x}
self.callback = KerasLogger(self.trainer, self.aetros_backend,
self.job_model, general_logger)
self.callback.log_epoch = False
self.callback.model = self.model
self.callback.confusion_matrix = self.confusion_matrix
return self.callback
def main(self, args):
from aetros import network
import aetros.const
from aetros.AetrosBackend import AetrosBackend
from aetros.GeneralLogger import GeneralLogger
from aetros.JobModel import JobModel
from aetros.Trainer import Trainer
# from aetros.network import ensure_dir
# from aetros.starter import start
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
prog=aetros.const.__prog__ + ' upload-weights')
parser.add_argument('id',
nargs='?',
help='Network name or training id')
parser.add_argument(
'--weights',
help=
"Weights path. Per default we try to find it in the ./weights/ folder."
)
parser.add_argument(
'--accuracy',
help=
"If you specified network name, you should also specify the accuracy this weights got."
)
parser.add_argument(
'--latest',
action="store_true",
help="Instead of best epoch we upload latest weights.")
parsed_args = parser.parse_args(args)
aetros_backend = AetrosBackend(parsed_args.id)
if '/' in parsed_args.id and '@' not in parsed_args.id:
job_id = aetros_backend.create_job(parsed_args.id)
aetros_backend.job_id = job_id
job = aetros_backend.get_light_job()
job_id = job['id']
aetros_backend.job_id = job_id
job = aetros_backend.get_job()
if job is None:
raise Exception("Training not found")
job_model = JobModel(aetros_backend, job)
weights_path = job_model.get_weights_filepath_best()
if parsed_args.weights:
weights_path = parsed_args.weights
print(("Validate weights in %s ..." % (weights_path, )))
network.job_prepare(job_model.job)
general_logger = GeneralLogger(job)
trainer = Trainer(aetros_backend, job_model, general_logger)
job_model.set_input_shape(trainer)
print("Loading model ...")
model_provider = job_model.get_model_provider()
model = model_provider.get_model(trainer)
loss = model_provider.get_loss(trainer)
optimizer = model_provider.get_optimizer(trainer)
print("Compiling ...")
model_provider.compile(trainer, model, loss, optimizer)
print(("Validate weights %s ..." % (weights_path, )))
job_model.load_weights(model, weights_path)
print("Validated.")
print("Uploading weights to %s of %s ..." % (job_id, job['networkId']))
aetros_backend.upload_weights('best.hdf5', weights_path,
float(parsed_args.accuracy))
print("Done")
def main(self, args):
from aetros import keras_model_utils
import aetros.const
from aetros.backend import JobBackend
from aetros.logger import GeneralLogger
from aetros.Trainer import Trainer
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
prog=aetros.const.__prog__ + ' upload-weights')
parser.add_argument('id', nargs='?', help='model name or job id')
parser.add_argument(
'--secure-key',
help="Secure key. Alternatively use API_KEY environment varibale.")
parser.add_argument(
'--weights',
help=
"Weights path. Per default we try to find it in the ./weights/ folder."
)
parser.add_argument(
'--accuracy',
help=
"If you specified model name, you should also specify the accuracy this weights got."
)
parser.add_argument(
'--latest',
action="store_true",
help="Instead of best epoch we upload latest weights.")
parsed_args = parser.parse_args(args)
job_backend = JobBackend(api_token=parsed_args.secure_key)
if '/' in parsed_args.id and '@' not in parsed_args.id:
job_backend.create(parsed_args.id)
job_backend.load(parsed_args.id)
if job_backend.job is None:
raise Exception("Job not found")
job_model = job_backend.get_job_model()
weights_path = job_model.get_weights_filepath_best()
if parsed_args.weights:
weights_path = parsed_args.weights
print(("Validate weights in %s ..." % (weights_path, )))
keras_model_utils.job_prepare(job_model)
general_logger = GeneralLogger()
trainer = Trainer(job_backend, general_logger)
job_model.set_input_shape(trainer)
print("Loading model ...")
model_provider = job_model.get_model_provider()
model = model_provider.get_model(trainer)
loss = model_provider.get_loss(trainer)
optimizer = model_provider.get_optimizer(trainer)
print("Compiling ...")
model_provider.compile(trainer, model, loss, optimizer)
print(("Validate weights %s ..." % (weights_path, )))
job_model.load_weights(model, weights_path)
print("Validated.")
print("Uploading weights to %s of %s ..." %
(job_backend.job_id, job_backend.model_id))
job_backend.upload_weights(
'best.hdf5', weights_path,
float(parsed_args.accuracy) if parsed_args.accuracy else None)
print("Done")