def create_simple(id, last_weights=False):
job_backend = JobBackend()
job_backend.load(id)
job_model = job_backend.get_job_model()
if last_weights:
weights_path = job_model.get_weights_filepath_latest()
else:
weights_path = job_model.get_weights_filepath_best()
if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0:
weight_url = job_backend.get_best_weight_url(id)
if not weight_url:
raise Exception("No weights available for this job.")
print(("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
model.job_prepare(job_model)
general_logger = GeneralLogger(job_backend=job_backend)
trainer = Trainer(job_backend, general_logger)
job_model.set_input_shape(trainer)
model = job_model.get_built_model(trainer)
job_model.load_weights(model, weights_path)
return job_model, model
def start_model(self, parsed_args):
from aetros import keras_model_utils
from aetros.backend import JobBackend
from aetros.logger import GeneralLogger
from aetros.Trainer import Trainer
from aetros.keras_model_utils import ensure_dir
if not parsed_args.id:
print("No job id given.")
sys.exit(1)
print("...")
self.lock.acquire()
job_backend = JobBackend(parsed_args.id)
job_backend.load_light_job()
self.job_model = job_backend.get_job_model()
if parsed_args.weights:
weights_path = parsed_args.weights
elif parsed_args.latest:
weights_path = self.job_model.get_weights_filepath_latest()
else:
weights_path = self.job_model.get_weights_filepath_best()
print("Check weights ...")
if not os.path.exists(weights_path) or os.path.getsize(
weights_path) == 0:
weight_url = job_backend.get_best_weight_url(parsed_args.id)
if not weight_url:
print("No weights available for this job.")
exit(1)
print(
("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
keras_model_utils.job_prepare(job_backend)
trainer = Trainer(job_backend)
self.job_model.set_input_shape(trainer)
print("Loading model ...")
model = self.job_model.get_built_model(trainer)
print(("Load weights %s ..." % (weights_path, )))
self.job_model.load_weights(model, weights_path)
print("Locked and loaded.")
self.lock.release()
return model
def start_model(self, parsed_args):
from aetros import keras_model_utils
from aetros.backend import JobBackend
from aetros.logger import GeneralLogger
from aetros.Trainer import Trainer
from aetros.keras_model_utils import ensure_dir
if not parsed_args.id:
print("No job id given.")
sys.exit(1)
print("...")
self.lock.acquire()
job_backend = JobBackend(parsed_args.id)
job_backend.load_light_job()
self.job_model = job_backend.get_job_model()
if parsed_args.weights:
weights_path = parsed_args.weights
elif parsed_args.latest:
weights_path = self.job_model.get_weights_filepath_latest()
else:
weights_path = self.job_model.get_weights_filepath_best()
print("Check weights ...")
if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0:
weight_url = job_backend.get_best_weight_url(parsed_args.id)
if not weight_url:
print("No weights available for this job.")
exit(1)
print(("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
trainer = Trainer(job_backend)
self.job_model.set_input_shape(trainer)
print("Loading model ...")
model = self.job_model.get_built_model(trainer)
print(("Load weights %s ..." % (weights_path,)))
self.job_model.load_weights(model, weights_path)
print("Locked and loaded.")
self.lock.release()
return model
def main(self, args):
from aetros import keras_model_utils
import aetros.const
from aetros.backend import JobBackend
from aetros.logger import GeneralLogger
from aetros.Trainer import Trainer
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
prog=aetros.const.__prog__ + ' upload-weights')
parser.add_argument('id', nargs='?', help='model name or job id')
parser.add_argument(
'--secure-key',
help="Secure key. Alternatively use API_KEY environment varibale.")
parser.add_argument(
'--weights',
help=
"Weights path. Per default we try to find it in the ./weights/ folder."
)
parser.add_argument(
'--accuracy',
help=
"If you specified model name, you should also specify the accuracy this weights got."
)
parser.add_argument(
'--latest',
action="store_true",
help="Instead of best epoch we upload latest weights.")
parsed_args = parser.parse_args(args)
job_backend = JobBackend(api_token=parsed_args.secure_key)
if '/' in parsed_args.id and '@' not in parsed_args.id:
job_backend.create(parsed_args.id)
job_backend.load(parsed_args.id)
if job_backend.job is None:
raise Exception("Job not found")
job_model = job_backend.get_job_model()
weights_path = job_model.get_weights_filepath_best()
if parsed_args.weights:
weights_path = parsed_args.weights
print(("Validate weights in %s ..." % (weights_path, )))
keras_model_utils.job_prepare(job_model)
general_logger = GeneralLogger()
trainer = Trainer(job_backend, general_logger)
job_model.set_input_shape(trainer)
print("Loading model ...")
model_provider = job_model.get_model_provider()
model = model_provider.get_model(trainer)
loss = model_provider.get_loss(trainer)
optimizer = model_provider.get_optimizer(trainer)
print("Compiling ...")
model_provider.compile(trainer, model, loss, optimizer)
print(("Validate weights %s ..." % (weights_path, )))
job_model.load_weights(model, weights_path)
print("Validated.")
print("Uploading weights to %s of %s ..." %
(job_backend.job_id, job_backend.model_id))
job_backend.upload_weights(
'best.hdf5', weights_path,
float(parsed_args.accuracy) if parsed_args.accuracy else None)
print("Done")