def create_simple(id, last_weights=False):
job_backend = JobBackend()
job_backend.load(id)
job_model = job_backend.get_job_model()
if last_weights:
weights_path = job_model.get_weights_filepath_latest()
else:
weights_path = job_model.get_weights_filepath_best()
if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0:
weight_url = job_backend.get_best_weight_url(id)
if not weight_url:
raise Exception("No weights available for this job.")
print(("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
model.job_prepare(job_model)
general_logger = GeneralLogger(job_backend=job_backend)
trainer = Trainer(job_backend, general_logger)
job_model.set_input_shape(trainer)
model = job_model.get_built_model(trainer)
job_model.load_weights(model, weights_path)
return job_model, model
def start_model(self, parsed_args):
from aetros import keras_model_utils
from aetros.backend import JobBackend
from aetros.logger import GeneralLogger
from aetros.Trainer import Trainer
from aetros.keras_model_utils import ensure_dir
if not parsed_args.id:
print("No job id given.")
sys.exit(1)
print("...")
self.lock.acquire()
job_backend = JobBackend(parsed_args.id)
job_backend.load_light_job()
self.job_model = job_backend.get_job_model()
if parsed_args.weights:
weights_path = parsed_args.weights
elif parsed_args.latest:
weights_path = self.job_model.get_weights_filepath_latest()
else:
weights_path = self.job_model.get_weights_filepath_best()
print("Check weights ...")
if not os.path.exists(weights_path) or os.path.getsize(
weights_path) == 0:
weight_url = job_backend.get_best_weight_url(parsed_args.id)
if not weight_url:
print("No weights available for this job.")
exit(1)
print(
("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
keras_model_utils.job_prepare(job_backend)
trainer = Trainer(job_backend)
self.job_model.set_input_shape(trainer)
print("Loading model ...")
model = self.job_model.get_built_model(trainer)
print(("Load weights %s ..." % (weights_path, )))
self.job_model.load_weights(model, weights_path)
print("Locked and loaded.")
self.lock.release()
return model
def predict(job_id,
file_paths,
insights=False,
weights_path=None,
api_key=None):
print("Prepare model ...")
job_backend = JobBackend(api_key=api_key)
job_backend.load(job_id)
job_model = job_backend.get_job_model()
log = io.open(tempfile.mktemp(), 'w', encoding='utf8')
log.truncate()
keras_model_utils.job_prepare(job_model)
if not weights_path:
weight_path = job_model.get_weights_filepath_best()
if not os.path.exists(weight_path) or os.path.getsize(
weight_path) == 0:
weight_url = job_backend.get_best_weight_url(job_id)
if not weight_url:
print("No weights available for this job.")
exit(1)
print("Download weights %s to %s .." % (weight_url, weight_path))
ensure_dir(os.path.dirname(weight_path))
f = open(weight_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
from .logger import GeneralLogger
from .Trainer import Trainer
general_logger = GeneralLogger(log, job_backend)
trainer = Trainer(job_backend, general_logger)
job_model.set_input_shape(trainer)
print("Load model and compile ...")
model = job_model.get_built_model(trainer)
from aetros.keras import load_weights
load_weights(model, weights_path)
inputs = []
for idx, file_path in enumerate(file_paths):
inputs.append(
job_model.convert_file_to_input_node(
file_path, job_model.get_input_node(idx)))
print("Start prediction ...")
prediction = job_model.predict(model, np.array(inputs))
print(json.dumps(prediction, indent=4, default=invalid_json_values))
def start_model(self, parsed_args):
from aetros import keras_model_utils
from aetros.backend import JobBackend
from aetros.logger import GeneralLogger
from aetros.Trainer import Trainer
from aetros.keras_model_utils import ensure_dir
if not parsed_args.id:
print("No job id given.")
sys.exit(1)
print("...")
self.lock.acquire()
job_backend = JobBackend(parsed_args.id)
job_backend.load_light_job()
self.job_model = job_backend.get_job_model()
if parsed_args.weights:
weights_path = parsed_args.weights
elif parsed_args.latest:
weights_path = self.job_model.get_weights_filepath_latest()
else:
weights_path = self.job_model.get_weights_filepath_best()
print("Check weights ...")
if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0:
weight_url = job_backend.get_best_weight_url(parsed_args.id)
if not weight_url:
print("No weights available for this job.")
exit(1)
print(("Download weights %s to %s .." % (weight_url, weights_path)))
ensure_dir(os.path.dirname(weights_path))
f = open(weights_path, 'wb')
f.write(urllib.urlopen(weight_url).read())
f.close()
trainer = Trainer(job_backend)
self.job_model.set_input_shape(trainer)
print("Loading model ...")
model = self.job_model.get_built_model(trainer)
print(("Load weights %s ..." % (weights_path,)))
self.job_model.load_weights(model, weights_path)
print("Locked and loaded.")
self.lock.release()
return model