def create_simple(id, last_weights=False): job_backend = JobBackend() job_backend.load(id) job_model = job_backend.get_job_model() if last_weights: weights_path = job_model.get_weights_filepath_latest() else: weights_path = job_model.get_weights_filepath_best() if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0: weight_url = job_backend.get_best_weight_url(id) if not weight_url: raise Exception("No weights available for this job.") print(("Download weights %s to %s .." % (weight_url, weights_path))) ensure_dir(os.path.dirname(weights_path)) f = open(weights_path, 'wb') f.write(urllib.urlopen(weight_url).read()) f.close() model.job_prepare(job_model) general_logger = GeneralLogger(job_backend=job_backend) trainer = Trainer(job_backend, general_logger) job_model.set_input_shape(trainer) model = job_model.get_built_model(trainer) job_model.load_weights(model, weights_path) return job_model, model
def start_model(self, parsed_args): from aetros import keras_model_utils from aetros.backend import JobBackend from aetros.logger import GeneralLogger from aetros.Trainer import Trainer from aetros.keras_model_utils import ensure_dir if not parsed_args.id: print("No job id given.") sys.exit(1) print("...") self.lock.acquire() job_backend = JobBackend(parsed_args.id) job_backend.load_light_job() self.job_model = job_backend.get_job_model() if parsed_args.weights: weights_path = parsed_args.weights elif parsed_args.latest: weights_path = self.job_model.get_weights_filepath_latest() else: weights_path = self.job_model.get_weights_filepath_best() print("Check weights ...") if not os.path.exists(weights_path) or os.path.getsize( weights_path) == 0: weight_url = job_backend.get_best_weight_url(parsed_args.id) if not weight_url: print("No weights available for this job.") exit(1) print( ("Download weights %s to %s .." % (weight_url, weights_path))) ensure_dir(os.path.dirname(weights_path)) f = open(weights_path, 'wb') f.write(urllib.urlopen(weight_url).read()) f.close() keras_model_utils.job_prepare(job_backend) trainer = Trainer(job_backend) self.job_model.set_input_shape(trainer) print("Loading model ...") model = self.job_model.get_built_model(trainer) print(("Load weights %s ..." % (weights_path, ))) self.job_model.load_weights(model, weights_path) print("Locked and loaded.") self.lock.release() return model
def start_model(self, parsed_args): from aetros import keras_model_utils from aetros.backend import JobBackend from aetros.logger import GeneralLogger from aetros.Trainer import Trainer from aetros.keras_model_utils import ensure_dir if not parsed_args.id: print("No job id given.") sys.exit(1) print("...") self.lock.acquire() job_backend = JobBackend(parsed_args.id) job_backend.load_light_job() self.job_model = job_backend.get_job_model() if parsed_args.weights: weights_path = parsed_args.weights elif parsed_args.latest: weights_path = self.job_model.get_weights_filepath_latest() else: weights_path = self.job_model.get_weights_filepath_best() print("Check weights ...") if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0: weight_url = job_backend.get_best_weight_url(parsed_args.id) if not weight_url: print("No weights available for this job.") exit(1) print(("Download weights %s to %s .." % (weight_url, weights_path))) ensure_dir(os.path.dirname(weights_path)) f = open(weights_path, 'wb') f.write(urllib.urlopen(weight_url).read()) f.close() trainer = Trainer(job_backend) self.job_model.set_input_shape(trainer) print("Loading model ...") model = self.job_model.get_built_model(trainer) print(("Load weights %s ..." % (weights_path,))) self.job_model.load_weights(model, weights_path) print("Locked and loaded.") self.lock.release() return model
def main(self, args): from aetros import keras_model_utils import aetros.const from aetros.backend import JobBackend from aetros.logger import GeneralLogger from aetros.Trainer import Trainer parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, prog=aetros.const.__prog__ + ' upload-weights') parser.add_argument('id', nargs='?', help='model name or job id') parser.add_argument( '--secure-key', help="Secure key. Alternatively use API_KEY environment varibale.") parser.add_argument( '--weights', help= "Weights path. Per default we try to find it in the ./weights/ folder." ) parser.add_argument( '--accuracy', help= "If you specified model name, you should also specify the accuracy this weights got." ) parser.add_argument( '--latest', action="store_true", help="Instead of best epoch we upload latest weights.") parsed_args = parser.parse_args(args) job_backend = JobBackend(api_token=parsed_args.secure_key) if '/' in parsed_args.id and '@' not in parsed_args.id: job_backend.create(parsed_args.id) job_backend.load(parsed_args.id) if job_backend.job is None: raise Exception("Job not found") job_model = job_backend.get_job_model() weights_path = job_model.get_weights_filepath_best() if parsed_args.weights: weights_path = parsed_args.weights print(("Validate weights in %s ..." % (weights_path, ))) keras_model_utils.job_prepare(job_model) general_logger = GeneralLogger() trainer = Trainer(job_backend, general_logger) job_model.set_input_shape(trainer) print("Loading model ...") model_provider = job_model.get_model_provider() model = model_provider.get_model(trainer) loss = model_provider.get_loss(trainer) optimizer = model_provider.get_optimizer(trainer) print("Compiling ...") model_provider.compile(trainer, model, loss, optimizer) print(("Validate weights %s ..." % (weights_path, ))) job_model.load_weights(model, weights_path) print("Validated.") print("Uploading weights to %s of %s ..." % (job_backend.job_id, job_backend.model_id)) job_backend.upload_weights( 'best.hdf5', weights_path, float(parsed_args.accuracy) if parsed_args.accuracy else None) print("Done")