def create_simple(id, last_weights=False): job_backend = JobBackend() job_backend.load(id) job_model = job_backend.get_job_model() if last_weights: weights_path = job_model.get_weights_filepath_latest() else: weights_path = job_model.get_weights_filepath_best() if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0: weight_url = job_backend.get_best_weight_url(id) if not weight_url: raise Exception("No weights available for this job.") print(("Download weights %s to %s .." % (weight_url, weights_path))) ensure_dir(os.path.dirname(weights_path)) f = open(weights_path, 'wb') f.write(urllib.urlopen(weight_url).read()) f.close() model.job_prepare(job_model) general_logger = GeneralLogger(job_backend=job_backend) trainer = Trainer(job_backend, general_logger) job_model.set_input_shape(trainer) model = job_model.get_built_model(trainer) job_model.load_weights(model, weights_path) return job_model, model
def setup(self, x=None, nb_epoch=1, batch_size=16): graph = self.model_to_graph(self.model) from keras.preprocessing.image import Iterator if isinstance(x, Iterator): batch_size = x.batch_size settings = { 'epochs': nb_epoch, 'batchSize': batch_size, 'optimizer': type(self.model.optimizer).__name__ if hasattr(self.model, 'optimizer') else '' } self.job_backend.ensure_model(self.job_backend.model_id, settings=settings, type=self.model_type) self.trainer = Trainer(self.job_backend, self.job_backend.general_logger_stdout) self.trainer.model = self.model self.trainer.data_train = {'x': x} self.job_backend.set_graph(graph) self.callback = KerasLogger(self.trainer, self.job_backend, self.job_backend.general_logger_stdout, force_insights=self.insights) self.callback.log_epoch = False self.callback.model = self.model self.callback.insight_layer = self.insight_layer self.callback.confusion_matrix = self.confusion_matrix return self.callback
def start_model(self, parsed_args): from aetros import keras_model_utils from aetros.backend import JobBackend from aetros.logger import GeneralLogger from aetros.Trainer import Trainer from aetros.keras_model_utils import ensure_dir if not parsed_args.id: print("No job id given.") sys.exit(1) print("...") self.lock.acquire() job_backend = JobBackend(parsed_args.id) job_backend.load_light_job() self.job_model = job_backend.get_job_model() if parsed_args.weights: weights_path = parsed_args.weights elif parsed_args.latest: weights_path = self.job_model.get_weights_filepath_latest() else: weights_path = self.job_model.get_weights_filepath_best() print("Check weights ...") if not os.path.exists(weights_path) or os.path.getsize( weights_path) == 0: weight_url = job_backend.get_best_weight_url(parsed_args.id) if not weight_url: print("No weights available for this job.") exit(1) print( ("Download weights %s to %s .." % (weight_url, weights_path))) ensure_dir(os.path.dirname(weights_path)) f = open(weights_path, 'wb') f.write(urllib.urlopen(weight_url).read()) f.close() keras_model_utils.job_prepare(job_backend) trainer = Trainer(job_backend) self.job_model.set_input_shape(trainer) print("Loading model ...") model = self.job_model.get_built_model(trainer) print(("Load weights %s ..." % (weights_path, ))) self.job_model.load_weights(model, weights_path) print("Locked and loaded.") self.lock.release() return model
def start_model(self, parsed_args): from aetros import network from aetros.AetrosBackend import AetrosBackend from aetros.GeneralLogger import GeneralLogger from aetros.JobModel import JobModel from aetros.Trainer import Trainer from aetros.network import ensure_dir print("...") self.lock.acquire() aetros_backend = AetrosBackend(parsed_args.id) job = aetros_backend.get_light_job() self.job_model = JobModel(aetros_backend, job) if parsed_args.weights: weights_path = parsed_args.weights elif parsed_args.latest: weights_path = self.job_model.get_weights_filepath_latest() else: weights_path = self.job_model.get_weights_filepath_best() print("Check weights ...") if not os.path.exists(weights_path) or os.path.getsize(weights_path) == 0: weight_url = aetros_backend.get_best_weight_url(parsed_args.id) if not weight_url: print("No weights available for this job.") exit(1) print(("Download weights %s to %s .." % (weight_url, weights_path))) ensure_dir(os.path.dirname(weights_path)) f = open(weights_path, 'wb') f.write(urllib.urlopen(weight_url).read()) f.close() network.job_prepare(self.job_model.job) general_logger = GeneralLogger(job) trainer = Trainer(aetros_backend, self.job_model, general_logger) self.job_model.set_input_shape(trainer) print("Loading model ...") model = self.job_model.get_built_model(trainer) print(("Load weights %s ..." % (weights_path,))) self.job_model.load_weights(model, weights_path) print("Locked and loaded.") self.lock.release() return model
def setup(self, x=None, nb_epoch=1, batch_size=16): graph = self.model_to_graph(self.model) from keras.preprocessing.image import Iterator if isinstance(x, Iterator): batch_size = x.batch_size settings = { 'epochs': nb_epoch, 'batchSize': batch_size, 'optimizer': type(self.model.optimizer).__name__ if hasattr( self.model, 'optimizer') else '' } self.job_backend.ensure_model(self.id, self.model.to_json(), settings=settings, type=self.model_type, graph=graph) job_id = self.job_backend.create(self.id, insights=self.insights) self.job_backend.start() print( "AETROS job '%s' created and started. Open http://%s/trainer/app#/job=%s to monitor the training." % (job_id, self.job_backend.host, job_id)) job = self.job_backend.load_light_job() general_logger = GeneralLogger(job, job_backend=self.job_backend) self.trainer = Trainer(self.job_backend, general_logger) self.monitoringThread = MonitoringThread(self.job_backend, self.trainer) self.monitoringThread.daemon = True self.monitoringThread.start() self.trainer.model = self.model self.trainer.data_train = {'x': x} self.callback = KerasLogger(self.trainer, self.job_backend, general_logger) self.callback.log_epoch = False self.callback.model = self.model self.callback.confusion_matrix = self.confusion_matrix return self.callback
def setup(self, x=None, nb_epoch=1, batch_size=16): graph = self.model_to_graph(self.model) from keras.preprocessing.image import Iterator if isinstance(x, Iterator): batch_size = x.batch_size settings = { 'epochs': nb_epoch, 'batchSize': batch_size, 'optimizer': type(self.model.optimizer).__name__ if hasattr( self.model, 'optimizer') else '' } self.aetros_backend.ensure_network(self.network_name, self.model.to_json(), settings=settings, network_type=self.network_type, graph=graph) self.aetros_backend.create_job(self.network_name, insights=self.insights) job = self.aetros_backend.get_light_job() self.job_model = JobModel(self.aetros_backend, job) general_logger = GeneralLogger(job, aetros_backend=self.aetros_backend) self.trainer = Trainer(self.aetros_backend, self.job_model, general_logger) self.monitoringThread = MonitoringThread(self.aetros_backend, self.trainer) self.monitoringThread.daemon = True self.monitoringThread.start() network.collect_system_information(self.trainer) self.trainer.model = self.model self.trainer.data_train = {'x': x} self.callback = KerasLogger(self.trainer, self.aetros_backend, self.job_model, general_logger) self.callback.log_epoch = False self.callback.model = self.model self.callback.confusion_matrix = self.confusion_matrix return self.callback
def main(self, args): from aetros import network import aetros.const from aetros.AetrosBackend import AetrosBackend from aetros.GeneralLogger import GeneralLogger from aetros.JobModel import JobModel from aetros.Trainer import Trainer # from aetros.network import ensure_dir # from aetros.starter import start parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, prog=aetros.const.__prog__ + ' upload-weights') parser.add_argument('id', nargs='?', help='Network name or training id') parser.add_argument( '--weights', help= "Weights path. Per default we try to find it in the ./weights/ folder." ) parser.add_argument( '--accuracy', help= "If you specified network name, you should also specify the accuracy this weights got." ) parser.add_argument( '--latest', action="store_true", help="Instead of best epoch we upload latest weights.") parsed_args = parser.parse_args(args) aetros_backend = AetrosBackend(parsed_args.id) if '/' in parsed_args.id and '@' not in parsed_args.id: job_id = aetros_backend.create_job(parsed_args.id) aetros_backend.job_id = job_id job = aetros_backend.get_light_job() job_id = job['id'] aetros_backend.job_id = job_id job = aetros_backend.get_job() if job is None: raise Exception("Training not found") job_model = JobModel(aetros_backend, job) weights_path = job_model.get_weights_filepath_best() if parsed_args.weights: weights_path = parsed_args.weights print(("Validate weights in %s ..." % (weights_path, ))) network.job_prepare(job_model.job) general_logger = GeneralLogger(job) trainer = Trainer(aetros_backend, job_model, general_logger) job_model.set_input_shape(trainer) print("Loading model ...") model_provider = job_model.get_model_provider() model = model_provider.get_model(trainer) loss = model_provider.get_loss(trainer) optimizer = model_provider.get_optimizer(trainer) print("Compiling ...") model_provider.compile(trainer, model, loss, optimizer) print(("Validate weights %s ..." % (weights_path, ))) job_model.load_weights(model, weights_path) print("Validated.") print("Uploading weights to %s of %s ..." % (job_id, job['networkId'])) aetros_backend.upload_weights('best.hdf5', weights_path, float(parsed_args.accuracy)) print("Done")
def main(self, args): from aetros import keras_model_utils import aetros.const from aetros.backend import JobBackend from aetros.logger import GeneralLogger from aetros.Trainer import Trainer parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, prog=aetros.const.__prog__ + ' upload-weights') parser.add_argument('id', nargs='?', help='model name or job id') parser.add_argument( '--secure-key', help="Secure key. Alternatively use API_KEY environment varibale.") parser.add_argument( '--weights', help= "Weights path. Per default we try to find it in the ./weights/ folder." ) parser.add_argument( '--accuracy', help= "If you specified model name, you should also specify the accuracy this weights got." ) parser.add_argument( '--latest', action="store_true", help="Instead of best epoch we upload latest weights.") parsed_args = parser.parse_args(args) job_backend = JobBackend(api_token=parsed_args.secure_key) if '/' in parsed_args.id and '@' not in parsed_args.id: job_backend.create(parsed_args.id) job_backend.load(parsed_args.id) if job_backend.job is None: raise Exception("Job not found") job_model = job_backend.get_job_model() weights_path = job_model.get_weights_filepath_best() if parsed_args.weights: weights_path = parsed_args.weights print(("Validate weights in %s ..." % (weights_path, ))) keras_model_utils.job_prepare(job_model) general_logger = GeneralLogger() trainer = Trainer(job_backend, general_logger) job_model.set_input_shape(trainer) print("Loading model ...") model_provider = job_model.get_model_provider() model = model_provider.get_model(trainer) loss = model_provider.get_loss(trainer) optimizer = model_provider.get_optimizer(trainer) print("Compiling ...") model_provider.compile(trainer, model, loss, optimizer) print(("Validate weights %s ..." % (weights_path, ))) job_model.load_weights(model, weights_path) print("Validated.") print("Uploading weights to %s of %s ..." % (job_backend.job_id, job_backend.model_id)) job_backend.upload_weights( 'best.hdf5', weights_path, float(parsed_args.accuracy) if parsed_args.accuracy else None) print("Done")