def run_solver_loop(self): solver_def = 'solver.prototxt' solver_param = caffe_pb2.SolverParameter() text_format.Merge(open(solver_def).read(), solver_param) solver = caffe.get_solver_from_string(solver_param.SerializeToString()) for i in range(50): solver.step(1)
def run_solver_loop(self): solver_def='solver.prototxt' solver_param = caffe_pb2.SolverParameter() text_format.Merge(open(solver_def).read(), solver_param) solver = caffe.get_solver_from_string(solver_param.SerializeToString()) for i in range(50): solver.step(1)
def __init__(self, prototxt, final_file=None, snap_file=None, solver='Adam', log_file=None, **kwargs): self.running = False self.force_snapshot = False self.last_interrupt = 0 self.final_file, self.snap_file = final_file, snap_file if final_file is None: print("Are you sure you dont want to save the model?") solver_str = 'train_net: "%s"\n' % prototxt if solver is not None: # Get the cases right if solver.upper() == "ADAM": solver = "Adam" if solver.upper() == "ADADELTA": solver = "AdaDelta" if solver.upper() == "ADAGRAD": solver = "AdaGrad" if solver.upper() == "NESTEROV": solver = "Nesterov" if solver.upper() == "RMSPROP": solver = "RMSProp" if solver.upper() == "SGD": solver = "SGD" solver_str += 'type: "%s"\n' % solver if solver == "RMSProp": if 'rms_decay' not in kwargs: kwargs['rms_decay'] = 0.9 if solver == "SGD": if 'momentum' not in kwargs: kwargs['momentum'] = 0.9 if solver == "Adam": if 'momentum' not in kwargs: kwargs['momentum'] = 0.9 if 'momentum2' not in kwargs: kwargs['momentum2'] = 0.99 if 'base_lr' not in kwargs: kwargs['base_lr'] = 0.001 if 'lr_policy' not in kwargs: kwargs['lr_policy'] = 'fixed' for i in kwargs: if isinstance(kwargs[i], str): solver_str += '%s: "%s"\n' % (i, kwargs[i]) elif isinstance(kwargs[i], int): solver_str += '%s: %d\n' % (i, kwargs[i]) else: solver_str += '%s: %f\n' % (i, kwargs[i]) self.solver = caffe.get_solver_from_string(solver_str) self.solver.add_callback(self.on_start, self.on_gradient) self.log = None if log_file: self.log = open(log_file, 'w')
def __init__(self, solver_def, solver_state=None, weights=None, gpus=[0], log_dir='log', log_db_prefix='log_db'): """ Acts as a driver for training with smart logging. Logs will be stored to a MySQL database. All paths can be set relative to the location of the solver prototxt. :param solver_def: prototxt that defines the solver :param solver_state: optional: a .solverstate file from which to resume training \ NEVER SET THESE TWO :param weights: optional: a .caffemodel file from which to begin finetuning / AT THE SAME TIME :param gpus: optional: a list of GPU IDs to use for (multi-)GPU training if set to None, caffe will operate in CPU mode :param log_dir: optional: will log into this directory under solver.prototxt :param log_db_prefix: prefix for both SQLite db names and table names The following parameters should to be set in the solver prototxt file: log_interval log per this number of iterations (simple log) [default = 20] viz_interval log visualization per this number of iterations (net blobs snapshot) [default = 100] test_iter: The number of iterations for each test net. """ if not os.path.isabs(solver_def): if not os.path.isfile(os.path.join(os.getcwd(), solver_def)): os.chdir('..') solver_def = os.path.join(os.getcwd(), solver_def) else: solver_def = os.path.join(os.getcwd(), solver_def) self.solver_dir = solver_def[:solver_def.rfind('/')] os.chdir(self.solver_dir) self.log_dir = os.path.join(self.solver_dir, log_dir) if not os.path.exists(self.log_dir): os.mkdir(self.log_dir) self.logprint("Logging to {}".format(self.log_dir)) self.log_db_prefix = log_db_prefix # used for db name and as a prefix for tables self.solver_param = caffe_pb2.SolverParameter() text_format.Merge(open(solver_def).read(), self.solver_param) # read params from solver definition self.iterations = self.solver_param.max_iter self.log_interval = self.solver_param.log_interval self.viz_interval = self.solver_param.viz_interval self.test_interval = self.solver_param.test_interval self.se_index_blob = self.solver_param.se_index_blob self.se_error_blob = self.solver_param.se_error_blob self.se_interval = self.solver_param.se_interval self.se_indices = [] self.se_errors = [] # make solver param for net fail safe if not os.path.isabs(self.solver_param.net): self.solver_param.net = os.path.join(self.solver_dir, self.solver_param.net) if not os.path.isfile(self.solver_param.net): raise Exception( 'could not find net definition from solver prototxt!') self.gpus = gpus if gpus: self.solver_param.device_id = gpus[0] caffe.set_device(gpus[0]) caffe.set_mode_gpu() caffe.set_solver_count(len(gpus)) self.solver = caffe.get_solver_from_string( self.solver_param.SerializeToString()) if solver_state: # check if this file is in the current (or parent) directory or if the solver path needs to be prepended if not os.path.isfile(os.path.join('..', solver_state)): if not os.path.isfile(solver_state): solver_state = os.path.join(self.solver_dir, solver_state) if not os.path.isfile(solver_state): raise Exception( 'could not find solver state specified!') else: solver_state = os.path.join('..', solver_state) self.solver.restore(solver_state) if weights: raise Exception( 'should not specify both solverstate and caffemodel! Preference will be given to solverstate.' ) if weights and not solver_state: self.solver.net.copy_from(weights) self.sync = None self.viz_thread = None self.log_thread = None self.test_input = None self.test_out_blobs = None self.test_start_layer = None self.iteration = 0 # check if blobs update_sample_errors exist if self.se_interval and not self.se_index_blob in self.solver.net.blobs: self.logprint( "WARNING: index_blob not found in net! Won't send errors to Net." ) self.se_interval = 0 if self.se_interval and not self.se_error_blob in self.solver.net.blobs: self.logprint( "WARNING: error_blob not found in net! Won't send errors to Net." ) self.se_interval = 0
def __init__(self, solver_def, solver_state=None, weights=None, gpus=[0], log_dir='log', log_db_prefix='log_db'): """ Acts as a driver for training with smart logging. Logs will be stored to a MySQL database. All paths can be set relative to the location of the solver prototxt. :param solver_def: prototxt that defines the solver :param solver_state: optional: a .solverstate file from which to resume training \ NEVER SET THESE TWO :param weights: optional: a .caffemodel file from which to begin finetuning / AT THE SAME TIME :param gpus: optional: a list of GPU IDs to use for (multi-)GPU training if set to None, caffe will operate in CPU mode :param log_dir: optional: will log into this directory under solver.prototxt :param log_db_prefix: prefix for both SQLite db names and table names The following parameters should to be set in the solver prototxt file: log_interval log per this number of iterations (simple log) [default = 20] viz_interval log visualization per this number of iterations (net blobs snapshot) [default = 100] test_iter: The number of iterations for each test net. """ if not os.path.isabs(solver_def): if not os.path.isfile(os.path.join(os.getcwd(), solver_def)): os.chdir('..') solver_def = os.path.join(os.getcwd(), solver_def) else: solver_def = os.path.join(os.getcwd(), solver_def) self.solver_dir = solver_def[:solver_def.rfind('/')] os.chdir(self.solver_dir) self.log_dir = os.path.join(self.solver_dir, log_dir) if not os.path.exists(self.log_dir): os.mkdir(self.log_dir) self.logprint("Logging to {}".format(self.log_dir)) self.log_db_prefix = log_db_prefix # used for db name and as a prefix for tables self.solver_param = caffe_pb2.SolverParameter() text_format.Merge(open(solver_def).read(), self.solver_param) # read params from solver definition self.iterations = self.solver_param.max_iter self.log_interval = self.solver_param.log_interval self.viz_interval = self.solver_param.viz_interval self.test_interval = self.solver_param.test_interval self.se_index_blob = self.solver_param.se_index_blob self.se_error_blob = self.solver_param.se_error_blob self.se_interval = self.solver_param.se_interval self.se_indices = [] self.se_errors = [] # make solver param for net fail safe if not os.path.isabs(self.solver_param.net): self.solver_param.net = os.path.join(self.solver_dir, self.solver_param.net) if not os.path.isfile(self.solver_param.net): raise Exception('could not find net definition from solver prototxt!') self.gpus = gpus if gpus: self.solver_param.device_id = gpus[0] caffe.set_device(gpus[0]) caffe.set_mode_gpu() caffe.set_solver_count(len(gpus)) self.solver = caffe.get_solver_from_string(self.solver_param.SerializeToString()) if solver_state: # check if this file is in the current (or parent) directory or if the solver path needs to be prepended if not os.path.isfile(os.path.join('..', solver_state)): if not os.path.isfile(solver_state): solver_state = os.path.join(self.solver_dir, solver_state) if not os.path.isfile(solver_state): raise Exception('could not find solver state specified!') else: solver_state = os.path.join('..', solver_state) self.solver.restore(solver_state) if weights: raise Exception( 'should not specify both solverstate and caffemodel! Preference will be given to solverstate.') if weights and not solver_state: self.solver.net.copy_from(weights) self.sync = None self.viz_thread = None self.log_thread = None self.test_input = None self.test_out_blobs = None self.test_start_layer = None self.iteration = 0 # check if blobs update_sample_errors exist if self.se_interval and not self.se_index_blob in self.solver.net.blobs: self.logprint("WARNING: index_blob not found in net! Won't send errors to Net.") self.se_interval = 0 if self.se_interval and not self.se_error_blob in self.solver.net.blobs: self.logprint("WARNING: error_blob not found in net! Won't send errors to Net.") self.se_interval = 0