def __init__(self, freq=None, first=0, tag='', log_to_file=None, rank=None, gen_heartbeat=False, num_epochs=300, test_freq=None, test_first=0): ''' Constructor. ''' if freq is None: freq = sys.maxsize if test_freq is None: test_freq = sys.maxsize super(ProgressPrinter, self).__init__(freq, first, test_freq, test_first) self.loss_since_start = 0 self.metric_since_start = 0 self.samples_since_start = 0 self.updates_since_start = 0 self.loss_since_last = 0 self.metric_since_last = 0 self.samples_since_last = 0 self.total_updates = 0 self.epochs = 0 self.freq = freq self.first = first self.test_freq = test_freq self.tag = '' if not tag else "[{}] ".format(tag) self.epoch_start_time = time.time() self.progress_timer_time = 0 self.log_to_file = log_to_file self.gen_heartbeat = gen_heartbeat self.num_epochs = num_epochs # print out data about CNTK build cntk_py.print_built_info() self.logfilename = None if self.log_to_file is not None: self.logfilename = self.log_to_file if rank is not None: self.logfilename = self.logfilename + 'rank' + str(rank) # print to stdout print("Redirecting log to file " + self.logfilename) with open(self.logfilename, "w") as logfile: logfile.write(self.logfilename + "\n") self.___logprint('CNTKCommandTrainInfo: train : ' + str(num_epochs)) self.___logprint('CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : ' + str(num_epochs)) self.___logprint('CNTKCommandTrainBegin: train') if freq == 0: self.___logprint(' average since average since examples') self.___logprint(' loss last metric last ') self.___logprint(' ------------------------------------------------------')
def __init__(self, freq=None, first=0, tag='', log_to_file=None, rank=None, gen_heartbeat=False, num_epochs=300, test_freq=None, test_first=0): ''' Constructor. Args: freq (`int` or `None`, default `None`): determines how often printing will occur. The value of 0 means an geometric schedule (1,2,4,...). A value > 0 means a arithmetic schedule (a log print for minibatch number: ``freq``, a log print for minibatch number: 2*``freq``, a log print for minibatch number: 3*``freq``,...), and a value of None means no per-minibatch log. first (`int`, default 0): Only start logging after the minibatch number is greater or equal to ``first``. tag (`string`, default EmptyString): prepend minibatch log lines with your own string log_to_file (`string` or `None`, default `None`): if None, output log data to stdout. If a string is passed, the string is path to a file for log data. rank (`int` or `None`, default `None`): set this to distributed.rank if you are using distributed parallelism -- each rank's log will go to separate file. gen_heartbeat (`bool`, default `False`): If True output a progress message every 10 seconds or so to stdout. num_epochs (`int`, default 300): The total number of epochs to be trained. Used for some metadata. This parameter is optional. test_freq (`int` or `None`, default `None`): similar to ``freq``, but applies to printing intermediate test results. test_first (`int`, default 0): similar to ``first``, but applies to printing intermediate test results. ''' if freq is None: freq = sys.maxsize if test_freq is None: test_freq = sys.maxsize super(ProgressPrinter, self).__init__(freq, first, test_freq, test_first) self.loss_since_start = 0 self.metric_since_start = 0 self.samples_since_start = 0 self.updates_since_start = 0 self.loss_since_last = 0 self.metric_since_last = 0 self.samples_since_last = 0 self.total_updates = 0 self.epochs = 0 self.freq = freq self.first = first self.test_freq = test_freq self.tag = '' if not tag else "[{}] ".format(tag) self.epoch_start_time = time.time() self.progress_timer_time = 0 self.log_to_file = log_to_file self.gen_heartbeat = gen_heartbeat self.num_epochs = num_epochs # print out data about CNTK build cntk_py.print_built_info() self.logfilename = None if self.log_to_file is not None: self.logfilename = self.log_to_file if rank is not None: self.logfilename = self.logfilename + 'rank' + str(rank) # print to stdout print("Redirecting log to file " + self.logfilename) with open(self.logfilename, "w") as logfile: logfile.write(self.logfilename + "\n") self.___logprint('CNTKCommandTrainInfo: train : ' + str(num_epochs)) self.___logprint( 'CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : ' + str(num_epochs)) self.___logprint('CNTKCommandTrainBegin: train') if freq == 0: self.___logprint( ' average since average since examples') self.___logprint( ' loss last metric last ') self.___logprint( ' ------------------------------------------------------')
def __init__(self, freq=None, first=0, tag='', log_to_file=None, rank=None, gen_heartbeat=False, num_epochs=None, test_freq=None, test_first=0, metric_is_pct=True, distributed_freq=None, distributed_first=0): ''' Constructor. ''' if freq is None: freq = sys.maxsize if test_freq is None: test_freq = sys.maxsize if distributed_freq is None: distributed_freq = sys.maxsize super(ProgressPrinter, self).__init__(freq, first, test_freq, test_first, distributed_freq, distributed_first) self.loss_since_start = 0 self.metric_since_start = 0 self.samples_since_start = 0 self.updates_since_start = 0 self.loss_since_last = 0 self.metric_since_last = 0 self.samples_since_last = 0 self.total_updates = 0 self.epochs = 0 self.freq = freq self.first = first self.test_freq = test_freq self.tag = '' if not tag else "[{}] ".format(tag) self.epoch_start_time = time.time() self.progress_timer_time = 0 self.log_to_file = log_to_file self.gen_heartbeat = gen_heartbeat self.num_epochs = num_epochs self.metric_is_pct = metric_is_pct if metric_is_pct: self.metric_multiplier = 100.0 else: self.metric_multiplier = 1.0 self.__disown__() # print out data about CNTK build # TODO: this is for internal purposes, so find better way cntk_py.print_built_info() self.logfilename = None if self.log_to_file is not None: self.logfilename = self.log_to_file if rank is not None: self.logfilename = self.logfilename + 'rank' + str(rank) # print to stdout print("Redirecting log to file " + self.logfilename) with open(self.logfilename, "w") as logfile: logfile.write(self.logfilename + "\n") self.___logprint('CNTKCommandTrainInfo: train : ' + str(num_epochs if num_epochs is not None else 300)) self.___logprint('CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : ' + str(num_epochs if num_epochs is not None else 300)) self.___logprint('CNTKCommandTrainBegin: train') if freq == 0: self.___logprint(' average since average since examples') self.___logprint(' loss last metric last ') self.___logprint(' ------------------------------------------------------')
def __init__(self, freq=None, first=0, tag='', log_to_file=None, rank=None, gen_heartbeat=False, num_epochs=300, tensorboard_log_dir=None, model=None): ''' Constructor. The optional ``freq`` parameter determines how often printing will occur. The value of 0 means an geometric schedule (1,2,4,...). A value > 0 means a arithmetic schedule (freq, 2*freq, 3*freq,...), and a value of None means no per-minibatch log. set log_to_file if you want the output to go file instead of stdout. set rank to distributed.rank if you are using distibuted parallelism -- each rank's log will go to seperate file. ''' from sys import maxsize if freq is None: freq = maxsize self.loss_since_start = 0 self.metric_since_start = 0 self.samples_since_start = 0 self.updates_since_start = 0 self.loss_since_last = 0 self.metric_since_last = 0 self.samples_since_last = 0 self.total_updates = 0 self.epochs = 0 self.freq = freq self.first = first self.tag = '' if not tag else "[{}] ".format(tag) self.epoch_start_time = time.time() self.progress_timer_time = 0 self.log_to_file = log_to_file self.rank = rank self.gen_heartbeat = gen_heartbeat self.num_epochs = num_epochs self.trainer = None # print out data about CNTK build print_built_info() # Create TensorBoardFileWriter if the path to a log directory was provided. self.tensorboard_writer = None if tensorboard_log_dir is not None: tb_run_name = tag.lower() if tag else '' if self.rank is not None: tb_run_name += 'rank' + str(self.rank) if tb_run_name: tensorboard_log_dir = os.path.join(tensorboard_log_dir, tb_run_name) self.tensorboard_writer = TensorBoardFileWriter( tensorboard_log_dir, model) self.logfilename = None if self.log_to_file is not None: self.logfilename = self.log_to_file if self.rank != None: self.logfilename = self.logfilename + 'rank' + str(self.rank) # print to stdout print("Redirecting log to file " + self.logfilename) with open(self.logfilename, "w") as logfile: logfile.write(self.logfilename + "\n") self.___logprint('CNTKCommandTrainInfo: train : ' + str(num_epochs)) self.___logprint( 'CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : ' + str(num_epochs)) self.___logprint('CNTKCommandTrainBegin: train') if freq == 0: self.___logprint( ' average since average since examples') self.___logprint( ' loss last metric last ') self.___logprint( ' ------------------------------------------------------')
def __init__(self, freq=None, first=0, tag='', log_to_file=None, rank=None, gen_heartbeat=False, num_epochs=300, test_freq=None, test_first=0): ''' Constructor. Args: freq (`int` or `None`, default `None`): determines how often printing will occur. The value of 0 means an geometric schedule (1,2,4,...). A value > 0 means a arithmetic schedule (a log print for minibatch number: ``freq``, a log print for minibatch number: 2*``freq``, a log print for minibatch number: 3*``freq``,...), and a value of None means no per-minibatch log. first (`int`, default 0): Only start logging after the minibatch number is greater or equal to ``first``. tag (`string`, default EmptyString): prepend minibatch log lines with your own string log_to_file (`string` or `None`, default `None`): if None, output log data to stdout. If a string is passed, the string is path to a file for log data. rank (`int` or `None`, default `None`): set this to distributed.rank if you are using distributed parallelism -- each rank's log will go to separate file. gen_heartbeat (`bool`, default `False`): If True output a progress message every 10 seconds or so to stdout. num_epochs (`int`, default 300): The total number of epochs to be trained. Used for some metadata. This parameter is optional. test_freq (`int` or `None`, default `None`): similar to ``freq``, but applies to printing intermediate test results. test_first (`int`, default 0): similar to ``first``, but applies to printing intermediate test results. ''' if freq is None: freq = sys.maxsize if test_freq is None: test_freq = sys.maxsize super(ProgressPrinter, self).__init__(freq, first, test_freq, test_first) self.loss_since_start = 0 self.metric_since_start = 0 self.samples_since_start = 0 self.updates_since_start = 0 self.loss_since_last = 0 self.metric_since_last = 0 self.samples_since_last = 0 self.total_updates = 0 self.epochs = 0 self.freq = freq self.first = first self.test_freq = test_freq self.tag = '' if not tag else "[{}] ".format(tag) self.epoch_start_time = time.time() self.progress_timer_time = 0 self.log_to_file = log_to_file self.gen_heartbeat = gen_heartbeat self.num_epochs = num_epochs # print out data about CNTK build cntk_py.print_built_info() self.logfilename = None if self.log_to_file is not None: self.logfilename = self.log_to_file if rank is not None: self.logfilename = self.logfilename + 'rank' + str(rank) # print to stdout print("Redirecting log to file " + self.logfilename) with open(self.logfilename, "w") as logfile: logfile.write(self.logfilename + "\n") self.___logprint('CNTKCommandTrainInfo: train : ' + str(num_epochs)) self.___logprint('CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : ' + str(num_epochs)) self.___logprint('CNTKCommandTrainBegin: train') if freq == 0: self.___logprint(' average since average since examples') self.___logprint(' loss last metric last ') self.___logprint(' ------------------------------------------------------')