def run(self): njobs = len(self.jobs) dataset = Dataset(self.path) if self.n_cache <= 1: cache_limit = max(2, int(0.12 * njobs)) else: cache_limit = int(self.n_cache) # ====== indices ====== # databases = defaultdictkey( lambda key: MmapDict(path=os.path.join(dataset.path, key), cache_size=10000, read_only=False)) last_start = defaultdict(int) # ====== statistic ====== # # load old statistics stats = defaultdict(lambda: [0, 0]) # name -> (sum1, sum2) for key in dataset.keys(): if 'sum1' == key[-4]: stats[key[:-4]][0] = dataset[key][:] elif 'sum2' == key[-4:]: stats[key[:-4]][1] = dataset[key][:] # all data are cached for periodically flushed cache = defaultdict(list) n_processed = [0] # store the value as reference # ====== helper ====== # def flush_feature(feat_name, X_cached): if len(X_cached) > 0: X_cached = np.concatenate(X_cached, 0) # flush data if feat_name in dataset: dataset[feat_name].append(X_cached) else: dataset[(feat_name, 'memmap')] = X_cached # ====== repeated for each result returned ====== # def post_processing(result): # search for file name if self.identifier not in result: raise RuntimeError( "Cannot find identifier '%s' in returned dictionary" % self.identifier) file_name = result[self.identifier] # invalid file_name if not is_string(file_name): raise RuntimeError( "Cannot find file name in returned features " "list, the file name can be specified in key: 'name', 'path' " "and the type of the value must be string. All available " "keys are: %s" % str(result.keys())) # store all new indices # mapping [X.shape[0]] -> [feat_name, feat_name, ...] all_indices = {} # processing for feat_name, X in result.items(): # some invalid feat_name if feat_name in ('config', 'pipeline', 'sum1', 'sum2'): raise RuntimeError( "Returned features' name cannot be one " "of the following: 'config', 'pipeline', 'sum1', 'sum2'." ) # ignore some feat_name if feat_name in ('name'): continue # if numpy ndarray, save to MmapData if isinstance(X, np.ndarray) or \ 'sum1' == feat_name[-4:] or \ 'sum2' == feat_name[-4:]: # save statistics instead if 'sum1' == feat_name[-4:]: stats[feat_name[:-4]][0] += X elif 'sum2' == feat_name[-4:]: stats[feat_name[:-4]][1] += X # save features array else: all_indices[feat_name] = X.shape[0] # cache data, only if we have more than 0 sample if X.shape[0] > 0: cache[feat_name].append(X) # else all other kind of data save to MmapDict else: databases[feat_name][file_name] = X # remove data del X # ====== update indices ====== # if len(all_indices) > 0: for feat_name, n in all_indices.items(): ids_name = 'indices_%s' % feat_name databases[ids_name][file_name] = (last_start[ids_name], last_start[ids_name] + n) last_start[ids_name] += n # ====== flush cache ====== # n_processed[0] += 1 if n_processed[0] % cache_limit == 0: # 12 + 8 for feat_name, X_cached in cache.items(): flush_feature(feat_name, X_cached) cache.clear() # ====== update progress ====== # return file_name # ====== mapping function ====== # def _map_func(dat): try: ret = self.extractor.transform(dat) except Exception as e: # Non-handled exception ret = '\n========\n' ret += 'Time : `%s`\n' % str( get_formatted_datetime(only_number=False)) ret += 'Error : `%s`\n' % str(e) ret += 'Input : `%s`\n' % str(dat) import traceback etype, value, tb = sys.exc_info() for line in traceback.TracebackException( type(value), value, tb, limit=None).format(chain=True): ret += line return ret # ====== processing ====== # mpi = MPI(jobs=self.jobs, func=_map_func, ncpu=self.n_cpu, batch=1, hwm=self.n_cpu * 3, backend='python') # initialize prog = Progbar(target=njobs, name=self.path, interval=0.12, print_report=True, print_summary=True) start_time = time.time() last_time = time.time() last_count = 0 with open(self._log_path, 'w') as flog: # writing the log head flog.write('============================\n') flog.write('Start Time : %s\n' % get_formatted_datetime(only_number=False)) flog.write('Outpath : %s\n' % self.path) flog.write('Extractor : %s\n' % '->'.join( [s[-1].__class__.__name__ for s in self.extractor.steps])) flog.write('#Jobs : %d\n' % njobs) flog.write('#CPU : %d\n' % self.n_cpu) flog.write('#Cache : %d\n' % cache_limit) flog.write('============================\n') flog.flush() # start processing the file list for count, result in enumerate(mpi): # Non-handled exception if isinstance(result, string_types): flog.write(result) flog.flush() self._error_log.append(result) if self.stop_on_failure: raise RuntimeError(result) # some error might happened elif isinstance(result, ExtractorSignal): flog.write(str(result)) flog.flush() if result.action == 'error': prog.add_notification(str(result)) raise RuntimeError( "ExtractorSignal requests terminating processor!") elif result.action == 'warn': prog.add_notification(str(result)) elif result.action == 'ignore': self._error_log.append(result) else: raise RuntimeError( "Unknown action from ExtractorSignal: %s" % result.action) prog['File'] = '%-48s' % result.message[:48] # otherwise, no error happened, do post-processing else: name = post_processing(result) prog['File'] = '%-48s' % str(name)[:48] # update progress prog.add(1) # manually write to external log file if (count + 1) % max(1, int(0.01 * njobs)) == 0: curr_time = time.time() elap = curr_time - start_time avg_speed = (count + 1) / elap cur_speed = (count + 1 - last_count) / (curr_time - last_time) avg_est = (njobs - count - 1) / avg_speed cur_est = (njobs - count - 1) / cur_speed flog.write( '[%s] Processed: %d(files) Remain: %d(files) Elap.: %.2f(secs)\n' ' Avg.Spd: %.2f(obj/sec) Avg.Est.: %.2f(secs)\n' ' Cur.Spd: %.2f(obj/sec) Cur.Est.: %.2f(secs)\n' % (get_formatted_datetime(only_number=False), count + 1, njobs - count - 1, elap, avg_speed, avg_est, cur_speed, cur_est)) flog.flush() last_time = curr_time last_count = count + 1 # ====== end, flush the last time ====== # for feat_name, X_cached in cache.items(): flush_feature(feat_name, X_cached) cache.clear() cache = None dataset.flush() prog.add_notification("Flushed all data to disk") # ====== saving indices ====== # for name, db in databases.items(): db.flush(save_all=True) db_size = len(db) db.close() prog.add_notification( 'Flush MmapDict "%s" to disk, size: %s' % (ctext(name, 'yellow'), ctext(str(db_size), 'yellow'))) # ====== save mean and std ====== # def save_mean_std(sum1, sum2, name): N = dataset[name.split('_')[0]].shape[0] mean = sum1 / N std = np.sqrt(sum2 / N - np.power(mean, 2)) if np.any(np.isnan(mean)): wprint('Mean contains NaN, name: %s' % name) if np.any(np.isnan(std)): wprint('Std contains NaN, name: %s' % name) dataset[name + 'sum1'] = sum1 dataset[name + 'sum2'] = sum2 dataset[name + 'mean'] = mean dataset[name + 'std'] = std # save all stats if len(stats) > 0: for feat_name, (sum1, sum2) in stats.items(): save_mean_std(sum1, sum2, feat_name) prog.add_notification( 'Saved statistics of: %s, shape: %s' % (ctext(feat_name.split('_')[0], 'yellow'), ctext(str(sum1.shape), 'yellow'))) # ====== dataset flush() ====== # dataset.flush() dataset.close() # ====== saving the extractor ====== # # not good idea to save the extractor all the time # pipeline_path = os.path.join(dataset.path, 'pipeline') # with open(pipeline_path, 'wb') as f: # cPickle.dump(self.extractor, f, protocol=2) # prog.add_notification("Saved Extractor pipeline at: %s" % # ctext(pipeline_path, 'yellow')) # ====== saving the configuration ====== # config_path = os.path.join(dataset.path, 'config') config = MmapDict(config_path) config['__configuration_time__'] = time.time() config['__processor__'] = self.path for i in dir(self): if _default_module.match(i) is not None: continue j = getattr(self, i) if isinstance(j, (Number, string_types, bool)): config[i] = j config.flush(save_all=True) self.config = {i: j for i, j in config} config.close() prog.add_notification("Saved configuration at: %s" % ctext(config_path, 'yellow')) # ====== final notification ====== # prog.add_notification("Closed all dataset.") prog.add_notification("Dataset at path: %s" % ctext(dataset.path, 'yellow'))
class Task(object): """ Parameters ---------- func: call-able function will be executed for each iteration data: single or list of odin.fuel.Data, numpy.ndarray iterate over all these data and execute function on the data. epoch: int how many epoch will be repeated p: float (0.0 - 1.0) probability the `func` will be execute for each iteration batch_size: int (> 0) number of samples for each iteration seed: int random seed for shuffling the data shuffle_level: int (0, 1, 2) if 0, shuffle the file lists if 1, shuffle the buffer (i.e. list of processing files) and all the previous if 2, shuffle the returned batch and all the previous callbacks: None, or list of `odin.training.Callback` callback will be promoted during the execution of the task labels: None, or list of string labels for printing the confusion matrix in `odin.utils.Progbar` name: None or string unique name for Task identity. verbose : {0, 1, 2, 3, 4} specific verbose level controlling the log output 0 - Turn off all log 1 - progress off, only notification 2 - progress off, notification and summary 3 - progress on, nothing else 4 - progress on, notification and summary 5 - progress on, notification, summary and batch report """ def __init__(self, func, data, epoch=1, p=1.0, batch_size=128, seed=None, shuffle_level=2, callbacks=None, labels=None, name=None, verbose=2): super(Task, self).__init__() self.set_func(func, data) # this Progbar will record the history as well self._labels = [str(l) for l in labels] \ if labels is not None else None self._progbar = Progbar(target=self.nb_samples, name=name, interval=0., print_report=True, print_summary=True) self._progbar.set_labels(self._labels) # ====== set callback and verbose ====== # self._callback = CallbackList(callbacks) self.set_verbose(verbose) # ====== assign other arguments ====== # self._nb_epoch = epoch self._p = np.clip(p, 0., 1.) self._seed = seed self.set_batch(batch_size, seed, shuffle_level) self._name = name # ====== current info ====== # self._curr_epoch = 0 self._curr_iter = 0 self._curr_samples = 0 self._curr_epoch_iter = 0 self._curr_epoch_samples = 0 self._callback_msg = [] # ====== iter tracking ====== # self._created_iter = None self._stop = False def __str__(self): return "<Task:'%s' p:%s bs:%s #ep:%s/%s #it:%s/%s #n:%s/%s %s>" % \ (ctext(self.name, 'lightyellow'), ctext(self.probability, 'cyan'), ctext(self.batch_size, 'cyan'), ctext(self.curr_epoch, 'lightcyan'), ctext(self.nb_epoch, 'cyan'), ctext(self.curr_epoch_iter, 'lightcyan'), ctext(self.curr_iter, 'cyan'), ctext(self.curr_epoch_samples, 'lightcyan'), ctext(self.curr_samples, 'cyan'), ','.join([ctext(i.__class__.__name__, 'cyan') for i in self._callback._callbacks])) def __getstate__(self): return (self._progbar, self._nb_epoch, self._p, self._name, self._batch_size, self._rng, self._seed, self._shuffle_level, self._verbose) def __setstate__(self, states): (self._progbar, self._nb_epoch, self._p, self._name, self._batch_size, self._rng, self._seed, self._shuffle_level, self._verbose) = states # ====== current info ====== # self._curr_epoch = 0 self._curr_iter = 0 self._curr_samples = 0 self._curr_epoch_iter = 0 self._curr_epoch_samples = 0 self._callback_msg = [] # ====== iter tracking ====== # self._created_iter = None self._stop = False # ====== reset value of func and data ====== # self._func = None self._data = None def set_callbacks(self, callbacks): self._callback.set_callbacks(callbacks) if self._verbose == 0: self._callback.set_notification(False) else: self._callback.set_notification(True) return self def set_verbose(self, verbose): verbose = int(verbose) self._verbose = verbose if verbose == 0: # turn off everything self._callback.set_notification(False) self._progbar.print_progress = False self._progbar.print_summary = False self._progbar.print_report = False elif verbose == 1: # progress off, only notification self._callback.set_notification(True) self._progbar.print_progress = False self._progbar.print_summary = False self._progbar.print_report = False elif verbose == 2: # progress off, notification + summary self._callback.set_notification(True) self._progbar.print_progress = False self._progbar.print_summary = True self._progbar.print_report = False elif verbose == 3: # progress on, nothing else self._callback.set_notification(False) self._progbar.print_progress = True self._progbar.print_summary = False self._progbar.print_report = False elif verbose == 4: # progress on, notification + summary self._callback.set_notification(True) self._progbar.print_progress = True self._progbar.print_summary = True self._progbar.print_report = False elif verbose == 5: # progress on, notification, report, summary self._callback.set_notification(True) self._progbar.print_progress = True self._progbar.print_summary = True self._progbar.print_report = True else: raise ValueError( "Only support verbose value: 0, 1, 2, 3, 4, 5; but given: %s" % str(verbose)) def set_func(self, func, data): # ====== check function ====== # self._func = func if isinstance(func, K.Function): self._output_info = [(o.name, o.shape.as_list()) for o in self._func.outputs] elif hasattr(func, '__call__'): self._output_info = [] # No info (normal function) else: raise ValueError("No support for function type: %s" % func.__class__.__name__) # ====== check data ====== # if not isinstance(data, (tuple, list)): data = [data] self._data = [fuel.as_data(i, copy=not isinstance(i, fuel.Feeder)) for i in data] self._nb_samples = min([d.iter_len for d in self._data]) return self def set_batch(self, batch_size=None, seed=-1, shuffle_level=None): if batch_size is not None: self._batch_size = batch_size if seed is None or seed >= 0: if seed is not None: self._rng = np.random.RandomState(seed) else: self._rng = struct() self._rng.randint = lambda x: None self._rng.rand = get_rng().rand if shuffle_level is not None: self._shuffle_level = min(max(int(shuffle_level), 0), 2) return self # ==================== Properties ==================== # @property def history(self): """ Return : dictionary type {epoch_id : {tensor_name0: [batch_return1, batch_return2, ...], tensor_name1: [batch_return1, batch_return2, ...], ...}, 1 : {tensor_name0: [batch_return1, batch_return2, ...], tensor_name1: [batch_return1, batch_return2, ...], ...}, ... } Example ------- >>> for task_name, task_hist in task.history.items(): >>> print(task_name) >>> for epoch_id, values in task_hist.items(): >>> print(' Epoch:', epoch_id) >>> for tensor_name, v in values.items(): >>> print(' ', tensor_name, len(v)) """ return self._progbar.history @property def progbar(self): return self._progbar @property def name(self): return str(self._name) @property def labels(self): return self._labels @property def nb_epoch(self): return self._nb_epoch @property def nb_samples(self): ''' Estimated number of iteration for each epoch ''' return self._nb_samples @property def probability(self): """Chance that the func will be execute during iteration""" return self._p @property def iter_per_epoch(self): ''' Estimated number of iteration for each epoch ''' return int(np.ceil(self._nb_samples / self._batch_size)) @property def batch_size(self): return self._batch_size @property def curr_epoch(self): """Total number of epoch finished since the beginning of the Task""" return self._curr_epoch @property def curr_iter(self): """Total number of iteration finished since the beginning of the Task""" return self._curr_iter @property def curr_samples(self): """Total number of samples finished since the beginning of the Task""" return self._curr_samples @property def curr_epoch_iter(self): """Number of iteration within current epoch""" return self._curr_epoch_iter @property def curr_epoch_samples(self): """Number of samples within current epoch""" return self._curr_epoch_samples @property def callback_msg(self): return self._callback_msg # ==================== control function ==================== # def stop(self): """ Stop all iterations running for this Task""" if self._created_iter is not None: self._stop = True # just run to end of the iterators for i in self._created_iter: pass self._stop = False self._created_iter = None def copy(self): return Task(self._func, self._data, epoch=self.nb_epoch, p=self.probability, batch_size=self.batch_size, seed=self._seed, shuffle_level=self._shuffle_level, name=self._name, verbose=self._verbose) def __iter(self): ''' Return ------ One of the following: * 'task_start': * 'epoch_start' : beginning of epoch * 'epoch_end' : epoch ended * 'task_end' : task ended * (results, nb_iter, nb_samples, nb_total_samples, nb_epoch) : results of execute function on data Note ---- 'end_task' also end of final epoch ''' yield None # just for initalize the iterator self._callback_msg = self._callback.task_start(self) yield 'task_start' if self._stop: yield 'task_end' else: # ====== start of training ====== # while self._curr_epoch < self._nb_epoch: self._callback_msg = self._callback.epoch_start(self, self._data) yield 'epoch_start' seed = self._rng.randint(10e8) # if only 1 Data, don't need zip or we will mess up if len(self._data) == 1: data_it = iter(self._data[0].set_batch(batch_size=self._batch_size, seed=seed, shuffle_level=self._shuffle_level)) data = data_it else: data_it = [iter(d.set_batch(batch_size=self._batch_size, seed=seed, shuffle_level=self._shuffle_level)) for d in self._data] data = zip(*data_it) # ====== start the iteration ====== # self._curr_epoch_samples = 0 self._curr_epoch_iter = 0 with self._progbar.safe_progress(): for i, x in enumerate(data): # alread terminated, try to exhausted the iterator # if forced_to_terminate: continue # preprocessed the data if not isinstance(x, (tuple, list)): x = [x] # update some info shape0 = x[0].shape[0] self._curr_samples += shape0 self._curr_iter += 1 self._curr_epoch_samples += shape0 self._curr_epoch_iter += 1 self._callback_msg = self._callback.batch_start(self, x) # apply the function if self.probability >= 1. or self._rng.rand() < self.probability: results = self._func(*x) # add msg from batch_end event self._callback_msg += self._callback.batch_end(self, results) # return results yield results # update the progress bar for (name, shape), res in zip(self._output_info, as_tuple(results)): if len(shape) == 0: # return single value self._progbar[name] = res else: # return tensor self._progbar[name] = res self._progbar.add(shape0) # check TERMINATE signal if self._stop: # send signal to the data iterators also for i in data_it: if hasattr(i, 'stop'): i.stop() else: # just iterate all over for _ in i: pass # break the epoch loop break ### Epoch end signaling self._curr_epoch += 1 self._callback_msg = self._callback.epoch_end( self, self._progbar.history[self._curr_epoch - 1]) yield 'epoch_end' # ====== check if we got the right number for epoch iter ====== # if self._curr_epoch_samples != self._nb_samples: # just for sure should not smaller than the real number self._nb_samples = self._curr_epoch_samples # ====== end_epoch or task ====== # if self._stop or self._curr_epoch >= self._nb_epoch: self._callback_msg = self._callback.task_end( self, self._progbar.history) yield 'task_end' # showing notification if self._verbose >= 1 and self._verbose != 3: self._progbar.add_notification('Task "%s" ended!' % str(self.name)) break # ====== end of iteration ====== # self._created_iter = None def __iter__(self): if self._created_iter is None: # reset all information self._curr_epoch = 0 self._curr_iter = 0 self._curr_samples = 0 self._curr_epoch_iter = 0 self._curr_epoch_samples = 0 self._callback_msg = [] # create new iter self._created_iter = self.__iter() # initialize the iteration next(self._created_iter) return self._created_iter def __del__(self): self.stop()
class Task(object): """ Parameters ---------- func: call-able function will be executed for each iteration data: single or list of odin.fuel.Data, numpy.ndarray iterate over all these data and execute function on the data. epoch: int how many epoch will be repeated p: float (0.0 - 1.0) probability the `func` will be execute for each iteration batch_size: int (> 0) number of samples for each iteration seed: int random seed for shuffling the data shuffle_level: int (0, 1, 2) if 0, shuffle the file lists if 1, shuffle the buffer (i.e. list of processing files) and all the previous if 2, shuffle the returned batch and all the previous callbacks: None, or list of `odin.training.Callback` callback will be promoted during the execution of the task labels: None, or list of string labels for printing the confusion matrix in `odin.utils.Progbar` name: None or string unique name for Task identity. verbose : {0, 1, 2, 3, 4} specific verbose level controlling the log output 0 - Turn off all log 1 - progress off, only notification 2 - progress off, notification and summary 3 - progress on, nothing else 4 - progress on, notification and summary 5 - progress on, notification, summary and batch report """ def __init__(self, func, data, epoch=1, p=1.0, batch_size=128, seed=None, shuffle_level=2, callbacks=None, labels=None, name=None, verbose=2): super(Task, self).__init__() self.set_func(func, data) # this Progbar will record the history as well self._labels = [str(l) for l in labels] \ if labels is not None else None self._progbar = Progbar(target=self.nb_samples, name=name, interval=0., print_report=True, print_summary=True) self._progbar.set_labels(self._labels) # ====== set callback and verbose ====== # self._callback = CallbackList(callbacks) self.set_verbose(verbose) # ====== assign other arguments ====== # self._nb_epoch = epoch self._p = np.clip(p, 0., 1.) self._seed = seed self.set_batch(batch_size, seed, shuffle_level) self._name = name # ====== current info ====== # self._curr_epoch = 0 self._curr_iter = 0 self._curr_samples = 0 self._curr_epoch_iter = 0 self._curr_epoch_samples = 0 self._callback_msg = [] # ====== iter tracking ====== # self._created_iter = None self._stop = False def __str__(self): return "<Task:'%s' p:%s bs:%s #ep:%s/%s #it:%s/%s #n:%s/%s %s>" % \ (ctext(self.name, 'lightyellow'), ctext(self.probability, 'cyan'), ctext(self.batch_size, 'cyan'), ctext(self.curr_epoch, 'lightcyan'), ctext(self.nb_epoch, 'cyan'), ctext(self.curr_epoch_iter, 'lightcyan'), ctext(self.curr_iter, 'cyan'), ctext(self.curr_epoch_samples, 'lightcyan'), ctext(self.curr_samples, 'cyan'), ','.join([ctext(i.__class__.__name__, 'cyan') for i in self._callback._callbacks])) def __getstate__(self): return (self._progbar, self._nb_epoch, self._p, self._name, self._batch_size, self._rng, self._seed, self._shuffle_level, self._verbose) def __setstate__(self, states): (self._progbar, self._nb_epoch, self._p, self._name, self._batch_size, self._rng, self._seed, self._shuffle_level, self._verbose) = states # ====== current info ====== # self._curr_epoch = 0 self._curr_iter = 0 self._curr_samples = 0 self._curr_epoch_iter = 0 self._curr_epoch_samples = 0 self._callback_msg = [] # ====== iter tracking ====== # self._created_iter = None self._stop = False # ====== reset value of func and data ====== # self._func = None self._data = None def set_callbacks(self, callbacks): self._callback.set_callbacks(callbacks) if self._verbose == 0: self._callback.set_notification(False) else: self._callback.set_notification(True) return self def set_verbose(self, verbose): verbose = int(verbose) self._verbose = verbose if verbose == 0: # turn off everything self._callback.set_notification(False) self._progbar.print_progress = False self._progbar.print_summary = False self._progbar.print_report = False elif verbose == 1: # progress off, only notification self._callback.set_notification(True) self._progbar.print_progress = False self._progbar.print_summary = False self._progbar.print_report = False elif verbose == 2: # progress off, notification + summary self._callback.set_notification(True) self._progbar.print_progress = False self._progbar.print_summary = True self._progbar.print_report = False elif verbose == 3: # progress on, nothing else self._callback.set_notification(False) self._progbar.print_progress = True self._progbar.print_summary = False self._progbar.print_report = False elif verbose == 4: # progress on, notification + summary self._callback.set_notification(True) self._progbar.print_progress = True self._progbar.print_summary = True self._progbar.print_report = False elif verbose == 5: # progress on, notification, report, summary self._callback.set_notification(True) self._progbar.print_progress = True self._progbar.print_summary = True self._progbar.print_report = True else: raise ValueError( "Only support verbose value: 0, 1, 2, 3, 4, 5; but given: %s" % str(verbose)) def set_func(self, func, data): # ====== check function ====== # self._func = func if isinstance(func, K.Function): self._output_info = [(o.name, o.shape.as_list()) for o in self._func.outputs] elif hasattr(func, '__call__'): self._output_info = [] # No info (normal function) else: raise ValueError("No support for function type: %s" % func.__class__.__name__) # ====== check data ====== # if not isinstance(data, (tuple, list)): data = [data] self._data = [fuel.as_data(i, copy=not isinstance(i, fuel.Feeder)) for i in data] self._nb_samples = min([d.iter_len for d in self._data]) return self def set_batch(self, batch_size=None, seed=-1, shuffle_level=None): if batch_size is not None: self._batch_size = batch_size if seed is None or seed >= 0: if seed is not None: self._rng = np.random.RandomState(seed) else: self._rng = struct() self._rng.randint = lambda x: None self._rng.rand = get_rng().rand if shuffle_level is not None: self._shuffle_level = min(max(int(shuffle_level), 0), 2) return self # ==================== Properties ==================== # @property def history(self): """ Return : dictionary type {epoch_id : {tensor_name0: [batch_return1, batch_return2, ...], tensor_name1: [batch_return1, batch_return2, ...], ...}, 1 : {tensor_name0: [batch_return1, batch_return2, ...], tensor_name1: [batch_return1, batch_return2, ...], ...}, ... } Example ------- >>> for task_name, task_hist in task.history.items(): >>> print(task_name) >>> for epoch_id, values in task_hist.items(): >>> print(' Epoch:', epoch_id) >>> for tensor_name, v in values.items(): >>> print(' ', tensor_name, len(v)) """ return self._progbar.history @property def progbar(self): return self._progbar @property def name(self): return str(self._name) @property def labels(self): return self._labels @property def nb_epoch(self): return self._nb_epoch @property def nb_samples(self): ''' Estimated number of iteration for each epoch ''' return self._nb_samples @property def probability(self): """Chance that the func will be execute during iteration""" return self._p @property def iter_per_epoch(self): ''' Estimated number of iteration for each epoch ''' return int(np.ceil(self._nb_samples / self._batch_size)) @property def batch_size(self): return self._batch_size @property def curr_epoch(self): """Total number of epoch finished since the beginning of the Task""" return self._curr_epoch @property def curr_iter(self): """Total number of iteration finished since the beginning of the Task""" return self._curr_iter @property def curr_samples(self): """Total number of samples finished since the beginning of the Task""" return self._curr_samples @property def curr_epoch_iter(self): """Number of iteration within current epoch""" return self._curr_epoch_iter @property def curr_epoch_samples(self): """Number of samples within current epoch""" return self._curr_epoch_samples @property def callback_msg(self): return self._callback_msg # ==================== control function ==================== # def stop(self): """ Stop all iterations running for this Task""" if self._created_iter is not None: self._stop = True # just run to end of the iterators for i in self._created_iter: pass self._stop = False self._created_iter = None def copy(self): return Task(self._func, self._data, epoch=self.nb_epoch, p=self.probability, batch_size=self.batch_size, seed=self._seed, shuffle_level=self._shuffle_level, name=self._name, verbose=self._verbose) def __iter(self): ''' Return ------ One of the following: * 'task_start': * 'epoch_start' : beginning of epoch * 'epoch_end' : epoch ended * 'task_end' : task ended * (results, nb_iter, nb_samples, nb_total_samples, nb_epoch) : results of execute function on data Note ---- 'end_task' also end of final epoch ''' yield None # just for initalize the iterator self._callback_msg = self._callback.task_start(self) yield 'task_start' if self._stop: yield 'task_end' else: # ====== start of training ====== # while self._curr_epoch < self._nb_epoch: self._callback_msg = self._callback.epoch_start(self, self._data) yield 'epoch_start' seed = self._rng.randint(10e8) # if only 1 Data, don't need zip or we will mess up if len(self._data) == 1: data_it = iter(self._data[0].set_batch(batch_size=self._batch_size, seed=seed, shuffle_level=self._shuffle_level)) data = data_it else: data_it = [iter(d.set_batch(batch_size=self._batch_size, seed=seed, shuffle_level=self._shuffle_level)) for d in self._data] data = zip(*data_it) # ====== start the iteration ====== # self._curr_epoch_samples = 0 self._curr_epoch_iter = 0 with self._progbar.safe_progress(): for i, x in enumerate(data): # alread terminated, try to exhausted the iterator # if forced_to_terminate: continue # preprocessed the data if not isinstance(x, (tuple, list)): x = [x] # update some info shape0 = x[0].shape[0] self._curr_samples += shape0 self._curr_iter += 1 self._curr_epoch_samples += shape0 self._curr_epoch_iter += 1 self._callback_msg = self._callback.batch_start(self, x) # apply the function if self.probability >= 1. or self._rng.rand() < self.probability: results = self._func(*x) # add msg from batch_end event self._callback_msg += self._callback.batch_end(self, results) # return results yield results # update the progress bar for (name, shape), res in zip(self._output_info, as_tuple(results)): if len(shape) == 0: # return single value self._progbar[name] = res else: # return tensor self._progbar[name] = res self._progbar.add(shape0) # check TERMINATE signal if self._stop: # send signal to the data iterators also for i in data_it: if hasattr(i, 'stop'): i.stop() else: # just iterate all over for _ in i: pass # break the epoch loop break ### Epoch end signaling self._curr_epoch += 1 self._callback_msg = self._callback.epoch_end( self, self._progbar.history[self._curr_epoch - 1]) yield 'epoch_end' # ====== check if we got the right number for epoch iter ====== # if self._curr_epoch_samples != self._nb_samples: # just for sure should not smaller than the real number self._nb_samples = self._curr_epoch_samples # ====== end_epoch or task ====== # if self._stop or self._curr_epoch >= self._nb_epoch: self._callback_msg = self._callback.task_end( self, self._progbar.history) yield 'task_end' # showing notification if self._verbose >= 1 and self._verbose != 3: self._progbar.add_notification('Task "%s" ended!' % str(self.name)) break # ====== end of iteration ====== # self._created_iter = None def __iter__(self): if self._created_iter is None: # reset all information self._curr_epoch = 0 self._curr_iter = 0 self._curr_samples = 0 self._curr_epoch_iter = 0 self._curr_epoch_samples = 0 self._callback_msg = [] # create new iter self._created_iter = self.__iter() # initialize the iteration next(self._created_iter) return self._created_iter def __del__(self): self.stop()
def run(self): njobs = len(self.jobs) dataset = Dataset(self.path) if self.n_cache <= 1: cache_limit = max(2, int(0.12 * njobs)) else: cache_limit = int(self.n_cache) # ====== indices ====== # databases = defaultdictkey(lambda key: MmapDict(path=os.path.join(dataset.path, key), cache_size=10000, read_only=False)) last_start = defaultdict(int) # ====== statistic ====== # # load old statistics stats = defaultdict(lambda: [0, 0]) # name -> (sum1, sum2) for key in dataset.keys(): if 'sum1' == key[-4]: stats[key[:-4]][0] = dataset[key][:] elif 'sum2' == key[-4:]: stats[key[:-4]][1] = dataset[key][:] # all data are cached for periodically flushed cache = defaultdict(list) n_processed = [0] # store the value as reference # ====== helper ====== # def flush_feature(feat_name, X_cached): if len(X_cached) > 0: X_cached = np.concatenate(X_cached, 0) # flush data if feat_name in dataset: dataset[feat_name].append(X_cached) else: dataset[(feat_name, 'memmap')] = X_cached # ====== repeated for each result returned ====== # def post_processing(result): # search for file name if self.identifier not in result: raise RuntimeError( "Cannot find identifier '%s' in returned dictionary" % self.identifier) file_name = result[self.identifier] # invalid file_name if not is_string(file_name): raise RuntimeError("Cannot find file name in returned features " "list, the file name can be specified in key: 'name', 'path' " "and the type of the value must be string. All available " "keys are: %s" % str(result.keys())) # store all new indices # mapping [X.shape[0]] -> [feat_name, feat_name, ...] all_indices = {} # processing for feat_name, X in result.items(): # some invalid feat_name if feat_name in ('config', 'pipeline', 'sum1', 'sum2'): raise RuntimeError("Returned features' name cannot be one " "of the following: 'config', 'pipeline', 'sum1', 'sum2'.") # ignore some feat_name if feat_name in ('name'): continue # if numpy ndarray, save to MmapData if isinstance(X, np.ndarray) or \ 'sum1' == feat_name[-4:] or \ 'sum2' == feat_name[-4:]: # save statistics instead if 'sum1' == feat_name[-4:]: stats[feat_name[:-4]][0] += X elif 'sum2' == feat_name[-4:]: stats[feat_name[:-4]][1] += X # save features array else: all_indices[feat_name] = X.shape[0] # cache data, only if we have more than 0 sample if X.shape[0] > 0: cache[feat_name].append(X) # else all other kind of data save to MmapDict else: databases[feat_name][file_name] = X # remove data del X # ====== update indices ====== # if len(all_indices) > 0: for feat_name, n in all_indices.items(): ids_name = 'indices_%s' % feat_name databases[ids_name][file_name] = (last_start[ids_name], last_start[ids_name] + n) last_start[ids_name] += n # ====== flush cache ====== # n_processed[0] += 1 if n_processed[0] % cache_limit == 0: # 12 + 8 for feat_name, X_cached in cache.items(): flush_feature(feat_name, X_cached) cache.clear() # ====== update progress ====== # return file_name # ====== mapping function ====== # def _map_func(dat): try: ret = self.extractor.transform(dat) except Exception as e: # Non-handled exception ret = '\n========\n' ret += 'Time : `%s`\n' % str(get_formatted_datetime(only_number=False)) ret += 'Error : `%s`\n' % str(e) ret += 'Input : `%s`\n' % str(dat) import traceback etype, value, tb = sys.exc_info() for line in traceback.TracebackException( type(value), value, tb, limit=None).format(chain=True): ret += line return ret # ====== processing ====== # mpi = MPI(jobs=self.jobs, func=_map_func, ncpu=self.n_cpu, batch=1, hwm=self.n_cpu * 3, backend='python') # initialize prog = Progbar(target=njobs, name=self.path, interval=0.12, print_report=True, print_summary=True) start_time = time.time() last_time = time.time() last_count = 0 with open(self._log_path, 'w') as flog: # writing the log head flog.write('============================\n') flog.write('Start Time : %s\n' % get_formatted_datetime(only_number=False)) flog.write('Outpath : %s\n' % self.path) flog.write('Extractor : %s\n' % '->'.join([s[-1].__class__.__name__ for s in self.extractor.steps])) flog.write('#Jobs : %d\n' % njobs) flog.write('#CPU : %d\n' % self.n_cpu) flog.write('#Cache : %d\n' % cache_limit) flog.write('============================\n') flog.flush() # start processing the file list for count, result in enumerate(mpi): # Non-handled exception if isinstance(result, string_types): flog.write(result) flog.flush() self._error_log.append(result) if self.stop_on_failure: raise RuntimeError(result) # some error might happened elif isinstance(result, ExtractorSignal): flog.write(str(result)); flog.flush() if result.action == 'error': prog.add_notification(str(result)) raise RuntimeError("ExtractorSignal requests terminating processor!") elif result.action == 'warn': prog.add_notification(str(result)) elif result.action == 'ignore': self._error_log.append(result) else: raise RuntimeError("Unknown action from ExtractorSignal: %s" % result.action) prog['File'] = '%-48s' % result.message[:48] # otherwise, no error happened, do post-processing else: name = post_processing(result) prog['File'] = '%-48s' % str(name)[:48] # update progress prog.add(1) # manually write to external log file if (count + 1) % max(1, int(0.01 * njobs)) == 0: curr_time = time.time() elap = curr_time - start_time avg_speed = (count + 1) / elap cur_speed = (count + 1 - last_count) / (curr_time - last_time) avg_est = (njobs - count - 1) / avg_speed cur_est = (njobs - count - 1) / cur_speed flog.write('[%s] Processed: %d(files) Remain: %d(files) Elap.: %.2f(secs)\n' ' Avg.Spd: %.2f(obj/sec) Avg.Est.: %.2f(secs)\n' ' Cur.Spd: %.2f(obj/sec) Cur.Est.: %.2f(secs)\n' % (get_formatted_datetime(only_number=False), count + 1, njobs - count - 1, elap, avg_speed, avg_est, cur_speed, cur_est)) flog.flush() last_time = curr_time last_count = count + 1 # ====== end, flush the last time ====== # for feat_name, X_cached in cache.items(): flush_feature(feat_name, X_cached) cache.clear() cache = None dataset.flush() prog.add_notification("Flushed all data to disk") # ====== saving indices ====== # for name, db in databases.items(): db.flush(save_all=True) db_size = len(db) db.close() prog.add_notification('Flush MmapDict "%s" to disk, size: %s' % (ctext(name, 'yellow'), ctext(str(db_size), 'yellow'))) # ====== save mean and std ====== # def save_mean_std(sum1, sum2, name): N = dataset[name.split('_')[0]].shape[0] mean = sum1 / N std = np.sqrt(sum2 / N - np.power(mean, 2)) if np.any(np.isnan(mean)): wprint('Mean contains NaN, name: %s' % name) if np.any(np.isnan(std)): wprint('Std contains NaN, name: %s' % name) dataset[name + 'sum1'] = sum1 dataset[name + 'sum2'] = sum2 dataset[name + 'mean'] = mean dataset[name + 'std'] = std # save all stats if len(stats) > 0: for feat_name, (sum1, sum2) in stats.items(): save_mean_std(sum1, sum2, feat_name) prog.add_notification('Saved statistics of: %s, shape: %s' % (ctext(feat_name.split('_')[0], 'yellow'), ctext(str(sum1.shape), 'yellow'))) # ====== dataset flush() ====== # dataset.flush() dataset.close() # ====== saving the extractor ====== # # not good idea to save the extractor all the time # pipeline_path = os.path.join(dataset.path, 'pipeline') # with open(pipeline_path, 'wb') as f: # cPickle.dump(self.extractor, f, protocol=2) # prog.add_notification("Saved Extractor pipeline at: %s" % # ctext(pipeline_path, 'yellow')) # ====== saving the configuration ====== # config_path = os.path.join(dataset.path, 'config') config = MmapDict(config_path) config['__configuration_time__'] = time.time() config['__processor__'] = self.path for i in dir(self): if _default_module.match(i) is not None: continue j = getattr(self, i) if isinstance(j, (Number, string_types, bool)): config[i] = j config.flush(save_all=True) self.config = {i: j for i, j in config} config.close() prog.add_notification("Saved configuration at: %s" % ctext(config_path, 'yellow')) # ====== final notification ====== # prog.add_notification("Closed all dataset.") prog.add_notification("Dataset at path: %s" % ctext(dataset.path, 'yellow'))