def __init__(self, workflow, **kwargs):
self.name = kwargs.get("name")
self.view_group = kwargs.get("view_group")
self._demanded = set()
self._id = str(uuid.uuid4())
self._links_from = {}
self._links_to = {}
super(Unit, self).__init__(**kwargs)
validate_kwargs(self, **kwargs)
self.verify_interface(IUnit)
self._gate_block = Bool(False)
self._gate_skip = Bool(False)
self._ignores_gate = Bool(kwargs.get("ignore_gate", False))
self._run_calls = 0
self._remembers_gates = True
timings = get(root.common.timings, None)
if timings is not None and isinstance(timings, set):
timings = self.__class__.__name__ in timings
else:
timings = False
self._timings = kwargs.get("timings", timings)
assert isinstance(self._timings, bool)
self.workflow = workflow
self.add_method_to_storage("initialize")
self.add_method_to_storage("run")
self.add_method_to_storage("stop")
if hasattr(self, "generate_data_for_master"):
self.add_method_to_storage("generate_data_for_master")
if hasattr(self, "apply_data_from_master"):
self.add_method_to_storage("apply_data_from_master")
if hasattr(self, "generate_data_for_slave"):
self.add_method_to_storage("generate_data_for_slave")
if hasattr(self, "apply_data_from_slave"):
self.add_method_to_storage("apply_data_from_slave")
def __init__(self, workflow, **kwargs):
kwargs["view_group"] = kwargs.get("view_group", "EVALUATOR")
super(GeneticsOptimizer, self).__init__(workflow, **kwargs)
self._model_ = kwargs["model"]
self._config = copy.deepcopy(kwargs.get("config", root))
if "config" not in kwargs:
del self.config.common
self.plotters_are_disabled = kwargs.get(
"plotters_are_disabled", root.common.genetics.disable.plotting)
self._tuneables = []
process_config(self.config, Range, self._add_tuneable)
if len(self.tuneables) == 0:
raise ValueError(
"There are no tunable parameters in the supplied configuration"
" %s. Wrap at least one into veles.genetics.Range class." %
self.config.__path__)
self._chromosome_index = 0
self.generation_changed = Bool()
if self.is_slave:
self.complete = Bool()
return
self._population = ConfigPopulation(
lambda *a, **k: ConfigChromosome(self, *a, **k),
len(self.tuneables), [x.min_value for x in self.tuneables],
[x.max_value for x in self.tuneables],
kwargs["size"],
rand=kwargs.get("rand", prng.get()),
max_generations=kwargs.get("generations"))
self.population.on_generation_changed_callback = \
self._set_generation_changed
self._best_config = "" # actual type is veles.config.Config
self.complete = ~self.population.improved
def __init__(self, workflow, **kwargs):
kwargs["view_group"] = kwargs.get("view_group", "TRAINER")
self.complete = Bool(False)
super(DecisionBase, self).__init__(workflow, **kwargs)
self.verify_interface(IDecision)
self.max_epochs = kwargs.get("max_epochs", None)
self.improved = Bool(False)
self.train_improved = Bool(False)
self.snapshot_suffix = ""
self.epoch_timestamp = False
self.demand("last_minibatch", "minibatch_class",
"class_lengths", "epoch_number", "epoch_ended")
def __init__(self, workflow, **kwargs):
kwargs["view_group"] = "LOADER"
self.last_minibatch = Bool()
super(Loader, self).__init__(workflow, **kwargs)
self.verify_interface(ILoader)
self.prng = kwargs.get("prng", random_generator.get())
if not self.testing:
self.shuffle_limit = kwargs.get("shuffle_limit",
numpy.iinfo(numpy.uint32).max)
else:
self.shuffle_limit = 0
self._max_minibatch_size = kwargs.get("minibatch_size", 100)
if self._max_minibatch_size < 1:
raise ValueError("minibatch_size must be greater than zero")
self._class_lengths = [0] * len(CLASS_NAME)
self._class_end_offsets = [0] * len(CLASS_NAME)
self._has_labels = False
self.epoch_ended = Bool()
self.epoch_number = 0
self.train_ended = Bool()
self.test_ended = Bool()
self.samples_served = 0
self._global_offset = 0
self.minibatch_class = 0
self.minibatch_data = memory.Array(shallow_pickle=True)
self.minibatch_indices = memory.Array(shallow_pickle=True)
self.minibatch_labels = memory.Array(shallow_pickle=True)
self._raw_minibatch_labels = []
self._labels_mapping = {}
self._reversed_labels_mapping = []
self._samples_mapping = defaultdict(set)
self.failed_minibatches = []
self._total_failed = 0
self._on_initialized = nothing
self._unique_labels_count = 1 # "None" label
self.shuffled_indices = memory.Array()
self.normalization_type = kwargs.get("normalization_type", "none")
self.normalization_parameters = kwargs.get("normalization_parameters",
{})
self.train_ratio = kwargs.get("train_ratio", self.train_ratio)
def __init__(self, workflow, **kwargs):
kwargs["view_group"] = kwargs.get("view_group", "EVALUATOR")
super(EnsembleModelManagerBase, self).__init__(workflow, **kwargs)
self._model_ = kwargs["model"]
self._model_index = 0
self._results = []
self._complete = Bool(lambda: None not in self.results)
def __init__(self, workflow, **kwargs):
super(Timer, self).__init__(workflow, **kwargs)
self.sync_iterations = set(kwargs.get("sync_iterations", []))
self.n_it = kwargs.get("n_it", 11) # one for dry-run
self.it = 0
self.complete = Bool(False)
self.times = [time.time()]
def __init__(self, workflow, **kwargs):
super(InteractiveLoader, self).__init__(workflow, **kwargs)
self._event = threading.Event()
self._event.clear()
self._max_minibatch_size = 1
self._loadtxt_kwargs = kwargs.get("loadtxt_kwargs", {})
self.complete = Bool(False)
def __init__(self, workflow, **kwargs):
super(DecisionGD, self).__init__(workflow, **kwargs)
self.fail_iterations = kwargs.get("fail_iterations", 100)
self.gd_skip = Bool()
self.epoch_n_err = [self.BIGNUM] * 3
self.epoch_n_evaluated_samples = [0] * 3
self.epoch_n_err_pt = [100.0] * 3
self.best_n_err_pt = [100.0] * 3
self.minibatch_n_err = None # memory.Array()
# minimum validation error and its epoch number
self.min_validation_n_err_pt = 100.0
self.min_validation_n_err_pt_epoch_number = -1
# train error when validation was minimal
self.min_train_validation_n_err_pt = 100.0
# minimum train error and its epoch number
self.min_train_n_err_pt = 100.0
self.min_train_n_err_pt_epoch_number = -1
self.prev_train_err_pt = 100.0
self.confusion_matrixes = [None] * 3
self.minibatch_confusion_matrix = None # memory.Array()
self.max_err_y_sums = [0] * 3
self.minibatch_max_err_y_sum = None # memory.Array()
self.demand("minibatch_size")
def __init__(self, workflow, **kwargs):
super(DecisionGD, self).__init__(workflow, **kwargs)
self.fail_iterations = kwargs.get("fail_iterations", 100)
self.gd_skip = Bool()
# Values for the current epoch
self.epoch_n_err = [None] * 3
self.epoch_n_evaluated_samples = [0] * 3
self.epoch_n_err_pt = [None] * 3
# Best achieved errors, independently for each class
self.best_n_err_pt = [None] * 3
# and its epoch numbers
self.best_n_err_pt_epoch_number = [None] * 3
# errors for other classes when the given class was the best
self.best_n_err_pt_others = [[None] * 3] * 3
self._store_best_n_err_pt_others = [False] * 3
# Errors at the epoch where
# max of train and validation errors was the best
self.best_minimax_n_err_pt = [None] * 3
# and it's epoch number
self.best_minimax_n_err_pt_epoch_number = -1
self.minibatch_n_err = None # memory.Array()
self.minibatch_confusion_matrix = None # memory.Array()
self.minibatch_max_err_y_sum = None # memory.Array()
self.confusion_matrixes = [None] * 3
self.max_err_y_sums = [0] * 3
self.autoencoder = False
self.demand("minibatch_size")
def __init__(self, workflow, **kwargs):
"""
Args:
max_iteration: loop is repeated max_iteration iterations
"""
super(IterationCounter, self).__init__(workflow, **kwargs)
self.max_iterations = kwargs["max_iterations"]
self.iteration = 0
self.complete = Bool(False)
def __init__(self, workflow, **kwargs):
super(FixAccumulator, self).__init__(workflow)
self.bars = kwargs.get("bars", 200)
self.type = kwargs.get("type", "relu")
self.input = None
self.output = Array()
self.reset_flag = Bool(True)
self.n_bars = [0]
self.max = 100
self.min = 0
def __init__(self, workflow, **kwargs):
super(HDFSTextLoader, self).__init__(workflow, **kwargs)
self.file_name = kwargs["file"]
self.chunk_lines_number = kwargs.get("chunk", 1000)
client_kwargs = dict(kwargs)
del client_kwargs["file"]
if "chunk" in kwargs:
del client_kwargs["chunk"]
self.hdfs_client = Client(**client_kwargs)
self.output = [""] * self.chunk_lines_number
self.finished = Bool()
def __init__(self, workflow, **kwargs):
kwargs["name"] = kwargs.get("name", "ImagePlotter")
super(ImagePlotter, self).__init__(workflow, **kwargs)
self.yuv = Bool(kwargs.get("yuv", False))
self.cm = None
self.pp = None
self.show_figure = nothing
self.demand("inputs", "input_fields")
self.inputs = []
self.input_fields = []
self._pics_to_draw = []
self.redraw_threshold = 1.5
def __init__(self, workflow, **kwargs):
super(RangeAccumulator, self).__init__(workflow)
self.bars = kwargs.get("bars", 20)
self.x = []
self.input = None
self.first_minibatch = True
self.y = []
self.d = 0
self.x_out = []
self.y_out = []
self.squash = kwargs.get("squash", True)
self.reset_flag = Bool(False)
self.gl_min = sys.float_info.max
self.gl_max = sys.float_info.min
self.residue_bars = 0
self.inside_bar = False
def __init__(self, workflow, **kwargs):
kwargs["name"] = kwargs.get("name", "Weights")
super(Weights2D, self).__init__(workflow, **kwargs)
self.demand("input")
self.color_space = kwargs.get("color_space", "RGB")
self.get_shape_from = None
self.limit = kwargs.get("limit", 64)
self.transposed = kwargs.get('transposed', False)
self.yuv = Bool(kwargs.get("yuv", False))
self.split_channels = kwargs.get("split_channels", False)
self.column_align = kwargs.get("column_align", 4)
self.cm = None
self.pp = None
self.show_figure = nothing
self._pics_to_draw = []
self.redraw_threshold = 1.5
def __init__(self, workflow, **kwargs):
kwargs["view_group"] = kwargs.get("view_group", "SERVICE")
super(SnapshotterBase, self).__init__(workflow, **kwargs)
self.verify_interface(ISnapshotter)
self.prefix = kwargs.get("prefix", "")
if "model_index" in root.common.ensemble:
self.prefix = ("%04d_" % root.common.ensemble.model_index) + \
self.prefix
self._destination = ""
self.compression = kwargs.get("compression", "gz")
self.compression_level = kwargs.get("compression_level", 6)
self.interval = kwargs.get("interval", 1)
self.time_interval = kwargs.get("time_interval", 15)
self.time = 0
self._skipped_counter = 0
self.skip = Bool(False)
self._warn_about_size = kwargs.get("warn_about_size", True)
self.demand("suffix")
def create_workflow(self):
self.loader = self.real_loader = BenchmarkLoader(
self, minibatch_size=BenchmarkLoader.BATCH,
force_numpy=True) # do not preload all dataset to device
self.loader.link_from(self.start_point)
self.t0 = Timer(self, name="Timer 0",
sync_iterations=(1, )).link_from(self.loader)
self.repeater.link_from(self.t0)
self.link_forwards(("input", "minibatch_data"), self.loader)
self.forwards[0].unlink_before()
self.forwards[0].link_from(self.repeater)
self.t1 = Timer(self, name="Timer 1",
sync_iterations=(1, )).link_from(self.forwards[-2])
self.repeater.link_from(self.t1)
self.forwards[-1].gate_block = ~self.t1.complete
self.forwards[0].gate_block = self.t1.complete
self.link_evaluator(self.forwards[-1])
self.link_decision(self.evaluator)
self.decision.gate_skip = Bool(True)
last_gd = self.link_gds(self.decision)
self.t2 = Timer(self, name="Timer 2",
sync_iterations=(1, )).link_from(self.gds[-1])
self.repeater2 = Repeater(self).link_from(self.t2)
self.gds[-2].unlink_before()
self.gds[-2].link_from(self.repeater2)
self.t3 = Timer(self, name="Timer 3",
sync_iterations=(1, )).link_from(last_gd)
self.repeater2.link_from(self.t3)
self.end_point.link_from(self.t3)
self.end_point.gate_block = ~self.t3.complete
self.repeater2.gate_block = self.t3.complete
def __init__(self,
chromosome_factory,
optimization_size,
min_values,
max_values,
population_size,
accuracy=0.00001,
rand=prng.get(),
max_generations=None,
crossing_attempts=10):
super(Population, self).__init__()
self.rand = rand
self.size = population_size
self.chromosome_factory = chromosome_factory
self.chromosomes = []
self.optimization = InlineObject()
self.optimization.choice = "betw"
self.optimization.code = "float"
self.optimization.size = optimization_size
self.optimization.min_values = min_values
self.optimization.max_values = max_values
assert len(min_values) == len(max_values)
self.optimization.accuracy = accuracy
self.fitness = None
self.average_fit = None
self.best_fit = None
self.worst_fit = None
self.median_fit = None
self.prev = InlineObject()
self.prev.fitness = -1.0e30
self.prev.average_fit = -1.0e30
self.prev.best_fit = -1.0e30
self.prev.worst_fit = -1.0e30
self.prev.median_fit = -1.0e30
self.roulette_select_size = 0.75
self.random_select_size = 0.5
self.tournament_size = 0.5
self.tournament_select_size = 0.1
self.crossing = InlineObject()
self.crossing.pointed_crossings = 0.2
self.crossing.pointed_points = 0.08
self.crossing.pointed_probability = 1.0
self.crossing.uniform_crossings = 0.15
self.crossing.uniform_probability = 0.9
self.crossing.arithmetic_crossings = 0.15
self.crossing.arithmetic_probability = 0.9
self.crossing.geometric_crossings = 0.2
self.crossing.geometric_probability = 0.9
self.crossing.pipeline = [
self.cross_uniform, self.cross_arithmetic, self.cross_geometric
]
self.delimeter = None
self.codes = None
self.mutations = {
"binary_point": {
"use": False,
"chromosomes": 0.2,
"points": 0.06,
"probability": 0.35
},
"gaussian": {
"use": True,
"chromosomes": 0.35,
"points": 0.05,
"probability": 0.7
},
"uniform": {
"use": True,
"chromosomes": 0.35,
"points": 0.05,
"probability": 0.7
},
"altering": {
"use": False,
"chromosomes": 0.1,
"points": None,
"probability": 0.35
}
}
self.generation = 0
self.max_generations = max_generations or self.MAX_GENERATIONS
self.crossing_attempts = crossing_attempts
self.improved = Bool(True)
self.on_generation_changed_callback = lambda: None
for _ in range(self.size):
self.add(self.new(size=self.optimization.size))
if self.optimization.code == "gray":
self.compute_gray_codes()
def __init__(self, workflow, **kwargs):
super(RestfulLoader, self).__init__(workflow, **kwargs)
self.complete = Bool(False)
self.max_response_time = kwargs.get("max_response_time", 0.1)
self._requests = []
def __init__(self, workflow, **kwargs):
super(NNSnapshotterBase, self).__init__(workflow, **kwargs)
self.has_invalid_values = Bool(False)
