def evaluate(self, chromo):
for tune, val in zip(self.tuneables, chromo.numeric):
tune <<= val
chromo.config = copy.deepcopy(self.config)
with NamedTemporaryFile(mode="wb", prefix="veles-optimization-config-",
suffix=".%d.pickle" % best_protocol) as fcfg:
pickle.dump(self.config, fcfg)
fcfg.flush()
with NamedTemporaryFile(
mode="r", prefix="veles-optimization-result-",
suffix=".%d.pickle" % best_protocol) as fres:
argv = ["--result-file", fres.name, "--stealth", "--log-id",
self.launcher.log_id] + self._filtered_argv_ + \
["root.common.disable.publishing=True"]
if self.plotters_are_disabled:
argv = ["-p", ""] + argv
i = -1
while "=" in argv[i]:
i -= 1
argv[i] = fcfg.name
result = self._exec(argv, fres)
if result is None:
raise EvaluationError()
try:
chromo.fitness = result["EvaluationFitness"]
except KeyError:
raise from_none(EvaluationError(
"Failed to find \"EvaluationFitness\" in the evaluation "
"results"))
chromo.snapshot = result.get("Snapshot")
self.info("Chromosome #%d was evaluated to %f", self._chromosome_index,
chromo.fitness)
def initialize(self, **kwargs):
if self.shuffle_limit != 0:
raise error.VelesException(
"You must disable shuffling in your loader (set shuffle_limit "
"to 0)")
self._file_ = open(self.file_name, "wb")
pickle.dump(self.get_header_data(), self.file, protocol=best_protocol)
def save_to_file(self, file_name, labels):
if len(labels) != len(self.outputs):
raise Exception("Labels and outputs size mismatch (" +
str(len(labels)) + " vs " +
str(len(self.outputs)) + ")")
logging.debug("Saving %d results", len(labels))
root = {"version": "1.0", "files": {}}
indices_map = sorted(range(0, len(labels)), key=lambda x: labels[x])
labels.sort()
for j in range(0, len(labels)):
i = indices_map[j]
label = labels[j]
file_element = {"features": {}}
for features in self.features:
feat_element = {
"description":
features.description({
"sampling_rate": self.outputs[i][1],
"channels": self.outputs[i][2]
})
}
if self.outputs[i]:
feat_element["value"] = self.outputs[i][0][features.name]
file_element["features"][features.name] = feat_element
root["files"][label] = file_element
fout = open(file_name, "wb")
pickle.dump(root, fout, protocol=best_protocol)
def evaluate(self, chromo):
for tune, val in zip(self.tuneables, chromo.numeric):
tune <<= val
chromo.config = copy.deepcopy(self.config)
with NamedTemporaryFile(mode="wb",
prefix="veles-optimization-config-",
suffix=".%d.pickle" % best_protocol) as fcfg:
pickle.dump(self.config, fcfg)
fcfg.flush()
with NamedTemporaryFile(mode="r",
prefix="veles-optimization-result-",
suffix=".%d.pickle" %
best_protocol) as fres:
argv = ["--result-file", fres.name, "--stealth", "--log-id",
self.launcher.log_id] + self._filtered_argv_ + \
["root.common.disable.publishing=True"]
if self.plotters_are_disabled:
argv = ["-p", ""] + argv
i = -1
while "=" in argv[i]:
i -= 1
argv[i] = fcfg.name
result = self._exec(argv, fres)
if result is None:
raise EvaluationError()
try:
chromo.fitness = result["EvaluationFitness"]
except KeyError:
raise from_none(
EvaluationError(
"Failed to find \"EvaluationFitness\" in the evaluation "
"results"))
chromo.snapshot = result.get("Snapshot")
self.info("Chromosome #%d was evaluated to %f", self._chromosome_index,
chromo.fitness)
def stop(self):
self.info("Writing %d items to %s...", self.size, self.file_name)
with open(self.file_name, "wb") as fout:
pickle.dump({u.name: vals
for u, vals in self.stats.items()},
fout,
protocol=best_protocol)
def stop(self):
if self.file.closed:
return
pos = self.file.tell()
pickle.dump(self.offset_table, self.file, protocol=best_protocol)
self.debug("Offset table took %d bytes", self.file.tell() - pos)
self.file.close()
self.info("Wrote %s", self.file_name)
def run(self):
prepared = self.prepare_chunk_data()
chunk_size = self.effective_class_chunk_sizes[self.minibatch_class]
chunks_number = int(numpy.ceil(self.max_minibatch_size / chunk_size))
for i in range(chunks_number):
self.offset_table.append(numpy.uint64(self.file.tell()))
file = MinibatchesSaver.CODECS[self.compression](
self.file, self.compression_level)
self.fill_chunk_data(
prepared, (i * chunk_size, (i + 1) * chunk_size))
pickle.dump(prepared, file, protocol=best_protocol)
file.flush()
def load_data(self):
self._file_ = open(self.file_name, "rb")
(codec, class_lengths, self.old_max_minibatch_size,
self.class_chunk_lengths,
self.minibatch_data_shape, self.minibatch_data_dtype,
self.minibatch_labels_shape, self.minibatch_labels_dtype,
self._labels_mapping) = \
pickle.load(self.file)
self.class_lengths[:] = class_lengths
self._has_labels = self.minibatch_labels_shape is not None
self._reversed_labels_mapping[:] = sorted(self.labels_mapping)
self.decompress = MinibatchesLoader.CODECS[codec]
self.chunk_numbers = []
for ci, cl in enumerate(self.class_lengths):
mb_chunks = int(numpy.ceil(self.old_max_minibatch_size /
self.class_chunk_lengths[ci]))
mb_count = int(numpy.ceil(cl / self.old_max_minibatch_size))
self.chunk_numbers.append(mb_chunks * mb_count)
class BytesMeasurer(object):
def __init__(self):
self.size = 0
def write(self, data):
self.size += len(data)
bm = BytesMeasurer()
fake_table = [numpy.uint64(i) for i in range(sum(self.chunk_numbers))]
pickle.dump(fake_table, bm, protocol=best_protocol)
self.file.seek(-bm.size, SEEK_END)
try:
self.offset_table = pickle.load(self.file)
except pickle.UnpicklingError as e:
self.error("Failed to read the offset table (table offset was %d)",
bm.size)
raise from_none(e)
for i, offset in enumerate(self.offset_table):
self.offset_table[i] = int(offset)
# Virtual end
self.offset_table.append(self.file.tell() - bm.size)
self.debug("Offsets: %s", self.offset_table)
if self.class_lengths[TRAIN] == 0:
assert self.normalization_type == "none", \
"You specified \"%s\" normalization but there are no train " \
"samples to analyze." % self.normalization_type
self.normalizer.analyze(self.minibatch_data.mem)
def export(self):
self._destination = ".".join(
(self.prefix, self.suffix, str(best_protocol)))
fio = BytesIO()
self.info("Preparing the snapshot...")
with self._open_fobj(fio) as fout:
pickle.dump(self.workflow, fout, protocol=best_protocol)
self.check_snapshot_size(len(fio.getvalue()))
binary = pyodbc.Binary(fio.getvalue())
self.info("Executing SQL insert into \"%s\"...", self.table)
now = datetime.now()
self._cursor_.execute(
"insert into %s(timestamp, id, log_id, workflow, name, codec, data"
") values (?, ?, ?, ?, ?, ?, ?);" % self.table, now,
self.launcher.id, self.launcher.log_id,
self.launcher.workflow.name, self.destination, self.compression,
binary)
self._db_.commit()
self.info("Successfully wrote %d bytes as %s @ %s",
len(binary), self.destination, now)
def export(self):
self._destination = ".".join(
(self.prefix, self.suffix, str(best_protocol)))
fio = BytesIO()
self.info("Preparing the snapshot...")
with self._open_fobj(fio) as fout:
pickle.dump(self.workflow, fout, protocol=best_protocol)
self.check_snapshot_size(len(fio.getvalue()))
binary = pyodbc.Binary(fio.getvalue())
self.info("Executing SQL insert into \"%s\"...", self.table)
now = datetime.now()
self._cursor_.execute(
"insert into %s(timestamp, id, log_id, workflow, name, codec, data"
") values (?, ?, ?, ?, ?, ?, ?);" % self.table, now,
self.launcher.id, self.launcher.log_id,
self.launcher.workflow.name, self.destination, self.compression,
binary)
self._db_.commit()
self.info("Successfully wrote %d bytes as %s @ %s", len(binary),
self.destination, now)
def export(self):
ext = ("." + self.compression) if self.compression else ""
rel_file_name = "%s_%s.%d.pickle%s" % (
self.prefix, self.suffix, best_protocol, ext)
self._destination = os.path.abspath(os.path.join(
self.directory, rel_file_name))
self.info("Snapshotting to %s..." % self.destination)
with self._open_file() as fout:
pickle.dump(self.workflow, fout, protocol=best_protocol)
self.check_snapshot_size(os.path.getsize(self.destination))
file_name_link = os.path.join(
self.directory, "%s_current.%d.pickle%s" % (
self.prefix, best_protocol, ext))
# Link creation may fail when several processes do this all at once,
# so try-except here:
try:
os.remove(file_name_link)
except OSError:
pass
try:
os.symlink(rel_file_name, file_name_link)
except OSError:
pass
def export(self):
ext = ("." + self.compression) if self.compression else ""
rel_file_name = "%s_%s.%d.pickle%s" % (self.prefix, self.suffix,
best_protocol, ext)
self._destination = os.path.abspath(
os.path.join(self.directory, rel_file_name))
self.info("Snapshotting to %s..." % self.destination)
with self._open_file() as fout:
pickle.dump(self.workflow, fout, protocol=best_protocol)
self.check_snapshot_size(os.path.getsize(self.destination))
file_name_link = os.path.join(
self.directory,
"%s_current.%d.pickle%s" % (self.prefix, best_protocol, ext))
# Link creation may fail when several processes do this all at once,
# so try-except here:
try:
os.remove(file_name_link)
except OSError:
pass
try:
os.symlink(rel_file_name, file_name_link)
except OSError:
pass
def save_to_file(self, file_name, labels):
if len(labels) != len(self.outputs):
raise Exception("Labels and outputs size mismatch (" +
str(len(labels)) + " vs " +
str(len(self.outputs)) + ")")
logging.debug("Saving %d results", len(labels))
root = {"version": "1.0", "files": {}}
indices_map = sorted(range(0, len(labels)), key=lambda x: labels[x])
labels.sort()
for j in range(0, len(labels)):
i = indices_map[j]
label = labels[j]
file_element = {"features": {}}
for features in self.features:
feat_element = {"description": features.description(
{"sampling_rate": self.outputs[i][1],
"channels": self.outputs[i][2]})}
if self.outputs[i]:
feat_element["value"] = self.outputs[i][0][features.name]
file_element["features"][features.name] = feat_element
root["files"][label] = file_element
fout = open(file_name, "wb")
pickle.dump(root, fout, protocol=best_protocol)
def save(self, file_name):
"""
Stores object's current state in the specified file.
"""
data = self.generate_data_for_slave()
pickle.dump(data, file_name, protocol=best_protocol)
def save(self, file_name):
"""
Stores object's current state in the specified file.
"""
data = self.generate_data_for_slave()
pickle.dump(data, file_name, protocol=best_protocol)
def stop(self):
self.info("Writing %d items to %s...", self.size, self.file_name)
with open(self.file_name, "wb") as fout:
pickle.dump({u.name: vals for u, vals in self.stats.items()}, fout,
protocol=best_protocol)