def cb(frame_len, orth):
if frame_len >= options.max_seq_frame_len:
return
orth_syms = parse_orthography(orth)
if len(orth_syms) >= options.max_seq_orth_len:
return
Stats.count += 1
Stats.total_frame_len += frame_len
if options.dump_orth_syms:
print("Orth:", "".join(orth_syms), file=log.v3)
if options.filter_orth_sym:
if options.filter_orth_sym in orth_syms:
print("Found orth:", "".join(orth_syms), file=log.v3)
if options.filter_orth_syms_seq:
filter_seq = parse_orthography_into_symbols(
options.filter_orth_syms_seq)
if found_sub_seq(filter_seq, orth_syms):
print("Found orth:", "".join(orth_syms), file=log.v3)
Stats.orth_syms_set.update(orth_syms)
Stats.total_orth_len += len(orth_syms)
# Show some progress if it takes long.
if time.time() - Stats.process_last_time > 2:
Stats.process_last_time = time.time()
if options.collect_time:
print("Collect process, total frame len so far:",
hms(Stats.total_frame_len *
(options.frame_time / 1000.0)),
file=log.v3)
else:
print("Collect process, total orth len so far:",
human_size(Stats.total_orth_len),
file=log.v3)
def cb(frame_len, orth):
if frame_len >= options.max_seq_frame_len:
return
orth_syms = parse_orthography(orth)
if len(orth_syms) >= options.max_seq_orth_len:
return
Stats.count += 1
Stats.total_frame_len += frame_len
if options.dump_orth_syms:
print >> log.v3, "Orth:", "".join(orth_syms)
if options.filter_orth_sym:
if options.filter_orth_sym in orth_syms:
print >> log.v3, "Found orth:", "".join(orth_syms)
if options.filter_orth_syms_seq:
filter_seq = parse_orthography_into_symbols(options.filter_orth_syms_seq)
if found_sub_seq(filter_seq, orth_syms):
print >> log.v3, "Found orth:", "".join(orth_syms)
Stats.orth_syms_set.update(orth_syms)
Stats.total_orth_len += len(orth_syms)
# Show some progress if it takes long.
if time.time() - Stats.process_last_time > 2:
Stats.process_last_time = time.time()
if options.collect_time:
print >> log.v3, "Collect process, total frame len so far:", hms(Stats.total_frame_len * (options.frame_time / 1000.0))
else:
print >> log.v3, "Collect process, total orth len so far:", human_size(Stats.total_orth_len)
def _callback(self, orth):
orth_words = parse_orthography(orth, prefix=[], postfix=[], word_based=True)
self.seq_count += 1
if self.options.dump_orth:
print("Orth:", orth_words, file=log.v3)
self.words.update(orth_words)
self.total_word_len += len(orth_words)
# Show some progress if it takes long.
if time.time() - self.process_last_time > 2:
self.process_last_time = time.time()
print("Collect process, total word len so far:", human_size(self.total_word_len), file=log.v3)
def __init__(self, options, iter_corpus):
"""
:param options: argparse.Namespace
"""
self.options = options
self.seq_count = 0
self.words = set()
self.total_word_len = 0
self.process_last_time = time.time()
iter_corpus(self._callback)
print("Total word len:", self.total_word_len, "(%s)" % human_size(self.total_word_len), file=log.v3)
print("Average orth len:", float(self.total_word_len) / self.seq_count, file=log.v3)
print("Num word symbols:", len(self.words), file=log.v3)
def _callback(self, orth):
orth_words = parse_orthography(orth,
prefix=[],
postfix=[],
word_based=True)
self.seq_count += 1
if self.options.dump_orth:
print("Orth:", orth_words, file=log.v3)
self.words.update(orth_words)
self.total_word_len += len(orth_words)
# Show some progress if it takes long.
if time.time() - self.process_last_time > 2:
self.process_last_time = time.time()
print("Collect process, total word len so far:",
human_size(self.total_word_len),
file=log.v3)
def __init__(self, options, iter_corpus):
"""
:param options: argparse.Namespace
"""
self.options = options
self.seq_count = 0
self.words = set()
self.total_word_len = 0
self.process_last_time = time.time()
iter_corpus(self._callback)
print("Total word len:",
self.total_word_len,
"(%s)" % human_size(self.total_word_len),
file=log.v3)
print("Average orth len:",
float(self.total_word_len) / self.seq_count,
file=log.v3)
print("Num word symbols:", len(self.words), file=log.v3)
def hdf_dump_from_dataset(dataset, hdf_dataset, parser_args):
"""
:param Dataset dataset: could be any dataset implemented as child of Dataset
:type hdf_dataset: h5py._hl.files.File
:param parser_args: argparse object from main()
:return:
"""
print("Work on epoch: %i" % parser_args.epoch, file=log.v3)
dataset.init_seq_order(parser_args.epoch)
data_keys = sorted(dataset.get_data_keys())
print("Data keys:", data_keys, file=log.v3)
if "orth" in data_keys:
data_keys.remove("orth")
# We need to do one run through the dataset to collect some stats like total len.
print("Collect stats, iterate through all data...", file=log.v3)
seq_idx = parser_args.start_seq
seq_idxs = []
seq_tags = []
seq_lens = []
total_seq_len = NumbersDict(0)
max_tag_len = 0
dataset_num_seqs = try_run(lambda: dataset.num_seqs,
default=None) # can be unknown
if parser_args.end_seq != float("inf"):
if dataset_num_seqs is not None:
dataset_num_seqs = min(dataset_num_seqs, parser_args.end_seq)
else:
dataset_num_seqs = parser_args.end_seq
if dataset_num_seqs is not None:
dataset_num_seqs -= parser_args.start_seq
assert dataset_num_seqs > 0
while dataset.is_less_than_num_seqs(
seq_idx) and seq_idx <= parser_args.end_seq:
seq_idxs += [seq_idx]
dataset.load_seqs(seq_idx, seq_idx + 1)
seq_len = dataset.get_seq_length(seq_idx)
seq_lens += [seq_len]
tag = dataset.get_tag(seq_idx)
seq_tags += [tag]
max_tag_len = max(len(tag), max_tag_len)
total_seq_len += seq_len
if dataset_num_seqs is not None:
progress_bar_with_time(
float(seq_idx - parser_args.start_seq) / dataset_num_seqs)
seq_idx += 1
num_seqs = len(seq_idxs)
assert num_seqs > 0
shapes = {}
for data_key in data_keys:
assert data_key in total_seq_len.dict
shape = [total_seq_len[data_key]]
shape += dataset.get_data_shape(data_key)
print("Total len of %r is %s, shape %r, dtype %s" %
(data_key, human_size(
shape[0]), shape, dataset.get_data_dtype(data_key)),
file=log.v3)
shapes[data_key] = shape
print("Set seq tags...", file=log.v3)
hdf_dataset.create_dataset('seqTags',
shape=(num_seqs, ),
dtype="S%i" % (max_tag_len + 1))
for i, tag in enumerate(seq_tags):
hdf_dataset['seqTags'][i] = numpy.array(tag,
dtype="S%i" %
(max_tag_len + 1))
progress_bar_with_time(float(i) / num_seqs)
print("Set seq len info...", file=log.v3)
hdf_dataset.create_dataset(HDFDataset.attr_seqLengths,
shape=(num_seqs, 2),
dtype="int32")
for i, seq_len in enumerate(seq_lens):
data_len = seq_len["data"]
targets_len = seq_len["classes"]
for data_key in dataset.get_target_list():
if data_key == "orth":
continue
assert seq_len[
data_key] == targets_len, "different lengths in multi-target not supported"
if targets_len is None:
targets_len = data_len
hdf_dataset[HDFDataset.attr_seqLengths][i] = [data_len, targets_len]
progress_bar_with_time(float(i) / num_seqs)
print("Create arrays in HDF...", file=log.v3)
hdf_dataset.create_group('targets/data')
hdf_dataset.create_group('targets/size')
hdf_dataset.create_group('targets/labels')
for data_key in data_keys:
if data_key == "data":
hdf_dataset.create_dataset('inputs',
shape=shapes[data_key],
dtype=dataset.get_data_dtype(data_key))
else:
hdf_dataset['targets/data'].create_dataset(
data_key,
shape=shapes[data_key],
dtype=dataset.get_data_dtype(data_key))
hdf_dataset['targets/size'].attrs[data_key] = dataset.num_outputs[
data_key]
if data_key in dataset.labels:
labels = dataset.labels[data_key]
assert len(labels) == dataset.num_outputs[data_key][0]
else:
labels = [
"%s-class-%i" % (data_key, i)
for i in range(dataset.get_data_dim(data_key))
]
print("Labels for %s:" % data_key, labels[:3], "...", file=log.v5)
max_label_len = max(map(len, labels))
if data_key != "data":
hdf_dataset['targets/labels'].create_dataset(
data_key, (len(labels), ), dtype="S%i" % (max_label_len + 1))
for i, label in enumerate(labels):
hdf_dataset['targets/labels'][data_key][i] = numpy.array(
label, dtype="S%i" % (max_label_len + 1))
# Again iterate through dataset, and set the data
print("Write data...", file=log.v3)
dataset.init_seq_order(parser_args.epoch)
offsets = NumbersDict(0)
for seq_idx, tag in zip(seq_idxs, seq_tags):
dataset.load_seqs(seq_idx, seq_idx + 1)
tag_ = dataset.get_tag(seq_idx)
assert tag == tag_ # Just a check for sanity. We expect the same order.
seq_len = dataset.get_seq_length(seq_idx)
for data_key in data_keys:
if data_key == "data":
hdf_data = hdf_dataset['inputs']
else:
hdf_data = hdf_dataset['targets/data'][data_key]
data = dataset.get_data(seq_idx, data_key)
hdf_data[offsets[data_key]:offsets[data_key] +
seq_len[data_key]] = data
progress_bar_with_time(float(offsets["data"]) / total_seq_len["data"])
offsets += seq_len
assert offsets == total_seq_len # Sanity check.
# Set some old-format attribs. Not needed for newer CRNN versions.
hdf_dataset.attrs[HDFDataset.attr_inputPattSize] = dataset.num_inputs
hdf_dataset.attrs[HDFDataset.attr_numLabels] = dataset.num_outputs.get(
"classes", (0, 0))[0]
print("All done.", file=log.v3)
def hdf_dump_from_dataset(dataset, hdf_dataset, parser_args):
"""
:param Dataset dataset: could be any dataset implemented as child of Dataset
:type hdf_dataset: h5py._hl.files.File
:param parser_args: argparse object from main()
:return:
"""
print >> log.v3, "Work on epoch: %i" % parser_args.epoch
dataset.init_seq_order(parser_args.epoch)
data_keys = sorted(dataset.get_data_keys())
print >> log.v3, "Data keys:", data_keys
# We need to do one run through the dataset to collect some stats like total len.
print >> log.v3, "Collect stats, iterate through all data..."
seq_idx = parser_args.start_seq
seq_idxs = []
seq_tags = []
seq_lens = []
total_seq_len = NumbersDict(0)
max_tag_len = 0
dataset_num_seqs = try_run(lambda: dataset.num_seqs, default=None) # can be unknown
if parser_args.end_seq != float("inf"):
if dataset_num_seqs is not None:
dataset_num_seqs = min(dataset_num_seqs, parser_args.end_seq)
else:
dataset_num_seqs = parser_args.end_seq
if dataset_num_seqs is not None:
dataset_num_seqs -= parser_args.start_seq
assert dataset_num_seqs > 0
while dataset.is_less_than_num_seqs(seq_idx) and seq_idx <= parser_args.end_seq:
seq_idxs += [seq_idx]
dataset.load_seqs(seq_idx, seq_idx + 1)
seq_len = dataset.get_seq_length(seq_idx)
seq_lens += [seq_len]
tag = dataset.get_tag(seq_idx)
seq_tags += [tag]
max_tag_len = max(len(tag), max_tag_len)
total_seq_len += seq_len
if dataset_num_seqs is not None:
progress_bar_with_time(float(seq_idx - parser_args.start_seq) / dataset_num_seqs)
seq_idx += 1
num_seqs = len(seq_idxs)
assert num_seqs > 0
shapes = {}
for data_key in data_keys:
assert data_key in total_seq_len.dict
shape = [total_seq_len[data_key]]
shape += dataset.get_data_shape(data_key)
print >> log.v3, "Total len of %r is %s, shape %r, dtype %s" % (
data_key, human_size(shape[0]), shape, dataset.get_data_dtype(data_key))
shapes[data_key] = shape
print >> log.v3, "Set seq tags..."
hdf_dataset.create_dataset('seqTags', shape=(num_seqs,), dtype="S%i" % (max_tag_len + 1))
for i, tag in enumerate(seq_tags):
hdf_dataset['seqTags'][i] = tag
progress_bar_with_time(float(i) / num_seqs)
print >> log.v3, "Set seq len info..."
hdf_dataset.create_dataset(HDFDataset.attr_seqLengths, shape=(num_seqs, 2), dtype="int32")
for i, seq_len in enumerate(seq_lens):
data_len = seq_len["data"]
targets_len = seq_len["classes"]
for data_key in dataset.get_target_list():
assert seq_len[data_key] == targets_len, "different lengths in multi-target not supported"
if targets_len is None:
targets_len = data_len
hdf_dataset[HDFDataset.attr_seqLengths][i] = [data_len, targets_len]
progress_bar_with_time(float(i) / num_seqs)
print >> log.v3, "Create arrays in HDF..."
hdf_dataset.create_group('targets/data')
hdf_dataset.create_group('targets/size')
hdf_dataset.create_group('targets/labels')
for data_key in data_keys:
if data_key == "data":
hdf_dataset.create_dataset(
'inputs', shape=shapes[data_key], dtype=dataset.get_data_dtype(data_key))
else:
hdf_dataset['targets/data'].create_dataset(
data_key, shape=shapes[data_key], dtype=dataset.get_data_dtype(data_key))
hdf_dataset['targets/size'].attrs[data_key] = dataset.num_outputs[data_key]
if data_key in dataset.labels:
labels = dataset.labels[data_key]
assert len(labels) == dataset.num_outputs[data_key][0]
else:
labels = ["%s-class-%i" % (data_key, i) for i in range(dataset.get_data_dim(data_key))]
print >> log.v5, "Labels for %s:" % data_key, labels[:3], "..."
max_label_len = max(map(len, labels))
hdf_dataset['targets/labels'].create_dataset(data_key, (len(labels),), dtype="S%i" % (max_label_len + 1))
for i, label in enumerate(labels):
hdf_dataset['targets/labels'][data_key][i] = label
# Again iterate through dataset, and set the data
print >> log.v3, "Write data..."
dataset.init_seq_order(parser_args.epoch)
offsets = NumbersDict(0)
for seq_idx, tag in zip(seq_idxs, seq_tags):
dataset.load_seqs(seq_idx, seq_idx + 1)
tag_ = dataset.get_tag(seq_idx)
assert tag == tag_ # Just a check for sanity. We expect the same order.
seq_len = dataset.get_seq_length(seq_idx)
for data_key in data_keys:
if data_key == "data":
hdf_data = hdf_dataset['inputs']
else:
hdf_data = hdf_dataset['targets/data'][data_key]
data = dataset.get_data(seq_idx, data_key)
hdf_data[offsets[data_key]:offsets[data_key] + seq_len[data_key]] = data
progress_bar_with_time(float(offsets["data"]) / total_seq_len["data"])
offsets += seq_len
assert offsets == total_seq_len # Sanity check.
# Set some old-format attribs. Not needed for newer CRNN versions.
hdf_dataset.attrs[HDFDataset.attr_inputPattSize] = dataset.num_inputs
hdf_dataset.attrs[HDFDataset.attr_numLabels] = dataset.num_outputs.get("classes", (0, 0))[0]
print >> log.v3, "All done."
def collect_stats(options, iter_corpus):
"""
:param options: argparse.Namespace
"""
orth_symbols_filename = options.output
if orth_symbols_filename:
assert not os.path.exists(orth_symbols_filename)
class Stats:
count = 0
process_last_time = time.time()
total_frame_len = 0
total_orth_len = 0
orth_syms_set = set()
if options.add_numbers:
Stats.orth_syms_set.update(
map(chr, list(range(ord("0"),
ord("9") + 1))))
if options.add_lower_alphabet:
Stats.orth_syms_set.update(
map(chr, list(range(ord("a"),
ord("z") + 1))))
if options.add_upper_alphabet:
Stats.orth_syms_set.update(
map(chr, list(range(ord("A"),
ord("Z") + 1))))
def cb(frame_len, orth):
if frame_len >= options.max_seq_frame_len:
return
orth_syms = parse_orthography(orth)
if len(orth_syms) >= options.max_seq_orth_len:
return
Stats.count += 1
Stats.total_frame_len += frame_len
if options.dump_orth_syms:
print("Orth:", "".join(orth_syms), file=log.v3)
if options.filter_orth_sym:
if options.filter_orth_sym in orth_syms:
print("Found orth:", "".join(orth_syms), file=log.v3)
if options.filter_orth_syms_seq:
filter_seq = parse_orthography_into_symbols(
options.filter_orth_syms_seq)
if found_sub_seq(filter_seq, orth_syms):
print("Found orth:", "".join(orth_syms), file=log.v3)
Stats.orth_syms_set.update(orth_syms)
Stats.total_orth_len += len(orth_syms)
# Show some progress if it takes long.
if time.time() - Stats.process_last_time > 2:
Stats.process_last_time = time.time()
if options.collect_time:
print("Collect process, total frame len so far:",
hms(Stats.total_frame_len *
(options.frame_time / 1000.0)),
file=log.v3)
else:
print("Collect process, total orth len so far:",
human_size(Stats.total_orth_len),
file=log.v3)
iter_corpus(cb)
if options.remove_symbols:
filter_syms = parse_orthography_into_symbols(options.remove_symbols)
Stats.orth_syms_set -= set(filter_syms)
if options.collect_time:
print("Total frame len:",
Stats.total_frame_len,
"time:",
hms(Stats.total_frame_len * (options.frame_time / 1000.0)),
file=log.v3)
else:
print("No time stats (--collect_time False).", file=log.v3)
print("Total orth len:",
Stats.total_orth_len,
"(%s)" % human_size(Stats.total_orth_len),
end=' ',
file=log.v3)
if options.collect_time:
print("fraction:",
float(Stats.total_orth_len) / Stats.total_frame_len,
file=log.v3)
else:
print("", file=log.v3)
print("Average orth len:",
float(Stats.total_orth_len) / Stats.count,
file=log.v3)
print("Num symbols:", len(Stats.orth_syms_set), file=log.v3)
if orth_symbols_filename:
orth_syms_file = open(orth_symbols_filename, "wb")
for orth_sym in sorted(Stats.orth_syms_set):
orth_syms_file.write("%s\n" % unicode(orth_sym).encode("utf8"))
orth_syms_file.close()
print("Wrote orthography symbols to",
orth_symbols_filename,
file=log.v3)
else:
print("Provide --output to save the symbols.", file=log.v3)
def collect_stats(options, iter_corpus):
"""
:param options: argparse.Namespace
"""
orth_symbols_filename = options.output
if orth_symbols_filename:
assert not os.path.exists(orth_symbols_filename)
class Stats:
count = 0
process_last_time = time.time()
total_frame_len = 0
total_orth_len = 0
orth_syms_set = set()
if options.add_numbers:
Stats.orth_syms_set.update(map(chr, range(ord("0"), ord("9") + 1)))
if options.add_lower_alphabet:
Stats.orth_syms_set.update(map(chr, range(ord("a"), ord("z") + 1)))
if options.add_upper_alphabet:
Stats.orth_syms_set.update(map(chr, range(ord("A"), ord("Z") + 1)))
def cb(frame_len, orth):
if frame_len >= options.max_seq_frame_len:
return
orth_syms = parse_orthography(orth)
if len(orth_syms) >= options.max_seq_orth_len:
return
Stats.count += 1
Stats.total_frame_len += frame_len
if options.dump_orth_syms:
print >> log.v3, "Orth:", "".join(orth_syms)
if options.filter_orth_sym:
if options.filter_orth_sym in orth_syms:
print >> log.v3, "Found orth:", "".join(orth_syms)
if options.filter_orth_syms_seq:
filter_seq = parse_orthography_into_symbols(options.filter_orth_syms_seq)
if found_sub_seq(filter_seq, orth_syms):
print >> log.v3, "Found orth:", "".join(orth_syms)
Stats.orth_syms_set.update(orth_syms)
Stats.total_orth_len += len(orth_syms)
# Show some progress if it takes long.
if time.time() - Stats.process_last_time > 2:
Stats.process_last_time = time.time()
if options.collect_time:
print >> log.v3, "Collect process, total frame len so far:", hms(Stats.total_frame_len * (options.frame_time / 1000.0))
else:
print >> log.v3, "Collect process, total orth len so far:", human_size(Stats.total_orth_len)
iter_corpus(cb)
if options.remove_symbols:
filter_syms = parse_orthography_into_symbols(options.remove_symbols)
Stats.orth_syms_set -= set(filter_syms)
if options.collect_time:
print >> log.v3, "Total frame len:", Stats.total_frame_len, "time:", hms(Stats.total_frame_len * (options.frame_time / 1000.0))
else:
print >> log.v3, "No time stats (--collect_time False)."
print >> log.v3, "Total orth len:", Stats.total_orth_len, "(%s)" % human_size(Stats.total_orth_len),
if options.collect_time:
print >> log.v3, "fraction:", float(Stats.total_orth_len) / Stats.total_frame_len
else:
print >> log.v3, ""
print >> log.v3, "Average orth len:", float(Stats.total_orth_len) / Stats.count
print >> log.v3, "Num symbols:", len(Stats.orth_syms_set)
if orth_symbols_filename:
orth_syms_file = open(orth_symbols_filename, "wb")
for orth_sym in sorted(Stats.orth_syms_set):
orth_syms_file.write("%s\n" % unicode(orth_sym).encode("utf8"))
orth_syms_file.close()
print >> log.v3, "Wrote orthography symbols to", orth_symbols_filename
else:
print >> log.v3, "Provide --output to save the symbols."
def dump_from_dataset(self, dataset, epoch=1, start_seq=0, end_seq=float("inf"), use_progress_bar=True):
"""
:param Dataset dataset: could be any dataset implemented as child of Dataset
:param int epoch: for dataset
:param int start_seq:
:param int|float end_seq:
:param bool use_progress_bar:
"""
from Util import NumbersDict, human_size, progress_bar_with_time, try_run, PY3
hdf_dataset = self.file
print("Work on epoch: %i" % epoch, file=log.v3)
dataset.init_seq_order(epoch)
data_keys = sorted(dataset.get_data_keys())
print("Data keys:", data_keys, file=log.v3)
if "orth" in data_keys: # special workaround for now, not handled
data_keys.remove("orth")
data_target_keys = [key for key in dataset.get_target_list() if key in data_keys]
data_input_keys = [key for key in data_keys if key not in data_target_keys]
assert len(data_input_keys) > 0 and len(data_target_keys) > 0
if len(data_input_keys) > 1:
if "data" in data_input_keys:
default_data_input_key = "data"
else:
raise Exception("not sure which input data key to use from %r" % (data_input_keys,))
else:
default_data_input_key = data_input_keys[0]
print("Using input data key:", default_data_input_key)
if len(data_target_keys) > 1:
if "classes" in data_target_keys:
default_data_target_key = "classes"
else:
raise Exception("not sure which target data key to use from %r" % (data_target_keys,))
else:
default_data_target_key = data_target_keys[0]
print("Using target data key:", default_data_target_key)
hdf_data_key_map = {key: key for key in data_keys if key != default_data_input_key}
if "data" in hdf_data_key_map:
hdf_data_key_map["data"] = "classes" # Replace "data" which is reserved for input key in HDFDataset.
assert "classes" not in hdf_data_key_map
# We need to do one run through the dataset to collect some stats like total len.
print("Collect stats, iterate through all data...", file=log.v3)
seq_idx = start_seq
seq_idxs = []
seq_tags = []
seq_lens = []
total_seq_len = NumbersDict(0)
max_tag_len = 0
dataset_num_seqs = try_run(lambda: dataset.num_seqs, default=None) # can be unknown
if end_seq != float("inf"):
if dataset_num_seqs is not None:
dataset_num_seqs = min(dataset_num_seqs, end_seq)
else:
dataset_num_seqs = end_seq
if dataset_num_seqs is not None:
dataset_num_seqs -= start_seq
assert dataset_num_seqs > 0
while dataset.is_less_than_num_seqs(seq_idx) and seq_idx <= end_seq:
seq_idxs += [seq_idx]
dataset.load_seqs(seq_idx, seq_idx + 1)
seq_len = dataset.get_seq_length(seq_idx)
seq_lens += [seq_len]
tag = dataset.get_tag(seq_idx)
seq_tags += [tag]
max_tag_len = max(len(tag), max_tag_len)
total_seq_len += seq_len
if use_progress_bar and dataset_num_seqs is not None:
progress_bar_with_time(float(seq_idx - start_seq) / dataset_num_seqs)
seq_idx += 1
num_seqs = len(seq_idxs)
assert num_seqs > 0
shapes = {}
for data_key in data_keys:
assert data_key in total_seq_len.dict
shape = [total_seq_len[data_key]]
shape += dataset.get_data_shape(data_key)
print("Total len of %r is %s, shape %r, dtype %s" % (
data_key, human_size(shape[0]), shape, dataset.get_data_dtype(data_key)), file=log.v3)
shapes[data_key] = shape
print("Set seq tags...", file=log.v3)
hdf_dataset.create_dataset('seqTags', shape=(num_seqs,), dtype="S%i" % (max_tag_len + 1))
for i, tag in enumerate(seq_tags):
hdf_dataset['seqTags'][i] = numpy.array(tag, dtype="S%i" % (max_tag_len + 1))
if use_progress_bar:
progress_bar_with_time(float(i) / num_seqs)
print("Set seq len info...", file=log.v3)
hdf_dataset.create_dataset(attr_seqLengths, shape=(num_seqs, 2), dtype="int32")
for i, seq_len in enumerate(seq_lens):
data_len = seq_len[default_data_input_key]
targets_len = seq_len[default_data_target_key]
for data_key in data_target_keys:
assert seq_len[data_key] == targets_len, "different lengths in multi-target not supported"
if targets_len is None:
targets_len = data_len
hdf_dataset[attr_seqLengths][i] = [data_len, targets_len]
if use_progress_bar:
progress_bar_with_time(float(i) / num_seqs)
print("Create arrays in HDF...", file=log.v3)
hdf_dataset.create_group('targets/data')
hdf_dataset.create_group('targets/size')
hdf_dataset.create_group('targets/labels')
for data_key in data_keys:
if data_key == default_data_input_key:
hdf_dataset.create_dataset(
'inputs', shape=shapes[data_key], dtype=dataset.get_data_dtype(data_key))
else:
hdf_dataset['targets/data'].create_dataset(
hdf_data_key_map[data_key], shape=shapes[data_key], dtype=dataset.get_data_dtype(data_key))
hdf_dataset['targets/size'].attrs[hdf_data_key_map[data_key]] = dataset.num_outputs[data_key]
if data_key in dataset.labels:
labels = dataset.labels[data_key]
if PY3:
labels = [label.encode("utf8") for label in labels]
assert len(labels) == dataset.num_outputs[data_key][0]
else:
labels = ["%s-class-%i" % (data_key, i) for i in range(dataset.get_data_dim(data_key))]
print("Labels for %s:" % data_key, labels[:3], "...", file=log.v5)
max_label_len = max(map(len, labels))
if data_key != default_data_input_key:
hdf_dataset['targets/labels'].create_dataset(hdf_data_key_map[data_key],
(len(labels),), dtype="S%i" % (max_label_len + 1))
for i, label in enumerate(labels):
hdf_dataset['targets/labels'][hdf_data_key_map[data_key]][i] = numpy.array(
label, dtype="S%i" % (max_label_len + 1))
# Again iterate through dataset, and set the data
print("Write data...", file=log.v3)
dataset.init_seq_order(epoch)
offsets = NumbersDict(0)
for seq_idx, tag in zip(seq_idxs, seq_tags):
dataset.load_seqs(seq_idx, seq_idx + 1)
tag_ = dataset.get_tag(seq_idx)
assert tag == tag_ # Just a check for sanity. We expect the same order.
seq_len = dataset.get_seq_length(seq_idx)
for data_key in data_keys:
if data_key == default_data_input_key:
hdf_data = hdf_dataset['inputs']
else:
hdf_data = hdf_dataset['targets/data'][hdf_data_key_map[data_key]]
data = dataset.get_data(seq_idx, data_key)
hdf_data[offsets[data_key]:offsets[data_key] + seq_len[data_key]] = data
if use_progress_bar:
progress_bar_with_time(float(offsets[default_data_input_key]) / total_seq_len[default_data_input_key])
offsets += seq_len
assert offsets == total_seq_len # Sanity check.
# Set some old-format attribs. Not needed for newer CRNN versions.
hdf_dataset.attrs[attr_inputPattSize] = dataset.num_inputs
hdf_dataset.attrs[attr_numLabels] = dataset.num_outputs.get(default_data_target_key, (0, 0))[0]
print("All done.", file=log.v3)
def dump_from_dataset(self, dataset, epoch=1, start_seq=0, end_seq=float("inf"), use_progress_bar=True):
"""
:param Dataset dataset: could be any dataset implemented as child of Dataset
:param int epoch: for dataset
:param int start_seq:
:param int|float end_seq:
:param bool use_progress_bar:
"""
from Util import NumbersDict, human_size, progress_bar_with_time, try_run, PY3
hdf_dataset = self.file
print("Work on epoch: %i" % epoch, file=log.v3)
dataset.init_seq_order(epoch)
data_keys = sorted(dataset.get_data_keys())
print("Data keys:", data_keys, file=log.v3)
if "orth" in data_keys: # special workaround for now, not handled
data_keys.remove("orth")
data_target_keys = [key for key in dataset.get_target_list() if key in data_keys]
data_input_keys = [key for key in data_keys if key not in data_target_keys]
assert len(data_input_keys) > 0 and len(data_target_keys) > 0
if len(data_input_keys) > 1:
if "data" in data_input_keys:
default_data_input_key = "data"
else:
raise Exception("not sure which input data key to use from %r" % (data_input_keys,))
else:
default_data_input_key = data_input_keys[0]
print("Using input data key:", default_data_input_key)
if len(data_target_keys) > 1:
if "classes" in data_target_keys:
default_data_target_key = "classes"
else:
raise Exception("not sure which target data key to use from %r" % (data_target_keys,))
else:
default_data_target_key = data_target_keys[0]
print("Using target data key:", default_data_target_key)
hdf_data_key_map = {key: key for key in data_keys if key != default_data_input_key}
if "data" in hdf_data_key_map:
hdf_data_key_map["data"] = "classes" # Replace "data" which is reserved for input key in HDFDataset.
assert "classes" not in hdf_data_key_map
# We need to do one run through the dataset to collect some stats like total len.
print("Collect stats, iterate through all data...", file=log.v3)
seq_idx = start_seq
seq_idxs = []
seq_tags = []
seq_lens = []
total_seq_len = NumbersDict(0)
max_tag_len = 0
dataset_num_seqs = try_run(lambda: dataset.num_seqs, default=None) # can be unknown
if end_seq != float("inf"):
if dataset_num_seqs is not None:
dataset_num_seqs = min(dataset_num_seqs, end_seq)
else:
dataset_num_seqs = end_seq
if dataset_num_seqs is not None:
dataset_num_seqs -= start_seq
assert dataset_num_seqs > 0
while dataset.is_less_than_num_seqs(seq_idx) and seq_idx <= end_seq:
seq_idxs += [seq_idx]
dataset.load_seqs(seq_idx, seq_idx + 1)
seq_len = dataset.get_seq_length(seq_idx)
seq_lens += [seq_len]
tag = dataset.get_tag(seq_idx)
seq_tags += [tag]
max_tag_len = max(len(tag), max_tag_len)
total_seq_len += seq_len
if use_progress_bar and dataset_num_seqs is not None:
progress_bar_with_time(float(seq_idx - start_seq) / dataset_num_seqs)
seq_idx += 1
num_seqs = len(seq_idxs)
assert num_seqs > 0
shapes = {}
for data_key in data_keys:
assert data_key in total_seq_len.dict
shape = [total_seq_len[data_key]]
shape += dataset.get_data_shape(data_key)
print("Total len of %r is %s, shape %r, dtype %s" % (
data_key, human_size(shape[0]), shape, dataset.get_data_dtype(data_key)), file=log.v3)
shapes[data_key] = shape
print("Set seq tags...", file=log.v3)
hdf_dataset.create_dataset('seqTags', shape=(num_seqs,), dtype="S%i" % (max_tag_len + 1))
for i, tag in enumerate(seq_tags):
hdf_dataset['seqTags'][i] = numpy.array(tag, dtype="S%i" % (max_tag_len + 1))
if use_progress_bar:
progress_bar_with_time(float(i) / num_seqs)
print("Set seq len info...", file=log.v3)
hdf_dataset.create_dataset(attr_seqLengths, shape=(num_seqs, 2), dtype="int32")
for i, seq_len in enumerate(seq_lens):
data_len = seq_len[default_data_input_key]
targets_len = seq_len[default_data_target_key]
for data_key in data_target_keys:
assert seq_len[data_key] == targets_len, "different lengths in multi-target not supported"
if targets_len is None:
targets_len = data_len
hdf_dataset[attr_seqLengths][i] = [data_len, targets_len]
if use_progress_bar:
progress_bar_with_time(float(i) / num_seqs)
print("Create arrays in HDF...", file=log.v3)
hdf_dataset.create_group('targets/data')
hdf_dataset.create_group('targets/size')
hdf_dataset.create_group('targets/labels')
for data_key in data_keys:
if data_key == default_data_input_key:
hdf_dataset.create_dataset(
'inputs', shape=shapes[data_key], dtype=dataset.get_data_dtype(data_key))
else:
hdf_dataset['targets/data'].create_dataset(
hdf_data_key_map[data_key], shape=shapes[data_key], dtype=dataset.get_data_dtype(data_key))
hdf_dataset['targets/size'].attrs[hdf_data_key_map[data_key]] = dataset.num_outputs[data_key]
if data_key in dataset.labels:
labels = dataset.labels[data_key]
if PY3:
labels = [label.encode("utf8") for label in labels]
assert len(labels) == dataset.num_outputs[data_key][0]
else:
labels = ["%s-class-%i" % (data_key, i) for i in range(dataset.get_data_dim(data_key))]
print("Labels for %s:" % data_key, labels[:3], "...", file=log.v5)
max_label_len = max(map(len, labels))
if data_key != default_data_input_key:
hdf_dataset['targets/labels'].create_dataset(hdf_data_key_map[data_key],
(len(labels),), dtype="S%i" % (max_label_len + 1))
for i, label in enumerate(labels):
hdf_dataset['targets/labels'][hdf_data_key_map[data_key]][i] = numpy.array(
label, dtype="S%i" % (max_label_len + 1))
# Again iterate through dataset, and set the data
print("Write data...", file=log.v3)
dataset.init_seq_order(epoch)
offsets = NumbersDict(0)
for seq_idx, tag in zip(seq_idxs, seq_tags):
dataset.load_seqs(seq_idx, seq_idx + 1)
tag_ = dataset.get_tag(seq_idx)
assert tag == tag_ # Just a check for sanity. We expect the same order.
seq_len = dataset.get_seq_length(seq_idx)
for data_key in data_keys:
if data_key == default_data_input_key:
hdf_data = hdf_dataset['inputs']
else:
hdf_data = hdf_dataset['targets/data'][hdf_data_key_map[data_key]]
data = dataset.get_data(seq_idx, data_key)
hdf_data[offsets[data_key]:offsets[data_key] + seq_len[data_key]] = data
if use_progress_bar:
progress_bar_with_time(float(offsets[default_data_input_key]) / total_seq_len[default_data_input_key])
offsets += seq_len
assert offsets == total_seq_len # Sanity check.
# Set some old-format attribs. Not needed for newer CRNN versions.
hdf_dataset.attrs[attr_inputPattSize] = dataset.num_inputs
hdf_dataset.attrs[attr_numLabels] = dataset.num_outputs.get(default_data_target_key, (0, 0))[0]
print("All done.", file=log.v3)