def _make_example(ids, raw_num_bytes):
if FLAGS.combine_to_length > 0:
ids += [0] * (FLAGS.combine_to_length - len(ids))
return generator_utils.to_example({
"targets": ids,
"inputs": [0],
"raw_num_bytes": [raw_num_bytes]
}).SerializeToString()
def __call__(self):
for case in self._gen:
source_ints = case["inputs"]
target_ints = case["targets"]
yield generator_utils.to_example({
"inputs": source_ints,
"targets": target_ints
})
def _make_example(ids, raw_num_bytes):
if FLAGS.combine_to_length > 0:
ids += [0] * (FLAGS.combine_to_length - len(ids))
return generator_utils.to_example({
"targets": ids,
"inputs": [0],
"raw_num_bytes": [raw_num_bytes]
}).SerializeToString()
def _get_online_predictions(self, lines: List[str]) -> List[int]:
"""retrieves predictions by triggering google cloud function, which
invokes ml-engine to make a prediction for each line.
"""
# constructs instances for predictions
contexts = self._get_line_context(lines, n=CONTEXT_N_LINES)
instances = []
for i, line in enumerate(lines):
context = contexts[i]
if MAX_LENGTH > 0:
if len(line) > MAX_LENGTH:
line = line[:MAX_LENGTH]
context = ''
elif (len(line) + len(context)) > MAX_LENGTH:
context = context[:MAX_LENGTH-len(line)]
assert (len(line) + len(context)) <= MAX_LENGTH
instances.append({'inputs': line, 'context': context})
if self.verbosity > 0:
print(f'making "line type" predictions for {len(instances)} lines...')
if self.verbosity > 1:
raw_instances = instances.copy()
problem_class = PROBLEM_CLASSES[PROBLEM]
problem = problem_class()
encoders = problem.feature_encoders(data_dir=DATA_DIR)
instances_b64 = []
for instance in instances:
if 'label' not in instance:
instance['label'] = 0
encoded_instance = problem.encode_example(instance, encoders)
# encoded_sample.pop('targets')
# encoded_sample.pop('context')
serialized_instance = to_example(encoded_instance).SerializeToString()
instances_b64.append({"b64": base64.b64encode(serialized_instance).decode('utf-8')})
instances = instances_b64
preds = []
batch_generator = batchify(instances, BATCH_SIZE)
if self.verbosity > 0:
batch_generator = tqdm(batch_generator, total=np.ceil(len(instances)/BATCH_SIZE).astype(int))
for batch in batch_generator:
if LOCAL:
res = requests.post(LOCAL_URL, data=json.dumps({"instances": batch}),
headers={"Content-Type": "application/json"})
else:
res = self._get_cloud_predictions(project=PROJECT, model=MODEL, instances=batch, version=VERSION)
assert res.ok, f'request failed. Reason: {res.reason}.'
predictions = json.loads(res.content)
predictions = predictions['predictions']
for i, pred in enumerate(predictions):
preds.append(pred['outputs'][0][0][0])
if self.verbosity > 1:
for i, pred in enumerate(preds):
instance = raw_instances[i]
if 'label' in instance:
instance.pop('label')
print(f'INPUT: {instance}\nOUTPUT: {pred}')
return preds
def save(df_row, save_folder, num_rows, num_cols, use_sparse=False):
dictionaries = pandas2dict(df_row, num_rows, num_cols, use_sparse)
for i, dictionary in enumerate(dictionaries):
filename = get_save_path(df_row.image_path, save_folder, use_sparse,
i + 1)
print(filename)
tf_example = to_example(dictionary).SerializeToString()
writer = tf.python_io.TFRecordWriter(filename)
writer.write(tf_example)
writer.close()
def _mean_to_example(mean_stdev):
"""Converts the found mean and stdev to tfrecords example."""
# mean_stdev is a dict
mean_stdev['mean'] = np.reshape(mean_stdev['mean'],
[10]).astype(np.float32).tolist()
mean_stdev['variance'] = np.reshape(mean_stdev['variance'],
[10]).astype(np.float32).tolist()
mean_stdev['stddev'] = np.reshape(mean_stdev['stddev'],
[10]).astype(np.float32).tolist()
mean_stdev['count'] = np.reshape(mean_stdev['count'],
[1]).astype(np.int64).tolist()
return generator_utils.to_example(mean_stdev)
def process(self, source_target_list):
example_dicts = [
self._make_spm_example_dict(source_text, target_text)
for source_text, target_text in source_target_list
]
if self._packed_examples:
example_dicts = pack_examples(
example_dicts, has_inputs=True, packed_length=self._packed_length)
for example_dict in example_dicts:
try:
padded_example_dict = self._pad_example_dict(example_dict)
except ValueError:
metrics.Metrics.counter('err_too_long', 'count').inc()
else:
yield to_example(padded_example_dict).SerializeToString()
def my_fn(records):
"""Function from list of TFRecords to list of TFRecords."""
examples = []
for record in records:
x = tf.train.Example()
x.ParseFromString(record)
example_dict = {}
if self.has_inputs:
example_dict["inputs"] = [
int(i) for i in x.features.feature["inputs"].int64_list.value]
example_dict["targets"] = [
int(i) for i in x.features.feature["targets"].int64_list.value]
examples.append(example_dict)
examples = list(self._maybe_pack_examples(examples))
return [
generator_utils.to_example(x).SerializeToString() for x in examples]
def predict_hansard_line_type4(request):
"""HTTP Cloud Function.
Args:
request (flask.Request): The request object.
<http://flask.pocoo.org/docs/1.0/api/#flask.Request>
Returns:
The response text, or any set of values that can be turned into a
Response object using `make_response`
<http://flask.pocoo.org/docs/1.0/api/#flask.Flask.make_response>.
"""
if request.method == 'POST':
assert 'content-type' in request.headers and request.headers['content-type'] == 'application/json'
# retrieve problem encoder.
problem = HansardLineType4()
assert problem.vocab_type == VocabType.CHARACTER
encoder = problem.get_or_create_vocab(data_dir=None, tmp_dir=None)
# serialize raw string instances
request_json = request.get_json(silent=True)
request_args = request.args
raw_instances = request_json['instances']
model_name = escape(request_args['model_name']) if 'model_name' in request_args else MODEL_NAME
serialized_instances = []
for instance in raw_instances:
if isinstance(instance, str):
instance = {'inputs': instance}
inputs = escape(instance['inputs'])
context = escape(instance['context']) if 'context' in instance else ''
encoded_instance = problem.encode_example({"inputs": inputs, "context": context, "label": 0}, encoder)
# encoded_sample.pop('targets')
# encoded_sample.pop('context')
serialized_instance = to_example(encoded_instance).SerializeToString()
serialized_instances.append(serialized_instance)
instances = []
for serialized_instance in serialized_instances:
instances.append({"b64": base64.b64encode(serialized_instance).decode('utf-8')})
# model_name = "hansard_line_type4_predict_lstm_encoder_lstm_attention"
# url = "https://us-central1-hansardparser.cloudfunctions.net/predictHansardLineType4"
# headers = {"Content-Type": "application/json"}
# makes predict request.
predictions = _predict_json(PROJECT_NAME, model_name, instances, version=None)
return jsonify(predictions)
return abort(405)
def _create_example(pathuni):
"""Bulk of dataset processing. Converts str path to serialized tf.Example."""
path, uni, binary_fp = pathuni
final = {}
# zoom out
path = svg_utils.zoom_out(path)
# make clockwise
path = svg_utils.canonicalize(path)
# render path for training
final['rendered'] = svg_utils.per_step_render(path, absolute=True)
# make path relative
path = svg_utils.make_relative(path)
# convert to vector
vector = svg_utils.path_to_vector(path, categorical=True)
# make simple vector
vector = np.array(vector)
vector = np.concatenate(
[np.take(vector, [0, 4, 5, 9], axis=-1), vector[..., -6:]], axis=-1)
# count some stats
final['seq_len'] = np.shape(vector)[0]
final['class'] = int(svg_utils.map_uni_to_alphanum(uni))
final['binary_fp'] = str(binary_fp)
# append eos
vector = svg_utils.append_eos(vector.tolist(), True, 10)
# pad path to 51 (with eos)
final['sequence'] = np.concatenate(
(vector, np.zeros(((50 - final['seq_len']), 10))), 0)
# make pure list:
final['rendered'] = np.reshape(final['rendered'][..., 0],
[64 * 64]).astype(np.float32).tolist()
final['sequence'] = np.reshape(final['sequence'],
[51 * 10]).astype(np.float32).tolist()
final['class'] = np.reshape(final['class'],
[1]).astype(np.int64).tolist()
final['seq_len'] = np.reshape(final['seq_len'],
[1]).astype(np.int64).tolist()
return generator_utils.to_example(final).SerializeToString()
def random_load_from_generator(self,
generator,
prefix_tag_length=4,
max_num_examples=-1,
log_every=100):
"""Builds TFExample from dict, serializes, and writes to CBT."""
prefix = self.prefix
table = self.table
for i, example_dict in enumerate(generator):
if not isinstance(example_dict, dict):
msg = "Expected generator to yield dict's, saw {}.".format(
type(example_dict))
raise ValueError(msg)
example = to_example(example_dict)
example = example.SerializeToString()
# Random target key
target_key = random_key(prefix=prefix, length=prefix_tag_length).encode()
row = table.row(target_key)
row.set_cell(column_family_id="tfexample",
column="example",
value=example,
timestamp=datetime.datetime(1970, 1, 1))
# Don't set a timestamp so we set instead of
# append cell values.
#timestamp=datetime.datetime.utcnow())
table.mutate_rows([row])
if log_every > 0 and i % log_every == 0:
tf.logging.info("Generated {} examples...".format(i))
if max_num_examples > 0 and i >= max_num_examples:
break
tf.logging.info("Generated {} examples.".format(i))
return i
def process(self, kv):
# Seed random number generator based on key so that hop times are
# deterministic.
key, ns_str = kv
m = hashlib.md5(key)
random.seed(int(m.hexdigest(), 16))
# Deserialize NoteSequence proto.
ns = music_pb2.NoteSequence.FromString(ns_str)
# Apply sustain pedal.
ns = sequences_lib.apply_sustain_control_changes(ns)
# Remove control changes as there are potentially a lot of them and they are
# no longer needed.
del ns.control_changes[:]
for _ in range(self._num_replications):
for augment_fn in self._augment_fns:
# Augment and encode the performance.
try:
augmented_performance_sequence = augment_fn(ns)
except DataAugmentationError:
Metrics.counter('extract_examples',
'augment_performance_failed').inc()
continue
seq = self._encode_performance_fn(
augmented_performance_sequence)
# feed in performance as both input/output to music transformer
# chopping sequence into length 2048 (throw out shorter sequences)
if len(seq) >= 2048:
max_offset = len(seq) - 2048
offset = random.randrange(max_offset + 1)
cropped_seq = seq[offset:offset + 2048]
example_dict = {
'inputs': cropped_seq,
'targets': cropped_seq
}
yield generator_utils.to_example(example_dict)
def process(self, kv):
# Seed random number generator based on key so that hop times are
# deterministic.
key, ns_str = kv
m = hashlib.md5(key)
random.seed(int(m.hexdigest(), 16))
# Deserialize NoteSequence proto.
ns = note_seq.NoteSequence.FromString(ns_str)
# Apply sustain pedal.
ns = sequences_lib.apply_sustain_control_changes(ns)
# Remove control changes as there are potentially a lot of them and they are
# no longer needed.
del ns.control_changes[:]
for _ in range(self._num_replications):
for augment_fn in self._augment_fns:
# Augment and encode the performance.
try:
augmented_performance_sequence = augment_fn(ns)
except DataAugmentationError:
Metrics.counter('extract_examples',
'augment_performance_failed').inc()
continue
seq = self._encode_performance_fn(
augmented_performance_sequence)
# feed in performance as both input/output to music transformer
# chopping sequence into length 2048 (throw out shorter sequences)
if len(seq) >= 2048:
max_offset = len(seq) - 2048
offset = random.randrange(max_offset + 1)
cropped_seq = seq[offset:offset + 2048]
example_dict = {
'inputs': cropped_seq,
'targets': cropped_seq
}
if self._melody:
# decode truncated performance sequence for melody inference
decoded_midi = self._decode_performance_fn(cropped_seq)
decoded_ns = note_seq.midi_io.midi_file_to_note_sequence(
decoded_midi)
# extract melody from cropped performance sequence
melody_instrument = melody_inference.infer_melody_for_sequence(
decoded_ns,
melody_interval_scale=2.0,
rest_prob=0.1,
instantaneous_non_max_pitch_prob=1e-15,
instantaneous_non_empty_rest_prob=0.0,
instantaneous_missing_pitch_prob=1e-15)
# remove non-melody notes from score
score_sequence = copy.deepcopy(decoded_ns)
score_notes = []
for note in score_sequence.notes:
if note.instrument == melody_instrument:
score_notes.append(note)
del score_sequence.notes[:]
score_sequence.notes.extend(score_notes)
# encode melody
encode_score_fn = self._encode_score_fns['melody']
example_dict['melody'] = encode_score_fn(
score_sequence)
# make sure performance input also matches targets; needed for
# compatibility of both perf and (mel & perf) autoencoders
if self._noisy:
# randomly sample a pitch shift to construct noisy performance
all_pitches = [x.pitch for x in decoded_ns.notes]
min_val = min(all_pitches)
max_val = max(all_pitches)
transpose_range = range(-(min_val - 21),
108 - max_val + 1)
try:
transpose_range.remove(
0) # make sure you transpose
except ValueError:
pass
transpose_amount = random.choice(transpose_range)
augmented_ns, _ = sequences_lib.transpose_note_sequence(
decoded_ns,
transpose_amount,
min_allowed_pitch=21,
max_allowed_pitch=108,
in_place=False)
aug_seq = self._encode_performance_fn(augmented_ns)
example_dict['performance'] = aug_seq
else:
example_dict['performance'] = example_dict[
'targets']
del example_dict['inputs']
Metrics.counter('extract_examples',
'encoded_example').inc()
Metrics.distribution(
'extract_examples',
'performance_length_in_seconds').update(
int(augmented_performance_sequence.total_time))
yield generator_utils.to_example(example_dict)
def process(self, kv):
# Seed random number generator based on key so that hop times are
# deterministic.
key, ns_str = kv
m = hashlib.md5(key.encode('utf-8'))
random.seed(int(m.hexdigest(), 16))
# Deserialize NoteSequence proto.
ns = note_seq.NoteSequence.FromString(ns_str)
# Apply sustain pedal.
ns = sequences_lib.apply_sustain_control_changes(ns)
# Remove control changes as there are potentially a lot of them and they are
# no longer needed.
del ns.control_changes[:]
if (self._min_hop_size_seconds
and ns.total_time < self._min_hop_size_seconds):
Metrics.counter('extract_examples', 'sequence_too_short').inc()
return
sequences = []
for _ in range(self._num_replications):
if self._max_hop_size_seconds:
if self._max_hop_size_seconds == self._min_hop_size_seconds:
# Split using fixed hop size.
sequences += sequences_lib.split_note_sequence(
ns, self._max_hop_size_seconds)
else:
# Sample random hop positions such that each segment size is within
# the specified range.
hop_times = [0.0]
while hop_times[
-1] <= ns.total_time - self._min_hop_size_seconds:
if hop_times[
-1] + self._max_hop_size_seconds < ns.total_time:
# It's important that we get a valid hop size here, since the
# remainder of the sequence is too long.
max_offset = min(
self._max_hop_size_seconds, ns.total_time -
self._min_hop_size_seconds - hop_times[-1])
else:
# It's okay if the next hop time is invalid (in which case we'll
# just stop).
max_offset = self._max_hop_size_seconds
offset = random.uniform(self._min_hop_size_seconds,
max_offset)
hop_times.append(hop_times[-1] + offset)
# Split at the chosen hop times (ignoring zero and the final invalid
# time).
sequences += sequences_lib.split_note_sequence(
ns, hop_times[1:-1])
else:
sequences += [ns]
for performance_sequence in sequences:
if self._encode_score_fns:
# We need to extract a score.
if not self._absolute_timing:
# Beats are required to extract a score with metric timing.
beats = [
ta for ta in performance_sequence.text_annotations
if ta.annotation_type == BEAT
and ta.time <= performance_sequence.total_time
]
if len(beats) < 2:
Metrics.counter('extract_examples',
'not_enough_beats').inc()
continue
# Ensure the sequence starts and ends on a beat.
performance_sequence = sequences_lib.extract_subsequence(
performance_sequence,
start_time=min(beat.time for beat in beats),
end_time=max(beat.time for beat in beats))
# Infer beat-aligned chords (only for relative timing).
try:
chord_inference.infer_chords_for_sequence(
performance_sequence,
chord_change_prob=0.25,
chord_note_concentration=50.0,
add_key_signatures=True)
except chord_inference.ChordInferenceError:
Metrics.counter('extract_examples',
'chord_inference_failed').inc()
continue
# Infer melody regardless of relative/absolute timing.
try:
melody_instrument = melody_inference.infer_melody_for_sequence(
performance_sequence,
melody_interval_scale=2.0,
rest_prob=0.1,
instantaneous_non_max_pitch_prob=1e-15,
instantaneous_non_empty_rest_prob=0.0,
instantaneous_missing_pitch_prob=1e-15)
except melody_inference.MelodyInferenceError:
Metrics.counter('extract_examples',
'melody_inference_failed').inc()
continue
if not self._absolute_timing:
# Now rectify detected beats to occur at fixed tempo.
# TODO(iansimon): also include the alignment
score_sequence, unused_alignment = sequences_lib.rectify_beats(
performance_sequence, beats_per_minute=SCORE_BPM)
else:
# Score uses same timing as performance.
score_sequence = copy.deepcopy(performance_sequence)
# Remove melody notes from performance.
performance_notes = []
for note in performance_sequence.notes:
if note.instrument != melody_instrument:
performance_notes.append(note)
del performance_sequence.notes[:]
performance_sequence.notes.extend(performance_notes)
# Remove non-melody notes from score.
score_notes = []
for note in score_sequence.notes:
if note.instrument == melody_instrument:
score_notes.append(note)
del score_sequence.notes[:]
score_sequence.notes.extend(score_notes)
# Remove key signatures and beat/chord annotations from performance.
del performance_sequence.key_signatures[:]
del performance_sequence.text_annotations[:]
Metrics.counter('extract_examples', 'extracted_score').inc()
for augment_fn in self._augment_fns:
# Augment and encode the performance.
try:
augmented_performance_sequence = augment_fn(
performance_sequence)
except DataAugmentationError:
Metrics.counter('extract_examples',
'augment_performance_failed').inc()
continue
example_dict = {
'targets':
self._encode_performance_fn(augmented_performance_sequence)
}
if not example_dict['targets']:
Metrics.counter('extract_examples',
'skipped_empty_targets').inc()
continue
if (self._random_crop_length and len(example_dict['targets']) >
self._random_crop_length):
# Take a random crop of the encoded performance.
max_offset = len(
example_dict['targets']) - self._random_crop_length
offset = random.randrange(max_offset + 1)
example_dict['targets'] = example_dict['targets'][
offset:offset + self._random_crop_length]
if self._encode_score_fns:
# Augment the extracted score.
try:
augmented_score_sequence = augment_fn(score_sequence)
except DataAugmentationError:
Metrics.counter('extract_examples',
'augment_score_failed').inc()
continue
# Apply all score encoding functions.
skip = False
for name, encode_score_fn in self._encode_score_fns.items(
):
example_dict[name] = encode_score_fn(
augmented_score_sequence)
if not example_dict[name]:
Metrics.counter('extract_examples',
'skipped_empty_%s' % name).inc()
skip = True
break
if skip:
continue
Metrics.counter('extract_examples', 'encoded_example').inc()
Metrics.distribution(
'extract_examples',
'performance_length_in_seconds').update(
int(augmented_performance_sequence.total_time))
yield generator_utils.to_example(example_dict)
def _get_online_predictions(self, lines: List[str], types: List[str] = None) -> List[int]:
"""retrieves predictions by triggering google cloud function, which
invokes ml-engine to make a prediction for each line.
"""
contexts = self._get_line_context(lines, n=CONTEXT_N_LINES)
instances = []
for i, line in enumerate(lines):
context = contexts[i]
if MAX_LENGTH > 0:
if len(line) > MAX_LENGTH:
line = line[:MAX_LENGTH]
context = ''
elif (len(line) + len(context)) > MAX_LENGTH:
context = context[:MAX_LENGTH-len(line)]
assert (len(line) + len(context)) <= MAX_LENGTH
instances.append({'inputs': line, 'context': context})
if self.verbosity > 1:
raw_instances = instances.copy()
if LABEL_SPEECHES_ONLY:
assert types is not None, '`types` must be provided when LABEL_SPEECHES_ONLY == True.'
assert len(types) == len(lines), f'types must have same length as lines, but {len(types)} != {len(lines)}.'
speeches = []
speeches_idx = []
for i, instance in enumerate(instances):
if types[i] == 'speech':
speeches.append(instance)
speeches_idx.append(i)
instances = speeches
if self.verbosity > 0:
print(f'Making "speaker span" predictions for {len(instances)} lines...')
problem_class = PROBLEM_CLASSES[PROBLEM]
problem = problem_class()
encoders = problem.feature_encoders(data_dir=DATA_DIR)
instances_b64 = []
for instance in instances:
if 'targets' not in instance:
instance['targets'] = ''
encoded_instance = problem.encode_example(instance, encoders)
# encoded_sample.pop('targets')
# encoded_sample.pop('context')
serialized_instance = to_example(encoded_instance).SerializeToString()
instances_b64.append({"b64": base64.b64encode(serialized_instance).decode('utf-8')})
instances = instances_b64
preds = []
batch_generator = batchify(instances, BATCH_SIZE)
if self.verbosity > 0:
batch_generator = tqdm(batch_generator, total=np.ceil(len(instances)/BATCH_SIZE).astype(int))
for batch in batch_generator:
try:
# print([len(inst['inputs']) + len(inst['context']) for inst in raw_instances[len(preds):len(preds)+BATCH_SIZE]])
if LOCAL:
res = requests.post(LOCAL_URL, data=json.dumps({"instances": batch}),
headers={"Content-Type": "application/json"})
else:
res = self._get_cloud_predictions(project=PROJECT, model=MODEL, instances=batch, version=VERSION)
assert res.ok, f'request failed. Reason: {res.reason}.'
predictions = json.loads(res.content)
predictions = predictions['predictions']
for i, pred in enumerate(predictions):
pred_out = pred['outputs']
pred_out = encoders['targets'].decode(pred_out)
# removes spaces.
pred_out = re.sub(r'\s+', '', pred_out)
try:
eos_idx = pred_out.lower().index('<eos>')
pred_out = pred_out[:eos_idx]
except ValueError:
if self.verbosity > 1:
logging.warn(f'<eos> not found in prediction: {pred_out}')
preds.append(pred_out)
# preds.append([token_pred[0][0] for token_pred in pred['outputs']])
except AssertionError as e:
print(e)
for i in range(len(batch)):
preds.append(None)
if LABEL_SPEECHES_ONLY:
preds_all_lines = []
for i, line in enumerate(lines):
pred = 'O' * len(line)
preds_all_lines.append(pred)
n_preds = 0
assert len(speeches_idx) == len(preds)
for i, idx in enumerate(speeches_idx):
preds_all_lines[idx] = preds[i]
n_preds += 1
# sanity check.
assert n_preds == len(preds)
preds = preds_all_lines
if self.verbosity > 1:
for i, pred in enumerate(preds):
instance = raw_instances[i]
if 'targets' in instance:
instance.pop('targets')
if 'label' in instance:
instance.pop('label')
print(f'INPUT (len={len(instance["inputs"])}): {instance}\nOUTPUT (len={len(pred) if pred is not None else None}): {pred}')
return preds
def process(self, kv):
# Seed random number generator based on key so that hop times are
# deterministic.
key, ns_str = kv
m = hashlib.md5(key)
random.seed(int(m.hexdigest(), 16))
# Deserialize NoteSequence proto.
ns = music_pb2.NoteSequence.FromString(ns_str)
# Apply sustain pedal.
ns = sequences_lib.apply_sustain_control_changes(ns)
# Remove control changes as there are potentially a lot of them and they are
# no longer needed.
del ns.control_changes[:]
if (self._min_hop_size_seconds and
ns.total_time < self._min_hop_size_seconds):
Metrics.counter('extract_examples', 'sequence_too_short').inc()
return
sequences = []
for _ in range(self._num_replications):
if self._max_hop_size_seconds:
if self._max_hop_size_seconds == self._min_hop_size_seconds:
# Split using fixed hop size.
sequences += sequences_lib.split_note_sequence(
ns, self._max_hop_size_seconds)
else:
# Sample random hop positions such that each segment size is within
# the specified range.
hop_times = [0.0]
while hop_times[-1] <= ns.total_time - self._min_hop_size_seconds:
if hop_times[-1] + self._max_hop_size_seconds < ns.total_time:
# It's important that we get a valid hop size here, since the
# remainder of the sequence is too long.
max_offset = min(
self._max_hop_size_seconds,
ns.total_time - self._min_hop_size_seconds - hop_times[-1])
else:
# It's okay if the next hop time is invalid (in which case we'll
# just stop).
max_offset = self._max_hop_size_seconds
offset = random.uniform(self._min_hop_size_seconds, max_offset)
hop_times.append(hop_times[-1] + offset)
# Split at the chosen hop times (ignoring zero and the final invalid
# time).
sequences += sequences_lib.split_note_sequence(ns, hop_times[1:-1])
else:
sequences += [ns]
for performance_sequence in sequences:
if self._encode_score_fns:
# We need to extract a score.
if not self._absolute_timing:
# Beats are required to extract a score with metric timing.
beats = [
ta for ta in performance_sequence.text_annotations
if (ta.annotation_type ==
music_pb2.NoteSequence.TextAnnotation.BEAT)
and ta.time <= performance_sequence.total_time
]
if len(beats) < 2:
Metrics.counter('extract_examples', 'not_enough_beats').inc()
continue
# Ensure the sequence starts and ends on a beat.
performance_sequence = sequences_lib.extract_subsequence(
performance_sequence,
start_time=min(beat.time for beat in beats),
end_time=max(beat.time for beat in beats)
)
# Infer beat-aligned chords (only for relative timing).
try:
chord_inference.infer_chords_for_sequence(
performance_sequence,
chord_change_prob=0.25,
chord_note_concentration=50.0,
add_key_signatures=True)
except chord_inference.ChordInferenceError:
Metrics.counter('extract_examples', 'chord_inference_failed').inc()
continue
# Infer melody regardless of relative/absolute timing.
try:
melody_instrument = melody_inference.infer_melody_for_sequence(
performance_sequence,
melody_interval_scale=2.0,
rest_prob=0.1,
instantaneous_non_max_pitch_prob=1e-15,
instantaneous_non_empty_rest_prob=0.0,
instantaneous_missing_pitch_prob=1e-15)
except melody_inference.MelodyInferenceError:
Metrics.counter('extract_examples', 'melody_inference_failed').inc()
continue
if not self._absolute_timing:
# Now rectify detected beats to occur at fixed tempo.
# TODO(iansimon): also include the alignment
score_sequence, unused_alignment = sequences_lib.rectify_beats(
performance_sequence, beats_per_minute=SCORE_BPM)
else:
# Score uses same timing as performance.
score_sequence = copy.deepcopy(performance_sequence)
# Remove melody notes from performance.
performance_notes = []
for note in performance_sequence.notes:
if note.instrument != melody_instrument:
performance_notes.append(note)
del performance_sequence.notes[:]
performance_sequence.notes.extend(performance_notes)
# Remove non-melody notes from score.
score_notes = []
for note in score_sequence.notes:
if note.instrument == melody_instrument:
score_notes.append(note)
del score_sequence.notes[:]
score_sequence.notes.extend(score_notes)
# Remove key signatures and beat/chord annotations from performance.
del performance_sequence.key_signatures[:]
del performance_sequence.text_annotations[:]
Metrics.counter('extract_examples', 'extracted_score').inc()
for augment_fn in self._augment_fns:
# Augment and encode the performance.
try:
augmented_performance_sequence = augment_fn(performance_sequence)
except DataAugmentationError:
Metrics.counter(
'extract_examples', 'augment_performance_failed').inc()
continue
example_dict = {
'targets': self._encode_performance_fn(
augmented_performance_sequence)
}
if not example_dict['targets']:
Metrics.counter('extract_examples', 'skipped_empty_targets').inc()
continue
if self._encode_score_fns:
# Augment the extracted score.
try:
augmented_score_sequence = augment_fn(score_sequence)
except DataAugmentationError:
Metrics.counter('extract_examples', 'augment_score_failed').inc()
continue
# Apply all score encoding functions.
skip = False
for name, encode_score_fn in self._encode_score_fns.items():
example_dict[name] = encode_score_fn(augmented_score_sequence)
if not example_dict[name]:
Metrics.counter('extract_examples',
'skipped_empty_%s' % name).inc()
skip = True
break
if skip:
continue
Metrics.counter('extract_examples', 'encoded_example').inc()
Metrics.distribution(
'extract_examples', 'performance_length_in_seconds').update(
int(augmented_performance_sequence.total_time))
yield generator_utils.to_example(example_dict)
def dataset(self,
mode,
data_dir=None,
num_threads=None,
output_buffer_size=None,
shuffle_files=None,
hparams=None,
preprocess=True,
dataset_split=None,
shard=None,
partition_id=0,
num_partitions=1):
"""Build a Dataset for this problem.
Args:
mode: tf.estimator.ModeKeys; determines which files to read from.
data_dir: directory that contains data files.
num_threads: int, number of threads to use for decode and preprocess
Dataset.map calls.
output_buffer_size: int, how many elements to prefetch in Dataset.map
calls.
shuffle_files: whether to shuffle input files. Default behavior (i.e. when
shuffle_files=None) is to shuffle if mode == TRAIN.
hparams: tf.contrib.training.HParams; hparams to be passed to
Problem.preprocess_example and Problem.hparams. If None, will use a
default set that is a no-op.
preprocess: bool, whether to map the Dataset through
Problem.preprocess_example.
dataset_split: tf.estimator.ModeKeys + ["test"], which split to read data
from (TRAIN:"-train", EVAL:"-dev", "test":"-test"). Defaults to mode.
shard: int, if provided, will only read data from the specified shard.
Returns:
Dataset containing dict<feature name, Tensor>.
"""
if dataset_split or (mode in ["train", "eval"]):
# In case when pathes to preprocessed files pointed out or if train mode
# launched, we save preprocessed data first, and then create dataset from
# that files.
dataset_split = dataset_split or mode
assert data_dir
if not hasattr(hparams, "data_dir"):
hparams.add_hparam("data_dir", data_dir)
if not hparams.data_dir:
hparams.data_dir = data_dir
# Construct the Problem's hparams so that items within it are accessible
_ = self.get_hparams(hparams)
data_fields, data_items_to_decoders = self.example_reading_spec()
if data_items_to_decoders is None:
data_items_to_decoders = {
field: tf.contrib.slim.tfexample_decoder.Tensor(field)
for field in data_fields}
is_training = mode == tf.estimator.ModeKeys.TRAIN
data_filepattern = self.filepattern(data_dir, dataset_split, shard=shard)
tf.logging.info("Reading data files from %s", data_filepattern)
data_files = tf.contrib.slim.parallel_reader.get_data_files(
data_filepattern)
if shuffle_files or shuffle_files is None and is_training:
random.shuffle(data_files)
else:
# In case when pathes to preprocessed files not pointed out, we create
# dataset from generator object.
eos_list = [] if EOS is None else [EOS]
data_list = []
with tf.gfile.GFile(self.test_path, mode="r") as source_file:
for line in source_file:
if line:
if "\t" in line:
parts = line.split("\t", 1)
source, target = parts[0].strip(), parts[1].strip()
source_ints = self.source_vocab.encode(source) + eos_list
target_ints = self.target_vocab.encode(target) + eos_list
data_list.append({"inputs":source_ints, "targets":target_ints})
else:
source_ints = self.source_vocab.encode(line) + eos_list
data_list.append(generator_utils.to_example(
{"inputs":source_ints}))
gen = Gen(self.generator(self.test_path, self.source_vocab,
self.target_vocab))
dataset = dataset_ops.Dataset.from_generator(gen, tf.string)
preprocess = False
def decode_record(record):
"""Serialized Example to dict of <feature name, Tensor>."""
decoder = tf.contrib.slim.tfexample_decoder.TFExampleDecoder(
data_fields, data_items_to_decoders)
decode_items = list(data_items_to_decoders)
decoded = decoder.decode(record, items=decode_items)
return dict(zip(decode_items, decoded))
def _preprocess(example):
"""Whether preprocess data into required format."""
example = self.preprocess_example(example, mode, hparams)
self.maybe_reverse_features(example)
self.maybe_copy_features(example)
return example
dataset = (tf.data.Dataset.from_tensor_slices(data_files)
.interleave(lambda x:
tf.data.TFRecordDataset(x).map(decode_record,
num_parallel_calls=4),
cycle_length=4, block_length=16))
if preprocess:
dataset = dataset.map(_preprocess, num_parallel_calls=4)
return dataset
def _create_example(pathuni):
"""Bulk of dataset processing. Converts str path to serialized tf.Example."""
path, uni, binary_fp = pathuni
final = {}
# print(path)
#f = open('/home1/wangyz/magenta-master/data-anypair/all_paths_' + str(uni) +'.txt','a')
#f.write(str(path) + '\n')
# f.close()
f = open('/home1/wangyz/magenta-master/data-anypair/all_paths_' +
str(uni) + '.txt', 'r')
lines = f.read().split('\n')
idx = random.randint(0, len(lines)-2)
print(uni, binary_fp)
#source_path = list(lines[idx])
# print(ast.literal_eval(lines[idx]))
source_path = ast.literal_eval(lines[idx])
# input()
# zoom out
path = svg_utils.zoom_out(path)
source_path = svg_utils.zoom_out(source_path)
# make clockwise
path = svg_utils.canonicalize(path)
source_path = svg_utils.canonicalize(source_path)
# render path for training
final['rendered'] = svg_utils.per_step_render(path, absolute=True)
final['source_rendered'] = svg_utils.per_step_render(
source_path, absolute=True)
# make path relative
path = svg_utils.make_relative(path)
source_path = svg_utils.make_relative(source_path)
# convert to vector
vector = svg_utils.path_to_vector(path, categorical=True)
source_vector = svg_utils.path_to_vector(source_path, categorical=True)
# make simple vector
vector = np.array(vector)
source_vector = np.array(source_vector)
vector = np.concatenate(
[np.take(vector, [0, 4, 5, 9], axis=-1), vector[..., -6:]], axis=-1)
source_vector = np.concatenate(
[np.take(source_vector, [0, 4, 5, 9], axis=-1), source_vector[..., -6:]], axis=-1)
# count some stats
# print(np.shape(vector))
# print(np.shape(source_vector))
# input()
final['seq_len'] = np.shape(vector)[0]
final['source_seq_len'] = np.shape(source_vector)[0]
final['class'] = int(svg_utils.map_uni_to_alphanum(uni))
final['binary_fp'] = str(binary_fp)
# append eos
vector = svg_utils.append_eos(vector.tolist(), True, 10)
source_vector = svg_utils.append_eos(source_vector.tolist(), True, 10)
# pad path to 51 (with eos)
final['sequence'] = np.concatenate(
(vector, np.zeros(((50 - final['seq_len']), 10))), 0)
final['source_sequence'] = np.concatenate(
(source_vector, np.zeros(((50 - final['source_seq_len']), 10))), 0)
# make pure list:
final['rendered'] = np.reshape(final['rendered'][..., 0],
[64*64]).astype(np.float32).tolist()
final['source_rendered'] = np.reshape(final['source_rendered'][..., 0],
[64*64]).astype(np.float32).tolist()
final['sequence'] = np.reshape(final['sequence'],
[51*10]).astype(np.float32).tolist()
final['source_sequence'] = np.reshape(final['source_sequence'],
[51*10]).astype(np.float32).tolist()
final['class'] = np.reshape(final['class'],
[1]).astype(np.int64).tolist()
final['seq_len'] = np.reshape(final['seq_len'],
[1]).astype(np.int64).tolist()
final['source_seq_len'] = np.reshape(final['source_seq_len'],
[1]).astype(np.int64).tolist()
return generator_utils.to_example(final).SerializeToString()
