def callback(ch, method, properties, body):
data = json.loads(body)
# We need:
# - name: name of the current sheet
# - partials: file names of the additional MEI files (including extension) (list).
# These "partial" MEIs need to be in the "whole" folder of the sheet already, just like the skeleton.mei
sheet_name = data['name']
partial_file_names = data['partials']
# Get sheet id (for status queue)
sheet_id = str(db[cfg.col_sheet].find_one({"name": sheet_name})["_id"])
whole_dir = fsm.get_sheet_whole_directory(sheet_name)
skeleton_path = whole_dir / 'aligned.mei'
partial_paths = [whole_dir / partial for partial in partial_file_names]
# skeleton always has 1 section which just contains the measures and some additional tags
skeleton_document = xml.parse(str(skeleton_path)).documentElement
skeleton_section = skeleton_document.getElementsByTagName("section")[0]
skeleton_section_xml = tt.purge_non_element_nodes(skeleton_section).toxml()
partial_sections_xml = []
for partial_path in partial_paths:
if partial_path.is_file():
partial = xml.parse(str(partial_path))
# We have to extract the measures and put them under a "fake" section root to get a similar structure as the skeleton
partial = tt.replace_child_nodes(
tt.create_element_node("section"),
partial.getElementsByTagName("measure"))
partial = tt.purge_non_element_nodes(partial)
partial_sections_xml.append(partial.toxml())
# Perform the alignments and node picking
aligned_trees = ta.align_trees_multiple([skeleton_section_xml] +
partial_sections_xml)
final_section_tree, _ = ta.build_consensus_tree(
aligned_trees, consensus_method=ta.consensus_bnd_enrich_skeleton)
# The final tree only aligned the section with measures, so we need to put the contents of that section back now
tt.replace_child_nodes(skeleton_section, final_section_tree.childNodes)
# Write the final tree to a file
with open(whole_dir / 'aligned.mei', 'w') as aligned_mei_file:
# We also purge everything that is not an element, to keep the tree clean and easily output a prettified XML file
aligned_mei_file.write(
tt.purge_non_element_nodes(skeleton_document).toprettyxml())
# Update status
status_update_msg = {
'_id': sheet_id,
'module': 'aligner',
'status': 'complete',
'name': sheet_name
}
global channel
channel.queue_declare(queue=cfg.mq_omr_planner_status)
channel.basic_publish(exchange="",
routing_key=cfg.mq_omr_planner_status,
body=json.dumps(status_update_msg))
def populate_payload_xml(score, tree, task):
task_type_name = task["type"]
task_step_name = task["step"]
measures = tree.getElementsByTagName("measure")
first_measure_index = measures[0].getAttribute("n")
# Scoredefs
if task_type_name == "0_fix_scoredef" and scoredef_chance > rnd.random():
node = tt.create_element_node(
"scoreDef", {
"key.sig": rnd.choice(key_sigs),
"key.mode": rnd.choice(key_modes),
"crmp_id": str(uuid.uuid4())
})
print(
f" Putting scoreDef {node.toxml()} before measure n={first_measure_index}"
)
tree.insertBefore(node, measures[0]).parentNode
# Clef detection
elif task_type_name == "1_detect_clefs" and task_step_name == "edit" and clef_chance > rnd.random(
):
node = tt.create_element_node("clef", {"shape": "G", "line": "2"})
staff = tree.getElementsByTagName("staff")[0]
first_staff_index = staff.getAttribute('n')
print(
f" Putting clef {node.toxml()} in measure n={first_measure_index}, staff n={first_staff_index}"
)
for layer in staff.getElementsByTagName("layer"):
layer.appendChild(node.cloneNode(deep=True))
break
# Note editing
elif task_type_name == "2_add_notes" and task_step_name == "edit":
staff = tree.getElementsByTagName("staff")[0]
staff_index = staff.getAttribute('n')
for layer in staff.getElementsByTagName("layer"):
for i in [_ for _ in range(4) if note_chance > rnd.random()]:
note_attr = {
"dur": rnd.choice(note_dur),
"oct": rnd.choice(note_oct),
"pname": rnd.choice(note_pname)
}
node = tt.create_element_node("note", note_attr)
print(
f" Putting note {node.toxml()} in measure n={first_measure_index}, staff n={staff_index}"
)
layer.appendChild(node.cloneNode(deep=True))
return tt.purge_non_element_nodes(tree).toxml()
def to_db_dict(self):
"""
Get a dictionary representation of the object for storage in the database.
"""
measures = []
for measure in self.measures:
staffs = [staff._asdict() for staff in measure.staffs]
measures.append(
Measure(staffs, measure.index, measure.xml, measure.context,
measure.score_def_before_measure)._asdict())
return {
"name": self.name,
"measures": measures,
"context": tt.purge_non_element_nodes(self.context).toprettyxml()
}
def callback(ch, method, properties, body):
data = json.loads(body)
sheet_name = data['name']
task_id = data['task_id']
client = MongoClient(cfg.mongodb_address.ip, cfg.mongodb_address.port)
db = client[cfg.db_name]
# Get MEI file and measures
mei_path = fsm.get_sheet_whole_directory(sheet_name) / "aligned.mei"
mei_xml_tree = xml.parse(str(mei_path))
# mei_measures = mei_xml.getElementsByTagName("measure")
# Obtain corresponding task and slice
task = db[cfg.col_task].find_one({"_id": ObjectId(task_id)})
# measure_staff_slice = db[cfg.col_slice].find_one({"_id" : ObjectId(task["slice_id"])})
# slice_measures = mei_measures[measure_staff_slice["start"]: measure_staff_slice["end"]]
# Get aggregated XML
aggregated_result = db[cfg.col_aggregated_result].find_one({
"task_id":
task_id,
"step":
task["step"]
})
if aggregated_result:
aggregated_xml = aggregated_result["result"]
# Temporary solution: give the slice somewhat more context by inserting only the header of the previous measure into it
tree = xml.parseString(aggregated_xml).documentElement
index = int(tree.getElementsByTagName("measure")[0].getAttribute(
"n")) - 1 # n-index is shifted up by 1
if index > 0:
measure = mei_xml_tree.getElementsByTagName("measure")[
index - 1].cloneNode(deep=True) # get the previous measure
measure.childNodes = []
tree.insertBefore(measure, tree.childNodes[0])
aggregated_xml = tree.toxml()
# Perform combination with original MEI via tree aligner
mei_section = mei_xml_tree.getElementsByTagName("section")[0]
mei_section_xml = mei_section.toxml()
aligned_trees = ta.align_trees_multiple(
[mei_section_xml, aggregated_xml],
distance_function=ta.node_distance_anchored)
final_section_tree, _ = ta.build_consensus_tree(
aligned_trees, consensus_method=ta.consensus_bnd_override_inner)
tt.replace_child_nodes(mei_section, final_section_tree.childNodes)
# Write MEI file
with open(str(mei_path), 'w') as mei_file:
mei_file.write(
tt.purge_non_element_nodes(
mei_xml_tree.documentElement).toprettyxml())
status_update_msg = {
'_id': task_id,
'module': 'score_rebuilder',
'status': 'complete'
}
else:
print(
f"Aggregated result for task with id {task_id} at step {task['step']} did not exist!"
)
status_update_msg = {
'_id': task_id,
'module': 'score_rebuilder',
'status': 'failed'
}
global channel
channel.queue_declare(queue=cfg.mq_task_scheduler_status)
channel.basic_publish(exchange="",
routing_key=cfg.mq_task_scheduler_status,
body=json.dumps(status_update_msg))
def align_trees_multiple(trees,
distance_function=node_distance,
gap_penalty=GAP_PENALTY,
verbose=False):
# If we just have one tree, don't do anything fancy and just return that one
if len(trees) == 1:
return [tt.purge_non_element_nodes(xml.parseString(trees[0]))]
# Create all distinct xml pairs
pairs = {}
for i in range(len(trees)):
for j in range(i + 1, len(trees)):
a = tt.purge_non_element_nodes(xml.parseString(trees[i]))
b = tt.purge_non_element_nodes(xml.parseString(trees[j]))
pairs[(i, j)] = (a, b)
# Perform pairwise alignments
distances = np.full((len(trees), len(trees)), np.inf)
for i, j in pairs:
a, b = pairs[i, j]
distances[i, j] = align_trees_pairwise(
a, b, distance_function=distance_function, gap_penalty=gap_penalty)
distance_bins = [0] * len(trees)
for i, j in pairs:
distance_bins[i] += distances[i, j]
distance_bins[j] += distances[i, j]
# Tree with the smallest overall distance will be a good candidate
main_tree_index = np.argmin(distance_bins)
# Use the pairwise alignments with a heuristic to come up with a multiple alignment solution
mas = []
candidate_pairs = sorted([(i, j)
for i, j in pairs if main_tree_index in (i, j)],
key=lambda t: distances[t[0], t[1]])
closest_pair_index = candidate_pairs[0]
closest_pair = pairs[closest_pair_index]
mas.append(closest_pair[closest_pair_index.index(
main_tree_index)]) # The closest tree
mas.append(closest_pair[not closest_pair_index.index(main_tree_index)]
) # The "other" tree
for pair_index in candidate_pairs[1:]:
pair = pairs[pair_index]
main_tree = pair[pair_index.index(main_tree_index)]
cand_tree = pair[not pair_index.index(main_tree_index)]
if verbose:
print(f"ITERATION {candidate_pairs.index(pair_index)}:")
print("===MAIN:")
print(main_tree.toprettyxml())
print("===CAND:")
print(cand_tree.toprettyxml())
print()
# Copy the gaps
for tree in mas:
copy_gaps(main_tree, tree)
mas.append(cand_tree)
if verbose:
print("===RESULT SO FAR:")
for tree in mas:
print(tree.toprettyxml())
print("============")
print()
print()
print()
print("------------------------------------------------")
if verbose:
print("===FINAL RESULTS:")
for index, tree in enumerate(mas):
print(f"TREE AT INDEX {index}:")
print(tree.toprettyxml())
print("============")
return mas
def callback(ch, method, properties, body):
data = json.loads(body)
sheet_name = data['name']
post_processing_steps = data["steps"]
task_id = data['task_id']
# Get MEI file
mei_path = fsm.get_sheet_whole_directory(sheet_name) / "aligned.mei"
mei_xml_tree = tt.purge_non_element_nodes(xml.parse(str(mei_path)))
mei_section = mei_xml_tree.getElementsByTagName("section")[0]
if "clef" in post_processing_steps:
print(f"Performing clef post-processing for sheet {sheet_name}")
for layer in mei_xml_tree.getElementsByTagName("layer"):
element = layer.firstChild
if element != None and element.tagName=="clef":
staff = layer.parentNode
measure = staff.parentNode
clef_line = element.getAttribute("line")
clef_shape = element.getAttribute("shape")
layer.removeChild(element)
prev = measure.previousSibling
scoreDef = None
while prev:
if prev.tagName == "measure":
break
if prev.tagName == "scoreDef":
scoreDef = prev
break
prev = prev.previousSibling
# TODO: actually generalize this code
if not scoreDef:
scoreDef = tt.create_element_node("scoreDef")
mei_section.insertBefore(scoreDef, measure)
staffGrp = tt.first_or_none(scoreDef, "staffGrp")
if not staffGrp:
staffGrp = tt.create_element_node("staffGrp")
scoreDef.appendChild(staffGrp)
staffDef = tt.first_or_none(staffGrp, "staffDef", lambda e: e.getAttribute("n") == staff.getAttribute("n"))
if not staffDef:
staffDef = tt.create_element_node("staffDef", {"n": staff.getAttribute("n")})
staffGrp.appendChild(staffDef)
staffDef.setAttribute("clef.line", clef_line)
staffDef.setAttribute("clef.shape", clef_shape)
# Write MEI file if there were changes
if post_processing_steps:
with open(str(mei_path), 'w') as mei_file:
mei_file.write(tt.purge_non_element_nodes(mei_xml_tree.documentElement).toprettyxml())
status_update_msg = {
'_id': task_id,
'module': 'post_processing',
'status': 'complete'
}
global channel
channel.queue_declare(queue=cfg.mq_task_scheduler_status)
channel.basic_publish(exchange="", routing_key=cfg.mq_task_scheduler_status, body=json.dumps(status_update_msg))