def test_bbox_to_integer_bounds(self):
# Arrange
expected = (44, 18, 56, 93)
expected2 = (44, 18, 56, 93)
# Act
actual = Node.round_bounding_box_to_integer(44.2, 18.9, 55.1, 92.99)
actual2 = Node.round_bounding_box_to_integer(44, 18, 56, 92.99)
# Assert
self.assertEqual(actual, expected)
self.assertEqual(actual2, expected2)
def build_staff_nodes(nodes: List[Node]) -> List[Node]:
"""Derives staff objects from staffline objects.
Assumes each staff has 5 stafflines.
Assumes the stafflines have already been merged."""
stafflines = [c for c in nodes
if c.class_name == _CONST.STAFFLINE_CLASS_NAME and
not __has_parent_staff(c, nodes)]
if len(stafflines) == 0:
return []
staffline_bboxes = [c.bounding_box for c in stafflines]
canvas, (_t, _l) = draw_nodes_on_empty_canvas(stafflines)
logging.warning('Creating staff bboxes and masks.')
# - Go top-down and group the stafflines by five to get staves.
# (The staffline bboxes are already sorted top-down.)
staff_bboxes = []
staff_masks = []
n_stafflines = len(stafflines)
for i in range(n_stafflines // 5):
_sbb = staffline_bboxes[5 * i:5 * (i + 1)]
_st = min([bb[0] for bb in _sbb])
_sl = min([bb[1] for bb in _sbb])
_sb = max([bb[2] for bb in _sbb])
_sr = max([bb[3] for bb in _sbb])
staff_bboxes.append((_st, _sl, _sb, _sr))
staff_masks.append(canvas[_st - _t:_sb - _t, _sl - _l:_sr - _l])
logging.info('Creating staff Nodes')
next_node_id = max([c.id for c in nodes]) + 1
dataset = nodes[0].dataset
document = nodes[0].document
staffs = []
for s_bb, s_m in zip(staff_bboxes, staff_masks):
t, l, b, r = s_bb
staff = Node(id_=next_node_id,
class_name=_CONST.STAFF_CLASS_NAME,
top=t, left=l, height=b - t, width=r - l,
mask=s_m,
dataset=dataset, document=document)
staffs.append(staff)
next_node_id += 1
for i, sc in enumerate(staffs):
sl_from = 5 * i
sl_to = 5 * (i + 1)
for sl in stafflines[sl_from:sl_to]:
sl.inlinks.append(sc.id)
sc.outlinks.append(sl.id)
return staffs
def upgrade_xml_file(nodes: List[Node]) -> List[Node]:
new_nodes = []
for node in nodes:
new_class_name = node.class_name
if node.class_name in CLASS_NAME_MAPPING:
new_class_name = CLASS_NAME_MAPPING[node.class_name]
new_node = Node(node.id, new_class_name, node.top, node.left,
node.width, node.height, node.outlinks, node.inlinks,
node.mask, node.dataset, node.document, node.data)
new_nodes.append(new_node)
return new_nodes
def map_class_names(self, nodes: List[Node]) -> List[Node]:
with open(
os.path.join(
self.path_of_this_file,
"MuscimaPlusPlusSymbolClassMapping.json")) as file:
symbol_class_mapping = json.load(file)
reclassified_nodes = []
for node in nodes:
reclassified_nodes.append(
Node(node.id, symbol_class_mapping[node.class_name], node.top,
node.left, node.width, node.height, node.outlinks,
node.inlinks, node.mask, node.dataset, node.document,
node.data))
return reclassified_nodes
def test_xml_export(self):
"""Ensure that XML export of node works as expected with 3-digit floating point rounding performed for probability values
"""
node = Node(0, 'test1', 10, 10, 20, 20,
[1 / 7, 4 / 7, 1 / 7, 0, 0, 0, 1 / 7], [2], [0.85], [3],
[0.7], np.zeros((20, 20)))
expected = \
"""<Node>
<Id>0</Id>
<ClassName>test1</ClassName>
<ClassLikelihoods>0.143 0.571 0.143 0.000 0.000 0.000 0.143</ClassLikelihoods>
<Top>10</Top>
<Left>10</Left>
<Width>20</Width>
<Height>20</Height>
<Mask>0:400</Mask>
<Inlinks>3</Inlinks>
<InlinksLikelihoods>0.700</InlinksLikelihoods>
<Outlinks>2</Outlinks>
<OutlinksLikelihoods>0.850</OutlinksLikelihoods>
</Node>"""
self.assertEqual(str(node), expected)
def get_features_distance_relative_bbox_and_clsname(
self, from_node: Node, to_node: Node) -> Dict:
"""Extract a feature vector from the given pair of Nodes.
Does *NOT* convert the class names to integers.
Features: bbox(c_to) - bbox(c_from), class_name(c_from), class_name(c_to)
Target: 1 if there is a link from u to v
Returns a tuple.
"""
target = 0
if from_node.document == to_node.document:
if to_node.id in from_node.outlinks:
target = 1
distance = from_node.distance_to(to_node)
features = (distance, to_node.top - from_node.top, to_node.left -
from_node.left, to_node.bottom - from_node.bottom,
to_node.right - from_node.right, from_node.class_name,
to_node.class_name, target)
dist, dt, dl, db, dr, cu, cv, tgt = features
if cu.startswith('letter'):
cu = 'letter'
if cu.startswith('numeral'):
cu = 'numeral'
if cv.startswith('letter'):
cv = 'letter'
if cv.startswith('numeral'):
cv = 'numeral'
feature_dict = {
'dist': dist,
'dt': dt,
'dl': dl,
'db': db,
'dr': dr,
'cls_from': cu,
'cls_to': cv,
'target': tgt
}
return feature_dict
def __compute_patch_center(m_from: Node, m_to: Node):
"""Computing the patch center for the given pair
of objects. Gets returned as (row, column) coordinates
with respect to the input image.
Option 1: take their center of gravity.
Option 2: if intersection, take center of intersection bbox.
If not, take center of line connecting closest points.
"""
intersection_bbox = m_from.bounding_box_intersection(m_to.bounding_box)
if intersection_bbox is not None:
it, il, ib, ir = intersection_bbox
i_center_x = (it + ib) // 2
i_center_y = (il + ir) // 2
p_center_x, p_center_y = i_center_x + m_from.top, i_center_y + m_from.left
else:
# The "closest point" computation, which can actually be implemented
# as taking the middle of the bounding box that is the intersection
# of extending the objects' bounding boxes towards each other.
# Vertical: take max of top, min of bottom, sort
p_bounds_vertical = max(m_from.top,
m_to.top), min(m_from.bottom, m_to.bottom)
# i_bounds_top, i_bounds_bottom = min(i_bounds_vertical), max(i_bounds_vertical)
# Horizontal: max of left, min of right, sort
p_bounds_horizontal = max(m_from.left,
m_to.left), min(m_from.right, m_to.right)
# i_bounds_left, i_bounds_right = min(i_bounds_horizontal), max(i_bounds_horizontal)
p_center_x = sum(p_bounds_vertical) // 2
p_center_y = sum(p_bounds_horizontal) // 2
return p_center_x, p_center_y
def __read_objects(xml_file):
document = minidom.parse(xml_file)
width = int(document.getElementsByTagName("width")[0].firstChild.nodeValue)
height = int(
document.getElementsByTagName("height")[0].firstChild.nodeValue)
nodes = []
for node in document.getElementsByTagName('object'):
left = float(
node.getElementsByTagName("xmin")[0].firstChild.nodeValue) * width
right = float(
node.getElementsByTagName("xmax")[0].firstChild.nodeValue) * width
top = float(
node.getElementsByTagName("ymin")[0].firstChild.nodeValue) * height
bottom = float(
node.getElementsByTagName("ymax")[0].firstChild.nodeValue) * height
new_node = Node(
0,
node.getElementsByTagName("name")[0].firstChild.nodeValue,
round(top), round(left), round(right - left), round(bottom - top))
if new_node.class_name == "staffLine":
nodes.append(new_node)
nodes = sorted(nodes, key=lambda node: node.top)
return nodes
def test_join_likelihoods(self):
"""Ensure that links and classes with their likelihoods are accurately stored when nodes are joined
"""
node1 = Node(0, 'test1', 10, 10, 20, 20, [0.9, 0.05, 0, 0, 0, 0, 0.04],
[2], [0.85], [3], [0.7], np.zeros((20, 20)))
node2 = Node(1, 'test2', 15, 15, 30, 30,
[0.1, 0.7, 0.1, 0.05, 0.05, 0, 0], [4], [0.5], [5], [0.5],
np.zeros((30, 30)))
node1.join(node2)
node_accurate_join = Node(0, 'test1', 10, 10, 35, 35,
[0.9, 0.05, 0, 0, 0, 0, 0.04], [2, 4],
[0.85, 0.5], [3, 5], [0.7, 0.5],
np.zeros((35, 35)))
self.assertEqual(node1.class_likelihoods,
node_accurate_join.class_likelihoods)
self.assertEqual(node1.inlinks, node_accurate_join.inlinks)
self.assertEqual(node1.inlinks_likelihoods,
node_accurate_join.inlinks_likelihoods)
self.assertEqual(node1.outlinks, node_accurate_join.outlinks)
self.assertEqual(node1.outlinks_likelihoods,
node_accurate_join.outlinks_likelihoods)
def main(args):
logging.info('Starting main...')
_start_time = time.clock()
########################################################
# Load gt image.
logging.info('Loading staffline image.')
# - Initialize Dataset. This checks for the root.
if args.staff_imfile is None:
cvc_dataset = CvcMuscimaDataset(root=args.root)
args.staff_imfile = cvc_dataset.imfile(page=args.number,
writer=args.writer,
distortion='ideal',
mode='staff_only')
# - Load the image.
gt = (imread(args.staff_imfile, as_grey=True) * 255).astype('uint8')
# - Cast as binary mask.
gt[gt > 0] = 1
########################################################
# Locate stafflines in gt image.
logging.info('Getting staffline connected components.')
# - Get connected components in gt image.
connected_components, labels, bboxes = compute_connected_components(gt)
# - Use vertical dimension of CCs to determine which ones belong together
# to form stafflines. (Criterion: row overlap.)
n_rows, n_cols = gt.shape
intervals = [[] for _ in range(n_rows)
] # For each row: which CCs have pxs on that row?
for label, (t, l, b, r) in list(bboxes.items()):
if label == 0:
continue
# Ignore very short staffline segments that can easily be artifacts
# and should not affect the vertical range of the staffline anyway.
if (r - l) < 8:
continue
for row in range(t, b):
intervals[row].append(label)
logging.info('Grouping staffline connected components into stafflines.')
staffline_components = [
] # For each staffline, we collect the CCs that it is made of
_in_staffline = False
_current_staffline_components = []
for r_labels in intervals:
if not _in_staffline:
# Last row did not contain staffline components.
if len(r_labels) == 0:
# No staffline component on current row
continue
else:
_in_staffline = True
_current_staffline_components += r_labels
else:
# Last row contained staffline components.
if len(r_labels) == 0:
# Current staffline has no more rows.
staffline_components.append(set(_current_staffline_components))
_current_staffline_components = []
_in_staffline = False
continue
else:
# Current row contains staffline components: the current
# staffline continues.
_current_staffline_components += r_labels
logging.info('No. of stafflines, with component groups: {0}'
''.format(len(staffline_components)))
# Now: merge the staffline components into one bbox/mask.
logging.info(
'Merging staffline components into staffline bboxes and masks.')
staffline_bboxes = []
staffline_masks = []
for sc in sorted(staffline_components,
key=lambda c: min([bboxes[cc][0]
for cc in c])): # Sorted top-down
st, sl, sb, sr = n_rows, n_cols, 0, 0
for component in sc:
t, l, b, r = bboxes[component]
st, sl, sb, sr = min(t, st), min(l, sl), max(b, sb), max(r, sr)
_sm = gt[st:sb, sl:sr]
staffline_bboxes.append((st, sl, sb, sr))
staffline_masks.append(_sm)
# Check if n. of stafflines is divisible by 5
n_stafflines = len(staffline_bboxes)
logging.info('\tTotal stafflines: {0}'.format(n_stafflines))
if n_stafflines % 5 != 0:
import matplotlib.pyplot as plt
stafllines_mask_image = numpy.zeros(gt.shape)
for i, (_sb, _sm) in enumerate(zip(staffline_bboxes, staffline_masks)):
t, l, b, r = _sb
stafllines_mask_image[t:b, l:r] = min(255, (i * 333) % 255 + 40)
plt.imshow(stafllines_mask_image, cmap='jet', interpolation='nearest')
plt.show()
raise ValueError('No. of stafflines is not divisible by 5!')
logging.info('Creating staff bboxes and masks.')
# - Go top-down and group the stafflines by five to get staves.
# (The staffline bboxes are already sorted top-down.)
staff_bboxes = []
staff_masks = []
for i in range(n_stafflines // 5):
_sbb = staffline_bboxes[5 * i:5 * (i + 1)]
_st = min([bb[0] for bb in _sbb])
_sl = min([bb[1] for bb in _sbb])
_sb = max([bb[2] for bb in _sbb])
_sr = max([bb[3] for bb in _sbb])
staff_bboxes.append((_st, _sl, _sb, _sr))
staff_masks.append(gt[_st:_sb, _sl:_sr])
logging.info('Total staffs: {0}'.format(len(staff_bboxes)))
##################################################################
# (Optionally fill in missing pixels, based on full image.)
logging.info('SKIP: fill in missing pixels based on full image.')
# - Load full image
# - Find gap regions
# - Obtain gap region masks from full image
# - Add gap region mask to staffline mask.
# Create the Nodes for stafflines and staffs:
# - Load corresponding annotation, to which the stafflines and
# staves should be added. (This is needed to correctly set docname
# and node_ids.)
if not args.annot:
nodes = []
next_node_id = 0
dataset = 'FCNOMR'
document = os.path.splitext(os.path.basename(args.staff_imfile))[0]
else:
if not os.path.isfile(args.annot):
raise ValueError('Annotation file {0} does not exist!'.format(
args.annot))
logging.info('Creating Nodes...')
nodes = read_nodes_from_file(args.annot)
logging.info('Non-staffline Nodes: {0}'.format(len(nodes)))
next_node_id = max([c.id for c in nodes]) + 1
dataset = nodes[0].dataset
document = nodes[0].document
# - Create the staffline Nodes
stafflines = []
for sl_bb, sl_m in zip(staffline_bboxes, staffline_masks):
t, l, b, r = sl_bb
c = Node(id_=next_node_id,
class_name=_CONST.STAFFLINE_CLASS_NAME,
top=t,
left=l,
height=b - t,
width=r - l,
mask=sl_m,
dataset=dataset,
document=document)
stafflines.append(c)
next_node_id += 1
if not args.stafflines_only:
# - Create the staff Nodes
staffs = []
for s_bb, s_m in zip(staff_bboxes, staff_masks):
t, l, b, r = s_bb
c = Node(id_=next_node_id,
class_name=_CONST.STAFF_CLASS_NAME,
top=t,
left=l,
height=b - t,
width=r - l,
mask=s_m,
dataset=dataset,
document=document)
staffs.append(c)
next_node_id += 1
# - Add the inlinks/outlinks
for i, sc in enumerate(staffs):
sl_from = 5 * i
sl_to = 5 * (i + 1)
for sl in stafflines[sl_from:sl_to]:
sl.inlinks.append(sc.id)
sc.outlinks.append(sl.id)
# Add the staffspaces.
staffspaces = []
for i, staff in enumerate(staffs):
current_stafflines = [
sc for sc in stafflines if sc.id in staff.outlinks
]
sorted_stafflines = sorted(current_stafflines, key=lambda x: x.top)
current_staffspaces = []
# Percussion single-line staves do not have staffspaces.
if len(sorted_stafflines) == 1:
continue
# Internal staffspace
for s1, s2 in zip(sorted_stafflines[:-1], sorted_stafflines[1:]):
# s1 is the UPPER staffline, s2 is the LOWER staffline
# Left and right limits: to simplify things, we take the column
# *intersection* of (s1, s2). This gives the invariant that
# the staffspace is limited from top and bottom in each of its columns.
l = max(s1.left, s2.left)
r = min(s1.right, s2.right)
# Shift s1, s2 to the right by this much to have the cols. align
# All of these are non-negative.
dl1, dl2 = l - s1.left, l - s2.left
dr1, dr2 = s1.right - r, s2.right - r
# The stafflines are not necessarily straight,
# so top is given for the *topmost bottom edge* of the top staffline + 1
# First create mask
canvas = numpy.zeros((s2.bottom - s1.top, r - l),
dtype='uint8')
# Paste masks into canvas.
# This assumes that the top of the bottom staffline is below
# the top of the top staffline... and that the bottom
# of the top staffline is above the bottom of the bottom
# staffline. This may not hold in very weird situations,
# but it's good for now.
logging.debug(s1.bounding_box, s1.mask.shape)
logging.debug(s2.bounding_box, s2.mask.shape)
logging.debug(canvas.shape)
logging.debug(
'l={0}, dl1={1}, dl2={2}, r={3}, dr1={4}, dr2={5}'
''.format(l, dl1, dl2, r, dr1, dr2))
# canvas[:s1.height, :] += s1.mask[:, dl1:s1.width-dr1]
# canvas[-s2.height:, :] += s2.mask[:, dl2:s2.width-dr2]
# We have to deal with staffline interruptions.
# One way to do this
# is watershed fill: put markers along the bottom and top
# edge, use mask * 10000 as elevation
s1_above, s1_below = staffline_surroundings_mask(s1)
s2_above, s2_below = staffline_surroundings_mask(s2)
# Get bounding boxes of the individual stafflines' masks
# that intersect with the staffspace bounding box, in terms
# of the staffline bounding box.
s1_t, s1_l, s1_b, s1_r = 0, dl1, \
s1.height, s1.width - dr1
s1_h, s1_w = s1_b - s1_t, s1_r - s1_l
s2_t, s2_l, s2_b, s2_r = canvas.shape[0] - s2.height, dl2, \
canvas.shape[0], s2.width - dr2
s2_h, s2_w = s2_b - s2_t, s2_r - s2_l
logging.debug(s1_t, s1_l, s1_b, s1_r, (s1_h, s1_w))
# We now take the intersection of s1_below and s2_above.
# If there is empty space in the middle, we fill it in.
staffspace_mask = numpy.ones(canvas.shape)
staffspace_mask[s1_t:s1_b, :] -= (
1 - s1_below[:, dl1:s1.width - dr1])
staffspace_mask[s2_t:s2_b, :] -= (
1 - s2_above[:, dl2:s2.width - dr2])
ss_top = s1.top
ss_bottom = s2.bottom
ss_left = l
ss_right = r
staffspace = Node(next_node_id,
_CONST.STAFFSPACE_CLASS_NAME,
top=ss_top,
left=ss_left,
height=ss_bottom - ss_top,
width=ss_right - ss_left,
mask=staffspace_mask,
dataset=dataset,
document=document)
staffspace.inlinks.append(staff.id)
staff.outlinks.append(staffspace.id)
current_staffspaces.append(staffspace)
next_node_id += 1
# Add top and bottom staffspace.
# These outer staffspaces will have the width
# of their bottom neighbor, and height derived
# from its mask columns.
# This is quite approximate, but it should do.
# Upper staffspace
tsl = sorted_stafflines[0]
tsl_heights = tsl.mask.sum(axis=0)
tss = current_staffspaces[0]
tss_heights = tss.mask.sum(axis=0)
uss_top = max(0, tss.top - max(tss_heights))
uss_left = tss.left
uss_width = tss.width
# We use 1.5, so that large noteheads
# do not "hang out" of the staffspace.
uss_height = int(tss.height / 1.2)
# Shift because of height downscaling:
uss_top += tss.height - uss_height
uss_mask = tss.mask[:uss_height, :] * 1
staffspace = Node(next_node_id,
_CONST.STAFFSPACE_CLASS_NAME,
top=uss_top,
left=uss_left,
height=uss_height,
width=uss_width,
mask=uss_mask,
dataset=dataset,
document=document)
current_staffspaces.append(staffspace)
staff.outlinks.append(staffspace.id)
staffspace.inlinks.append(staff.id)
next_node_id += 1
# Lower staffspace
bss = current_staffspaces[-1]
bss_heights = bss.mask.sum(axis=0)
bsl = sorted_stafflines[-1]
bsl_heights = bsl.mask.sum(axis=0)
lss_top = bss.bottom # + max(bsl_heights)
lss_left = bss.left
lss_width = bss.width
lss_height = int(bss.height / 1.2)
lss_mask = bss.mask[:lss_height, :] * 1
staffspace = Node(next_node_id,
_CONST.STAFFSPACE_CLASS_NAME,
top=lss_top,
left=lss_left,
height=lss_height,
width=lss_width,
mask=lss_mask,
dataset=dataset,
document=document)
current_staffspaces.append(staffspace)
staff.outlinks.append(staffspace.id)
staffspace.inlinks.append(staff.id)
next_node_id += 1
# ################ End of dealing with upper/lower staffspace ######
# Add to current list
staffspaces += current_staffspaces
# - Join the lists together
nodes_with_staffs = nodes \
+ stafflines \
+ staffspaces \
+ staffs
else:
nodes_with_staffs = nodes + stafflines
logging.info('Exporting the new Node list: {0} objects'
''.format(len(nodes_with_staffs)))
# - Export the combined list.
nodes_string = export_node_list(nodes_with_staffs)
if args.export is not None:
with open(args.export, 'w') as hdl:
hdl.write(nodes_string)
else:
print(nodes_string)
_end_time = time.clock()
logging.info('add_staffline_symbols.py done in {0:.3f} s'
''.format(_end_time - _start_time))
def test_overlaps(self):
# Arrange
node = Node(0, 'test', 10, 100, height=20, width=10)
# Act and Assert
self.assertEqual(node.bounding_box, (10, 100, 30, 110))
self.assertTrue(node.overlaps((10, 100, 30, 110))) # Exact match
self.assertFalse(node.overlaps((0, 100, 8, 110))) # Row mismatch
self.assertFalse(node.overlaps((10, 0, 30, 89))) # Column mismatch
self.assertFalse(node.overlaps((0, 0, 8, 89))) # Total mismatch
self.assertTrue(node.overlaps((9, 99, 31, 111))) # Encompasses Node
self.assertTrue(node.overlaps((11, 101, 29, 109))) # Within Node
self.assertTrue(node.overlaps((9, 101, 31, 109))) # Encompass horz., within vert.
self.assertTrue(node.overlaps((11, 99, 29, 111))) # Encompasses vert., within horz.
self.assertTrue(node.overlaps((11, 101, 31, 111))) # Corner within: top left
self.assertTrue(node.overlaps((11, 99, 31, 109))) # Corner within: top right
self.assertTrue(node.overlaps((9, 101, 29, 111))) # Corner within: bottom left
self.assertTrue(node.overlaps((9, 99, 29, 109))) # Corner within: bottom right
def read_nodes_from_file(filename: str) -> List[Node]:
"""From a xml file with a Nodes as the top element, parse
a list of nodes. (See ``Node`` class documentation
for a description of the XMl format.)
Let's test whether the parsing function works:
>>> test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)),
... 'test', 'test_data')
>>> file = os.path.join(test_data_dir, '01_basic.xml')
>>> nodes = read_nodes_from_file(file)
>>> len(nodes)
48
Let's also test the ``data`` attribute:
>>> file_with_data = os.path.join(test_data_dir, '01_basic_binary_2.0.xml')
>>> nodes = read_nodes_from_file(file_with_data)
>>> nodes[0].data['pitch_step']
'G'
>>> nodes[0].data['midi_pitch_code']
79
>>> nodes[0].data['precedence_outlinks']
[8, 17]
>>> nodes[0].dataset
'MUSCIMA-pp_2.0'
>>> nodes[0].document
'01_basic_binary'
:returns: A list of ``Node``s.
"""
if not os.path.exists(filename):
print("Could not find {0} on disk. Absolute path: {1}".format(filename, os.path.abspath(filename)))
return None
tree = etree.parse(filename)
root = tree.getroot()
logging.debug('XML parsed.')
nodes = []
dataset = root.attrib['dataset']
document = root.attrib['document']
for i, node in enumerate(root.iter('Node')):
######################################################
logging.debug('Parsing Node {0}'.format(i))
node_id = int(float(node.findall('Id')[0].text))
class_name = node.findall('ClassName')[0].text
top = int(node.findall('Top')[0].text)
left = int(node.findall('Left')[0].text)
width = int(node.findall('Width')[0].text)
height = int(node.findall('Height')[0].text)
#################################
# Parsing the graph structure (Can deal with missing Inlinks/Outlinks)
inlinks = []
i_s = node.findall('Inlinks')
if len(i_s) > 0:
i_s_text = node.findall('Inlinks')[0].text
if i_s_text is not None: # Zero-length links
inlinks = list(map(int, i_s_text.split(' ')))
outlinks = []
o_s = node.findall('Outlinks')
if len(o_s) > 0:
o_s_text = node.findall('Outlinks')[0].text
if o_s_text is not None:
outlinks = list(map(int, o_s_text.split(' ')))
#################################
data = node.findall('Data')
data_dict = None
if len(data) > 0:
data = data[0]
data_dict = {}
for data_item in data.findall('DataItem'):
key = data_item.get('key')
value_type = data_item.get('type')
value = data_item.text
# logging.debug('Creating data entry: key={0}, type={1},'
# ' value={2}'.format(key, value_type, value))
if value_type == 'int':
value = int(value)
elif value_type == 'float':
value = float(value)
elif value_type.startswith('list'):
if value is None:
value = []
else:
vt_factory = str
if value_type.endswith('[int]'):
vt_factory = int
elif value_type.endswith('[float]'):
vt_factory = float
value = list(map(vt_factory, value.split()))
data_dict[key] = value
#################################
# Create the object.
new_node = Node(id_=node_id,
class_name=class_name,
top=top,
left=left,
width=width,
height=height,
inlinks=inlinks,
outlinks=outlinks,
dataset=dataset,
document=document,
data=data_dict)
#################################
# Add mask.
# We do this only after the Node has been created,
# to make sure that the width & height used to reshape
# the flattened mask reflects what is in the Node.
mask = None
mask_elements = node.findall('Mask')
if len(mask_elements) > 0:
mask = Node.decode_mask(mask_elements[0].text, shape=(new_node.height, new_node.width))
new_node.set_mask(mask)
nodes.append(new_node)
logging.debug('Nodes loaded.')
if not validate_nodes_graph_structure(nodes):
raise ValueError('Invalid Node graph structure! Check warnings'
' in log for the individual errors.')
return nodes
def merge_staffline_segments(nodes: List[Node], margin: int = 10) -> List[Node]:
"""Given a list of Nodes that contain some staffline
objects, generates a new list where the stafflines
are merged based on their horizontal projections.
Basic step for going from the staffline detection masks to
the actual staffline objects.
Assumes that stafflines are straight: their bounding boxes
do not touch or overlap.
:return: A modified Node list: the original staffline-class
symbols are replaced by the merged ones. If the original stafflines
had any inlinks, they are preserved (mapped to the new staffline).
"""
already_processed_stafflines = [node for node in nodes
if (node.class_name == _CONST.STAFFLINE_CLASS_NAME) and
__has_parent_staff(node, nodes)]
# margin is used to avoid the stafflines touching the edges,
# which could perhaps break some assumptions down the line.
old_staffline_nodes = [c for c in nodes
if (c.class_name == _CONST.STAFFLINE_CLASS_NAME) and
not __has_parent_staff(c, nodes)]
if len(old_staffline_nodes) == 0:
logging.info('merge_staffline_segments: nothing new to do!')
return nodes
canvas, (_t, _l) = draw_nodes_on_empty_canvas(old_staffline_nodes)
_staffline_bboxes, staffline_masks = staffline_bboxes_and_masks_from_horizontal_merge(canvas)
# Bounding boxes need to be adjusted back with respect to the original image!
staffline_bboxes = [(t + _t, l + _l, b + _t, r + _l) for t, l, b, r in _staffline_bboxes]
# Create the Nodes.
next_node_id = max([c.id for c in nodes]) + 1
dataset = nodes[0].dataset
document = nodes[0].document
# - Create the staffline Nodes
staffline_nodes = []
for sl_bb, sl_m in zip(staffline_bboxes, staffline_masks):
t, l, b, r = sl_bb
c = Node(id_=next_node_id,
class_name=_CONST.STAFFLINE_CLASS_NAME,
top=t, left=l, height=b - t, width=r - l,
mask=sl_m,
dataset=dataset, document=document)
staffline_nodes.append(c)
next_node_id += 1
non_staffline_nodes = [c for c in nodes
if c.class_name != _CONST.STAFFLINE_CLASS_NAME]
old_staffline_ids = set([c.id for c in old_staffline_nodes])
old2new_staffline_id_map = {}
for os in old_staffline_nodes:
for ns in staffline_nodes:
if os.overlaps(ns):
old2new_staffline_id_map[os.id] = ns
logging.info('Re-linking from the old staffline objects to new ones.')
for c in non_staffline_nodes:
new_outlinks = []
for o in c.outlinks:
if o in old_staffline_ids:
new_staffline = old2new_staffline_id_map[o]
new_outlinks.append(new_staffline.id)
new_staffline.inlinks.append(c.id)
else:
new_outlinks.append(o)
output = non_staffline_nodes + staffline_nodes + already_processed_stafflines
return output
def build_staffspace_nodes(nodes: List[Node]) -> List[Node]:
"""Creates the staffspace objects based on stafflines
and staffs. There is a staffspace between each two stafflines,
one on the top side of each staff, and one on the bottom
side for each staff (corresponding e.g. to positions of g5 and d4
with the standard G-clef).
Note that staffspaces do not assume anything about the number
of stafflines per staff. However, single-staffline staffs will
not have the outer staffspaces generated (there is nothing to derive
their size from), for now.
:param nodes: A list of Nodes that must contain
all the relevant stafflines and staffs.
:return: A list of staffspace Nodes.
"""
next_node_id = max([c.id for c in nodes]) + 1
dataset = nodes[0].dataset
document = nodes[0].document
staff_nodes = [node for node in nodes
if node.class_name == _CONST.STAFF_CLASS_NAME
and not __has_child_staffspace(node, nodes)]
staffline_nodes = [node for node in nodes
if node.class_name == _CONST.STAFFLINE_CLASS_NAME
and not __has_neighbor_staffspace(node, nodes)]
staff_spaces = []
for i, staff in enumerate(staff_nodes):
current_stafflines = [sc for sc in staffline_nodes
if sc.id in staff.outlinks]
sorted_stafflines = sorted(current_stafflines, key=lambda x: x.top)
current_staffspace_nodes = []
# Percussion single-line staves do not have staffspaces.
if len(sorted_stafflines) == 1:
continue
#################
# Internal staffspace
for s1, s2 in zip(sorted_stafflines[:-1], sorted_stafflines[1:]):
# s1 is the UPPER staffline, s2 is the LOWER staffline
# Left and right limits: to simplify things, we take the column
# *intersection* of (s1, s2). This gives the invariant that
# the staffspace is limited from top and bottom in each of its columns.
l = max(s1.left, s2.left)
r = min(s1.right, s2.right)
# Shift s1, s2 to the right by this much to have the cols. align
# All of these are non-negative.
dl1, dl2 = l - s1.left, l - s2.left
dr1, dr2 = s1.right - r, s2.right - r
# The stafflines are not necessarily straight,
# so top is given for the *topmost bottom edge* of the top staffline + 1
# First create mask
canvas = numpy.zeros((s2.bottom - s1.top, r - l), dtype='uint8')
# Paste masks into canvas.
# This assumes that the top of the bottom staffline is below
# the top of the top staffline... and that the bottom
# of the top staffline is above the bottom of the bottom
# staffline. This may not hold in very weird situations,
# but it's good for now.
logging.debug(s1.bounding_box, s1.mask.shape)
logging.debug(s2.bounding_box, s2.mask.shape)
logging.debug(canvas.shape)
logging.debug('l={0}, dl1={1}, dl2={2}, r={3}, dr1={4}, dr2={5}'
''.format(l, dl1, dl2, r, dr1, dr2))
# canvas[:s1.height, :] += s1.mask[:, dl1:s1.width-dr1]
# canvas[-s2.height:, :] += s2.mask[:, dl2:s2.width-dr2]
# We have to deal with staffline interruptions.
# One way to do this
# is watershed fill: put markers along the bottom and top
# edge, use mask * 10000 as elevation
s1_above, s1_below = staffline_surroundings_mask(s1)
s2_above, s2_below = staffline_surroundings_mask(s2)
# Get bounding boxes of the individual stafflines' masks
# that intersect with the staffspace bounding box, in terms
# of the staffline bounding box.
s1_t, s1_l, s1_b, s1_r = 0, dl1, \
s1.height, s1.width - dr1
s1_h, s1_w = s1_b - s1_t, s1_r - s1_l
s2_t, s2_l, s2_b, s2_r = canvas.shape[0] - s2.height, dl2, \
canvas.shape[0], s2.width - dr2
s2_h, s2_w = s2_b - s2_t, s2_r - s2_l
logging.debug(s1_t, s1_l, s1_b, s1_r, (s1_h, s1_w))
# We now take the intersection of s1_below and s2_above.
# If there is empty space in the middle, we fill it in.
staffspace_mask = numpy.ones(canvas.shape)
staffspace_mask[s1_t:s1_b, :] -= (1 - s1_below[:, dl1:s1.width - dr1])
staffspace_mask[s2_t:s2_b, :] -= (1 - s2_above[:, dl2:s2.width - dr2])
ss_top = s1.top
ss_bottom = s2.bottom
ss_left = l
ss_right = r
staff_space = Node(next_node_id, _CONST.STAFFSPACE_CLASS_NAME,
top=ss_top, left=ss_left,
height=ss_bottom - ss_top,
width=ss_right - ss_left,
mask=staffspace_mask,
dataset=dataset, document=document)
staff_space.inlinks.append(staff.id)
staff.outlinks.append(staff_space.id)
current_staffspace_nodes.append(staff_space)
next_node_id += 1
##########
# Add top and bottom staffspace.
# These outer staffspaces will have the width
# of their bottom neighbor, and height derived
# from its mask columns.
# This is quite approximate, but it should do.
# Upper staffspace
tsl = sorted_stafflines[0]
tsl_heights = tsl.mask.sum(axis=0)
tss = current_staffspace_nodes[0]
tss_heights = tss.mask.sum(axis=0)
uss_top = max(0, tss.top - max(tss_heights))
uss_left = tss.left
uss_width = tss.width
# We use 1.5, so that large noteheads
# do not "hang out" of the staffspace.
uss_height = int(tss.height / 1.2)
# Shift because of height downscaling:
uss_top += tss.height - uss_height
uss_mask = tss.mask[:uss_height, :] * 1
staff_space = Node(next_node_id, _CONST.STAFFSPACE_CLASS_NAME,
top=uss_top, left=uss_left,
height=uss_height,
width=uss_width,
mask=uss_mask,
dataset=dataset, document=document)
current_staffspace_nodes.append(staff_space)
staff.outlinks.append(staff_space.id)
staff_space.inlinks.append(staff.id)
next_node_id += 1
# Lower staffspace
bss = current_staffspace_nodes[-2]
bss_heights = bss.mask.sum(axis=0)
bsl = sorted_stafflines[-1]
bsl_heights = bsl.mask.sum(axis=0)
lss_top = bss.bottom # + max(bsl_heights)
lss_left = bss.left
lss_width = bss.width
lss_height = int(bss.height / 1.2)
lss_mask = bss.mask[:lss_height, :] * 1
staff_space = Node(next_node_id, _CONST.STAFFSPACE_CLASS_NAME,
top=lss_top, left=lss_left,
height=lss_height,
width=lss_width,
mask=lss_mask,
dataset=dataset, document=document)
current_staffspace_nodes.append(staff_space)
staff.outlinks.append(staff_space.id)
staff_space.inlinks.append(staff.id)
next_node_id += 1
staff_spaces += current_staffspace_nodes
return staff_spaces
def process_compound_nodes(self, nodes: List[Node]) -> List[Node]:
print("Processing compound objects ...", flush=True)
with open(
os.path.join(
self.path_of_this_file,
"MuscimaPlusPlusClassesThatNeedComposition.json")) as file:
classes_that_need_composition = json.load(file)
nodes_for_composition = [
node for node in nodes
if node.class_name in classes_that_need_composition
]
final_nodes = []
quarter_notes = []
half_notes = []
eighth_notes = []
sixteenth_notes = []
node_dict = {c.unique_id: c for c in nodes} # type: Dict[str, Node]
for c in nodes_for_composition:
if c.class_name not in ['noteheadFull', 'noteheadHalf']:
continue
has_stem = False
has_beam = False
has_flag = False
stem_object = None
flag_objects = []
for o in c.outlinks:
uid_of_outlink = Node.UID_DELIMITER.join(
[c.dataset, c.document, str(o)])
if uid_of_outlink not in node_dict:
continue # The targeted object has been filtered by broken-list or ignored classes
outgoing_object = node_dict[uid_of_outlink]
if outgoing_object.class_name == 'stem':
has_stem = True
stem_object = outgoing_object
elif outgoing_object.class_name == 'beam':
has_beam = True
elif outgoing_object.class_name.startswith('flag'):
has_flag = True
flag_objects.append(outgoing_object)
if not has_stem:
continue
if has_beam:
pass
elif has_flag:
if len(flag_objects) == 1:
eighth_notes.append((c, stem_object, flag_objects))
elif len(flag_objects) == 2:
sixteenth_notes.append((c, stem_object, flag_objects))
else:
if c.class_name == 'noteheadFull':
quarter_notes.append((c, stem_object))
else:
half_notes.append((c, stem_object))
for half_note in half_notes:
note_head, stem = half_note
note_head.join(stem)
final_nodes.append(
Node(note_head.id, "Half-Note", note_head.top, note_head.left,
note_head.width, note_head.height, note_head.outlinks,
note_head.inlinks, note_head.mask, note_head.dataset,
note_head.document, note_head.data))
for quarter_note in quarter_notes:
note_head, stem = quarter_note
note_head.join(stem)
final_nodes.append(
Node(note_head.id, "Quarter-Note", note_head.top,
note_head.left, note_head.width, note_head.height,
note_head.outlinks, note_head.inlinks, note_head.mask,
note_head.dataset, note_head.document, note_head.data))
for eighth_note in eighth_notes:
note_head, stem, flags = eighth_note
note_head.join(stem)
for flag in flags:
note_head.join(flag)
final_nodes.append(
Node(note_head.id, "Eighth-Note", note_head.top,
note_head.left, note_head.width, note_head.height,
note_head.outlinks, note_head.inlinks, note_head.mask,
note_head.dataset, note_head.document, note_head.data))
for sixteenth_note in sixteenth_notes:
note_head, stem, flags = sixteenth_note
note_head.join(stem)
for flag in flags:
note_head.join(flag)
final_nodes.append(
Node(note_head.id, "Sixteenth-Note", note_head.top,
note_head.left, note_head.width, note_head.height,
note_head.outlinks, note_head.inlinks, note_head.mask,
note_head.dataset, note_head.document, note_head.data))
return final_nodes
