def test_convolve_with_concat(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 2
for i in result:
assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
def test_emulate_teuthology_noceph(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('teuthology/no-ceph')
assert len(result) == 11
assert self.fragment_occurences(result, 'vps.yaml') == 1 / 11.0
def test_emulate_teuthology_noceph(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('teuthology/no-ceph')
assert len(result) == 11
assert self.fragment_occurences(result, 'vps.yaml') == 1 / 11.0
def test_convolve_with_concat(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 2
for i in result:
assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
def test_random_dollar_sign_with_concat(self):
fake_fs = {
'd0_0': {
'$': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 1
if result[0][0][1:].startswith('d1_2'):
for i in result:
assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
def test_random_dollar_sign_with_convolve(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'$': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 4
fake_fs1 = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2$': {
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
self.stop_patchers()
self.start_patchers(fake_fs1)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 4
def test_random_dollar_sign_with_convolve(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'$': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 4
fake_fs1 = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2$': {
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
self.stop_patchers()
self.start_patchers(fake_fs1)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 4
def test_random_dollar_sign_with_concat(self):
fake_fs = {
'd0_0': {
'$': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
fake_fs1 = {
'd0_0$': {
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
for info in [(fake_fs, 'd0_0'), (fake_fs1, 'd0_0$')]:
fsv = info[0]
dval = info[1]
self.start_patchers(fsv)
result = build_matrix.build_matrix(dval)
assert len(result) == 1
if result[0][0][1:].startswith('d1_2'):
for i in result:
assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
if dval == 'd0_0':
self.stop_patchers()
def test_random_dollar_sign_with_concat(self):
fake_fs = {
'd0_0': {
'$': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
fake_fs1 = {
'd0_0$': {
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
for info in [(fake_fs,'d0_0'), (fake_fs1,'d0_0$')]:
fsv = info[0]
dval = info[1]
self.start_patchers(fsv)
result = build_matrix.build_matrix(dval)
assert len(result) == 1
if result[0][0][1:].startswith('d1_2'):
for i in result:
assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
if dval == 'd0_0':
self.stop_patchers()
def test_random_dollar_sign_with_concat(self):
fake_fs = {
'd0_0': {
'$': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
fake_fs1 = {
'd0_0$': {
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
for fs, root in [(fake_fs,'d0_0'), (fake_fs1,'d0_0$')]:
self.start_patchers(fs)
result = build_matrix.build_matrix(root)
assert len(result) == 1
if result[0][0][1:].startswith('d1_2'):
for i in result:
assert os.path.join(root, 'd1_2/d1_2_0.yaml') in i[1]
assert os.path.join(root, 'd1_2/d1_2_1.yaml') in i[1]
assert os.path.join(root, 'd1_2/d1_2_2.yaml') in i[1]
assert os.path.join(root, 'd1_2/d1_2_3.yaml') in i[1]
if root == 'd0_0':
self.stop_patchers()
def test_random_dollar_sign_with_concat(self):
fake_fs = {
'd0_0': {
'$': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
fake_fs1 = {
'd0_0$': {
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
for fs, root in [(fake_fs,'d0_0'), (fake_fs1,'d0_0$')]:
self.start_patchers(fs)
result = build_matrix.build_matrix(root)
assert len(result) == 1
if result[0][0][1:].startswith('d1_2'):
for i in result:
assert os.path.join(root, 'd1_2/d1_2_0.yaml') in i[1]
assert os.path.join(root, 'd1_2/d1_2_1.yaml') in i[1]
assert os.path.join(root, 'd1_2/d1_2_2.yaml') in i[1]
assert os.path.join(root, 'd1_2/d1_2_3.yaml') in i[1]
if root == 'd0_0':
self.stop_patchers()
def test_convolve_2x2(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 4
assert self.fragment_occurences(result, 'd1_1_1.yaml') == 0.5
def test_convolve_2x2(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 4
assert self.fragment_occurences(result, 'd1_1_1.yaml') == 0.5
def test_concatenate_1x2x3(self):
fake_fs = {
'd0_0': {
'+': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 1
def test_concatenate_1x2x3(self):
fake_fs = {
'd0_0': {
'+': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 1
def test_sort_order(self):
# This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
# fragments are sorted.
fake_fs = {
'thrash': {
'%': None,
'ceph-thrash': {
'default.yaml': None
},
'ceph': {
'base.yaml': None
},
'clusters': {
'mds-1active-1standby.yaml': None
},
'debug': {
'mds_client.yaml': None
},
'fs': {
'btrfs.yaml': None
},
'msgr-failures': {
'none.yaml': None
},
'overrides': {
'whitelist_wrongly_marked_down.yaml': None
},
'tasks': {
'cfuse_workunit_suites_fsstress.yaml': None
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('thrash')
assert len(result) == 1
assert self.fragment_occurences(result, 'base.yaml') == 1
fragments = result[0][1]
assert fragments[0] == 'thrash/ceph/base.yaml'
assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
def test_random_dollar_sign_2x2x3(self):
fake_fs = {
'd0_0': {
'$': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('d0_0')
assert len(result) == 1
def test_sort_order(self):
# This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
# fragments are sorted.
fake_fs = {
'thrash': {
'%': None,
'ceph-thrash': {'default.yaml': None},
'ceph': {'base.yaml': None},
'clusters': {'mds-1active-1standby.yaml': None},
'debug': {'mds_client.yaml': None},
'fs': {'btrfs.yaml': None},
'msgr-failures': {'none.yaml': None},
'overrides': {'whitelist_wrongly_marked_down.yaml': None},
'tasks': {'cfuse_workunit_suites_fsstress.yaml': None},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('thrash')
assert len(result) == 1
assert self.fragment_occurences(result, 'base.yaml') == 1
fragments = result[0][1]
assert fragments[0] == 'thrash/ceph/base.yaml'
assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
def test_disable_extension(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('teuthology/no-ceph')
self.stop_patchers()
fake_fs2 = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
'forcefilevps_ubuntu14.04.yaml.disable': None,
'forcefilevps_ubuntu14.04.yaml.anotherextension': None,
},
'tasks': {
'teuthology.yaml': None,
'forcefilevps_ubuntu14.04notyaml': None,
},
'forcefilevps_ubuntu14.04notyaml': None,
'tasks.disable': {
'teuthology2.yaml': None,
'forcefilevps_ubuntu14.04notyaml': None,
},
},
},
}
self.start_patchers(fake_fs2)
result2 = build_matrix.build_matrix('teuthology/no-ceph')
assert len(result) == 11
assert len(result2) == len(result)
def test_empty_dirs(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('teuthology/no-ceph')
self.stop_patchers()
fake_fs2 = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'empty': {},
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
'empty': {},
},
},
}
self.start_patchers(fake_fs2)
result2 = build_matrix.build_matrix('teuthology/no-ceph')
assert len(result) == 11
assert len(result2) == len(result)
def test_empty_dirs(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('teuthology/no-ceph')
self.stop_patchers()
fake_fs2 = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'empty': {},
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
'empty': {},
},
},
}
self.start_patchers(fake_fs2)
result2 = build_matrix.build_matrix('teuthology/no-ceph')
assert len(result) == 11
assert len(result2) == len(result)
def test_disable_extension(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
self.start_patchers(fake_fs)
result = build_matrix.build_matrix('teuthology/no-ceph')
self.stop_patchers()
fake_fs2 = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
'forcefilevps_ubuntu14.04.yaml.disable': None,
'forcefilevps_ubuntu14.04.yaml.anotherextension': None,
},
'tasks': {
'teuthology.yaml': None,
'forcefilevps_ubuntu14.04notyaml': None,
},
'forcefilevps_ubuntu14.04notyaml': None,
'tasks.disable': {
'teuthology2.yaml': None,
'forcefilevps_ubuntu14.04notyaml': None,
},
},
},
}
self.start_patchers(fake_fs2)
result2 = build_matrix.build_matrix('teuthology/no-ceph')
assert len(result) == 11
assert len(result2) == len(result)
def get_combinations(suite_dir, fields, subset, limit, filter_in, filter_out,
include_facet):
"""
Describes the combinations of a suite, optionally limiting
or filtering output based on the given parameters. Includes
columns for the subsuite and facets when include_facet is True.
Returns a tuple of (headers, rows) where both elements are lists
of strings.
"""
configs = [(combine_path(suite_dir, item[0]), item[1])
for item in build_matrix(suite_dir, subset)]
num_listed = 0
rows = []
facet_headers = set()
dirs = {}
max_dir_depth = 0
for _, fragment_paths in configs:
if limit > 0 and num_listed >= limit:
break
if filter_in and not any(
[f in path for f in filter_in for path in fragment_paths]):
continue
if filter_out and any(
[f in path for f in filter_out for path in fragment_paths]):
continue
fragment_fields = [
extract_info(path, fields) for path in fragment_paths
]
# merge fields from multiple fragments by joining their values with \n
metadata = {}
for fragment_meta in fragment_fields:
for field, value in fragment_meta.items():
if value == '':
continue
if field in metadata:
metadata[field] += '\n' + str(value)
else:
metadata[field] = str(value)
if include_facet:
# map final dir (facet) -> filename without the .yaml suffix
for path in fragment_paths:
facet_dir = os.path.dirname(path)
facet = os.path.basename(facet_dir)
metadata[facet] = os.path.basename(path)[:-5]
facet_headers.add(facet)
facet_dirs = facet_dir.split('/')[:-1]
for i, dir_ in enumerate(facet_dirs):
if i not in dirs:
dirs[i] = set()
dirs[i].add(dir_)
metadata['_dir_' + str(i)] = os.path.basename(dir_)
max_dir_depth = max(max_dir_depth, i)
rows.append(metadata)
num_listed += 1
subsuite_headers = []
if include_facet:
first_subsuite_depth = max_dir_depth
for i in range(max_dir_depth):
if len(dirs[i]) > 1:
first_subsuite_depth = i
break
subsuite_headers = [
'subsuite depth ' + str(i)
for i in range(0, max_dir_depth - first_subsuite_depth + 1)
]
for row in rows:
for i in range(first_subsuite_depth, max_dir_depth + 1):
row[subsuite_headers[i - first_subsuite_depth]] = \
row.get('_dir_' + str(i), '')
headers = subsuite_headers + sorted(facet_headers) + fields
return headers, sorted([[row.get(field, '') for field in headers]
for row in rows])
def schedule_suite(self):
"""
Schedule the suite-run. Returns the number of jobs scheduled.
"""
name = self.name
if self.args.arch:
arch = self.args.arch
log.debug("Using '%s' as an arch" % arch)
else:
arch = util.get_arch(self.base_config.machine_type)
suite_name = self.base_config.suite
suite_path = os.path.normpath(
os.path.join(
self.suite_repo_path,
self.args.suite_relpath,
'suites',
self.base_config.suite.replace(':', '/'),
))
log.debug('Suite %s in %s' % (suite_name, suite_path))
configs = build_matrix(suite_path,
subset=self.args.subset,
seed=self.args.seed)
log.info('Suite %s in %s generated %d jobs (not yet filtered)' %
(suite_name, suite_path, len(configs)))
if self.args.dry_run:
log.debug("Base job config:\n%s" % self.base_config)
# create, but do not write, the temp file here, so it can be
# added to the args in collect_jobs, but not filled until
# any backtracking is done
base_yaml_path = NamedTemporaryFile(prefix='schedule_suite_',
delete=False).name
self.base_yaml_paths.insert(0, base_yaml_path)
# compute job limit in respect of --sleep-before-teardown
job_limit = self.args.limit or 0
sleep_before_teardown = int(self.args.sleep_before_teardown or 0)
if sleep_before_teardown:
if job_limit == 0:
log.warning('The --sleep-before-teardown option was provided: '
'only 1 job will be scheduled. '
'Use --limit to run more jobs')
# give user a moment to read this warning
time.sleep(5)
job_limit = 1
elif self.args.non_interactive:
log.warning('The --sleep-before-teardown option is active. '
'There will be a maximum {} jobs running '
'which will fall asleep for {}'.format(
job_limit,
format_timespan(sleep_before_teardown)))
elif job_limit > 4:
are_you_insane = (
'There are {total} configs and {maximum} job limit is used. '
'Do you really want to lock all machines needed for '
'this run for {that_long}? (y/N):'.format(
that_long=format_timespan(sleep_before_teardown),
total=len(configs),
maximum=job_limit))
while True:
insane = (input(are_you_insane) or 'n').lower()
if insane == 'y':
break
elif insane == 'n':
exit(0)
# if newest, do this until there are no missing packages
# if not, do it once
backtrack = 0
limit = self.args.newest
while backtrack <= limit:
jobs_missing_packages, jobs_to_schedule = \
self.collect_jobs(arch,
util.filter_configs(configs,
filter_in=self.args.filter_in,
filter_out=self.args.filter_out,
filter_all=self.args.filter_all,
filter_fragments=self.args.filter_fragments,
suite_name=suite_name),
self.args.newest, job_limit)
if jobs_missing_packages and self.args.newest:
new_sha1 = \
util.find_git_parent('ceph', self.base_config.sha1)
if new_sha1 is None:
util.schedule_fail('Backtrack for --newest failed', name)
# rebuild the base config to resubstitute sha1
self.config_input['ceph_hash'] = new_sha1
self.base_config = self.build_base_config()
backtrack += 1
continue
if backtrack:
log.info("--newest supplied, backtracked %d commits to %s" %
(backtrack, self.base_config.sha1))
break
else:
if self.args.newest:
util.schedule_fail(
'Exceeded %d backtracks; raise --newest value' % limit,
name,
)
if self.args.dry_run:
log.debug("Base job config:\n%s" % self.base_config)
with open(base_yaml_path, 'w+b') as base_yaml:
base_yaml.write(str(self.base_config).encode())
if jobs_to_schedule:
self.write_rerun_memo()
# Before scheduling jobs, check the priority
if self.args.priority and jobs_to_schedule and not self.args.force_priority:
self.check_priority(len(jobs_to_schedule))
self.check_num_jobs(len(jobs_to_schedule))
self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name)
os.remove(base_yaml_path)
count = len(jobs_to_schedule)
missing_count = len(jobs_missing_packages)
log.info('Suite %s in %s scheduled %d jobs.' %
(suite_name, suite_path, count))
log.info('%d/%d jobs were filtered out.', (len(configs) - count),
len(configs))
if missing_count:
log.warning('Scheduled %d/%d jobs that are missing packages!',
missing_count, count)
return count
def schedule_suite(self):
"""
Schedule the suite-run. Returns the number of jobs scheduled.
"""
name = self.name
arch = util.get_arch(self.base_config.machine_type)
suite_name = self.base_config.suite
suite_path = os.path.normpath(os.path.join(
self.suite_repo_path,
self.args.suite_relpath,
'suites',
self.base_config.suite.replace(':', '/'),
))
log.debug('Suite %s in %s' % (suite_name, suite_path))
configs = [
(combine_path(suite_name, item[0]), item[1]) for item in
build_matrix(suite_path, subset=self.args.subset, seed=self.args.seed)
]
log.info('Suite %s in %s generated %d jobs (not yet filtered)' % (
suite_name, suite_path, len(configs)))
if self.args.dry_run:
log.debug("Base job config:\n%s" % self.base_config)
# create, but do not write, the temp file here, so it can be
# added to the args in collect_jobs, but not filled until
# any backtracking is done
base_yaml_path = NamedTemporaryFile(
prefix='schedule_suite_', delete=False
).name
self.base_yaml_paths.insert(0, base_yaml_path)
# if newest, do this until there are no missing packages
# if not, do it once
backtrack = 0
limit = self.args.newest
while backtrack <= limit:
jobs_missing_packages, jobs_to_schedule = \
self.collect_jobs(arch, configs, self.args.newest)
if jobs_missing_packages and self.args.newest:
new_sha1 = \
util.find_git_parent('ceph', self.base_config.sha1)
if new_sha1 is None:
util.schedule_fail('Backtrack for --newest failed', name)
# rebuild the base config to resubstitute sha1
self.config_input['ceph_hash'] = new_sha1
self.base_config = self.build_base_config()
backtrack += 1
continue
if backtrack:
log.info("--newest supplied, backtracked %d commits to %s" %
(backtrack, self.base_config.sha1))
break
else:
if self.args.newest:
util.schedule_fail(
'Exceeded %d backtracks; raise --newest value' % limit,
name,
)
if self.args.dry_run:
log.debug("Base job config:\n%s" % self.base_config)
with open(base_yaml_path, 'w+b') as base_yaml:
base_yaml.write(str(self.base_config))
if jobs_to_schedule:
self.write_rerun_memo()
self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name)
os.remove(base_yaml_path)
count = len(jobs_to_schedule)
missing_count = len(jobs_missing_packages)
log.info(
'Suite %s in %s scheduled %d jobs.' %
(suite_name, suite_path, count)
)
log.info('%d/%d jobs were filtered out.',
(len(configs) - count),
len(configs))
if missing_count:
log.warn('Scheduled %d/%d jobs that are missing packages!',
missing_count, count)
return count
def get_combinations(suite_dir, fields, subset,
limit, filter_in, filter_out,
include_facet):
"""
Describes the combinations of a suite, optionally limiting
or filtering output based on the given parameters. Includes
columns for the subsuite and facets when include_facet is True.
Returns a tuple of (headers, rows) where both elements are lists
of strings.
"""
configs = [(combine_path(suite_dir, item[0]), item[1]) for item in
build_matrix(suite_dir, subset)]
num_listed = 0
rows = []
facet_headers = set()
dirs = {}
max_dir_depth = 0
for _, fragment_paths in configs:
if limit > 0 and num_listed >= limit:
break
if filter_in and not any([f in path for f in filter_in
for path in fragment_paths]):
continue
if filter_out and any([f in path for f in filter_out
for path in fragment_paths]):
continue
fragment_fields = [extract_info(path, fields)
for path in fragment_paths]
# merge fields from multiple fragments by joining their values with \n
metadata = {}
for fragment_meta in fragment_fields:
for field, value in fragment_meta.items():
if value == '':
continue
if field in metadata:
metadata[field] += '\n' + str(value)
else:
metadata[field] = str(value)
if include_facet:
# map final dir (facet) -> filename without the .yaml suffix
for path in fragment_paths:
facet_dir = os.path.dirname(path)
facet = os.path.basename(facet_dir)
metadata[facet] = os.path.basename(path)[:-5]
facet_headers.add(facet)
facet_dirs = facet_dir.split('/')[:-1]
for i, dir_ in enumerate(facet_dirs):
if i not in dirs:
dirs[i] = set()
dirs[i].add(dir_)
metadata['_dir_' + str(i)] = os.path.basename(dir_)
max_dir_depth = max(max_dir_depth, i)
rows.append(metadata)
num_listed += 1
subsuite_headers = []
if include_facet:
first_subsuite_depth = max_dir_depth
for i in range(max_dir_depth):
if len(dirs[i]) > 1:
first_subsuite_depth = i
break
subsuite_headers = ['subsuite depth ' + str(i)
for i in
range(0, max_dir_depth - first_subsuite_depth + 1)]
for row in rows:
for i in range(first_subsuite_depth, max_dir_depth + 1):
row[subsuite_headers[i - first_subsuite_depth]] = \
row.get('_dir_' + str(i), '')
headers = subsuite_headers + sorted(facet_headers) + fields
return headers, sorted([[row.get(field, '') for field in headers]
for row in rows])
