Пример #1
0
 def test_disable_extension(self):
     fake_fs = {
         "teuthology": {
             "no-ceph": {
                 "%": None,
                 "clusters": {"single.yaml": None},
                 "distros": {
                     "baremetal.yaml": None,
                     "rhel7.0.yaml": None,
                     "ubuntu12.04.yaml": None,
                     "ubuntu14.04.yaml": None,
                     "vps.yaml": None,
                     "vps_centos6.5.yaml": None,
                     "vps_debian7.yaml": None,
                     "vps_rhel6.4.yaml": None,
                     "vps_rhel6.5.yaml": None,
                     "vps_rhel7.0.yaml": None,
                     "vps_ubuntu14.04.yaml": None,
                 },
                 "tasks": {"teuthology.yaml": None},
             }
         }
     }
     fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
     result = suite.build_matrix("teuthology/no-ceph", fake_isfile, fake_isdir, fake_listdir)
     fake_fs2 = {
         "teuthology": {
             "no-ceph": {
                 "%": None,
                 "clusters": {"single.yaml": None},
                 "distros": {
                     "baremetal.yaml": None,
                     "rhel7.0.yaml": None,
                     "ubuntu12.04.yaml": None,
                     "ubuntu14.04.yaml": None,
                     "vps.yaml": None,
                     "vps_centos6.5.yaml": None,
                     "vps_debian7.yaml": None,
                     "vps_rhel6.4.yaml": None,
                     "vps_rhel6.5.yaml": None,
                     "vps_rhel7.0.yaml": None,
                     "vps_ubuntu14.04.yaml": None,
                     "forcefilevps_ubuntu14.04.yaml.disable": None,
                     "forcefilevps_ubuntu14.04.yaml.anotherextension": None,
                 },
                 "tasks": {"teuthology.yaml": None, "forcefilevps_ubuntu14.04notyaml": None},
                 "forcefilevps_ubuntu14.04notyaml": None,
                 "tasks.disable": {"teuthology2.yaml": None, "forcefilevps_ubuntu14.04notyaml": None},
             }
         }
     }
     fake_listdir2, fake_isfile2, fake_isdir2 = make_fake_fstools(fake_fs2)
     result2 = suite.build_matrix("teuthology/no-ceph", fake_isfile2, fake_isdir2, fake_listdir2)
     assert len(result) == 11
     assert len(result2) == len(result)
Пример #2
0
 def test_emulate_teuthology_noceph(self):
     fake_fs = {
         'teuthology': {
             'no-ceph': {
                 '%': None,
                 'clusters': {
                     'single.yaml': None,
                 },
                 'distros': {
                     'baremetal.yaml': None,
                     'rhel7.0.yaml': None,
                     'ubuntu12.04.yaml': None,
                     'ubuntu14.04.yaml': None,
                     'vps.yaml': None,
                     'vps_centos6.5.yaml': None,
                     'vps_debian7.yaml': None,
                     'vps_rhel6.4.yaml': None,
                     'vps_rhel6.5.yaml': None,
                     'vps_rhel7.0.yaml': None,
                     'vps_ubuntu14.04.yaml': None,
                 },
                 'tasks': {
                     'teuthology.yaml': None,
                 },
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
                                 fake_isdir, fake_listdir)
     assert len(result) == 11
     assert self.fragment_occurences(result, 'vps.yaml') == 1 / 11.0
Пример #3
0
 def test_convolve_with_concat(self):
     fake_fs = {
         'd0_0': {
             '%': None,
             'd1_0': {
                 'd1_0_0.yaml': None,
             },
             'd1_1': {
                 'd1_1_0.yaml': None,
                 'd1_1_1.yaml': None,
             },
             'd1_2': {
                 '+': None,
                 'd1_2_0.yaml': None,
                 'd1_2_1.yaml': None,
                 'd1_2_2.yaml': None,
                 'd1_2_3.yaml': None,
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
                                 fake_listdir)
     assert len(result) == 2
     for i in result:
         assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
         assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
         assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
         assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
Пример #4
0
 def test_emulate_teuthology_noceph(self):
     fake_fs = {
         'teuthology': {
             'no-ceph': {
                 '%': None,
                 'clusters': {
                     'single.yaml': None,
                 },
                 'distros': {
                     'baremetal.yaml': None,
                     'rhel7.0.yaml': None,
                     'ubuntu12.04.yaml': None,
                     'ubuntu14.04.yaml': None,
                     'vps.yaml': None,
                     'vps_centos6.5.yaml': None,
                     'vps_debian7.yaml': None,
                     'vps_rhel6.4.yaml': None,
                     'vps_rhel6.5.yaml': None,
                     'vps_rhel7.0.yaml': None,
                     'vps_ubuntu14.04.yaml': None,
                 },
                 'tasks': {
                     'teuthology.yaml': None,
                 },
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
                                 fake_isdir, fake_listdir)
     assert len(result) == 11
     assert self.fragment_occurences(result, 'vps.yaml') == 1 / 11.0
Пример #5
0
 def test_emulate_teuthology_noceph(self):
     fake_fs = {
         "teuthology": {
             "no-ceph": {
                 "%": None,
                 "clusters": {"single.yaml": None},
                 "distros": {
                     "baremetal.yaml": None,
                     "rhel7.0.yaml": None,
                     "ubuntu12.04.yaml": None,
                     "ubuntu14.04.yaml": None,
                     "vps.yaml": None,
                     "vps_centos6.5.yaml": None,
                     "vps_debian7.yaml": None,
                     "vps_rhel6.4.yaml": None,
                     "vps_rhel6.5.yaml": None,
                     "vps_rhel7.0.yaml": None,
                     "vps_ubuntu14.04.yaml": None,
                 },
                 "tasks": {"teuthology.yaml": None},
             }
         }
     }
     fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
     result = suite.build_matrix("teuthology/no-ceph", fake_isfile, fake_isdir, fake_listdir)
     assert len(result) == 11
     assert self.fragment_occurences(result, "vps.yaml") == 1 / 11.0
Пример #6
0
 def test_convolve_with_concat(self):
     fake_fs = {
         'd0_0': {
             '%': None,
             'd1_0': {
                 'd1_0_0.yaml': None,
             },
             'd1_1': {
                 'd1_1_0.yaml': None,
                 'd1_1_1.yaml': None,
             },
             'd1_2': {
                 '+': None,
                 'd1_2_0.yaml': None,
                 'd1_2_1.yaml': None,
                 'd1_2_2.yaml': None,
                 'd1_2_3.yaml': None,
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
                                 fake_listdir)
     assert len(result) == 2
     for i in result:
         assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
         assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
         assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
         assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
Пример #7
0
 def test_convolve_2x2(self):
     fake_fs = {
         "d0_0": {
             "%": None,
             "d1_0": {"d1_0_0.yaml": None, "d1_0_1.yaml": None},
             "d1_1": {"d1_1_0.yaml": None, "d1_1_1.yaml": None},
         }
     }
     fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
     result = suite.build_matrix("d0_0", fake_isfile, fake_isdir, fake_listdir)
     assert len(result) == 4
     assert self.fragment_occurences(result, "d1_1_1.yaml") == 0.5
Пример #8
0
 def test_concatenate_1x2x3(self):
     fake_fs = {
         "d0_0": {
             "+": None,
             "d1_0": {"d1_0_0.yaml": None},
             "d1_1": {"d1_1_0.yaml": None, "d1_1_1.yaml": None},
             "d1_2": {"d1_2_0.yaml": None, "d1_2_1.yaml": None, "d1_2_2.yaml": None},
         }
     }
     fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
     result = suite.build_matrix("d0_0", fake_isfile, fake_isdir, fake_listdir)
     assert len(result) == 1
Пример #9
0
 def test_convolve_with_concat(self):
     fake_fs = {
         "d0_0": {
             "%": None,
             "d1_0": {"d1_0_0.yaml": None},
             "d1_1": {"d1_1_0.yaml": None, "d1_1_1.yaml": None},
             "d1_2": {"+": None, "d1_2_0.yaml": None, "d1_2_1.yaml": None, "d1_2_2.yaml": None, "d1_2_3.yaml": None},
         }
     }
     fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
     result = suite.build_matrix("d0_0", fake_isfile, fake_isdir, fake_listdir)
     assert len(result) == 2
     for i in result:
         assert "d0_0/d1_2/d1_2_0.yaml" in i[1]
         assert "d0_0/d1_2/d1_2_1.yaml" in i[1]
         assert "d0_0/d1_2/d1_2_2.yaml" in i[1]
         assert "d0_0/d1_2/d1_2_3.yaml" in i[1]
Пример #10
0
 def test_convolve_2x2(self):
     fake_fs = {
         'd0_0': {
             '%': None,
             'd1_0': {
                 'd1_0_0.yaml': None,
                 'd1_0_1.yaml': None,
             },
             'd1_1': {
                 'd1_1_0.yaml': None,
                 'd1_1_1.yaml': None,
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
                                 fake_listdir)
     assert len(result) == 4
     assert self.fragment_occurences(result, 'd1_1_1.yaml') == 0.5
Пример #11
0
 def test_convolve_2x2(self):
     fake_fs = {
         'd0_0': {
             '%': None,
             'd1_0': {
                 'd1_0_0.yaml': None,
                 'd1_0_1.yaml': None,
             },
             'd1_1': {
                 'd1_1_0.yaml': None,
                 'd1_1_1.yaml': None,
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
                                 fake_listdir)
     assert len(result) == 4
     assert self.fragment_occurences(result, 'd1_1_1.yaml') == 0.5
Пример #12
0
 def test_sort_order(self):
     # This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
     # fragments are sorted.
     fake_fs = {
         'thrash': {
             '%': None,
             'ceph-thrash': {
                 'default.yaml': None
             },
             'ceph': {
                 'base.yaml': None
             },
             'clusters': {
                 'mds-1active-1standby.yaml': None
             },
             'debug': {
                 'mds_client.yaml': None
             },
             'fs': {
                 'btrfs.yaml': None
             },
             'msgr-failures': {
                 'none.yaml': None
             },
             'overrides': {
                 'whitelist_wrongly_marked_down.yaml': None
             },
             'tasks': {
                 'cfuse_workunit_suites_fsstress.yaml': None
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('thrash', fake_isfile, fake_isdir,
                                 fake_listdir)
     assert len(result) == 1
     assert self.fragment_occurences(result, 'base.yaml') == 1
     fragments = result[0][1]
     assert fragments[0] == 'thrash/ceph/base.yaml'
     assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
Пример #13
0
 def test_concatenate_1x2x3(self):
     fake_fs = {
         'd0_0': {
             '+': None,
             'd1_0': {
                 'd1_0_0.yaml': None,
             },
             'd1_1': {
                 'd1_1_0.yaml': None,
                 'd1_1_1.yaml': None,
             },
             'd1_2': {
                 'd1_2_0.yaml': None,
                 'd1_2_1.yaml': None,
                 'd1_2_2.yaml': None,
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
                                 fake_listdir)
     assert len(result) == 1
Пример #14
0
 def test_concatenate_1x2x3(self):
     fake_fs = {
         'd0_0': {
             '+': None,
             'd1_0': {
                 'd1_0_0.yaml': None,
             },
             'd1_1': {
                 'd1_1_0.yaml': None,
                 'd1_1_1.yaml': None,
             },
             'd1_2': {
                 'd1_2_0.yaml': None,
                 'd1_2_1.yaml': None,
                 'd1_2_2.yaml': None,
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
                                 fake_listdir)
     assert len(result) == 1
Пример #15
0
 def test_sort_order(self):
     # This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
     # fragments are sorted.
     fake_fs = {
         "thrash": {
             "%": None,
             "ceph-thrash": {"default.yaml": None},
             "ceph": {"base.yaml": None},
             "clusters": {"mds-1active-1standby.yaml": None},
             "debug": {"mds_client.yaml": None},
             "fs": {"btrfs.yaml": None},
             "msgr-failures": {"none.yaml": None},
             "overrides": {"whitelist_wrongly_marked_down.yaml": None},
             "tasks": {"cfuse_workunit_suites_fsstress.yaml": None},
         }
     }
     fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
     result = suite.build_matrix("thrash", fake_isfile, fake_isdir, fake_listdir)
     assert len(result) == 1
     assert self.fragment_occurences(result, "base.yaml") == 1
     fragments = result[0][1]
     assert fragments[0] == "thrash/ceph/base.yaml"
     assert fragments[1] == "thrash/ceph-thrash/default.yaml"
Пример #16
0
 def test_convolve_1x2x4(self):
     fake_fs = {
         'd0_0': {
             '%': None,
             'd1_0': {
                 'd1_0_0.yaml': None,
             },
             'd1_1': {
                 'd1_1_0.yaml': None,
                 'd1_1_1.yaml': None,
             },
             'd1_2': {
                 'd1_2_0.yaml': None,
                 'd1_2_1.yaml': None,
                 'd1_2_2.yaml': None,
                 'd1_2_3.yaml': None,
             },
         },
     }
     fake_listdir = make_fake_listdir(fake_fs)
     result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
                                 fake_listdir)
     assert len(result) == 8
     assert self.fragment_occurences(result, 'd1_2_2.yaml') == 0.25
Пример #17
0
 def test_sort_order(self):
     # This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
     # fragments are sorted.
     fake_fs = {
         'thrash': {
             '%': None,
             'ceph-thrash': {'default.yaml': None},
             'ceph': {'base.yaml': None},
             'clusters': {'mds-1active-1standby.yaml': None},
             'debug': {'mds_client.yaml': None},
             'fs': {'btrfs.yaml': None},
             'msgr-failures': {'none.yaml': None},
             'overrides': {'whitelist_wrongly_marked_down.yaml': None},
             'tasks': {'cfuse_workunit_suites_fsstress.yaml': None},
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('thrash', fake_isfile,
                                 fake_isdir, fake_listdir)
     assert len(result) == 1
     assert self.fragment_occurences(result, 'base.yaml') == 1
     fragments = result[0][1]
     assert fragments[0] == 'thrash/ceph/base.yaml'
     assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
Пример #18
0
def get_combinations(suite_dir,
                     fields,
                     subset,
                     limit,
                     filter_in,
                     filter_out,
                     include_facet,
                     _isdir=os.path.isdir,
                     _open=open,
                     _isfile=os.path.isfile,
                     _listdir=os.listdir):
    """
    Describes the combinations of a suite, optionally limiting
    or filtering output based on the given parameters. Includes
    columns for the subsuite and facets when include_facet is True.

    Returns a tuple of (headers, rows) where both elements are lists
    of strings.
    """
    configs = [
        (combine_path(suite_dir, item[0]), item[1])
        for item in build_matrix(suite_dir, _isfile, _isdir, _listdir, subset)
    ]

    num_listed = 0
    rows = []

    facet_headers = set()
    dirs = {}
    max_dir_depth = 0

    for _, fragment_paths in configs:
        if limit > 0 and num_listed >= limit:
            break
        if filter_in and not any(
            [f in path for f in filter_in for path in fragment_paths]):
            continue
        if filter_out and any(
            [f in path for f in filter_out for path in fragment_paths]):
            continue

        fragment_fields = [
            extract_info(path, fields, _isdir, _open)
            for path in fragment_paths
        ]

        # merge fields from multiple fragments by joining their values with \n
        metadata = {}
        for fragment_meta in fragment_fields:
            for field, value in fragment_meta.items():
                if value == '':
                    continue
                if field in metadata:
                    metadata[field] += '\n' + str(value)
                else:
                    metadata[field] = str(value)

        if include_facet:
            # map final dir (facet) -> filename without the .yaml suffix
            for path in fragment_paths:
                facet_dir = os.path.dirname(path)
                facet = os.path.basename(facet_dir)
                metadata[facet] = os.path.basename(path)[:-5]
                facet_headers.add(facet)
                facet_dirs = facet_dir.split('/')[:-1]
                for i, dir_ in enumerate(facet_dirs):
                    if i not in dirs:
                        dirs[i] = set()
                    dirs[i].add(dir_)
                    metadata['_dir_' + str(i)] = os.path.basename(dir_)
                    max_dir_depth = max(max_dir_depth, i)

        rows.append(metadata)
        num_listed += 1

    subsuite_headers = []
    if include_facet:
        first_subsuite_depth = max_dir_depth
        for i in range(max_dir_depth):
            if len(dirs[i]) > 1:
                first_subsuite_depth = i
                break

        subsuite_headers = [
            'subsuite depth ' + str(i)
            for i in range(0, max_dir_depth - first_subsuite_depth + 1)
        ]

        for row in rows:
            for i in range(first_subsuite_depth, max_dir_depth + 1):
                row[subsuite_headers[i - first_subsuite_depth]] = \
                    row.get('_dir_' + str(i), '')

    headers = subsuite_headers + sorted(facet_headers) + fields
    return headers, sorted([[row.get(field, '') for field in headers]
                            for row in rows])
Пример #19
0
 def test_disable_extension(self):
     fake_fs = {
         'teuthology': {
             'no-ceph': {
                 '%': None,
                 'clusters': {
                     'single.yaml': None,
                 },
                 'distros': {
                     'baremetal.yaml': None,
                     'rhel7.0.yaml': None,
                     'ubuntu12.04.yaml': None,
                     'ubuntu14.04.yaml': None,
                     'vps.yaml': None,
                     'vps_centos6.5.yaml': None,
                     'vps_debian7.yaml': None,
                     'vps_rhel6.4.yaml': None,
                     'vps_rhel6.5.yaml': None,
                     'vps_rhel7.0.yaml': None,
                     'vps_ubuntu14.04.yaml': None,
                 },
                 'tasks': {
                     'teuthology.yaml': None,
                 },
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
                                 fake_isdir, fake_listdir)
     fake_fs2 = {
         'teuthology': {
             'no-ceph': {
                 '%': None,
                 'clusters': {
                     'single.yaml': None,
                 },
                 'distros': {
                     'baremetal.yaml': None,
                     'rhel7.0.yaml': None,
                     'ubuntu12.04.yaml': None,
                     'ubuntu14.04.yaml': None,
                     'vps.yaml': None,
                     'vps_centos6.5.yaml': None,
                     'vps_debian7.yaml': None,
                     'vps_rhel6.4.yaml': None,
                     'vps_rhel6.5.yaml': None,
                     'vps_rhel7.0.yaml': None,
                     'vps_ubuntu14.04.yaml': None,
                     'forcefilevps_ubuntu14.04.yaml.disable': None,
                     'forcefilevps_ubuntu14.04.yaml.anotherextension': None,
                 },
                 'tasks': {
                     'teuthology.yaml': None,
                     'forcefilevps_ubuntu14.04notyaml': None,
                 },
                 'forcefilevps_ubuntu14.04notyaml': None,
                 'tasks.disable': {
                     'teuthology2.yaml': None,
                     'forcefilevps_ubuntu14.04notyaml': None,
                 },
             },
         },
     }
     fake_listdir2, fake_isfile2, fake_isdir2, _ = make_fake_fstools(
         fake_fs2)
     result2 = suite.build_matrix('teuthology/no-ceph', fake_isfile2,
                                  fake_isdir2, fake_listdir2)
     assert len(result) == 11
     assert len(result2) == len(result)
Пример #20
0
 def test_empty_dirs(self):
     fake_fs = {
         'teuthology': {
             'no-ceph': {
                 '%': None,
                 'clusters': {
                     'single.yaml': None,
                 },
                 'distros': {
                     'baremetal.yaml': None,
                     'rhel7.0.yaml': None,
                     'ubuntu12.04.yaml': None,
                     'ubuntu14.04.yaml': None,
                     'vps.yaml': None,
                     'vps_centos6.5.yaml': None,
                     'vps_debian7.yaml': None,
                     'vps_rhel6.4.yaml': None,
                     'vps_rhel6.5.yaml': None,
                     'vps_rhel7.0.yaml': None,
                     'vps_ubuntu14.04.yaml': None,
                 },
                 'tasks': {
                     'teuthology.yaml': None,
                 },
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
     result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
                                 fake_isdir, fake_listdir)
     fake_fs2 = {
         'teuthology': {
             'no-ceph': {
                 '%': None,
                 'clusters': {
                     'single.yaml': None,
                 },
                 'distros': {
                     'empty': {},
                     'baremetal.yaml': None,
                     'rhel7.0.yaml': None,
                     'ubuntu12.04.yaml': None,
                     'ubuntu14.04.yaml': None,
                     'vps.yaml': None,
                     'vps_centos6.5.yaml': None,
                     'vps_debian7.yaml': None,
                     'vps_rhel6.4.yaml': None,
                     'vps_rhel6.5.yaml': None,
                     'vps_rhel7.0.yaml': None,
                     'vps_ubuntu14.04.yaml': None,
                 },
                 'tasks': {
                     'teuthology.yaml': None,
                 },
                 'empty': {},
             },
         },
     }
     fake_listdir2, fake_isfile2, fake_isdir2 = make_fake_fstools(fake_fs2)
     result2 = suite.build_matrix('teuthology/no-ceph', fake_isfile2,
                                  fake_isdir2, fake_listdir2)
     assert len(result) == 11
     assert len(result2) == len(result)
Пример #21
0
 def test_disable_extension(self):
     fake_fs = {
         'teuthology': {
             'no-ceph': {
                 '%': None,
                 'clusters': {
                     'single.yaml': None,
                 },
                 'distros': {
                     'baremetal.yaml': None,
                     'rhel7.0.yaml': None,
                     'ubuntu12.04.yaml': None,
                     'ubuntu14.04.yaml': None,
                     'vps.yaml': None,
                     'vps_centos6.5.yaml': None,
                     'vps_debian7.yaml': None,
                     'vps_rhel6.4.yaml': None,
                     'vps_rhel6.5.yaml': None,
                     'vps_rhel7.0.yaml': None,
                     'vps_ubuntu14.04.yaml': None,
                 },
                 'tasks': {
                     'teuthology.yaml': None,
                 },
             },
         },
     }
     fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
     result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
                                 fake_isdir, fake_listdir)
     fake_fs2 = {
         'teuthology': {
             'no-ceph': {
                 '%': None,
                 'clusters': {
                     'single.yaml': None,
                 },
                 'distros': {
                     'baremetal.yaml': None,
                     'rhel7.0.yaml': None,
                     'ubuntu12.04.yaml': None,
                     'ubuntu14.04.yaml': None,
                     'vps.yaml': None,
                     'vps_centos6.5.yaml': None,
                     'vps_debian7.yaml': None,
                     'vps_rhel6.4.yaml': None,
                     'vps_rhel6.5.yaml': None,
                     'vps_rhel7.0.yaml': None,
                     'vps_ubuntu14.04.yaml': None,
                     'forcefilevps_ubuntu14.04.yaml.disable': None,
                     'forcefilevps_ubuntu14.04.yaml.anotherextension': None,
                 },
                 'tasks': {
                     'teuthology.yaml': None,
                     'forcefilevps_ubuntu14.04notyaml': None,
                 },
                 'forcefilevps_ubuntu14.04notyaml': None,
                 'tasks.disable': {
                     'teuthology2.yaml': None,
                     'forcefilevps_ubuntu14.04notyaml': None,
                 },
             },
         },
     }
     fake_listdir2, fake_isfile2, fake_isdir2, _ = make_fake_fstools(fake_fs2)
     result2 = suite.build_matrix('teuthology/no-ceph', fake_isfile2,
                                  fake_isdir2, fake_listdir2)
     assert len(result) == 11
     assert len(result2) == len(result)
Пример #22
0
def get_combinations(suite_dir, fields, subset,
                     limit, filter_in, filter_out,
                     include_facet, _isdir=os.path.isdir, _open=open,
                     _isfile=os.path.isfile, _listdir=os.listdir):
    """
    Describes the combinations of a suite, optionally limiting
    or filtering output based on the given parameters. Includes
    columns for the subsuite and facets when include_facet is True.

    Returns a tuple of (headers, rows) where both elements are lists
    of strings.
    """
    configs = [(combine_path(suite_dir, item[0]), item[1]) for item in
               build_matrix(suite_dir, _isfile, _isdir, _listdir, subset)]

    num_listed = 0
    rows = []

    facet_headers = set()
    dirs = {}
    max_dir_depth = 0

    for _, fragment_paths in configs:
        if limit > 0 and num_listed >= limit:
            break
        if filter_in and not any([f in path for f in filter_in
                                  for path in fragment_paths]):
            continue
        if filter_out and any([f in path for f in filter_out
                               for path in fragment_paths]):
            continue

        fragment_fields = [extract_info(path, fields, _isdir, _open)
                           for path in fragment_paths]

        # merge fields from multiple fragments by joining their values with \n
        metadata = {}
        for fragment_meta in fragment_fields:
            for field, value in fragment_meta.items():
                if value == '':
                    continue
                if field in metadata:
                    metadata[field] += '\n' + str(value)
                else:
                    metadata[field] = str(value)

        if include_facet:
            # map final dir (facet) -> filename without the .yaml suffix
            for path in fragment_paths:
                facet_dir = os.path.dirname(path)
                facet = os.path.basename(facet_dir)
                metadata[facet] = os.path.basename(path)[:-5]
                facet_headers.add(facet)
                facet_dirs = facet_dir.split('/')[:-1]
                for i, dir_ in enumerate(facet_dirs):
                    if i not in dirs:
                        dirs[i] = set()
                    dirs[i].add(dir_)
                    metadata['_dir_' + str(i)] = os.path.basename(dir_)
                    max_dir_depth = max(max_dir_depth, i)

        rows.append(metadata)
        num_listed += 1

    subsuite_headers = []
    if include_facet:
        first_subsuite_depth = max_dir_depth
        for i in range(max_dir_depth):
            if len(dirs[i]) > 1:
                first_subsuite_depth = i
                break

        subsuite_headers = ['subsuite depth ' + str(i)
                            for i in
                            range(0, max_dir_depth - first_subsuite_depth + 1)]

        for row in rows:
            for i in range(first_subsuite_depth, max_dir_depth + 1):
                row[subsuite_headers[i - first_subsuite_depth]] = \
                    row.get('_dir_' + str(i), '')

    headers = subsuite_headers + sorted(facet_headers) + fields
    return headers, sorted([[row.get(field, '') for field in headers]
                            for row in rows])