def test_disable_extension(self):
fake_fs = {
"teuthology": {
"no-ceph": {
"%": None,
"clusters": {"single.yaml": None},
"distros": {
"baremetal.yaml": None,
"rhel7.0.yaml": None,
"ubuntu12.04.yaml": None,
"ubuntu14.04.yaml": None,
"vps.yaml": None,
"vps_centos6.5.yaml": None,
"vps_debian7.yaml": None,
"vps_rhel6.4.yaml": None,
"vps_rhel6.5.yaml": None,
"vps_rhel7.0.yaml": None,
"vps_ubuntu14.04.yaml": None,
},
"tasks": {"teuthology.yaml": None},
}
}
}
fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
result = suite.build_matrix("teuthology/no-ceph", fake_isfile, fake_isdir, fake_listdir)
fake_fs2 = {
"teuthology": {
"no-ceph": {
"%": None,
"clusters": {"single.yaml": None},
"distros": {
"baremetal.yaml": None,
"rhel7.0.yaml": None,
"ubuntu12.04.yaml": None,
"ubuntu14.04.yaml": None,
"vps.yaml": None,
"vps_centos6.5.yaml": None,
"vps_debian7.yaml": None,
"vps_rhel6.4.yaml": None,
"vps_rhel6.5.yaml": None,
"vps_rhel7.0.yaml": None,
"vps_ubuntu14.04.yaml": None,
"forcefilevps_ubuntu14.04.yaml.disable": None,
"forcefilevps_ubuntu14.04.yaml.anotherextension": None,
},
"tasks": {"teuthology.yaml": None, "forcefilevps_ubuntu14.04notyaml": None},
"forcefilevps_ubuntu14.04notyaml": None,
"tasks.disable": {"teuthology2.yaml": None, "forcefilevps_ubuntu14.04notyaml": None},
}
}
}
fake_listdir2, fake_isfile2, fake_isdir2 = make_fake_fstools(fake_fs2)
result2 = suite.build_matrix("teuthology/no-ceph", fake_isfile2, fake_isdir2, fake_listdir2)
assert len(result) == 11
assert len(result2) == len(result)
def test_emulate_teuthology_noceph(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
fake_isdir, fake_listdir)
assert len(result) == 11
assert self.fragment_occurences(result, 'vps.yaml') == 1 / 11.0
def test_convolve_with_concat(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
fake_listdir)
assert len(result) == 2
for i in result:
assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
def test_emulate_teuthology_noceph(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
fake_isdir, fake_listdir)
assert len(result) == 11
assert self.fragment_occurences(result, 'vps.yaml') == 1 / 11.0
def test_emulate_teuthology_noceph(self):
fake_fs = {
"teuthology": {
"no-ceph": {
"%": None,
"clusters": {"single.yaml": None},
"distros": {
"baremetal.yaml": None,
"rhel7.0.yaml": None,
"ubuntu12.04.yaml": None,
"ubuntu14.04.yaml": None,
"vps.yaml": None,
"vps_centos6.5.yaml": None,
"vps_debian7.yaml": None,
"vps_rhel6.4.yaml": None,
"vps_rhel6.5.yaml": None,
"vps_rhel7.0.yaml": None,
"vps_ubuntu14.04.yaml": None,
},
"tasks": {"teuthology.yaml": None},
}
}
}
fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
result = suite.build_matrix("teuthology/no-ceph", fake_isfile, fake_isdir, fake_listdir)
assert len(result) == 11
assert self.fragment_occurences(result, "vps.yaml") == 1 / 11.0
def test_convolve_with_concat(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'+': None,
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
fake_listdir)
assert len(result) == 2
for i in result:
assert 'd0_0/d1_2/d1_2_0.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_1.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_2.yaml' in i[1]
assert 'd0_0/d1_2/d1_2_3.yaml' in i[1]
def test_convolve_2x2(self):
fake_fs = {
"d0_0": {
"%": None,
"d1_0": {"d1_0_0.yaml": None, "d1_0_1.yaml": None},
"d1_1": {"d1_1_0.yaml": None, "d1_1_1.yaml": None},
}
}
fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
result = suite.build_matrix("d0_0", fake_isfile, fake_isdir, fake_listdir)
assert len(result) == 4
assert self.fragment_occurences(result, "d1_1_1.yaml") == 0.5
def test_concatenate_1x2x3(self):
fake_fs = {
"d0_0": {
"+": None,
"d1_0": {"d1_0_0.yaml": None},
"d1_1": {"d1_1_0.yaml": None, "d1_1_1.yaml": None},
"d1_2": {"d1_2_0.yaml": None, "d1_2_1.yaml": None, "d1_2_2.yaml": None},
}
}
fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
result = suite.build_matrix("d0_0", fake_isfile, fake_isdir, fake_listdir)
assert len(result) == 1
def test_convolve_with_concat(self):
fake_fs = {
"d0_0": {
"%": None,
"d1_0": {"d1_0_0.yaml": None},
"d1_1": {"d1_1_0.yaml": None, "d1_1_1.yaml": None},
"d1_2": {"+": None, "d1_2_0.yaml": None, "d1_2_1.yaml": None, "d1_2_2.yaml": None, "d1_2_3.yaml": None},
}
}
fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
result = suite.build_matrix("d0_0", fake_isfile, fake_isdir, fake_listdir)
assert len(result) == 2
for i in result:
assert "d0_0/d1_2/d1_2_0.yaml" in i[1]
assert "d0_0/d1_2/d1_2_1.yaml" in i[1]
assert "d0_0/d1_2/d1_2_2.yaml" in i[1]
assert "d0_0/d1_2/d1_2_3.yaml" in i[1]
def test_convolve_2x2(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
fake_listdir)
assert len(result) == 4
assert self.fragment_occurences(result, 'd1_1_1.yaml') == 0.5
def test_convolve_2x2(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
'd1_0_1.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
fake_listdir)
assert len(result) == 4
assert self.fragment_occurences(result, 'd1_1_1.yaml') == 0.5
def test_sort_order(self):
# This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
# fragments are sorted.
fake_fs = {
'thrash': {
'%': None,
'ceph-thrash': {
'default.yaml': None
},
'ceph': {
'base.yaml': None
},
'clusters': {
'mds-1active-1standby.yaml': None
},
'debug': {
'mds_client.yaml': None
},
'fs': {
'btrfs.yaml': None
},
'msgr-failures': {
'none.yaml': None
},
'overrides': {
'whitelist_wrongly_marked_down.yaml': None
},
'tasks': {
'cfuse_workunit_suites_fsstress.yaml': None
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('thrash', fake_isfile, fake_isdir,
fake_listdir)
assert len(result) == 1
assert self.fragment_occurences(result, 'base.yaml') == 1
fragments = result[0][1]
assert fragments[0] == 'thrash/ceph/base.yaml'
assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
def test_concatenate_1x2x3(self):
fake_fs = {
'd0_0': {
'+': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
fake_listdir)
assert len(result) == 1
def test_concatenate_1x2x3(self):
fake_fs = {
'd0_0': {
'+': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
fake_listdir)
assert len(result) == 1
def test_sort_order(self):
# This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
# fragments are sorted.
fake_fs = {
"thrash": {
"%": None,
"ceph-thrash": {"default.yaml": None},
"ceph": {"base.yaml": None},
"clusters": {"mds-1active-1standby.yaml": None},
"debug": {"mds_client.yaml": None},
"fs": {"btrfs.yaml": None},
"msgr-failures": {"none.yaml": None},
"overrides": {"whitelist_wrongly_marked_down.yaml": None},
"tasks": {"cfuse_workunit_suites_fsstress.yaml": None},
}
}
fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
result = suite.build_matrix("thrash", fake_isfile, fake_isdir, fake_listdir)
assert len(result) == 1
assert self.fragment_occurences(result, "base.yaml") == 1
fragments = result[0][1]
assert fragments[0] == "thrash/ceph/base.yaml"
assert fragments[1] == "thrash/ceph-thrash/default.yaml"
def test_convolve_1x2x4(self):
fake_fs = {
'd0_0': {
'%': None,
'd1_0': {
'd1_0_0.yaml': None,
},
'd1_1': {
'd1_1_0.yaml': None,
'd1_1_1.yaml': None,
},
'd1_2': {
'd1_2_0.yaml': None,
'd1_2_1.yaml': None,
'd1_2_2.yaml': None,
'd1_2_3.yaml': None,
},
},
}
fake_listdir = make_fake_listdir(fake_fs)
result = suite.build_matrix('d0_0', fake_isfile, fake_isdir,
fake_listdir)
assert len(result) == 8
assert self.fragment_occurences(result, 'd1_2_2.yaml') == 0.25
def test_sort_order(self):
# This test ensures that 'ceph' comes before 'ceph-thrash' when yaml
# fragments are sorted.
fake_fs = {
'thrash': {
'%': None,
'ceph-thrash': {'default.yaml': None},
'ceph': {'base.yaml': None},
'clusters': {'mds-1active-1standby.yaml': None},
'debug': {'mds_client.yaml': None},
'fs': {'btrfs.yaml': None},
'msgr-failures': {'none.yaml': None},
'overrides': {'whitelist_wrongly_marked_down.yaml': None},
'tasks': {'cfuse_workunit_suites_fsstress.yaml': None},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('thrash', fake_isfile,
fake_isdir, fake_listdir)
assert len(result) == 1
assert self.fragment_occurences(result, 'base.yaml') == 1
fragments = result[0][1]
assert fragments[0] == 'thrash/ceph/base.yaml'
assert fragments[1] == 'thrash/ceph-thrash/default.yaml'
def get_combinations(suite_dir,
fields,
subset,
limit,
filter_in,
filter_out,
include_facet,
_isdir=os.path.isdir,
_open=open,
_isfile=os.path.isfile,
_listdir=os.listdir):
"""
Describes the combinations of a suite, optionally limiting
or filtering output based on the given parameters. Includes
columns for the subsuite and facets when include_facet is True.
Returns a tuple of (headers, rows) where both elements are lists
of strings.
"""
configs = [
(combine_path(suite_dir, item[0]), item[1])
for item in build_matrix(suite_dir, _isfile, _isdir, _listdir, subset)
]
num_listed = 0
rows = []
facet_headers = set()
dirs = {}
max_dir_depth = 0
for _, fragment_paths in configs:
if limit > 0 and num_listed >= limit:
break
if filter_in and not any(
[f in path for f in filter_in for path in fragment_paths]):
continue
if filter_out and any(
[f in path for f in filter_out for path in fragment_paths]):
continue
fragment_fields = [
extract_info(path, fields, _isdir, _open)
for path in fragment_paths
]
# merge fields from multiple fragments by joining their values with \n
metadata = {}
for fragment_meta in fragment_fields:
for field, value in fragment_meta.items():
if value == '':
continue
if field in metadata:
metadata[field] += '\n' + str(value)
else:
metadata[field] = str(value)
if include_facet:
# map final dir (facet) -> filename without the .yaml suffix
for path in fragment_paths:
facet_dir = os.path.dirname(path)
facet = os.path.basename(facet_dir)
metadata[facet] = os.path.basename(path)[:-5]
facet_headers.add(facet)
facet_dirs = facet_dir.split('/')[:-1]
for i, dir_ in enumerate(facet_dirs):
if i not in dirs:
dirs[i] = set()
dirs[i].add(dir_)
metadata['_dir_' + str(i)] = os.path.basename(dir_)
max_dir_depth = max(max_dir_depth, i)
rows.append(metadata)
num_listed += 1
subsuite_headers = []
if include_facet:
first_subsuite_depth = max_dir_depth
for i in range(max_dir_depth):
if len(dirs[i]) > 1:
first_subsuite_depth = i
break
subsuite_headers = [
'subsuite depth ' + str(i)
for i in range(0, max_dir_depth - first_subsuite_depth + 1)
]
for row in rows:
for i in range(first_subsuite_depth, max_dir_depth + 1):
row[subsuite_headers[i - first_subsuite_depth]] = \
row.get('_dir_' + str(i), '')
headers = subsuite_headers + sorted(facet_headers) + fields
return headers, sorted([[row.get(field, '') for field in headers]
for row in rows])
def test_disable_extension(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
fake_isdir, fake_listdir)
fake_fs2 = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
'forcefilevps_ubuntu14.04.yaml.disable': None,
'forcefilevps_ubuntu14.04.yaml.anotherextension': None,
},
'tasks': {
'teuthology.yaml': None,
'forcefilevps_ubuntu14.04notyaml': None,
},
'forcefilevps_ubuntu14.04notyaml': None,
'tasks.disable': {
'teuthology2.yaml': None,
'forcefilevps_ubuntu14.04notyaml': None,
},
},
},
}
fake_listdir2, fake_isfile2, fake_isdir2, _ = make_fake_fstools(
fake_fs2)
result2 = suite.build_matrix('teuthology/no-ceph', fake_isfile2,
fake_isdir2, fake_listdir2)
assert len(result) == 11
assert len(result2) == len(result)
def test_empty_dirs(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
fake_listdir, fake_isfile, fake_isdir = make_fake_fstools(fake_fs)
result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
fake_isdir, fake_listdir)
fake_fs2 = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'empty': {},
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
'empty': {},
},
},
}
fake_listdir2, fake_isfile2, fake_isdir2 = make_fake_fstools(fake_fs2)
result2 = suite.build_matrix('teuthology/no-ceph', fake_isfile2,
fake_isdir2, fake_listdir2)
assert len(result) == 11
assert len(result2) == len(result)
def test_disable_extension(self):
fake_fs = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
},
'tasks': {
'teuthology.yaml': None,
},
},
},
}
fake_listdir, fake_isfile, fake_isdir, _ = make_fake_fstools(fake_fs)
result = suite.build_matrix('teuthology/no-ceph', fake_isfile,
fake_isdir, fake_listdir)
fake_fs2 = {
'teuthology': {
'no-ceph': {
'%': None,
'clusters': {
'single.yaml': None,
},
'distros': {
'baremetal.yaml': None,
'rhel7.0.yaml': None,
'ubuntu12.04.yaml': None,
'ubuntu14.04.yaml': None,
'vps.yaml': None,
'vps_centos6.5.yaml': None,
'vps_debian7.yaml': None,
'vps_rhel6.4.yaml': None,
'vps_rhel6.5.yaml': None,
'vps_rhel7.0.yaml': None,
'vps_ubuntu14.04.yaml': None,
'forcefilevps_ubuntu14.04.yaml.disable': None,
'forcefilevps_ubuntu14.04.yaml.anotherextension': None,
},
'tasks': {
'teuthology.yaml': None,
'forcefilevps_ubuntu14.04notyaml': None,
},
'forcefilevps_ubuntu14.04notyaml': None,
'tasks.disable': {
'teuthology2.yaml': None,
'forcefilevps_ubuntu14.04notyaml': None,
},
},
},
}
fake_listdir2, fake_isfile2, fake_isdir2, _ = make_fake_fstools(fake_fs2)
result2 = suite.build_matrix('teuthology/no-ceph', fake_isfile2,
fake_isdir2, fake_listdir2)
assert len(result) == 11
assert len(result2) == len(result)
def get_combinations(suite_dir, fields, subset,
limit, filter_in, filter_out,
include_facet, _isdir=os.path.isdir, _open=open,
_isfile=os.path.isfile, _listdir=os.listdir):
"""
Describes the combinations of a suite, optionally limiting
or filtering output based on the given parameters. Includes
columns for the subsuite and facets when include_facet is True.
Returns a tuple of (headers, rows) where both elements are lists
of strings.
"""
configs = [(combine_path(suite_dir, item[0]), item[1]) for item in
build_matrix(suite_dir, _isfile, _isdir, _listdir, subset)]
num_listed = 0
rows = []
facet_headers = set()
dirs = {}
max_dir_depth = 0
for _, fragment_paths in configs:
if limit > 0 and num_listed >= limit:
break
if filter_in and not any([f in path for f in filter_in
for path in fragment_paths]):
continue
if filter_out and any([f in path for f in filter_out
for path in fragment_paths]):
continue
fragment_fields = [extract_info(path, fields, _isdir, _open)
for path in fragment_paths]
# merge fields from multiple fragments by joining their values with \n
metadata = {}
for fragment_meta in fragment_fields:
for field, value in fragment_meta.items():
if value == '':
continue
if field in metadata:
metadata[field] += '\n' + str(value)
else:
metadata[field] = str(value)
if include_facet:
# map final dir (facet) -> filename without the .yaml suffix
for path in fragment_paths:
facet_dir = os.path.dirname(path)
facet = os.path.basename(facet_dir)
metadata[facet] = os.path.basename(path)[:-5]
facet_headers.add(facet)
facet_dirs = facet_dir.split('/')[:-1]
for i, dir_ in enumerate(facet_dirs):
if i not in dirs:
dirs[i] = set()
dirs[i].add(dir_)
metadata['_dir_' + str(i)] = os.path.basename(dir_)
max_dir_depth = max(max_dir_depth, i)
rows.append(metadata)
num_listed += 1
subsuite_headers = []
if include_facet:
first_subsuite_depth = max_dir_depth
for i in range(max_dir_depth):
if len(dirs[i]) > 1:
first_subsuite_depth = i
break
subsuite_headers = ['subsuite depth ' + str(i)
for i in
range(0, max_dir_depth - first_subsuite_depth + 1)]
for row in rows:
for i in range(first_subsuite_depth, max_dir_depth + 1):
row[subsuite_headers[i - first_subsuite_depth]] = \
row.get('_dir_' + str(i), '')
headers = subsuite_headers + sorted(facet_headers) + fields
return headers, sorted([[row.get(field, '') for field in headers]
for row in rows])
