def test_to_index_html_004(self):
data = """
[
{
"columns": [
],
"database_name": "orcl",
"row_count": 200000,
"schema_name": "public",
"table_name": "t1",
"timestamp": "2016-05-01T16:15:18.028309"
}
]
"""
# max_panels is not specified, so 'tag7' could be appeared.
html = DbProfilerFormatter.to_index_html(json.loads(data),
tags=[['tag%s' % x, 0]
for x in range(1, 8)],
reponame='testrepo')
# print(html)
self.assertTrue(html.find('tag6') > 0) # found
self.assertTrue(html.find('tag7') > 0) # found
self.assertTrue(html.find('index-tags.html') < 0) # not found
# max_panels=6, so 'tag7' must not be appeared.
html = DbProfilerFormatter.to_index_html(json.loads(data),
tags=[['tag%s' % x, 0]
for x in range(1, 8)],
reponame='testrepo',
max_panels=6)
# print(html)
self.assertTrue(html.find('tag6') > 0) # found
self.assertTrue(html.find('tag7') < 0) # not found
self.assertTrue(html.find('index-tags.html') > 0) # found
def test_get_non_null_ratio_001(self):
self.assertEqual('75.00 %',
DbProfilerFormatter.get_non_null_ratio(100, 25))
self.assertEqual('100.00 %',
DbProfilerFormatter.get_non_null_ratio(100, 0))
# N/A
self.assertEqual('N/A',
DbProfilerFormatter.get_non_null_ratio(100, None))
self.assertEqual('N/A',
DbProfilerFormatter.get_non_null_ratio(None, 20))
def test_filter_markdown2html_001(self):
md = ''
a = ''
self.assertEqual(a, DbProfilerFormatter.filter_markdown2html(md))
md = None
a = ''
self.assertEqual(a, DbProfilerFormatter.filter_markdown2html(md))
md = "# title 1"
a = '<h1>title 1</h1>'
self.assertEqual(a, DbProfilerFormatter.filter_markdown2html(md))
def test_filter_glossaryterms_001(self):
terms = [{'term': 'PV', 'description_short': 'Page Views'}]
html = 'foo'
a = 'foo'
self.assertEqual(a,
DbProfilerFormatter.filter_glossaryterms(html, terms))
html = 'foo PV bar'
a = u'foo <a tabindex="0" data-toggle="popover" data-trigger="focus" data-html="true" title="PV" data-content="Page Views<br/><div align=right><a href=\'glossary.html#PV\' target=\'_glossary\'>詳細…</a></div>"><span class="glossary-term">PV</span></a> bar'
self.assertEqual(a,
DbProfilerFormatter.filter_glossaryterms(html, terms))
def test_filter_term2popover_001(self):
a = (
'aaa <a tabindex="0" data-toggle="popover" data-trigger="focus" data-html="true" title="abc" data-content="xyz"><span class="glossary-term">abc</span></a> bbb',
153)
b = DbProfilerFormatter.filter_term2popover('aaa abc bbb', 4, 'abc',
'xyz')
self.assertEqual(a, b)
a = (
'aaa <a tabindex="0" data-toggle="popover" data-trigger="focus" data-html="true" title="abc" data-content="xyz"><span class="glossary-term">jkl</span></a> bbb',
153)
b = DbProfilerFormatter.filter_term2popover('aaa abc bbb', 4, 'jkl',
'xyz', 'abc')
self.assertEqual(a, b)
def test_create_popover_content_001(self):
t = {}
t['term'] = 'aa'
t['description_short'] = 'desc'
a = u"desc<br/><div align=right><a href='glossary.html#aa' target='_glossary'>\u8a73\u7d30\u2026</a></div>"
self.assertEqual(a, DbProfilerFormatter.create_popover_content(t))
t['synonyms'] = ['a', 'b']
a = u"desc<br/><br/>同義語: a, b<div align=right><a href='glossary.html#aa' target='_glossary'>\u8a73\u7d30\u2026</a></div>"
self.assertEqual(a, DbProfilerFormatter.create_popover_content(t))
t['related_terms'] = ['c', 'd']
a = u"desc<br/><br/>同義語: a, b<br/>類義語: c, d<div align=right><a href='glossary.html#aa' target='_glossary'>\u8a73\u7d30\u2026</a></div>"
self.assertEqual(a, DbProfilerFormatter.create_popover_content(t))
t['assigned_assets'] = ['e', 'f']
a = u"desc<br/><br/>同義語: a, b<br/>類義語: c, d<br/>関連テーブル: e, f<div align=right><a href='glossary.html#aa' target='_glossary'>\u8a73\u7d30\u2026</a></div>"
self.assertEqual(a, DbProfilerFormatter.create_popover_content(t))
def test_to_index_html_002(self):
data = """
[
{
"columns": [
{
"cardinality": 199999,
"column_name": "c1",
"data_type": [
"NUMBER",
"22"
],
"max": "200000",
"min": "1",
"nulls": 0,
"validation": {}
},
{
"cardinality": 200000,
"column_name": "c2",
"data_type": [
"NUMBER",
"22"
],
"max": "200000",
"min": "1",
"nulls": 3,
"validation": {}
}
],
"database_name": "orcl",
"row_count": 200000,
"schema_name": "public",
"table_name": "t1",
"timestamp": "2016-05-01T16:15:18.028309"
}
]
"""
with self.assertRaises(ValueError):
DbProfilerFormatter.to_index_html(json.loads(data),
reponame='testrepo',
template_file='foo.html')
def test_to_glossary_html_001(self):
data = None
self.assertIsNotNone(DbProfilerFormatter.to_glossary_html(data))
data = [{
'term': u'Term1',
'description_short': u'short description of Term1',
'description_long': u'long description of Term1',
'categories': ['term1cat', 'term1cat2'],
'assigned_terms': ['Term2', 'Term4'],
'assigned_assets': ['table1', 'table2'],
'assigned_assets2': {
'table1': ['dqwbtest.public.table1']
},
'owned_by': 'snaga'
}, {
'term': u'Term2',
'description_short': u'short description of Term2 for c1'
}]
html = DbProfilerFormatter.to_glossary_html(data)
# print(html)
# term, short desc, long desc, categories, related terms, related assets, owner
a = u'<td class="wrap"><a name="Term1" class="nodeco">Term1</a></td>'
self.assertIsNotNone(re.search(a, html))
a = u'<td class="wrap">short description of '
self.assertIsNotNone(re.search(a, html))
a = u'<td class="wrap"><p>long description of '
self.assertIsNotNone(re.search(a, html))
a = u'<td class="wrap">term1cat, term1cat2</td>'
self.assertIsNotNone(re.search(a, html))
a = u'<td class="wrap">short description of <a tabindex="0" data-toggle="popover" data-trigger="focus" data-html="true" title="Term1" data-content="short description of Term1<br/><br/>関連テーブル: table1, table2<div align=right><a href=\'glossary.html#Term1\' target=\'_glossary\'>詳細…</a></div>"><span class="glossary-term">Term1</span></a></td>'
self.assertIsNotNone(re.search(a, html))
a = u'<td class="wrap"><a href="dqwbtest.public.table1.html" target=_blank >table1</a>, table2</td>'
self.assertIsNotNone(re.search(a, html))
a = u'<td class="wrap">snaga</td>'
self.assertIsNotNone(re.search(a, html))
a = u'<td class="wrap"><a name="Term2" class="nodeco">Term2</a></td>'
self.assertIsNotNone(re.search(a, html))
def test_to_index_html_005(self):
data = """
[
{
"columns": [
],
"database_name": "orcl",
"row_count": 200000,
"schema_name": "public",
"table_name": "t1",
"timestamp": "2016-05-01T16:15:18.028309"
}
]
"""
# index page for the tag 'tag0'
html = DbProfilerFormatter.to_index_html(json.loads(data),
tags=[['tag0', 0]],
files=['aaa.txt'],
reponame='testrepo')
# print(html)
# tag comment with attached file.
self.assertTrue(re.search(u'関連ファイル:', html))
self.assertTrue(
re.search(u'<li><a href="attachments/aaa.txt">aaa.txt</a></li>',
html))
# index page for the schema 'orcl'
html = DbProfilerFormatter.to_index_html(
json.loads(data),
schemas=[['orcl', 'public', 0]],
files=['bbb.txt'],
reponame='testrepo')
# print(html)
# tag comment with attached file.
self.assertTrue(re.search(u'関連ファイル:', html))
self.assertTrue(
re.search(u'<li><a href="attachments/bbb.txt">bbb.txt</a></li>',
html))
def test_get_cardinality_001(self):
self.assertEqual('25.00 %',
DbProfilerFormatter.get_cardinality(100, 25, 0))
self.assertEqual('50.00 %',
DbProfilerFormatter.get_cardinality(100, 25, 50))
self.assertEqual('N/A',
DbProfilerFormatter.get_cardinality(100, 0, 100))
# N/A
self.assertEqual('N/A',
DbProfilerFormatter.get_cardinality(None, 25, 50))
self.assertEqual('N/A',
DbProfilerFormatter.get_cardinality(100, None, 50))
self.assertEqual('N/A',
DbProfilerFormatter.get_cardinality(100, 25, None))
def test_get_value_freq_ratio_001(self):
self.assertEqual('0.00 %',
DbProfilerFormatter.get_value_freq_ratio(100, 0, 0))
self.assertEqual('50.00 %',
DbProfilerFormatter.get_value_freq_ratio(100, 0, 50))
self.assertEqual('100.00 %',
DbProfilerFormatter.get_value_freq_ratio(100, 50, 50))
# empty table
self.assertEqual('0.00 %',
DbProfilerFormatter.get_value_freq_ratio(0, 0, 0))
# N/A
self.assertEqual('N/A',
DbProfilerFormatter.get_value_freq_ratio(None, 0, 0))
self.assertEqual('N/A',
DbProfilerFormatter.get_value_freq_ratio(0, None, 0))
self.assertEqual('N/A',
DbProfilerFormatter.get_value_freq_ratio(0, 0, None))
def test_to_index_html_001(self):
data = """
[
{
"columns": [
{
"cardinality": 199999,
"column_name": "c1",
"data_type": [
"NUMBER",
"22"
],
"max": "200000",
"min": "1",
"nulls": 0,
"validation": {}
},
{
"cardinality": 200000,
"column_name": "c2",
"data_type": [
"NUMBER",
"22"
],
"max": "200000",
"min": "1",
"nulls": 3,
"validation": {}
}
],
"database_name": "orcl",
"row_count": 200000,
"schema_name": "public",
"table_name": "t1",
"timestamp": "2016-05-01T16:15:18.028309",
"comment": "this is table comment. foo PV bar."
}
]
"""
terms = [{'term': 'PV', 'description_short': 'Page Views'}]
html = DbProfilerFormatter.to_index_html(json.loads(data),
glossary_terms=terms,
reponame='testrepo')
self.assertIsNotNone(html)
# print(html)
t = """
<td>public.t1</td>
<td>orcl</td>
<td>public</td>
<td><a href="orcl.public.t1.html" target="_blank">t1</a></td>
<td></td>
<td>200,000</td>
<td>2</td>
"""
self.assertTrue(html.find(t) > 0)
# table comment with tooltip
self.assertIsNotNone(
re.search(
'#t1_comment.* data-toggle="tooltip" data-html="true" data-placement="top" title=" this is table comment. foo PV bar. ">',
html))
# glossary markup in table comment.
a = u'foo <a tabindex="0" data-toggle="popover" data-trigger="focus" data-html="true" title="PV" data-content="Page Views<br/><div align=right><a href=\'glossary.html#PV\' target=\'_glossary\'>詳細…</a></div>"><span class="glossary-term">PV</span></a> bar'
self.assertIsNotNone(re.search(a, html))
def test_coalesce2_001(self):
c = {}
c['foo'] = 'bar'
self.assertEqual('bar', DbProfilerFormatter.coalesce2(c, 'foo', 'baz'))
self.assertEqual('baz', DbProfilerFormatter.coalesce2(c, 'bar', 'baz'))
self.assertEqual('baz', DbProfilerFormatter.coalesce2(c, None, 'baz'))
def test_to_table_html_001(self):
data = """
{
"columns": [
{
"cardinality": 199999,
"column_name": "c1",
"data_type": [
"NUMBER",
"22"
],
"max": "200001",
"min": "1",
"nulls": 0,
"validation": {},
"fk_ref": [ "public.t2.c2" ],
"comment": "this is column comment on c1 with Term2."
},
{
"cardinality": 200000,
"column_name": "c2",
"data_type": [
"NUMBER",
"23"
],
"max": "200000",
"min": "1",
"nulls": 3,
"validation": {},
"fk": [ "?public.t3.c1" ]
}
],
"database_name": "orcl",
"row_count": 200000,
"schema_name": "public",
"table_name": "t1",
"timestamp": "2016-05-01T16:15:18.028309",
"comment": "this is table comment. glossary Term1."
}
"""
terms = [{
'term': u'Term1',
'description_short': u'short description of Term1'
}, {
'term': u'Term2',
'description_short': u'short description of Term2 for c1'
}]
html = DbProfilerFormatter.to_table_html(json.loads(data),
glossary_terms=terms,
files=['aaa.txt'])
self.assertIsNotNone(html)
# print(html)
# column name, data type, min/max values and cardinality
self.assertIsNotNone(
re.search('<tr>.*c1.*NUMBER \(22\).*\[ 1, 200001 \].*100.00 %',
html))
self.assertIsNotNone(
re.search('<tr>.*c2.*NUMBER \(23\).*\[ 1, 200000 \].*100.00 %',
html))
# column comment with tooltip
self.assertIsNotNone(
re.search(
'#c1_comment.* data-toggle="tooltip" data-html="true" data-placement="top" title=" this is column comment on c1 with Term2. ">',
html))
# glossary markup in column comment.
self.assertIsNotNone(
re.search(
u'<a tabindex="0" data-toggle="popover" data-trigger="focus" data-html="true" title="Term2" data-content="short description of Term2 for c1<br/><div align=right><a href=\'glossary.html#Term2\' target=\'_glossary\'>詳細…</a></div>"><span class="glossary-term">Term2</span></a>',
html))
# table comment with attached file
self.assertIsNotNone(re.search('this is table comment.', html))
self.assertIsNotNone(re.search(u'関連ファイル:', html))
self.assertIsNotNone(
re.search(u'<a href="attachments/aaa.txt">aaa.txt</a></li>', html))
# glossary markup in table comment.
self.assertIsNotNone(
re.search(
u'<a tabindex="0" data-toggle="popover" data-trigger="focus" data-html="true" title="Term1" data-content="short description of Term1<br/><div align=right><a href=\'glossary.html#Term1\' target=\'_glossary\'>詳細…</a></div>"><span class="glossary-term">Term1</span></a>.',
html))
# foreign keys
self.assertIsNotNone(
re.search(u'orcl\.public\.t2\.html#c2.*テーブル public.t2 の c2 から参照',
html))
self.assertIsNotNone(
re.search(
u'orcl\.public\.t3\.html#c1.*テーブル public.t3 の c1 を参照（推定）',
html))