def test_write(self):
_, parser_config = self.get_test_objects()
output_dir = googletest.GetTempDir()
generate_lib.write_docs(output_dir, parser_config, yaml_toc=True)
# Check redirects
redirects_file = os.path.join(output_dir, '_redirects.yaml')
self.assertTrue(os.path.exists(redirects_file))
with open(redirects_file) as f:
redirects = f.read()
self.assertEqual(redirects.split(), [
'redirects:', '-', 'from:', '/api_docs/python/tf/test_function',
'to:', '/api_docs/python/tf/TestModule/test_function'
])
# Make sure that the right files are written to disk.
self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, 'tf.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, '_toc.yaml')))
self.assertTrue(
os.path.exists(os.path.join(output_dir, 'tf/TestModule.md')))
self.assertFalse(
os.path.exists(os.path.join(output_dir, 'tf/test_function.md')))
self.assertTrue(
os.path.exists(
os.path.join(output_dir, 'tf/TestModule/TestClass.md')))
self.assertTrue(
os.path.exists(
os.path.join(output_dir,
'tf/TestModule/TestClass/ChildClass.md')))
self.assertTrue(
os.path.exists(
os.path.join(
output_dir,
'tf/TestModule/TestClass/ChildClass/GrandChildClass.md')))
# Make sure that duplicates are not written
self.assertTrue(
os.path.exists(
os.path.join(output_dir, 'tf/TestModule/test_function.md')))
def __init__(self,
num_workers,
dispatcher_port=0,
work_dir=TMP_WORK_DIR,
fault_tolerant_mode=True,
job_gc_check_interval_ms=None,
job_gc_timeout_ms=None,
start=True):
"""Creates a tf.data service test cluster.
Args:
num_workers: The number of workers to initially add to the cluster.
dispatcher_port: The port to use for the dispatcher.
work_dir: The work directory to use for the dispatcher. If set to
`TMP_WORK_DIR`, the cluster will create a new temporary directory to use
as the work directory. If set to `NO_WORK_DIR`, no work directory will
be used.
fault_tolerant_mode: Whether the dispatcher should write its state to a
journal so that it can recover from restarts.
job_gc_check_interval_ms: How often the dispatcher should scan through to
delete old and unused jobs, in milliseconds.
job_gc_timeout_ms: How long a job needs to be unused before it becomes a
candidate for garbage collection, in milliseconds.
start: Whether to immediately start the servers in the cluster. If
`False`, the servers can be started later by calling
`start_dispatcher()` and `start_workers()`.
"""
if work_dir == TMP_WORK_DIR:
work_dir = tempfile.mkdtemp(dir=googletest.GetTempDir())
self.dispatcher = server_lib.DispatchServer(
server_lib.DispatcherConfig(
port=dispatcher_port,
work_dir=work_dir,
fault_tolerant_mode=fault_tolerant_mode,
job_gc_check_interval_ms=job_gc_check_interval_ms,
job_gc_timeout_ms=job_gc_timeout_ms),
start=start)
self.workers = []
for _ in range(num_workers):
self.add_worker(start=start)
def save_and_load_benchmark(app):
"""Util for saved model benchmarks."""
trials = 3
model = app(weights=None)
model_name = app.__name__
tmp_dir = googletest.GetTempDir()
gfile.MakeDirs(tmp_dir)
save_dir = tempfile.mkdtemp(dir=tmp_dir)
total_save_time = 0
total_load_time = 0
# Run one untimed iteration of saving/loading.
model.save(save_dir, save_format='tf')
tf.keras.models.load_model(save_dir)
for _ in range(trials):
start_time = time.time()
model.save(save_dir, save_format='tf')
total_save_time += time.time() - start_time
start_time = time.time()
tf.keras.models.load_model(save_dir)
total_load_time += time.time() - start_time
save_result = {
'iters': trials,
'wall_time': total_save_time / trials,
'name': '{}.save'.format(model_name)
}
load_result = {
'iters': trials,
'wall_time': total_load_time / trials,
'name': '{}.load'.format(model_name)
}
gfile.DeleteRecursively(save_dir)
return save_result, load_result
def test_end_to_end(self):
generate2.MIN_NUM_FILES_EXPECTED = 1
output_dir = pathlib.Path(googletest.GetTempDir())/'output'
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
generate2.build_docs(
output_dir=output_dir,
code_url_prefix='',
search_hints=True,
)
raw_ops_page = (output_dir/'tf/raw_ops.md').read_text()
self.assertIn('/tf/raw_ops/Add.md', raw_ops_page)
toc = yaml.safe_load((output_dir / 'tf/_toc.yaml').read_text())
self.assertEqual({
'title': 'Overview',
'path': '/tf_overview'
}, toc['toc'][0]['section'][0])
redirects = yaml.safe_load((output_dir / 'tf/_redirects.yaml').read_text())
self.assertIn({'from': '/tf_overview', 'to': '/tf'}, redirects['redirects'])
def test_update_id_tags_inplace(self):
test_dir = googletest.GetTempDir()
test_sub_dir = os.path.join(test_dir, 'a/b')
os.makedirs(test_sub_dir)
test_path1 = os.path.join(test_dir, 'file1.md')
test_path2 = os.path.join(test_sub_dir, 'file2.md')
test_path3 = os.path.join(test_sub_dir, 'file3.notmd')
with open(test_path1, 'w') as f:
f.write('## abc&123')
with open(test_path2, 'w') as f:
f.write('# A Level 1 Heading\n')
f.write('## A Level 2 Heading')
with open(test_path3, 'w') as f:
f.write("## don\'t change this")
generate_lib.update_id_tags_inplace(test_dir)
with open(test_path1) as f:
content = f.read()
self.assertEqual(content, '<h2 id="abc_123">abc&123</h2>')
with open(test_path2) as f:
content = f.read()
self.assertEqual(
content, '# A Level 1 Heading\n'
'<h2 id="A_Level_2_Heading">A Level 2 Heading</h2>')
with open(test_path3) as f:
content = f.read()
self.assertEqual(content, "## don\'t change this")
class Flags(object):
resource_root = resource_loader.get_root_dir_with_all_resources()
src_dir = os.path.join(resource_root, 'tensorflow/docs_src')
base_dir = os.path.join(resource_root, 'tensorflow/')
output_dir = googletest.GetTempDir()
def get_temp_dir(self):
if not self._tempdir:
self._tempdir = googletest.GetTempDir()
return self._tempdir
def test_write(self):
module = sys.modules[__name__]
index = {
'tf':
sys, # Can be any module, this test doesn't care about content.
'tf.TestModule':
module,
'tf.test_function':
test_function,
'tf.TestModule.test_function':
test_function,
'tf.TestModule.TestClass':
TestClass,
'tf.TestModule.TestClass.ChildClass':
TestClass.ChildClass,
'tf.TestModule.TestClass.ChildClass.GrandChildClass':
TestClass.ChildClass.GrandChildClass,
}
tree = {
'tf': ['TestModule', 'test_function'],
'tf.TestModule': ['test_function', 'TestClass'],
'tf.TestModule.TestClass': ['ChildClass'],
'tf.TestModule.TestClass.ChildClass': ['GrandChildClass'],
'tf.TestModule.TestClass.ChildClass.GrandChildClass': []
}
duplicate_of = {'tf.test_function': 'tf.TestModule.test_function'}
duplicates = {
'tf.TestModule.test_function':
['tf.test_function', 'tf.TestModule.test_function']
}
base_dir = os.path.dirname(__file__)
visitor = DummyVisitor(index, duplicate_of)
reference_resolver = parser.ReferenceResolver.from_visitor(
visitor=visitor, doc_index={}, py_module_names=['tf'])
parser_config = parser.ParserConfig(
reference_resolver=reference_resolver,
duplicates=duplicates,
duplicate_of=duplicate_of,
tree=tree,
index=index,
reverse_index={},
guide_index={},
base_dir=base_dir)
output_dir = googletest.GetTempDir()
generate_lib.write_docs(output_dir, parser_config, yaml_toc=True)
# Make sure that the right files are written to disk.
self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, 'tf.md')))
self.assertTrue(os.path.exists(os.path.join(output_dir, '_toc.yaml')))
self.assertTrue(
os.path.exists(os.path.join(output_dir, 'tf/TestModule.md')))
self.assertFalse(
os.path.exists(os.path.join(output_dir, 'tf/test_function.md')))
self.assertTrue(
os.path.exists(
os.path.join(output_dir, 'tf/TestModule/TestClass.md')))
self.assertTrue(
os.path.exists(
os.path.join(output_dir,
'tf/TestModule/TestClass/ChildClass.md')))
self.assertTrue(
os.path.exists(
os.path.join(
output_dir,
'tf/TestModule/TestClass/ChildClass/GrandChildClass.md')))
# Make sure that duplicates are not written
self.assertTrue(
os.path.exists(
os.path.join(output_dir, 'tf/TestModule/test_function.md')))
def test_end_to_end(self):
output_dir = os.path.join(googletest.GetTempDir(), 'output')
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
generate2.build_docs(output_dir=output_dir, code_url_prefix='')
def get_temp_dir(self):
if not self._tempdir:
self._tempdir = tempfile.mkdtemp(dir=googletest.GetTempDir())
return self._tempdir
"""Tests for functions."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.ops import data_flow_ops
from tensorflow.python.ops import io_ops
from tensorflow.python.platform import test, gfile, googletest
from tensorflow.python.util import compat
import os
os.environ["IPTF_OFFLINE"] = "1"
os.environ["IPTF_PATH"] = googletest.GetTempDir()
import iptf
import json
class FileSystemTest(test.TestCase):
def testReadWrite(self):
with self.test_session() as sess:
contents = "ASDASDASDASDASDAS"
filename = "iptf://repo/root/foo"
meta_filename = "iptf://meta/repo/root/foo"
wf = io_ops.write_file(filename=constant_op.constant(filename),
contents=constant_op.constant(contents))
reader = io_ops.WholeFileReader("test_reader")
def test_replace_refes(self):
test_dir = googletest.GetTempDir()
test_in_dir = os.path.join(test_dir, 'in')
test_in_dir_a = os.path.join(test_dir, 'in/a')
test_in_dir_b = os.path.join(test_dir, 'in/b')
os.makedirs(test_in_dir)
os.makedirs(test_in_dir_a)
os.makedirs(test_in_dir_b)
test_out_dir = os.path.join(test_dir, 'out')
os.makedirs(test_out_dir)
test_path1 = os.path.join(test_in_dir_a, 'file1.md')
test_path2 = os.path.join(test_in_dir_b, 'file2.md')
test_path3 = os.path.join(test_in_dir_b, 'file3.notmd')
test_path4 = os.path.join(test_in_dir_b, 'OWNERS')
with open(test_path1, 'w') as f:
f.write('Use `tf.test_function` to test things.')
with open(test_path2, 'w') as f:
f.write(
'Use @{tf.TestModule.TestClass.ChildClass} to test things.\n'
"`tf.whatever` doesn't exist")
with open(test_path3, 'w') as f:
file3_content = (
'Not a .md file. Should be copied unchanged:'
'@{tf.TestModule.TestClass.ChildClass}, `tf.test_function`')
f.write(file3_content)
with open(test_path4, 'w') as f:
f.write('')
reference_resolver, _ = self.get_test_objects()
generate_lib.replace_refs(test_in_dir, test_out_dir,
reference_resolver, '*.md')
with open(os.path.join(test_out_dir, 'a/file1.md')) as f:
content = f.read()
self.assertEqual(
content,
'Use <a href="../api_docs/python/tf/TestModule/test_function.md">'
'<code>tf.test_function</code></a> to test things.')
with open(os.path.join(test_out_dir, 'b/file2.md')) as f:
content = f.read()
self.assertEqual(
content, 'Use '
'<a href="../api_docs/python/tf/TestModule/TestClass/ChildClass.md">'
'<code>tf.TestModule.TestClass.ChildClass</code></a> '
'to test things.\n'
'`tf.whatever` doesn\'t exist')
with open(os.path.join(test_out_dir, 'b/file3.notmd')) as f:
content = f.read()
self.assertEqual(content, file3_content)
with self.assertRaises(IOError):
# This should fail. The OWNERS file should not be copied
with open(os.path.join(test_out_dir, 'b/OWNERS')) as f:
content = f.read()
def _start_dispatcher(self, dispatcher_port):
work_dir = tempfile.mkdtemp(dir=googletest.GetTempDir())
self._dispatcher = server_lib.DispatchServer(
server_lib.DispatcherConfig(
port=dispatcher_port, work_dir=work_dir, protocol="grpc"),
start=True)
