def test_very_long_cells(self):
"""
Torture test that long cells do not cause issues
"""
lorem_ipsum_text = textwrap.dedent("""\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec
dignissim, ipsum non facilisis tempus, dui felis tincidunt metus,
nec pulvinar neque odio eget risus. Nulla nisi lectus, cursus
suscipit interdum at, ultrices sit amet orci. Mauris facilisis
imperdiet elit, vitae scelerisque ipsum dignissim non. Integer
consequat malesuada neque sit amet pulvinar. Curabitur pretium
ut turpis eget aliquet. Maecenas sagittis lacus sed lectus
volutpat, eu adipiscing purus pulvinar. Maecenas consequat
luctus urna, eget cursus quam mollis a. Aliquam vitae ornare
erat, non hendrerit urna. Sed eu diam nec massa egestas pharetra
at nec tellus. Fusce feugiat lacus quis urna sollicitudin volutpat.
Quisque at sapien non nibh feugiat tempus ac ultricies purus.
""")
lorem_ipsum_text = lorem_ipsum_text.replace("\n"," ") + "\n\n"
large_lorem_ipsum_text = "".join([lorem_ipsum_text]*3000)
notebook_name = "lorem_ipsum_long.ipynb"
nb = v4.new_notebook(
cells=[
v4.new_markdown_cell(source=large_lorem_ipsum_text)
]
)
with TemporaryDirectory() as td:
nbfile = os.path.join(td, notebook_name)
with open(nbfile, 'w') as f:
write(nb, f, 4)
(output, resources) = LatexExporter(template_file='article').from_filename(nbfile)
assert len(output) > 0
def convert_figure(self, data_format, data):
"""
Convert a single SVG figure to PDF. Returns converted data.
"""
# Work in a temporary directory
with TemporaryDirectory() as tmpdir:
# Write fig to temp file
input_filename = os.path.join(tmpdir, 'figure.svg')
# SVG data is unicode text
with io.open(input_filename, 'w', encoding='utf8') as f:
f.write(cast_unicode_py2(data))
# Call conversion application
output_filename = os.path.join(tmpdir, 'figure.pdf')
shell = self.command.format(from_filename=input_filename,
to_filename=output_filename)
subprocess.call(
shell, shell=True) # Shell=True okay since input is trusted.
# Read output from drive
# return value expects a filename
if os.path.isfile(output_filename):
with open(output_filename, 'rb') as f:
# PDF is a nb supported binary, data type, so base64 encode.
return base64.encodebytes(f.read())
else:
raise TypeError("Inkscape svg to pdf conversion failed")
def test_simple_addition_model(self):
# Tests a case where packaging works correctly and
# the model output matches the expected output
with TemporaryDirectory() as test_dir:
self.package_simple_addition_model(
test_dir, custom_ops=[self.custom_op_path, self.second_custom_op]
)
def test_simple_addition_model_failure(self):
# Tests a case where the output does not match the expected output
with TemporaryDirectory() as test_dir:
with self.assertRaises(ValueError):
self.package_simple_addition_model(
test_dir, do_fail=True, custom_ops=[self.custom_op_path]
)
def test_make_sdist():
# Smoke test of making a complete sdist
builder = sdist.SdistBuilder.from_ini_path(
osp.join(samples_dir, 'package1-pkg.ini'))
with TemporaryDirectory() as td:
builder.build(td)
assert_isfile(osp.join(td, 'package1-0.1.tar.gz'))
def test_python_deps(self):
# Test that we can correctly load two different models with the same dependencies
with TemporaryDirectory() as test_dir1:
with TemporaryDirectory() as test_dir2:
path1 = self.package_sklearn_model(test_dir1)
path2 = self.package_sklearn_model(test_dir2)
with load_neuropod(path1) as n1:
with load_neuropod(path2) as n2:
input_data = {
"x": np.array([[4, 5]], dtype=np.float64)
}
self.assertAlmostEqual(
n1.infer(input_data)["out"][0], 17)
self.assertAlmostEqual(
n2.infer(input_data)["out"][0], 17)
def test_prepare_metadata_for_build_wheel():
hooks = get_hooks('pkg2')
with TemporaryDirectory() as metadatadir:
with modified_env({'PYTHONPATH': BUILDSYS_PKGS}):
hooks.prepare_metadata_for_build_wheel(metadatadir, {})
assert_isfile(pjoin(metadatadir, 'pkg2-0.5.dist-info', 'METADATA'))
def package_simple_addition_model(self, do_fail=False):
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
model_code_dir = os.path.join(test_dir, "model_code")
os.makedirs(model_code_dir)
with open(os.path.join(model_code_dir, "addition_model.py"),
"w") as f:
f.write(ADDITION_MODEL_SOURCE)
# `create_python_neuropod` runs inference with the test data immediately
# after creating the neuropod. Raises a ValueError if the model output
# does not match the expected output.
create_python_neuropod(
neuropod_path=neuropod_path,
model_name="addition_model",
data_paths=[],
code_path_spec=[{
"python_root":
model_code_dir,
"dirs_to_package": [
"" # Package everything in the python_root
],
}],
entrypoint_package="addition_model",
entrypoint="get_model",
# Get the input/output spec along with test data
**get_addition_model_spec(do_fail=do_fail))
# Run some additional checks
check_addition_model(neuropod_path)
def notebook_server():
info = {}
with TemporaryDirectory() as td:
nbdir = info['nbdir'] = pjoin(td, 'notebooks')
os.makedirs(pjoin(nbdir, u'sub ∂ir1', u'sub ∂ir 1a'))
os.makedirs(pjoin(nbdir, u'sub ∂ir2', u'sub ∂ir 1b'))
info['extra_env'] = {
'JUPYTER_CONFIG_DIR': pjoin(td, 'jupyter_config'),
'JUPYTER_RUNTIME_DIR': pjoin(td, 'jupyter_runtime'),
'IPYTHONDIR': pjoin(td, 'ipython'),
}
env = os.environ.copy()
env.update(info['extra_env'])
command = [sys.executable, '-m', 'notebook',
'--no-browser',
'--notebook-dir', nbdir,
# run with a base URL that would be escaped,
# to test that we don't double-escape URLs
'--NotebookApp.base_url=/a@b/',
]
print("command=", command)
proc = info['popen'] = Popen(command, cwd=nbdir, env=env)
info_file_path = pjoin(td, 'jupyter_runtime',
'nbserver-%i.json' % proc.pid)
info.update(_wait_for_server(proc, info_file_path))
print("Notebook server info:", info)
yield info
# Shut the server down
requests.post(urljoin(info['url'], 'api/shutdown'),
headers={'Authorization': 'token '+info['token']})
def package_devices_model(self):
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
# `create_torchscript_neuropod` runs inference with the test data immediately
# after creating the neuropod. Raises a ValueError if the model output
# does not match the expected output.
create_torchscript_neuropod(
neuropod_path=neuropod_path,
model_name="devices_model",
module=DevicesModel(),
input_spec=[
{
"name": "x",
"dtype": "float32",
"shape": (None, )
},
{
"name": "y",
"dtype": "float32",
"shape": (None, )
},
],
output_spec=[
{
"name": "x",
"dtype": "int64",
"shape": (None, )
},
{
"name": "y",
"dtype": "int64",
"shape": (None, )
},
],
test_input_data={
"x": np.arange(5).astype(np.float32),
"y": np.arange(5).astype(np.float32),
},
test_expected_out={
"x": np.array([0], dtype=np.int64),
"y": np.array([1], dtype=np.int64),
},
input_tensor_device={"x": "CPU"},
default_input_tensor_device="GPU",
)
# Ensure all inputs are moved to CPU if we run with no visible GPUs
load_and_test_neuropod(
neuropod_path,
test_input_data={
"x": np.arange(5).astype(np.float32),
"y": np.arange(5).astype(np.float32),
},
test_expected_out={
"x": np.array([0], dtype=np.int64),
"y": np.array([0], dtype=np.int64),
},
neuropod_load_args={"visible_gpu": None},
)
def notebook_server():
info = {}
temp_dir = TemporaryDirectory()
td = temp_dir.name
# do not use context manager because of https://github.com/vatlab/sos-notebook/issues/214
if True:
nbdir = info['nbdir'] = pjoin(td, 'notebooks')
os.makedirs(pjoin(nbdir, u'sub ∂ir1', u'sub ∂ir 1a'))
os.makedirs(pjoin(nbdir, u'sub ∂ir2', u'sub ∂ir 1b'))
# print(nbdir)
info['extra_env'] = {
'JUPYTER_CONFIG_DIR': pjoin(td, 'jupyter_config'),
'JUPYTER_RUNTIME_DIR': pjoin(td, 'jupyter_runtime'),
'IPYTHONDIR': pjoin(td, 'ipython'),
}
env = os.environ.copy()
env.update(info['extra_env'])
command = [
sys.executable,
'-m',
'notebook',
'--no-browser',
'--notebook-dir',
nbdir,
# run with a base URL that would be escaped,
# to test that we don't double-escape URLs
'--NotebookApp.base_url=/a@b/',
]
print("command=", command)
proc = info['popen'] = Popen(command, cwd=nbdir, env=env)
info_file_path = pjoin(td, 'jupyter_runtime',
'nbserver-%i.json' % proc.pid)
info.update(_wait_for_server(proc, info_file_path))
print("Notebook server info:", info)
yield info
# manually try to clean up, which would fail under windows because
# a permission error caused by iPython history.sqlite.
try:
temp_dir.cleanup()
except Exception as ex:
print(ex)
# Shut the server down
requests.post(urljoin(info['url'], 'api/shutdown'),
headers={'Authorization': 'token ' + info['token']})
def test_strings_model(self):
# Tests a case where packaging works correctly and
# the model output matches the expected output
for model in [
StringsModel, StringsModelDictInput, StringsModelListOutput
]:
with TemporaryDirectory() as test_dir:
package_strings_model(test_dir, model=model)
def test_download():
wd = WheelDownloader("astsearch==0.1.2", "3.5.1", 64)
wheel = wd.fetch()
assert_isfile(wheel)
with TemporaryDirectory() as td:
extract_wheel(wheel, target_dir=td)
assert_isfile(pjoin(td, 'astsearch.py'))
def test_build_editable():
with TemporaryDirectory() as td, cwd(osp.join(samples_dir,'pep517')):
filename = buildapi.build_editable(td)
assert filename.endswith('.whl'), filename
assert_isfile(osp.join(td, filename))
assert zipfile.is_zipfile(osp.join(td, filename))
with zipfile.ZipFile(osp.join(td, filename)) as zip:
assert "module1.py" not in zip.namelist()
assert "module1.pth" in zip.namelist()
def compressed_data():
data_path = yatest.common.source_path(os.path.join("catboost", "pytest", "data"))
tmp_dir = TemporaryDirectory()
for file_name in os.listdir(data_path):
if file_name.endswith('.zip'):
with zipfile.ZipFile(os.path.join(data_path, file_name)) as zip_file:
zip_file.extractall(path=tmp_dir.name)
return tmp_dir
def test_build_sdist(mock_pip_install):
with modified_env({'PYTHONPATH': BUILDSYS_PKGS}), \
TemporaryDirectory() as outdir:
filename = build_sdist(pjoin(SAMPLES_DIR, 'pkg1'), outdir)
assert_isfile(pjoin(outdir, filename))
assert tarfile.is_tarfile(pjoin(outdir, filename))
assert mock_pip_install.call_count == 2
assert mock_pip_install.call_args_list[0] == call(['eg_buildsys'])
assert mock_pip_install.call_args_list[1] == call(['frog'])
def test_save_load_test_data(self):
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
os.mkdir(neuropod_path)
test_input = {"x": 3, "y": 4}
test_expected_output = {"out": 7}
save_test_data(neuropod_path, test_input, test_expected_output)
test_data = load_test_data(neuropod_path)
self.assertEquals(test_input, test_data["test_input"])
self.assertEquals(test_expected_output, test_data["test_output"])
def test_build_wheel():
hooks = get_hooks('pkg1')
with TemporaryDirectory() as builddir:
with modified_env({'PYTHONPATH': BUILDSYS_PKGS}):
whl_file = hooks.build_wheel(builddir, {})
assert whl_file.endswith('.whl')
assert os.sep not in whl_file
whl_file = pjoin(builddir, whl_file)
assert_isfile(whl_file)
assert zipfile.is_zipfile(whl_file)
def package_named_tuple_model(self, do_fail=False):
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
# `create_torchscript_neuropod` runs inference with the test data immediately
# after creating the neuropod. Raises a ValueError if the model output
# does not match the expected output.
create_torchscript_neuropod(
neuropod_path=neuropod_path,
model_name="named_tuple_model",
module=NamedTupleModel(),
input_spec=[
{
"name": "x",
"dtype": "float32",
"shape": ("batch_size", )
},
{
"name": "y",
"dtype": "float32",
"shape": ("batch_size", )
},
],
output_spec=[
{
"name": "sum",
"dtype": "float32",
"shape": ("batch_size", )
},
{
"name": "difference",
"dtype": "float32",
"shape": ("batch_size", ),
},
{
"name": "product",
"dtype": "float32",
"shape": ("batch_size", )
},
],
test_input_data={
"x": np.arange(5, dtype=np.float32),
"y": np.arange(5, dtype=np.float32),
},
test_expected_out={
"sum":
np.zeros(5) if do_fail else np.arange(5) + np.arange(5),
"difference":
np.zeros(5) if do_fail else np.arange(5) - np.arange(5),
"product":
np.zeros(5) if do_fail else np.arange(5) * np.arange(5),
},
)
def package_mixed_types_model(self, do_fail=False):
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
# `create_torchscript_neuropod` runs inference with the test data immediately
# after creating the neuropod. Raises a ValueError if the model output
# does not match the expected output.
create_torchscript_neuropod(
neuropod_path=neuropod_path,
model_name="mixed_types_model",
module=MixedReturnTypesModel(),
# Get the input/output spec along with test data
**get_mixed_model_spec(do_fail=do_fail))
def test_prepare_metadata_for_build_wheel_no_fallback():
hooks = get_hooks('pkg2')
with TemporaryDirectory() as metadatadir:
with modified_env({'PYTHONPATH': BUILDSYS_PKGS}):
with pytest.raises(HookMissing) as exc_info:
hooks.prepare_metadata_for_build_wheel(
metadatadir, {}, _allow_fallback=False
)
e = exc_info.value
assert 'prepare_metadata_for_build_wheel' == e.hook_name
assert 'prepare_metadata_for_build_wheel' in str(e)
def test_consistent_hash(self):
# Packages the same model twice and ensures it has the same hash
shas = []
for i in range(2):
with TemporaryDirectory() as test_dir:
self.package_simple_addition_model(
test_dir, custom_ops=[self.custom_op_path, self.second_custom_op]
)
neuropod_path = os.path.join(test_dir, "test_neuropod")
shas.append(sha256sum(neuropod_path))
self.assertEqual(shas[0], shas[1])
def test_stateful_model(self):
# `init_op` can be passed a list of strings or a string
for init_op_name_as_list in [False, True]:
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
self.package_accumulator_model(neuropod_path,
init_op_name_as_list)
neuropod_obj = load_neuropod(neuropod_path)
np.testing.assert_equal(neuropod_obj.name, "accumulator_model")
np.testing.assert_equal(neuropod_obj.platform, "tensorflow")
np.testing.assert_equal(
neuropod_obj.infer({"x": np.float32(2.0)}), {"out": 2.0})
np.testing.assert_equal(
neuropod_obj.infer({"x": np.float32(4.0)}), {"out": 6.0})
def test_issue_104():
hooks = get_hooks('test-for-issue-104')
with TemporaryDirectory() as outdir:
with modified_env({
'PYTHONPATH': BUILDSYS_PKGS,
'PEP517_ISSUE104_OUTDIR': outdir,
}):
hooks.get_requires_for_build_wheel({})
with open(pjoin(outdir, 'out.json')) as f:
children = json.load(f)
assert set(children) <= {
'__init__.py', '__init__.pyc', '_in_process.py', '_in_process.pyc',
'__pycache__',
}
def package_simple_addition_model(self, do_fail=False):
for model in [CustomOpModel]:
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
# `create_torchscript_neuropod` runs inference with the test data immediately
# after creating the neuropod. Raises a ValueError if the model output
# does not match the expected output.
create_torchscript_neuropod(
neuropod_path=neuropod_path,
model_name="addition_model",
module=model(),
custom_ops=[self.custom_op_path, self.second_custom_op],
# Get the input/output spec along with test data
**get_addition_model_spec(do_fail=do_fail))
def test_noncontiguous_array(self):
x = np.arange(16).astype(np.int64).reshape(4, 4)
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
model_code_dir = os.path.join(test_dir, "model_code")
os.makedirs(model_code_dir)
with open(os.path.join(model_code_dir, "splitter_model.py"),
"w") as f:
f.write(NONCONTIGUOUS_MODEL_SOURCE)
create_python_neuropod(
neuropod_path=neuropod_path,
model_name="splitter",
data_paths=[],
code_path_spec=[{
"python_root":
model_code_dir,
"dirs_to_package": [
"" # Package everything in the python_root
],
}],
entrypoint_package="splitter_model",
entrypoint="get_model",
input_spec=[{
"name": "x",
"dtype": "int64",
"shape": (4, 4)
}],
output_spec=[
{
"name": "x1",
"dtype": "int64",
"shape": (4, 2)
},
{
"name": "x2",
"dtype": "int64",
"shape": (4, 2)
},
],
test_input_data={"x": x},
test_expected_out={
"x1": x[:, :2],
"x2": x[:, 2:]
},
)
def test_build_sdist():
hooks = get_hooks('pkg1')
with TemporaryDirectory() as sdistdir:
with modified_env({'PYTHONPATH': BUILDSYS_PKGS}):
sdist = hooks.build_sdist(sdistdir, {})
assert sdist.endswith('.tar.gz')
assert os.sep not in sdist
sdist = pjoin(sdistdir, sdist)
assert_isfile(sdist)
assert tarfile.is_tarfile(sdist)
with tarfile.open(sdist) as tf:
contents = tf.getnames()
assert 'pkg1-0.5/pyproject.toml' in contents
def test_mixed_types_model_failure_duplicate_item(self):
# Tests a model that returns duplicate items across multiple dictionaries
# This is either a CalledProcessError or a RuntimeError depending on whether
# we're using the native bindings or not
with self.assertRaises((CalledProcessError, RuntimeError)):
with TemporaryDirectory() as test_dir:
neuropod_path = os.path.join(test_dir, "test_neuropod")
# `create_torchscript_neuropod` runs inference with the test data immediately
# after creating the neuropod. Raises a ValueError if the model output
# does not match the expected output.
create_torchscript_neuropod(
neuropod_path=neuropod_path,
model_name="mixed_types_model",
module=MixedReturnTypesModelDuplicateItem(),
# Get the input/output spec along with test data
**get_mixed_model_spec())
def test_prepare_bin_dir():
with TemporaryDirectory() as td:
td = Path(td)
commands.prepare_bin_directory(td, cmds)
assert_isfile(td / 'acommand.exe')
script_file = td / 'acommand-script.py'
assert_isfile(script_file)
with script_file.open() as f:
script_contents = f.read()
assert script_contents.startswith("#!python")
assert_in('import extra', script_contents)
assert_in('somefunc()', script_contents)
_rewrite_shebangs.main(['_rewrite_shebangs.py', str(td)])
with script_file.open() as f:
assert f.read().startswith('#!"')
def setUp(self):
self.td = TemporaryDirectory()
self.addCleanup(self.td.cleanup)
self.file_path = os.path.join(self.td.name, 'afile')
with open(self.file_path, 'w') as f:
f.write('Blah')
self.dir_path = os.path.join(self.td.name, 'adir')
os.mkdir(self.dir_path)
self.link_path = os.path.join(self.td.name, 'alink')
if os.name == 'posix':
# Symlinks are rarely usable on Windows, because a special
# permission is needed to create them.
os.symlink(self.file_path, self.link_path)
self.nonexistant_path = os.path.join(self.td.name, 'doesntexist')
