def test_update_yaml(fs: FakeFilesystem) -> None:
"""
Test ``update_yaml``.
"""
path = Path("/path/to/blog/test.yaml")
fs.create_dir(path.parent)
# create new file
with update_yaml(path) as config:
config["foo"] = "bar"
# check contents
with update_yaml(path) as config:
assert config["foo"] == "bar"
# test error
with open(path, "w", encoding="utf-8") as output:
output.write("{{ test }}")
with pytest.raises(yaml.constructor.ConstructorError) as excinfo:
with update_yaml(path) as config:
pass
assert (str(excinfo.value) == """while constructing a mapping
in "<unicode string>", line 1, column 1:
{{ test }}
^
found unhashable key
in "<unicode string>", line 1, column 2:
{{ test }}
^""")
def test_terraform_wrapper(mocker, fs: FakeFilesystem):
mocker.patch("vim_adaptor.terraform.TerraformWrapper.init")
fs.create_file(
"/my-template-dir/template.tf",
contents=("Context variable: {{ my_var }}"),
)
terraform = TerraformWrapper(
workdir=Path("/workdir"),
templates=[Path("/my-template-dir/template.tf")],
context={"my_var": "value"},
tf_vars={},
)
# Template(s) should have been compiled
assert terraform.workdir.exists()
rendered_template_file: Path = terraform.workdir / "template.tf"
assert rendered_template_file.exists()
with rendered_template_file.open() as f:
assert "Context variable: value" == f.read()
# `init()` should have been called
terraform.init.assert_called()
# Remove working directory
terraform.remove_workdir()
assert not rendered_template_file.exists()
assert not terraform.workdir.exists()
def test_find_modified_files(
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test ``find_modified_files``.
"""
publisher = Publisher(root, config, "generic")
fs.create_dir(root / "build/generic")
fs.create_dir(root / "build/generic/subdir")
with freeze_time("2021-01-01T00:00:00Z"):
(root / "build/generic/one").touch()
with freeze_time("2021-01-02T00:00:00Z"):
(root / "build/generic/subdir/two").touch()
since = datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
modified_files = list(
publisher.find_modified_files(force=False, since=since))
assert modified_files == [Path("/path/to/blog/build/generic/subdir/two")]
since = datetime(2021, 1, 2, 12, 0, 0, tzinfo=timezone.utc)
modified_files = list(
publisher.find_modified_files(force=False, since=since))
assert modified_files == []
since = datetime(2021, 1, 2, 12, 0, 0, tzinfo=timezone.utc)
modified_files = list(
publisher.find_modified_files(force=True, since=since))
assert modified_files == [
Path("/path/to/blog/build/generic/one"),
Path("/path/to/blog/build/generic/subdir/two"),
]
def config(fs: FakeFilesystem, root: Path) -> Iterator[Config]:
"""
Create configuration file.
"""
fs.create_file(root / CONFIG_FILENAME, contents=yaml.dump(CONFIG))
yield Config(**CONFIG)
async def test_publish(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test ``publish``.
"""
FTP = mocker.patch("nefelibata.publishers.ftp.FTP")
ftp = FTP.return_value.__enter__.return_value
mocker.patch.object(
FTPPublisher,
"find_modified_files",
return_value=iter([root / "build/generic/one"]),
)
fs.create_file(root / "build/generic/one", contents="Hello, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
)
await publisher.publish()
FTP.assert_called_with("ftp.example.com", "username", "password")
ftp.pwd.assert_called_with()
ftp.storbinary.assert_called() # second argument is an internal object
async def test_unmount_error(
gstate: GlobalState,
fs: fake_filesystem.FakeFilesystem,
mocker: MockerFixture,
) -> None:
async def mock_call(
cmd: List[str],
) -> Tuple[int, Optional[str], Optional[str]]:
raise Exception("Failed unmount.")
mocker.patch(
"gravel.controllers.nodes.systemdisk.aqr_run_cmd", new=mock_call
)
from gravel.controllers.nodes.systemdisk import MountError, SystemDisk
systemdisk = SystemDisk(gstate)
throws = False
try:
await systemdisk.unmount()
except MountError as e:
assert "does not exist" in e.message
throws = True
assert throws
fs.create_dir("/var/lib/aquarium-system")
throws = False
try:
await systemdisk.unmount()
except MountError as e:
assert "failed unmount" in e.message.lower()
throws = True
assert throws
async def test_publish_create_directory(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test ``publish`` when the directories need to be created.
"""
FTP = mocker.patch("nefelibata.publishers.ftp.FTP")
ftp = FTP.return_value.__enter__.return_value
ftp.cwd.side_effect = [ftplib.error_perm, ""]
mocker.patch.object(
FTPPublisher,
"find_modified_files",
return_value=iter([root / "build/generic/subdir1/one"]),
)
fs.create_file(root / "build/generic/subdir1/one", contents="Hello, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
)
await publisher.publish()
ftp.mkd.assert_called_with("subdir1")
def test_model_status(app: Flask, fs: FakeFilesystem) -> None:
fs.add_real_directory('./testdata/test_model/test_instance')
fs.add_real_directory('./testdata/test_model/test_instance_unreleased')
client = app.test_client()
with app.test_request_context():
resp = client.get('/models?model=test_model',
content_type='application/json')
assert resp.status == '200 OK'
result = json.loads(resp.data)
assert result
assert result['instances'] == ['test_instance', 'test_instance_unreleased']
assert result['preferred'] == 'test_instance'
# Pick two random test topics to assert
assert result['topics']['Asylum-seekers - refugees'] == {
'name': 'Asylum-seekers - refugees',
'quality': 1.0,
'samples': 260
}
assert result['topics']['National Human Rights Institution'] == {
'name': 'National Human Rights Institution',
'quality': 1.0,
'samples': 552
}
async def test_publish_last_published(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test publishing when ``last_published`` is present.
"""
FTP = mocker.patch("nefelibata.publishers.ftp.FTP")
ftp = FTP.return_value.__enter__.return_value
with freeze_time("2021-01-01T00:00:00Z"):
fs.create_file(root / "build/generic/one", contents="Hello, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
)
await publisher.publish(since=datetime(2021, 1, 2, tzinfo=timezone.utc))
ftp.storbinary.assert_not_called()
await publisher.publish(force=True)
ftp.storbinary.assert_called()
def test_model_status(app: Flask, fs: FakeFilesystem) -> None:
fs.add_real_directory('./testdata/test_model/test_instance')
fs.add_real_directory('./testdata/test_model/test_instance_unreleased')
client = app.test_client()
with app.test_request_context():
resp = client.get('/models?model=test_model',
content_type='application/json')
assert resp.status == '200 OK'
result = json.loads(resp.data)
assert result
assert result['instances'] == ['test_instance', 'test_instance_unreleased']
assert result['preferred'] == 'test_instance'
# Pick random test topics to assert
assert result['topics']['Poverty'] == {
'name': 'Poverty',
'quality': 0.81,
'samples': 178
}
# Pick topic with not enough samples for threshold optimization
assert result['topics']['nan'] == {
'name': 'nan',
'samples': 0,
'quality': 0.0
}
async def test_publish_tls(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test ``publish`` with TLS.
"""
FTP_TLS = mocker.patch("nefelibata.publishers.ftp.FTP_TLS")
ftp = FTP_TLS.return_value.__enter__.return_value
mocker.patch.object(
FTPPublisher,
"find_modified_files",
return_value=iter([root / "build/generic/one"]),
)
fs.create_file(root / "build/generic/one", contents="Hello, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
use_tls=True,
)
await publisher.publish()
ftp.prot_p.assert_called()
async def test_mount_error(
gstate: GlobalState,
fs: fake_filesystem.FakeFilesystem,
mocker: MockerFixture,
) -> None:
async def mock_call(
cmd: List[str],
) -> Tuple[int, Optional[str], Optional[str]]:
raise Exception("Failed mount.")
mocker.patch(
"gravel.controllers.nodes.systemdisk.aqr_run_cmd", new=mock_call
)
from gravel.controllers.nodes.systemdisk import MountError, SystemDisk
systemdisk = SystemDisk(gstate)
asserted = False
try:
await systemdisk.mount()
except AssertionError:
asserted = True
assert asserted
fs.create_file("/dev/mapper/aquarium-systemdisk")
throws = False
try:
await systemdisk.mount()
except MountError:
throws = True
assert throws
assert fs.exists("/var/lib/aquarium-system")
def test_preferred_instance(self, fs: FakeFilesystem) -> None:
model = 'test_model'
fs.add_real_directory('./testdata/test_model/test_instance')
fs.add_real_directory('./testdata/test_model/test_instance_unreleased')
s = ModelStatus(self.BASE_CLASSIFIER_PATH, model)
result = s.get_preferred_model_instance()
assert result == 'test_instance'
def test_missing_labels_file(self, fs: FakeFilesystem) -> None:
fs.add_real_directory('./testdata/test_model/test_instance')
fs.remove_object('./testdata/test_model/test_instance/label.vocab')
with pytest.raises(
Exception,
match=(r'Failure to load labels file from {0} with exception').
format('./testdata/test_model/test_instance/label.vocab')):
Classifier(self.BASE_CLASSIFIER_PATH, 'test_model')
def test_missing_instance_dir(self, fs: FakeFilesystem) -> None:
fs.add_real_directory('./testdata/test_model/test_instance_unreleased')
model_path = os.path.join(self.BASE_CLASSIFIER_PATH, 'test_model')
with pytest.raises(Exception,
match=('No valid instance of model found in %s, ' +
'instances were %s') %
(model_path, r'\[\'test_instance_unreleased\'\]')):
Classifier(self.BASE_CLASSIFIER_PATH, 'test_model')
def test_ctor(gstate: GlobalState, fs: fake_filesystem.FakeFilesystem) -> None:
NodeMgr(gstate)
assert fs.exists("/etc/aquarium/node.json")
# clean up
for f in fs.listdir("/etc/aquarium"):
fs.remove(f"/etc/aquarium/{f}")
def test_missing_model(self, fs: FakeFilesystem) -> None:
instance_path = os.path.join(self.BASE_CLASSIFIER_PATH, 'test_model',
'test_instance_missing_model')
fs.add_real_directory(instance_path)
with pytest.raises(Exception,
match=('SavedModel file does not exist at: {0}'
).format(instance_path)):
Classifier(self.BASE_CLASSIFIER_PATH, 'test_model')
def test_processes_given_clip(mocker: MockFixture, fs: FakeFilesystem) -> None:
fs.create_file("testclip.mp4")
mock_abridge_clip = mocker.patch("abridge.processor.abridge_clip")
sys.argv = ["", "testclip.mp4"]
main()
mock_abridge_clip.assert_called_with("testclip.mp4", "processed", 20, 5)
def test_provides_thresholds(mocker: MockFixture, fs: FakeFilesystem) -> None:
fs.create_file("testclip.mp4")
mock_abridge_clip = mocker.patch("abridge.processor.abridge_clip")
sys.argv = ["", "-r", "10", "-t", "30", "testclip.mp4"]
main()
mock_abridge_clip.assert_called_with("testclip.mp4", "processed", 30, 10)
def test_missing_variables(self, fs: FakeFilesystem) -> None:
instance_path = os.path.join(self.BASE_CLASSIFIER_PATH, 'test_model',
'test_instance_missing_variables')
fs.add_real_directory(instance_path)
with pytest.raises(
Exception,
match=('{0}/variables; No such file or directory'.format(
instance_path))):
Classifier(self.BASE_CLASSIFIER_PATH, 'test_model')
def test_threadpool_executes_clips(fs: FakeFilesystem,
mock_threadpool: MagicMock) -> None:
fs.create_file("somefile.mp4")
sys.argv = ["", "somefile.mp4"]
main()
mock_threadpool().submit.assert_called_with(abridge_clip, "somefile.mp4",
"processed", 20, 5)
def ceph_conf_file_fs(request: SubRequest, fs: fake_filesystem.FakeFilesystem):
""" This fixture uses pyfakefs to stub filesystem calls and return
any files created with the parent `fs` fixture. """
fs.add_real_file( # pyright: reportUnknownMemberType=false
os.path.join(
DATA_DIR,
request.param # pyright: reportUnknownArgumentType=false
),
target_path='/etc/ceph/ceph.conf')
yield fs
def post(fs: FakeFilesystem, root: Path, config: Config) -> Iterator[Post]:
"""
Create a post.
"""
post_directory = root / "posts/first"
post_path = post_directory / "index.mkd"
with freeze_time("2021-01-01T00:00:00Z"):
fs.create_file(post_path, contents=POST_CONTENT)
post = build_post(root, config, post_path)
yield post
async def test_build_post(fs: FakeFilesystem, root: Path,
config: Config) -> None:
"""
Test ``build_post``.
"""
path = Path(root / "posts/first/index.mkd")
# create post
with freeze_time("2021-01-01T00:00:00Z"):
fs.create_file(path, contents=POST_CONTENT)
local_date = formatdate(1609459200.0, localtime=True)
config.announcers = {"antenna": AnnouncerModel(plugin="antenna")}
post = build_post(root, config, path)
assert post.path == path
assert post.title == "This is your first post"
assert post.timestamp == datetime(2021, 1, 1, 0, 0, tzinfo=timezone.utc)
assert post.metadata == {
"keywords": "welcome, blog",
"summary": "Hello, world!",
}
assert post.tags == {"welcome", "blog"}
assert post.categories == {"stem"}
assert post.type == "post"
assert post.url == "first/index"
assert (post.content == """# Welcome #
This is your first post. It should be written using Markdown.
Read more about [Nefelibata](https://nefelibata.readthedocs.io/).""")
assert post.announcers == {"antenna"}
# check to file was updated with the ``date`` header
with open(path, encoding="utf-8") as input_:
content = input_.read()
assert (content == f"""subject: This is your first post
keywords: welcome, blog
summary: Hello, world!
date: {local_date}
announce-on: antenna
# Welcome #
This is your first post. It should be written using Markdown.
Read more about [Nefelibata](https://nefelibata.readthedocs.io/).""")
# build again, and test that the file wasn't modified since it
# already has all the required headers
last_update = path.stat().st_mtime
build_post(root, config, path)
assert path.stat().st_mtime == last_update
def test_threadpool_sets_workers(mocker: MockFixture, fs: FakeFilesystem,
mock_threadpool: MagicMock) -> None:
mocker.patch("abridge.processor.abridge_clip", autospec=True)
fs.create_file("somefile.mp4")
sys.argv = ["", "-w", "5", "somefile.mp4"]
main()
mock_threadpool.assert_called_with(max_workers=5)
def test_makes_output_directory(mocker: MockFixture,
fs: FakeFilesystem) -> None:
fs.create_file("testclip.mp4")
mocker.patch("abridge.processor.abridge_clip")
sys.argv = ["", "testclip.mp4", "-o", "outputdir"]
main()
os.path.isdir("outputdir")
def test_not_mounted(fs: fake_filesystem.FakeFilesystem) -> None:
fs.add_real_file(
source_path=os.path.join(DATA_DIR, "mounts_without_aquarium.raw"),
target_path="/proc/mounts",
)
from gravel.controllers.nodes.systemdisk import SystemDisk
systemdisk = SystemDisk()
assert not systemdisk.mounted
def test_exits_when_one_clip_doesnt_exist(mocker: MockFixture,
fs: FakeFilesystem) -> None:
fs.create_file("exists.mp4")
mocker.patch("abridge.processor.abridge_clip")
sys_spy = mocker.spy(abridge.cli, "sys")
sys.argv = ["", "exists.mp4", "something.mp4"]
main()
sys_spy.exit.assert_called_with("something.mp4: No such file or directory")
def test_instances(self, fs: FakeFilesystem) -> None:
model = 'test_model'
fs.add_real_directory(path.join(self.BASE_CLASSIFIER_PATH, model))
s = ModelStatus(self.BASE_CLASSIFIER_PATH, model_name=model)
result = s.list_model_instances()
assert result == [
'test_instance', 'test_instance_missing_model',
'test_instance_missing_variables',
'test_instance_missing_variables_data',
'test_instance_missing_variables_index', 'test_instance_unreleased'
]
def test_silly_mounts(fs: fake_filesystem.FakeFilesystem) -> None:
fs.add_real_file(
source_path=os.path.join(DATA_DIR, "mounts_silly.raw"),
target_path="/proc/mounts",
)
from gravel.controllers.nodes.systemdisk import MountEntry, get_mounts
lst: List[MountEntry] = get_mounts()
assert len(lst) == 2
assert lst[0].source == "foo" and lst[0].dest == "bar"
assert lst[1].source == "asd" and lst[1].dest == "fgh"
