async def test_publish_tls(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test ``publish`` with TLS.
"""
FTP_TLS = mocker.patch("nefelibata.publishers.ftp.FTP_TLS")
ftp = FTP_TLS.return_value.__enter__.return_value
mocker.patch.object(
FTPPublisher,
"find_modified_files",
return_value=iter([root / "build/generic/one"]),
)
fs.create_file(root / "build/generic/one", contents="Hello, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
use_tls=True,
)
await publisher.publish()
ftp.prot_p.assert_called()
async def test_publish_create_directory(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test ``publish`` when the directories need to be created.
"""
FTP = mocker.patch("nefelibata.publishers.ftp.FTP")
ftp = FTP.return_value.__enter__.return_value
ftp.cwd.side_effect = [ftplib.error_perm, ""]
mocker.patch.object(
FTPPublisher,
"find_modified_files",
return_value=iter([root / "build/generic/subdir1/one"]),
)
fs.create_file(root / "build/generic/subdir1/one", contents="Hello, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
)
await publisher.publish()
ftp.mkd.assert_called_with("subdir1")
def test_terraform_wrapper(mocker, fs: FakeFilesystem):
mocker.patch("vim_adaptor.terraform.TerraformWrapper.init")
fs.create_file(
"/my-template-dir/template.tf",
contents=("Context variable: {{ my_var }}"),
)
terraform = TerraformWrapper(
workdir=Path("/workdir"),
templates=[Path("/my-template-dir/template.tf")],
context={"my_var": "value"},
tf_vars={},
)
# Template(s) should have been compiled
assert terraform.workdir.exists()
rendered_template_file: Path = terraform.workdir / "template.tf"
assert rendered_template_file.exists()
with rendered_template_file.open() as f:
assert "Context variable: value" == f.read()
# `init()` should have been called
terraform.init.assert_called()
# Remove working directory
terraform.remove_workdir()
assert not rendered_template_file.exists()
assert not terraform.workdir.exists()
def config(fs: FakeFilesystem, root: Path) -> Iterator[Config]:
"""
Create configuration file.
"""
fs.create_file(root / CONFIG_FILENAME, contents=yaml.dump(CONFIG))
yield Config(**CONFIG)
async def test_mount_error(
gstate: GlobalState,
fs: fake_filesystem.FakeFilesystem,
mocker: MockerFixture,
) -> None:
async def mock_call(
cmd: List[str],
) -> Tuple[int, Optional[str], Optional[str]]:
raise Exception("Failed mount.")
mocker.patch(
"gravel.controllers.nodes.systemdisk.aqr_run_cmd", new=mock_call
)
from gravel.controllers.nodes.systemdisk import MountError, SystemDisk
systemdisk = SystemDisk(gstate)
asserted = False
try:
await systemdisk.mount()
except AssertionError:
asserted = True
assert asserted
fs.create_file("/dev/mapper/aquarium-systemdisk")
throws = False
try:
await systemdisk.mount()
except MountError:
throws = True
assert throws
assert fs.exists("/var/lib/aquarium-system")
async def test_publish(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test ``publish``.
"""
FTP = mocker.patch("nefelibata.publishers.ftp.FTP")
ftp = FTP.return_value.__enter__.return_value
mocker.patch.object(
FTPPublisher,
"find_modified_files",
return_value=iter([root / "build/generic/one"]),
)
fs.create_file(root / "build/generic/one", contents="Hello, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
)
await publisher.publish()
FTP.assert_called_with("ftp.example.com", "username", "password")
ftp.pwd.assert_called_with()
ftp.storbinary.assert_called() # second argument is an internal object
async def test_publish_last_published(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test publishing when ``last_published`` is present.
"""
FTP = mocker.patch("nefelibata.publishers.ftp.FTP")
ftp = FTP.return_value.__enter__.return_value
with freeze_time("2021-01-01T00:00:00Z"):
fs.create_file(root / "build/generic/one", contents="Hello, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
)
await publisher.publish(since=datetime(2021, 1, 2, tzinfo=timezone.utc))
ftp.storbinary.assert_not_called()
await publisher.publish(force=True)
ftp.storbinary.assert_called()
def test_provides_thresholds(mocker: MockFixture, fs: FakeFilesystem) -> None:
fs.create_file("testclip.mp4")
mock_abridge_clip = mocker.patch("abridge.processor.abridge_clip")
sys.argv = ["", "-r", "10", "-t", "30", "testclip.mp4"]
main()
mock_abridge_clip.assert_called_with("testclip.mp4", "processed", 30, 10)
def test_processes_given_clip(mocker: MockFixture, fs: FakeFilesystem) -> None:
fs.create_file("testclip.mp4")
mock_abridge_clip = mocker.patch("abridge.processor.abridge_clip")
sys.argv = ["", "testclip.mp4"]
main()
mock_abridge_clip.assert_called_with("testclip.mp4", "processed", 20, 5)
def test_threadpool_executes_clips(fs: FakeFilesystem,
mock_threadpool: MagicMock) -> None:
fs.create_file("somefile.mp4")
sys.argv = ["", "somefile.mp4"]
main()
mock_threadpool().submit.assert_called_with(abridge_clip, "somefile.mp4",
"processed", 20, 5)
def test_threadpool_sets_workers(mocker: MockFixture, fs: FakeFilesystem,
mock_threadpool: MagicMock) -> None:
mocker.patch("abridge.processor.abridge_clip", autospec=True)
fs.create_file("somefile.mp4")
sys.argv = ["", "-w", "5", "somefile.mp4"]
main()
mock_threadpool.assert_called_with(max_workers=5)
async def test_build_post(fs: FakeFilesystem, root: Path,
config: Config) -> None:
"""
Test ``build_post``.
"""
path = Path(root / "posts/first/index.mkd")
# create post
with freeze_time("2021-01-01T00:00:00Z"):
fs.create_file(path, contents=POST_CONTENT)
local_date = formatdate(1609459200.0, localtime=True)
config.announcers = {"antenna": AnnouncerModel(plugin="antenna")}
post = build_post(root, config, path)
assert post.path == path
assert post.title == "This is your first post"
assert post.timestamp == datetime(2021, 1, 1, 0, 0, tzinfo=timezone.utc)
assert post.metadata == {
"keywords": "welcome, blog",
"summary": "Hello, world!",
}
assert post.tags == {"welcome", "blog"}
assert post.categories == {"stem"}
assert post.type == "post"
assert post.url == "first/index"
assert (post.content == """# Welcome #
This is your first post. It should be written using Markdown.
Read more about [Nefelibata](https://nefelibata.readthedocs.io/).""")
assert post.announcers == {"antenna"}
# check to file was updated with the ``date`` header
with open(path, encoding="utf-8") as input_:
content = input_.read()
assert (content == f"""subject: This is your first post
keywords: welcome, blog
summary: Hello, world!
date: {local_date}
announce-on: antenna
# Welcome #
This is your first post. It should be written using Markdown.
Read more about [Nefelibata](https://nefelibata.readthedocs.io/).""")
# build again, and test that the file wasn't modified since it
# already has all the required headers
last_update = path.stat().st_mtime
build_post(root, config, path)
assert path.stat().st_mtime == last_update
def test_makes_output_directory(mocker: MockFixture,
fs: FakeFilesystem) -> None:
fs.create_file("testclip.mp4")
mocker.patch("abridge.processor.abridge_clip")
sys.argv = ["", "testclip.mp4", "-o", "outputdir"]
main()
os.path.isdir("outputdir")
def post(fs: FakeFilesystem, root: Path, config: Config) -> Iterator[Post]:
"""
Create a post.
"""
post_directory = root / "posts/first"
post_path = post_directory / "index.mkd"
with freeze_time("2021-01-01T00:00:00Z"):
fs.create_file(post_path, contents=POST_CONTENT)
post = build_post(root, config, post_path)
yield post
def test_exits_when_one_clip_doesnt_exist(mocker: MockFixture,
fs: FakeFilesystem) -> None:
fs.create_file("exists.mp4")
mocker.patch("abridge.processor.abridge_clip")
sys_spy = mocker.spy(abridge.cli, "sys")
sys.argv = ["", "exists.mp4", "something.mp4"]
main()
sys_spy.exit.assert_called_with("something.mp4: No such file or directory")
def test_updates_clips_process_bar(mocker: MockFixture,
fs: FakeFilesystem) -> None:
mock_ui_manager = mocker.patch("abridge.ui._MANAGER", autospec=True)
mocker.patch("abridge.processor.abridge_clip", autospec=True)
fs.create_file("somefile.mp4")
fs.create_file("somefile2.mp4")
sys.argv = ["", "somefile.mp4", "somefile2.mp4"]
main()
assert mock_ui_manager.counter().update.call_count == 2
def test_save_paths_obtain_and_check_error_on_existing_file(
fs: FakeFilesystem):
command = Mock()
path_existing_base_dir = "existing/path/"
os.makedirs(path_existing_base_dir)
path_existing_config = os.path.join(path_existing_base_dir,
"my_config.json")
fs.create_file(path_existing_config)
command.option.return_value = path_existing_config
with pytest.raises(ValueError,
match=r".*{reason}.*".format(reason="already exists")):
save_paths_obtain_and_check(command)
def test_find_directory(fs: FakeFilesystem) -> None:
"""
Test ``find_directory``.
"""
fs.create_dir("/path/to/blog/posts/first/css")
fs.create_file("/path/to/blog/nefelibata.yaml")
path = find_directory(Path("/path/to/blog/posts/first/css"))
assert path == Path("/path/to/blog")
with pytest.raises(SystemExit) as excinfo:
find_directory(Path("/path/to"))
assert str(excinfo.value) == "No configuration found!"
def test_keyboard_interrupt_shutsdown_pool(self, mocker: MockFixture,
fs: FakeFilesystem) -> None:
mock_splice = mocker.patch("abridge.processor.abridge_clip",
side_effect=KeyboardInterrupt())
mock_shutdown = mocker.patch("abridge.cli._shutdown")
fs.create_file("somefile.mp4")
sys.argv = ["", "somefile.mp4"]
main()
mock_splice.assert_called()
mock_shutdown.assert_called()
def test_creates_clip_processing_progress(mocker: MockFixture,
fs: FakeFilesystem) -> None:
mock_ui_manager = mocker.patch("abridge.ui._MANAGER", autospec=True)
mocker.patch("abridge.processor.abridge_clip", autospec=True)
fs.create_file("somefile.mp4")
fs.create_file("somefile2.mp4")
sys.argv = ["", "somefile.mp4", "somefile2.mp4"]
main()
mock_ui_manager.counter.assert_called_with(desc="Processed clips",
total=2,
unit="clips")
async def test_set_hostname(
mocker: MockerFixture,
fs: fake_filesystem.FakeFilesystem,
get_data_contents: Callable[[str, str], str],
) -> None:
called_set_hostname = False
async def mock_call(
cmd: List[str], ) -> Tuple[int, Optional[str], Optional[str]]:
nonlocal called_set_hostname
called_set_hostname = True
assert cmd[0] == "hostnamectl"
assert cmd[1] == "set-hostname"
assert cmd[2] == "foobar"
return 0, None, None
async def mock_call_fail(
cmd: List[str], ) -> Tuple[int, Optional[str], Optional[str]]:
return 1, None, "oops"
from gravel.controllers.nodes.host import HostnameCtlError, set_hostname
mocker.patch("socket.gethostname", return_value="myhostname")
mocker.patch("gravel.controllers.nodes.host.aqr_run_cmd",
new=mock_call_fail)
throws = False
try:
await set_hostname("foobar")
except HostnameCtlError as e:
assert "oops" in e.message
throws = True
assert throws
assert not fs.exists("/etc/hosts")
mocker.patch("gravel.controllers.nodes.host.aqr_run_cmd", new=mock_call)
fs.create_file("/etc/hosts")
hosts = get_data_contents(DATA_DIR, "hosts.raw")
with open("/etc/hosts", "w") as f:
f.write(hosts)
await set_hostname("foobar")
assert called_set_hostname
with open("/etc/hosts", "r") as f:
text = f.read()
assert text.count("myhostname") == 0
assert text.count("foobar") == 2
def test_load_extra_metadata(mocker: MockerFixture,
fs: FakeFilesystem) -> None:
"""
Test ``load_extra_metadata``.
"""
_logger = mocker.patch("nefelibata.utils._logger")
fs.create_file("/path/to/blog/test.yaml", contents=yaml.dump(dict(a=42)))
fs.create_file("/path/to/blog/broken.yaml", contents="{[")
metadata = load_extra_metadata(Path("/path/to/blog"))
assert metadata == {"test": {"a": 42}}
_logger.warning.assert_called_with(
"Invalid file: %s",
Path("/path/to/blog/broken.yaml"),
)
async def test_publish_with_basedir(
mocker: MockerFixture,
fs: FakeFilesystem,
root: Path,
config: Config,
) -> None:
"""
Test ``publish`` when a base directory is specified.
"""
FTP = mocker.patch("nefelibata.publishers.ftp.FTP")
ftp = FTP.return_value.__enter__.return_value
ftp.pwd.return_value = "/ftp/upload"
mocker.patch.object(
FTPPublisher,
"find_modified_files",
return_value=iter(
[root / "build/generic/subdir1/one", root / "build/generic/subdir2/two"],
),
)
fs.create_file(root / "build/generic/subdir1/one", contents="Hello, world!")
fs.create_file(root / "build/generic/subdir2/two", contents="Goodbye, world!")
publisher = FTPPublisher(
root,
config,
"generic",
"ftp.example.com",
"username",
"password",
"ftp/upload",
)
await publisher.publish()
ftp.cwd.assert_has_calls(
[
mocker.call("ftp/upload"),
mocker.call("subdir1"),
mocker.call("/ftp/upload"),
mocker.call("subdir2"),
],
)
async def test_get_posts(fs: FakeFilesystem, root: Path,
config: Config) -> None:
"""
Test ``get_posts``.
"""
with freeze_time("2021-01-01T00:00:00Z"):
fs.create_dir(root / "posts/one")
fs.create_file(root / "posts/one/index.mkd")
with freeze_time("2021-01-02T00:00:00Z"):
fs.create_dir(root / "posts/two")
fs.create_file(root / "posts/two/index.mkd")
posts = get_posts(root, config)
assert len(posts) == 2
assert posts[0].path == Path(root / "posts/two/index.mkd")
assert posts[1].path == Path(root / "posts/one/index.mkd")
# test limited number of posts returned
posts = get_posts(root, config, 1)
assert len(posts) == 1
def test_invalid_bert(self, fs: FakeFilesystem) -> None:
bad_bert_path = './bad/path/to/bert'
config = """
{
"bert": "%s",
"labels": "label.vocab",
"is_released": true,
"description": "This is the latest model from Sascha.",
"metadata": {
"thesaurus": "issues"
}
}
""" % (bad_bert_path)
fs.add_real_directory('./testdata/test_model/test_instance')
fs.remove_object('./testdata/test_model/test_instance/config.json')
fs.create_file('./testdata/test_model/test_instance/config.json',
contents=config)
with pytest.raises(Exception,
match='SavedModel file does not exist at'):
c = Classifier(self.BASE_CLASSIFIER_PATH, 'test_model')
# Bad bert is only used on uncached embed.
c.classify(['some string'])
def test_update_config(fs: fake_filesystem.FakeFilesystem, ) -> None:
from gravel.controllers.resources.network import Network
config = Path("/etc/sysconfig/network/config")
fs.create_file(config,
contents='NETCONFIG_DNS_STATIC_SERVERS=""\n# comment\n')
network = Network(5.0)
network._update_config(
config,
{
"NETCONFIG_DNS_STATIC_SERVERS": "8.8.8.8",
"NEW_THING": "very important data",
"ANOTHER_NEW_THING": "more important data",
},
)
with config.open("r") as f:
contents = f.readlines()
assert contents == [
'NETCONFIG_DNS_STATIC_SERVERS="8.8.8.8"\n',
"# comment\n",
'NEW_THING="very important data"\n',
'ANOTHER_NEW_THING="more important data"\n',
]
def test_load_yaml(mocker: MockerFixture, fs: FakeFilesystem) -> None:
"""
Test ``load_yaml``.
"""
assert load_yaml(Path("/path/to/blog/missing.yaml"), BaseModel) == {}
fs.create_file(
"/path/to/blog/existing.yaml",
contents="""
reply,gemini://ew.srht.site/en/2021/20210915-re-changing-old-code-is-risky.gmi:
id: reply,gemini://ew.srht.site/en/2021/20210915-re-changing-old-code-is-risky.gmi
name: '2021-09-15 ~ew''s FlightLog: Re: Changing old code is risky'
timestamp: null
type: reply
url: gemini://ew.srht.site/en/2021/20210915-re-changing-old-code-is-risky.gmi
""",
)
assert load_yaml(Path("/path/to/blog/existing.yaml"), Interaction) == {
("reply,gemini://ew.srht.site/en/2021/"
"20210915-re-changing-old-code-is-risky.gmi"):
Interaction(
id=
"reply,gemini://ew.srht.site/en/2021/20210915-re-changing-old-code-is-risky.gmi",
name="2021-09-15 ~ew's FlightLog: Re: Changing old code is risky",
url=
"gemini://ew.srht.site/en/2021/20210915-re-changing-old-code-is-risky.gmi",
type="reply",
timestamp=None,
),
}
path = Path("/path/to/blog/invalid.yaml")
fs.create_file(path, contents="[1,2,3")
_logger = mocker.patch("nefelibata.utils._logger")
with pytest.raises(yaml.parser.ParserError):
load_yaml(path, BaseModel)
assert _logger.warning.called_with("Invalid YAML file: %s", path)
def test_processes_clip_glob(mocker: MockFixture, fs: FakeFilesystem) -> None:
fs.create_file("testclip.mp4")
fs.create_file("anotherclip.mp4")
fs.create_file("thirdclip.mp4")
mock_abridge_clip = mocker.patch("abridge.processor.abridge_clip")
sys.argv = ["", "*.mp4"]
main()
mock_abridge_clip.assert_has_calls(
[
call("testclip.mp4", "processed", 20, 5),
call("anotherclip.mp4", "processed", 20, 5),
call("thirdclip.mp4", "processed", 20, 5),
],
any_order=True,
)
def host_sysfs(fs: fake_filesystem.FakeFilesystem):
"""Create a fake filesystem to represent sysfs"""
enc_path = '/sys/class/scsi_generic/sg2/device/enclosure/0:0:1:0'
dev_path = '/sys/class/scsi_generic/sg2/device'
slot_count = 12
fs.create_dir(dev_path)
fs.create_file(os.path.join(dev_path, 'vendor'), contents="EnclosuresInc")
fs.create_file(os.path.join(dev_path, 'model'), contents="D12")
fs.create_file(os.path.join(enc_path, 'id'), contents='1')
fs.create_file(os.path.join(enc_path, 'components'), contents=str(slot_count))
for slot_num in range(slot_count):
slot_dir = os.path.join(enc_path, str(slot_num))
fs.create_file(os.path.join(slot_dir, 'locate'), contents='0')
fs.create_file(os.path.join(slot_dir, 'fault'), contents='0')
fs.create_file(os.path.join(slot_dir, 'slot'), contents=str(slot_num))
if slot_num < 6:
fs.create_file(os.path.join(slot_dir, 'status'), contents='Ok')
slot_dev = os.path.join(slot_dir, 'device')
fs.create_dir(slot_dev)
fs.create_file(os.path.join(slot_dev, 'vpd_pg80'), contents=f'fake{slot_num:0>3}')
else:
fs.create_file(os.path.join(slot_dir, 'status'), contents='not installed')
yield fs
def template_file_syntax_error(fs: fake_filesystem.FakeFilesystem) -> Path:
"""Fixture for create a template file."""
template_uri = "/template_with_syntax_error.mako"
fs.create_file(template_uri, contents="hello ${wor")
return Path(template_uri)
