def test_successful_job_events():
with mock(
"test.test1", """
from middlewared.service import job
@job()
def mock(self, job, *args):
return 42
"""):
with client() as c:
events = []
def callback(type, **message):
events.append((type, message))
c.subscribe("core.get_jobs", callback, sync=True)
c.call("test.test1", job=True)
assert len(events) == 3, pprint.pformat(events, indent=2)
assert events[0][0] == "ADDED"
assert events[0][1]["fields"]["state"] == "WAITING"
assert events[1][0] == "CHANGED"
assert events[1][1]["fields"]["state"] == "RUNNING"
assert events[2][0] == "CHANGED"
assert events[2][1]["fields"]["state"] == "SUCCESS"
assert events[2][1]["fields"]["result"] == 42
def test_client_job_callback():
with mock(
"test.test1", """
from middlewared.service import job
@job()
def mock(self, job, *args):
import time
time.sleep(2)
return 42
"""):
with client() as c:
results = []
c.call("test.test1",
job=True,
callback=lambda job: results.append(job.copy()))
# callback is called in a separate thread, allow it to settle
time.sleep(2)
assert len(results) == 2, pprint.pformat(results, indent=2)
assert results[0]['state'] == 'RUNNING'
assert results[1]['state'] == 'SUCCESS'
assert results[1]['result'] == 42
def test_filesystem__file_tail_follow__grouping():
ssh("echo > /tmp/file_tail_follow.txt")
with client() as c:
received = []
def append(type, **kwargs):
received.append((time.monotonic(), kwargs["fields"]["data"]))
c.subscribe("filesystem.file_tail_follow:/tmp/file_tail_follow.txt",
append)
ssh("for i in `seq 1 200`; do echo test >> /tmp/file_tail_follow.txt; sleep 0.01; done"
)
# Settle down things
time.sleep(1)
received = received[1:] # Initial file contents
# We were sending this for 2-3 seconds so we should have received 4-6 blocks with 0.5 sec interval
assert 4 <= len(received) <= 6, str(received)
# All blocks should have been received uniformly in time
assert all(
0.4 <= b2[0] - b1[0] <= 1.0
for b1, b2 in zip(received[:-1], received[1:])), str(received)
# All blocks should contains more or less same amount of data
assert all(30 <= len(block[1].split("\n")) <= 60
for block in received[:-1]), str(received)
# One single send
ssh("echo finish >> /tmp/file_tail_follow.txt")
time.sleep(1)
assert received[-1][1] == "finish\n"
def test_no_upload_to_unchecked_pipe():
with session() as s:
r = s.post(
f"{url()}/api/v2.0/resttest/test_input_unchecked_pipe",
headers={"Content-type": "application/json"},
data='{"key": "value"}',
)
r.raise_for_status()
job_id = r.json()
with client() as c:
assert c.call("core.job_wait", job_id,
job=True) == '{"key": "value"}NONE'
def test_download_from_download_endpoint():
with client() as c:
job_id, path = c.call("core.download", "resttest.test_download_pipe",
[{
"key": "value"
}], "file.bin")
r = requests.get(f"{url()}{path}")
r.raise_for_status()
assert r.headers[
"Content-Disposition"] == "attachment; filename=\"file.bin\""
assert r.headers["Content-Type"] == "application/octet-stream"
assert r.text == '{"key": "value"}'
def test_upload(method):
with session() as s:
r = s.post(
f"{url()}/api/v2.0/resttest/{method}",
files={
"data": (None, io.StringIO('{"key": "value"}')),
"file": (None, io.StringIO("FILE")),
},
)
r.raise_for_status()
job_id = r.json()
with client() as c:
assert c.call("core.job_wait", job_id,
job=True) == '{"key": "value"}FILE'
def test_private_params_do_not_leak_to_logs():
with mock(
"test.test1", """
from middlewared.service import accepts
from middlewared.schema import Dict, Str
@accepts(Dict("test", Str("password", private=True)))
async def mock(self, args):
raise Exception()
"""):
log_before = ssh("cat /var/log/middlewared.log")
with client(py_exceptions=False) as c:
with pytest.raises(Exception):
c.call("test.test1", {"password": "******"})
log = ssh("cat /var/log/middlewared.log")[len(log_before):]
assert "Exception while calling test.test1(*[{'password': '******'}])" in log
def test_no_lock():
with mock(
"test.test1", """
from middlewared.service import lock
async def mock(self, *args):
import asyncio
await asyncio.sleep(5)
"""):
start = time.monotonic()
with client() as c:
c1 = c.call("test.test1", background=True)
c2 = c.call("test.test1", background=True)
c.wait(c1)
c.wait(c2)
assert time.monotonic() - start < 6
def test_threading_lock():
with mock(
"test.test1", """
from middlewared.service import lock
@lock("test")
def mock(self, *args):
import time
time.sleep(5)
"""):
start = time.monotonic()
with client() as c:
c1 = c.call("test.test1", background=True)
c2 = c.call("test.test1", background=True)
c.wait(c1)
c.wait(c2)
assert time.monotonic() - start >= 10
def test_upload_to_upload_endpoint():
with session() as s:
r = s.post(
f"{url()}/_upload",
files={
"data": (None,
io.StringIO(
json.dumps({
"method": "resttest.test_input_pipe",
"params": [{
"key": "value"
}]
}))),
"file": (None, io.StringIO("FILE")),
},
)
r.raise_for_status()
job_id = r.json()["job_id"]
with client() as c:
assert c.call("core.job_wait", job_id,
job=True) == '{"key": "value"}FILE'
def test_block_hooks(block):
hook_name = str(uuid.uuid4())
with mock("test.test1", """
async def mock(self, hook_name, blocked_hooks):
from pathlib import Path
sentinel = Path("/tmp/block_hooks_sentinel")
async def hook(middleware):
sentinel.write_text("")
self.middleware.register_hook(hook_name, hook, blockable=True, sync=True)
sentinel.unlink(missing_ok=True)
with self.middleware.block_hooks(*blocked_hooks):
await self.middleware.call_hook(hook_name)
return sentinel.exists()
"""):
with client() as c:
assert c.call("test.test1", hook_name, [hook_name] if block else []) == (not block)
