async def test_subscribe_glob_filter():
"""
Ensure that globs only match matching files.
"""
with run_server():
async with make_connection() as tree:
expect = None
count = 0
async def on_changed(changes: {str: [str]}):
# Sort all files in the changes list for stability.
changes = {k: list(sorted(v)) for k, v in changes.items()}
nonlocal count
assert expect is not None
assert expect == (count, changes)
count += 1
for name in ['a', 'b', 'c', 'aa']:
await tree.create_file("/", name)
await tree.watch_matching_files("/?", on_changed)
expect = (0, {"foo": ["/a"]})
await tree.set_file("/a", "foo")
await tree.set_file("/aa", "foo")
expect = (1, {"bar": ["/b"]})
await tree.set_file("/b", "bar")
await tree.set_file("/aa", "bar")
expect = (2, {"baz": ["/c"]})
await tree.set_file("/c", "baz")
await tree.set_file("/aa", "baz")
async def test_subscribe_glob_multi():
"""
Ensure that globs return all matching path events in a single message.
"""
with run_server():
async with make_connection() as tree:
expect = None
count = 0
async def on_changed(changes):
# Sort all files in the changes list for stability.
changes = {k: list(sorted(v)) for k, v in changes.items()}
nonlocal count
assert expect is not None
assert expect == (count, changes)
count += 1
for a in ['a', 'b', 'c']:
await tree.create_directory("/", a)
for b in ['a', 'b', 'c']:
await tree.create_directory("/{}".format(a), b)
for c in ['foo', 'bar', 'baz']:
await tree.create_file("/{}/{}".format(a, b), c)
await tree.watch_matching_files("/a/**/foo", on_changed)
expect = (0, {"a": ["/a/a/foo"]})
await tree.set_file("/a/a/foo", "a")
expect = (1, {"b": ["/a/a/foo"]})
await tree.set_matching_files("/a/a/*", "b")
await tree.set_matching_files("/**/bar", "c")
expect = (2, {"c": ["/a/a/foo", "/a/b/foo", "/a/c/foo"]})
await tree.set_matching_files("/**/foo", "c")
async def test_remove_errors():
with run_server():
async with make_connection() as tree:
for exc_type, chars in InvalidChars.items():
for c in chars:
with pytest.raises(exc_type):
await tree.remove_node("/", "a" + c + "b")
if c != '/':
with pytest.raises(exc_type):
await tree.remove_node("/a" + c + "b", "foo")
for exc_type, names in InvalidNames.items():
for name in names:
with pytest.raises(exc_type):
await tree.remove_node("/", name)
with pytest.raises(exc_type):
await tree.remove_node("/" + name, "foo")
for exc_type, names in EmptyComponentPaths.items():
for name in names:
with pytest.raises(exc_type):
await tree.remove_node(name, "foo")
with pytest.raises(db.NoSuchNode):
await tree.remove_node("/", "a")
await tree.create_directory("/", "a")
await tree.create_directory("/a", "b")
with pytest.raises(db.DirectoryNotEmpty):
await tree.remove_node("/", "a")
await tree.remove_node("/a", "b")
async def test_create_errors():
with run_server():
async with make_connection() as tree:
await tree.create_directory("/", "dir")
await tree.create_file("/", "file")
for path in ("/", "/dir"):
for ty in NodeTypes:
for exc_type, chars in InvalidChars.items():
for c in chars:
with pytest.raises(exc_type):
await create_node(tree, ty, path, "a" + c + "b")
if c != '/':
with pytest.raises(exc_type):
await create_node(tree, ty, "/a" + c + "b", "foo")
for exc_type, names in InvalidNames.items():
for name in names:
with pytest.raises(exc_type):
await create_node(tree, ty, path, name)
with pytest.raises(exc_type):
await create_node(tree, ty, "/" + name, "foo")
for exc_type, names in EmptyComponentPaths.items():
for name in names:
with pytest.raises(exc_type):
await create_node(tree, ty, name, "foo")
with pytest.raises(db.NoSuchNode):
await create_node(tree, ty, "/b", "a")
with pytest.raises(db.NodeAlreadyExists):
await create_node(tree, ty, "/", "dir")
with pytest.raises(db.NotDirectory):
await create_node(tree, ty, "/file", 'foo')
async def test_initial_tree():
"""
Ensure that the tree starts empty.
"""
with run_server():
async with make_connection() as tree:
assert await tree.list_directory("/") == []
async def test_basic_formula_input():
with run_server():
async with make_connection() as tree:
await tree.create_file("/", "a0")
await tree.set_file("/a0", "Hello, World!")
await tree.create_formula("/", "result", {'a0': '/a0'}, 'a0')
data = await tree.get_file("/result")
assert data == "Hello, World!"
async def test_watch_errors():
with run_server():
async with make_connection() as tree:
async def target(**_):
pass
with pytest.raises(db.Dotfile):
await tree.watch_matching_files("/../../usr/lib/libGL.so", target)
async def test_data_errors():
with run_server():
async with make_connection() as tree:
with pytest.raises(db.Dotfile):
await tree.set_file("/.", "")
with pytest.raises(db.Dotfile):
await tree.get_file("/.")
with pytest.raises(db.NonAbsolutePath):
await tree.set_file("a/b", "")
with pytest.raises(db.NonAbsolutePath):
await tree.get_file("a/b")
async def test_data():
"""
Check that basic create/get/set/remove works as expected.
"""
with run_server():
async with make_connection() as tree:
await tree.create_file("/", "a")
await tree.set_file("/a", "flinfniffle")
data = await tree.get_file("/a")
assert data == "flinfniffle"
await tree.remove_node("/", "a")
async def test_set_glob_basic():
"""
Ensure that globs only match matching files.
"""
with run_server():
async with make_connection() as tree:
for name in "abcd":
await tree.create_file("/", name)
await tree.set_matching_files("/*", "hello")
data = await tree.get_matching_files("/*")
for name in "abcd":
assert data["/" + name] == "hello"
async def test_subscribe_same_client_data():
"""
Ensure that subscriptions to data work and that we can:
* make multiple subscriptions to the same path on a single client.
* touch a subpath without being notified in the parent
* remove one subscription of multiple
"""
with run_server():
async with make_connection() as tree:
count1 = 0
notify1 = asyncio.Future()
async def on_child_changed1(changes):
assert changes == {"foo": ["/a"]}
nonlocal count1, notify1
count1 += 1
notify1.set_result(...)
count2 = 0
notify2 = asyncio.Future()
async def on_child_changed2(changes):
assert changes == {"foo": ["/a"]}
nonlocal count2, notify2
count2 += 1
notify2.set_result(...)
# Create and subscribe to data node.
await tree.create_file("/", "a")
await tree.create_file("/", "b")
subid1 = await tree.watch_matching_files("/a", on_child_changed1)
subid2 = await tree.watch_matching_files("/a", on_child_changed2)
# Check that we get messages when we change the data, but not when we set siblings, or query it.
await tree.set_file("/a", "foo")
await tree.set_file("/b", "foo")
data = await tree.get_file("/a")
assert data == "foo"
await asyncio.gather(notify1, notify2)
assert count1 == 1
assert count2 == 1
# Reset notifications; unsubscribe #1, then check that we only get the notice on 2.
notify1 = asyncio.Future()
notify2 = asyncio.Future()
await tree.unwatch(subid1)
await tree.set_file("/a", "foo")
await asyncio.sleep(0.1) # we don't expect a response from 1, but give it some time to be more sure.
await notify2
assert count1 == 1
assert count2 == 2
async def test_subscribe_glob_basic_file():
"""
Ensure that subscribing to a glob works properly.
"""
with run_server():
async with make_connection() as tree:
expect = None
count = 0
async def on_foo_changed(changes: {str: [str]}):
# Sort all files in the changes list for stability.
changes = {k: list(sorted(v)) for k, v in changes.items()}
nonlocal count
assert expect is not None
assert expect == (count, changes)
count += 1
sid = await tree.watch_matching_files("/{a,b}-foo", on_foo_changed)
# We should be able to create a new path and have the glob pick it up.
await tree.create_file("/", "a-foo")
# Try setting file contents.
expect = (0, {"0": ["/a-foo"]})
await tree.set_file("/a-foo", "0")
expect = (1, {"": ["/a-foo"]})
await tree.set_file("/a-foo", "")
expect = None
# We should not get a notification on removing the watched node.
await tree.remove_node("/", "a-foo")
# We should not get a notification for inserting into a non-matching directory.
await tree.create_file("/", "a-bar")
await tree.set_file("/a-bar", "test")
await tree.remove_node("/", "a-bar")
# Check for multiple matches with overlapping.
await tree.create_file("/", "a-foo")
await tree.create_file("/", "b-foo")
await tree.create_file("/", "c-foo")
expect = (2, {"2": ["/a-foo"]})
await tree.set_matching_files("/{a,c}-foo", "2")
expect = (3, {"3": ["/b-foo"]})
await tree.set_matching_files("/{b,c}-foo", "3")
expect = (4, {"4": ["/a-foo", "/b-foo"]})
await tree.set_matching_files("/{a,b,c}-foo", "4")
expect = None
async def test_formula_subscription_all():
with run_server():
async with make_connection() as tree:
await tree.create_file("/", "a0")
await tree.set_file("/a0", "Hello, World!")
await tree.create_formula("/", "result", {'a0': '/a0'}, 'a0')
count = 0
expect = None
async def on_result_changed(changes):
nonlocal count
assert expect == (count, changes)
count += 1
await tree.watch_matching_files("/*", on_result_changed)
expect = (0, {"foobar": ["/a0", "/result"]})
await tree.set_file("/a0", "foobar")
assert count == 1
async def test_formula_nested():
with run_server():
async with make_connection() as tree:
expect = None
count = 0
async def on_result_changed(changes):
nonlocal count
assert expect == (count, changes)
count += 1
await tree.create_file("/", "a")
await tree.create_formula("/", "b", {"a": "/a"}, "a")
await tree.create_formula("/", "c", {"b": "/b"}, "b")
await tree.watch_matching_files("/{a,c}", on_result_changed)
expect = (0, {"foobar": ["/a", "/c"]})
await tree.set_file("/a", "foobar")
assert count == 1
assert await tree.get_file("/c") == "foobar"
async def test_formula_multi_input():
"""
Use a formula to ensure that the basic use works.
"""
with run_server():
async with make_connection() as tree:
expect = None
count = 0
async def on_result_changed(changes):
nonlocal count
assert expect == (count, changes)
count += 1
await tree.create_formula("/", "result", {'a0': "/arg0", 'a1': "/arg1"},
'(join "" a0 a1)')
with pytest.raises(db.FormulaInputNotFound):
await tree.get_file("/result")
await tree.watch_matching_files("/*", on_result_changed)
await tree.create_file("/", "arg0")
with pytest.raises(db.FormulaInputNotFound):
await tree.get_file("/result")
await tree.create_file("/", "arg1")
await tree.get_file("/result")
expect = (0, {"foo": ["/arg0", "/result"]})
await tree.set_file("/arg0", "foo")
assert await tree.get_file("/result") == "foo"
expect = (1, {"bar": ["/arg1"], "foobar": ["/result"]})
await tree.set_file("/arg1", "bar")
assert await tree.get_file("/result") == "foobar"
expect = (2, {"baz": ["/arg0"], "bazbar": ["/result"]})
await tree.set_file("/arg0", "baz")
assert await tree.get_file("/result") == "bazbar"
async def test_tree_sync():
"""
Create a large tree using the sync API and verify the content and layout.
"""
with run_server():
async with make_connection() as tree:
for a in "abcd":
await tree.create_directory("/", a)
for b in "efgh":
await tree.create_directory("/{}".format(a), b)
for c in "ijkl":
await tree.create_directory("/{}/{}".format(a, b), c)
for d in "mnop":
await tree.create_directory("/{}/{}/{}".format(a, b, c), d)
path = "/"
assert "".join(sorted(await tree.list_directory(path))) == "abcd"
for a in await tree.list_directory(path):
a_path = path + a
assert "".join(sorted(await tree.list_directory(a_path))) == "efgh"
for b in await tree.list_directory(a_path):
b_path = a_path + "/" + b
assert "".join(sorted(await tree.list_directory(b_path))) == "ijkl"
for c in await tree.list_directory(b_path):
c_path = b_path + "/" + c
assert "".join(sorted(await tree.list_directory(c_path))) == "mnop"
for d in await tree.list_directory(c_path):
d_path = c_path + "/" + d
assert "".join(sorted(await tree.list_directory(d_path))) == ""
path = "/"
for a in await tree.list_directory(path):
for b in await tree.list_directory("/{}".format(a)):
for c in await tree.list_directory("/{}/{}".format(a, b)):
for d in await tree.list_directory("/{}/{}/{}".format(a, b, c)):
await tree.remove_node("/{}/{}/{}".format(a, b, c), d)
await tree.remove_node("/{}/{}".format(a, b), c)
await tree.remove_node("/{}".format(a), b)
await tree.remove_node("/", a)
async def test_tree_async():
"""
Do a large amount of work using async calls.
"""
with run_server():
async with make_connection() as tree:
futures = []
children = "abcdefghijklmnopqrstuvwxyz"
for i, a in enumerate(children):
if i % 2 == 0:
futures.append(await tree.create_directory_async("/", a))
else:
# Note that this is safe because we're using TCP under the hood. If we ever do anything non-serial,
# this test will need to change pretty dramatically.
futures.append(await tree.create_file_async("/", a))
futures.append(await tree.set_file_async("/" + a, a))
await asyncio.gather(*futures)
future = await tree.list_directory_async("/")
result = await future
assert "".join(sorted(result.children)) == children
futures = []
for i, a in enumerate(children):
if i % 2 == 1:
futures.append(await tree.get_file_async("/" + a))
results = await asyncio.gather(*futures)
assert "".join(sorted([rv.data for rv in results])) == children[1::2]
futures = []
vowels = "aeiou"
for a in vowels:
futures.append(await tree.remove_node_async("/", a))
await asyncio.gather(*futures)
future = await tree.list_directory_async("/")
result = await future
assert "".join(sorted(result.children)) == \
"bcdfghjklmnpqrstvwxyz"
async def test_subscribe_multiple_clients():
"""
Ensure that causing an event on one client reports that event on a different client.
"""
with run_server():
async with make_connection() as treeA:
async with make_connection() as treeB:
count = 0
notify = asyncio.Future()
async def on_child_changed1(changes):
nonlocal count
count += 1
if 'bar' in changes:
notify.set_result(...)
await treeA.create_file("/", "a")
await treeA.watch_matching_files("/a", on_child_changed1)
await treeB.set_file("/a", "foo")
await treeB.set_file("/a", "bar")
await notify
assert count == 2
#!/usr/bin/env python
import os
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
with pushd('cpp'):
run('make')
run('./gender')
with pushd('malware_classification'):
pass
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('cpp'):
run('python', 'waf', 'configure')
run('python', 'waf')
run('./build/ml_update')
run('./build/ml_analysis')
async def test_basic_formula_get():
with run_server():
async with make_connection() as tree:
await tree.create_formula("/", "result", {}, '"Hello, World!"')
data = await tree.get_file("/result")
assert data == 'Hello, World!'
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
run('python', 'python/shogun.py')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
run('python', 'python/gender.py')
with pushd('twitter_streaming_lang'):
with run_server('jubaclassifier', '-f', 'twitter_streaming_lang.json'):
pass
with pushd('twitter_streaming_location'):
with run_server('jubaclassifier', '-f', 'twitter_streaming_location.json'):
pass
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('python'):
run('python', 'ml_update.py')
run('python', 'ml_analysis.py')
with pushd('npb_similar_player'):
with run_server('jubarecommender', '-f', 'npb_similar_player.json'):
run('python', 'update.py')
async def test_basic_formula_types():
with run_server():
async with make_connection() as tree:
await tree.create_formula("/", "result", {}, '42')
with pytest.raises(db.FormulaTypeError):
await tree.get_file("/result")
#!/usr/bin/env python
#
# Copyright (c) 2007-2009 The PyAMF Project.
# See LICENSE.txt for details.
"""
Echo test server.
You can use this example with the echo_test.swf client on the
U{EchoTest<http://pyamf.org/wiki/EchoTest>} wiki page.
@since: 0.1.0
"""
import sys
import echo
from util import parse_args, run_server
options = parse_args(sys.argv[1:])
services = {
options[0].service: echo.echo,
'Red5Echo': echo
}
try:
run_server('Echo Test', options[0], services)
except KeyboardInterrupt:
pass
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
with pushd('python'):
run('python', 'shogun.py')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
with pushd('python'):
run('python', 'gender.py')
with pushd('twitter_streaming_lang'):
with run_server('jubaclassifier', '-f', 'twitter_streaming_lang.json'):
pass
with pushd('twitter_streaming_location'):
with run_server('jubaclassifier', '-f',
'twitter_streaming_location.json'):
pass
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('python'):
run('python', 'ml_update.py')
run('python', 'ml_analysis.py')
async def test_basic_formula_stmt():
with run_server():
async with make_connection() as tree:
await tree.create_formula("/", "result", {}, '(format "~s" 42)')
data = await tree.get_file("/result")
assert data == '42'
#!/usr/bin/env python
"""
Being in the sandbox, this file requires that the echo example dir is in the
path
"""
import sys
from util import parse_args, run_server
import acpb
options = parse_args(sys.argv[1:])
services = {'acpb':acpb.acpb}
run_server('ArrayCollection Test', options[0], services)
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
with pushd('java'):
run('mvn', 'clean')
run('sh', 'run.sh')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
with pushd('java'):
run('mvn', 'clean')
run('sh', 'run.sh')
with pushd('twitter_streaming_location'):
with run_server('jubaclassifier', '-f', 'twitter_streaming_location.json'):
pass
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('java'):
run('mvn', 'clean')
run('sh', 'run-update.sh')
run('sh', 'run-analyze.sh')
async def test_basic_formula_missing_input():
with run_server():
async with make_connection() as tree:
await tree.create_formula("/", "result", {'a0': '/a0'}, 'a0')
with pytest.raises(db.FormulaInputNotFound):
await tree.get_file("/result")
async def test_unwatch_errors():
with run_server():
async with make_connection() as tree:
with pytest.raises(db.NoSuchSubscription):
await tree.unwatch(10)
async def test_basic_formula_no_assign():
with run_server():
async with make_connection() as tree:
await tree.create_formula("/", "result", {}, '"foo"')
with pytest.raises(db.NotFile):
await tree.set_file("/result", "anything")
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
run('ruby', 'ruby/shogun.rb')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
run('ruby', 'ruby/gender.rb')
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('ruby'):
run('ruby', 'ml_update.rb')
run('ruby', 'ml_analysis.rb')
with pushd('npb_similar_player'):
with run_server('jubarecommender', '-f', 'npb_similar_player.json'):
run('ruby', 'update.rb')
run('ruby', 'analyze.rb')
with pushd('rent'):
with run_server('jubaregression', '-f', 'rent.json'):
run('ruby', 'ruby/train.rb', 'dat/rent-data.csv')
run('ruby', 'ruby/test.rb', input='19.9\n2\n22\n4\nW\n')
with pushd('language_detection'):
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
with pushd('ruby'):
run('ruby', 'shogun.rb')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
with pushd('ruby'):
run('ruby', 'gender.rb')
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('ruby'):
run('ruby', 'ml_update.rb')
run('ruby', 'ml_analysis.rb')
with pushd('npb_similar_player'):
with run_server('jubarecommender', '-f', 'npb_similar_player.json'):
with pushd('ruby'):
run('ruby', 'update.rb')
run('ruby', 'analyze.rb')
with pushd('rent'):
with run_server('jubaregression', '-f', 'rent.json'):
with pushd('ruby'):
