def test_snapshots():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
snapshot_1 = os.path.join(t, "snapshot_1.ap")
snapshot_2 = os.path.join(t, "snapshot_2.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("cosine", data=np.cos(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("tangent", data=np.tan(2.*np.pi*frequency*time))
""" % (snapshot_1, snapshot_2))
calc_angular.run()
paper.close()
# Open the snapshot files to verify they are valid ActivePapers
ActivePaper(snapshot_1, 'r').close()
ActivePaper(snapshot_2, 'r').close()
# Check the contents
paper = h5py.File(filename)
snapshot_1 = h5py.File(snapshot_1)
snapshot_2 = h5py.File(snapshot_2)
for item in ['/data/time', '/data/frequency', '/data/angular/sine',
'/code/calc_angular']:
assert item in paper
assert item in snapshot_1
assert item in snapshot_2
assert '/data/angular/cosine' in paper
assert '/data/angular/cosine' not in snapshot_1
assert '/data/angular/cosine' in snapshot_2
assert '/data/angular/tangent' in paper
assert '/data/angular/tangent' not in snapshot_1
assert '/data/angular/tangent' not in snapshot_2
for root in [snapshot_1, snapshot_2]:
#time_ref = root['/data/angular/time'][()]
#assert root[time_ref].name == '/data/time'
time_ref = root['/data/angular'].attrs['time']
assert root[time_ref].name == '/data/time'
def test_import_math():
# math is an extension module, so this should fail
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
paper.import_module('math')
paper.close()
def test_dummy_datasets():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_angular.run()
paper.replace_by_dummy('/data/angular/sine')
dummy = paper.data_group['angular/sine']
assert dummy.attrs.get('ACTIVE_PAPER_GENERATING_CODELET') \
== '/code/calc_angular'
assert dummy.attrs.get('ACTIVE_PAPER_DUMMY_DATASET', False)
passed = True
try:
paper.replace_by_dummy('/data/time')
except AssertionError:
passed = False
assert not passed
paper.close()
def test_modified_scripts():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=42)
group = data.create_group('group1')
group.mark_as_data_item()
group['value'] = 1
group = data.create_group('group2')
group['value'] = 2
""")
script.run()
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/code/script', '/data/foo',
'/data/group1', '/data/group2/value']
assert (paper.data['foo'][...] == 42)
assert (paper.data['group1/value'][...] == 1)
assert (paper.data['group2/value'][...] == 2)
script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=1)
""")
script.run()
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/code/script', '/data/foo']
assert (paper.data['foo'][...] == 1)
paper.close()
def make_paper_with_internal_module(filename):
paper = ActivePaper(filename, "w")
paper.add_module(
"my_math", """
import numpy as np
def my_func(x):
return np.sin(x)
""")
paper.data.create_dataset("frequency", data=0.2)
paper.data.create_dataset("time", data=0.1 * np.arange(100))
calc_sine = paper.create_calclet(
"calc_sine", """
from activepapers.contents import data
import numpy as np
from my_math import my_func
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def test_groups():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group = paper.data.create_group('group')
subgroup = group.create_group('subgroup')
group['data1'] = np.arange(10)
group['data2'] = 42
assert sorted([g.name for g in paper.iter_groups()]) \
== ['/data/group', '/data/group/subgroup']
assert sorted(list(node for node in group)) \
== ['data1', 'data2', 'subgroup']
assert group['data1'][...].shape == (10,)
assert group['data2'][...] == 42
assert paper.data.parent is paper.data
assert group.parent is paper.data
assert group['data1'].parent is group
assert group['data2'].parent is group
script = paper.create_calclet("script",
"""
from activepapers.contents import data
assert data.parent is data
assert data._codelet is not None
assert data._codelet.path == '/code/script'
group = data['group']
assert group.parent is data
assert group._codelet is not None
assert group._codelet.path == '/code/script'
""")
script.run()
paper.close()
def test_simple_paper():
with tempdir.TempDir() as t:
filename1 = os.path.join(t, "simple1.ap")
filename2 = os.path.join(t, "simple2.ap")
make_simple_paper(filename1)
all_paths = [
'README', 'code', 'code/calc_sine', 'code/initialize', 'data',
'data/frequency', 'data/sine', 'data/time', 'documentation',
'external-dependencies', 'history'
]
all_items = [
'/code/calc_sine', '/code/initialize', '/data/frequency',
'/data/sine', '/data/time'
]
all_deps = {
'/data/sine': ["/code/calc_sine", "/data/frequency", "/data/time"],
'/data/time': [],
'/data/frequency': [],
'/code/calc_sine': [],
'/code/initialize': []
}
sine_deps = ["/code/calc_sine", "/data/frequency", "/data/time"]
hierarchy = [[
'/code/calc_sine', '/code/initialize', '/data/frequency',
'/data/time'
], ['/data/sine']]
check_hdf5_file(filename1, all_paths, sine_deps)
check_paper(filename1, all_items, all_deps, hierarchy)
with ActivePaper(filename1, "r") as paper:
paper.rebuild(filename2)
check_hdf5_file(filename2, all_paths, sine_deps)
check_paper(filename2, all_items, all_deps, hierarchy)
def check_paper(filename, ref_items, ref_deps, ref_hierarchy):
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == ref_items
items_with_deps = sorted([
item.name for item in paper.iter_items()
if paper.has_dependencies(item)
])
assert items_with_deps == ['/data/sine']
deps = dict(
(ascii(item.name),
sorted(list(ascii(dep.name)
for dep in paper.iter_dependencies(item))))
for item in paper.iter_items())
assert deps == ref_deps
graph = collections.defaultdict(set)
for item, deps in ref_deps.items():
for d in deps:
graph[d].add(item)
assert graph == paper.dependency_graph()
hierarchy = [
sorted([ascii(item.name) for item in items])
for items in paper.dependency_hierarchy()
]
assert hierarchy == ref_hierarchy
calclets = paper.calclets()
assert len(calclets) == 1
assert ascii(calclets['/code/calc_sine'].path) == '/code/calc_sine'
paper.close()
def make_simple_paper(filename):
paper = ActivePaper(filename, "w")
#paper.data.create_dataset("frequency", data=0.2)
#paper.data.create_dataset("time", data=0.1*np.arange(100))
init = paper.create_importlet(
"initialize", """
from activepapers.contents import data
import numpy as np
data['frequency'] = 0.2
data['time'] = 0.1*np.arange(100)
""")
init.run()
calc_sine = paper.create_calclet(
"calc_sine", """
from activepapers.contents import data
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def test_zenodo_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.5281/zenodo.7648")
assert local_name == os.path.join(t, "10.5281/zenodo.7648.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group['python-packages/mosaic/__init__'].
attrs['ACTIVE_PAPER_DATATYPE']) == 'module'
paper.close()
def test_figshare_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.6084/m9.figshare.692144")
assert local_name == os.path.join(t, "10.6084/m9.figshare.692144.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group['python-packages/immutable/__init__'].
attrs['ACTIVE_PAPER_DATATYPE']) == 'module'
paper.close()
def test_simple_paper():
with tempdir.TempDir() as t:
filename1 = os.path.join(t, "paper1.ap")
filename2 = os.path.join(t, "paper2.ap")
make_paper(filename1)
check_paper(filename1)
with ActivePaper(filename1, "r") as paper:
paper.rebuild(filename2)
check_paper(filename2)
def test_import_ctypes():
# ctypes is not in the "allowed module" list, so this should fail
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
script = paper.create_calclet("test", """
import ctypes
""")
script.run()
paper.close()
def test_import_forbidden():
# distutils is a forbidden module from the standard library
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
script = paper.create_calclet("script",
"""
import distutils
""")
script.run()
paper.close()
def make_simple_paper_with_copies(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.create_copy("/data/frequency", paper_ref)
paper.create_copy("/data/time", paper_ref)
paper.create_copy("/code/calc_sine", paper_ref)
paper.run_codelet('calc_sine')
paper.close()
def make_simple_paper_with_data_and_code_refs(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.create_data_ref("frequency", paper_ref)
paper.create_data_ref("time", paper_ref)
paper.create_code_ref("calc_sine", paper_ref)
paper.run_codelet('calc_sine')
paper.close()
def make_library_paper(filename):
paper = ActivePaper(filename, "w")
paper.add_module(
"my_math", """
import numpy as np
def my_func(x):
return np.sin(x)
""")
paper.close()
def make_paper_with_module(filename, value):
paper = ActivePaper(filename, "w")
paper.add_module("some_values", """
a_value = %d
""" % value)
script = paper.create_calclet(
"test", """
from activepapers.contents import data
from some_values import a_value
data['a_value'] = a_value
""")
script.run()
paper.close()
def check_paper_with_refs(filename, with_name_postfix, refs, additional_items):
time_ds_name = '/data/time_from_ref' if with_name_postfix else '/data/time'
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == sorted(
['/code/calc_sine', '/data/frequency', '/data/sine', time_ds_name] +
additional_items)
for item_name in refs:
assert paper.data_group[item_name].attrs['ACTIVE_PAPER_DATATYPE'] \
== 'reference'
assert_almost_equal(paper.data["sine"][...],
np.sin(0.04 * np.pi * np.arange(100)), 1.e-10)
paper.close()
def make_paper(filename):
paper = ActivePaper(filename, "w")
paper.import_module('foo')
paper.import_module('foo.bar')
script = paper.create_calclet(
"test", """
from activepapers.contents import data
import foo
from foo.bar import frobnicate
data['result'] = frobnicate(2)
assert frobnicate(foo.__version__) == '42'
""")
script.run()
paper.close()
def make_local_paper(filename):
paper = ActivePaper(filename, "w")
paper.data.create_dataset("frequency", data=0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
paper.add_module("my_math",
"""
import numpy as np
def my_func(x):
return np.sin(x)
""")
paper.close()
def test_groups_as_items():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group1 = paper.data.create_group('group1')
group1.create_dataset('value', data=42)
group2 = paper.data.create_group('group2')
group2.mark_as_data_item()
group2.create_dataset('array', data=np.arange(10))
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/data/group1/value', '/data/group2']
groups = sorted([group.name for group in paper.iter_groups()])
assert groups == ['/data/group1']
script = paper.create_calclet("script1",
"""
from activepapers.contents import data
x1 = data['group2']['array'][...]
x2 = data['group1']['value'][...]
data.create_dataset('sum1', data=x1+x2)
""")
script.run()
assert (paper.data['sum1'][...] == np.arange(42,52)).all()
script = paper.create_calclet("script2",
"""
from activepapers.contents import data
x1 = data['/group2/array'][...]
g = data['group1']
x2 = g['/group1/value'][...]
data.create_dataset('sum2', data=x1+x2)
""")
script.run()
assert (paper.data['sum2'][...] == np.arange(42,52)).all()
deps = [sorted([ascii(item.name) for item in level])
for level in paper.dependency_hierarchy()]
assert deps == [['/code/script1', '/code/script2',
'/data/group1/value', '/data/group2'],
['/data/sum1', '/data/sum2']]
deps = paper.data['sum1']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script1',
'/data/group1/value', '/data/group2']
deps = paper.data['sum2']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script2',
'/data/group1/value', '/data/group2']
paper.close()
def test_attrs():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group = paper.data.create_group('group')
ds = group.create_dataset('value', data=42)
group.mark_as_data_item()
assert len(group.attrs) == 0
group.attrs['foo'] = 'bar'
assert len(group.attrs) == 1
assert list(group.attrs) == ['foo']
assert group.attrs['foo'] == 'bar'
assert len(ds.attrs) == 0
ds.attrs['foo'] = 'bar'
assert len(ds.attrs) == 1
assert list(ds.attrs) == ['foo']
assert ds.attrs['foo'] == 'bar'
paper.close()
def make_simple_paper_with_data_refs(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.create_data_ref("frequency", paper_ref)
paper.create_data_ref("time_from_ref", paper_ref, "time")
calc_sine = paper.create_calclet(
"calc_sine", """
from activepapers.contents import data
import numpy as np
frequency = data['frequency'][...]
time = data['time_from_ref'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def check_paper(filename):
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == [
"/code/python-packages/foo/__init__", "/code/python-packages/foo/bar",
"/code/test", "/data/result"
]
deps = [
sorted(item.name for item in level)
for level in paper.dependency_hierarchy()
]
assert deps == [[
'/code/python-packages/foo/__init__', '/code/python-packages/foo/bar',
'/code/test'
], ['/data/result']]
for path in ['foo/__init__', 'foo/bar']:
node = paper.code_group['python-packages'][path]
assert_is_python_module(node)
paper.close()
def test_copy():
with tempdir.TempDir() as t:
library.library = [t]
os.mkdir(os.path.join(t, "local"))
filename1 = os.path.join(t, "local/simple1.ap")
filename2 = os.path.join(t, "simple2.ap")
make_simple_paper(filename1)
make_simple_paper_with_copies(filename2, "local:simple1")
check_paper_with_refs(filename2, False, [], [])
paper = ActivePaper(filename2, 'r')
for path in ['/code/calc_sine', '/data/frequency', '/data/time']:
item = paper.file[path]
source = item.attrs.get('ACTIVE_PAPER_COPIED_FROM')
assert source is not None
paper_ref, ref_path = source
if h5py.version.version_tuple[:2] <= (2, 2):
paper_ref = paper_ref.flat[0]
ref_path = ref_path.flat[0]
assert ascii(paper_ref) == "local:simple1"
assert ascii(ref_path) == path
def make_simple_paper_with_library_refs(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.data.create_dataset("frequency", data=0.2)
paper.data.create_dataset("time", data=0.1 * np.arange(100))
paper.create_module_ref("my_math", paper_ref)
calc_sine = paper.create_calclet(
"calc_sine", """
from activepapers.contents import data
import numpy as np
from my_math import my_func
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def test_datasets():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
dset = paper.data.create_dataset("MyDataset", (10,10,10), 'f')
assert len(dset) == 10
assert dset[0, 0, 0].shape == ()
assert dset[0, 2:10, 1:9:3].shape == (8, 3)
assert dset[:, ::2, 5].shape == (10, 5)
assert dset[0].shape == (10, 10)
assert dset[1, 5].shape == (10,)
assert dset[0, ...].shape == (10, 10)
assert dset[..., 6].shape == (10, 10)
array = np.arange(100)
dset = paper.data.create_dataset("MyArray", data=array)
assert len(dset) == 100
assert (dset[array > 50] == np.arange(51, 100)).all()
dset[:20] = 42
assert (dset[...] == np.array(20*[42]+list(range(20, 100)))).all()
paper.data['a_number'] = 42
assert paper.data['a_number'][()] == 42
paper.close()
def test_overwrite_internal_file():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("write1",
"""
from activepapers.contents import open
f = open('numbers', 'w')
for i in range(10):
f.write(str(i)+'\\n')
f.close()
""")
script.run()
script = paper.create_calclet("write2",
"""
from activepapers.contents import open
with open('numbers', 'w') as f:
for i in range(10):
f.write(str(i)+'\\n')
""")
script.run()
paper.close()
def test_dependencies():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset('e', data = np.e)
paper.data.create_dataset('pi', data = np.pi)
script = paper.create_calclet("script",
"""
from activepapers.contents import data
import numpy as np
e = data['e'][...]
sum = data.create_dataset('sum', shape=(1,), dtype=np.float)
pi = data['pi'][...]
sum[0] = e+pi
""")
script.run()
deps = [ascii(item.name)
for item in paper.iter_dependencies(paper.data['sum']._node)]
assert sorted(deps) == ['/code/script', '/data/e', '/data/pi']
assert not paper.is_stale(paper.data['sum']._node)
del paper.data['e']
paper.data['e'] = 0.
assert paper.is_stale(paper.data['sum']._node)
paper.close()
