def test_groups():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group = paper.data.create_group('group')
subgroup = group.create_group('subgroup')
group['data1'] = np.arange(10)
group['data2'] = 42
assert sorted([g.name for g in paper.iter_groups()]) \
== ['/data/group', '/data/group/subgroup']
assert sorted(list(node for node in group)) \
== ['data1', 'data2', 'subgroup']
assert group['data1'][...].shape == (10,)
assert group['data2'][...] == 42
assert paper.data.parent is paper.data
assert group.parent is paper.data
assert group['data1'].parent is group
assert group['data2'].parent is group
script = paper.create_calclet("script",
"""
from activepapers.contents import data
assert data.parent is data
assert data._codelet is not None
assert data._codelet.path == '/code/script'
group = data['group']
assert group.parent is data
assert group._codelet is not None
assert group._codelet.path == '/code/script'
""")
script.run()
paper.close()
def test_import_math():
# math is an extension module, so this should fail
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
paper.import_module('math')
paper.close()
def make_simple_paper(filename):
paper = ActivePaper(filename, "w")
#paper.data.create_dataset("frequency", data=0.2)
#paper.data.create_dataset("time", data=0.1*np.arange(100))
init = paper.create_importlet("initialize",
"""
from activepapers.contents import data
import numpy as np
data['frequency'] = 0.2
data['time'] = 0.1*np.arange(100)
""")
init.run()
calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def test_figshare_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.6084/m9.figshare.692144")
assert local_name == os.path.join(t, "10.6084/m9.figshare.692144.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group["python-packages/immutable/__init__"].attrs["ACTIVE_PAPER_DATATYPE"]) == "module"
paper.close()
def test_zenodo_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.5281/zenodo.7648")
assert local_name == os.path.join(t, "10.5281/zenodo.7648.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group["python-packages/mosaic/__init__"].attrs["ACTIVE_PAPER_DATATYPE"]) == "module"
paper.close()
def test_zenodo_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.5281/zenodo.7648")
assert local_name == os.path.join(t, "10.5281/zenodo.7648.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group['python-packages/mosaic/__init__'].
attrs['ACTIVE_PAPER_DATATYPE']) == 'module'
paper.close()
def test_figshare_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.6084/m9.figshare.692144")
assert local_name == os.path.join(t, "10.6084/m9.figshare.692144.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group['python-packages/immutable/__init__'].
attrs['ACTIVE_PAPER_DATATYPE']) == 'module'
paper.close()
def test_import_ctypes():
# ctypes is not in the "allowed module" list, so this should fail
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
script = paper.create_calclet("test", """
import ctypes
""")
script.run()
paper.close()
def test_dummy_datasets():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_angular.run()
paper.replace_by_dummy('/data/angular/sine')
dummy = paper.data_group['angular/sine']
assert dummy.attrs.get('ACTIVE_PAPER_GENERATING_CODELET') \
== '/code/calc_angular'
assert dummy.attrs.get('ACTIVE_PAPER_DUMMY_DATASET', False)
passed = True
try:
paper.replace_by_dummy('/data/time')
except AssertionError:
passed = False
assert not passed
paper.close()
def test_import_forbidden():
# distutils is a forbidden module from the standard library
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
script = paper.create_calclet("script",
"""
import distutils
""")
script.run()
paper.close()
def test_import_forbidden():
# distutils is a forbidden module from the standard library
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
script = paper.create_calclet("script",
"""
import distutils
""")
script.run()
paper.close()
def test_import_ctypes():
# ctypes is not in the "allowed module" list, so this should fail
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
script = paper.create_calclet("test",
"""
import ctypes
""")
script.run()
paper.close()
def check_paper_with_refs(filename, with_name_postfix, refs, additional_items):
time_ds_name = '/data/time_from_ref' if with_name_postfix else '/data/time'
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == sorted(['/code/calc_sine', '/data/frequency',
'/data/sine', time_ds_name] + additional_items)
for item_name in refs:
assert paper.data_group[item_name].attrs['ACTIVE_PAPER_DATATYPE'] \
== 'reference'
assert_almost_equal(paper.data["sine"][...],
np.sin(0.04*np.pi*np.arange(100)),
1.e-10)
paper.close()
def make_library_paper(filename):
paper = ActivePaper(filename, "w")
paper.add_module("my_math",
"""
import numpy as np
def my_func(x):
return np.sin(x)
""")
paper.close()
def check_paper_with_refs(filename, with_name_postfix, refs, additional_items):
time_ds_name = '/data/time_from_ref' if with_name_postfix else '/data/time'
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == sorted(
['/code/calc_sine', '/data/frequency', '/data/sine', time_ds_name] +
additional_items)
for item_name in refs:
assert paper.data_group[item_name].attrs['ACTIVE_PAPER_DATATYPE'] \
== 'reference'
assert_almost_equal(paper.data["sine"][...],
np.sin(0.04 * np.pi * np.arange(100)), 1.e-10)
paper.close()
def test_modified_scripts():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=42)
group = data.create_group('group1')
group.mark_as_data_item()
group['value'] = 1
group = data.create_group('group2')
group['value'] = 2
""")
script.run()
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/code/script', '/data/foo',
'/data/group1', '/data/group2/value']
assert (paper.data['foo'][...] == 42)
assert (paper.data['group1/value'][...] == 1)
assert (paper.data['group2/value'][...] == 2)
script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=1)
""")
script.run()
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/code/script', '/data/foo']
assert (paper.data['foo'][...] == 1)
paper.close()
def test_dummy_datasets():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_angular.run()
paper.replace_by_dummy('/data/angular/sine')
dummy = paper.data_group['angular/sine']
assert dummy.attrs.get('ACTIVE_PAPER_GENERATING_CODELET') \
== '/code/calc_angular'
assert dummy.attrs.get('ACTIVE_PAPER_DUMMY_DATASET', False)
passed = True
try:
paper.replace_by_dummy('/data/time')
except AssertionError:
passed = False
assert not passed
paper.close()
def test_snapshots():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
snapshot_1 = os.path.join(t, "snapshot_1.ap")
snapshot_2 = os.path.join(t, "snapshot_2.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("cosine", data=np.cos(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("tangent", data=np.tan(2.*np.pi*frequency*time))
""" % (snapshot_1, snapshot_2))
calc_angular.run()
paper.close()
# Open the snapshot files to verify they are valid ActivePapers
ActivePaper(snapshot_1, 'r').close()
ActivePaper(snapshot_2, 'r').close()
# Check the contents
paper = h5py.File(filename)
snapshot_1 = h5py.File(snapshot_1)
snapshot_2 = h5py.File(snapshot_2)
for item in ['/data/time', '/data/frequency', '/data/angular/sine',
'/code/calc_angular']:
assert item in paper
assert item in snapshot_1
assert item in snapshot_2
assert '/data/angular/cosine' in paper
assert '/data/angular/cosine' not in snapshot_1
assert '/data/angular/cosine' in snapshot_2
assert '/data/angular/tangent' in paper
assert '/data/angular/tangent' not in snapshot_1
assert '/data/angular/tangent' not in snapshot_2
for root in [snapshot_1, snapshot_2]:
#time_ref = root['/data/angular/time'][()]
#assert root[time_ref].name == '/data/time'
time_ref = root['/data/angular'].attrs['time']
assert root[time_ref].name == '/data/time'
def test_simple_paper():
with tempdir.TempDir() as t:
filename1 = os.path.join(t, "simple1.ap")
filename2 = os.path.join(t, "simple2.ap")
make_simple_paper(filename1)
all_paths = [
'README', 'code', 'code/calc_sine', 'code/initialize', 'data',
'data/frequency', 'data/sine', 'data/time', 'documentation',
'external-dependencies', 'history'
]
all_items = [
'/code/calc_sine', '/code/initialize', '/data/frequency',
'/data/sine', '/data/time'
]
all_deps = {
'/data/sine': ["/code/calc_sine", "/data/frequency", "/data/time"],
'/data/time': [],
'/data/frequency': [],
'/code/calc_sine': [],
'/code/initialize': []
}
sine_deps = ["/code/calc_sine", "/data/frequency", "/data/time"]
hierarchy = [[
'/code/calc_sine', '/code/initialize', '/data/frequency',
'/data/time'
], ['/data/sine']]
check_hdf5_file(filename1, all_paths, sine_deps)
check_paper(filename1, all_items, all_deps, hierarchy)
with ActivePaper(filename1, "r") as paper:
paper.rebuild(filename2)
check_hdf5_file(filename2, all_paths, sine_deps)
check_paper(filename2, all_items, all_deps, hierarchy)
def check_paper(filename):
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == ["/code/python-packages/foo/__init__",
"/code/python-packages/foo/bar",
"/code/test",
"/data/result"]
deps = [sorted(item.name for item in level)
for level in paper.dependency_hierarchy()]
assert deps == [['/code/python-packages/foo/__init__',
'/code/python-packages/foo/bar',
'/code/test'],
['/data/result']]
for path in ['foo/__init__', 'foo/bar']:
node = paper.code_group['python-packages'][path]
assert_is_python_module(node)
paper.close()
def make_paper(filename):
paper = ActivePaper(filename, "w")
paper.import_module('foo')
paper.import_module('foo.bar')
script = paper.create_calclet(
"test", """
from activepapers.contents import data
import foo
from foo.bar import frobnicate
data['result'] = frobnicate(2)
assert frobnicate(foo.__version__) == '42'
""")
script.run()
paper.close()
def make_paper_with_internal_module(filename):
paper = ActivePaper(filename, "w")
paper.add_module(
"my_math", """
import numpy as np
def my_func(x):
return np.sin(x)
""")
paper.data.create_dataset("frequency", data=0.2)
paper.data.create_dataset("time", data=0.1 * np.arange(100))
calc_sine = paper.create_calclet(
"calc_sine", """
from activepapers.contents import data
import numpy as np
from my_math import my_func
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def test_modified_scripts():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=42)
group = data.create_group('group1')
group.mark_as_data_item()
group['value'] = 1
group = data.create_group('group2')
group['value'] = 2
""")
script.run()
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/code/script', '/data/foo',
'/data/group1', '/data/group2/value']
assert (paper.data['foo'][...] == 42)
assert (paper.data['group1/value'][...] == 1)
assert (paper.data['group2/value'][...] == 2)
script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=1)
""")
script.run()
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/code/script', '/data/foo']
assert (paper.data['foo'][...] == 1)
paper.close()
def test_simple_paper():
with tempdir.TempDir() as t:
filename1 = os.path.join(t, "paper1.ap")
filename2 = os.path.join(t, "paper2.ap")
make_paper(filename1)
check_paper(filename1)
with ActivePaper(filename1, "r") as paper:
paper.rebuild(filename2)
check_paper(filename2)
def test_attrs():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group = paper.data.create_group('group')
ds = group.create_dataset('value', data=42)
group.mark_as_data_item()
assert len(group.attrs) == 0
group.attrs['foo'] = 'bar'
assert len(group.attrs) == 1
assert list(group.attrs) == ['foo']
assert group.attrs['foo'] == 'bar'
assert len(ds.attrs) == 0
ds.attrs['foo'] = 'bar'
assert len(ds.attrs) == 1
assert list(ds.attrs) == ['foo']
assert ds.attrs['foo'] == 'bar'
paper.close()
def test_attrs():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group = paper.data.create_group('group')
ds = group.create_dataset('value', data=42)
group.mark_as_data_item()
assert len(group.attrs) == 0
group.attrs['foo'] = 'bar'
assert len(group.attrs) == 1
assert list(group.attrs) == ['foo']
assert group.attrs['foo'] == 'bar'
assert len(ds.attrs) == 0
ds.attrs['foo'] = 'bar'
assert len(ds.attrs) == 1
assert list(ds.attrs) == ['foo']
assert ds.attrs['foo'] == 'bar'
paper.close()
def make_simple_paper(filename):
paper = ActivePaper(filename, "w")
paper.data.create_dataset("frequency", data=0.2)
paper.data.create_dataset("time", data=0.1 * np.arange(100))
calc_sine = paper.create_calclet(
"calc_sine", """
from activepapers.contents import data
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def test_groups_as_items():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group1 = paper.data.create_group('group1')
group1.create_dataset('value', data=42)
group2 = paper.data.create_group('group2')
group2.mark_as_data_item()
group2.create_dataset('array', data=np.arange(10))
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/data/group1/value', '/data/group2']
groups = sorted([group.name for group in paper.iter_groups()])
assert groups == ['/data/group1']
script = paper.create_calclet("script1",
"""
from activepapers.contents import data
x1 = data['group2']['array'][...]
x2 = data['group1']['value'][...]
data.create_dataset('sum1', data=x1+x2)
""")
script.run()
assert (paper.data['sum1'][...] == np.arange(42,52)).all()
script = paper.create_calclet("script2",
"""
from activepapers.contents import data
x1 = data['/group2/array'][...]
g = data['group1']
x2 = g['/group1/value'][...]
data.create_dataset('sum2', data=x1+x2)
""")
script.run()
assert (paper.data['sum2'][...] == np.arange(42,52)).all()
deps = [sorted([ascii(item.name) for item in level])
for level in paper.dependency_hierarchy()]
assert deps == [['/code/script1', '/code/script2',
'/data/group1/value', '/data/group2'],
['/data/sum1', '/data/sum2']]
deps = paper.data['sum1']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script1',
'/data/group1/value', '/data/group2']
deps = paper.data['sum2']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script2',
'/data/group1/value', '/data/group2']
paper.close()
def make_paper_with_internal_module(filename):
paper = ActivePaper(filename, "w")
paper.add_module("my_math",
"""
import numpy as np
def my_func(x):
return np.sin(x)
""")
paper.data.create_dataset("frequency", data=0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np
from my_math import my_func
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def make_paper(filename):
paper = ActivePaper(filename, "w")
paper.import_module('foo')
paper.import_module('foo.bar')
script = paper.create_calclet("test",
"""
from activepapers.contents import data
import foo
from foo.bar import frobnicate
data['result'] = frobnicate(2)
assert frobnicate(foo.__version__) == '42'
""")
script.run()
paper.close()
def test_groups():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group = paper.data.create_group('group')
subgroup = group.create_group('subgroup')
group['data1'] = np.arange(10)
group['data2'] = 42
assert sorted([g.name for g in paper.iter_groups()]) \
== ['/data/group', '/data/group/subgroup']
assert sorted(list(node for node in group)) \
== ['data1', 'data2', 'subgroup']
assert group['data1'][...].shape == (10,)
assert group['data2'][...] == 42
assert paper.data.parent is paper.data
assert group.parent is paper.data
assert group['data1'].parent is group
assert group['data2'].parent is group
script = paper.create_calclet("script",
"""
from activepapers.contents import data
assert data.parent is data
assert data._codelet is not None
assert data._codelet.path == '/code/script'
group = data['group']
assert group.parent is data
assert group._codelet is not None
assert group._codelet.path == '/code/script'
""")
script.run()
paper.close()
def test_import_math():
# math is an extension module, so this should fail
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, "w")
paper.import_module('math')
paper.close()
def test_datasets():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
dset = paper.data.create_dataset("MyDataset", (10,10,10), 'f')
assert len(dset) == 10
assert dset[0, 0, 0].shape == ()
assert dset[0, 2:10, 1:9:3].shape == (8, 3)
assert dset[:, ::2, 5].shape == (10, 5)
assert dset[0].shape == (10, 10)
assert dset[1, 5].shape == (10,)
assert dset[0, ...].shape == (10, 10)
assert dset[..., 6].shape == (10, 10)
array = np.arange(100)
dset = paper.data.create_dataset("MyArray", data=array)
assert len(dset) == 100
assert (dset[array > 50] == np.arange(51, 100)).all()
dset[:20] = 42
assert (dset[...] == np.array(20*[42]+list(range(20, 100)))).all()
paper.data['a_number'] = 42
assert paper.data['a_number'][()] == 42
paper.close()
def test_datasets():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
dset = paper.data.create_dataset("MyDataset", (10,10,10), 'f')
assert len(dset) == 10
assert dset[0, 0, 0].shape == ()
assert dset[0, 2:10, 1:9:3].shape == (8, 3)
assert dset[:, ::2, 5].shape == (10, 5)
assert dset[0].shape == (10, 10)
assert dset[1, 5].shape == (10,)
assert dset[0, ...].shape == (10, 10)
assert dset[..., 6].shape == (10, 10)
array = np.arange(100)
dset = paper.data.create_dataset("MyArray", data=array)
assert len(dset) == 100
assert (dset[array > 50] == np.arange(51, 100)).all()
dset[:20] = 42
assert (dset[...] == np.array(20*[42]+list(range(20, 100)))).all()
paper.data['a_number'] = 42
assert paper.data['a_number'][()] == 42
paper.close()
def make_simple_paper_with_data_refs(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.create_data_ref("frequency", paper_ref)
paper.create_data_ref("time_from_ref", paper_ref, "time")
calc_sine = paper.create_calclet(
"calc_sine", """
from activepapers.contents import data
import numpy as np
frequency = data['frequency'][...]
time = data['time_from_ref'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def test_overwrite_internal_file():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("write1",
"""
from activepapers.contents import open
f = open('numbers', 'w')
for i in range(10):
f.write(str(i)+'\\n')
f.close()
""")
script.run()
script = paper.create_calclet("write2",
"""
from activepapers.contents import open
with open('numbers', 'w') as f:
for i in range(10):
f.write(str(i)+'\\n')
""")
script.run()
paper.close()
def make_paper_with_module(filename, value):
paper = ActivePaper(filename, "w")
paper.add_module("some_values", """
a_value = %d
""" % value)
script = paper.create_calclet(
"test", """
from activepapers.contents import data
from some_values import a_value
data['a_value'] = a_value
""")
script.run()
paper.close()
def test_groups_as_items():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group1 = paper.data.create_group('group1')
group1.create_dataset('value', data=42)
group2 = paper.data.create_group('group2')
group2.mark_as_data_item()
group2.create_dataset('array', data=np.arange(10))
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/data/group1/value', '/data/group2']
groups = sorted([group.name for group in paper.iter_groups()])
assert groups == ['/data/group1']
script = paper.create_calclet("script1",
"""
from activepapers.contents import data
x1 = data['group2']['array'][...]
x2 = data['group1']['value'][...]
data.create_dataset('sum1', data=x1+x2)
""")
script.run()
assert (paper.data['sum1'][...] == np.arange(42,52)).all()
script = paper.create_calclet("script2",
"""
from activepapers.contents import data
x1 = data['/group2/array'][...]
g = data['group1']
x2 = g['/group1/value'][...]
data.create_dataset('sum2', data=x1+x2)
""")
script.run()
assert (paper.data['sum2'][...] == np.arange(42,52)).all()
deps = [sorted([ascii(item.name) for item in level])
for level in paper.dependency_hierarchy()]
assert deps == [['/code/script1', '/code/script2',
'/data/group1/value', '/data/group2'],
['/data/sum1', '/data/sum2']]
deps = paper.data['sum1']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script1',
'/data/group1/value', '/data/group2']
deps = paper.data['sum2']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script2',
'/data/group1/value', '/data/group2']
paper.close()
def test_snapshots():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
snapshot_1 = os.path.join(t, "snapshot_1.ap")
snapshot_2 = os.path.join(t, "snapshot_2.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("cosine", data=np.cos(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("tangent", data=np.tan(2.*np.pi*frequency*time))
""" % (snapshot_1, snapshot_2))
calc_angular.run()
paper.close()
# Open the snapshot files to verify they are valid ActivePapers
ActivePaper(snapshot_1, 'r').close()
ActivePaper(snapshot_2, 'r').close()
# Check the contents
paper = h5py.File(filename)
snapshot_1 = h5py.File(snapshot_1)
snapshot_2 = h5py.File(snapshot_2)
for item in ['/data/time', '/data/frequency', '/data/angular/sine',
'/code/calc_angular']:
assert item in paper
assert item in snapshot_1
assert item in snapshot_2
assert '/data/angular/cosine' in paper
assert '/data/angular/cosine' not in snapshot_1
assert '/data/angular/cosine' in snapshot_2
assert '/data/angular/tangent' in paper
assert '/data/angular/tangent' not in snapshot_1
assert '/data/angular/tangent' not in snapshot_2
for root in [snapshot_1, snapshot_2]:
#time_ref = root['/data/angular/time'][()]
#assert root[time_ref].name == '/data/time'
time_ref = root['/data/angular'].attrs['time']
assert root[time_ref].name == '/data/time'
def make_simple_paper_with_data_refs(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.create_data_ref("frequency", paper_ref)
paper.create_data_ref("time_from_ref", paper_ref, "time")
calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np
frequency = data['frequency'][...]
time = data['time_from_ref'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def make_library_paper(filename):
paper = ActivePaper(filename, "w")
paper.add_module(
"my_math", """
import numpy as np
def my_func(x):
return np.sin(x)
""")
paper.close()
class ActivePaper(object):
def __init__(self, file_or_ref, use_code=True):
global _paper_for_code
try:
self.paper = open_paper_ref(file_or_ref)
except ValueError:
self.paper = ActivePaperStorage(file_or_ref, 'r')
if use_code and ("python-packages" not in self.paper.code_group \
or len(self.paper.code_group["python-packages"]) == 0):
# The paper contains no importable modules or packages.
use_code = False
if use_code and _paper_for_code is not None:
raise IOError("Only one ActivePaper per process can use code.")
self.data = self.paper.data
self.documentation = self.paper.documentation_group
self.code = self.paper.code_group
try:
self.__doc__ = self.open_documentation('README').read()
except KeyError:
pass
if use_code:
_paper_for_code = self.paper
def close(self):
global _paper_for_code
if _paper_for_code is self.paper:
_paper_for_code = None
def _open(self, path, section, mode='r'):
if mode not in ['r', 'rb']:
raise ValueError("invalid mode: " + repr(mode))
path = path_in_section(path, section)
if not path.startswith('/'):
path = section + '/' + path
return self.paper.open_internal_file(path, mode, 'utf8', None)
def open(self, path, mode='r'):
return self._open(path, '/data', mode)
def open_documentation(self, path, mode='r'):
return self._open(path, '/documentation', mode)
def read_code(self, file):
return self.code[file][...].ravel()[0].decode('utf-8')
def test_dependencies():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset('e', data = np.e)
paper.data.create_dataset('pi', data = np.pi)
script = paper.create_calclet("script",
"""
from activepapers.contents import data
import numpy as np
e = data['e'][...]
sum = data.create_dataset('sum', shape=(1,), dtype=np.float)
pi = data['pi'][...]
sum[0] = e+pi
""")
script.run()
deps = [ascii(item.name)
for item in paper.iter_dependencies(paper.data['sum']._node)]
assert sorted(deps) == ['/code/script', '/data/e', '/data/pi']
assert not paper.is_stale(paper.data['sum']._node)
del paper.data['e']
paper.data['e'] = 0.
assert paper.is_stale(paper.data['sum']._node)
paper.close()
def make_paper_with_module(filename, value):
paper = ActivePaper(filename, "w")
paper.add_module("some_values",
"""
a_value = %d
""" % value)
script = paper.create_calclet("test",
"""
from activepapers.contents import data
from some_values import a_value
data['a_value'] = a_value
""")
script.run()
paper.close()
def test_copy():
with tempdir.TempDir() as t:
library.library = [t]
os.mkdir(os.path.join(t, "local"))
filename1 = os.path.join(t, "local/simple1.ap")
filename2 = os.path.join(t, "simple2.ap")
make_simple_paper(filename1)
make_simple_paper_with_copies(filename2, "local:simple1")
check_paper_with_refs(filename2, False, [], [])
paper = ActivePaper(filename2, 'r')
for path in ['/code/calc_sine', '/data/frequency', '/data/time']:
item = paper.file[path]
source = item.attrs.get('ACTIVE_PAPER_COPIED_FROM')
assert source is not None
paper_ref, ref_path = source
if h5py.version.version_tuple[:2] <= (2, 2):
paper_ref = paper_ref.flat[0]
ref_path = ref_path.flat[0]
assert ascii(paper_ref) == "local:simple1"
assert ascii(ref_path) == path
def __init__(self, file_or_ref, use_code=True):
global _paper_for_code
try:
self.paper = open_paper_ref(file_or_ref)
except ValueError:
self.paper = ActivePaperStorage(file_or_ref, 'r')
if use_code and ("python-packages" not in self.paper.code_group \
or len(self.paper.code_group["python-packages"]) == 0):
# The paper contains no importable modules or packages.
use_code = False
if use_code and _paper_for_code is not None:
raise IOError("Only one ActivePaper per process can use code.")
self.data = self.paper.data
self.documentation = self.paper.documentation_group
self.code = self.paper.code_group
try:
self.__doc__ = self.open_documentation('README').read()
except KeyError:
pass
if use_code:
_paper_for_code = self.paper
def make_local_paper(filename):
paper = ActivePaper(filename, "w")
paper.data.create_dataset("frequency", data=0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
paper.add_module("my_math",
"""
import numpy as np
def my_func(x):
return np.sin(x)
""")
paper.close()
def test_dependencies():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset('e', data = np.e)
paper.data.create_dataset('pi', data = np.pi)
script = paper.create_calclet("script",
"""
from activepapers.contents import data
import numpy as np
e = data['e'][...]
sum = data.create_dataset('sum', shape=(1,), dtype=np.float)
pi = data['pi'][...]
sum[0] = e+pi
""")
script.run()
deps = [ascii(item.name)
for item in paper.iter_dependencies(paper.data['sum']._node)]
assert sorted(deps) == ['/code/script', '/data/e', '/data/pi']
assert not paper.is_stale(paper.data['sum']._node)
del paper.data['e']
paper.data['e'] = 0.
assert paper.is_stale(paper.data['sum']._node)
paper.close()
def make_simple_paper_with_library_refs(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.data.create_dataset("frequency", data=0.2)
paper.data.create_dataset("time", data=0.1 * np.arange(100))
paper.create_module_ref("my_math", paper_ref)
calc_sine = paper.create_calclet(
"calc_sine", """
from activepapers.contents import data
import numpy as np
from my_math import my_func
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def check_paper(filename):
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == [
"/code/python-packages/foo/__init__", "/code/python-packages/foo/bar",
"/code/test", "/data/result"
]
deps = [
sorted(item.name for item in level)
for level in paper.dependency_hierarchy()
]
assert deps == [[
'/code/python-packages/foo/__init__', '/code/python-packages/foo/bar',
'/code/test'
], ['/data/result']]
for path in ['foo/__init__', 'foo/bar']:
node = paper.code_group['python-packages'][path]
assert_is_python_module(node)
paper.close()
def make_simple_paper_with_library_refs(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
paper.create_module_ref("my_math", paper_ref)
calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np
from my_math import my_func
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
def test_overwrite_internal_file():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("write1",
"""
from activepapers.contents import open
f = open('numbers', 'w')
for i in range(10):
f.write(str(i)+'\\n')
f.close()
""")
script.run()
script = paper.create_calclet("write2",
"""
from activepapers.contents import open
with open('numbers', 'w') as f:
for i in range(10):
f.write(str(i)+'\\n')
""")
script.run()
paper.close()
from activepapers.storage import ActivePaper
import numpy as np
paper = ActivePaper("plotting.ap", "w",
dependencies = ["matplotlib"])
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
plot_sine = paper.create_calclet("plot_sine",
"""
from activepapers.contents import open, data
import matplotlib
# Make matplotlib ignore the user's .matplotlibrc
matplotlib.rcdefaults()
# Use the SVG backend. Must be done *before* importing pyplot.
matplotlib.use('SVG')
import matplotlib.pyplot as plt
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
sine = np.sin(2.*np.pi*frequency*time)
plt.plot(time, sine)
# Save plot to a file, which is simulated by a HDF5 byte array
with open('sine_plot.svg', 'w') as output:
plt.savefig(output)
""")
plot_sine.run()
def make_simple_paper_with_data_and_code_refs(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.create_data_ref("frequency", paper_ref)
paper.create_data_ref("time", paper_ref)
paper.create_code_ref("calc_sine", paper_ref)
paper.run_codelet('calc_sine')
paper.close()
from activepapers.storage import ActivePaper
import numpy as np
paper = ActivePaper("simple.ap", "w")
paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))
calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np
frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_sine.run()
paper.close()
import os
os.environ['ACTIVEPAPERS_LIBRARY'] = os.getcwd()
from activepapers.storage import ActivePaper
import numpy as np
paper = ActivePaper("ref_to_simple.ap", "w")
paper.create_data_ref("frequency", "local:simple")
paper.create_data_ref("time", "local:simple", "time")
paper.create_code_ref("calc_sine", "local:simple", "calc_sine")
paper.run_codelet('calc_sine')
paper.close()
def make_simple_paper_with_copies(filename, paper_ref):
paper = ActivePaper(filename, "w")
paper.create_copy("/data/frequency", paper_ref)
paper.create_copy("/data/time", paper_ref)
paper.create_copy("/code/calc_sine", paper_ref)
paper.run_codelet('calc_sine')
paper.close()
def check_paper(filename, ref_items, ref_deps, ref_hierarchy):
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == ref_items
items_with_deps = sorted([item.name for item in paper.iter_items()
if paper.has_dependencies(item)])
assert items_with_deps == ['/data/sine']
deps = dict((ascii(item.name),
sorted(list(ascii(dep.name)
for dep in paper.iter_dependencies(item))))
for item in paper.iter_items())
assert deps == ref_deps
graph = collections.defaultdict(set)
for item, deps in ref_deps.items():
for d in deps:
graph[d].add(item)
assert graph == paper.dependency_graph()
hierarchy = [sorted([ascii(item.name) for item in items])
for items in paper.dependency_hierarchy()]
assert hierarchy == ref_hierarchy
calclets = paper.calclets()
assert len(calclets) == 1
assert ascii(calclets['/code/calc_sine'].path) == '/code/calc_sine'
paper.close()
# a specific module on the fly. This generic module
# is used when activepapers.contents is imported from a
# standard Python script. It is meant to be facilitate
# development of codelets for ActivePaper in a standard
# Python development environment.
# Locate the (hopefully only) ActivePaper in the current directory
import os
apfiles = [fn for fn in os.listdir('.') if fn.endswith('.ap')]
if len(apfiles) != 1:
raise IOError("directory contains %s ActivePapers" % len(apfiles))
del os
# Open the paper read-only
from activepapers.storage import ActivePaper
_paper = ActivePaper(apfiles[0], 'r')
del apfiles
del ActivePaper
# Emulate the internal activepapers.contents module
data = _paper.data
def _open(filename, mode='r'):
from activepapers.utility import path_in_section
section = '/data'
path = path_in_section(path, section)
if not path.startswith('/'):
path = section + '/' + path
assert mode == 'r'
return _paper.open_internal_file(path, 'r', None)
def test_internal_files():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("write1",
"""
from activepapers.contents import open
f = open('numbers1', 'w')
for i in range(10):
f.write(str(i)+'\\n')
f.close()
""")
script.run()
script = paper.create_calclet("write2",
"""
from activepapers.contents import open
with open('numbers', 'w') as f:
for i in range(10):
f.write(str(i)+'\\n')
""")
script.run()
script = paper.create_calclet("write3",
"""
from activepapers.contents import open
with open('empty', 'w') as f:
pass
""")
script.run()
script = paper.create_calclet("read1",
"""
from activepapers.contents import open
f = open('numbers')
for i in range(10):
assert f.readline().strip() == str(i)
f.close()
""")
script.run()
script = paper.create_calclet("read2",
"""
from activepapers.contents import open
f = open('numbers')
data = [int(line.strip()) for line in f]
f.close()
assert data == list(range(10))
""")
script.run()
script = paper.create_calclet("read3",
"""
from activepapers.contents import open
f = open('empty')
data = f.read()
f.close()
assert len(data) == 0
""")
script.run()
script = paper.create_calclet("convert_to_binary",
"""
from activepapers.contents import open
import struct
with open('numbers') as f:
data = [int(line.strip()) for line in f]
f = open('binary_numbers', 'wb')
f.write(struct.pack(len(data)*'h', *data))
f.close()
""")
script.run()
script = paper.create_calclet("read_binary",
"""
from activepapers.contents import open
import struct
f = open('binary_numbers', 'rb')
assert struct.unpack(10*'h', f.read()) == tuple(range(10))
f.close()
""")
script.run()
script = paper.create_calclet("write_documentation",
"""
from activepapers.contents import open_documentation
with open_documentation('hello.txt', 'w') as f:
f.write('Hello world!\\n')
""")
script.run()
h = [sorted(list(ascii(item.name) for item in step))
for step in paper.dependency_hierarchy()]
assert h == [['/code/convert_to_binary',
'/code/read1', '/code/read2', '/code/read3',
'/code/read_binary',
'/code/write1', '/code/write2', '/code/write3',
'/code/write_documentation'],
['/data/empty', '/data/numbers', '/data/numbers1',
'/documentation/hello.txt'],
['/data/binary_numbers']]
paper.close()