def test_groups():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        group = paper.data.create_group('group')
        subgroup = group.create_group('subgroup')
        group['data1'] = np.arange(10)
        group['data2'] = 42
        assert sorted([g.name for g in paper.iter_groups()]) \
               == ['/data/group', '/data/group/subgroup']
        assert sorted(list(node for node in group)) \
               == ['data1', 'data2', 'subgroup']
        assert group['data1'][...].shape == (10,)
        assert group['data2'][...] == 42
        assert paper.data.parent is paper.data
        assert group.parent is paper.data
        assert group['data1'].parent is group
        assert group['data2'].parent is group
        script = paper.create_calclet("script",
"""
from activepapers.contents import data
assert data.parent is data
assert data._codelet is not None
assert data._codelet.path == '/code/script'
group = data['group']
assert group.parent is data
assert group._codelet is not None
assert group._codelet.path == '/code/script'
""")
        script.run()
        paper.close()
def test_import_math():
    # math is an extension module, so this should fail
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, "w")
        paper.import_module('math')
        paper.close()
def make_simple_paper(filename):

    paper = ActivePaper(filename, "w")

    #paper.data.create_dataset("frequency", data=0.2)
    #paper.data.create_dataset("time", data=0.1*np.arange(100))

    init = paper.create_importlet("initialize",
"""
from activepapers.contents import data
import numpy as np

data['frequency'] = 0.2
data['time'] = 0.1*np.arange(100)
""")
    init.run()

    calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np

frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
    calc_sine.run()

    paper.close()
def test_figshare_download():
    with tempdir.TempDir() as t:
        library.library = [t]
        local_name = library.find_in_library("doi:10.6084/m9.figshare.692144")
        assert local_name == os.path.join(t, "10.6084/m9.figshare.692144.ap")
        paper = ActivePaper(local_name)
        assert ascii(paper.code_group["python-packages/immutable/__init__"].attrs["ACTIVE_PAPER_DATATYPE"]) == "module"
        paper.close()
def test_zenodo_download():
    with tempdir.TempDir() as t:
        library.library = [t]
        local_name = library.find_in_library("doi:10.5281/zenodo.7648")
        assert local_name == os.path.join(t, "10.5281/zenodo.7648.ap")
        paper = ActivePaper(local_name)
        assert ascii(paper.code_group["python-packages/mosaic/__init__"].attrs["ACTIVE_PAPER_DATATYPE"]) == "module"
        paper.close()
Esempio n. 6
0
def test_zenodo_download():
    with tempdir.TempDir() as t:
        library.library = [t]
        local_name = library.find_in_library("doi:10.5281/zenodo.7648")
        assert local_name == os.path.join(t, "10.5281/zenodo.7648.ap")
        paper = ActivePaper(local_name)
        assert ascii(paper.code_group['python-packages/mosaic/__init__'].
                     attrs['ACTIVE_PAPER_DATATYPE']) == 'module'
        paper.close()
Esempio n. 7
0
def test_figshare_download():
    with tempdir.TempDir() as t:
        library.library = [t]
        local_name = library.find_in_library("doi:10.6084/m9.figshare.692144")
        assert local_name == os.path.join(t, "10.6084/m9.figshare.692144.ap")
        paper = ActivePaper(local_name)
        assert ascii(paper.code_group['python-packages/immutable/__init__'].
                     attrs['ACTIVE_PAPER_DATATYPE']) == 'module'
        paper.close()
Esempio n. 8
0
def test_import_ctypes():
    # ctypes is not in the "allowed module" list, so this should fail
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, "w")
        script = paper.create_calclet("test", """
import ctypes
""")
        script.run()
        paper.close()
Esempio n. 9
0
def test_dummy_datasets():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        paper.data.create_dataset("frequency", data = 0.2)
        paper.data.create_dataset("time", data=0.1*np.arange(100))
        calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np

frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
        calc_angular.run()
        paper.replace_by_dummy('/data/angular/sine')
        dummy = paper.data_group['angular/sine']
        assert dummy.attrs.get('ACTIVE_PAPER_GENERATING_CODELET') \
            ==  '/code/calc_angular'
        assert dummy.attrs.get('ACTIVE_PAPER_DUMMY_DATASET', False)
        passed = True
        try:
            paper.replace_by_dummy('/data/time')
        except AssertionError:
            passed = False
        assert not passed
        paper.close()
Esempio n. 10
0
def test_import_forbidden():
    # distutils is a forbidden module from the standard library
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, "w")
        script = paper.create_calclet("script",
"""
import distutils
""")
        script.run()
        paper.close()
def test_import_forbidden():
    # distutils is a forbidden module from the standard library
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, "w")
        script = paper.create_calclet("script",
"""
import distutils
""")
        script.run()
        paper.close()
def test_import_ctypes():
    # ctypes is not in the "allowed module" list, so this should fail
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, "w")
        script = paper.create_calclet("test",
"""
import ctypes
""")
        script.run()
        paper.close()
def check_paper_with_refs(filename, with_name_postfix, refs, additional_items):
    time_ds_name = '/data/time_from_ref' if with_name_postfix else '/data/time'
    paper = ActivePaper(filename, "r")
    items = sorted([item.name for item in paper.iter_items()])
    assert items == sorted(['/code/calc_sine', '/data/frequency',
                            '/data/sine', time_ds_name] + additional_items)
    for item_name in refs:
        assert paper.data_group[item_name].attrs['ACTIVE_PAPER_DATATYPE'] \
                    == 'reference'
    assert_almost_equal(paper.data["sine"][...],
                        np.sin(0.04*np.pi*np.arange(100)),
                        1.e-10)
    paper.close()
def make_library_paper(filename):

    paper = ActivePaper(filename, "w")

    paper.add_module("my_math",
"""
import numpy as np

def my_func(x):
    return np.sin(x)
""")

    paper.close()
def check_paper_with_refs(filename, with_name_postfix, refs, additional_items):
    time_ds_name = '/data/time_from_ref' if with_name_postfix else '/data/time'
    paper = ActivePaper(filename, "r")
    items = sorted([item.name for item in paper.iter_items()])
    assert items == sorted(
        ['/code/calc_sine', '/data/frequency', '/data/sine', time_ds_name] +
        additional_items)
    for item_name in refs:
        assert paper.data_group[item_name].attrs['ACTIVE_PAPER_DATATYPE'] \
                    == 'reference'
    assert_almost_equal(paper.data["sine"][...],
                        np.sin(0.04 * np.pi * np.arange(100)), 1.e-10)
    paper.close()
Esempio n. 16
0
def test_modified_scripts():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=42)
group = data.create_group('group1')
group.mark_as_data_item()
group['value'] = 1
group = data.create_group('group2')
group['value'] = 2
""")
        script.run()
        items = sorted([item.name for item in paper.iter_items()])
        assert items == ['/code/script', '/data/foo',
                         '/data/group1', '/data/group2/value']
        assert (paper.data['foo'][...] == 42)
        assert (paper.data['group1/value'][...] == 1)
        assert (paper.data['group2/value'][...] == 2)
        script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=1)
""")
        script.run()
        items = sorted([item.name for item in paper.iter_items()])
        assert items == ['/code/script', '/data/foo']
        assert (paper.data['foo'][...] == 1)
        paper.close()
def test_dummy_datasets():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        paper.data.create_dataset("frequency", data = 0.2)
        paper.data.create_dataset("time", data=0.1*np.arange(100))
        calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np

frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
        calc_angular.run()
        paper.replace_by_dummy('/data/angular/sine')
        dummy = paper.data_group['angular/sine']
        assert dummy.attrs.get('ACTIVE_PAPER_GENERATING_CODELET') \
            ==  '/code/calc_angular'
        assert dummy.attrs.get('ACTIVE_PAPER_DUMMY_DATASET', False)
        passed = True
        try:
            paper.replace_by_dummy('/data/time')
        except AssertionError:
            passed = False
        assert not passed
        paper.close()
def test_snapshots():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        snapshot_1 = os.path.join(t, "snapshot_1.ap")
        snapshot_2 = os.path.join(t, "snapshot_2.ap")
        paper = ActivePaper(filename, 'w')
        paper.data.create_dataset("frequency", data = 0.2)
        paper.data.create_dataset("time", data=0.1*np.arange(100))
        calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np

frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("cosine", data=np.cos(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("tangent", data=np.tan(2.*np.pi*frequency*time))
""" % (snapshot_1, snapshot_2))
        calc_angular.run()
        paper.close()
        # Open the snapshot files to verify they are valid ActivePapers
        ActivePaper(snapshot_1, 'r').close()
        ActivePaper(snapshot_2, 'r').close()
        # Check the contents
        paper = h5py.File(filename)
        snapshot_1 = h5py.File(snapshot_1)
        snapshot_2 = h5py.File(snapshot_2)
        for item in ['/data/time', '/data/frequency', '/data/angular/sine',
                     '/code/calc_angular']:
            assert item in paper
            assert item in snapshot_1
            assert item in snapshot_2
        assert '/data/angular/cosine' in paper
        assert '/data/angular/cosine' not in snapshot_1
        assert '/data/angular/cosine' in snapshot_2
        assert '/data/angular/tangent' in paper
        assert '/data/angular/tangent' not in snapshot_1
        assert '/data/angular/tangent' not in snapshot_2
        for root in [snapshot_1, snapshot_2]:
            #time_ref = root['/data/angular/time'][()]
            #assert root[time_ref].name == '/data/time'
            time_ref = root['/data/angular'].attrs['time']
            assert root[time_ref].name == '/data/time'
Esempio n. 19
0
def test_simple_paper():
    with tempdir.TempDir() as t:
        filename1 = os.path.join(t, "simple1.ap")
        filename2 = os.path.join(t, "simple2.ap")
        make_simple_paper(filename1)
        all_paths = [
            'README', 'code', 'code/calc_sine', 'code/initialize', 'data',
            'data/frequency', 'data/sine', 'data/time', 'documentation',
            'external-dependencies', 'history'
        ]
        all_items = [
            '/code/calc_sine', '/code/initialize', '/data/frequency',
            '/data/sine', '/data/time'
        ]
        all_deps = {
            '/data/sine': ["/code/calc_sine", "/data/frequency", "/data/time"],
            '/data/time': [],
            '/data/frequency': [],
            '/code/calc_sine': [],
            '/code/initialize': []
        }
        sine_deps = ["/code/calc_sine", "/data/frequency", "/data/time"]
        hierarchy = [[
            '/code/calc_sine', '/code/initialize', '/data/frequency',
            '/data/time'
        ], ['/data/sine']]
        check_hdf5_file(filename1, all_paths, sine_deps)
        check_paper(filename1, all_items, all_deps, hierarchy)
        with ActivePaper(filename1, "r") as paper:
            paper.rebuild(filename2)
        check_hdf5_file(filename2, all_paths, sine_deps)
        check_paper(filename2, all_items, all_deps, hierarchy)
def check_paper(filename):
    paper = ActivePaper(filename, "r")
    items = sorted([item.name for item in paper.iter_items()])
    assert items == ["/code/python-packages/foo/__init__",
                     "/code/python-packages/foo/bar",
                     "/code/test",
                     "/data/result"]
    deps = [sorted(item.name for item in level)
            for level in paper.dependency_hierarchy()]
    assert deps == [['/code/python-packages/foo/__init__',
                     '/code/python-packages/foo/bar',
                     '/code/test'],
                    ['/data/result']]
    for path in ['foo/__init__', 'foo/bar']:
        node = paper.code_group['python-packages'][path]
        assert_is_python_module(node)
    paper.close()
Esempio n. 21
0
def make_paper(filename):
    paper = ActivePaper(filename, "w")
    paper.import_module('foo')
    paper.import_module('foo.bar')
    script = paper.create_calclet(
        "test", """
from activepapers.contents import data
import foo
from foo.bar import frobnicate
data['result'] = frobnicate(2)
assert frobnicate(foo.__version__) == '42'
""")
    script.run()
    paper.close()
Esempio n. 22
0
def make_paper_with_internal_module(filename):

    paper = ActivePaper(filename, "w")

    paper.add_module(
        "my_math", """
import numpy as np

def my_func(x):
    return np.sin(x)
""")

    paper.data.create_dataset("frequency", data=0.2)
    paper.data.create_dataset("time", data=0.1 * np.arange(100))

    calc_sine = paper.create_calclet(
        "calc_sine", """
from activepapers.contents import data
import numpy as np
from my_math import my_func

frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
""")
    calc_sine.run()

    paper.close()
def test_modified_scripts():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=42)
group = data.create_group('group1')
group.mark_as_data_item()
group['value'] = 1
group = data.create_group('group2')
group['value'] = 2
""")
        script.run()
        items = sorted([item.name for item in paper.iter_items()])
        assert items == ['/code/script', '/data/foo',
                         '/data/group1', '/data/group2/value']
        assert (paper.data['foo'][...] == 42)
        assert (paper.data['group1/value'][...] == 1)
        assert (paper.data['group2/value'][...] == 2)
        script = paper.create_calclet("script",
"""
from activepapers.contents import data
data.create_dataset('foo', data=1)
""")
        script.run()
        items = sorted([item.name for item in paper.iter_items()])
        assert items == ['/code/script', '/data/foo']
        assert (paper.data['foo'][...] == 1)
        paper.close()
Esempio n. 24
0
def test_simple_paper():
    with tempdir.TempDir() as t:
        filename1 = os.path.join(t, "paper1.ap")
        filename2 = os.path.join(t, "paper2.ap")
        make_paper(filename1)
        check_paper(filename1)
        with ActivePaper(filename1, "r") as paper:
            paper.rebuild(filename2)
        check_paper(filename2)
Esempio n. 25
0
def test_attrs():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        group = paper.data.create_group('group')
        ds = group.create_dataset('value', data=42)
        group.mark_as_data_item()
        assert len(group.attrs) == 0
        group.attrs['foo'] = 'bar'
        assert len(group.attrs) == 1
        assert list(group.attrs) == ['foo']
        assert group.attrs['foo'] == 'bar'
        assert len(ds.attrs) == 0
        ds.attrs['foo'] = 'bar'
        assert len(ds.attrs) == 1
        assert list(ds.attrs) == ['foo']
        assert ds.attrs['foo'] == 'bar'
        paper.close()
def test_attrs():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        group = paper.data.create_group('group')
        ds = group.create_dataset('value', data=42)
        group.mark_as_data_item()
        assert len(group.attrs) == 0
        group.attrs['foo'] = 'bar'
        assert len(group.attrs) == 1
        assert list(group.attrs) == ['foo']
        assert group.attrs['foo'] == 'bar'
        assert len(ds.attrs) == 0
        ds.attrs['foo'] = 'bar'
        assert len(ds.attrs) == 1
        assert list(ds.attrs) == ['foo']
        assert ds.attrs['foo'] == 'bar'
        paper.close()
def make_simple_paper(filename):

    paper = ActivePaper(filename, "w")

    paper.data.create_dataset("frequency", data=0.2)
    paper.data.create_dataset("time", data=0.1 * np.arange(100))

    calc_sine = paper.create_calclet(
        "calc_sine", """
from activepapers.contents import data
import numpy as np

frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
    calc_sine.run()

    paper.close()
Esempio n. 28
0
def test_groups_as_items():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        group1 = paper.data.create_group('group1')
        group1.create_dataset('value', data=42)
        group2 = paper.data.create_group('group2')
        group2.mark_as_data_item()
        group2.create_dataset('array', data=np.arange(10))
        items = sorted([item.name for item in paper.iter_items()])
        assert items == ['/data/group1/value', '/data/group2']
        groups = sorted([group.name for group in paper.iter_groups()])
        assert groups == ['/data/group1']
        script = paper.create_calclet("script1",
"""
from activepapers.contents import data
x1 = data['group2']['array'][...]
x2 = data['group1']['value'][...]
data.create_dataset('sum1', data=x1+x2)
""")
        script.run()
        assert (paper.data['sum1'][...] == np.arange(42,52)).all()
        script = paper.create_calclet("script2",
"""
from activepapers.contents import data
x1 = data['/group2/array'][...]
g = data['group1']
x2 = g['/group1/value'][...]
data.create_dataset('sum2', data=x1+x2)
""")
        script.run()
        assert (paper.data['sum2'][...] == np.arange(42,52)).all()
        deps = [sorted([ascii(item.name) for item in level])
                for level in paper.dependency_hierarchy()]
        assert deps == [['/code/script1', '/code/script2',
                         '/data/group1/value', '/data/group2'],
                        ['/data/sum1', '/data/sum2']]
        deps = paper.data['sum1']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
        deps = sorted(ascii(d) for d in deps)
        assert deps == ['/code/script1',
                        '/data/group1/value', '/data/group2']
        deps = paper.data['sum2']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
        deps = sorted(ascii(d) for d in deps)
        assert deps == ['/code/script2',
                        '/data/group1/value', '/data/group2']
        paper.close()
Esempio n. 29
0
def make_paper_with_internal_module(filename):

    paper = ActivePaper(filename, "w")

    paper.add_module("my_math",
"""
import numpy as np

def my_func(x):
    return np.sin(x)
""")

    paper.data.create_dataset("frequency", data=0.2)
    paper.data.create_dataset("time", data=0.1*np.arange(100))

    calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np
from my_math import my_func

frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
""")
    calc_sine.run()

    paper.close()
def make_paper(filename):
    paper = ActivePaper(filename, "w")
    paper.import_module('foo')
    paper.import_module('foo.bar')
    script = paper.create_calclet("test",
"""
from activepapers.contents import data
import foo
from foo.bar import frobnicate
data['result'] = frobnicate(2)
assert frobnicate(foo.__version__) == '42'
""")
    script.run()
    paper.close()
Esempio n. 31
0
def test_groups():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        group = paper.data.create_group('group')
        subgroup = group.create_group('subgroup')
        group['data1'] = np.arange(10)
        group['data2'] = 42
        assert sorted([g.name for g in paper.iter_groups()]) \
               == ['/data/group', '/data/group/subgroup']
        assert sorted(list(node for node in group)) \
               == ['data1', 'data2', 'subgroup']
        assert group['data1'][...].shape == (10,)
        assert group['data2'][...] == 42
        assert paper.data.parent is paper.data
        assert group.parent is paper.data
        assert group['data1'].parent is group
        assert group['data2'].parent is group
        script = paper.create_calclet("script",
"""
from activepapers.contents import data
assert data.parent is data
assert data._codelet is not None
assert data._codelet.path == '/code/script'
group = data['group']
assert group.parent is data
assert group._codelet is not None
assert group._codelet.path == '/code/script'
""")
        script.run()
        paper.close()
Esempio n. 32
0
def test_import_math():
    # math is an extension module, so this should fail
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, "w")
        paper.import_module('math')
        paper.close()
def test_datasets():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        dset = paper.data.create_dataset("MyDataset", (10,10,10), 'f')
        assert len(dset) == 10
        assert dset[0, 0, 0].shape == ()
        assert dset[0, 2:10, 1:9:3].shape == (8, 3)
        assert dset[:, ::2, 5].shape == (10, 5)
        assert dset[0].shape == (10, 10)
        assert dset[1, 5].shape == (10,)
        assert dset[0, ...].shape == (10, 10)
        assert dset[..., 6].shape == (10, 10)
        array = np.arange(100)
        dset = paper.data.create_dataset("MyArray", data=array)
        assert len(dset) == 100
        assert (dset[array > 50] == np.arange(51, 100)).all()
        dset[:20] = 42
        assert (dset[...] == np.array(20*[42]+list(range(20, 100)))).all()
        paper.data['a_number'] = 42
        assert paper.data['a_number'][()] == 42
        paper.close()
Esempio n. 34
0
def test_datasets():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        dset = paper.data.create_dataset("MyDataset", (10,10,10), 'f')
        assert len(dset) == 10
        assert dset[0, 0, 0].shape == ()
        assert dset[0, 2:10, 1:9:3].shape == (8, 3)
        assert dset[:, ::2, 5].shape == (10, 5)
        assert dset[0].shape == (10, 10)
        assert dset[1, 5].shape == (10,)
        assert dset[0, ...].shape == (10, 10)
        assert dset[..., 6].shape == (10, 10)
        array = np.arange(100)
        dset = paper.data.create_dataset("MyArray", data=array)
        assert len(dset) == 100
        assert (dset[array > 50] == np.arange(51, 100)).all()
        dset[:20] = 42
        assert (dset[...] == np.array(20*[42]+list(range(20, 100)))).all()
        paper.data['a_number'] = 42
        assert paper.data['a_number'][()] == 42
        paper.close()
def make_simple_paper_with_data_refs(filename, paper_ref):

    paper = ActivePaper(filename, "w")

    paper.create_data_ref("frequency", paper_ref)
    paper.create_data_ref("time_from_ref", paper_ref, "time")

    calc_sine = paper.create_calclet(
        "calc_sine", """
from activepapers.contents import data
import numpy as np

frequency = data['frequency'][...]
time = data['time_from_ref'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
    calc_sine.run()

    paper.close()
def test_overwrite_internal_file():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        script = paper.create_calclet("write1",
"""
from activepapers.contents import open
f = open('numbers', 'w')
for i in range(10):
    f.write(str(i)+'\\n')
f.close()
""")
        script.run()
        script = paper.create_calclet("write2",
"""
from activepapers.contents import open

with open('numbers', 'w') as f:
    for i in range(10):
        f.write(str(i)+'\\n')
""")
        script.run()
        paper.close()
Esempio n. 37
0
def make_paper_with_module(filename, value):
    paper = ActivePaper(filename, "w")
    paper.add_module("some_values", """
a_value = %d
""" % value)
    script = paper.create_calclet(
        "test", """
from activepapers.contents import data
from some_values import a_value
data['a_value'] = a_value
""")
    script.run()
    paper.close()
def test_groups_as_items():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        group1 = paper.data.create_group('group1')
        group1.create_dataset('value', data=42)
        group2 = paper.data.create_group('group2')
        group2.mark_as_data_item()
        group2.create_dataset('array', data=np.arange(10))
        items = sorted([item.name for item in paper.iter_items()])
        assert items == ['/data/group1/value', '/data/group2']
        groups = sorted([group.name for group in paper.iter_groups()])
        assert groups == ['/data/group1']
        script = paper.create_calclet("script1",
"""
from activepapers.contents import data
x1 = data['group2']['array'][...]
x2 = data['group1']['value'][...]
data.create_dataset('sum1', data=x1+x2)
""")
        script.run()
        assert (paper.data['sum1'][...] == np.arange(42,52)).all()
        script = paper.create_calclet("script2",
"""
from activepapers.contents import data
x1 = data['/group2/array'][...]
g = data['group1']
x2 = g['/group1/value'][...]
data.create_dataset('sum2', data=x1+x2)
""")
        script.run()
        assert (paper.data['sum2'][...] == np.arange(42,52)).all()
        deps = [sorted([ascii(item.name) for item in level])
                for level in paper.dependency_hierarchy()]
        assert deps == [['/code/script1', '/code/script2',
                         '/data/group1/value', '/data/group2'],
                        ['/data/sum1', '/data/sum2']]
        deps = paper.data['sum1']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
        deps = sorted(ascii(d) for d in deps)
        assert deps == ['/code/script1',
                        '/data/group1/value', '/data/group2']
        deps = paper.data['sum2']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
        deps = sorted(ascii(d) for d in deps)
        assert deps == ['/code/script2',
                        '/data/group1/value', '/data/group2']
        paper.close()
Esempio n. 39
0
def test_snapshots():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        snapshot_1 = os.path.join(t, "snapshot_1.ap")
        snapshot_2 = os.path.join(t, "snapshot_2.ap")
        paper = ActivePaper(filename, 'w')
        paper.data.create_dataset("frequency", data = 0.2)
        paper.data.create_dataset("time", data=0.1*np.arange(100))
        calc_angular = paper.create_calclet("calc_angular",
"""
from activepapers.contents import data, snapshot
import numpy as np

frequency = data['frequency'][...]
time = data['time'][...]
angular = data.create_group('angular')
angular.attrs['time'] = data['time'].ref
angular.create_dataset("time", data=data['time'].ref)
angular.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("cosine", data=np.cos(2.*np.pi*frequency*time))
snapshot('%s')
angular.create_dataset("tangent", data=np.tan(2.*np.pi*frequency*time))
""" % (snapshot_1, snapshot_2))
        calc_angular.run()
        paper.close()
        # Open the snapshot files to verify they are valid ActivePapers
        ActivePaper(snapshot_1, 'r').close()
        ActivePaper(snapshot_2, 'r').close()
        # Check the contents
        paper = h5py.File(filename)
        snapshot_1 = h5py.File(snapshot_1)
        snapshot_2 = h5py.File(snapshot_2)
        for item in ['/data/time', '/data/frequency', '/data/angular/sine',
                     '/code/calc_angular']:
            assert item in paper
            assert item in snapshot_1
            assert item in snapshot_2
        assert '/data/angular/cosine' in paper
        assert '/data/angular/cosine' not in snapshot_1
        assert '/data/angular/cosine' in snapshot_2
        assert '/data/angular/tangent' in paper
        assert '/data/angular/tangent' not in snapshot_1
        assert '/data/angular/tangent' not in snapshot_2
        for root in [snapshot_1, snapshot_2]:
            #time_ref = root['/data/angular/time'][()]
            #assert root[time_ref].name == '/data/time'
            time_ref = root['/data/angular'].attrs['time']
            assert root[time_ref].name == '/data/time'
def make_simple_paper_with_data_refs(filename, paper_ref):

    paper = ActivePaper(filename, "w")

    paper.create_data_ref("frequency", paper_ref)
    paper.create_data_ref("time_from_ref", paper_ref, "time")

    calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np

frequency = data['frequency'][...]
time = data['time_from_ref'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
    calc_sine.run()

    paper.close()
def make_library_paper(filename):

    paper = ActivePaper(filename, "w")

    paper.add_module(
        "my_math", """
import numpy as np

def my_func(x):
    return np.sin(x)
""")

    paper.close()
Esempio n. 42
0
class ActivePaper(object):
    def __init__(self, file_or_ref, use_code=True):
        global _paper_for_code
        try:
            self.paper = open_paper_ref(file_or_ref)
        except ValueError:
            self.paper = ActivePaperStorage(file_or_ref, 'r')
        if use_code and ("python-packages" not in self.paper.code_group \
                         or len(self.paper.code_group["python-packages"]) == 0):
            # The paper contains no importable modules or packages.
            use_code = False
        if use_code and _paper_for_code is not None:
            raise IOError("Only one ActivePaper per process can use code.")
        self.data = self.paper.data
        self.documentation = self.paper.documentation_group
        self.code = self.paper.code_group
        try:
            self.__doc__ = self.open_documentation('README').read()
        except KeyError:
            pass
        if use_code:
            _paper_for_code = self.paper

    def close(self):
        global _paper_for_code
        if _paper_for_code is self.paper:
            _paper_for_code = None

    def _open(self, path, section, mode='r'):
        if mode not in ['r', 'rb']:
            raise ValueError("invalid mode: " + repr(mode))
        path = path_in_section(path, section)
        if not path.startswith('/'):
            path = section + '/' + path
        return self.paper.open_internal_file(path, mode, 'utf8', None)

    def open(self, path, mode='r'):
        return self._open(path, '/data', mode)

    def open_documentation(self, path, mode='r'):
        return self._open(path, '/documentation', mode)

    def read_code(self, file):
        return self.code[file][...].ravel()[0].decode('utf-8')
Esempio n. 43
0
def test_dependencies():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        paper.data.create_dataset('e', data = np.e)
        paper.data.create_dataset('pi', data = np.pi)
        script = paper.create_calclet("script",
"""
from activepapers.contents import data
import numpy as np
e = data['e'][...]
sum = data.create_dataset('sum', shape=(1,), dtype=np.float)
pi = data['pi'][...]
sum[0] = e+pi
""")
        script.run()
        deps = [ascii(item.name)
                for item in paper.iter_dependencies(paper.data['sum']._node)]
        assert sorted(deps) == ['/code/script', '/data/e', '/data/pi']
        assert not paper.is_stale(paper.data['sum']._node)
        del paper.data['e']
        paper.data['e'] = 0.
        assert paper.is_stale(paper.data['sum']._node)
        paper.close()
def make_paper_with_module(filename, value):
    paper = ActivePaper(filename, "w")
    paper.add_module("some_values",
"""
a_value = %d
""" % value)
    script = paper.create_calclet("test",
"""
from activepapers.contents import data
from some_values import a_value
data['a_value'] = a_value
""")
    script.run()
    paper.close()
def test_copy():
    with tempdir.TempDir() as t:
        library.library = [t]
        os.mkdir(os.path.join(t, "local"))
        filename1 = os.path.join(t, "local/simple1.ap")
        filename2 = os.path.join(t, "simple2.ap")
        make_simple_paper(filename1)
        make_simple_paper_with_copies(filename2, "local:simple1")
        check_paper_with_refs(filename2, False, [], [])
        paper = ActivePaper(filename2, 'r')
        for path in ['/code/calc_sine', '/data/frequency', '/data/time']:
            item = paper.file[path]
            source = item.attrs.get('ACTIVE_PAPER_COPIED_FROM')
            assert source is not None
            paper_ref, ref_path = source
            if h5py.version.version_tuple[:2] <= (2, 2):
                paper_ref = paper_ref.flat[0]
                ref_path = ref_path.flat[0]
            assert ascii(paper_ref) == "local:simple1"
            assert ascii(ref_path) == path
Esempio n. 46
0
 def __init__(self, file_or_ref, use_code=True):
     global _paper_for_code
     try:
         self.paper = open_paper_ref(file_or_ref)
     except ValueError:
         self.paper = ActivePaperStorage(file_or_ref, 'r')
     if use_code and ("python-packages" not in self.paper.code_group \
                      or len(self.paper.code_group["python-packages"]) == 0):
         # The paper contains no importable modules or packages.
         use_code = False
     if use_code and _paper_for_code is not None:
         raise IOError("Only one ActivePaper per process can use code.")
     self.data = self.paper.data
     self.documentation = self.paper.documentation_group
     self.code = self.paper.code_group
     try:
         self.__doc__ = self.open_documentation('README').read()
     except KeyError:
         pass
     if use_code:
         _paper_for_code = self.paper
Esempio n. 47
0
def make_local_paper(filename):

    paper = ActivePaper(filename, "w")

    paper.data.create_dataset("frequency", data=0.2)
    paper.data.create_dataset("time", data=0.1*np.arange(100))

    paper.add_module("my_math",
"""
import numpy as np

def my_func(x):
    return np.sin(x)
""")

    paper.close()
def test_dependencies():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        paper.data.create_dataset('e', data = np.e)
        paper.data.create_dataset('pi', data = np.pi)
        script = paper.create_calclet("script",
"""
from activepapers.contents import data
import numpy as np
e = data['e'][...]
sum = data.create_dataset('sum', shape=(1,), dtype=np.float)
pi = data['pi'][...]
sum[0] = e+pi
""")
        script.run()
        deps = [ascii(item.name)
                for item in paper.iter_dependencies(paper.data['sum']._node)]
        assert sorted(deps) == ['/code/script', '/data/e', '/data/pi']
        assert not paper.is_stale(paper.data['sum']._node)
        del paper.data['e']
        paper.data['e'] = 0.
        assert paper.is_stale(paper.data['sum']._node)
        paper.close()
def make_simple_paper_with_library_refs(filename, paper_ref):

    paper = ActivePaper(filename, "w")

    paper.data.create_dataset("frequency", data=0.2)
    paper.data.create_dataset("time", data=0.1 * np.arange(100))

    paper.create_module_ref("my_math", paper_ref)

    calc_sine = paper.create_calclet(
        "calc_sine", """
from activepapers.contents import data
import numpy as np
from my_math import my_func

frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
    """)
    calc_sine.run()

    paper.close()
Esempio n. 50
0
def check_paper(filename):
    paper = ActivePaper(filename, "r")
    items = sorted([item.name for item in paper.iter_items()])
    assert items == [
        "/code/python-packages/foo/__init__", "/code/python-packages/foo/bar",
        "/code/test", "/data/result"
    ]
    deps = [
        sorted(item.name for item in level)
        for level in paper.dependency_hierarchy()
    ]
    assert deps == [[
        '/code/python-packages/foo/__init__', '/code/python-packages/foo/bar',
        '/code/test'
    ], ['/data/result']]
    for path in ['foo/__init__', 'foo/bar']:
        node = paper.code_group['python-packages'][path]
        assert_is_python_module(node)
    paper.close()
def make_simple_paper_with_library_refs(filename, paper_ref):

    paper = ActivePaper(filename, "w")

    paper.data.create_dataset("frequency", data = 0.2)
    paper.data.create_dataset("time", data=0.1*np.arange(100))

    paper.create_module_ref("my_math", paper_ref)

    calc_sine = paper.create_calclet("calc_sine",
    """
from activepapers.contents import data
import numpy as np
from my_math import my_func

frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=my_func(2.*np.pi*frequency*time))
    """)
    calc_sine.run()

    paper.close()
Esempio n. 52
0
def test_overwrite_internal_file():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        script = paper.create_calclet("write1",
"""
from activepapers.contents import open
f = open('numbers', 'w')
for i in range(10):
    f.write(str(i)+'\\n')
f.close()
""")
        script.run()
        script = paper.create_calclet("write2",
"""
from activepapers.contents import open

with open('numbers', 'w') as f:
    for i in range(10):
        f.write(str(i)+'\\n')
""")
        script.run()
        paper.close()
Esempio n. 53
0
from activepapers.storage import ActivePaper
import numpy as np

paper = ActivePaper("plotting.ap", "w",
                    dependencies = ["matplotlib"])

paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))

plot_sine = paper.create_calclet("plot_sine",
"""
from activepapers.contents import open, data
import matplotlib
# Make matplotlib ignore the user's .matplotlibrc
matplotlib.rcdefaults()
# Use the SVG backend. Must be done *before* importing pyplot.
matplotlib.use('SVG')
import matplotlib.pyplot as plt

import numpy as np

frequency = data['frequency'][...]
time = data['time'][...]
sine = np.sin(2.*np.pi*frequency*time)

plt.plot(time, sine)
# Save plot to a file, which is simulated by a HDF5 byte array
with open('sine_plot.svg', 'w') as output:
    plt.savefig(output)
""")
plot_sine.run()
def make_simple_paper_with_data_and_code_refs(filename, paper_ref):

    paper = ActivePaper(filename, "w")

    paper.create_data_ref("frequency", paper_ref)
    paper.create_data_ref("time", paper_ref)

    paper.create_code_ref("calc_sine", paper_ref)
    paper.run_codelet('calc_sine')

    paper.close()
Esempio n. 55
0
from activepapers.storage import ActivePaper
import numpy as np

paper = ActivePaper("simple.ap", "w")

paper.data.create_dataset("frequency", data = 0.2)
paper.data.create_dataset("time", data=0.1*np.arange(100))

calc_sine = paper.create_calclet("calc_sine",
"""
from activepapers.contents import data
import numpy as np

frequency = data['frequency'][...]
time = data['time'][...]
data.create_dataset("sine", data=np.sin(2.*np.pi*frequency*time))
""")
calc_sine.run()

paper.close()
import os
os.environ['ACTIVEPAPERS_LIBRARY'] = os.getcwd()

from activepapers.storage import ActivePaper
import numpy as np

paper = ActivePaper("ref_to_simple.ap", "w")

paper.create_data_ref("frequency", "local:simple")
paper.create_data_ref("time", "local:simple", "time")

paper.create_code_ref("calc_sine", "local:simple", "calc_sine")
paper.run_codelet('calc_sine')

paper.close()
def make_simple_paper_with_copies(filename, paper_ref):

    paper = ActivePaper(filename, "w")

    paper.create_copy("/data/frequency", paper_ref)
    paper.create_copy("/data/time", paper_ref)

    paper.create_copy("/code/calc_sine", paper_ref)
    paper.run_codelet('calc_sine')

    paper.close()
Esempio n. 58
0
def check_paper(filename, ref_items, ref_deps, ref_hierarchy):
    paper = ActivePaper(filename, "r")
    items = sorted([item.name for item in paper.iter_items()])
    assert items == ref_items
    items_with_deps = sorted([item.name for item in paper.iter_items()
                              if paper.has_dependencies(item)])
    assert items_with_deps == ['/data/sine']
    deps = dict((ascii(item.name),
                 sorted(list(ascii(dep.name)
                             for dep in paper.iter_dependencies(item))))
                for item in paper.iter_items())
    assert deps == ref_deps
    graph = collections.defaultdict(set)
    for item, deps in ref_deps.items():
        for d in deps:
            graph[d].add(item)
    assert graph == paper.dependency_graph()
    hierarchy = [sorted([ascii(item.name) for item in items])
                 for items in paper.dependency_hierarchy()]
    assert hierarchy == ref_hierarchy
    calclets = paper.calclets()
    assert len(calclets) == 1
    assert ascii(calclets['/code/calc_sine'].path) == '/code/calc_sine'
    paper.close()
Esempio n. 59
0
# a specific module on the fly. This generic module
# is used when activepapers.contents is imported from a
# standard Python script. It is meant to be facilitate
# development of codelets for ActivePaper in a standard
# Python development environment.

# Locate the (hopefully only) ActivePaper in the current directory
import os
apfiles = [fn for fn in os.listdir('.') if fn.endswith('.ap')]
if len(apfiles) != 1:
    raise IOError("directory contains %s ActivePapers" % len(apfiles))
del os

# Open the paper read-only
from activepapers.storage import ActivePaper
_paper = ActivePaper(apfiles[0], 'r')
del apfiles
del ActivePaper

# Emulate the internal activepapers.contents module
data = _paper.data


def _open(filename, mode='r'):
    from activepapers.utility import path_in_section
    section = '/data'
    path = path_in_section(path, section)
    if not path.startswith('/'):
        path = section + '/' + path
    assert mode == 'r'
    return _paper.open_internal_file(path, 'r', None)
def test_internal_files():
    with tempdir.TempDir() as t:
        filename = os.path.join(t, "paper.ap")
        paper = ActivePaper(filename, 'w')
        script = paper.create_calclet("write1",
"""
from activepapers.contents import open

f = open('numbers1', 'w')
for i in range(10):
    f.write(str(i)+'\\n')
f.close()
""")
        script.run()
        script = paper.create_calclet("write2",
"""
from activepapers.contents import open

with open('numbers', 'w') as f:
    for i in range(10):
        f.write(str(i)+'\\n')
""")
        script.run()
        script = paper.create_calclet("write3",
"""
from activepapers.contents import open

with open('empty', 'w') as f:
    pass
""")
        script.run()
        script = paper.create_calclet("read1",
"""
from activepapers.contents import open

f = open('numbers')
for i in range(10):
    assert f.readline().strip() == str(i)
f.close()
""")
        script.run()
        script = paper.create_calclet("read2",
"""
from activepapers.contents import open

f = open('numbers')
data = [int(line.strip()) for line in f]
f.close()
assert data == list(range(10))
""")
        script.run()
        script = paper.create_calclet("read3",
"""
from activepapers.contents import open

f = open('empty')
data = f.read()
f.close()
assert len(data) == 0
""")
        script.run()
        script = paper.create_calclet("convert_to_binary",
"""
from activepapers.contents import open
import struct

with open('numbers') as f:
    data = [int(line.strip()) for line in f]
f = open('binary_numbers', 'wb')
f.write(struct.pack(len(data)*'h', *data))
f.close()
""")
        script.run()
        script = paper.create_calclet("read_binary",
"""
from activepapers.contents import open
import struct

f = open('binary_numbers', 'rb')
assert struct.unpack(10*'h', f.read()) == tuple(range(10))
f.close()
""")
        script.run()
        script = paper.create_calclet("write_documentation",
"""
from activepapers.contents import open_documentation

with open_documentation('hello.txt', 'w') as f:
    f.write('Hello world!\\n')
""")
        script.run()
        h = [sorted(list(ascii(item.name) for item in step))
             for step in paper.dependency_hierarchy()]
        assert h == [['/code/convert_to_binary',
                      '/code/read1', '/code/read2', '/code/read3',
                      '/code/read_binary',
                      '/code/write1', '/code/write2', '/code/write3',
                      '/code/write_documentation'],
                     ['/data/empty', '/data/numbers', '/data/numbers1',
                      '/documentation/hello.txt'],
                     ['/data/binary_numbers']]
        paper.close()