def check_paper(filename, ref_items, ref_deps, ref_hierarchy):
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == ref_items
items_with_deps = sorted([
item.name for item in paper.iter_items()
if paper.has_dependencies(item)
])
assert items_with_deps == ['/data/sine']
deps = dict(
(ascii(item.name),
sorted(list(ascii(dep.name)
for dep in paper.iter_dependencies(item))))
for item in paper.iter_items())
assert deps == ref_deps
graph = collections.defaultdict(set)
for item, deps in ref_deps.items():
for d in deps:
graph[d].add(item)
assert graph == paper.dependency_graph()
hierarchy = [
sorted([ascii(item.name) for item in items])
for items in paper.dependency_hierarchy()
]
assert hierarchy == ref_hierarchy
calclets = paper.calclets()
assert len(calclets) == 1
assert ascii(calclets['/code/calc_sine'].path) == '/code/calc_sine'
paper.close()
def check_hdf5_file(filename, ref_all_paths, ref_deps):
h5file = h5py.File(filename, "r")
all_paths = []
h5file.visit(all_paths.append)
all_paths.sort()
assert all_paths == ref_all_paths
assert_valid_paper(h5file)
assert_almost_equal(h5file["data/frequency"][...], 0.2, 1.e-15)
assert_almost_equal(h5file["data/time"][...],
0.1*np.arange(100),
1.e-15)
assert_almost_equal(h5file["data/sine"][...],
np.sin(0.04*np.pi*np.arange(100)),
1.e-10)
for path in ['data/frequency', 'data/sine', 'data/time']:
assert h5file[path].attrs['ACTIVE_PAPER_DATATYPE'] == "data"
assert h5file[path].attrs['ACTIVE_PAPER_TIMESTAMP'] > 1.e9
for path in ['code/calc_sine']:
assert h5file[path].attrs['ACTIVE_PAPER_DATATYPE'] == "calclet"
deps = h5file["data/sine"].attrs['ACTIVE_PAPER_DEPENDENCIES']
assert list(ascii(p) for p in deps) \
== [ascii(p) for p in ref_deps]
assert h5file["data/sine"].attrs['ACTIVE_PAPER_GENERATING_CODELET'] \
== "/code/calc_sine"
h5file.close()
def _convert(self, data):
if self._binary:
return data
elif self._encoding is not None:
return data.decode(self._encoding)
else:
return ascii(data)
def _run(self, environment):
logging.info("Running %s %s"
% (self.__class__.__name__.lower(), self.path))
self.paper.remove_owned_by(self.path)
# A string uniquely identifying the paper from which the
# calclet is called. Used in Importer.
paper_id = hex(id(self.paper))[2:]
script = ascii(self.node[...].flat[0])
script = compile(script, ':'.join([paper_id, self.path]), 'exec')
self._contents_module = imp.new_module('activepapers.contents')
self._contents_module.data = DataGroup(self.paper, None,
self.paper.data_group, self)
self._contents_module.open = self.open_data_file
self._contents_module.open_documentation = self.open_documentation_file
self._contents_module.snapshot = self.paper.snapshot
# The remaining part of this method is not thread-safe because
# of the way the global state in sys.modules is modified.
with codelet_lock:
try:
codelet_registry[(paper_id, self.path)] = self
for name, module in self.paper._local_modules.items():
assert name not in sys.modules
sys.modules[name] = module
execstring(script, environment)
finally:
del codelet_registry[(paper_id, self.path)]
self._contents_module = None
if 'activepapers.contents' in sys.modules:
del sys.modules['activepapers.contents']
for name, module in self.paper._local_modules.items():
del sys.modules[name]
def import_module(self, name, python_path=sys.path):
if name in self.imported_modules:
return self.imported_modules[name]
if '.' in name:
# Submodule, add the underlying package first
package, _, module = name.rpartition('.')
path = [self.import_module(package, python_path)]
else:
module = name
path = python_path
file, filename, (suffix, mode, kind) = imp.find_module(module, path)
if kind == imp.PKG_DIRECTORY:
package = filename
file = open(os.path.join(filename, '__init__.py'))
name = name + '/__init__'
else:
package = None
if file is None:
raise ValueError("%s is not a Python module" % name)
if kind != imp.PY_SOURCE:
file.close()
raise ValueError("%s is not a Python source code file" %
filename)
self.add_module(name, ascii(file.read()))
file.close()
self.imported_modules[name] = package
return package
def import_module(self, name, python_path=sys.path):
if name in self.imported_modules:
return self.imported_modules[name]
if '.' in name:
# Submodule, add the underlying package first
package, _, module = name.rpartition('.')
path = [self.import_module(package, python_path)]
else:
module = name
path = python_path
file, filename, (suffix, mode, kind) = imp.find_module(module, path)
if kind == imp.PKG_DIRECTORY:
package = filename
file = open(os.path.join(filename, '__init__.py'))
name = name + '/__init__'
else:
package = None
if file is None:
raise ValueError("%s is not a Python module" % name)
if kind != imp.PY_SOURCE:
raise ValueError("%s is not a Python source code file"
% filename)
self.add_module(name, ascii(file.read()))
self.imported_modules[name] = package
return package
def load_module(self, fullname):
assert fullname == self.fullname
if fullname in sys.modules:
module = sys.modules[fullname]
loader = getattr(module, '__loader__', None)
if isinstance(loader, ModuleLoader):
assert loader.paper is self.paper
return module
code = ascii(self.node[...].flat[0])
module = imp.new_module(fullname)
module.__file__ = os.path.abspath(self.node.file.filename) + ':' + \
self.node.name
module.__loader__ = self
if self._is_package:
module.__path__ = []
module.__package__ = fullname
else:
module.__package__ = fullname.rpartition('.')[0]
sys.modules[fullname] = module
self.paper._local_modules[fullname] = module
try:
execstring(code, module.__dict__)
except:
del sys.modules[fullname]
del self.paper._local_modules[fullname]
raise
return module
def _convert(self, data):
if self._binary:
return data
elif self._encoding is not None:
return data.decode(self._encoding)
else:
return ascii(data)
def test_copy():
with tempdir.TempDir() as t:
library.library = [t]
os.mkdir(os.path.join(t, "local"))
filename1 = os.path.join(t, "local/simple1.ap")
filename2 = os.path.join(t, "simple2.ap")
make_simple_paper(filename1)
make_simple_paper_with_copies(filename2, "local:simple1")
check_paper_with_refs(filename2, False, [], [])
paper = ActivePaper(filename2, 'r')
for path in ['/code/calc_sine', '/data/frequency', '/data/time']:
item = paper.file[path]
source = item.attrs.get('ACTIVE_PAPER_COPIED_FROM')
assert source is not None
paper_ref, ref_path = source
assert ascii(paper_ref.flat[0]) == "local:simple1"
assert ascii(ref_path.flat[0]) == path
def test_figshare_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.6084/m9.figshare.692144")
assert local_name == os.path.join(t, "10.6084/m9.figshare.692144.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group["python-packages/immutable/__init__"].attrs["ACTIVE_PAPER_DATATYPE"]) == "module"
paper.close()
def test_groups_as_items():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group1 = paper.data.create_group('group1')
group1.create_dataset('value', data=42)
group2 = paper.data.create_group('group2')
group2.mark_as_data_item()
group2.create_dataset('array', data=np.arange(10))
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/data/group1/value', '/data/group2']
groups = sorted([group.name for group in paper.iter_groups()])
assert groups == ['/data/group1']
script = paper.create_calclet("script1",
"""
from activepapers.contents import data
x1 = data['group2']['array'][...]
x2 = data['group1']['value'][...]
data.create_dataset('sum1', data=x1+x2)
""")
script.run()
assert (paper.data['sum1'][...] == np.arange(42,52)).all()
script = paper.create_calclet("script2",
"""
from activepapers.contents import data
x1 = data['/group2/array'][...]
g = data['group1']
x2 = g['/group1/value'][...]
data.create_dataset('sum2', data=x1+x2)
""")
script.run()
assert (paper.data['sum2'][...] == np.arange(42,52)).all()
deps = [sorted([ascii(item.name) for item in level])
for level in paper.dependency_hierarchy()]
assert deps == [['/code/script1', '/code/script2',
'/data/group1/value', '/data/group2'],
['/data/sum1', '/data/sum2']]
deps = paper.data['sum1']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script1',
'/data/group1/value', '/data/group2']
deps = paper.data['sum2']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script2',
'/data/group1/value', '/data/group2']
paper.close()
def test_groups_as_items():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
group1 = paper.data.create_group('group1')
group1.create_dataset('value', data=42)
group2 = paper.data.create_group('group2')
group2.mark_as_data_item()
group2.create_dataset('array', data=np.arange(10))
items = sorted([item.name for item in paper.iter_items()])
assert items == ['/data/group1/value', '/data/group2']
groups = sorted([group.name for group in paper.iter_groups()])
assert groups == ['/data/group1']
script = paper.create_calclet("script1",
"""
from activepapers.contents import data
x1 = data['group2']['array'][...]
x2 = data['group1']['value'][...]
data.create_dataset('sum1', data=x1+x2)
""")
script.run()
assert (paper.data['sum1'][...] == np.arange(42,52)).all()
script = paper.create_calclet("script2",
"""
from activepapers.contents import data
x1 = data['/group2/array'][...]
g = data['group1']
x2 = g['/group1/value'][...]
data.create_dataset('sum2', data=x1+x2)
""")
script.run()
assert (paper.data['sum2'][...] == np.arange(42,52)).all()
deps = [sorted([ascii(item.name) for item in level])
for level in paper.dependency_hierarchy()]
assert deps == [['/code/script1', '/code/script2',
'/data/group1/value', '/data/group2'],
['/data/sum1', '/data/sum2']]
deps = paper.data['sum1']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script1',
'/data/group1/value', '/data/group2']
deps = paper.data['sum2']._node.attrs['ACTIVE_PAPER_DEPENDENCIES']
deps = sorted(ascii(d) for d in deps)
assert deps == ['/code/script2',
'/data/group1/value', '/data/group2']
paper.close()
def test_zenodo_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.5281/zenodo.7648")
assert local_name == os.path.join(t, "10.5281/zenodo.7648.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group["python-packages/mosaic/__init__"].attrs["ACTIVE_PAPER_DATATYPE"]) == "module"
paper.close()
def dependency_attributes(self):
if self._dependencies is None:
return {'ACTIVE_PAPER_GENERATING_CODELET': self.path}
else:
deps = list(self._dependencies)
deps.append(ascii(self.path))
deps.sort()
return {'ACTIVE_PAPER_GENERATING_CODELET': self.path,
'ACTIVE_PAPER_DEPENDENCIES': deps}
def test_figshare_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.6084/m9.figshare.692144")
assert local_name == os.path.join(t, "10.6084/m9.figshare.692144.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group['python-packages/immutable/__init__'].
attrs['ACTIVE_PAPER_DATATYPE']) == 'module'
paper.close()
def test_zenodo_download():
with tempdir.TempDir() as t:
library.library = [t]
local_name = library.find_in_library("doi:10.5281/zenodo.7648")
assert local_name == os.path.join(t, "10.5281/zenodo.7648.ap")
paper = ActivePaper(local_name)
assert ascii(paper.code_group['python-packages/mosaic/__init__'].
attrs['ACTIVE_PAPER_DATATYPE']) == 'module'
paper.close()
def test_copy():
with tempdir.TempDir() as t:
library.library = [t]
os.mkdir(os.path.join(t, "local"))
filename1 = os.path.join(t, "local/simple1.ap")
filename2 = os.path.join(t, "simple2.ap")
make_simple_paper(filename1)
make_simple_paper_with_copies(filename2, "local:simple1")
check_paper_with_refs(filename2, False, [], [])
paper = ActivePaper(filename2, 'r')
for path in ['/code/calc_sine', '/data/frequency', '/data/time']:
item = paper.file[path]
source = item.attrs.get('ACTIVE_PAPER_COPIED_FROM')
assert source is not None
paper_ref, ref_path = source
if h5py.version.version_tuple[:2] <= (2, 2):
paper_ref = paper_ref.flat[0]
ref_path = ref_path.flat[0]
assert ascii(paper_ref) == "local:simple1"
assert ascii(ref_path) == path
def check_hdf5_file(filename, ref_all_paths, ref_deps):
h5file = h5py.File(filename, "r")
all_paths = []
h5file.visit(all_paths.append)
all_paths.sort()
assert all_paths == ref_all_paths
assert_valid_paper(h5file)
assert_almost_equal(h5file["data/frequency"][...], 0.2, 1.e-15)
assert_almost_equal(h5file["data/time"][...], 0.1 * np.arange(100), 1.e-15)
assert_almost_equal(h5file["data/sine"][...],
np.sin(0.04 * np.pi * np.arange(100)), 1.e-10)
for path in ['data/frequency', 'data/sine', 'data/time']:
assert h5file[path].attrs['ACTIVE_PAPER_DATATYPE'] == "data"
assert h5file[path].attrs['ACTIVE_PAPER_TIMESTAMP'] > 1.e9
for path in ['code/calc_sine']:
assert h5file[path].attrs['ACTIVE_PAPER_DATATYPE'] == "calclet"
deps = h5file["data/sine"].attrs['ACTIVE_PAPER_DEPENDENCIES']
assert list(ascii(p) for p in deps) \
== [ascii(p) for p in ref_deps]
assert h5file["data/sine"].attrs['ACTIVE_PAPER_GENERATING_CODELET'] \
== "/code/calc_sine"
h5file.close()
def check_paper(filename, ref_items, ref_deps, ref_hierarchy):
paper = ActivePaper(filename, "r")
items = sorted([item.name for item in paper.iter_items()])
assert items == ref_items
items_with_deps = sorted([item.name for item in paper.iter_items()
if paper.has_dependencies(item)])
assert items_with_deps == ['/data/sine']
deps = dict((ascii(item.name),
sorted(list(ascii(dep.name)
for dep in paper.iter_dependencies(item))))
for item in paper.iter_items())
assert deps == ref_deps
graph = collections.defaultdict(set)
for item, deps in ref_deps.items():
for d in deps:
graph[d].add(item)
assert graph == paper.dependency_graph()
hierarchy = [sorted([ascii(item.name) for item in items])
for items in paper.dependency_hierarchy()]
assert hierarchy == ref_hierarchy
calclets = paper.calclets()
assert len(calclets) == 1
assert ascii(calclets['/code/calc_sine'].path) == '/code/calc_sine'
paper.close()
def assert_valid_paper(h5file):
assert h5file.attrs['DATA_MODEL'] == ascii('active-papers-py')
assert h5file.attrs['DATA_MODEL_MAJOR_VERSION'] == 0
assert h5file.attrs['DATA_MODEL_MINOR_VERSION'] == 1
for group in ['code', 'data', 'documentation']:
assert group in h5file
assert isinstance(h5file[group], h5py.Group)
history = h5file['history']
assert history.shape == (1,)
opened = history[0]['opened']
closed = history[0]['closed']
def check_timestamp(name, node):
t = node.attrs.get('ACTIVE_PAPER_TIMESTAMP', None)
if t is not None:
assert t >= opened
assert t <= closed
h5file.visititems(check_timestamp)
def find_module(self, fullname, path=None):
codelet, paper = get_codelet_and_paper()
if paper is None:
return None
node = paper.get_local_module(fullname)
if node is None:
# No corresponding node found
return None
is_package = False
if node.is_group():
# Node is a group, so this should be a package
if '__init__' not in node:
# Not a package
return None
is_package = True
node = node['__init__']
if datatype(node) != "module" \
or ascii(node.attrs.get("ACTIVE_PAPER_LANGUAGE", "")) != "python":
# Node found but is not a Python module
return None
return ModuleLoader(paper, fullname, node, is_package)
def assert_valid_paper(h5file):
assert h5file.attrs['DATA_MODEL'] == ascii('active-papers-py')
assert h5file.attrs['DATA_MODEL_MAJOR_VERSION'] == 0
assert h5file.attrs['DATA_MODEL_MINOR_VERSION'] == 1
for group in ['code', 'data', 'documentation']:
assert group in h5file
assert isinstance(h5file[group], h5py.Group)
history = h5file['history']
assert history.shape == (1, )
opened = history[0]['opened']
closed = history[0]['closed']
def check_timestamp(name, node):
t = node.attrs.get('ACTIVE_PAPER_TIMESTAMP', None)
if t is not None:
assert t >= opened
assert t <= closed
h5file.visititems(check_timestamp)
def test_dependencies():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset('e', data = np.e)
paper.data.create_dataset('pi', data = np.pi)
script = paper.create_calclet("script",
"""
from activepapers.contents import data
import numpy as np
e = data['e'][...]
sum = data.create_dataset('sum', shape=(1,), dtype=np.float)
pi = data['pi'][...]
sum[0] = e+pi
""")
script.run()
deps = [ascii(item.name)
for item in paper.iter_dependencies(paper.data['sum']._node)]
assert sorted(deps) == ['/code/script', '/data/e', '/data/pi']
assert not paper.is_stale(paper.data['sum']._node)
del paper.data['e']
paper.data['e'] = 0.
assert paper.is_stale(paper.data['sum']._node)
paper.close()
def test_dependencies():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
paper.data.create_dataset('e', data = np.e)
paper.data.create_dataset('pi', data = np.pi)
script = paper.create_calclet("script",
"""
from activepapers.contents import data
import numpy as np
e = data['e'][...]
sum = data.create_dataset('sum', shape=(1,), dtype=np.float)
pi = data['pi'][...]
sum[0] = e+pi
""")
script.run()
deps = [ascii(item.name)
for item in paper.iter_dependencies(paper.data['sum']._node)]
assert sorted(deps) == ['/code/script', '/data/e', '/data/pi']
assert not paper.is_stale(paper.data['sum']._node)
del paper.data['e']
paper.data['e'] = 0.
assert paper.is_stale(paper.data['sum']._node)
paper.close()
def test_internal_files():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("write1",
"""
from activepapers.contents import open
f = open('numbers1', 'w')
for i in range(10):
f.write(str(i)+'\\n')
f.close()
""")
script.run()
script = paper.create_calclet("write2",
"""
from activepapers.contents import open
with open('numbers', 'w') as f:
for i in range(10):
f.write(str(i)+'\\n')
""")
script.run()
script = paper.create_calclet("write3",
"""
from activepapers.contents import open
with open('empty', 'w') as f:
pass
""")
script.run()
script = paper.create_calclet("write4",
u"""
from activepapers.contents import open
with open('utf8', 'w', encoding='utf-8') as f:
f.write(u'déjà')
""")
script.run()
script = paper.create_calclet("read1",
"""
from activepapers.contents import open
f = open('numbers')
for i in range(10):
assert f.readline().strip() == str(i)
f.close()
""")
script.run()
script = paper.create_calclet("read2",
"""
from activepapers.contents import open
f = open('numbers')
data = [int(line.strip()) for line in f]
f.close()
assert data == list(range(10))
""")
script.run()
script = paper.create_calclet("read3",
"""
from activepapers.contents import open
f = open('empty')
data = f.read()
f.close()
assert len(data) == 0
""")
script.run()
script = paper.create_calclet("read4",
u"""
from activepapers.contents import open
f = open('utf8', encoding='utf-8')
data = f.read()
f.close()
assert data == u'déjà'
""")
script.run()
script = paper.create_calclet("convert_to_binary",
"""
from activepapers.contents import open
import struct
with open('numbers') as f:
data = [int(line.strip()) for line in f]
f = open('binary_numbers', 'wb')
f.write(struct.pack(len(data)*'h', *data))
f.close()
""")
script.run()
script = paper.create_calclet("read_binary",
"""
from activepapers.contents import open
import struct
f = open('binary_numbers', 'rb')
assert struct.unpack(10*'h', f.read()) == tuple(range(10))
f.close()
""")
script.run()
script = paper.create_calclet("write_documentation",
"""
from activepapers.contents import open_documentation
with open_documentation('hello.txt', 'w') as f:
f.write('Hello world!\\n')
""")
script.run()
h = [sorted(list(ascii(item.name) for item in step))
for step in paper.dependency_hierarchy()]
print(h)
assert h == [['/code/convert_to_binary',
'/code/read1', '/code/read2', '/code/read3',
'/code/read4', '/code/read_binary',
'/code/write1', '/code/write2', '/code/write3',
'/code/write4', '/code/write_documentation'],
['/data/empty', '/data/numbers', '/data/numbers1',
'/data/utf8', '/documentation/hello.txt'],
['/data/binary_numbers']]
paper.close()
def __init__(self, filename, mode="r", dependencies=None):
self.filename = filename
self.file = h5py.File(filename, mode)
self.open = True
self.writable = False
if mode[0] == 'r':
assert dependencies is None
if ascii(self.file.attrs['DATA_MODEL']) != 'active-papers-py':
raise ValueError("File %s is not an ActivePaper" % filename)
self.code_group = self.file["code"]
self.data_group = self.file["data"]
self.documentation_group = self.file["documentation"]
self.writable = '+' in mode
self.history = self.file['history']
deps = self.file.get('external-dependencies/'
'python-packages', None)
if deps is None:
self.dependencies = []
else:
self.dependencies = [ascii(n) for n in deps]
for module_name in self.dependencies:
importlib.import_module(module_name)
elif mode[0] == 'w':
self.file.attrs['DATA_MODEL'] = ascii('active-papers-py')
self.file.attrs['DATA_MODEL_MAJOR_VERSION'] = 0
self.file.attrs['DATA_MODEL_MINOR_VERSION'] = 1
self.code_group = self.file.create_group("code")
self.data_group = self.file.create_group("data")
self.documentation_group = self.file.create_group("documentation")
deps = self.file.create_group('external-dependencies')
if dependencies is None:
self.dependencies = []
else:
for module_name in dependencies:
assert isstring(module_name)
importlib.import_module(module_name)
self.dependencies = dependencies
ds = deps.create_dataset('python-packages',
dtype=h5vstring,
shape=(len(dependencies), ))
ds[:] = dependencies
htype = np.dtype(
[('opened', np.int64), ('closed',
np.int64), ('platform', h5vstring),
('hostname', h5vstring), ('username', h5vstring)] +
[(name + "_version", h5vstring) for name in
['activepapers', 'python', 'numpy', 'h5py', 'hdf5'] +
self.dependencies])
self.history = self.file.create_dataset("history",
shape=(0, ),
dtype=htype,
chunks=(1, ),
maxshape=(None, ))
readme = self.file.create_dataset("README",
dtype=h5vstring,
shape=())
readme[...] = readme_text
self.writable = True
if self.writable:
self.update_history(close=False)
import activepapers.utility
self.data = DataGroup(self, None, self.data_group, ExternalCode(self))
self.imported_modules = {}
self._local_modules = {}
paper_registry[self._id()] = self
def dereference(ref_node):
assert datatype(ref_node) == 'reference'
paper_ref, path = ref_node[()]
paper = open_paper_ref(ascii(paper_ref))
return paper, paper.file[path]
def dereference(ref_node):
assert datatype(ref_node) == 'reference'
paper_ref, path = ref_node[()]
paper = open_paper_ref(ascii(paper_ref))
return paper, paper.file[path]
def _convert(self, data):
if self._binary:
return data
else:
return ascii(data)
def __init__(self, filename, mode="r", dependencies=None):
self.filename = filename
self.file = h5py.File(filename, mode)
self.open = True
self.writable = False
if mode[0] == 'r':
assert dependencies is None
if ascii(self.file.attrs['DATA_MODEL']) != 'active-papers-py':
raise ValueError("File %s is not an ActivePaper" % filename)
self.code_group = self.file["code"]
self.data_group = self.file["data"]
self.documentation_group = self.file["documentation"]
self.writable = '+' in mode
self.history = self.file['history']
deps = self.file.get('external-dependencies/'
'python-packages', None)
if deps is None:
self.dependencies = []
else:
self.dependencies = [ascii(n) for n in deps]
for module_name in self.dependencies:
importlib.import_module(module_name)
elif mode[0] == 'w':
self.file.attrs['DATA_MODEL'] = ascii('active-papers-py')
self.file.attrs['DATA_MODEL_MAJOR_VERSION'] = 0
self.file.attrs['DATA_MODEL_MINOR_VERSION'] = 1
self.code_group = self.file.create_group("code")
self.data_group = self.file.create_group("data")
self.documentation_group = self.file.create_group("documentation")
deps = self.file.create_group('external-dependencies')
if dependencies is None:
self.dependencies = []
else:
for module_name in dependencies:
assert isstring(module_name)
importlib.import_module(module_name)
self.dependencies = dependencies
deps.create_dataset('python-packages',
data = dependencies)
htype = np.dtype([('opened', np.int64),
('closed', np.int64),
('platform', h5vstring),
('hostname', h5vstring),
('username', h5vstring)]
+ [(name+"_version", h5vstring)
for name in ['activepapers','python',
'numpy', 'h5py', 'hdf5']
+ self.dependencies])
self.history = self.file.create_dataset("history", shape=(0,),
dtype=htype,
chunks=(1,),
maxshape=(None,))
readme = self.file.create_dataset("README",
dtype=h5vstring, shape = ())
readme[...] = readme_text
self.writable = True
if self.writable:
self.update_history(close=False)
import activepapers.utility
self.data = DataGroup(self, None, self.data_group, ExternalCode(self))
self.imported_modules = {}
self._local_modules = {}
paper_id = hex(id(self))[2:]
paper_registry[paper_id] = self
def test_internal_files():
with tempdir.TempDir() as t:
filename = os.path.join(t, "paper.ap")
paper = ActivePaper(filename, 'w')
script = paper.create_calclet("write1",
"""
from activepapers.contents import open
f = open('numbers1', 'w')
for i in range(10):
f.write(str(i)+'\\n')
f.close()
""")
script.run()
script = paper.create_calclet("write2",
"""
from activepapers.contents import open
with open('numbers', 'w') as f:
for i in range(10):
f.write(str(i)+'\\n')
""")
script.run()
script = paper.create_calclet("write3",
"""
from activepapers.contents import open
with open('empty', 'w') as f:
pass
""")
script.run()
script = paper.create_calclet("read1",
"""
from activepapers.contents import open
f = open('numbers')
for i in range(10):
assert f.readline().strip() == str(i)
f.close()
""")
script.run()
script = paper.create_calclet("read2",
"""
from activepapers.contents import open
f = open('numbers')
data = [int(line.strip()) for line in f]
f.close()
assert data == list(range(10))
""")
script.run()
script = paper.create_calclet("read3",
"""
from activepapers.contents import open
f = open('empty')
data = f.read()
f.close()
assert len(data) == 0
""")
script.run()
script = paper.create_calclet("convert_to_binary",
"""
from activepapers.contents import open
import struct
with open('numbers') as f:
data = [int(line.strip()) for line in f]
f = open('binary_numbers', 'wb')
f.write(struct.pack(len(data)*'h', *data))
f.close()
""")
script.run()
script = paper.create_calclet("read_binary",
"""
from activepapers.contents import open
import struct
f = open('binary_numbers', 'rb')
assert struct.unpack(10*'h', f.read()) == tuple(range(10))
f.close()
""")
script.run()
script = paper.create_calclet("write_documentation",
"""
from activepapers.contents import open_documentation
with open_documentation('hello.txt', 'w') as f:
f.write('Hello world!\\n')
""")
script.run()
h = [sorted(list(ascii(item.name) for item in step))
for step in paper.dependency_hierarchy()]
assert h == [['/code/convert_to_binary',
'/code/read1', '/code/read2', '/code/read3',
'/code/read_binary',
'/code/write1', '/code/write2', '/code/write3',
'/code/write_documentation'],
['/data/empty', '/data/numbers', '/data/numbers1',
'/documentation/hello.txt'],
['/data/binary_numbers']]
paper.close()
def add_dependency(self, dependency):
assert isinstance(self._dependencies, set)
self._dependencies.add(ascii(dependency))
