def rebuild(self, filename):
"""
Rebuild all the dependent items in the paper in a new file.
First all items without dependencies are copied to the new
file, then all the calclets are run in the new file in the
order determined by the dependency graph in the original file.
"""
deps = self.dependency_hierarchy()
with ActivePaper(filename, 'w') as clone:
for item in next(deps):
# Make sure all the groups in the path exist
path = item.name.split('/')
name = path[-1]
groups = path[:-1]
dest = clone.file
while groups:
group_name = groups[0]
if len(group_name) > 0:
if group_name not in dest:
dest.create_group(group_name)
dest = dest[group_name]
del groups[0]
clone.file.copy(item, item.name, expand_refs=True)
timestamp(clone.file[item.name])
for items in deps:
calclets = set(item.attrs['ACTIVE_PAPER_GENERATING_CODELET']
for item in items)
for calclet in calclets:
clone.run_codelet(calclet)
def rebuild(self, filename):
"""
Rebuild all the dependent items in the paper in a new file.
First all items without dependencies are copied to the new
file, then all the calclets are run in the new file in the
order determined by the dependency graph in the original file.
"""
deps = self.dependency_hierarchy()
with ActivePaper(filename, 'w') as clone:
for item in next(deps):
# Make sure all the groups in the path exist
path = item.name.split('/')
name = path[-1]
groups = path[:-1]
dest = clone.file
while groups:
group_name = groups[0]
if len(group_name) > 0:
if group_name not in dest:
dest.create_group(group_name)
dest = dest[group_name]
del groups[0]
clone.file.copy(item, item.name, expand_refs=True)
timestamp(clone.file[item.name])
for items in deps:
calclets = set(item.attrs['ACTIVE_PAPER_GENERATING_CODELET']
for item in items)
for calclet in calclets:
clone.run_codelet(calclet)
def create_copy(self, path, paper_ref, ref_path=None):
if ref_path is None:
ref_path = path
paper = open_paper_ref(paper_ref)
item = paper.file[ref_path]
self.file.copy(item, path, expand_refs=True)
copy = self.file[path]
self._delete_dependency_attributes(copy)
timestamp(copy, mod_time(item))
ref_dtype = np.dtype([('paper_ref', h5vstring), ('path', h5vstring)])
copy.attrs.create('ACTIVE_PAPER_COPIED_FROM',
shape=(), dtype=ref_dtype,
data=np.array((paper_ref, ref_path), dtype=ref_dtype))
return copy
def create_copy(self, path, paper_ref, ref_path=None):
if ref_path is None:
ref_path = path
paper = open_paper_ref(paper_ref)
item = paper.file[ref_path]
self.file.copy(item, path, expand_refs=True)
copy = self.file[path]
self._delete_dependency_attributes(copy)
timestamp(copy, mod_time(item))
ref_dtype = np.dtype([('paper_ref', h5vstring), ('path', h5vstring)])
copy.attrs.create('ACTIVE_PAPER_COPIED_FROM',
shape=(), dtype=ref_dtype,
data=np.array((paper_ref, ref_path), dtype=ref_dtype))
return copy
def replace_by_dummy(self, item_name):
item = self.file[item_name]
codelet = owner(item)
assert codelet is not None
dtype = datatype(item)
mtime = mod_time(item)
deps = item.attrs.get('ACTIVE_PAPER_DEPENDENCIES')
del self.file[item_name]
ds = self.file.create_dataset(item_name,
data=np.zeros((), dtype=np.int))
stamp(ds, dtype,
dict(ACTIVE_PAPER_GENERATING_CODELET=codelet,
ACTIVE_PAPER_DEPENDENCIES=list(deps)))
timestamp(ds, mtime)
ds.attrs['ACTIVE_PAPER_DUMMY_DATASET'] = True
def replace_by_dummy(self, item_name):
item = self.file[item_name]
codelet = owner(item)
assert codelet is not None
dtype = datatype(item)
mtime = mod_time(item)
deps = item.attrs.get('ACTIVE_PAPER_DEPENDENCIES')
del self.file[item_name]
ds = self.file.create_dataset(item_name,
data=np.zeros((), dtype=np.int))
stamp(ds, dtype,
dict(ACTIVE_PAPER_GENERATING_CODELET=codelet,
ACTIVE_PAPER_DEPENDENCIES=list(deps)))
timestamp(ds, mtime)
ds.attrs['ACTIVE_PAPER_DUMMY_DATASET'] = True
def update_from_file(paper, filename, type=None,
force_update=False, dry_run=False,
dataset_name=None, create_new=True):
if not os.path.exists(filename):
raise ValueError("File %s not found" % filename)
mtime = os.path.getmtime(filename)
basename = filename
ext = ''
if dataset_name is not None:
item = paper.file.get(dataset_name, None)
if item is not None:
basename = item.name
else:
item = paper.file.get(basename, None)
if item is None:
basename, ext = os.path.splitext(filename)
item = paper.file.get(basename, None)
language = file_languages.get(ext, None)
if item is None:
if not create_new:
return
# Create new item
if type is None:
raise ValueError("Datatype required to create new item %s"
% basename)
if type in ['calclet', 'importlet', 'module']:
if not basename.startswith('code/'):
raise ValueError("Items of type %s must be"
" in the code section"
% type)
if language != 'python':
raise ValueError("Items of type %s must be Python code"
% type)
if type == 'module' and \
not basename.startswith('code/python-packages/'):
raise ValueError("Items of type %s must be in"
"code/python-packages"
% type)
elif type == 'file':
if not basename.startswith('data/') \
and not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the data or documentation section"
% type)
basename += ext
elif type == 'text':
if not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the documentation section"
% type)
else:
# Update existing item
if mtime <= mod_time(item) and not force_update:
if dry_run:
sys.stdout.write("Skip %s: file %s is not newer\n"
% (item.name, filename))
return
if type is not None and type != datatype(item):
raise ValueError("Cannot change datatype %s to %s"
% (datatype(item), type))
if type is None:
type = datatype(item)
if language is None:
language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None)
if dry_run:
sys.stdout.write("Delete %s\n" % item.name)
else:
del item.parent[item.name.split('/')[-1]]
if dry_run:
fulltype = type if language is None else '/'.join((type, language))
sys.stdout.write("Create item %s of type %s from file %s\n"
% (basename, fulltype, filename))
else:
if type in ['calclet', 'importlet', 'module']:
item = paper.store_python_code(basename[5:],
open(filename, 'rb').read())
stamp(item, type, {})
timestamp(item, mtime)
elif type in ['file', 'text']:
f = paper.open_internal_file(basename, 'w')
f.write(open(filename, 'rb').read())
f.close()
stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language})
timestamp(f._ds, mtime)
def update_from_file(paper,
filename,
type=None,
force_update=False,
dry_run=False,
dataset_name=None,
create_new=True):
if not os.path.exists(filename):
raise ValueError("File %s not found" % filename)
mtime = os.path.getmtime(filename)
basename = filename
ext = ''
if dataset_name is not None:
item = paper.file.get(dataset_name, None)
if item is not None:
basename = item.name
else:
item = paper.file.get(basename, None)
if item is None:
basename, ext = os.path.splitext(filename)
item = paper.file.get(basename, None)
language = file_languages.get(ext, None)
if item is None:
if not create_new:
return
# Create new item
if type is None:
raise ValueError("Datatype required to create new item %s" %
basename)
if type in ['calclet', 'importlet', 'module']:
if not basename.startswith('code/'):
raise ValueError("Items of type %s must be"
" in the code section" % type)
if language != 'python':
raise ValueError("Items of type %s must be Python code" % type)
if type == 'module' and \
not basename.startswith('code/python-packages/'):
raise ValueError("Items of type %s must be in"
"code/python-packages" % type)
elif type == 'file':
if not basename.startswith('data/') \
and not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the data or documentation section" %
type)
basename += ext
elif type == 'text':
if not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the documentation section" % type)
else:
# Update existing item
if mtime <= mod_time(item) and not force_update:
if dry_run:
sys.stdout.write("Skip %s: file %s is not newer\n" %
(item.name, filename))
return
if type is not None and type != datatype(item):
raise ValueError("Cannot change datatype %s to %s" %
(datatype(item), type))
if type is None:
type = datatype(item)
if language is None:
language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None)
if dry_run:
sys.stdout.write("Delete %s\n" % item.name)
else:
del item.parent[item.name.split('/')[-1]]
if dry_run:
fulltype = type if language is None else '/'.join((type, language))
sys.stdout.write("Create item %s of type %s from file %s\n" %
(basename, fulltype, filename))
else:
if type in ['calclet', 'importlet', 'module']:
code = open(filename, 'rb').read().decode('utf-8')
item = paper.store_python_code(basename[5:], code)
stamp(item, type, {})
timestamp(item, mtime)
elif type in ['file', 'text']:
f = paper.open_internal_file(basename, 'w')
f.write(open(filename, 'rb').read())
f.close()
stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language})
timestamp(f._ds, mtime)