def create_importlet(self, path, script):
path = codepath(path)
if not path.startswith('/'):
path = '/'.join([self.code_group.name, path])
ds = self.store_python_code(path, script)
stamp(ds, "importlet", {})
return Importlet(self, ds)
def create_importlet(self, path, script):
path = codepath(path)
if not path.startswith('/'):
path = '/'.join([self.code_group.name, path])
ds = self.store_python_code(path, script)
stamp(ds, "importlet", {})
return Importlet(self, ds)
def __setitem__(self, path, value):
path = datapath(path)
needs_stamp = False
if isinstance(value, (DataGroup, DatasetWrapper)):
value = value._node
else:
needs_stamp = True
self._node[path] = value
if needs_stamp:
node = self._node[path]
stamp(node, "data", self._codelet.dependency_attributes())
def replace_by_dummy(self, item_name):
item = self.file[item_name]
codelet = owner(item)
assert codelet is not None
dtype = datatype(item)
mtime = mod_time(item)
deps = item.attrs.get('ACTIVE_PAPER_DEPENDENCIES')
del self.file[item_name]
ds = self.file.create_dataset(item_name,
data=np.zeros((), dtype=np.int))
stamp(ds, dtype,
dict(ACTIVE_PAPER_GENERATING_CODELET=codelet,
ACTIVE_PAPER_DEPENDENCIES=list(deps)))
timestamp(ds, mtime)
ds.attrs['ACTIVE_PAPER_DUMMY_DATASET'] = True
def replace_by_dummy(self, item_name):
item = self.file[item_name]
codelet = owner(item)
assert codelet is not None
dtype = datatype(item)
mtime = mod_time(item)
deps = item.attrs.get('ACTIVE_PAPER_DEPENDENCIES')
del self.file[item_name]
ds = self.file.create_dataset(item_name,
data=np.zeros((), dtype=np.int))
stamp(ds, dtype,
dict(ACTIVE_PAPER_GENERATING_CODELET=codelet,
ACTIVE_PAPER_DEPENDENCIES=list(deps)))
timestamp(ds, mtime)
ds.attrs['ACTIVE_PAPER_DUMMY_DATASET'] = True
def _create_ref(self, path, paper_ref, ref_path, group, prefix):
if ref_path is None:
ref_path = path
if group is None:
group = 'file'
if prefix is None:
prefix = ''
else:
prefix += '/'
paper = open_paper_ref(paper_ref)
# Access the item to make sure it exists
item = getattr(paper, group)[ref_path]
ref_dtype = np.dtype([('paper_ref', h5vstring), ('path', h5vstring)])
ds = getattr(self, group).require_dataset(path, shape=(),
dtype=ref_dtype)
ds[...] = (paper_ref, prefix + ref_path)
stamp(ds, 'reference', {})
return ds
def _create_ref(self, path, paper_ref, ref_path, group, prefix):
if ref_path is None:
ref_path = path
if group is None:
group = 'file'
if prefix is None:
prefix = ''
else:
prefix += '/'
paper = open_paper_ref(paper_ref)
# Access the item to make sure it exists
item = getattr(paper, group)[ref_path]
ref_dtype = np.dtype([('paper_ref', h5vstring), ('path', h5vstring)])
ds = getattr(self, group).require_dataset(path, shape=(),
dtype=ref_dtype)
ds[...] = (paper_ref, prefix + ref_path)
stamp(ds, 'reference', {})
return ds
def add_module(self, name, module_code):
path = codepath('/'.join(['', 'python-packages'] + name.split('.')))
ds = self.store_python_code(path, module_code)
stamp(ds, "module", {})
def mark_as_data_item(self):
stamp(self._node, "data", self._codelet.dependency_attributes())
self._data_item = self
def _stamp_new_node(self, node, ap_type):
if self._data_item:
stamp(self._data_item._node, "data",
self._codelet.dependency_attributes())
else:
stamp(node, ap_type, self._codelet.dependency_attributes())
def resize(self, size, axis=None):
self._node.resize(size, axis)
stamp(self._node, "data", self._codelet.dependency_attributes())
def __setitem__(self, item, value):
self._node[item] = value
stamp(self._node, "data", self._codelet.dependency_attributes())
def _stamp(self):
if self.writable():
stamp(self._ds, "file", self._get_attributes())
def add_module(self, name, module_code):
path = codepath('/'.join(['', 'python-packages'] + name.split('.')))
ds = self.store_python_code(path, module_code)
stamp(ds, "module", {})
def update_from_file(paper, filename, type=None,
force_update=False, dry_run=False,
dataset_name=None, create_new=True):
if not os.path.exists(filename):
raise ValueError("File %s not found" % filename)
mtime = os.path.getmtime(filename)
basename = filename
ext = ''
if dataset_name is not None:
item = paper.file.get(dataset_name, None)
if item is not None:
basename = item.name
else:
item = paper.file.get(basename, None)
if item is None:
basename, ext = os.path.splitext(filename)
item = paper.file.get(basename, None)
language = file_languages.get(ext, None)
if item is None:
if not create_new:
return
# Create new item
if type is None:
raise ValueError("Datatype required to create new item %s"
% basename)
if type in ['calclet', 'importlet', 'module']:
if not basename.startswith('code/'):
raise ValueError("Items of type %s must be"
" in the code section"
% type)
if language != 'python':
raise ValueError("Items of type %s must be Python code"
% type)
if type == 'module' and \
not basename.startswith('code/python-packages/'):
raise ValueError("Items of type %s must be in"
"code/python-packages"
% type)
elif type == 'file':
if not basename.startswith('data/') \
and not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the data or documentation section"
% type)
basename += ext
elif type == 'text':
if not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the documentation section"
% type)
else:
# Update existing item
if mtime <= mod_time(item) and not force_update:
if dry_run:
sys.stdout.write("Skip %s: file %s is not newer\n"
% (item.name, filename))
return
if type is not None and type != datatype(item):
raise ValueError("Cannot change datatype %s to %s"
% (datatype(item), type))
if type is None:
type = datatype(item)
if language is None:
language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None)
if dry_run:
sys.stdout.write("Delete %s\n" % item.name)
else:
del item.parent[item.name.split('/')[-1]]
if dry_run:
fulltype = type if language is None else '/'.join((type, language))
sys.stdout.write("Create item %s of type %s from file %s\n"
% (basename, fulltype, filename))
else:
if type in ['calclet', 'importlet', 'module']:
item = paper.store_python_code(basename[5:],
open(filename, 'rb').read())
stamp(item, type, {})
timestamp(item, mtime)
elif type in ['file', 'text']:
f = paper.open_internal_file(basename, 'w')
f.write(open(filename, 'rb').read())
f.close()
stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language})
timestamp(f._ds, mtime)
def update_from_file(paper,
filename,
type=None,
force_update=False,
dry_run=False,
dataset_name=None,
create_new=True):
if not os.path.exists(filename):
raise ValueError("File %s not found" % filename)
mtime = os.path.getmtime(filename)
basename = filename
ext = ''
if dataset_name is not None:
item = paper.file.get(dataset_name, None)
if item is not None:
basename = item.name
else:
item = paper.file.get(basename, None)
if item is None:
basename, ext = os.path.splitext(filename)
item = paper.file.get(basename, None)
language = file_languages.get(ext, None)
if item is None:
if not create_new:
return
# Create new item
if type is None:
raise ValueError("Datatype required to create new item %s" %
basename)
if type in ['calclet', 'importlet', 'module']:
if not basename.startswith('code/'):
raise ValueError("Items of type %s must be"
" in the code section" % type)
if language != 'python':
raise ValueError("Items of type %s must be Python code" % type)
if type == 'module' and \
not basename.startswith('code/python-packages/'):
raise ValueError("Items of type %s must be in"
"code/python-packages" % type)
elif type == 'file':
if not basename.startswith('data/') \
and not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the data or documentation section" %
type)
basename += ext
elif type == 'text':
if not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the documentation section" % type)
else:
# Update existing item
if mtime <= mod_time(item) and not force_update:
if dry_run:
sys.stdout.write("Skip %s: file %s is not newer\n" %
(item.name, filename))
return
if type is not None and type != datatype(item):
raise ValueError("Cannot change datatype %s to %s" %
(datatype(item), type))
if type is None:
type = datatype(item)
if language is None:
language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None)
if dry_run:
sys.stdout.write("Delete %s\n" % item.name)
else:
del item.parent[item.name.split('/')[-1]]
if dry_run:
fulltype = type if language is None else '/'.join((type, language))
sys.stdout.write("Create item %s of type %s from file %s\n" %
(basename, fulltype, filename))
else:
if type in ['calclet', 'importlet', 'module']:
code = open(filename, 'rb').read().decode('utf-8')
item = paper.store_python_code(basename[5:], code)
stamp(item, type, {})
timestamp(item, mtime)
elif type in ['file', 'text']:
f = paper.open_internal_file(basename, 'w')
f.write(open(filename, 'rb').read())
f.close()
stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language})
timestamp(f._ds, mtime)
def _stamp(self):
if self.writable():
stamp(self._ds, "file", self._get_attributes())
def write_direct(source, source_sel=None, dest_sel=None):
self._node.write_direct(source, source_sel, dest_sel)
stamp(self._node, "data", self._codelet.dependency_attributes())
