def create_importlet(self, path, script): path = codepath(path) if not path.startswith('/'): path = '/'.join([self.code_group.name, path]) ds = self.store_python_code(path, script) stamp(ds, "importlet", {}) return Importlet(self, ds)
def __setitem__(self, path, value): path = datapath(path) needs_stamp = False if isinstance(value, (DataGroup, DatasetWrapper)): value = value._node else: needs_stamp = True self._node[path] = value if needs_stamp: node = self._node[path] stamp(node, "data", self._codelet.dependency_attributes())
def replace_by_dummy(self, item_name): item = self.file[item_name] codelet = owner(item) assert codelet is not None dtype = datatype(item) mtime = mod_time(item) deps = item.attrs.get('ACTIVE_PAPER_DEPENDENCIES') del self.file[item_name] ds = self.file.create_dataset(item_name, data=np.zeros((), dtype=np.int)) stamp(ds, dtype, dict(ACTIVE_PAPER_GENERATING_CODELET=codelet, ACTIVE_PAPER_DEPENDENCIES=list(deps))) timestamp(ds, mtime) ds.attrs['ACTIVE_PAPER_DUMMY_DATASET'] = True
def _create_ref(self, path, paper_ref, ref_path, group, prefix): if ref_path is None: ref_path = path if group is None: group = 'file' if prefix is None: prefix = '' else: prefix += '/' paper = open_paper_ref(paper_ref) # Access the item to make sure it exists item = getattr(paper, group)[ref_path] ref_dtype = np.dtype([('paper_ref', h5vstring), ('path', h5vstring)]) ds = getattr(self, group).require_dataset(path, shape=(), dtype=ref_dtype) ds[...] = (paper_ref, prefix + ref_path) stamp(ds, 'reference', {}) return ds
def add_module(self, name, module_code): path = codepath('/'.join(['', 'python-packages'] + name.split('.'))) ds = self.store_python_code(path, module_code) stamp(ds, "module", {})
def mark_as_data_item(self): stamp(self._node, "data", self._codelet.dependency_attributes()) self._data_item = self
def _stamp_new_node(self, node, ap_type): if self._data_item: stamp(self._data_item._node, "data", self._codelet.dependency_attributes()) else: stamp(node, ap_type, self._codelet.dependency_attributes())
def resize(self, size, axis=None): self._node.resize(size, axis) stamp(self._node, "data", self._codelet.dependency_attributes())
def __setitem__(self, item, value): self._node[item] = value stamp(self._node, "data", self._codelet.dependency_attributes())
def _stamp(self): if self.writable(): stamp(self._ds, "file", self._get_attributes())
def update_from_file(paper, filename, type=None, force_update=False, dry_run=False, dataset_name=None, create_new=True): if not os.path.exists(filename): raise ValueError("File %s not found" % filename) mtime = os.path.getmtime(filename) basename = filename ext = '' if dataset_name is not None: item = paper.file.get(dataset_name, None) if item is not None: basename = item.name else: item = paper.file.get(basename, None) if item is None: basename, ext = os.path.splitext(filename) item = paper.file.get(basename, None) language = file_languages.get(ext, None) if item is None: if not create_new: return # Create new item if type is None: raise ValueError("Datatype required to create new item %s" % basename) if type in ['calclet', 'importlet', 'module']: if not basename.startswith('code/'): raise ValueError("Items of type %s must be" " in the code section" % type) if language != 'python': raise ValueError("Items of type %s must be Python code" % type) if type == 'module' and \ not basename.startswith('code/python-packages/'): raise ValueError("Items of type %s must be in" "code/python-packages" % type) elif type == 'file': if not basename.startswith('data/') \ and not basename.startswith('documentation/'): raise ValueError("Items of type %s must be" " in the data or documentation section" % type) basename += ext elif type == 'text': if not basename.startswith('documentation/'): raise ValueError("Items of type %s must be" " in the documentation section" % type) else: # Update existing item if mtime <= mod_time(item) and not force_update: if dry_run: sys.stdout.write("Skip %s: file %s is not newer\n" % (item.name, filename)) return if type is not None and type != datatype(item): raise ValueError("Cannot change datatype %s to %s" % (datatype(item), type)) if type is None: type = datatype(item) if language is None: language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None) if dry_run: sys.stdout.write("Delete %s\n" % item.name) else: del item.parent[item.name.split('/')[-1]] if dry_run: fulltype = type if language is None else '/'.join((type, language)) sys.stdout.write("Create item %s of type %s from file %s\n" % (basename, fulltype, filename)) else: if type in ['calclet', 'importlet', 'module']: item = paper.store_python_code(basename[5:], open(filename, 'rb').read()) stamp(item, type, {}) timestamp(item, mtime) elif type in ['file', 'text']: f = paper.open_internal_file(basename, 'w') f.write(open(filename, 'rb').read()) f.close() stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language}) timestamp(f._ds, mtime)
def update_from_file(paper, filename, type=None, force_update=False, dry_run=False, dataset_name=None, create_new=True): if not os.path.exists(filename): raise ValueError("File %s not found" % filename) mtime = os.path.getmtime(filename) basename = filename ext = '' if dataset_name is not None: item = paper.file.get(dataset_name, None) if item is not None: basename = item.name else: item = paper.file.get(basename, None) if item is None: basename, ext = os.path.splitext(filename) item = paper.file.get(basename, None) language = file_languages.get(ext, None) if item is None: if not create_new: return # Create new item if type is None: raise ValueError("Datatype required to create new item %s" % basename) if type in ['calclet', 'importlet', 'module']: if not basename.startswith('code/'): raise ValueError("Items of type %s must be" " in the code section" % type) if language != 'python': raise ValueError("Items of type %s must be Python code" % type) if type == 'module' and \ not basename.startswith('code/python-packages/'): raise ValueError("Items of type %s must be in" "code/python-packages" % type) elif type == 'file': if not basename.startswith('data/') \ and not basename.startswith('documentation/'): raise ValueError("Items of type %s must be" " in the data or documentation section" % type) basename += ext elif type == 'text': if not basename.startswith('documentation/'): raise ValueError("Items of type %s must be" " in the documentation section" % type) else: # Update existing item if mtime <= mod_time(item) and not force_update: if dry_run: sys.stdout.write("Skip %s: file %s is not newer\n" % (item.name, filename)) return if type is not None and type != datatype(item): raise ValueError("Cannot change datatype %s to %s" % (datatype(item), type)) if type is None: type = datatype(item) if language is None: language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None) if dry_run: sys.stdout.write("Delete %s\n" % item.name) else: del item.parent[item.name.split('/')[-1]] if dry_run: fulltype = type if language is None else '/'.join((type, language)) sys.stdout.write("Create item %s of type %s from file %s\n" % (basename, fulltype, filename)) else: if type in ['calclet', 'importlet', 'module']: code = open(filename, 'rb').read().decode('utf-8') item = paper.store_python_code(basename[5:], code) stamp(item, type, {}) timestamp(item, mtime) elif type in ['file', 'text']: f = paper.open_internal_file(basename, 'w') f.write(open(filename, 'rb').read()) f.close() stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language}) timestamp(f._ds, mtime)
def write_direct(source, source_sel=None, dest_sel=None): self._node.write_direct(source, source_sel, dest_sel) stamp(self._node, "data", self._codelet.dependency_attributes())