def rebuild(self, filename): """ Rebuild all the dependent items in the paper in a new file. First all items without dependencies are copied to the new file, then all the calclets are run in the new file in the order determined by the dependency graph in the original file. """ deps = self.dependency_hierarchy() with ActivePaper(filename, 'w') as clone: for item in next(deps): # Make sure all the groups in the path exist path = item.name.split('/') name = path[-1] groups = path[:-1] dest = clone.file while groups: group_name = groups[0] if len(group_name) > 0: if group_name not in dest: dest.create_group(group_name) dest = dest[group_name] del groups[0] clone.file.copy(item, item.name, expand_refs=True) timestamp(clone.file[item.name]) for items in deps: calclets = set(item.attrs['ACTIVE_PAPER_GENERATING_CODELET'] for item in items) for calclet in calclets: clone.run_codelet(calclet)
def create_copy(self, path, paper_ref, ref_path=None): if ref_path is None: ref_path = path paper = open_paper_ref(paper_ref) item = paper.file[ref_path] self.file.copy(item, path, expand_refs=True) copy = self.file[path] self._delete_dependency_attributes(copy) timestamp(copy, mod_time(item)) ref_dtype = np.dtype([('paper_ref', h5vstring), ('path', h5vstring)]) copy.attrs.create('ACTIVE_PAPER_COPIED_FROM', shape=(), dtype=ref_dtype, data=np.array((paper_ref, ref_path), dtype=ref_dtype)) return copy
def replace_by_dummy(self, item_name): item = self.file[item_name] codelet = owner(item) assert codelet is not None dtype = datatype(item) mtime = mod_time(item) deps = item.attrs.get('ACTIVE_PAPER_DEPENDENCIES') del self.file[item_name] ds = self.file.create_dataset(item_name, data=np.zeros((), dtype=np.int)) stamp(ds, dtype, dict(ACTIVE_PAPER_GENERATING_CODELET=codelet, ACTIVE_PAPER_DEPENDENCIES=list(deps))) timestamp(ds, mtime) ds.attrs['ACTIVE_PAPER_DUMMY_DATASET'] = True
def update_from_file(paper, filename, type=None, force_update=False, dry_run=False, dataset_name=None, create_new=True): if not os.path.exists(filename): raise ValueError("File %s not found" % filename) mtime = os.path.getmtime(filename) basename = filename ext = '' if dataset_name is not None: item = paper.file.get(dataset_name, None) if item is not None: basename = item.name else: item = paper.file.get(basename, None) if item is None: basename, ext = os.path.splitext(filename) item = paper.file.get(basename, None) language = file_languages.get(ext, None) if item is None: if not create_new: return # Create new item if type is None: raise ValueError("Datatype required to create new item %s" % basename) if type in ['calclet', 'importlet', 'module']: if not basename.startswith('code/'): raise ValueError("Items of type %s must be" " in the code section" % type) if language != 'python': raise ValueError("Items of type %s must be Python code" % type) if type == 'module' and \ not basename.startswith('code/python-packages/'): raise ValueError("Items of type %s must be in" "code/python-packages" % type) elif type == 'file': if not basename.startswith('data/') \ and not basename.startswith('documentation/'): raise ValueError("Items of type %s must be" " in the data or documentation section" % type) basename += ext elif type == 'text': if not basename.startswith('documentation/'): raise ValueError("Items of type %s must be" " in the documentation section" % type) else: # Update existing item if mtime <= mod_time(item) and not force_update: if dry_run: sys.stdout.write("Skip %s: file %s is not newer\n" % (item.name, filename)) return if type is not None and type != datatype(item): raise ValueError("Cannot change datatype %s to %s" % (datatype(item), type)) if type is None: type = datatype(item) if language is None: language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None) if dry_run: sys.stdout.write("Delete %s\n" % item.name) else: del item.parent[item.name.split('/')[-1]] if dry_run: fulltype = type if language is None else '/'.join((type, language)) sys.stdout.write("Create item %s of type %s from file %s\n" % (basename, fulltype, filename)) else: if type in ['calclet', 'importlet', 'module']: item = paper.store_python_code(basename[5:], open(filename, 'rb').read()) stamp(item, type, {}) timestamp(item, mtime) elif type in ['file', 'text']: f = paper.open_internal_file(basename, 'w') f.write(open(filename, 'rb').read()) f.close() stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language}) timestamp(f._ds, mtime)
def update_from_file(paper, filename, type=None, force_update=False, dry_run=False, dataset_name=None, create_new=True): if not os.path.exists(filename): raise ValueError("File %s not found" % filename) mtime = os.path.getmtime(filename) basename = filename ext = '' if dataset_name is not None: item = paper.file.get(dataset_name, None) if item is not None: basename = item.name else: item = paper.file.get(basename, None) if item is None: basename, ext = os.path.splitext(filename) item = paper.file.get(basename, None) language = file_languages.get(ext, None) if item is None: if not create_new: return # Create new item if type is None: raise ValueError("Datatype required to create new item %s" % basename) if type in ['calclet', 'importlet', 'module']: if not basename.startswith('code/'): raise ValueError("Items of type %s must be" " in the code section" % type) if language != 'python': raise ValueError("Items of type %s must be Python code" % type) if type == 'module' and \ not basename.startswith('code/python-packages/'): raise ValueError("Items of type %s must be in" "code/python-packages" % type) elif type == 'file': if not basename.startswith('data/') \ and not basename.startswith('documentation/'): raise ValueError("Items of type %s must be" " in the data or documentation section" % type) basename += ext elif type == 'text': if not basename.startswith('documentation/'): raise ValueError("Items of type %s must be" " in the documentation section" % type) else: # Update existing item if mtime <= mod_time(item) and not force_update: if dry_run: sys.stdout.write("Skip %s: file %s is not newer\n" % (item.name, filename)) return if type is not None and type != datatype(item): raise ValueError("Cannot change datatype %s to %s" % (datatype(item), type)) if type is None: type = datatype(item) if language is None: language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None) if dry_run: sys.stdout.write("Delete %s\n" % item.name) else: del item.parent[item.name.split('/')[-1]] if dry_run: fulltype = type if language is None else '/'.join((type, language)) sys.stdout.write("Create item %s of type %s from file %s\n" % (basename, fulltype, filename)) else: if type in ['calclet', 'importlet', 'module']: code = open(filename, 'rb').read().decode('utf-8') item = paper.store_python_code(basename[5:], code) stamp(item, type, {}) timestamp(item, mtime) elif type in ['file', 'text']: f = paper.open_internal_file(basename, 'w') f.write(open(filename, 'rb').read()) f.close() stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language}) timestamp(f._ds, mtime)