def create_copy(self, path, paper_ref, ref_path=None):
if ref_path is None:
ref_path = path
paper = open_paper_ref(paper_ref)
item = paper.file[ref_path]
self.file.copy(item, path, expand_refs=True)
copy = self.file[path]
self._delete_dependency_attributes(copy)
timestamp(copy, mod_time(item))
ref_dtype = np.dtype([('paper_ref', h5vstring), ('path', h5vstring)])
copy.attrs.create('ACTIVE_PAPER_COPIED_FROM',
shape=(), dtype=ref_dtype,
data=np.array((paper_ref, ref_path), dtype=ref_dtype))
return copy
def create_copy(self, path, paper_ref, ref_path=None):
if ref_path is None:
ref_path = path
paper = open_paper_ref(paper_ref)
item = paper.file[ref_path]
self.file.copy(item, path, expand_refs=True)
copy = self.file[path]
self._delete_dependency_attributes(copy)
timestamp(copy, mod_time(item))
ref_dtype = np.dtype([('paper_ref', h5vstring), ('path', h5vstring)])
copy.attrs.create('ACTIVE_PAPER_COPIED_FROM',
shape=(), dtype=ref_dtype,
data=np.array((paper_ref, ref_path), dtype=ref_dtype))
return copy
def replace_by_dummy(self, item_name):
item = self.file[item_name]
codelet = owner(item)
assert codelet is not None
dtype = datatype(item)
mtime = mod_time(item)
deps = item.attrs.get('ACTIVE_PAPER_DEPENDENCIES')
del self.file[item_name]
ds = self.file.create_dataset(item_name,
data=np.zeros((), dtype=np.int))
stamp(ds, dtype,
dict(ACTIVE_PAPER_GENERATING_CODELET=codelet,
ACTIVE_PAPER_DEPENDENCIES=list(deps)))
timestamp(ds, mtime)
ds.attrs['ACTIVE_PAPER_DUMMY_DATASET'] = True
def replace_by_dummy(self, item_name):
item = self.file[item_name]
codelet = owner(item)
assert codelet is not None
dtype = datatype(item)
mtime = mod_time(item)
deps = item.attrs.get('ACTIVE_PAPER_DEPENDENCIES')
del self.file[item_name]
ds = self.file.create_dataset(item_name,
data=np.zeros((), dtype=np.int))
stamp(ds, dtype,
dict(ACTIVE_PAPER_GENERATING_CODELET=codelet,
ACTIVE_PAPER_DEPENDENCIES=list(deps)))
timestamp(ds, mtime)
ds.attrs['ACTIVE_PAPER_DUMMY_DATASET'] = True
def extract_to_file(paper, item, file=None, filename=None, directory=None):
if file is None:
if filename is not None:
filename = os.path.abspath(filename)
if directory is not None:
directory = os.path.abspath(directory)
if filename is not None and directory is not None:
if not filename.startswith(directory):
raise ValueError("% not in directory %s"
% (filename, directory))
if filename is None:
item_name = item.name.split('/')[1:]
filename = os.path.join(directory, *item_name)
if '.' not in item_name[-1]:
# Add a file extension using some heuristics
language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None)
filename += file_extensions.get((datatype(item), language), '')
directory, _ = os.path.split(filename)
if directory and not os.path.exists(directory):
os.makedirs(directory)
file = open(filename, 'wb')
close = True
else:
# If a file object is given, no other file specification is allowed
assert filename is None
assert directory is None
close = False
dt = datatype(item)
if dt in ['file', 'text']:
internal = activepapers.storage.InternalFile(item, 'rb')
file.write(internal.read())
elif dt in extractable_types:
file.write(item[...].flat[0])
else:
raise ValueError("cannot extract dataset %s of type %s"
% (item.name, dt))
if close:
file.close()
mtime = mod_time(item)
if mtime:
os.utime(filename, (mtime, mtime))
return filename
def extract_to_file(paper, item, file=None, filename=None, directory=None):
if file is None:
if filename is not None:
filename = os.path.abspath(filename)
if directory is not None:
directory = os.path.abspath(directory)
if filename is not None and directory is not None:
if not filename.startswith(directory):
raise ValueError("% not in directory %s" %
(filename, directory))
if filename is None:
item_name = item.name.split('/')[1:]
filename = os.path.join(directory, *item_name)
if '.' not in item_name[-1]:
# Add a file extension using some heuristics
language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None)
filename += file_extensions.get((datatype(item), language), '')
directory, _ = os.path.split(filename)
if directory and not os.path.exists(directory):
os.makedirs(directory)
file = open(filename, 'wb')
close = True
else:
# If a file object is given, no other file specification is allowed
assert filename is None
assert directory is None
close = False
dt = datatype(item)
if dt in ['file', 'text']:
internal = activepapers.storage.InternalFile(item, 'rb')
file.write(internal.read())
elif dt in extractable_types:
file.write(item[...].flat[0])
else:
raise ValueError("cannot extract dataset %s of type %s" %
(item.name, dt))
if close:
file.close()
mtime = mod_time(item)
if mtime:
os.utime(filename, (mtime, mtime))
return filename
def is_stale(self, item):
t = mod_time(item)
for dep in self.iter_dependencies(item):
if mod_time(dep) > t:
return True
return False
def is_stale(self, item):
t = mod_time(item)
for dep in self.iter_dependencies(item):
if mod_time(dep) > t:
return True
return False
def update_from_file(paper, filename, type=None,
force_update=False, dry_run=False,
dataset_name=None, create_new=True):
if not os.path.exists(filename):
raise ValueError("File %s not found" % filename)
mtime = os.path.getmtime(filename)
basename = filename
ext = ''
if dataset_name is not None:
item = paper.file.get(dataset_name, None)
if item is not None:
basename = item.name
else:
item = paper.file.get(basename, None)
if item is None:
basename, ext = os.path.splitext(filename)
item = paper.file.get(basename, None)
language = file_languages.get(ext, None)
if item is None:
if not create_new:
return
# Create new item
if type is None:
raise ValueError("Datatype required to create new item %s"
% basename)
if type in ['calclet', 'importlet', 'module']:
if not basename.startswith('code/'):
raise ValueError("Items of type %s must be"
" in the code section"
% type)
if language != 'python':
raise ValueError("Items of type %s must be Python code"
% type)
if type == 'module' and \
not basename.startswith('code/python-packages/'):
raise ValueError("Items of type %s must be in"
"code/python-packages"
% type)
elif type == 'file':
if not basename.startswith('data/') \
and not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the data or documentation section"
% type)
basename += ext
elif type == 'text':
if not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the documentation section"
% type)
else:
# Update existing item
if mtime <= mod_time(item) and not force_update:
if dry_run:
sys.stdout.write("Skip %s: file %s is not newer\n"
% (item.name, filename))
return
if type is not None and type != datatype(item):
raise ValueError("Cannot change datatype %s to %s"
% (datatype(item), type))
if type is None:
type = datatype(item)
if language is None:
language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None)
if dry_run:
sys.stdout.write("Delete %s\n" % item.name)
else:
del item.parent[item.name.split('/')[-1]]
if dry_run:
fulltype = type if language is None else '/'.join((type, language))
sys.stdout.write("Create item %s of type %s from file %s\n"
% (basename, fulltype, filename))
else:
if type in ['calclet', 'importlet', 'module']:
item = paper.store_python_code(basename[5:],
open(filename, 'rb').read())
stamp(item, type, {})
timestamp(item, mtime)
elif type in ['file', 'text']:
f = paper.open_internal_file(basename, 'w')
f.write(open(filename, 'rb').read())
f.close()
stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language})
timestamp(f._ds, mtime)
def update_from_file(paper,
filename,
type=None,
force_update=False,
dry_run=False,
dataset_name=None,
create_new=True):
if not os.path.exists(filename):
raise ValueError("File %s not found" % filename)
mtime = os.path.getmtime(filename)
basename = filename
ext = ''
if dataset_name is not None:
item = paper.file.get(dataset_name, None)
if item is not None:
basename = item.name
else:
item = paper.file.get(basename, None)
if item is None:
basename, ext = os.path.splitext(filename)
item = paper.file.get(basename, None)
language = file_languages.get(ext, None)
if item is None:
if not create_new:
return
# Create new item
if type is None:
raise ValueError("Datatype required to create new item %s" %
basename)
if type in ['calclet', 'importlet', 'module']:
if not basename.startswith('code/'):
raise ValueError("Items of type %s must be"
" in the code section" % type)
if language != 'python':
raise ValueError("Items of type %s must be Python code" % type)
if type == 'module' and \
not basename.startswith('code/python-packages/'):
raise ValueError("Items of type %s must be in"
"code/python-packages" % type)
elif type == 'file':
if not basename.startswith('data/') \
and not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the data or documentation section" %
type)
basename += ext
elif type == 'text':
if not basename.startswith('documentation/'):
raise ValueError("Items of type %s must be"
" in the documentation section" % type)
else:
# Update existing item
if mtime <= mod_time(item) and not force_update:
if dry_run:
sys.stdout.write("Skip %s: file %s is not newer\n" %
(item.name, filename))
return
if type is not None and type != datatype(item):
raise ValueError("Cannot change datatype %s to %s" %
(datatype(item), type))
if type is None:
type = datatype(item)
if language is None:
language = item.attrs.get('ACTIVE_PAPER_LANGUAGE', None)
if dry_run:
sys.stdout.write("Delete %s\n" % item.name)
else:
del item.parent[item.name.split('/')[-1]]
if dry_run:
fulltype = type if language is None else '/'.join((type, language))
sys.stdout.write("Create item %s of type %s from file %s\n" %
(basename, fulltype, filename))
else:
if type in ['calclet', 'importlet', 'module']:
code = open(filename, 'rb').read().decode('utf-8')
item = paper.store_python_code(basename[5:], code)
stamp(item, type, {})
timestamp(item, mtime)
elif type in ['file', 'text']:
f = paper.open_internal_file(basename, 'w')
f.write(open(filename, 'rb').read())
f.close()
stamp(f._ds, type, {'ACTIVE_PAPER_LANGUAGE': language})
timestamp(f._ds, mtime)
