def run(self,
setid,
selected_nodes=None,
deps=None,
force=None,
keep=None,
update_detail=None,
update_listing=None,
excluded_nodes=None,
cachesize=None):
excluded_nodes = excluded_nodes or []
dataset = Dataset.query.filter(Dataset.setid == str(setid))\
.options(db.joinedload(Dataset.files))\
.one()
collection = self.collections[dataset.collection]
selected_nodes = set(selected_nodes or [])
if not (selected_nodes or update_listing or update_detail):
selected_nodes.update(collection.listing_deps)
selected_nodes.update(collection.detail_deps)
persistent = collection.nodes
try:
nodes = [
persistent[name] if not ':' in name else find_obj(name)
for name in selected_nodes if name not in excluded_nodes
if name != 'dataset'
]
except KeyError as e:
raise UnknownNode(dataset.collection, e.args[0])
nodes.sort()
storedir = app.site.config.marv.storedir
store = Store(storedir, persistent)
changed = False
try:
if nodes:
changed = run_nodes(dataset,
nodes,
store,
force=force,
persistent=persistent,
deps=deps,
cachesize=cachesize)
except:
raise
else:
if changed or update_detail:
collection.render_detail(dataset)
log.verbose('%s detail rendered', setid)
if changed or update_listing:
collection.update_listings([dataset])
log.verbose('%s listing rendered', setid)
finally:
if not keep:
for tmpdir in store.pending.values():
store.logdebug('Cleaning up %r', tmpdir)
shutil.rmtree(tmpdir)
store.pending.clear()
async def run(self,
setid,
selected_nodes=None,
deps=None,
force=None,
keep=None,
force_dependent=None,
update_detail=None,
update_listing=None,
excluded_nodes=None,
cachesize=None):
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
assert not force_dependent or selected_nodes
excluded_nodes = set(excluded_nodes or [])
async with scoped_session(self.db) as txn:
dataset = await Dataset.get(setid=setid)\
.prefetch_related('collection', 'files')\
.using_db(txn)
collection = self.collections[dataset.collection.name]
selected_nodes = set(selected_nodes or [])
if not (selected_nodes or update_listing or update_detail):
selected_nodes.update(collection.listing_deps)
selected_nodes.update(collection.detail_deps)
persistent = collection.nodes
try:
nodes = {
persistent[name] if ':' not in name else Node.from_dag_node(
find_obj(name))
for name in selected_nodes if name not in excluded_nodes
if name != 'dataset'
}
except KeyError as exc:
raise ConfigError(
f'Collection {collection.name!r} has no node {exc}')
if force_dependent:
nodes.update(x for name in selected_nodes
for x in persistent[name].dependent)
nodes = sorted(nodes)
storedir = self.config.marv.storedir
store = Store(storedir, persistent)
changed = False
try:
if nodes:
changed = await run_nodes(dataset,
nodes,
store,
force=force,
persistent=persistent,
deps=deps,
cachesize=cachesize,
site=self)
finally:
if not keep:
for stream in store.pending:
if stream.streamfile:
stream.streamfile.close()
for stream in store.readstreams:
if stream.streamfile:
stream.streamfile.close()
for tmpdir, tmpdir_fd in store.pending.values():
store.logdebug('Cleaning up %r', tmpdir)
shutil.rmtree(tmpdir)
fcntl.flock(tmpdir_fd, fcntl.LOCK_UN)
os.close(tmpdir_fd)
store.pending.clear()
if changed or update_detail:
collection.render_detail(dataset)
log.verbose('%s detail rendered', setid)
if changed or update_listing:
await collection.update_listings([dataset])
log.verbose('%s listing rendered', setid)
return changed