def run(self, setid, selected_nodes=None, deps=None, force=None, keep=None, update_detail=None, update_listing=None, excluded_nodes=None, cachesize=None): excluded_nodes = excluded_nodes or [] dataset = Dataset.query.filter(Dataset.setid == str(setid))\ .options(db.joinedload(Dataset.files))\ .one() collection = self.collections[dataset.collection] selected_nodes = set(selected_nodes or []) if not (selected_nodes or update_listing or update_detail): selected_nodes.update(collection.listing_deps) selected_nodes.update(collection.detail_deps) persistent = collection.nodes try: nodes = [ persistent[name] if not ':' in name else find_obj(name) for name in selected_nodes if name not in excluded_nodes if name != 'dataset' ] except KeyError as e: raise UnknownNode(dataset.collection, e.args[0]) nodes.sort() storedir = app.site.config.marv.storedir store = Store(storedir, persistent) changed = False try: if nodes: changed = run_nodes(dataset, nodes, store, force=force, persistent=persistent, deps=deps, cachesize=cachesize) except: raise else: if changed or update_detail: collection.render_detail(dataset) log.verbose('%s detail rendered', setid) if changed or update_listing: collection.update_listings([dataset]) log.verbose('%s listing rendered', setid) finally: if not keep: for tmpdir in store.pending.values(): store.logdebug('Cleaning up %r', tmpdir) shutil.rmtree(tmpdir) store.pending.clear()
async def run(self, setid, selected_nodes=None, deps=None, force=None, keep=None, force_dependent=None, update_detail=None, update_listing=None, excluded_nodes=None, cachesize=None): # pylint: disable=too-many-arguments,too-many-locals,too-many-branches assert not force_dependent or selected_nodes excluded_nodes = set(excluded_nodes or []) async with scoped_session(self.db) as txn: dataset = await Dataset.get(setid=setid)\ .prefetch_related('collection', 'files')\ .using_db(txn) collection = self.collections[dataset.collection.name] selected_nodes = set(selected_nodes or []) if not (selected_nodes or update_listing or update_detail): selected_nodes.update(collection.listing_deps) selected_nodes.update(collection.detail_deps) persistent = collection.nodes try: nodes = { persistent[name] if ':' not in name else Node.from_dag_node( find_obj(name)) for name in selected_nodes if name not in excluded_nodes if name != 'dataset' } except KeyError as exc: raise ConfigError( f'Collection {collection.name!r} has no node {exc}') if force_dependent: nodes.update(x for name in selected_nodes for x in persistent[name].dependent) nodes = sorted(nodes) storedir = self.config.marv.storedir store = Store(storedir, persistent) changed = False try: if nodes: changed = await run_nodes(dataset, nodes, store, force=force, persistent=persistent, deps=deps, cachesize=cachesize, site=self) finally: if not keep: for stream in store.pending: if stream.streamfile: stream.streamfile.close() for stream in store.readstreams: if stream.streamfile: stream.streamfile.close() for tmpdir, tmpdir_fd in store.pending.values(): store.logdebug('Cleaning up %r', tmpdir) shutil.rmtree(tmpdir) fcntl.flock(tmpdir_fd, fcntl.LOCK_UN) os.close(tmpdir_fd) store.pending.clear() if changed or update_detail: collection.render_detail(dataset) log.verbose('%s detail rendered', setid) if changed or update_listing: await collection.update_listings([dataset]) log.verbose('%s listing rendered', setid) return changed