Exemplo n.º 1
0
    def _write_parallel(self, docnames, warnings, nproc):
        def write_process(docs):
            local_warnings = []
            self.env.set_warnfunc(lambda *args: local_warnings.append(args))
            for docname, doctree in docs:
                self.write_doc(docname, doctree)
            return local_warnings

        def add_warnings(docs, wlist):
            warnings.extend(wlist)

        # warm up caches/compile templates using the first document
        firstname, docnames = docnames[0], docnames[1:]
        doctree = self.env.get_and_resolve_doctree(firstname, self)
        self.write_doc_serialized(firstname, doctree)
        self.write_doc(firstname, doctree)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in self.app.status_iterator(
                chunks, 'writing output... ', darkgreen, len(chunks)):
            arg = []
            for i, docname in enumerate(chunk):
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                arg.append((docname, doctree))
            tasks.add_task(write_process, arg, add_warnings)

        # make sure all threads have finished
        self.info(bold('waiting for workers...'))
        tasks.join()

        for warning in warnings:
            self.warn(*warning)
Exemplo n.º 2
0
    def _read_parallel(self, docnames, nproc):
        # type: (List[unicode], int) -> None
        # clear all outdated docs at once
        for docname in docnames:
            self.app.emit('env-purge-doc', self.env, docname)
            self.env.clear_doc(docname)

        def read_process(docs):
            # type: (List[unicode]) -> bytes
            self.env.app = self.app
            for docname in docs:
                self.read_doc(docname)
            # allow pickling self to send it back
            return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)

        def merge(docs, otherenv):
            # type: (List[unicode], bytes) -> None
            env = pickle.loads(otherenv)
            self.env.merge_info_from(docs, env, self.app)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in status_iterator(chunks, 'reading sources... ', "purple",
                                     len(chunks), self.app.verbosity):
            tasks.add_task(read_process, chunk, merge)

        # make sure all threads have finished
        logger.info(bold('waiting for workers...'))
        tasks.join()
Exemplo n.º 3
0
    def _write_parallel(self, docnames, nproc):
        # type: (Sequence[unicode], int) -> None
        def write_process(docs):
            # type: (List[Tuple[unicode, nodes.Node]]) -> None
            self.app.phase = BuildPhase.WRITING
            for docname, doctree in docs:
                self.write_doc(docname, doctree)

        # warm up caches/compile templates using the first document
        firstname, docnames = docnames[0], docnames[1:]
        self.app.phase = BuildPhase.RESOLVING
        doctree = self.env.get_and_resolve_doctree(firstname, self)
        self.app.phase = BuildPhase.WRITING
        self.write_doc_serialized(firstname, doctree)
        self.write_doc(firstname, doctree)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        self.app.phase = BuildPhase.RESOLVING
        for chunk in status_iterator(chunks, __('writing output... '), "darkgreen",
                                     len(chunks), self.app.verbosity):
            arg = []
            for i, docname in enumerate(chunk):
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                arg.append((docname, doctree))
            tasks.add_task(write_process, arg)

        # make sure all threads have finished
        logger.info(bold(__('waiting for workers...')))
        tasks.join()
Exemplo n.º 4
0
def test_logging_in_ParallelTasks(app, status, warning):
    logging.setup(app, status, warning)
    logger = logging.getLogger(__name__)

    def child_process():
        logger.info('message1')
        logger.warning('message2', location='index')

    tasks = ParallelTasks(1)
    tasks.add_task(child_process)
    tasks.join()
    assert 'message1' in status.getvalue()
    assert 'index.txt: WARNING: message2' in warning.getvalue()
Exemplo n.º 5
0
    def _write_parallel(self, docnames, warnings, nproc):
        def write_process(docs):
            local_warnings = []
            self.env.set_warnfunc(lambda *args: local_warnings.append(args))
            for docname, doctree in docs:
                self.write_doc(docname, doctree)
            return local_warnings

        def add_warnings(docs, wlist):
            warnings.extend(wlist)

        # warm up caches/compile templates using the first document
        firstname, docnames = docnames[0], docnames[1:]
        doctree = self.env.get_and_resolve_doctree(firstname, self)
        self.write_doc_serialized(firstname, doctree)
        self.write_doc(firstname, doctree)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in self.app.status_iterator(chunks, 'writing output... ',
                                              darkgreen, len(chunks)):
            arg = []
            for i, docname in enumerate(chunk):
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                arg.append((docname, doctree))
            tasks.add_task(write_process, arg, add_warnings)

        # make sure all threads have finished
        self.info(bold('waiting for workers...'))
        tasks.join()

        for warning in warnings:
            self.warn(*warning)
Exemplo n.º 6
0
    def _read_parallel(self, docnames, app, nproc):
        # type: (List[unicode], Sphinx, int) -> None
        # clear all outdated docs at once
        for docname in docnames:
            app.emit('env-purge-doc', self, docname)
            self.clear_doc(docname)

        def read_process(docs):
            # type: (List[unicode]) -> BuildEnvironment
            self.app = app
            for docname in docs:
                self.read_doc(docname, app)
            # allow pickling self to send it back
            del self.app
            del self.domains
            del self.config.values
            del self.config
            return self

        def merge(docs, otherenv):
            # type: (List[unicode], BuildEnvironment) -> None
            self.merge_info_from(docs, otherenv, app)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in status_iterator(chunks, 'reading sources... ', "purple",
                                     len(chunks), self.app.verbosity):
            tasks.add_task(read_process, chunk, merge)

        # make sure all threads have finished
        logger.info(bold('waiting for workers...'))
        tasks.join()
Exemplo n.º 7
0
    def _read_parallel(self, docnames, nproc):
        # type: (List[unicode], int) -> None
        # clear all outdated docs at once
        for docname in docnames:
            self.app.emit('env-purge-doc', self.env, docname)
            self.env.clear_doc(docname)

        def read_process(docs):
            # type: (List[unicode]) -> unicode
            self.env.app = self.app
            for docname in docs:
                self.env.read_doc(docname, self.app)
            # allow pickling self to send it back
            return BuildEnvironment.dumps(self.env)

        def merge(docs, otherenv):
            # type: (List[unicode], unicode) -> None
            env = BuildEnvironment.loads(otherenv)
            self.env.merge_info_from(docs, env, self.app)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in status_iterator(chunks, 'reading sources... ', "purple",
                                     len(chunks), self.app.verbosity):
            tasks.add_task(read_process, chunk, merge)

        # make sure all threads have finished
        logger.info(bold('waiting for workers...'))
        tasks.join()
Exemplo n.º 8
0
    def _read_parallel(self, docnames: List[str], nproc: int) -> None:
        # clear all outdated docs at once
        for docname in docnames:
            self.events.emit('env-purge-doc', self.env, docname)
            self.env.clear_doc(docname)

        def read_process(docs: List[str]) -> bytes:
            self.env.app = self.app
            for docname in docs:
                self.read_doc(docname)
            # allow pickling self to send it back
            return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)

        def merge(docs: List[str], otherenv: bytes) -> None:
            env = pickle.loads(otherenv)
            self.env.merge_info_from(docs, env, self.app)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in status_iterator(chunks, __('reading sources... '), "purple",
                                     len(chunks), self.app.verbosity):
            tasks.add_task(read_process, chunk, merge)

        # make sure all threads have finished
        logger.info(bold(__('waiting for workers...')))
        tasks.join()
Exemplo n.º 9
0
    def _read_parallel(self, docnames: List[str], nproc: int) -> None:
        chunks = make_chunks(docnames, nproc)

        # create a status_iterator to step progressbar after reading a document
        # (see: ``merge()`` function)
        progress = status_iterator(chunks, __('reading sources... '), "purple",
                                   len(chunks), self.app.verbosity)

        # clear all outdated docs at once
        for docname in docnames:
            self.events.emit('env-purge-doc', self.env, docname)
            self.env.clear_doc(docname)

        def read_process(docs: List[str]) -> bytes:
            self.env.app = self.app
            for docname in docs:
                self.read_doc(docname)
            # allow pickling self to send it back
            return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)

        def merge(docs: List[str], otherenv: bytes) -> None:
            env = pickle.loads(otherenv)
            self.env.merge_info_from(docs, env, self.app)

            next(progress)

        tasks = ParallelTasks(nproc)
        for chunk in chunks:
            tasks.add_task(read_process, chunk, merge)

        # make sure all threads have finished
        tasks.join()
        logger.info('')
Exemplo n.º 10
0
    def _write_parallel(self, docnames, nproc):
        # type: (Sequence[unicode], int) -> None
        def write_process(docs):
            # type: (List[Tuple[unicode, nodes.Node]]) -> None
            self.app.phase = BuildPhase.WRITING
            for docname, doctree in docs:
                self.write_doc(docname, doctree)

        # warm up caches/compile templates using the first document
        firstname, docnames = docnames[0], docnames[1:]
        self.app.phase = BuildPhase.RESOLVING
        doctree = self.env.get_and_resolve_doctree(firstname, self)
        self.app.phase = BuildPhase.WRITING
        self.write_doc_serialized(firstname, doctree)
        self.write_doc(firstname, doctree)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        self.app.phase = BuildPhase.RESOLVING
        for chunk in status_iterator(chunks,
                                     __('writing output... '), "darkgreen",
                                     len(chunks), self.app.verbosity):
            arg = []
            for i, docname in enumerate(chunk):
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                arg.append((docname, doctree))
            tasks.add_task(write_process, arg)

        # make sure all threads have finished
        logger.info(bold(__('waiting for workers...')))
        tasks.join()
Exemplo n.º 11
0
    def _write_parallel(self, docnames, warnings, nproc):
        # type: (Iterable[unicode], List[Tuple[Tuple, Dict]], int) -> None
        def write_process(docs):
            # type: (List[Tuple[unicode, nodes.Node]]) -> List[Tuple[Tuple, Dict]]
            local_warnings = []

            def warnfunc(*args, **kwargs):
                local_warnings.append((args, kwargs))
            self.env.set_warnfunc(warnfunc)
            for docname, doctree in docs:
                self.write_doc(docname, doctree)
            return local_warnings

        def add_warnings(docs, wlist):
            warnings.extend(wlist)

        # warm up caches/compile templates using the first document
        firstname, docnames = docnames[0], docnames[1:]  # type: ignore
        doctree = self.env.get_and_resolve_doctree(firstname, self)
        self.write_doc_serialized(firstname, doctree)
        self.write_doc(firstname, doctree)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in self.app.status_iterator(
                chunks, 'writing output... ', darkgreen, len(chunks)):
            arg = []
            for i, docname in enumerate(chunk):
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                arg.append((docname, doctree))
            tasks.add_task(write_process, arg, add_warnings)

        # make sure all threads have finished
        self.info(bold('waiting for workers...'))
        tasks.join()

        for warning, kwargs in warnings:
            self.warn(*warning, **kwargs)
Exemplo n.º 12
0
    def _write_parallel(self, docnames, nproc):
        # type: (Iterable[unicode], int) -> None
        def write_process(docs):
            # type: (List[Tuple[unicode, nodes.Node]]) -> None
            for docname, doctree in docs:
                self.write_doc(docname, doctree)

        # warm up caches/compile templates using the first document
        firstname, docnames = docnames[0], docnames[1:]  # type: ignore
        doctree = self.env.get_and_resolve_doctree(firstname, self)
        self.write_doc_serialized(firstname, doctree)
        self.write_doc(firstname, doctree)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in self.app.status_iterator(chunks, 'writing output... ',
                                              darkgreen, len(chunks)):
            arg = []
            for i, docname in enumerate(chunk):
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                arg.append((docname, doctree))
            tasks.add_task(write_process, arg)

        # make sure all threads have finished
        logger.info(bold('waiting for workers...'))
        tasks.join()
Exemplo n.º 13
0
    def _read_parallel(self, docnames, app, nproc):
        # clear all outdated docs at once
        for docname in docnames:
            app.emit('env-purge-doc', self, docname)
            self.clear_doc(docname)

        def read_process(docs):
            self.app = app
            self.warnings = []
            self.set_warnfunc(lambda *args, **kwargs: self.warnings.append((args, kwargs)))
            for docname in docs:
                self.read_doc(docname, app)
            # allow pickling self to send it back
            self.set_warnfunc(None)
            del self.app
            del self.domains
            del self.config.values
            del self.config
            return self

        def merge(docs, otherenv):
            warnings.extend(otherenv.warnings)
            self.merge_info_from(docs, otherenv, app)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        warnings = []
        for chunk in app.status_iterator(
                chunks, 'reading sources... ', purple, len(chunks)):
            tasks.add_task(read_process, chunk, merge)

        # make sure all threads have finished
        app.info(bold('waiting for workers...'))
        tasks.join()

        for warning, kwargs in warnings:
            self._warnfunc(*warning, **kwargs)
Exemplo n.º 14
0
def generate_recipes(app):
    """
    Go through every folder in the `bioconda-recipes/recipes` dir,
    have a README.rst file generated and generate a recipes.rst from
    the collected data.
    """
    renderer = Renderer(app)
    load_config(os.path.join(os.path.dirname(RECIPE_DIR), "config.yml"))
    repodata = RepoData()
    repodata.set_cache(op.join(app.env.doctreedir, 'RepoDataCache.csv'))
    # force loading repodata to avoid duplicate loads from threads
    repodata.df  # pylint: disable=pointless-statement
    recipes: List[Dict[str, Any]] = []
    recipe_dirs = os.listdir(RECIPE_DIR)

    if parallel_available and len(recipe_dirs) > 5:
        nproc = app.parallel
    else:
        nproc = 1

    if nproc == 1:
        for folder in status_iterator(recipe_dirs,
                                      'Generating package READMEs...',
                                      "purple", len(recipe_dirs),
                                      app.verbosity):
            if not op.isdir(op.join(RECIPE_DIR, folder)):
                logger.error("Item '%s' in recipes folder is not a folder",
                             folder)
                continue
            recipes.extend(generate_readme(folder, repodata, renderer))
    else:
        tasks = ParallelTasks(nproc)
        chunks = make_chunks(recipe_dirs, nproc)

        def process_chunk(chunk):
            _recipes: List[Dict[str, Any]] = []
            for folder in chunk:
                if not op.isdir(op.join(RECIPE_DIR, folder)):
                    logger.error("Item '%s' in recipes folder is not a folder",
                                 folder)
                    continue
                _recipes.extend(generate_readme(folder, repodata, renderer))
            return _recipes

        def merge_chunk(_chunk, res):
            recipes.extend(res)

        for chunk in status_iterator(
                chunks,
                'Generating package READMEs with {} threads...'.format(nproc),
                "purple", len(chunks), app.verbosity):
            tasks.add_task(process_chunk, chunk, merge_chunk)
        logger.info("waiting for workers...")
        tasks.join()
Exemplo n.º 15
0
def generate_recipes(app):
    """
    Go through every folder in the `bioconda-recipes/recipes` dir,
    have a README.rst file generated and generate a recipes.rst from
    the collected data.
    """
    renderer = Renderer(app)
    repodata = RepoData()
    recipes = []
    recipe_dirs = os.listdir(RECIPE_DIR)

    if parallel_available and len(recipe_dirs) > 5:
        nproc = app.parallel
    else:
        nproc = 1

    if nproc == 1:
        for folder in status_iterator(recipe_dirs,
                                      'Generating package READMEs...',
                                      "purple", len(recipe_dirs),
                                      app.verbosity):
            recipes.extend(generate_readme(folder, repodata, renderer))
    else:
        tasks = ParallelTasks(nproc)
        chunks = make_chunks(recipe_dirs, nproc)

        def process_chunk(chunk):
            _recipes = []
            for folder in chunk:
                _recipes.extend(generate_readme(folder, repodata, renderer))
            return _recipes

        def merge_chunk(chunk, res):
            recipes.extend(res)

        for chunk in status_iterator(
                chunks,
                'Generating package READMEs with {} threads...'.format(nproc),
                "purple", len(chunks), app.verbosity):
            tasks.add_task(process_chunk, chunk, merge_chunk)
        logger.info("waiting for workers...")
        tasks.join()

    updated = renderer.render_to_file(
        "source/recipes.rst",
        "recipes.rst_t",
        {
            'recipes': recipes,
            # order of columns in the table; must be keys in template_options
            'keys': ['Package', 'Version', 'License', 'Linux', 'OSX']
        })
    if updated:
        logger.info("Updated source/recipes.rst")
Exemplo n.º 16
0
def test_logging_in_ParallelTasks(app, status, warning):
    logging.setup(app, status, warning)
    logger = logging.getLogger(__name__)

    def child_process():
        logger.info('message1')
        logger.warning('message2', location='index')

    tasks = ParallelTasks(1)
    tasks.add_task(child_process)
    tasks.join()
    assert 'message1' in status.getvalue()
    assert 'index.txt: WARNING: message2' in warning.getvalue()
Exemplo n.º 17
0
    def _write_parallel(self, docnames, warnings, nproc):
        # type: (Iterable[unicode], List[Tuple[Tuple, Dict]], int) -> None
        def write_process(docs):
            # type: (List[Tuple[unicode, nodes.Node]]) -> List[Tuple[Tuple, Dict]]
            local_warnings = []

            def warnfunc(*args, **kwargs):
                local_warnings.append((args, kwargs))

            self.env.set_warnfunc(warnfunc)
            for docname, doctree in docs:
                self.write_doc(docname, doctree)
            return local_warnings

        def add_warnings(docs, wlist):
            warnings.extend(wlist)

        # warm up caches/compile templates using the first document
        firstname, docnames = docnames[0], docnames[1:]  # type: ignore
        doctree = self.env.get_and_resolve_doctree(firstname, self)
        self.write_doc_serialized(firstname, doctree)
        self.write_doc(firstname, doctree)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        for chunk in self.app.status_iterator(chunks, 'writing output... ',
                                              darkgreen, len(chunks)):
            arg = []
            for i, docname in enumerate(chunk):
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                arg.append((docname, doctree))
            tasks.add_task(write_process, arg, add_warnings)

        # make sure all threads have finished
        self.info(bold('waiting for workers...'))
        tasks.join()

        for warning, kwargs in warnings:
            self.warn(*warning, **kwargs)
Exemplo n.º 18
0
    def _write_parallel(self, docnames: Sequence[str], nproc: int) -> None:
        def write_process(docs: List[Tuple[str, nodes.document]]) -> None:
            self.app.phase = BuildPhase.WRITING
            for docname, doctree in docs:
                self.write_doc(docname, doctree)

        # warm up caches/compile templates using the first document
        firstname, docnames = docnames[0], docnames[1:]
        self.app.phase = BuildPhase.RESOLVING
        doctree = self.env.get_and_resolve_doctree(firstname, self)
        self.app.phase = BuildPhase.WRITING
        self.write_doc_serialized(firstname, doctree)
        self.write_doc(firstname, doctree)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        # create a status_iterator to step progressbar after writing a document
        # (see: ``on_chunk_done()`` function)
        progress = status_iterator(chunks,
                                   __('writing output... '), "darkgreen",
                                   len(chunks), self.app.verbosity)

        def on_chunk_done(args: List[Tuple[str, NoneType]],
                          result: NoneType) -> None:
            next(progress)

        self.app.phase = BuildPhase.RESOLVING
        for chunk in chunks:
            arg = []
            for docname in chunk:
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                arg.append((docname, doctree))
            tasks.add_task(write_process, arg, on_chunk_done)

        # make sure all threads have finished
        tasks.join()
        logger.info('')
Exemplo n.º 19
0
    def _read_parallel(self, docnames, app, nproc):
        # clear all outdated docs at once
        for docname in docnames:
            app.emit('env-purge-doc', self, docname)
            self.clear_doc(docname)

        def read_process(docs):
            self.app = app
            self.warnings = []
            self.set_warnfunc(lambda *args, **kwargs: self.warnings.append((args, kwargs)))
            for docname in docs:
                self.read_doc(docname, app)
            # allow pickling self to send it back
            self.set_warnfunc(None)
            del self.app
            del self.domains
            del self.config.values
            del self.config
            return self

        def merge(docs, otherenv):
            warnings.extend(otherenv.warnings)
            self.merge_info_from(docs, otherenv, app)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docnames, nproc)

        warnings = []
        for chunk in app.status_iterator(
                chunks, 'reading sources... ', purple, len(chunks)):
            tasks.add_task(read_process, chunk, merge)

        # make sure all threads have finished
        app.info(bold('waiting for workers...'))
        tasks.join()

        for warning, kwargs in warnings:
            self._warnfunc(*warning, **kwargs)
Exemplo n.º 20
0
    def _write_parallel(self, docCfgList, warnings, nproc):
        def write_process(docs):
            local_warnings = []

            def warnfunc(*args, **kwargs):
                local_warnings.append((args, kwargs))

            self.env.set_warnfunc(warnfunc)
            for targetname, doctree in docs:
                self.write_doc(targetname, doctree)
            return local_warnings

        def add_warnings(_docs, wlist):
            warnings.extend(wlist)

        # warm up caches/compile templates using the first document
        docCfg, docCfgList = docCfgList[0], docCfgList[1:]

        self.write_doc_serialized(docCfg)
        self.write_doc(docCfg)

        tasks = ParallelTasks(nproc)
        chunks = make_chunks(docCfg, nproc)

        for chunk in self.app.status_iterator(chunks, 'writing output... ',
                                              darkgreen, len(chunks)):
            arg = []
            for docCfg in chunk:
                self.write_doc_serialized(docCfg)
                arg.append((docCfg, ))
            tasks.add_task(write_process, arg, add_warnings)

        # make sure all threads have finished
        self.info(bold('waiting for workers...'))
        tasks.join()

        for warning, kwargs in warnings:
            self.warn(*warning, **kwargs)
Exemplo n.º 21
0
def generate_recipes(app):
    """Generates recipe RST files

    - Checks out repository
    - Prepares `RepoData`
    - Selects recipes (if BIOCONDA_FILTER_RECIPES in environment)
    - Dispatches calls to `generate_readme` for each recipe
    - Removes old RST files
    """
    source_dir = app.env.srcdir
    doctree_dir = app.env.doctreedir  # .../build/doctrees
    repo_dir = op.join(op.dirname(app.env.srcdir), "_bioconda_recipes")
    recipe_basedir = op.join(repo_dir, app.config.bioconda_recipes_path)
    repodata_cache_file = op.join(doctree_dir, 'RepoDataCache.pkl')
    repo_config_file = os.path.join(repo_dir, app.config.bioconda_config_file)
    output_dir = op.join(source_dir, 'recipes')

    # Initialize Repo and point globals at the right place
    repo = BiocondaRepo(folder=repo_dir, home=app.config.bioconda_repo_url)
    repo.checkout_master()
    load_config(repo_config_file)
    logger.info("Preloading RepoData")
    repodata = RepoData()
    repodata.set_cache(repodata_cache_file)
    repodata.df  # pylint: disable=pointless-statement
    logger.info("Preloading RepoData (done)")

    # Collect recipe names
    recipe_dirs = os.listdir(recipe_basedir)
    if 'BIOCONDA_FILTER_RECIPES' in os.environ:
        limiter = os.environ['BIOCONDA_FILTER_RECIPES']
        try:
            recipe_dirs = recipe_dirs[:int(limiter)]
        except ValueError:
            match = re.compile(limiter)
            recipe_dirs = [recipe for recipe in recipe_dirs
                           if match.search(recipe)]

    # Set up renderer preparing recipe readme.rst files
    recipe_base_url = "{base}/tree/master/{recipes}/".format(
        base=app.config.bioconda_repo_url.rstrip(".git"),
        recipes=app.config.bioconda_recipes_path
    )
    renderer = Renderer(app, {'gh_recipes': recipe_base_url})

    recipes: List[str] = []

    if parallel_available and len(recipe_dirs) > 5:
        nproc = app.parallel
    else:
        nproc = 1

    if nproc == 1:
        for folder in status_iterator(
                recipe_dirs,
                'Generating package READMEs...',
                "purple", len(recipe_dirs), app.verbosity):
            if not op.isdir(op.join(recipe_basedir, folder)):
                logger.error("Item '%s' in recipes folder is not a folder",
                             folder)
                continue
            recipes.extend(generate_readme(recipe_basedir, output_dir, folder, repodata, renderer))
    else:
        tasks = ParallelTasks(nproc)
        chunks = make_chunks(recipe_dirs, nproc)

        def process_chunk(chunk):
            _recipes: List[Dict[str, Any]] = []
            for folder in chunk:
                if not op.isdir(op.join(recipe_basedir, folder)):
                    logger.error("Item '%s' in recipes folder is not a folder",
                                 folder)
                    continue
                _recipes.extend(generate_readme(recipe_basedir, output_dir, folder, repodata, renderer))
            return _recipes

        def merge_chunk(_chunk, res):
            recipes.extend(res)

        for chunk in status_iterator(
                chunks,
                'Generating package READMEs with {} threads...'.format(nproc),
                "purple", len(chunks), app.verbosity):
            tasks.add_task(process_chunk, chunk, merge_chunk)
        logger.info("waiting for workers...")
        tasks.join()

    files_wanted = set(recipes)
    for root, dirs, files in os.walk(output_dir, topdown=False):
        for fname in files:
            path = op.join(root, fname)
            if path not in files_wanted:
                os.unlink(path)
        for dname in dirs:
            try:
                os.rmdir(op.join(root, dname))
            except OSError:
                pass
Exemplo n.º 22
0
def generate_recipes(app):
    """Generates recipe RST files

    - Checks out repository
    - Prepares `RepoData`
    - Selects recipes (if `BIOCONDA_FILTER_RECIPES` in environment)
    - Dispatches calls to `generate_readme` for each recipe
    - Removes old RST files
    """
    source_dir = app.env.srcdir
    doctree_dir = app.env.doctreedir  # .../build/doctrees
    repo_dir = op.join(op.dirname(app.env.srcdir), "_bioconda_recipes")
    recipe_basedir = op.join(repo_dir, app.config.bioconda_recipes_path)
    repodata_cache_file = op.join(doctree_dir, 'RepoDataCache.pkl')
    repo_config_file = os.path.join(repo_dir, app.config.bioconda_config_file)
    output_dir = op.join(source_dir, 'recipes')

    # Initialize Repo and point globals at the right place
    repo = BiocondaRepo(folder=repo_dir, home=app.config.bioconda_repo_url)
    repo.checkout_master()
    load_config(repo_config_file)
    logger.info("Preloading RepoData")
    repodata = RepoData()
    repodata.set_cache(repodata_cache_file)
    repodata.df  # pylint: disable=pointless-statement
    logger.info("Preloading RepoData (done)")

    # Collect recipe names
    recipe_dirs = os.listdir(recipe_basedir)
    if 'BIOCONDA_FILTER_RECIPES' in os.environ:
        limiter = os.environ['BIOCONDA_FILTER_RECIPES']
        try:
            recipe_dirs = recipe_dirs[:int(limiter)]
        except ValueError:
            match = re.compile(limiter)
            recipe_dirs = [
                recipe for recipe in recipe_dirs if match.search(recipe)
            ]

    # Set up renderer preparing recipe readme.rst files
    recipe_base_url = "{base}/tree/master/{recipes}/".format(
        base=app.config.bioconda_repo_url.rstrip(".git"),
        recipes=app.config.bioconda_recipes_path)
    renderer = Renderer(app, {'gh_recipes': recipe_base_url})

    recipes: List[str] = []

    if parallel_available and len(recipe_dirs) > 5:
        nproc = app.parallel
    else:
        nproc = 1

    if nproc == 1:
        for folder in status_iterator(recipe_dirs,
                                      'Generating package READMEs...',
                                      "purple", len(recipe_dirs),
                                      app.verbosity):
            if not op.isdir(op.join(recipe_basedir, folder)):
                logger.error("Item '%s' in recipes folder is not a folder",
                             folder)
                continue
            recipes.extend(
                generate_readme(recipe_basedir, output_dir, folder, repodata,
                                renderer))
    else:
        tasks = ParallelTasks(nproc)
        chunks = make_chunks(recipe_dirs, nproc)

        def process_chunk(chunk):
            _recipes: List[Dict[str, Any]] = []
            for folder in chunk:
                if not op.isdir(op.join(recipe_basedir, folder)):
                    logger.error("Item '%s' in recipes folder is not a folder",
                                 folder)
                    continue
                _recipes.extend(
                    generate_readme(recipe_basedir, output_dir, folder,
                                    repodata, renderer))
            return _recipes

        def merge_chunk(_chunk, res):
            recipes.extend(res)

        for chunk in status_iterator(
                chunks,
                'Generating package READMEs with {} threads...'.format(nproc),
                "purple", len(chunks), app.verbosity):
            tasks.add_task(process_chunk, chunk, merge_chunk)
        logger.info("waiting for workers...")
        tasks.join()

    files_wanted = set(recipes)
    for root, dirs, files in os.walk(output_dir, topdown=False):
        for fname in files:
            path = op.join(root, fname)
            if path not in files_wanted:
                os.unlink(path)
        for dname in dirs:
            try:
                os.rmdir(op.join(root, dname))
            except OSError:
                pass
Exemplo n.º 23
0
def generate_recipes(app):
    """
    Go through every folder in the `ggd-recipes/recipes` dir,
    have a README.rst file generated and generate a recipes.rst from
    the collected data.
    """
    renderer = Renderer(app)
    load_config(os.path.join(os.path.dirname(RECIPE_DIR), "config.yaml"))
    repodata = RepoData()
    # Add ggd channels to repodata object
    #repodata.channels = ['ggd-genomics', 'conda-forge', 'bioconda', 'defaults']
    recipes = []
    ## Get each folder that contains a meat.yaml file
    recipe_dirs = []
    for root, dirs, files in os.walk(RECIPE_DIR):
        if "meta.yaml" in files:
            recipe_dirs.append(root)


    if parallel_available and len(recipe_dirs) > 5:
        nproc = app.parallel
    else:
        nproc = 1

    if nproc == 1:
        for folder in status_iterator(
                recipe_dirs,
                'Generating package READMEs...',
                "purple", len(recipe_dirs), app.verbosity):
            recipes.extend(generate_readme(folder, repodata, renderer))
    else:
        tasks = ParallelTasks(nproc)
        chunks = make_chunks(recipe_dirs, nproc)

        def process_chunk(chunk):
            _recipes = []
            for folder in chunk:
                _recipes.extend(generate_readme(folder, repodata, renderer))
            return _recipes

        def merge_chunk(chunk, res):
            recipes.extend(res)


        for chunk in status_iterator(
                chunks,
                'Generating package READMEs with {} threads...'.format(nproc),
                "purple", len(chunks), app.verbosity):
            tasks.add_task(process_chunk, chunk, merge_chunk)
        logger.info("waiting for workers...")
        tasks.join()

    updated = renderer.render_to_file("source/recipes.rst", "recipes.rst_t", {
        'recipes': recipes,
        # order of columns in the table; must be keys in template_options
        'keys': ['Package', 'Version', 'Linux', 'OSX', 'NOARCH'],
		'noarch_symbol': '<i class="fa fa-desktop"></i>',
		'linux_symbol': '<i class="fa fa-linux"></i>', 
		'osx_symbol': '<i class="fa fa-apple"></i>',
		'dot_symbol': '<i class="fa fa-dot-circle-o"></i>' 
    })
    if updated:
        logger.info("Updated source/recipes.rst")
Exemplo n.º 24
0
def generate_recipes(app):
    """
    Go through every folder in the `ggd-recipes/recipes` dir,
    have a README.rst file generated and generate a recipes.rst from
    the collected data.
    """
    renderer = Renderer(app)

    recipes = []

    ## Get each folder that contains a meat.yaml file
    recipe_dirs = []

    ## Get Species, Genome Builds, and
    species_build_dict = get_species(full_dict = True)
    ggd_channels = set(get_ggd_channels())

    ## Get recipe info
    curr_channel = ""
    curr_species = ""
    curr_genome_build = ""

    ## Get all sub_path combinations for channel, species, and build
    sub_paths = {op.join(channel,species,build): {"channel":channel, "species": species, "build": build} for channel in ggd_channels for species, builds in species_build_dict.items() for build in builds}

    for sub_path, info_dict in sub_paths.items():
        for root, dirs, files in os.walk(op.join(RECIPE_DIR,sub_path)):
            ## Get all paths to the yaml file
            if "meta.yaml" in files:
                info = copy.deepcopy(info_dict)
                info["meta_path"] = root
                recipe_dirs.append(info)

    if parallel_available and len(recipe_dirs) > 5:
        nproc = app.parallel
    else:
        nproc = 1

    if nproc == 1:
        for recipe_dict in status_iterator(
                recipe_dirs,
                'Generating package READMEs...',
                "purple", len(recipe_dirs), app.verbosity):
            recipes.append(generate_readme(recipe_dict, renderer))
    else:
        tasks = ParallelTasks(nproc)
        chunks = make_chunks(recipe_dirs, nproc)

        def process_chunk(chunk):
            _recipes = []
            for recipe_dict in chunk:
                _recipes.extend(generate_readme(recipe_dict, renderer))
            return _recipes

        def merge_chunk(chunk, res):
            recipes.extend(res)

        for chunk in status_iterator(
                chunks,
                'Generating package READMEs with {} threads...'.format(nproc),
                "purple", len(chunks), app.verbosity):
            tasks.add_task(process_chunk, chunk, merge_chunk)
        logger.info("waiting for workers...")
        tasks.join()

    
    ## Create json file based on genome, species, and genome build
    pkg_json = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(str))))
    for recipe_d in recipes:

        ## Get recipe info
        channel = recipe_d["ggd_channel"]
        species = recipe_d["species"]
        genome_build = recipe_d["genome_build"]
        name = recipe_d["name"]
        href = recipe_d["Package"]

        ## Add to dict
        pkg_json[channel][species][genome_build][name] = href
   
    ## Create a json file with recipe info in it 
    with open("build/html/recipes.json", "w") as j:
        j.write("recipe_data = ")
        json.dump(pkg_json, j)

    ## Update recipe.rst
    updated = renderer.render_to_file("source/recipes.rst", "recipes.rst_t", {})
    if updated:
        logger.info("Updated source/recipes.rst")