def _read_parallel(self, docnames: List[str], nproc: int) -> None:
chunks = make_chunks(docnames, nproc)
# create a status_iterator to step progressbar after reading a document
# (see: ``merge()`` function)
progress = status_iterator(chunks, __('reading sources... '), "purple",
len(chunks), self.app.verbosity)
# clear all outdated docs at once
for docname in docnames:
self.events.emit('env-purge-doc', self.env, docname)
self.env.clear_doc(docname)
def read_process(docs: List[str]) -> bytes:
self.env.app = self.app
for docname in docs:
self.read_doc(docname)
# allow pickling self to send it back
return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)
def merge(docs: List[str], otherenv: bytes) -> None:
env = pickle.loads(otherenv)
self.env.merge_info_from(docs, env, self.app)
next(progress)
tasks = ParallelTasks(nproc)
for chunk in chunks:
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
tasks.join()
logger.info('')
def _write_parallel(self, docnames, nproc):
# type: (Sequence[unicode], int) -> None
def write_process(docs):
# type: (List[Tuple[unicode, nodes.Node]]) -> None
self.app.phase = BuildPhase.WRITING
for docname, doctree in docs:
self.write_doc(docname, doctree)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:]
self.app.phase = BuildPhase.RESOLVING
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.app.phase = BuildPhase.WRITING
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
self.app.phase = BuildPhase.RESOLVING
for chunk in status_iterator(chunks,
__('writing output... '), "darkgreen",
len(chunks), self.app.verbosity):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg)
# make sure all threads have finished
logger.info(bold(__('waiting for workers...')))
tasks.join()
def _write_parallel(self, docnames, nproc):
# type: (Iterable[unicode], int) -> None
def write_process(docs):
# type: (List[Tuple[unicode, nodes.Node]]) -> None
for docname, doctree in docs:
self.write_doc(docname, doctree)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:] # type: ignore
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in self.app.status_iterator(chunks, 'writing output... ',
darkgreen, len(chunks)):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg)
# make sure all threads have finished
logger.info(bold('waiting for workers...'))
tasks.join()
def _read_parallel(self, docnames, nproc):
# type: (List[unicode], int) -> None
# clear all outdated docs at once
for docname in docnames:
self.app.emit('env-purge-doc', self.env, docname)
self.env.clear_doc(docname)
def read_process(docs):
# type: (List[unicode]) -> bytes
self.env.app = self.app
for docname in docs:
self.read_doc(docname)
# allow pickling self to send it back
return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)
def merge(docs, otherenv):
# type: (List[unicode], bytes) -> None
env = pickle.loads(otherenv)
self.env.merge_info_from(docs, env, self.app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in status_iterator(chunks, 'reading sources... ', "purple",
len(chunks), self.app.verbosity):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
logger.info(bold('waiting for workers...'))
tasks.join()
def _read_parallel(self, docnames: List[str], nproc: int) -> None:
# clear all outdated docs at once
for docname in docnames:
self.events.emit('env-purge-doc', self.env, docname)
self.env.clear_doc(docname)
def read_process(docs: List[str]) -> bytes:
self.env.app = self.app
for docname in docs:
self.read_doc(docname)
# allow pickling self to send it back
return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)
def merge(docs: List[str], otherenv: bytes) -> None:
env = pickle.loads(otherenv)
self.env.merge_info_from(docs, env, self.app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in status_iterator(chunks, __('reading sources... '), "purple",
len(chunks), self.app.verbosity):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
logger.info(bold(__('waiting for workers...')))
tasks.join()
def _write_parallel(self, docnames, nproc):
# type: (Sequence[unicode], int) -> None
def write_process(docs):
# type: (List[Tuple[unicode, nodes.Node]]) -> None
self.app.phase = BuildPhase.WRITING
for docname, doctree in docs:
self.write_doc(docname, doctree)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:]
self.app.phase = BuildPhase.RESOLVING
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.app.phase = BuildPhase.WRITING
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
self.app.phase = BuildPhase.RESOLVING
for chunk in status_iterator(chunks, __('writing output... '), "darkgreen",
len(chunks), self.app.verbosity):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg)
# make sure all threads have finished
logger.info(bold(__('waiting for workers...')))
tasks.join()
def _write_parallel(self, docnames, warnings, nproc):
def write_process(docs):
local_warnings = []
self.env.set_warnfunc(lambda *args: local_warnings.append(args))
for docname, doctree in docs:
self.write_doc(docname, doctree)
return local_warnings
def add_warnings(docs, wlist):
warnings.extend(wlist)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:]
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in self.app.status_iterator(chunks, 'writing output... ',
darkgreen, len(chunks)):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg, add_warnings)
# make sure all threads have finished
self.info(bold('waiting for workers...'))
tasks.join()
for warning in warnings:
self.warn(*warning)
def _read_parallel(self, docnames, app, nproc):
# type: (List[unicode], Sphinx, int) -> None
# clear all outdated docs at once
for docname in docnames:
app.emit('env-purge-doc', self, docname)
self.clear_doc(docname)
def read_process(docs):
# type: (List[unicode]) -> BuildEnvironment
self.app = app
for docname in docs:
self.read_doc(docname, app)
# allow pickling self to send it back
del self.app
del self.domains
del self.config.values
del self.config
return self
def merge(docs, otherenv):
# type: (List[unicode], BuildEnvironment) -> None
self.merge_info_from(docs, otherenv, app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in status_iterator(chunks, 'reading sources... ', "purple",
len(chunks), self.app.verbosity):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
logger.info(bold('waiting for workers...'))
tasks.join()
def _read_parallel(self, docnames, nproc):
# type: (List[unicode], int) -> None
# clear all outdated docs at once
for docname in docnames:
self.app.emit('env-purge-doc', self.env, docname)
self.env.clear_doc(docname)
def read_process(docs):
# type: (List[unicode]) -> unicode
self.env.app = self.app
for docname in docs:
self.env.read_doc(docname, self.app)
# allow pickling self to send it back
return BuildEnvironment.dumps(self.env)
def merge(docs, otherenv):
# type: (List[unicode], unicode) -> None
env = BuildEnvironment.loads(otherenv)
self.env.merge_info_from(docs, env, self.app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in status_iterator(chunks, 'reading sources... ', "purple",
len(chunks), self.app.verbosity):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
logger.info(bold('waiting for workers...'))
tasks.join()
def _write_parallel(self, docnames, warnings, nproc):
def write_process(docs):
local_warnings = []
self.env.set_warnfunc(lambda *args: local_warnings.append(args))
for docname, doctree in docs:
self.write_doc(docname, doctree)
return local_warnings
def add_warnings(docs, wlist):
warnings.extend(wlist)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:]
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in self.app.status_iterator(
chunks, 'writing output... ', darkgreen, len(chunks)):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg, add_warnings)
# make sure all threads have finished
self.info(bold('waiting for workers...'))
tasks.join()
for warning in warnings:
self.warn(*warning)
def generate_recipes(app):
"""
Go through every folder in the `bioconda-recipes/recipes` dir,
have a README.rst file generated and generate a recipes.rst from
the collected data.
"""
renderer = Renderer(app)
load_config(os.path.join(os.path.dirname(RECIPE_DIR), "config.yml"))
repodata = RepoData()
repodata.set_cache(op.join(app.env.doctreedir, 'RepoDataCache.csv'))
# force loading repodata to avoid duplicate loads from threads
repodata.df # pylint: disable=pointless-statement
recipes: List[Dict[str, Any]] = []
recipe_dirs = os.listdir(RECIPE_DIR)
if parallel_available and len(recipe_dirs) > 5:
nproc = app.parallel
else:
nproc = 1
if nproc == 1:
for folder in status_iterator(recipe_dirs,
'Generating package READMEs...',
"purple", len(recipe_dirs),
app.verbosity):
if not op.isdir(op.join(RECIPE_DIR, folder)):
logger.error("Item '%s' in recipes folder is not a folder",
folder)
continue
recipes.extend(generate_readme(folder, repodata, renderer))
else:
tasks = ParallelTasks(nproc)
chunks = make_chunks(recipe_dirs, nproc)
def process_chunk(chunk):
_recipes: List[Dict[str, Any]] = []
for folder in chunk:
if not op.isdir(op.join(RECIPE_DIR, folder)):
logger.error("Item '%s' in recipes folder is not a folder",
folder)
continue
_recipes.extend(generate_readme(folder, repodata, renderer))
return _recipes
def merge_chunk(_chunk, res):
recipes.extend(res)
for chunk in status_iterator(
chunks,
'Generating package READMEs with {} threads...'.format(nproc),
"purple", len(chunks), app.verbosity):
tasks.add_task(process_chunk, chunk, merge_chunk)
logger.info("waiting for workers...")
tasks.join()
def generate_recipes(app):
"""
Go through every folder in the `bioconda-recipes/recipes` dir,
have a README.rst file generated and generate a recipes.rst from
the collected data.
"""
renderer = Renderer(app)
repodata = RepoData()
recipes = []
recipe_dirs = os.listdir(RECIPE_DIR)
if parallel_available and len(recipe_dirs) > 5:
nproc = app.parallel
else:
nproc = 1
if nproc == 1:
for folder in status_iterator(recipe_dirs,
'Generating package READMEs...',
"purple", len(recipe_dirs),
app.verbosity):
recipes.extend(generate_readme(folder, repodata, renderer))
else:
tasks = ParallelTasks(nproc)
chunks = make_chunks(recipe_dirs, nproc)
def process_chunk(chunk):
_recipes = []
for folder in chunk:
_recipes.extend(generate_readme(folder, repodata, renderer))
return _recipes
def merge_chunk(chunk, res):
recipes.extend(res)
for chunk in status_iterator(
chunks,
'Generating package READMEs with {} threads...'.format(nproc),
"purple", len(chunks), app.verbosity):
tasks.add_task(process_chunk, chunk, merge_chunk)
logger.info("waiting for workers...")
tasks.join()
updated = renderer.render_to_file(
"source/recipes.rst",
"recipes.rst_t",
{
'recipes': recipes,
# order of columns in the table; must be keys in template_options
'keys': ['Package', 'Version', 'License', 'Linux', 'OSX']
})
if updated:
logger.info("Updated source/recipes.rst")
def test_logging_in_ParallelTasks(app, status, warning):
logging.setup(app, status, warning)
logger = logging.getLogger(__name__)
def child_process():
logger.info('message1')
logger.warning('message2', location='index')
tasks = ParallelTasks(1)
tasks.add_task(child_process)
tasks.join()
assert 'message1' in status.getvalue()
assert 'index.txt: WARNING: message2' in warning.getvalue()
def test_logging_in_ParallelTasks(app, status, warning):
logging.setup(app, status, warning)
logger = logging.getLogger(__name__)
def child_process():
logger.info('message1')
logger.warning('message2', location='index')
tasks = ParallelTasks(1)
tasks.add_task(child_process)
tasks.join()
assert 'message1' in status.getvalue()
assert 'index.txt: WARNING: message2' in warning.getvalue()
def _write_parallel(self, docnames, warnings, nproc):
# type: (Iterable[unicode], List[Tuple[Tuple, Dict]], int) -> None
def write_process(docs):
# type: (List[Tuple[unicode, nodes.Node]]) -> List[Tuple[Tuple, Dict]]
local_warnings = []
def warnfunc(*args, **kwargs):
local_warnings.append((args, kwargs))
self.env.set_warnfunc(warnfunc)
for docname, doctree in docs:
self.write_doc(docname, doctree)
return local_warnings
def add_warnings(docs, wlist):
warnings.extend(wlist)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:] # type: ignore
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in self.app.status_iterator(chunks, 'writing output... ',
darkgreen, len(chunks)):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg, add_warnings)
# make sure all threads have finished
self.info(bold('waiting for workers...'))
tasks.join()
for warning, kwargs in warnings:
self.warn(*warning, **kwargs)
def _write_parallel(self, docnames, warnings, nproc):
# type: (Iterable[unicode], List[Tuple[Tuple, Dict]], int) -> None
def write_process(docs):
# type: (List[Tuple[unicode, nodes.Node]]) -> List[Tuple[Tuple, Dict]]
local_warnings = []
def warnfunc(*args, **kwargs):
local_warnings.append((args, kwargs))
self.env.set_warnfunc(warnfunc)
for docname, doctree in docs:
self.write_doc(docname, doctree)
return local_warnings
def add_warnings(docs, wlist):
warnings.extend(wlist)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:] # type: ignore
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
for chunk in self.app.status_iterator(
chunks, 'writing output... ', darkgreen, len(chunks)):
arg = []
for i, docname in enumerate(chunk):
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg, add_warnings)
# make sure all threads have finished
self.info(bold('waiting for workers...'))
tasks.join()
for warning, kwargs in warnings:
self.warn(*warning, **kwargs)
def _write_parallel(self, docnames: Sequence[str], nproc: int) -> None:
def write_process(docs: List[Tuple[str, nodes.document]]) -> None:
self.app.phase = BuildPhase.WRITING
for docname, doctree in docs:
self.write_doc(docname, doctree)
# warm up caches/compile templates using the first document
firstname, docnames = docnames[0], docnames[1:]
self.app.phase = BuildPhase.RESOLVING
doctree = self.env.get_and_resolve_doctree(firstname, self)
self.app.phase = BuildPhase.WRITING
self.write_doc_serialized(firstname, doctree)
self.write_doc(firstname, doctree)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
# create a status_iterator to step progressbar after writing a document
# (see: ``on_chunk_done()`` function)
progress = status_iterator(chunks,
__('writing output... '), "darkgreen",
len(chunks), self.app.verbosity)
def on_chunk_done(args: List[Tuple[str, NoneType]],
result: NoneType) -> None:
next(progress)
self.app.phase = BuildPhase.RESOLVING
for chunk in chunks:
arg = []
for docname in chunk:
doctree = self.env.get_and_resolve_doctree(docname, self)
self.write_doc_serialized(docname, doctree)
arg.append((docname, doctree))
tasks.add_task(write_process, arg, on_chunk_done)
# make sure all threads have finished
tasks.join()
logger.info('')
def _read_parallel(self, docnames, app, nproc):
# clear all outdated docs at once
for docname in docnames:
app.emit('env-purge-doc', self, docname)
self.clear_doc(docname)
def read_process(docs):
self.app = app
self.warnings = []
self.set_warnfunc(lambda *args, **kwargs: self.warnings.append((args, kwargs)))
for docname in docs:
self.read_doc(docname, app)
# allow pickling self to send it back
self.set_warnfunc(None)
del self.app
del self.domains
del self.config.values
del self.config
return self
def merge(docs, otherenv):
warnings.extend(otherenv.warnings)
self.merge_info_from(docs, otherenv, app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
warnings = []
for chunk in app.status_iterator(
chunks, 'reading sources... ', purple, len(chunks)):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
app.info(bold('waiting for workers...'))
tasks.join()
for warning, kwargs in warnings:
self._warnfunc(*warning, **kwargs)
def _read_parallel(self, docnames, app, nproc):
# clear all outdated docs at once
for docname in docnames:
app.emit('env-purge-doc', self, docname)
self.clear_doc(docname)
def read_process(docs):
self.app = app
self.warnings = []
self.set_warnfunc(lambda *args, **kwargs: self.warnings.append((args, kwargs)))
for docname in docs:
self.read_doc(docname, app)
# allow pickling self to send it back
self.set_warnfunc(None)
del self.app
del self.domains
del self.config.values
del self.config
return self
def merge(docs, otherenv):
warnings.extend(otherenv.warnings)
self.merge_info_from(docs, otherenv, app)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docnames, nproc)
warnings = []
for chunk in app.status_iterator(
chunks, 'reading sources... ', purple, len(chunks)):
tasks.add_task(read_process, chunk, merge)
# make sure all threads have finished
app.info(bold('waiting for workers...'))
tasks.join()
for warning, kwargs in warnings:
self._warnfunc(*warning, **kwargs)
def _write_parallel(self, docCfgList, warnings, nproc):
def write_process(docs):
local_warnings = []
def warnfunc(*args, **kwargs):
local_warnings.append((args, kwargs))
self.env.set_warnfunc(warnfunc)
for targetname, doctree in docs:
self.write_doc(targetname, doctree)
return local_warnings
def add_warnings(_docs, wlist):
warnings.extend(wlist)
# warm up caches/compile templates using the first document
docCfg, docCfgList = docCfgList[0], docCfgList[1:]
self.write_doc_serialized(docCfg)
self.write_doc(docCfg)
tasks = ParallelTasks(nproc)
chunks = make_chunks(docCfg, nproc)
for chunk in self.app.status_iterator(chunks, 'writing output... ',
darkgreen, len(chunks)):
arg = []
for docCfg in chunk:
self.write_doc_serialized(docCfg)
arg.append((docCfg, ))
tasks.add_task(write_process, arg, add_warnings)
# make sure all threads have finished
self.info(bold('waiting for workers...'))
tasks.join()
for warning, kwargs in warnings:
self.warn(*warning, **kwargs)
def generate_recipes(app):
"""Generates recipe RST files
- Checks out repository
- Prepares `RepoData`
- Selects recipes (if BIOCONDA_FILTER_RECIPES in environment)
- Dispatches calls to `generate_readme` for each recipe
- Removes old RST files
"""
source_dir = app.env.srcdir
doctree_dir = app.env.doctreedir # .../build/doctrees
repo_dir = op.join(op.dirname(app.env.srcdir), "_bioconda_recipes")
recipe_basedir = op.join(repo_dir, app.config.bioconda_recipes_path)
repodata_cache_file = op.join(doctree_dir, 'RepoDataCache.pkl')
repo_config_file = os.path.join(repo_dir, app.config.bioconda_config_file)
output_dir = op.join(source_dir, 'recipes')
# Initialize Repo and point globals at the right place
repo = BiocondaRepo(folder=repo_dir, home=app.config.bioconda_repo_url)
repo.checkout_master()
load_config(repo_config_file)
logger.info("Preloading RepoData")
repodata = RepoData()
repodata.set_cache(repodata_cache_file)
repodata.df # pylint: disable=pointless-statement
logger.info("Preloading RepoData (done)")
# Collect recipe names
recipe_dirs = os.listdir(recipe_basedir)
if 'BIOCONDA_FILTER_RECIPES' in os.environ:
limiter = os.environ['BIOCONDA_FILTER_RECIPES']
try:
recipe_dirs = recipe_dirs[:int(limiter)]
except ValueError:
match = re.compile(limiter)
recipe_dirs = [recipe for recipe in recipe_dirs
if match.search(recipe)]
# Set up renderer preparing recipe readme.rst files
recipe_base_url = "{base}/tree/master/{recipes}/".format(
base=app.config.bioconda_repo_url.rstrip(".git"),
recipes=app.config.bioconda_recipes_path
)
renderer = Renderer(app, {'gh_recipes': recipe_base_url})
recipes: List[str] = []
if parallel_available and len(recipe_dirs) > 5:
nproc = app.parallel
else:
nproc = 1
if nproc == 1:
for folder in status_iterator(
recipe_dirs,
'Generating package READMEs...',
"purple", len(recipe_dirs), app.verbosity):
if not op.isdir(op.join(recipe_basedir, folder)):
logger.error("Item '%s' in recipes folder is not a folder",
folder)
continue
recipes.extend(generate_readme(recipe_basedir, output_dir, folder, repodata, renderer))
else:
tasks = ParallelTasks(nproc)
chunks = make_chunks(recipe_dirs, nproc)
def process_chunk(chunk):
_recipes: List[Dict[str, Any]] = []
for folder in chunk:
if not op.isdir(op.join(recipe_basedir, folder)):
logger.error("Item '%s' in recipes folder is not a folder",
folder)
continue
_recipes.extend(generate_readme(recipe_basedir, output_dir, folder, repodata, renderer))
return _recipes
def merge_chunk(_chunk, res):
recipes.extend(res)
for chunk in status_iterator(
chunks,
'Generating package READMEs with {} threads...'.format(nproc),
"purple", len(chunks), app.verbosity):
tasks.add_task(process_chunk, chunk, merge_chunk)
logger.info("waiting for workers...")
tasks.join()
files_wanted = set(recipes)
for root, dirs, files in os.walk(output_dir, topdown=False):
for fname in files:
path = op.join(root, fname)
if path not in files_wanted:
os.unlink(path)
for dname in dirs:
try:
os.rmdir(op.join(root, dname))
except OSError:
pass
def generate_recipes(app):
"""Generates recipe RST files
- Checks out repository
- Prepares `RepoData`
- Selects recipes (if `BIOCONDA_FILTER_RECIPES` in environment)
- Dispatches calls to `generate_readme` for each recipe
- Removes old RST files
"""
source_dir = app.env.srcdir
doctree_dir = app.env.doctreedir # .../build/doctrees
repo_dir = op.join(op.dirname(app.env.srcdir), "_bioconda_recipes")
recipe_basedir = op.join(repo_dir, app.config.bioconda_recipes_path)
repodata_cache_file = op.join(doctree_dir, 'RepoDataCache.pkl')
repo_config_file = os.path.join(repo_dir, app.config.bioconda_config_file)
output_dir = op.join(source_dir, 'recipes')
# Initialize Repo and point globals at the right place
repo = BiocondaRepo(folder=repo_dir, home=app.config.bioconda_repo_url)
repo.checkout_master()
load_config(repo_config_file)
logger.info("Preloading RepoData")
repodata = RepoData()
repodata.set_cache(repodata_cache_file)
repodata.df # pylint: disable=pointless-statement
logger.info("Preloading RepoData (done)")
# Collect recipe names
recipe_dirs = os.listdir(recipe_basedir)
if 'BIOCONDA_FILTER_RECIPES' in os.environ:
limiter = os.environ['BIOCONDA_FILTER_RECIPES']
try:
recipe_dirs = recipe_dirs[:int(limiter)]
except ValueError:
match = re.compile(limiter)
recipe_dirs = [
recipe for recipe in recipe_dirs if match.search(recipe)
]
# Set up renderer preparing recipe readme.rst files
recipe_base_url = "{base}/tree/master/{recipes}/".format(
base=app.config.bioconda_repo_url.rstrip(".git"),
recipes=app.config.bioconda_recipes_path)
renderer = Renderer(app, {'gh_recipes': recipe_base_url})
recipes: List[str] = []
if parallel_available and len(recipe_dirs) > 5:
nproc = app.parallel
else:
nproc = 1
if nproc == 1:
for folder in status_iterator(recipe_dirs,
'Generating package READMEs...',
"purple", len(recipe_dirs),
app.verbosity):
if not op.isdir(op.join(recipe_basedir, folder)):
logger.error("Item '%s' in recipes folder is not a folder",
folder)
continue
recipes.extend(
generate_readme(recipe_basedir, output_dir, folder, repodata,
renderer))
else:
tasks = ParallelTasks(nproc)
chunks = make_chunks(recipe_dirs, nproc)
def process_chunk(chunk):
_recipes: List[Dict[str, Any]] = []
for folder in chunk:
if not op.isdir(op.join(recipe_basedir, folder)):
logger.error("Item '%s' in recipes folder is not a folder",
folder)
continue
_recipes.extend(
generate_readme(recipe_basedir, output_dir, folder,
repodata, renderer))
return _recipes
def merge_chunk(_chunk, res):
recipes.extend(res)
for chunk in status_iterator(
chunks,
'Generating package READMEs with {} threads...'.format(nproc),
"purple", len(chunks), app.verbosity):
tasks.add_task(process_chunk, chunk, merge_chunk)
logger.info("waiting for workers...")
tasks.join()
files_wanted = set(recipes)
for root, dirs, files in os.walk(output_dir, topdown=False):
for fname in files:
path = op.join(root, fname)
if path not in files_wanted:
os.unlink(path)
for dname in dirs:
try:
os.rmdir(op.join(root, dname))
except OSError:
pass
def generate_recipes(app):
"""
Go through every folder in the `ggd-recipes/recipes` dir,
have a README.rst file generated and generate a recipes.rst from
the collected data.
"""
renderer = Renderer(app)
load_config(os.path.join(os.path.dirname(RECIPE_DIR), "config.yaml"))
repodata = RepoData()
# Add ggd channels to repodata object
#repodata.channels = ['ggd-genomics', 'conda-forge', 'bioconda', 'defaults']
recipes = []
## Get each folder that contains a meat.yaml file
recipe_dirs = []
for root, dirs, files in os.walk(RECIPE_DIR):
if "meta.yaml" in files:
recipe_dirs.append(root)
if parallel_available and len(recipe_dirs) > 5:
nproc = app.parallel
else:
nproc = 1
if nproc == 1:
for folder in status_iterator(
recipe_dirs,
'Generating package READMEs...',
"purple", len(recipe_dirs), app.verbosity):
recipes.extend(generate_readme(folder, repodata, renderer))
else:
tasks = ParallelTasks(nproc)
chunks = make_chunks(recipe_dirs, nproc)
def process_chunk(chunk):
_recipes = []
for folder in chunk:
_recipes.extend(generate_readme(folder, repodata, renderer))
return _recipes
def merge_chunk(chunk, res):
recipes.extend(res)
for chunk in status_iterator(
chunks,
'Generating package READMEs with {} threads...'.format(nproc),
"purple", len(chunks), app.verbosity):
tasks.add_task(process_chunk, chunk, merge_chunk)
logger.info("waiting for workers...")
tasks.join()
updated = renderer.render_to_file("source/recipes.rst", "recipes.rst_t", {
'recipes': recipes,
# order of columns in the table; must be keys in template_options
'keys': ['Package', 'Version', 'Linux', 'OSX', 'NOARCH'],
'noarch_symbol': '<i class="fa fa-desktop"></i>',
'linux_symbol': '<i class="fa fa-linux"></i>',
'osx_symbol': '<i class="fa fa-apple"></i>',
'dot_symbol': '<i class="fa fa-dot-circle-o"></i>'
})
if updated:
logger.info("Updated source/recipes.rst")
def generate_recipes(app):
"""
Go through every folder in the `ggd-recipes/recipes` dir,
have a README.rst file generated and generate a recipes.rst from
the collected data.
"""
renderer = Renderer(app)
recipes = []
## Get each folder that contains a meat.yaml file
recipe_dirs = []
## Get Species, Genome Builds, and
species_build_dict = get_species(full_dict = True)
ggd_channels = set(get_ggd_channels())
## Get recipe info
curr_channel = ""
curr_species = ""
curr_genome_build = ""
## Get all sub_path combinations for channel, species, and build
sub_paths = {op.join(channel,species,build): {"channel":channel, "species": species, "build": build} for channel in ggd_channels for species, builds in species_build_dict.items() for build in builds}
for sub_path, info_dict in sub_paths.items():
for root, dirs, files in os.walk(op.join(RECIPE_DIR,sub_path)):
## Get all paths to the yaml file
if "meta.yaml" in files:
info = copy.deepcopy(info_dict)
info["meta_path"] = root
recipe_dirs.append(info)
if parallel_available and len(recipe_dirs) > 5:
nproc = app.parallel
else:
nproc = 1
if nproc == 1:
for recipe_dict in status_iterator(
recipe_dirs,
'Generating package READMEs...',
"purple", len(recipe_dirs), app.verbosity):
recipes.append(generate_readme(recipe_dict, renderer))
else:
tasks = ParallelTasks(nproc)
chunks = make_chunks(recipe_dirs, nproc)
def process_chunk(chunk):
_recipes = []
for recipe_dict in chunk:
_recipes.extend(generate_readme(recipe_dict, renderer))
return _recipes
def merge_chunk(chunk, res):
recipes.extend(res)
for chunk in status_iterator(
chunks,
'Generating package READMEs with {} threads...'.format(nproc),
"purple", len(chunks), app.verbosity):
tasks.add_task(process_chunk, chunk, merge_chunk)
logger.info("waiting for workers...")
tasks.join()
## Create json file based on genome, species, and genome build
pkg_json = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(str))))
for recipe_d in recipes:
## Get recipe info
channel = recipe_d["ggd_channel"]
species = recipe_d["species"]
genome_build = recipe_d["genome_build"]
name = recipe_d["name"]
href = recipe_d["Package"]
## Add to dict
pkg_json[channel][species][genome_build][name] = href
## Create a json file with recipe info in it
with open("build/html/recipes.json", "w") as j:
j.write("recipe_data = ")
json.dump(pkg_json, j)
## Update recipe.rst
updated = renderer.render_to_file("source/recipes.rst", "recipes.rst_t", {})
if updated:
logger.info("Updated source/recipes.rst")