def STAGE3_loop(pkgdir_paths, nb_cpu):
total = len(pkgdir_paths)
job_queue = []
for pkgdir_path in pkgdir_paths:
try:
pkg = bbs.parse.getPkgFromDir(pkgdir_path)
version = bbs.parse.getVersionFromDir(pkgdir_path)
srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path)
except IOError:
print "BBS> Can't read DESCRIPTION file!"
else:
cmd = BBSbase.getSTAGE3cmd(pkgdir_path)
pkgdumps_prefix = pkg + '.buildsrc'
pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps,
BBSvars.buildsrc_rdir)
job_queue.append(job)
nb_jobs = len(job_queue)
print "BBS> BEGIN STAGE3 loop."
t0 = time.time()
nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
BBScorevars.r_cmd_timeout, True)
dt = time.time() - t0
print "BBS> END STAGE3 loop."
print "BBS> -------------------------------------------------------------"
print "BBS> STAGE3 SUMMARY:"
print "BBS> o Working dir: %s" % os.getcwd()
print "BBS> o %d pkg(s) listed in file BBS_CENTRAL_BASEURL/%s" % (
total, BBScorevars.meat_index_file)
print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs
print "BBS> o %d srcpkg file(s) produced" % nb_products
print "BBS> o Total time: %.2f seconds" % dt
print "BBS> -------------------------------------------------------------"
return
def makeNodeInfo():
# Generate the NodeInfo files (the files containing some node related info)
NodeInfo_subdir = "NodeInfo"
print "BBS> Updating BBS_WORK_TOPDIR/%s" % NodeInfo_subdir
NodeInfo_path = os.path.join(BBSvars.work_topdir, NodeInfo_subdir)
bbs.fileutils.remake_dir(NodeInfo_path)
os.chdir(NodeInfo_path)
writeRversion()
writeRconfig()
Rscript = "sessionInfo()"
bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), \
'R-sessionInfo.txt', 60.0, True) # ignore retcode
Rscript = "data.frame(installed.packages()[,c('Version','Built')])"
bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), \
'R-instpkgs.txt', 60.0, True) # ignore retcode
writeSysCommandVersion('CC')
writeSysCommandVersion('CXX')
writeSysCommandVersion('CXX98')
writeSysCommandVersion('CXX11')
writeSysCommandVersion('CXX14')
writeSysCommandVersion('F77')
writeSysCommandVersion('FC')
print "BBS> cd BBS_WORK_TOPDIR"
os.chdir(BBSvars.work_topdir)
BBSvars.Node_rdir.Put(NodeInfo_subdir, True, True)
return
def STAGE4_loop(srcpkg_paths, nb_cpu):
total = len(srcpkg_paths)
job_queue = []
for srcpkg_path in srcpkg_paths:
cmd = BBSbase.getSTAGE4cmd(srcpkg_path)
if cmd == None:
continue
pkg = bbs.parse.getPkgFromPath(srcpkg_path)
version = bbs.parse.getVersionFromPath(srcpkg_path)
check_dir = pkg + '.Rcheck'
pkgdumps_prefix = pkg + '.checksrc'
pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix)
job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps,
BBSvars.checksrc_rdir)
job_queue.append(job)
nb_jobs = len(job_queue)
print "BBS> BEGIN STAGE4 loop."
t0 = time.time()
bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
BBScorevars.r_cmd_timeout, True)
dt = time.time() - t0
print "BBS> END STAGE4 loop."
print "BBS> -------------------------------------------------------------"
print "BBS> STAGE4 SUMMARY:"
print "BBS> o Working dir: %s" % os.getcwd()
print "BBS> o %d srcpkg file(s) in working dir" % total
print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs
print "BBS> o Total time: %.2f seconds" % dt
print "BBS> -------------------------------------------------------------"
return
def STAGE5_loop(srcpkg_paths, nb_cpu):
total = len(srcpkg_paths)
job_queue = []
for srcpkg_path in srcpkg_paths:
cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
if cmd == None:
continue
pkg = bbs.parse.getPkgFromPath(srcpkg_path)
version = bbs.parse.getVersionFromPath(srcpkg_path)
fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
pkgdumps_prefix = pkg + '.buildbin'
pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps,
BBSvars.buildbin_rdir)
job_queue.append(job)
nb_jobs = len(job_queue)
print "BBS> BEGIN STAGE5 loop."
t0 = time.time()
nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
BBScorevars.r_cmd_timeout, True)
dt = time.time() - t0
print "BBS> END STAGE5 loop."
print "BBS> -------------------------------------------------------------"
print "BBS> STAGE5 SUMMARY:"
print "BBS> o Working dir: %s" % os.getcwd()
print "BBS> o %d srcpkg file(s) in working dir" % total
print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs
print "BBS> o %d binpkg file(s) produced" % nb_products
print "BBS> o Total time: %.2f seconds" % dt
print "BBS> -------------------------------------------------------------"
return
def makeNodeInfo():
# Generate the NodeInfo files (the files containing some node related info)
NodeInfo_subdir = 'NodeInfo'
print('BBS> Updating BBS_WORK_TOPDIR/%s' % NodeInfo_subdir)
NodeInfo_path = os.path.join(BBSvars.work_topdir, NodeInfo_subdir)
bbs.fileutils.remake_dir(NodeInfo_path)
os.chdir(NodeInfo_path)
write_R_version()
write_R_config()
write_sys_command_version('CC')
write_sys_command_version('CXX')
#write_sys_command_version('CXX98')
write_sys_command_version('CXX11')
write_sys_command_version('CXX14')
write_sys_command_version('CXX17')
#write_sys_command_version('F77')
#write_sys_command_version('FC')
Rexpr = 'sessionInfo()'
bbs.jobs.runJob(BBSbase.Rexpr2syscmd(Rexpr), \
'R-sessionInfo.txt', 60.0, True) # ignore retcode
Rexpr = "options(width=500);print(installed.packages()[,c('LibPath','Version','Built')],quote=FALSE)"
bbs.jobs.runJob(BBSbase.Rexpr2syscmd(Rexpr), \
'R-instpkgs.txt', 120.0, True) # ignore retcode
print('BBS> cd BBS_WORK_TOPDIR')
os.chdir(BBSvars.work_topdir)
BBSvars.Node_rdir.Put(NodeInfo_subdir, True, True)
return
def prepare_STAGE2_job_queue(target_pkgs, pkg_deps_list, installed_pkgs):
print "BBS> Preparing STAGE2 job queue ... ",
stage = 'install'
jobs = []
nb_target_pkgs_in_queue = nb_skipped_pkgs = 0
for pkg in pkg_deps_list.keys():
version = None
pkgdumps_prefix = pkg + '.' + stage
pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix)
if pkg in target_pkgs:
try:
version = bbs.parse.getVersionFromDir(pkg)
except IOError:
print "BBS> Can't read %s/DESCRIPTION file!" % pkg
cmd = BBSbase.getSTAGE2cmd(pkg, version)
nb_target_pkgs_in_queue += 1
else:
if pkg in installed_pkgs:
cmd = pkgdumps = None
nb_skipped_pkgs += 1
else:
cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg)
job = BBSbase.InstallPkg_Job(pkg, version, cmd,
pkgdumps, BBSvars.install_rdir)
jobs.append(job)
nb_jobs = len(jobs)
print "OK"
nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue
nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue
nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \
nb_skipped_pkgs
nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs
print "BBS> Job summary:"
print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \
(nb_not_needed, len(target_pkgs))
print "BBS> | => no need to install them"
print "BBS> | => they're not going in the installation queue"
print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \
nb_jobs
print "BBS> | o %d are target pkgs" % nb_target_pkgs_in_queue
print "BBS> | => will (re-)install them with 'R CMD INSTALL'"
print "BBS> | o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue
print "BBS> | - %d are already installed" % nb_skipped_pkgs
print "BBS> | => won't re-install them (job will be skipped)"
print "BBS> | - %d are not already installed" % \
nb_non_target_pkgs_to_install
print "BBS> | => will install them with"
print "BBS> | install.packages(pkg, repos=non_target_repos,"
print "BBS> | dep=FALSE, ...)"
print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install
print "BBS>"
job_queue = bbs.jobs.JobQueue(stage, jobs, pkg_deps_list)
job_queue._nb_pkgs_to_install = nb_pkgs_to_install
return job_queue
def prepare_STAGE2_job_queue(target_pkgs, pkg_dep_graph, installed_pkgs,
out_dir):
print('BBS> Preparing STAGE2 job queue ...', end=' ')
sys.stdout.flush()
stage = 'install'
jobs = []
nb_target_pkgs_in_queue = nb_skipped_pkgs = 0
for pkg in pkg_dep_graph.keys():
version = None
pkgdumps_prefix = pkg + '.' + stage
pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix)
if pkg in target_pkgs:
version = bbs.parse.get_Version_from_pkgsrctree(pkg)
cmd = BBSbase.getSTAGE2cmd(pkg, version)
nb_target_pkgs_in_queue += 1
else:
if pkg in installed_pkgs:
cmd = pkgdumps = None
nb_skipped_pkgs += 1
else:
cmd = BBSbase.get_install_cmd_for_non_target_pkg(pkg)
job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps, out_dir)
jobs.append(job)
nb_jobs = len(jobs)
print('OK')
sys.stdout.flush()
nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue
nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue
nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \
nb_skipped_pkgs
nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs
print('BBS> Job summary:')
print('BBS> | %d (out of %d) target pkgs are not supporting pkgs' % \
(nb_not_needed, len(target_pkgs)))
print('BBS> | => no need to install them')
print('BBS> | => they\'re not going in the installation queue')
print('BBS> | %d pkgs in the installation queue (all supporting pkgs):' % \
nb_jobs)
print('BBS> | o %d are target pkgs' % nb_target_pkgs_in_queue)
print('BBS> | => will (re-)install them with \'R CMD INSTALL\'')
print('BBS> | o %d are non-target pkgs:' % nb_non_target_pkgs_in_queue)
print('BBS> | - %d are already installed' % nb_skipped_pkgs)
print('BBS> | => won\'t re-install them (job will be skipped)')
print('BBS> | - %d are not already installed' % \
nb_non_target_pkgs_to_install)
print('BBS> | => will install them with')
print('BBS> | install.packages(pkg, repos=non_target_repos,')
print('BBS> | dep=FALSE, ...)')
print('BBS> | Total nb of packages to install: %d' % nb_pkgs_to_install)
print('BBS>')
job_queue = bbs.jobs.JobQueue(stage, jobs, pkg_dep_graph)
job_queue._nb_pkgs_to_install = nb_pkgs_to_install
return job_queue
def STAGE5_loop(srcpkg_paths, nb_cpu):
total = len(srcpkg_paths)
job_queue = []
for srcpkg_path in srcpkg_paths:
cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
if cmd == None:
continue
pkg = bbs.parse.getPkgFromPath(srcpkg_path)
version = bbs.parse.getVersionFromPath(srcpkg_path)
fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
pkgdumps_prefix = pkg + '.buildbin'
pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd,
pkgdumps, BBSvars.buildbin_rdir)
job_queue.append(job)
nb_jobs = len(job_queue)
print "BBS> BEGIN STAGE5 loop."
t0 = time.time()
nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
BBScorevars.r_cmd_timeout, True)
dt = time.time() - t0
print "BBS> END STAGE5 loop."
print "BBS> -------------------------------------------------------------"
print "BBS> STAGE5 SUMMARY:"
print "BBS> o Working dir: %s" % os.getcwd()
print "BBS> o %d srcpkg file(s) in working dir" % total
print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs
print "BBS> o %d binpkg file(s) produced" % nb_products
print "BBS> o Total time: %.2f seconds" % dt
print "BBS> -------------------------------------------------------------"
return
def STAGE3_loop(pkgdir_paths, nb_cpu):
total = len(pkgdir_paths)
job_queue = []
for pkgdir_path in pkgdir_paths:
try:
pkg = bbs.parse.getPkgFromDir(pkgdir_path)
version = bbs.parse.getVersionFromDir(pkgdir_path)
srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path)
except IOError:
print "BBS> Can't read DESCRIPTION file!"
else:
cmd = BBSbase.getSTAGE3cmd(pkgdir_path)
pkgdumps_prefix = pkg + '.buildsrc'
pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd,
pkgdumps, BBSvars.buildsrc_rdir)
job_queue.append(job)
nb_jobs = len(job_queue)
print "BBS> BEGIN STAGE3 loop."
t0 = time.time()
nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
BBScorevars.r_cmd_timeout, True)
dt = time.time() - t0
print "BBS> END STAGE3 loop."
print "BBS> -------------------------------------------------------------"
print "BBS> STAGE3 SUMMARY:"
print "BBS> o Working dir: %s" % os.getcwd()
print "BBS> o %d pkg(s) listed in file BBS_CENTRAL_BASEURL/%s" % (total, BBScorevars.meat_index_file)
print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs
print "BBS> o %d srcpkg file(s) produced" % nb_products
print "BBS> o Total time: %.2f seconds" % dt
print "BBS> -------------------------------------------------------------"
return
def STAGE4_loop(srcpkg_paths, nb_cpu):
total = len(srcpkg_paths)
job_queue = []
for srcpkg_path in srcpkg_paths:
cmd = BBSbase.getSTAGE4cmd(srcpkg_path)
if cmd == None:
continue
pkg = bbs.parse.getPkgFromPath(srcpkg_path)
version = bbs.parse.getVersionFromPath(srcpkg_path)
check_dir = pkg + '.Rcheck'
pkgdumps_prefix = pkg + '.checksrc'
pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix)
job = BBSbase.CheckSrc_Job(pkg, version, cmd,
pkgdumps, BBSvars.checksrc_rdir)
job_queue.append(job)
nb_jobs = len(job_queue)
print "BBS> BEGIN STAGE4 loop."
t0 = time.time()
nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
BBScorevars.r_cmd_timeout, True)
dt = time.time() - t0
print "BBS> END STAGE4 loop."
print "BBS> -------------------------------------------------------------"
print "BBS> STAGE4 SUMMARY:"
print "BBS> o Working dir: %s" % os.getcwd()
print "BBS> o %d srcpkg file(s) in working dir" % total
print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs
print "BBS> o Total time: %.2f seconds" % dt
print "BBS> -------------------------------------------------------------"
return
def STAGE2():
print "BBS> [STAGE2] STARTING STAGE2 at %s..." % time.asctime()
# We want to make sure the target repo is ready before we actually start
# (if it's not ready yet it probably means that the prerun.sh script did
# not finish on the main node, in which case we want to wait before we
# sync the local meat dir with the central MEAT0 dir).
waitForTargetRepoToBeReady()
meat_path = BBSvars.meat_path
BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True)
if BBSvars.MEAT0_type == 2:
srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path)
for srcpkg_file in srcpkg_files:
srcpkg_filepath = os.path.join(meat_path, srcpkg_file)
BBSbase.Untar(srcpkg_filepath, meat_path)
os.remove(srcpkg_filepath)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp"
STAGE2_tmp = os.path.join(BBSvars.work_topdir, "STAGE2_tmp")
bbs.fileutils.remake_dir(STAGE2_tmp)
os.chdir(STAGE2_tmp)
# Re-create architecture-specific Renviron.site files on multi-arch
# build machines.
if ('BBS_STAGE2_MODE' in os.environ
and os.environ['BBS_STAGE2_MODE'] == 'multiarch'
and BBScorevars.mode == "bioc"):
CreateREnvironFiles()
# Try to update all installed packages
CallRfunctionFromSTAGE2Script("updateNonTargetPkgs",
"updateNonTargetPkgs1.Rout")
# Extract list of target packages.
target_pkgs = extractTargetPkgListFromMeatIndex()
# Get 'pkg_deps_list' and 'installed_pkgs'.
pkg_deps_list = make_STAGE2_pkg_deps_list(target_pkgs)
installed_pkgs = get_installed_pkgs()
# Then re-install the supporting packages.
print "BBS> [STAGE2] cd BBS_MEAT_PATH"
os.chdir(BBSvars.meat_path)
BBSvars.install_rdir.RemakeMe(True)
STAGE2_loop(target_pkgs, pkg_deps_list, installed_pkgs, BBSvars.nb_cpu)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp"
os.chdir(STAGE2_tmp)
# Try again to update all installed packages (some updates could have
# failed in the previous attempt because of dependency issues).
CallRfunctionFromSTAGE2Script("updateNonTargetPkgs",
"updateNonTargetPkgs2.Rout")
updateNodeInfo()
print "BBS> [STAGE2] DONE at %s." % time.asctime()
return
def prepare_STAGE4_job_queue(srcpkg_paths, out_dir):
print("BBS> Preparing STAGE4 job queue ...", end=" ")
sys.stdout.flush()
stage = 'checksrc'
jobs = []
for srcpkg_path in srcpkg_paths:
cmd = BBSbase.getSTAGE4cmd(srcpkg_path)
pkg = bbs.parse.get_pkgname_from_srcpkg_path(srcpkg_path)
version = bbs.parse.get_version_from_srcpkg_path(srcpkg_path)
Rcheck_dir = pkg + '.Rcheck'
pkgdumps_prefix = pkg + '.' + stage
pkgdumps = BBSbase.PkgDumps(Rcheck_dir, pkgdumps_prefix)
job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, out_dir)
jobs.append(job)
print("OK")
sys.stdout.flush()
job_queue = bbs.jobs.JobQueue(stage, jobs, None)
job_queue._total = len(srcpkg_paths)
return job_queue
def prepare_STAGE5_job_queue(srcpkg_paths, out_dir):
print("BBS> Preparing STAGE5 job queue ...", end=" ")
sys.stdout.flush()
stage = 'buildbin'
jobs = []
for srcpkg_path in srcpkg_paths:
cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
pkg = bbs.parse.get_pkgname_from_srcpkg_path(srcpkg_path)
version = bbs.parse.get_version_from_srcpkg_path(srcpkg_path)
fileext = BBSutils.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
pkgdumps_prefix = pkg + '.' + stage
pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, out_dir)
jobs.append(job)
print("OK")
sys.stdout.flush()
job_queue = bbs.jobs.JobQueue(stage, jobs, None)
job_queue._total = len(srcpkg_paths)
return job_queue
def prepare_STAGE4_job_queue(srcpkg_paths):
print "BBS> Preparing STAGE4 job queue ... ",
stage = 'checksrc'
jobs = []
for srcpkg_path in srcpkg_paths:
cmd = BBSbase.getSTAGE4cmd(srcpkg_path)
if cmd == None:
continue
pkg = bbs.parse.getPkgFromPath(srcpkg_path)
version = bbs.parse.getVersionFromPath(srcpkg_path)
check_dir = pkg + '.Rcheck'
pkgdumps_prefix = pkg + '.' + stage
pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix)
job = BBSbase.CheckSrc_Job(pkg, version, cmd,
pkgdumps, BBSvars.checksrc_rdir)
jobs.append(job)
print "OK"
job_queue = bbs.jobs.JobQueue(stage, jobs, None)
job_queue._total = len(srcpkg_paths)
return job_queue
def extractSrcPkgTarballs(dest_dir):
MEAT0_path = BBSvars.MEAT0_rdir.path # Hopefully this is local!
srcpkg_files = bbs.fileutils.listSrcPkgFiles(MEAT0_path)
bbs.fileutils.remake_dir(dest_dir)
pkgs = []
for srcpkg_file in srcpkg_files:
pkg = bbs.parse.getPkgFromPath(srcpkg_file)
srcpkg_filepath = os.path.join(MEAT0_path, srcpkg_file)
BBSbase.Untar(srcpkg_filepath, dest_dir)
pkgs.append(pkg)
return pkgs
def prepare_STAGE1_job_queue(pkgdir_paths, dest_rdir):
print "BBS> Preparing STAGE1 job queue ... ",
stage = 'buildnovig'
jobs = []
for pkgdir_path in pkgdir_paths:
try:
pkg = bbs.parse.getPkgFromDir(pkgdir_path)
version = bbs.parse.getVersionFromDir(pkgdir_path)
srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path)
except IOError:
print "BBS> Can't read DESCRIPTION file!"
else:
cmd = BBSbase.getSTAGE1cmd(pkgdir_path)
pkgdumps_prefix = pkg + '.' + stage
pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir)
jobs.append(job)
print "OK"
job_queue = bbs.jobs.JobQueue(stage, jobs, None)
job_queue._total = len(pkgdir_paths)
return job_queue
def prepare_STAGE5_job_queue(srcpkg_paths):
print "BBS> Preparing STAGE5 job queue ... ",
stage = 'buildbin'
jobs = []
for srcpkg_path in srcpkg_paths:
cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
if cmd == None:
continue
pkg = bbs.parse.getPkgFromPath(srcpkg_path)
version = bbs.parse.getVersionFromPath(srcpkg_path)
fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
pkgdumps_prefix = pkg + '.' + stage
pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd,
pkgdumps, BBSvars.buildbin_rdir)
jobs.append(job)
print "OK"
job_queue = bbs.jobs.JobQueue(stage, jobs, None)
job_queue._total = len(srcpkg_paths)
return job_queue
def make_STAGE2_pkg_deps_list(target_pkgs):
# Make 'target_pkgs.txt' file.
target_pkgs_file = "target_pkgs.txt"
out = open(target_pkgs_file, 'w')
for pkg in target_pkgs:
out.write("%s\n" % pkg)
out.close()
print "BBS> [make_STAGE2_pkg_deps_list]",
print "%s pkgs written to %s" % (len(target_pkgs), target_pkgs_file)
# Make 'STAGE2_pkg_deps_list.txt' file.
Rfunction = "make_STAGE2_pkg_deps_list"
script_path = os.path.join(BBScorevars.BBS_home,
"utils",
"make_STAGE2_pkg_deps_list.R")
STAGE2_pkg_deps_list_path = "STAGE2_pkg_deps_list.txt"
print "BBS> [make_STAGE2_pkg_deps_list]",
print "Calling %s() defined in %s to make %s file ..." % \
(Rfunction, script_path, STAGE2_pkg_deps_list_path),
# Backslashes in the paths injected in 'Rscript' will be seen as escape
# characters by R so we need to replace them. Nothing will be replaced
# on a Unix-like platform, only on Windows where the paths can actually
# contain backslashes.
script_path2 = script_path.replace('\\', '/')
target_pkgs_file2 = target_pkgs_file.replace('\\', '/')
STAGE2_pkg_deps_list_path2 = STAGE2_pkg_deps_list_path.replace('\\', '/')
# Use short.list=TRUE to skip installation of target packages not needed
# by another target package for build or check.
#Rscript = "source('%s');%s('%s',outfile='%s',short.list=TRUE)" % \
Rscript = "source('%s');%s('%s',outfile='%s')" % \
(script_path2, Rfunction, target_pkgs_file2,
STAGE2_pkg_deps_list_path2)
out_file = Rfunction + ".Rout"
bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), out_file) # ignore retcode
print "OK"
# Load 'STAGE2_pkg_deps_list.txt' file.
print "BBS> [make_STAGE2_pkg_deps_list] Loading %s file ..." % \
STAGE2_pkg_deps_list_path,
f = open(STAGE2_pkg_deps_list_path, 'r')
pkg_deps_list = {}
EMPTY_STRING = ''
for line in f:
(pkg, deps) = line.split(":")
deps = deps.strip().split(" ")
if EMPTY_STRING in deps:
deps.remove(EMPTY_STRING)
pkg_deps_list[pkg] = deps
f.close()
print "OK (%s pkgs and their deps loaded)" % len(pkg_deps_list)
print "BBS> [make_STAGE2_pkg_deps_list] DONE."
return pkg_deps_list
def prepare_STAGE1_job_queue(pkgsrctrees, dest_rdir):
print("BBS> Preparing STAGE1 job queue ... ", end=" ")
stage = 'buildnovig'
jobs = []
for pkgsrctree in pkgsrctrees:
try:
pkg = bbs.parse.get_Package_from_pkgsrctree(pkgsrctree)
version = bbs.parse.get_Version_from_pkgsrctree(pkgsrctree)
srcpkg_file = bbs.parse.make_srcpkg_file_from_pkgsrctree(
pkgsrctree)
except IOError:
print("BBS> Can't read DESCRIPTION file!")
else:
cmd = BBSbase.getSTAGE1cmd(pkgsrctree)
pkgdumps_prefix = pkg + '.' + stage
pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir)
jobs.append(job)
print("OK")
job_queue = bbs.jobs.JobQueue(stage, jobs, None)
job_queue._total = len(pkgsrctrees)
return job_queue
def get_installed_pkgs():
installed_pkgs_path = "installed_pkgs.txt"
Rscript = "writeLines(rownames(installed.packages()),'%s')" % \
installed_pkgs_path
out_file = "get_installed_pkgs.Rout"
bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), out_file) # ignore retcode
installed_pkgs = []
f = open(installed_pkgs_path, 'r')
for line in f:
installed_pkgs.append(line.strip())
f.close()
print "BBS> [get_installed_pkgs] %s installed pkgs" % len(installed_pkgs)
return installed_pkgs
def get_installed_pkgs():
installed_pkgs_path = 'installed_pkgs.txt'
Rexpr = "writeLines(rownames(installed.packages()),'%s')" % \
installed_pkgs_path
out_file = 'get_installed_pkgs.Rout'
bbs.jobs.runJob(BBSbase.Rexpr2syscmd(Rexpr), out_file) # ignore retcode
installed_pkgs = []
f = open(installed_pkgs_path, 'r')
for line in f:
installed_pkgs.append(line.strip())
f.close()
print('BBS> [get_installed_pkgs] %s installed pkgs' % len(installed_pkgs))
return installed_pkgs
def CallRfunctionFromSTAGE2Script(Rfunction, out_file=None):
print "BBS> [%s] BEGIN ..." % Rfunction
script_path = BBSvars.STAGE2_r_script
# Backslahes in the paths injected in 'Rscript' will be seen as escape
# characters by R so we need to replace them. Nothing will be replaced
# on a Unix-like platform, only on Windows where the paths can actually
# contain backslahes.
script_path = script_path.replace('\\', '/')
Rscript = "source('%s');%s()" % (script_path, Rfunction)
if out_file == None:
out_file = '%s.Rout' % Rfunction
bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), out_file, 3600.0) # ignore retcode
print "BBS> [%s] END." % Rfunction
return
def write_PACKAGES(rdir):
doing_what = 'creating PACKAGES index file for target repo'
print('BBS> START %s at %s...' % (doing_what, time.asctime()))
sys.stdout.flush()
Rexpr = r'library(tools);write_PACKAGES(\".\")'
bbs.jobs.doOrDie(BBSbase.Rexpr2syscmd(Rexpr))
## write_PACKAGES() won't create an empty PACKAGES file if no packages
## are found so we create one.
if not os.path.exists('PACKAGES'):
f = open('PACKAGES', 'w')
f.close()
rdir.Put('PACKAGES', True, True)
print('BBS> DONE %s at %s.' % (doing_what, time.asctime()))
sys.stdout.flush()
return
def GetCranPkgs(work_topdir):
print "BBS> [prerun:get-cran-pkgs] cd BBS_WORK_TOPDIR"
os.chdir(work_topdir)
pkgs_dir = "pkgs"
if os.path.exists(pkgs_dir):
print "BBS> [prerun:get-cran-pkgs] rm -r %s" % pkgs_dir
bbs.fileutils.nuke_tree(pkgs_dir)
print "BBS> [prerun:get-cran-pkgs] Download CRAN packages to BBS_WORK_TOPDIR/pkgs"
cmd = "/usr/bin/rsync -ae ssh webadmin@cobra:'/extra/www/cran-mirror/src/contrib/*.tar.gz' %s" % pkgs_dir
bbs.jobs.doOrDie(cmd)
os.chdir(pkgs_dir)
srcpkg_files = bbs.fileutils.listSrcPkgFiles()
for srcpkg_file in srcpkg_files:
print "BBS> [prerun:get-cran-pkgs] tar zxf %s" % srcpkg_file
BBSbase.Untar(srcpkg_file)
return
def build_pkg_dep_graph(target_pkgs):
# Generate file 'target_pkgs.txt'.
target_pkgs_file = 'target_pkgs.txt'
out = open(target_pkgs_file, 'w')
for pkg in target_pkgs:
out.write('%s\n' % pkg)
out.close()
print('BBS> [build_pkg_dep_graph]', end=' ')
print('%s pkgs written to %s' % (len(target_pkgs), target_pkgs_file))
# Generate file 'pkg_dep_graph.txt'.
Rfunction = 'build_pkg_dep_graph'
script_path = os.path.join(BBSvars.BBS_home, 'utils',
'build_pkg_dep_graph.R')
print('BBS> [build_pkg_dep_graph]', end=' ')
print('Calling %s() defined in %s to generate file %s ...' % \
(Rfunction, script_path, BBSutils.pkg_dep_graph_file), end=' ')
# Backslashes in the paths injected in 'Rexpr' will be seen as escape
# characters by R so we need to replace them. Nothing will be replaced
# on a Unix-like platform, only on Windows where the paths can actually
# contain backslashes.
script_path2 = script_path.replace('\\', '/')
target_pkgs_file2 = target_pkgs_file.replace('\\', '/')
STAGE2_pkg_dep_graph_path2 = BBSutils.pkg_dep_graph_file.replace('\\', '/')
# Use short.list=TRUE for "smart STAGE2" i.e. to skip installation of
# target packages not needed by another target package for build or check.
#Rexpr = "source('%s');%s('%s',outfile='%s',short.list=TRUE)" % \
Rexpr = "source('%s');%s('%s',outfile='%s')" % \
(script_path2, Rfunction, target_pkgs_file2,
STAGE2_pkg_dep_graph_path2)
out_file = Rfunction + '.Rout'
bbs.jobs.runJob(BBSbase.Rexpr2syscmd(Rexpr), out_file) # ignore retcode
print('OK')
# Send files 'target_pkgs.txt' and 'pkg_dep_graph.txt' to central
# build node.
BBSvars.Node_rdir.Put(BBSutils.pkg_dep_graph_file, True, True)
# Load file 'pkg_dep_graph.txt'.
print('BBS> [build_pkg_dep_graph] Loading %s file ...' % \
BBSutils.pkg_dep_graph_file, end=' ')
pkg_dep_graph = bbs.parse.load_pkg_dep_graph(BBSutils.pkg_dep_graph_file)
print('OK (%s pkgs and their deps loaded)' % len(pkg_dep_graph))
print('BBS> [build_pkg_dep_graph] DONE.')
return pkg_dep_graph
def make_PROPAGATION_STATUS_DB(final_repo):
Rfunction = 'makePropagationStatusDb'
OUTGOING_dir = 'OUTGOING'
db_filepath = 'PROPAGATION_STATUS_DB.txt'
script_path = os.path.join(BBSvars.BBS_home,
"utils",
"makePropagationStatusDb.R")
Rexpr = "source('%s');%s('%s','%s','%s')" % \
(script_path, Rfunction, OUTGOING_dir, final_repo, db_filepath)
cmd = BBSbase.Rexpr2syscmd(Rexpr)
## Nasty things (that I don't really understand) can happen with
## subprocess.run() if this code is runned by the Task Scheduler
## on Windows (the child process tends to almost always return an
## error). Apparently using 'stderr=subprocess.STDOUT' fixes this pb.
subprocess.run(cmd, stdout=None, stderr=subprocess.STDOUT, shell=True,
check=True)
return
def prepare_STAGE1_job_queue(pkgdir_paths, dest_rdir):
print "BBS> Preparing STAGE1 job queue ... ",
stage = 'buildnovig'
jobs = []
for pkgdir_path in pkgdir_paths:
try:
pkg = bbs.parse.getPkgFromDir(pkgdir_path)
version = bbs.parse.getVersionFromDir(pkgdir_path)
srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path)
except IOError:
print "BBS> Can't read DESCRIPTION file!"
else:
cmd = BBSbase.getSTAGE1cmd(pkgdir_path)
pkgdumps_prefix = pkg + '.' + stage;
pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix)
job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir)
jobs.append(job)
print "OK"
job_queue = bbs.jobs.JobQueue(stage, jobs, None)
job_queue._total = len(pkgdir_paths)
return job_queue
def STAGE2():
print "BBS> [STAGE2] STARTING STAGE2 at %s..." % time.asctime()
# We want to make sure the target repo is ready before we actually start
# (if it's not ready yet it probably means that the prerun.sh script did
# not finish on the main node, in which case we want to wait before we
# sync the local meat dir with the central MEAT0 dir).
waitForTargetRepoToBeReady()
meat_path = BBSvars.meat_path
BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True)
if BBSvars.MEAT0_type == 2:
srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path)
for srcpkg_file in srcpkg_files:
srcpkg_filepath = os.path.join(meat_path, srcpkg_file)
BBSbase.Untar(srcpkg_filepath, meat_path)
os.remove(srcpkg_filepath)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/gitlog"
gitlog_path = BBSvars.gitlog_path
BBSvars.GITLOG_rdir.syncLocalDir(gitlog_path, True)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp"
STAGE2_tmp = os.path.join(BBSvars.work_topdir, "STAGE2_tmp")
bbs.fileutils.remake_dir(STAGE2_tmp)
os.chdir(STAGE2_tmp)
# Re-create architecture-specific Renviron.site files on multi-arch
# build machines.
if ('BBS_STAGE2_MODE' in os.environ and
os.environ['BBS_STAGE2_MODE'] == 'multiarch' and
BBScorevars.subbuilds == "bioc"):
CreateREnvironFiles()
# Try to update all installed packages.
CallRfunctionFromSTAGE2Script("updateNonTargetPkgs",
"updateNonTargetPkgs1.Rout")
# Extract list of target packages.
target_pkgs = extractTargetPkgListFromMeatIndex()
# Get 'pkg_deps_list' and 'installed_pkgs'.
pkg_deps_list = make_STAGE2_pkg_deps_list(target_pkgs)
installed_pkgs = get_installed_pkgs()
# Inject additional fields into DESCRIPTION.
print "BBS> [STAGE2] cd BBS_MEAT_PATH"
print "BBS> [STAGE2] Injecting fields into DESCRIPTION"
os.chdir(meat_path)
for pkg in target_pkgs:
gitlog_file = os.path.join(BBSvars.gitlog_path, "git-log-%s.dcf" % pkg)
if not os.path.exists(gitlog_file):
print "BBS> %s file does not exist --> skipping." % gitlog_file
continue
desc_file = os.path.join(BBSvars.meat_path, pkg, 'DESCRIPTION')
if not os.path.exists(desc_file):
print "BBS> %s file does not exist --> skipping." % desc_file
continue
bbs.parse.injectFieldsInDESCRIPTION(desc_file, gitlog_file)
# Then re-install the supporting packages.
print "BBS> [STAGE2] Re-install supporting packages"
os.chdir(meat_path)
BBSvars.install_rdir.RemakeMe(True)
job_queue = prepare_STAGE2_job_queue(target_pkgs, pkg_deps_list,
installed_pkgs)
STAGE2_loop(job_queue, BBSvars.nb_cpu)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp"
os.chdir(STAGE2_tmp)
# Try again to update all installed packages (some updates could have
# failed in the previous attempt because of dependency issues).
CallRfunctionFromSTAGE2Script("updateNonTargetPkgs",
"updateNonTargetPkgs2.Rout")
print "BBS> [STAGE2] DONE at %s." % time.asctime()
return
def STAGE2_loop(target_pkgs, pkg_deps_list, installed_pkgs, nb_cpu):
print "BBS> Preparing STAGE2 job queue ...",
job_queue = []
nb_target_pkgs_in_queue = nb_skipped_pkgs = 0
for pkg in pkg_deps_list.keys():
version = None
pkgdumps_prefix = pkg + '.install'
pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix)
if pkg in target_pkgs:
try:
version = bbs.parse.getVersionFromDir(pkg)
except IOError:
print "BBS> Can't read DESCRIPTION file!"
cmd = BBSbase.getSTAGE2cmd(pkg, version)
nb_target_pkgs_in_queue += 1
else:
if pkg in installed_pkgs:
cmd = pkgdumps = None
nb_skipped_pkgs += 1
else:
cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg)
job = BBSbase.InstallPkg_Job(pkg, version, cmd,
pkgdumps, BBSvars.install_rdir)
job_queue.append(job)
nb_jobs = len(job_queue)
print "OK"
nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue
nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue
nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \
nb_skipped_pkgs
nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs
print "BBS> Job summary:"
print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \
(nb_not_needed, len(target_pkgs))
print "BBS> | => no need to install them"
print "BBS> | => they're not going in the installation queue"
print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \
nb_jobs
print "BBS> | o %d are target pkgs" % nb_target_pkgs_in_queue
print "BBS> | => will (re-)install them with 'R CMD INSTALL'"
print "BBS> | o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue
print "BBS> | - %d are already installed" % nb_skipped_pkgs
print "BBS> | => won't re-install them (job will be skipped)"
print "BBS> | - %d are not already installed" % \
nb_non_target_pkgs_to_install
print "BBS> | => will install them with"
print "BBS> | install.packages(pkg, repos=non_target_repos,"
print "BBS> | dep=FALSE, ...)"
print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install
print "BBS>"
print "BBS> BEGIN STAGE2 loop."
t0 = time.time()
nb_installed = bbs.jobs.processJobQueue(job_queue, pkg_deps_list, nb_cpu,
BBScorevars.r_cmd_timeout, True)
dt = time.time() - t0
print "BBS> END STAGE2 loop."
nb_failures = nb_pkgs_to_install - nb_installed
print "BBS> -------------------------------------------------------------"
print "BBS> STAGE2 SUMMARY:"
print "BBS> o Working dir: %s" % os.getcwd()
print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs
print "BBS> o %d pkg(s) to (re-)install: %d successes / %d failures" % \
(nb_pkgs_to_install, nb_installed, nb_failures)
print "BBS> o Total time: %.2f seconds" % dt
print "BBS> -------------------------------------------------------------"
return
def STAGE2():
print('BBS> [STAGE2] STARTING STAGE2 at %s' % time.asctime())
# We want to make sure the target repo is ready before we actually start
# (if it's not ready yet it probably means that the prerun.sh script did
# not finish on the main node, in which case we want to wait before we
# sync the local meat dir with the central MEAT0 dir).
waitForTargetRepoToBeReady()
BBSvars.install_rdir.RemakeMe(True)
if asynchronous_mode:
out_dir = make_stage_out_dir('install')
else:
out_dir = BBSvars.install_rdir
meat_path = BBSvars.meat_path
BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True)
if BBSvars.MEAT0_type == 2:
srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path)
for srcpkg_file in srcpkg_files:
srcpkg_filepath = os.path.join(meat_path, srcpkg_file)
BBSbase.Untar(srcpkg_filepath, meat_path)
os.remove(srcpkg_filepath)
print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/gitlog')
gitlog_path = BBSutils.getenv('BBS_GITLOG_PATH')
BBSvars.GITLOG_rdir.syncLocalDir(gitlog_path, True)
print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp')
STAGE2_tmp = os.path.join(BBSvars.work_topdir, 'STAGE2_tmp')
bbs.fileutils.remake_dir(STAGE2_tmp)
os.chdir(STAGE2_tmp)
# Re-create architecture-specific Renviron.site files on multi-arch
# build machines.
#if ('BBS_STAGE2_MODE' in os.environ and
# os.environ['BBS_STAGE2_MODE'] == 'multiarch' and
# BBSvars.buildtype == 'bioc'):
# CreateREnvironFiles()
if BBSvars.buildtype in ['bioc', 'bioc-testing']:
# Update non-target packages.
print('BBS> [STAGE2] Update non-target packages (1st run) ...',
end=' ')
sys.stdout.flush()
cmd = BBSbase.get_update_cmd_for_non_target_pkgs()
bbs.jobs.runJob(cmd, 'updateNonTargetPkgs1.Rout', 3600.0)
print('OK')
sys.stdout.flush()
# Extract list of target packages.
target_pkgs = get_list_of_target_pkgs()
# Get 'pkg_dep_graph' and 'installed_pkgs'.
pkg_dep_graph = build_pkg_dep_graph(target_pkgs)
installed_pkgs = get_installed_pkgs()
# Inject additional fields into DESCRIPTION.
print('BBS> [STAGE2] cd BBS_MEAT_PATH')
print('BBS> [STAGE2] Injecting fields into DESCRIPTION')
os.chdir(meat_path)
for pkg in target_pkgs:
gitlog_file = os.path.join(gitlog_path, 'git-log-%s.dcf' % pkg)
if not os.path.exists(gitlog_file):
print('BBS> %s file does not exist --> skipping.' % gitlog_file)
continue
desc_file = os.path.join(BBSvars.meat_path, pkg, 'DESCRIPTION')
if not os.path.exists(desc_file):
print('BBS> %s file does not exist --> skipping.' % desc_file)
continue
bbs.parse.injectFieldsInDESCRIPTION(desc_file, gitlog_file)
# Then re-install the supporting packages.
print('BBS> [STAGE2] Re-install supporting packages')
os.chdir(meat_path)
job_queue = prepare_STAGE2_job_queue(target_pkgs, pkg_dep_graph,
installed_pkgs, out_dir)
STAGE2_loop(job_queue, BBSvars.install_nb_cpu, out_dir)
print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp')
os.chdir(STAGE2_tmp)
if BBSvars.buildtype in ['bioc', 'bioc-testing']:
# Try again to update non-target packages (some updates could have
# failed in the previous attempt because of dependency issues).
print('BBS> [STAGE2] Update non-target packages (2nd run) ...',
end=' ')
sys.stdout.flush()
cmd = BBSbase.get_update_cmd_for_non_target_pkgs()
bbs.jobs.runJob(cmd, 'updateNonTargetPkgs2.Rout', 3600.0)
print('OK')
sys.stdout.flush()
makeNodeInfo()
print('BBS> [STAGE2] DONE at %s.' % time.asctime())
return
def STAGE2_loop(target_pkgs, pkg_deps_list, installed_pkgs, nb_cpu):
print "BBS> Preparing STAGE2 job queue ...",
job_queue = []
nb_target_pkgs_in_queue = nb_skipped_pkgs = 0
for pkg in pkg_deps_list.keys():
version = None
pkgdumps_prefix = pkg + '.install'
pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix)
if pkg in target_pkgs:
try:
version = bbs.parse.getVersionFromDir(pkg)
except IOError:
print "BBS> Can't read DESCRIPTION file!"
cmd = BBSbase.getSTAGE2cmd(pkg, version)
nb_target_pkgs_in_queue += 1
else:
if pkg in installed_pkgs:
cmd = pkgdumps = None
nb_skipped_pkgs += 1
else:
cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg)
job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps,
BBSvars.install_rdir)
job_queue.append(job)
nb_jobs = len(job_queue)
print "OK"
nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue
nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue
nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \
nb_skipped_pkgs
nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs
print "BBS> Job summary:"
print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \
(nb_not_needed, len(target_pkgs))
print "BBS> | => no need to install them"
print "BBS> | => they're not going in the installation queue"
print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \
nb_jobs
print "BBS> | o %d are target pkgs" % nb_target_pkgs_in_queue
print "BBS> | => will (re-)install them with 'R CMD INSTALL'"
print "BBS> | o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue
print "BBS> | - %d are already installed" % nb_skipped_pkgs
print "BBS> | => won't re-install them (job will be skipped)"
print "BBS> | - %d are not already installed" % \
nb_non_target_pkgs_to_install
print "BBS> | => will install them with"
print "BBS> | install.packages(pkg, repos=non_target_repos,"
print "BBS> | dep=FALSE, ...)"
print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install
print "BBS>"
print "BBS> BEGIN STAGE2 loop."
t0 = time.time()
nb_installed = bbs.jobs.processJobQueue(job_queue, pkg_deps_list, nb_cpu,
BBScorevars.r_cmd_timeout, True)
dt = time.time() - t0
print "BBS> END STAGE2 loop."
nb_failures = nb_pkgs_to_install - nb_installed
print "BBS> -------------------------------------------------------------"
print "BBS> STAGE2 SUMMARY:"
print "BBS> o Working dir: %s" % os.getcwd()
print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs
print "BBS> o %d pkg(s) to (re-)install: %d successes / %d failures" % \
(nb_pkgs_to_install, nb_installed, nb_failures)
print "BBS> o Total time: %.2f seconds" % dt
print "BBS> -------------------------------------------------------------"
return
