def STAGE3_loop(pkgdir_paths, nb_cpu): total = len(pkgdir_paths) job_queue = [] for pkgdir_path in pkgdir_paths: try: pkg = bbs.parse.getPkgFromDir(pkgdir_path) version = bbs.parse.getVersionFromDir(pkgdir_path) srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path) except IOError: print "BBS> Can't read DESCRIPTION file!" else: cmd = BBSbase.getSTAGE3cmd(pkgdir_path) pkgdumps_prefix = pkg + '.buildsrc' pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.buildsrc_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "BBS> BEGIN STAGE3 loop." t0 = time.time() nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE3 loop." print "BBS> -------------------------------------------------------------" print "BBS> STAGE3 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d pkg(s) listed in file BBS_CENTRAL_BASEURL/%s" % ( total, BBScorevars.meat_index_file) print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs print "BBS> o %d srcpkg file(s) produced" % nb_products print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def makeNodeInfo(): # Generate the NodeInfo files (the files containing some node related info) NodeInfo_subdir = "NodeInfo" print "BBS> Updating BBS_WORK_TOPDIR/%s" % NodeInfo_subdir NodeInfo_path = os.path.join(BBSvars.work_topdir, NodeInfo_subdir) bbs.fileutils.remake_dir(NodeInfo_path) os.chdir(NodeInfo_path) writeRversion() writeRconfig() Rscript = "sessionInfo()" bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), \ 'R-sessionInfo.txt', 60.0, True) # ignore retcode Rscript = "data.frame(installed.packages()[,c('Version','Built')])" bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), \ 'R-instpkgs.txt', 60.0, True) # ignore retcode writeSysCommandVersion('CC') writeSysCommandVersion('CXX') writeSysCommandVersion('CXX98') writeSysCommandVersion('CXX11') writeSysCommandVersion('CXX14') writeSysCommandVersion('F77') writeSysCommandVersion('FC') print "BBS> cd BBS_WORK_TOPDIR" os.chdir(BBSvars.work_topdir) BBSvars.Node_rdir.Put(NodeInfo_subdir, True, True) return
def STAGE4_loop(srcpkg_paths, nb_cpu): total = len(srcpkg_paths) job_queue = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE4cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) check_dir = pkg + '.Rcheck' pkgdumps_prefix = pkg + '.checksrc' pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix) job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, BBSvars.checksrc_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "BBS> BEGIN STAGE4 loop." t0 = time.time() bbs.jobs.processJobQueue(job_queue, None, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE4 loop." print "BBS> -------------------------------------------------------------" print "BBS> STAGE4 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d srcpkg file(s) in working dir" % total print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def STAGE5_loop(srcpkg_paths, nb_cpu): total = len(srcpkg_paths) job_queue = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE5cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt') binpkg_file = "%s_%s.%s" % (pkg, version, fileext) pkgdumps_prefix = pkg + '.buildbin' pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.buildbin_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "BBS> BEGIN STAGE5 loop." t0 = time.time() nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE5 loop." print "BBS> -------------------------------------------------------------" print "BBS> STAGE5 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d srcpkg file(s) in working dir" % total print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs print "BBS> o %d binpkg file(s) produced" % nb_products print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def makeNodeInfo(): # Generate the NodeInfo files (the files containing some node related info) NodeInfo_subdir = 'NodeInfo' print('BBS> Updating BBS_WORK_TOPDIR/%s' % NodeInfo_subdir) NodeInfo_path = os.path.join(BBSvars.work_topdir, NodeInfo_subdir) bbs.fileutils.remake_dir(NodeInfo_path) os.chdir(NodeInfo_path) write_R_version() write_R_config() write_sys_command_version('CC') write_sys_command_version('CXX') #write_sys_command_version('CXX98') write_sys_command_version('CXX11') write_sys_command_version('CXX14') write_sys_command_version('CXX17') #write_sys_command_version('F77') #write_sys_command_version('FC') Rexpr = 'sessionInfo()' bbs.jobs.runJob(BBSbase.Rexpr2syscmd(Rexpr), \ 'R-sessionInfo.txt', 60.0, True) # ignore retcode Rexpr = "options(width=500);print(installed.packages()[,c('LibPath','Version','Built')],quote=FALSE)" bbs.jobs.runJob(BBSbase.Rexpr2syscmd(Rexpr), \ 'R-instpkgs.txt', 120.0, True) # ignore retcode print('BBS> cd BBS_WORK_TOPDIR') os.chdir(BBSvars.work_topdir) BBSvars.Node_rdir.Put(NodeInfo_subdir, True, True) return
def prepare_STAGE2_job_queue(target_pkgs, pkg_deps_list, installed_pkgs): print "BBS> Preparing STAGE2 job queue ... ", stage = 'install' jobs = [] nb_target_pkgs_in_queue = nb_skipped_pkgs = 0 for pkg in pkg_deps_list.keys(): version = None pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix) if pkg in target_pkgs: try: version = bbs.parse.getVersionFromDir(pkg) except IOError: print "BBS> Can't read %s/DESCRIPTION file!" % pkg cmd = BBSbase.getSTAGE2cmd(pkg, version) nb_target_pkgs_in_queue += 1 else: if pkg in installed_pkgs: cmd = pkgdumps = None nb_skipped_pkgs += 1 else: cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg) job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.install_rdir) jobs.append(job) nb_jobs = len(jobs) print "OK" nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \ nb_skipped_pkgs nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs print "BBS> Job summary:" print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \ (nb_not_needed, len(target_pkgs)) print "BBS> | => no need to install them" print "BBS> | => they're not going in the installation queue" print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \ nb_jobs print "BBS> | o %d are target pkgs" % nb_target_pkgs_in_queue print "BBS> | => will (re-)install them with 'R CMD INSTALL'" print "BBS> | o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue print "BBS> | - %d are already installed" % nb_skipped_pkgs print "BBS> | => won't re-install them (job will be skipped)" print "BBS> | - %d are not already installed" % \ nb_non_target_pkgs_to_install print "BBS> | => will install them with" print "BBS> | install.packages(pkg, repos=non_target_repos," print "BBS> | dep=FALSE, ...)" print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install print "BBS>" job_queue = bbs.jobs.JobQueue(stage, jobs, pkg_deps_list) job_queue._nb_pkgs_to_install = nb_pkgs_to_install return job_queue
def prepare_STAGE2_job_queue(target_pkgs, pkg_dep_graph, installed_pkgs, out_dir): print('BBS> Preparing STAGE2 job queue ...', end=' ') sys.stdout.flush() stage = 'install' jobs = [] nb_target_pkgs_in_queue = nb_skipped_pkgs = 0 for pkg in pkg_dep_graph.keys(): version = None pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix) if pkg in target_pkgs: version = bbs.parse.get_Version_from_pkgsrctree(pkg) cmd = BBSbase.getSTAGE2cmd(pkg, version) nb_target_pkgs_in_queue += 1 else: if pkg in installed_pkgs: cmd = pkgdumps = None nb_skipped_pkgs += 1 else: cmd = BBSbase.get_install_cmd_for_non_target_pkg(pkg) job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps, out_dir) jobs.append(job) nb_jobs = len(jobs) print('OK') sys.stdout.flush() nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \ nb_skipped_pkgs nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs print('BBS> Job summary:') print('BBS> | %d (out of %d) target pkgs are not supporting pkgs' % \ (nb_not_needed, len(target_pkgs))) print('BBS> | => no need to install them') print('BBS> | => they\'re not going in the installation queue') print('BBS> | %d pkgs in the installation queue (all supporting pkgs):' % \ nb_jobs) print('BBS> | o %d are target pkgs' % nb_target_pkgs_in_queue) print('BBS> | => will (re-)install them with \'R CMD INSTALL\'') print('BBS> | o %d are non-target pkgs:' % nb_non_target_pkgs_in_queue) print('BBS> | - %d are already installed' % nb_skipped_pkgs) print('BBS> | => won\'t re-install them (job will be skipped)') print('BBS> | - %d are not already installed' % \ nb_non_target_pkgs_to_install) print('BBS> | => will install them with') print('BBS> | install.packages(pkg, repos=non_target_repos,') print('BBS> | dep=FALSE, ...)') print('BBS> | Total nb of packages to install: %d' % nb_pkgs_to_install) print('BBS>') job_queue = bbs.jobs.JobQueue(stage, jobs, pkg_dep_graph) job_queue._nb_pkgs_to_install = nb_pkgs_to_install return job_queue
def STAGE3_loop(pkgdir_paths, nb_cpu): total = len(pkgdir_paths) job_queue = [] for pkgdir_path in pkgdir_paths: try: pkg = bbs.parse.getPkgFromDir(pkgdir_path) version = bbs.parse.getVersionFromDir(pkgdir_path) srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path) except IOError: print "BBS> Can't read DESCRIPTION file!" else: cmd = BBSbase.getSTAGE3cmd(pkgdir_path) pkgdumps_prefix = pkg + '.buildsrc' pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.buildsrc_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "BBS> BEGIN STAGE3 loop." t0 = time.time() nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE3 loop." print "BBS> -------------------------------------------------------------" print "BBS> STAGE3 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d pkg(s) listed in file BBS_CENTRAL_BASEURL/%s" % (total, BBScorevars.meat_index_file) print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs print "BBS> o %d srcpkg file(s) produced" % nb_products print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def STAGE4_loop(srcpkg_paths, nb_cpu): total = len(srcpkg_paths) job_queue = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE4cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) check_dir = pkg + '.Rcheck' pkgdumps_prefix = pkg + '.checksrc' pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix) job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, BBSvars.checksrc_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "BBS> BEGIN STAGE4 loop." t0 = time.time() nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE4 loop." print "BBS> -------------------------------------------------------------" print "BBS> STAGE4 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d srcpkg file(s) in working dir" % total print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def STAGE2(): print "BBS> [STAGE2] STARTING STAGE2 at %s..." % time.asctime() # We want to make sure the target repo is ready before we actually start # (if it's not ready yet it probably means that the prerun.sh script did # not finish on the main node, in which case we want to wait before we # sync the local meat dir with the central MEAT0 dir). waitForTargetRepoToBeReady() meat_path = BBSvars.meat_path BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True) if BBSvars.MEAT0_type == 2: srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path) for srcpkg_file in srcpkg_files: srcpkg_filepath = os.path.join(meat_path, srcpkg_file) BBSbase.Untar(srcpkg_filepath, meat_path) os.remove(srcpkg_filepath) print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp" STAGE2_tmp = os.path.join(BBSvars.work_topdir, "STAGE2_tmp") bbs.fileutils.remake_dir(STAGE2_tmp) os.chdir(STAGE2_tmp) # Re-create architecture-specific Renviron.site files on multi-arch # build machines. if ('BBS_STAGE2_MODE' in os.environ and os.environ['BBS_STAGE2_MODE'] == 'multiarch' and BBScorevars.mode == "bioc"): CreateREnvironFiles() # Try to update all installed packages CallRfunctionFromSTAGE2Script("updateNonTargetPkgs", "updateNonTargetPkgs1.Rout") # Extract list of target packages. target_pkgs = extractTargetPkgListFromMeatIndex() # Get 'pkg_deps_list' and 'installed_pkgs'. pkg_deps_list = make_STAGE2_pkg_deps_list(target_pkgs) installed_pkgs = get_installed_pkgs() # Then re-install the supporting packages. print "BBS> [STAGE2] cd BBS_MEAT_PATH" os.chdir(BBSvars.meat_path) BBSvars.install_rdir.RemakeMe(True) STAGE2_loop(target_pkgs, pkg_deps_list, installed_pkgs, BBSvars.nb_cpu) print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp" os.chdir(STAGE2_tmp) # Try again to update all installed packages (some updates could have # failed in the previous attempt because of dependency issues). CallRfunctionFromSTAGE2Script("updateNonTargetPkgs", "updateNonTargetPkgs2.Rout") updateNodeInfo() print "BBS> [STAGE2] DONE at %s." % time.asctime() return
def prepare_STAGE4_job_queue(srcpkg_paths, out_dir): print("BBS> Preparing STAGE4 job queue ...", end=" ") sys.stdout.flush() stage = 'checksrc' jobs = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE4cmd(srcpkg_path) pkg = bbs.parse.get_pkgname_from_srcpkg_path(srcpkg_path) version = bbs.parse.get_version_from_srcpkg_path(srcpkg_path) Rcheck_dir = pkg + '.Rcheck' pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(Rcheck_dir, pkgdumps_prefix) job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, out_dir) jobs.append(job) print("OK") sys.stdout.flush() job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(srcpkg_paths) return job_queue
def prepare_STAGE5_job_queue(srcpkg_paths, out_dir): print("BBS> Preparing STAGE5 job queue ...", end=" ") sys.stdout.flush() stage = 'buildbin' jobs = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE5cmd(srcpkg_path) pkg = bbs.parse.get_pkgname_from_srcpkg_path(srcpkg_path) version = bbs.parse.get_version_from_srcpkg_path(srcpkg_path) fileext = BBSutils.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt') binpkg_file = "%s_%s.%s" % (pkg, version, fileext) pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, out_dir) jobs.append(job) print("OK") sys.stdout.flush() job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(srcpkg_paths) return job_queue
def prepare_STAGE4_job_queue(srcpkg_paths): print "BBS> Preparing STAGE4 job queue ... ", stage = 'checksrc' jobs = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE4cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) check_dir = pkg + '.Rcheck' pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix) job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, BBSvars.checksrc_rdir) jobs.append(job) print "OK" job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(srcpkg_paths) return job_queue
def extractSrcPkgTarballs(dest_dir): MEAT0_path = BBSvars.MEAT0_rdir.path # Hopefully this is local! srcpkg_files = bbs.fileutils.listSrcPkgFiles(MEAT0_path) bbs.fileutils.remake_dir(dest_dir) pkgs = [] for srcpkg_file in srcpkg_files: pkg = bbs.parse.getPkgFromPath(srcpkg_file) srcpkg_filepath = os.path.join(MEAT0_path, srcpkg_file) BBSbase.Untar(srcpkg_filepath, dest_dir) pkgs.append(pkg) return pkgs
def prepare_STAGE1_job_queue(pkgdir_paths, dest_rdir): print "BBS> Preparing STAGE1 job queue ... ", stage = 'buildnovig' jobs = [] for pkgdir_path in pkgdir_paths: try: pkg = bbs.parse.getPkgFromDir(pkgdir_path) version = bbs.parse.getVersionFromDir(pkgdir_path) srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path) except IOError: print "BBS> Can't read DESCRIPTION file!" else: cmd = BBSbase.getSTAGE1cmd(pkgdir_path) pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir) jobs.append(job) print "OK" job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(pkgdir_paths) return job_queue
def prepare_STAGE5_job_queue(srcpkg_paths): print "BBS> Preparing STAGE5 job queue ... ", stage = 'buildbin' jobs = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE5cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt') binpkg_file = "%s_%s.%s" % (pkg, version, fileext) pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.buildbin_rdir) jobs.append(job) print "OK" job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(srcpkg_paths) return job_queue
def make_STAGE2_pkg_deps_list(target_pkgs): # Make 'target_pkgs.txt' file. target_pkgs_file = "target_pkgs.txt" out = open(target_pkgs_file, 'w') for pkg in target_pkgs: out.write("%s\n" % pkg) out.close() print "BBS> [make_STAGE2_pkg_deps_list]", print "%s pkgs written to %s" % (len(target_pkgs), target_pkgs_file) # Make 'STAGE2_pkg_deps_list.txt' file. Rfunction = "make_STAGE2_pkg_deps_list" script_path = os.path.join(BBScorevars.BBS_home, "utils", "make_STAGE2_pkg_deps_list.R") STAGE2_pkg_deps_list_path = "STAGE2_pkg_deps_list.txt" print "BBS> [make_STAGE2_pkg_deps_list]", print "Calling %s() defined in %s to make %s file ..." % \ (Rfunction, script_path, STAGE2_pkg_deps_list_path), # Backslashes in the paths injected in 'Rscript' will be seen as escape # characters by R so we need to replace them. Nothing will be replaced # on a Unix-like platform, only on Windows where the paths can actually # contain backslashes. script_path2 = script_path.replace('\\', '/') target_pkgs_file2 = target_pkgs_file.replace('\\', '/') STAGE2_pkg_deps_list_path2 = STAGE2_pkg_deps_list_path.replace('\\', '/') # Use short.list=TRUE to skip installation of target packages not needed # by another target package for build or check. #Rscript = "source('%s');%s('%s',outfile='%s',short.list=TRUE)" % \ Rscript = "source('%s');%s('%s',outfile='%s')" % \ (script_path2, Rfunction, target_pkgs_file2, STAGE2_pkg_deps_list_path2) out_file = Rfunction + ".Rout" bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), out_file) # ignore retcode print "OK" # Load 'STAGE2_pkg_deps_list.txt' file. print "BBS> [make_STAGE2_pkg_deps_list] Loading %s file ..." % \ STAGE2_pkg_deps_list_path, f = open(STAGE2_pkg_deps_list_path, 'r') pkg_deps_list = {} EMPTY_STRING = '' for line in f: (pkg, deps) = line.split(":") deps = deps.strip().split(" ") if EMPTY_STRING in deps: deps.remove(EMPTY_STRING) pkg_deps_list[pkg] = deps f.close() print "OK (%s pkgs and their deps loaded)" % len(pkg_deps_list) print "BBS> [make_STAGE2_pkg_deps_list] DONE." return pkg_deps_list
def prepare_STAGE1_job_queue(pkgsrctrees, dest_rdir): print("BBS> Preparing STAGE1 job queue ... ", end=" ") stage = 'buildnovig' jobs = [] for pkgsrctree in pkgsrctrees: try: pkg = bbs.parse.get_Package_from_pkgsrctree(pkgsrctree) version = bbs.parse.get_Version_from_pkgsrctree(pkgsrctree) srcpkg_file = bbs.parse.make_srcpkg_file_from_pkgsrctree( pkgsrctree) except IOError: print("BBS> Can't read DESCRIPTION file!") else: cmd = BBSbase.getSTAGE1cmd(pkgsrctree) pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir) jobs.append(job) print("OK") job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(pkgsrctrees) return job_queue
def get_installed_pkgs(): installed_pkgs_path = "installed_pkgs.txt" Rscript = "writeLines(rownames(installed.packages()),'%s')" % \ installed_pkgs_path out_file = "get_installed_pkgs.Rout" bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), out_file) # ignore retcode installed_pkgs = [] f = open(installed_pkgs_path, 'r') for line in f: installed_pkgs.append(line.strip()) f.close() print "BBS> [get_installed_pkgs] %s installed pkgs" % len(installed_pkgs) return installed_pkgs
def get_installed_pkgs(): installed_pkgs_path = 'installed_pkgs.txt' Rexpr = "writeLines(rownames(installed.packages()),'%s')" % \ installed_pkgs_path out_file = 'get_installed_pkgs.Rout' bbs.jobs.runJob(BBSbase.Rexpr2syscmd(Rexpr), out_file) # ignore retcode installed_pkgs = [] f = open(installed_pkgs_path, 'r') for line in f: installed_pkgs.append(line.strip()) f.close() print('BBS> [get_installed_pkgs] %s installed pkgs' % len(installed_pkgs)) return installed_pkgs
def CallRfunctionFromSTAGE2Script(Rfunction, out_file=None): print "BBS> [%s] BEGIN ..." % Rfunction script_path = BBSvars.STAGE2_r_script # Backslahes in the paths injected in 'Rscript' will be seen as escape # characters by R so we need to replace them. Nothing will be replaced # on a Unix-like platform, only on Windows where the paths can actually # contain backslahes. script_path = script_path.replace('\\', '/') Rscript = "source('%s');%s()" % (script_path, Rfunction) if out_file == None: out_file = '%s.Rout' % Rfunction bbs.jobs.runJob(BBSbase.Rscript2syscmd(Rscript), out_file, 3600.0) # ignore retcode print "BBS> [%s] END." % Rfunction return
def write_PACKAGES(rdir): doing_what = 'creating PACKAGES index file for target repo' print('BBS> START %s at %s...' % (doing_what, time.asctime())) sys.stdout.flush() Rexpr = r'library(tools);write_PACKAGES(\".\")' bbs.jobs.doOrDie(BBSbase.Rexpr2syscmd(Rexpr)) ## write_PACKAGES() won't create an empty PACKAGES file if no packages ## are found so we create one. if not os.path.exists('PACKAGES'): f = open('PACKAGES', 'w') f.close() rdir.Put('PACKAGES', True, True) print('BBS> DONE %s at %s.' % (doing_what, time.asctime())) sys.stdout.flush() return
def GetCranPkgs(work_topdir): print "BBS> [prerun:get-cran-pkgs] cd BBS_WORK_TOPDIR" os.chdir(work_topdir) pkgs_dir = "pkgs" if os.path.exists(pkgs_dir): print "BBS> [prerun:get-cran-pkgs] rm -r %s" % pkgs_dir bbs.fileutils.nuke_tree(pkgs_dir) print "BBS> [prerun:get-cran-pkgs] Download CRAN packages to BBS_WORK_TOPDIR/pkgs" cmd = "/usr/bin/rsync -ae ssh webadmin@cobra:'/extra/www/cran-mirror/src/contrib/*.tar.gz' %s" % pkgs_dir bbs.jobs.doOrDie(cmd) os.chdir(pkgs_dir) srcpkg_files = bbs.fileutils.listSrcPkgFiles() for srcpkg_file in srcpkg_files: print "BBS> [prerun:get-cran-pkgs] tar zxf %s" % srcpkg_file BBSbase.Untar(srcpkg_file) return
def build_pkg_dep_graph(target_pkgs): # Generate file 'target_pkgs.txt'. target_pkgs_file = 'target_pkgs.txt' out = open(target_pkgs_file, 'w') for pkg in target_pkgs: out.write('%s\n' % pkg) out.close() print('BBS> [build_pkg_dep_graph]', end=' ') print('%s pkgs written to %s' % (len(target_pkgs), target_pkgs_file)) # Generate file 'pkg_dep_graph.txt'. Rfunction = 'build_pkg_dep_graph' script_path = os.path.join(BBSvars.BBS_home, 'utils', 'build_pkg_dep_graph.R') print('BBS> [build_pkg_dep_graph]', end=' ') print('Calling %s() defined in %s to generate file %s ...' % \ (Rfunction, script_path, BBSutils.pkg_dep_graph_file), end=' ') # Backslashes in the paths injected in 'Rexpr' will be seen as escape # characters by R so we need to replace them. Nothing will be replaced # on a Unix-like platform, only on Windows where the paths can actually # contain backslashes. script_path2 = script_path.replace('\\', '/') target_pkgs_file2 = target_pkgs_file.replace('\\', '/') STAGE2_pkg_dep_graph_path2 = BBSutils.pkg_dep_graph_file.replace('\\', '/') # Use short.list=TRUE for "smart STAGE2" i.e. to skip installation of # target packages not needed by another target package for build or check. #Rexpr = "source('%s');%s('%s',outfile='%s',short.list=TRUE)" % \ Rexpr = "source('%s');%s('%s',outfile='%s')" % \ (script_path2, Rfunction, target_pkgs_file2, STAGE2_pkg_dep_graph_path2) out_file = Rfunction + '.Rout' bbs.jobs.runJob(BBSbase.Rexpr2syscmd(Rexpr), out_file) # ignore retcode print('OK') # Send files 'target_pkgs.txt' and 'pkg_dep_graph.txt' to central # build node. BBSvars.Node_rdir.Put(BBSutils.pkg_dep_graph_file, True, True) # Load file 'pkg_dep_graph.txt'. print('BBS> [build_pkg_dep_graph] Loading %s file ...' % \ BBSutils.pkg_dep_graph_file, end=' ') pkg_dep_graph = bbs.parse.load_pkg_dep_graph(BBSutils.pkg_dep_graph_file) print('OK (%s pkgs and their deps loaded)' % len(pkg_dep_graph)) print('BBS> [build_pkg_dep_graph] DONE.') return pkg_dep_graph
def make_PROPAGATION_STATUS_DB(final_repo): Rfunction = 'makePropagationStatusDb' OUTGOING_dir = 'OUTGOING' db_filepath = 'PROPAGATION_STATUS_DB.txt' script_path = os.path.join(BBSvars.BBS_home, "utils", "makePropagationStatusDb.R") Rexpr = "source('%s');%s('%s','%s','%s')" % \ (script_path, Rfunction, OUTGOING_dir, final_repo, db_filepath) cmd = BBSbase.Rexpr2syscmd(Rexpr) ## Nasty things (that I don't really understand) can happen with ## subprocess.run() if this code is runned by the Task Scheduler ## on Windows (the child process tends to almost always return an ## error). Apparently using 'stderr=subprocess.STDOUT' fixes this pb. subprocess.run(cmd, stdout=None, stderr=subprocess.STDOUT, shell=True, check=True) return
def prepare_STAGE1_job_queue(pkgdir_paths, dest_rdir): print "BBS> Preparing STAGE1 job queue ... ", stage = 'buildnovig' jobs = [] for pkgdir_path in pkgdir_paths: try: pkg = bbs.parse.getPkgFromDir(pkgdir_path) version = bbs.parse.getVersionFromDir(pkgdir_path) srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path) except IOError: print "BBS> Can't read DESCRIPTION file!" else: cmd = BBSbase.getSTAGE1cmd(pkgdir_path) pkgdumps_prefix = pkg + '.' + stage; pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir) jobs.append(job) print "OK" job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(pkgdir_paths) return job_queue
def STAGE2(): print "BBS> [STAGE2] STARTING STAGE2 at %s..." % time.asctime() # We want to make sure the target repo is ready before we actually start # (if it's not ready yet it probably means that the prerun.sh script did # not finish on the main node, in which case we want to wait before we # sync the local meat dir with the central MEAT0 dir). waitForTargetRepoToBeReady() meat_path = BBSvars.meat_path BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True) if BBSvars.MEAT0_type == 2: srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path) for srcpkg_file in srcpkg_files: srcpkg_filepath = os.path.join(meat_path, srcpkg_file) BBSbase.Untar(srcpkg_filepath, meat_path) os.remove(srcpkg_filepath) print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/gitlog" gitlog_path = BBSvars.gitlog_path BBSvars.GITLOG_rdir.syncLocalDir(gitlog_path, True) print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp" STAGE2_tmp = os.path.join(BBSvars.work_topdir, "STAGE2_tmp") bbs.fileutils.remake_dir(STAGE2_tmp) os.chdir(STAGE2_tmp) # Re-create architecture-specific Renviron.site files on multi-arch # build machines. if ('BBS_STAGE2_MODE' in os.environ and os.environ['BBS_STAGE2_MODE'] == 'multiarch' and BBScorevars.subbuilds == "bioc"): CreateREnvironFiles() # Try to update all installed packages. CallRfunctionFromSTAGE2Script("updateNonTargetPkgs", "updateNonTargetPkgs1.Rout") # Extract list of target packages. target_pkgs = extractTargetPkgListFromMeatIndex() # Get 'pkg_deps_list' and 'installed_pkgs'. pkg_deps_list = make_STAGE2_pkg_deps_list(target_pkgs) installed_pkgs = get_installed_pkgs() # Inject additional fields into DESCRIPTION. print "BBS> [STAGE2] cd BBS_MEAT_PATH" print "BBS> [STAGE2] Injecting fields into DESCRIPTION" os.chdir(meat_path) for pkg in target_pkgs: gitlog_file = os.path.join(BBSvars.gitlog_path, "git-log-%s.dcf" % pkg) if not os.path.exists(gitlog_file): print "BBS> %s file does not exist --> skipping." % gitlog_file continue desc_file = os.path.join(BBSvars.meat_path, pkg, 'DESCRIPTION') if not os.path.exists(desc_file): print "BBS> %s file does not exist --> skipping." % desc_file continue bbs.parse.injectFieldsInDESCRIPTION(desc_file, gitlog_file) # Then re-install the supporting packages. print "BBS> [STAGE2] Re-install supporting packages" os.chdir(meat_path) BBSvars.install_rdir.RemakeMe(True) job_queue = prepare_STAGE2_job_queue(target_pkgs, pkg_deps_list, installed_pkgs) STAGE2_loop(job_queue, BBSvars.nb_cpu) print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp" os.chdir(STAGE2_tmp) # Try again to update all installed packages (some updates could have # failed in the previous attempt because of dependency issues). CallRfunctionFromSTAGE2Script("updateNonTargetPkgs", "updateNonTargetPkgs2.Rout") print "BBS> [STAGE2] DONE at %s." % time.asctime() return
def STAGE2_loop(target_pkgs, pkg_deps_list, installed_pkgs, nb_cpu): print "BBS> Preparing STAGE2 job queue ...", job_queue = [] nb_target_pkgs_in_queue = nb_skipped_pkgs = 0 for pkg in pkg_deps_list.keys(): version = None pkgdumps_prefix = pkg + '.install' pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix) if pkg in target_pkgs: try: version = bbs.parse.getVersionFromDir(pkg) except IOError: print "BBS> Can't read DESCRIPTION file!" cmd = BBSbase.getSTAGE2cmd(pkg, version) nb_target_pkgs_in_queue += 1 else: if pkg in installed_pkgs: cmd = pkgdumps = None nb_skipped_pkgs += 1 else: cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg) job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.install_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "OK" nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \ nb_skipped_pkgs nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs print "BBS> Job summary:" print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \ (nb_not_needed, len(target_pkgs)) print "BBS> | => no need to install them" print "BBS> | => they're not going in the installation queue" print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \ nb_jobs print "BBS> | o %d are target pkgs" % nb_target_pkgs_in_queue print "BBS> | => will (re-)install them with 'R CMD INSTALL'" print "BBS> | o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue print "BBS> | - %d are already installed" % nb_skipped_pkgs print "BBS> | => won't re-install them (job will be skipped)" print "BBS> | - %d are not already installed" % \ nb_non_target_pkgs_to_install print "BBS> | => will install them with" print "BBS> | install.packages(pkg, repos=non_target_repos," print "BBS> | dep=FALSE, ...)" print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install print "BBS>" print "BBS> BEGIN STAGE2 loop." t0 = time.time() nb_installed = bbs.jobs.processJobQueue(job_queue, pkg_deps_list, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE2 loop." nb_failures = nb_pkgs_to_install - nb_installed print "BBS> -------------------------------------------------------------" print "BBS> STAGE2 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs print "BBS> o %d pkg(s) to (re-)install: %d successes / %d failures" % \ (nb_pkgs_to_install, nb_installed, nb_failures) print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def STAGE2(): print('BBS> [STAGE2] STARTING STAGE2 at %s' % time.asctime()) # We want to make sure the target repo is ready before we actually start # (if it's not ready yet it probably means that the prerun.sh script did # not finish on the main node, in which case we want to wait before we # sync the local meat dir with the central MEAT0 dir). waitForTargetRepoToBeReady() BBSvars.install_rdir.RemakeMe(True) if asynchronous_mode: out_dir = make_stage_out_dir('install') else: out_dir = BBSvars.install_rdir meat_path = BBSvars.meat_path BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True) if BBSvars.MEAT0_type == 2: srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path) for srcpkg_file in srcpkg_files: srcpkg_filepath = os.path.join(meat_path, srcpkg_file) BBSbase.Untar(srcpkg_filepath, meat_path) os.remove(srcpkg_filepath) print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/gitlog') gitlog_path = BBSutils.getenv('BBS_GITLOG_PATH') BBSvars.GITLOG_rdir.syncLocalDir(gitlog_path, True) print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp') STAGE2_tmp = os.path.join(BBSvars.work_topdir, 'STAGE2_tmp') bbs.fileutils.remake_dir(STAGE2_tmp) os.chdir(STAGE2_tmp) # Re-create architecture-specific Renviron.site files on multi-arch # build machines. #if ('BBS_STAGE2_MODE' in os.environ and # os.environ['BBS_STAGE2_MODE'] == 'multiarch' and # BBSvars.buildtype == 'bioc'): # CreateREnvironFiles() if BBSvars.buildtype in ['bioc', 'bioc-testing']: # Update non-target packages. print('BBS> [STAGE2] Update non-target packages (1st run) ...', end=' ') sys.stdout.flush() cmd = BBSbase.get_update_cmd_for_non_target_pkgs() bbs.jobs.runJob(cmd, 'updateNonTargetPkgs1.Rout', 3600.0) print('OK') sys.stdout.flush() # Extract list of target packages. target_pkgs = get_list_of_target_pkgs() # Get 'pkg_dep_graph' and 'installed_pkgs'. pkg_dep_graph = build_pkg_dep_graph(target_pkgs) installed_pkgs = get_installed_pkgs() # Inject additional fields into DESCRIPTION. print('BBS> [STAGE2] cd BBS_MEAT_PATH') print('BBS> [STAGE2] Injecting fields into DESCRIPTION') os.chdir(meat_path) for pkg in target_pkgs: gitlog_file = os.path.join(gitlog_path, 'git-log-%s.dcf' % pkg) if not os.path.exists(gitlog_file): print('BBS> %s file does not exist --> skipping.' % gitlog_file) continue desc_file = os.path.join(BBSvars.meat_path, pkg, 'DESCRIPTION') if not os.path.exists(desc_file): print('BBS> %s file does not exist --> skipping.' % desc_file) continue bbs.parse.injectFieldsInDESCRIPTION(desc_file, gitlog_file) # Then re-install the supporting packages. print('BBS> [STAGE2] Re-install supporting packages') os.chdir(meat_path) job_queue = prepare_STAGE2_job_queue(target_pkgs, pkg_dep_graph, installed_pkgs, out_dir) STAGE2_loop(job_queue, BBSvars.install_nb_cpu, out_dir) print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp') os.chdir(STAGE2_tmp) if BBSvars.buildtype in ['bioc', 'bioc-testing']: # Try again to update non-target packages (some updates could have # failed in the previous attempt because of dependency issues). print('BBS> [STAGE2] Update non-target packages (2nd run) ...', end=' ') sys.stdout.flush() cmd = BBSbase.get_update_cmd_for_non_target_pkgs() bbs.jobs.runJob(cmd, 'updateNonTargetPkgs2.Rout', 3600.0) print('OK') sys.stdout.flush() makeNodeInfo() print('BBS> [STAGE2] DONE at %s.' % time.asctime()) return