def STAGE5_loop(srcpkg_paths, nb_cpu): total = len(srcpkg_paths) job_queue = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE5cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt') binpkg_file = "%s_%s.%s" % (pkg, version, fileext) pkgdumps_prefix = pkg + '.buildbin' pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.buildbin_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "BBS> BEGIN STAGE5 loop." t0 = time.time() nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE5 loop." print "BBS> -------------------------------------------------------------" print "BBS> STAGE5 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d srcpkg file(s) in working dir" % total print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs print "BBS> o %d binpkg file(s) produced" % nb_products print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def STAGE3_loop(pkgdir_paths, nb_cpu): total = len(pkgdir_paths) job_queue = [] for pkgdir_path in pkgdir_paths: try: pkg = bbs.parse.getPkgFromDir(pkgdir_path) version = bbs.parse.getVersionFromDir(pkgdir_path) srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path) except IOError: print "BBS> Can't read DESCRIPTION file!" else: cmd = BBSbase.getSTAGE3cmd(pkgdir_path) pkgdumps_prefix = pkg + '.buildsrc' pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.buildsrc_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "BBS> BEGIN STAGE3 loop." t0 = time.time() nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE3 loop." print "BBS> -------------------------------------------------------------" print "BBS> STAGE3 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d pkg(s) listed in file BBS_CENTRAL_BASEURL/%s" % ( total, BBScorevars.meat_index_file) print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs print "BBS> o %d srcpkg file(s) produced" % nb_products print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def STAGE4_loop(srcpkg_paths, nb_cpu): total = len(srcpkg_paths) job_queue = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE4cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) check_dir = pkg + '.Rcheck' pkgdumps_prefix = pkg + '.checksrc' pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix) job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, BBSvars.checksrc_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "BBS> BEGIN STAGE4 loop." t0 = time.time() bbs.jobs.processJobQueue(job_queue, None, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE4 loop." print "BBS> -------------------------------------------------------------" print "BBS> STAGE4 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d srcpkg file(s) in working dir" % total print "BBS> o %d srcpkg file(s) queued and processed" % nb_jobs print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return
def prepare_STAGE2_job_queue(target_pkgs, pkg_deps_list, installed_pkgs): print "BBS> Preparing STAGE2 job queue ... ", stage = 'install' jobs = [] nb_target_pkgs_in_queue = nb_skipped_pkgs = 0 for pkg in pkg_deps_list.keys(): version = None pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix) if pkg in target_pkgs: try: version = bbs.parse.getVersionFromDir(pkg) except IOError: print "BBS> Can't read %s/DESCRIPTION file!" % pkg cmd = BBSbase.getSTAGE2cmd(pkg, version) nb_target_pkgs_in_queue += 1 else: if pkg in installed_pkgs: cmd = pkgdumps = None nb_skipped_pkgs += 1 else: cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg) job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.install_rdir) jobs.append(job) nb_jobs = len(jobs) print "OK" nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \ nb_skipped_pkgs nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs print "BBS> Job summary:" print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \ (nb_not_needed, len(target_pkgs)) print "BBS> | => no need to install them" print "BBS> | => they're not going in the installation queue" print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \ nb_jobs print "BBS> | o %d are target pkgs" % nb_target_pkgs_in_queue print "BBS> | => will (re-)install them with 'R CMD INSTALL'" print "BBS> | o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue print "BBS> | - %d are already installed" % nb_skipped_pkgs print "BBS> | => won't re-install them (job will be skipped)" print "BBS> | - %d are not already installed" % \ nb_non_target_pkgs_to_install print "BBS> | => will install them with" print "BBS> | install.packages(pkg, repos=non_target_repos," print "BBS> | dep=FALSE, ...)" print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install print "BBS>" job_queue = bbs.jobs.JobQueue(stage, jobs, pkg_deps_list) job_queue._nb_pkgs_to_install = nb_pkgs_to_install return job_queue
def prepare_STAGE2_job_queue(target_pkgs, pkg_dep_graph, installed_pkgs, out_dir): print('BBS> Preparing STAGE2 job queue ...', end=' ') sys.stdout.flush() stage = 'install' jobs = [] nb_target_pkgs_in_queue = nb_skipped_pkgs = 0 for pkg in pkg_dep_graph.keys(): version = None pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix) if pkg in target_pkgs: version = bbs.parse.get_Version_from_pkgsrctree(pkg) cmd = BBSbase.getSTAGE2cmd(pkg, version) nb_target_pkgs_in_queue += 1 else: if pkg in installed_pkgs: cmd = pkgdumps = None nb_skipped_pkgs += 1 else: cmd = BBSbase.get_install_cmd_for_non_target_pkg(pkg) job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps, out_dir) jobs.append(job) nb_jobs = len(jobs) print('OK') sys.stdout.flush() nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \ nb_skipped_pkgs nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs print('BBS> Job summary:') print('BBS> | %d (out of %d) target pkgs are not supporting pkgs' % \ (nb_not_needed, len(target_pkgs))) print('BBS> | => no need to install them') print('BBS> | => they\'re not going in the installation queue') print('BBS> | %d pkgs in the installation queue (all supporting pkgs):' % \ nb_jobs) print('BBS> | o %d are target pkgs' % nb_target_pkgs_in_queue) print('BBS> | => will (re-)install them with \'R CMD INSTALL\'') print('BBS> | o %d are non-target pkgs:' % nb_non_target_pkgs_in_queue) print('BBS> | - %d are already installed' % nb_skipped_pkgs) print('BBS> | => won\'t re-install them (job will be skipped)') print('BBS> | - %d are not already installed' % \ nb_non_target_pkgs_to_install) print('BBS> | => will install them with') print('BBS> | install.packages(pkg, repos=non_target_repos,') print('BBS> | dep=FALSE, ...)') print('BBS> | Total nb of packages to install: %d' % nb_pkgs_to_install) print('BBS>') job_queue = bbs.jobs.JobQueue(stage, jobs, pkg_dep_graph) job_queue._nb_pkgs_to_install = nb_pkgs_to_install return job_queue
def prepare_STAGE4_job_queue(srcpkg_paths, out_dir): print("BBS> Preparing STAGE4 job queue ...", end=" ") sys.stdout.flush() stage = 'checksrc' jobs = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE4cmd(srcpkg_path) pkg = bbs.parse.get_pkgname_from_srcpkg_path(srcpkg_path) version = bbs.parse.get_version_from_srcpkg_path(srcpkg_path) Rcheck_dir = pkg + '.Rcheck' pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(Rcheck_dir, pkgdumps_prefix) job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, out_dir) jobs.append(job) print("OK") sys.stdout.flush() job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(srcpkg_paths) return job_queue
def prepare_STAGE4_job_queue(srcpkg_paths): print "BBS> Preparing STAGE4 job queue ... ", stage = 'checksrc' jobs = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE4cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) check_dir = pkg + '.Rcheck' pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix) job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, BBSvars.checksrc_rdir) jobs.append(job) print "OK" job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(srcpkg_paths) return job_queue
def prepare_STAGE5_job_queue(srcpkg_paths, out_dir): print("BBS> Preparing STAGE5 job queue ...", end=" ") sys.stdout.flush() stage = 'buildbin' jobs = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE5cmd(srcpkg_path) pkg = bbs.parse.get_pkgname_from_srcpkg_path(srcpkg_path) version = bbs.parse.get_version_from_srcpkg_path(srcpkg_path) fileext = BBSutils.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt') binpkg_file = "%s_%s.%s" % (pkg, version, fileext) pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, out_dir) jobs.append(job) print("OK") sys.stdout.flush() job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(srcpkg_paths) return job_queue
def prepare_STAGE1_job_queue(pkgdir_paths, dest_rdir): print "BBS> Preparing STAGE1 job queue ... ", stage = 'buildnovig' jobs = [] for pkgdir_path in pkgdir_paths: try: pkg = bbs.parse.getPkgFromDir(pkgdir_path) version = bbs.parse.getVersionFromDir(pkgdir_path) srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path) except IOError: print "BBS> Can't read DESCRIPTION file!" else: cmd = BBSbase.getSTAGE1cmd(pkgdir_path) pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir) jobs.append(job) print "OK" job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(pkgdir_paths) return job_queue
def prepare_STAGE5_job_queue(srcpkg_paths): print "BBS> Preparing STAGE5 job queue ... ", stage = 'buildbin' jobs = [] for srcpkg_path in srcpkg_paths: cmd = BBSbase.getSTAGE5cmd(srcpkg_path) if cmd == None: continue pkg = bbs.parse.getPkgFromPath(srcpkg_path) version = bbs.parse.getVersionFromPath(srcpkg_path) fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt') binpkg_file = "%s_%s.%s" % (pkg, version, fileext) pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.buildbin_rdir) jobs.append(job) print "OK" job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(srcpkg_paths) return job_queue
def prepare_STAGE1_job_queue(pkgsrctrees, dest_rdir): print("BBS> Preparing STAGE1 job queue ... ", end=" ") stage = 'buildnovig' jobs = [] for pkgsrctree in pkgsrctrees: try: pkg = bbs.parse.get_Package_from_pkgsrctree(pkgsrctree) version = bbs.parse.get_Version_from_pkgsrctree(pkgsrctree) srcpkg_file = bbs.parse.make_srcpkg_file_from_pkgsrctree( pkgsrctree) except IOError: print("BBS> Can't read DESCRIPTION file!") else: cmd = BBSbase.getSTAGE1cmd(pkgsrctree) pkgdumps_prefix = pkg + '.' + stage pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix) job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir) jobs.append(job) print("OK") job_queue = bbs.jobs.JobQueue(stage, jobs, None) job_queue._total = len(pkgsrctrees) return job_queue
def STAGE2_loop(target_pkgs, pkg_deps_list, installed_pkgs, nb_cpu): print "BBS> Preparing STAGE2 job queue ...", job_queue = [] nb_target_pkgs_in_queue = nb_skipped_pkgs = 0 for pkg in pkg_deps_list.keys(): version = None pkgdumps_prefix = pkg + '.install' pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix) if pkg in target_pkgs: try: version = bbs.parse.getVersionFromDir(pkg) except IOError: print "BBS> Can't read DESCRIPTION file!" cmd = BBSbase.getSTAGE2cmd(pkg, version) nb_target_pkgs_in_queue += 1 else: if pkg in installed_pkgs: cmd = pkgdumps = None nb_skipped_pkgs += 1 else: cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg) job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps, BBSvars.install_rdir) job_queue.append(job) nb_jobs = len(job_queue) print "OK" nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \ nb_skipped_pkgs nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs print "BBS> Job summary:" print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \ (nb_not_needed, len(target_pkgs)) print "BBS> | => no need to install them" print "BBS> | => they're not going in the installation queue" print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \ nb_jobs print "BBS> | o %d are target pkgs" % nb_target_pkgs_in_queue print "BBS> | => will (re-)install them with 'R CMD INSTALL'" print "BBS> | o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue print "BBS> | - %d are already installed" % nb_skipped_pkgs print "BBS> | => won't re-install them (job will be skipped)" print "BBS> | - %d are not already installed" % \ nb_non_target_pkgs_to_install print "BBS> | => will install them with" print "BBS> | install.packages(pkg, repos=non_target_repos," print "BBS> | dep=FALSE, ...)" print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install print "BBS>" print "BBS> BEGIN STAGE2 loop." t0 = time.time() nb_installed = bbs.jobs.processJobQueue(job_queue, pkg_deps_list, nb_cpu, BBScorevars.r_cmd_timeout, True) dt = time.time() - t0 print "BBS> END STAGE2 loop." nb_failures = nb_pkgs_to_install - nb_installed print "BBS> -------------------------------------------------------------" print "BBS> STAGE2 SUMMARY:" print "BBS> o Working dir: %s" % os.getcwd() print "BBS> o %d pkg dir(s) queued and processed" % nb_jobs print "BBS> o %d pkg(s) to (re-)install: %d successes / %d failures" % \ (nb_pkgs_to_install, nb_installed, nb_failures) print "BBS> o Total time: %.2f seconds" % dt print "BBS> -------------------------------------------------------------" return