Exemplo n.º 1
0
def STAGE5_loop(srcpkg_paths, nb_cpu):
    total = len(srcpkg_paths)
    job_queue = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
        if cmd == None:
            continue
        pkg = bbs.parse.getPkgFromPath(srcpkg_path)
        version = bbs.parse.getVersionFromPath(srcpkg_path)
        fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
        binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkgdumps_prefix = pkg + '.buildbin'
        pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
        job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps,
                                   BBSvars.buildbin_rdir)
        job_queue.append(job)
    nb_jobs = len(job_queue)
    print "BBS> BEGIN STAGE5 loop."
    t0 = time.time()
    nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
                                           BBScorevars.r_cmd_timeout, True)
    dt = time.time() - t0
    print "BBS> END STAGE5 loop."
    print "BBS> -------------------------------------------------------------"
    print "BBS> STAGE5 SUMMARY:"
    print "BBS>   o Working dir: %s" % os.getcwd()
    print "BBS>   o %d srcpkg file(s) in working dir" % total
    print "BBS>   o %d srcpkg file(s) queued and processed" % nb_jobs
    print "BBS>   o %d binpkg file(s) produced" % nb_products
    print "BBS>   o Total time: %.2f seconds" % dt
    print "BBS> -------------------------------------------------------------"
    return
Exemplo n.º 2
0
def STAGE3_loop(pkgdir_paths, nb_cpu):
    total = len(pkgdir_paths)
    job_queue = []
    for pkgdir_path in pkgdir_paths:
        try:
            pkg = bbs.parse.getPkgFromDir(pkgdir_path)
            version = bbs.parse.getVersionFromDir(pkgdir_path)
            srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path)
        except IOError:
            print "BBS>   Can't read DESCRIPTION file!"
        else:
            cmd = BBSbase.getSTAGE3cmd(pkgdir_path)
            pkgdumps_prefix = pkg + '.buildsrc'
            pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix)
            job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps,
                                       BBSvars.buildsrc_rdir)
            job_queue.append(job)
    nb_jobs = len(job_queue)
    print "BBS> BEGIN STAGE3 loop."
    t0 = time.time()
    nb_products = bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
                                           BBScorevars.r_cmd_timeout, True)
    dt = time.time() - t0
    print "BBS> END STAGE3 loop."
    print "BBS> -------------------------------------------------------------"
    print "BBS> STAGE3 SUMMARY:"
    print "BBS>   o Working dir: %s" % os.getcwd()
    print "BBS>   o %d pkg(s) listed in file BBS_CENTRAL_BASEURL/%s" % (
        total, BBScorevars.meat_index_file)
    print "BBS>   o %d pkg dir(s) queued and processed" % nb_jobs
    print "BBS>   o %d srcpkg file(s) produced" % nb_products
    print "BBS>   o Total time: %.2f seconds" % dt
    print "BBS> -------------------------------------------------------------"
    return
Exemplo n.º 3
0
def STAGE4_loop(srcpkg_paths, nb_cpu):
    total = len(srcpkg_paths)
    job_queue = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE4cmd(srcpkg_path)
        if cmd == None:
            continue
        pkg = bbs.parse.getPkgFromPath(srcpkg_path)
        version = bbs.parse.getVersionFromPath(srcpkg_path)
        check_dir = pkg + '.Rcheck'
        pkgdumps_prefix = pkg + '.checksrc'
        pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix)
        job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps,
                                   BBSvars.checksrc_rdir)
        job_queue.append(job)
    nb_jobs = len(job_queue)
    print "BBS> BEGIN STAGE4 loop."
    t0 = time.time()
    bbs.jobs.processJobQueue(job_queue, None, nb_cpu,
                             BBScorevars.r_cmd_timeout, True)
    dt = time.time() - t0
    print "BBS> END STAGE4 loop."
    print "BBS> -------------------------------------------------------------"
    print "BBS> STAGE4 SUMMARY:"
    print "BBS>   o Working dir: %s" % os.getcwd()
    print "BBS>   o %d srcpkg file(s) in working dir" % total
    print "BBS>   o %d srcpkg file(s) queued and processed" % nb_jobs
    print "BBS>   o Total time: %.2f seconds" % dt
    print "BBS> -------------------------------------------------------------"
    return
Exemplo n.º 4
0
Arquivo: BBS-run.py Projeto: kant/BBS
def prepare_STAGE2_job_queue(target_pkgs, pkg_deps_list, installed_pkgs):
    print "BBS> Preparing STAGE2 job queue ... ",
    stage = 'install'
    jobs = []
    nb_target_pkgs_in_queue = nb_skipped_pkgs = 0
    for pkg in pkg_deps_list.keys():
        version = None
        pkgdumps_prefix = pkg + '.' + stage
        pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix)
        if pkg in target_pkgs:
            try:
                version = bbs.parse.getVersionFromDir(pkg)
            except IOError:
                print "BBS>   Can't read %s/DESCRIPTION file!" % pkg
            cmd = BBSbase.getSTAGE2cmd(pkg, version)
            nb_target_pkgs_in_queue += 1
        else:
            if pkg in installed_pkgs:
                cmd = pkgdumps = None
                nb_skipped_pkgs += 1
            else:
                cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg)
        job = BBSbase.InstallPkg_Job(pkg, version, cmd,
                                     pkgdumps, BBSvars.install_rdir)
        jobs.append(job)
    nb_jobs = len(jobs)
    print "OK"
    nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue
    nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue
    nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \
                                    nb_skipped_pkgs
    nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs
    print "BBS> Job summary:"
    print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \
          (nb_not_needed, len(target_pkgs))
    print "BBS> |   => no need to install them"
    print "BBS> |   => they're not going in the installation queue"
    print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \
          nb_jobs
    print "BBS> |   o %d are target pkgs" % nb_target_pkgs_in_queue
    print "BBS> |       => will (re-)install them with 'R CMD INSTALL'"
    print "BBS> |   o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue
    print "BBS> |     - %d are already installed" % nb_skipped_pkgs
    print "BBS> |         => won't re-install them (job will be skipped)"
    print "BBS> |     - %d are not already installed" % \
          nb_non_target_pkgs_to_install
    print "BBS> |         => will install them with"
    print "BBS> |              install.packages(pkg, repos=non_target_repos,"
    print "BBS> |                               dep=FALSE, ...)"
    print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install
    print "BBS>"
    job_queue = bbs.jobs.JobQueue(stage, jobs, pkg_deps_list)
    job_queue._nb_pkgs_to_install = nb_pkgs_to_install
    return job_queue
Exemplo n.º 5
0
def prepare_STAGE2_job_queue(target_pkgs, pkg_dep_graph, installed_pkgs,
                             out_dir):
    print('BBS> Preparing STAGE2 job queue ...', end=' ')
    sys.stdout.flush()
    stage = 'install'
    jobs = []
    nb_target_pkgs_in_queue = nb_skipped_pkgs = 0
    for pkg in pkg_dep_graph.keys():
        version = None
        pkgdumps_prefix = pkg + '.' + stage
        pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix)
        if pkg in target_pkgs:
            version = bbs.parse.get_Version_from_pkgsrctree(pkg)
            cmd = BBSbase.getSTAGE2cmd(pkg, version)
            nb_target_pkgs_in_queue += 1
        else:
            if pkg in installed_pkgs:
                cmd = pkgdumps = None
                nb_skipped_pkgs += 1
            else:
                cmd = BBSbase.get_install_cmd_for_non_target_pkg(pkg)
        job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps, out_dir)
        jobs.append(job)
    nb_jobs = len(jobs)
    print('OK')
    sys.stdout.flush()
    nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue
    nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue
    nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \
                                    nb_skipped_pkgs
    nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs
    print('BBS> Job summary:')
    print('BBS> | %d (out of %d) target pkgs are not supporting pkgs' % \
          (nb_not_needed, len(target_pkgs)))
    print('BBS> |   => no need to install them')
    print('BBS> |   => they\'re not going in the installation queue')
    print('BBS> | %d pkgs in the installation queue (all supporting pkgs):' % \
          nb_jobs)
    print('BBS> |   o %d are target pkgs' % nb_target_pkgs_in_queue)
    print('BBS> |       => will (re-)install them with \'R CMD INSTALL\'')
    print('BBS> |   o %d are non-target pkgs:' % nb_non_target_pkgs_in_queue)
    print('BBS> |     - %d are already installed' % nb_skipped_pkgs)
    print('BBS> |         => won\'t re-install them (job will be skipped)')
    print('BBS> |     - %d are not already installed' % \
          nb_non_target_pkgs_to_install)
    print('BBS> |         => will install them with')
    print('BBS> |              install.packages(pkg, repos=non_target_repos,')
    print('BBS> |                               dep=FALSE, ...)')
    print('BBS> | Total nb of packages to install: %d' % nb_pkgs_to_install)
    print('BBS>')
    job_queue = bbs.jobs.JobQueue(stage, jobs, pkg_dep_graph)
    job_queue._nb_pkgs_to_install = nb_pkgs_to_install
    return job_queue
Exemplo n.º 6
0
def prepare_STAGE4_job_queue(srcpkg_paths, out_dir):
    print("BBS> Preparing STAGE4 job queue ...", end=" ")
    sys.stdout.flush()
    stage = 'checksrc'
    jobs = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE4cmd(srcpkg_path)
        pkg = bbs.parse.get_pkgname_from_srcpkg_path(srcpkg_path)
        version = bbs.parse.get_version_from_srcpkg_path(srcpkg_path)
        Rcheck_dir = pkg + '.Rcheck'
        pkgdumps_prefix = pkg + '.' + stage
        pkgdumps = BBSbase.PkgDumps(Rcheck_dir, pkgdumps_prefix)
        job = BBSbase.CheckSrc_Job(pkg, version, cmd, pkgdumps, out_dir)
        jobs.append(job)
    print("OK")
    sys.stdout.flush()
    job_queue = bbs.jobs.JobQueue(stage, jobs, None)
    job_queue._total = len(srcpkg_paths)
    return job_queue
Exemplo n.º 7
0
Arquivo: BBS-run.py Projeto: kant/BBS
def prepare_STAGE4_job_queue(srcpkg_paths):
    print "BBS> Preparing STAGE4 job queue ... ",
    stage = 'checksrc'
    jobs = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE4cmd(srcpkg_path)
        if cmd == None:
            continue
        pkg = bbs.parse.getPkgFromPath(srcpkg_path)
        version = bbs.parse.getVersionFromPath(srcpkg_path)
        check_dir = pkg + '.Rcheck'
        pkgdumps_prefix = pkg + '.' + stage
        pkgdumps = BBSbase.PkgDumps(check_dir, pkgdumps_prefix)
        job = BBSbase.CheckSrc_Job(pkg, version, cmd,
                                   pkgdumps, BBSvars.checksrc_rdir)
        jobs.append(job)
    print "OK"
    job_queue = bbs.jobs.JobQueue(stage, jobs, None)
    job_queue._total = len(srcpkg_paths)
    return job_queue
Exemplo n.º 8
0
def prepare_STAGE5_job_queue(srcpkg_paths, out_dir):
    print("BBS> Preparing STAGE5 job queue ...", end=" ")
    sys.stdout.flush()
    stage = 'buildbin'
    jobs = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
        pkg = bbs.parse.get_pkgname_from_srcpkg_path(srcpkg_path)
        version = bbs.parse.get_version_from_srcpkg_path(srcpkg_path)
        fileext = BBSutils.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
        binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkgdumps_prefix = pkg + '.' + stage
        pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
        job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, out_dir)
        jobs.append(job)
    print("OK")
    sys.stdout.flush()
    job_queue = bbs.jobs.JobQueue(stage, jobs, None)
    job_queue._total = len(srcpkg_paths)
    return job_queue
Exemplo n.º 9
0
def prepare_STAGE1_job_queue(pkgdir_paths, dest_rdir):
    print "BBS> Preparing STAGE1 job queue ... ",
    stage = 'buildnovig'
    jobs = []
    for pkgdir_path in pkgdir_paths:
        try:
            pkg = bbs.parse.getPkgFromDir(pkgdir_path)
            version = bbs.parse.getVersionFromDir(pkgdir_path)
            srcpkg_file = bbs.parse.getSrcPkgFileFromDir(pkgdir_path)
        except IOError:
            print "BBS>   Can't read DESCRIPTION file!"
        else:
            cmd = BBSbase.getSTAGE1cmd(pkgdir_path)
            pkgdumps_prefix = pkg + '.' + stage
            pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix)
            job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir)
            jobs.append(job)
    print "OK"
    job_queue = bbs.jobs.JobQueue(stage, jobs, None)
    job_queue._total = len(pkgdir_paths)
    return job_queue
Exemplo n.º 10
0
Arquivo: BBS-run.py Projeto: kant/BBS
def prepare_STAGE5_job_queue(srcpkg_paths):
    print "BBS> Preparing STAGE5 job queue ... ",
    stage = 'buildbin'
    jobs = []
    for srcpkg_path in srcpkg_paths:
        cmd = BBSbase.getSTAGE5cmd(srcpkg_path)
        if cmd == None:
            continue
        pkg = bbs.parse.getPkgFromPath(srcpkg_path)
        version = bbs.parse.getVersionFromPath(srcpkg_path)
        fileext = BBScorevars.getNodeSpec(BBSvars.node_hostname, 'pkgFileExt')
        binpkg_file = "%s_%s.%s" % (pkg, version, fileext)
        pkgdumps_prefix = pkg + '.' + stage
        pkgdumps = BBSbase.PkgDumps(binpkg_file, pkgdumps_prefix)
        job = BBSbase.BuildPkg_Job(pkg, version, cmd,
                                   pkgdumps, BBSvars.buildbin_rdir)
        jobs.append(job)
    print "OK"
    job_queue = bbs.jobs.JobQueue(stage, jobs, None)
    job_queue._total = len(srcpkg_paths)
    return job_queue
Exemplo n.º 11
0
def prepare_STAGE1_job_queue(pkgsrctrees, dest_rdir):
    print("BBS> Preparing STAGE1 job queue ... ", end=" ")
    stage = 'buildnovig'
    jobs = []
    for pkgsrctree in pkgsrctrees:
        try:
            pkg = bbs.parse.get_Package_from_pkgsrctree(pkgsrctree)
            version = bbs.parse.get_Version_from_pkgsrctree(pkgsrctree)
            srcpkg_file = bbs.parse.make_srcpkg_file_from_pkgsrctree(
                pkgsrctree)
        except IOError:
            print("BBS>   Can't read DESCRIPTION file!")
        else:
            cmd = BBSbase.getSTAGE1cmd(pkgsrctree)
            pkgdumps_prefix = pkg + '.' + stage
            pkgdumps = BBSbase.PkgDumps(srcpkg_file, pkgdumps_prefix)
            job = BBSbase.BuildPkg_Job(pkg, version, cmd, pkgdumps, dest_rdir)
            jobs.append(job)
    print("OK")
    job_queue = bbs.jobs.JobQueue(stage, jobs, None)
    job_queue._total = len(pkgsrctrees)
    return job_queue
Exemplo n.º 12
0
def STAGE2_loop(target_pkgs, pkg_deps_list, installed_pkgs, nb_cpu):
    print "BBS> Preparing STAGE2 job queue ...",
    job_queue = []
    nb_target_pkgs_in_queue = nb_skipped_pkgs = 0
    for pkg in pkg_deps_list.keys():
        version = None
        pkgdumps_prefix = pkg + '.install'
        pkgdumps = BBSbase.PkgDumps(None, pkgdumps_prefix)
        if pkg in target_pkgs:
            try:
                version = bbs.parse.getVersionFromDir(pkg)
            except IOError:
                print "BBS>   Can't read DESCRIPTION file!"
            cmd = BBSbase.getSTAGE2cmd(pkg, version)
            nb_target_pkgs_in_queue += 1
        else:
            if pkg in installed_pkgs:
                cmd = pkgdumps = None
                nb_skipped_pkgs += 1
            else:
                cmd = BBSbase.getSTAGE2cmdForNonTargetPkg(pkg)
        job = BBSbase.InstallPkg_Job(pkg, version, cmd, pkgdumps,
                                     BBSvars.install_rdir)
        job_queue.append(job)
    nb_jobs = len(job_queue)
    print "OK"

    nb_not_needed = len(target_pkgs) - nb_target_pkgs_in_queue
    nb_non_target_pkgs_in_queue = nb_jobs - nb_target_pkgs_in_queue
    nb_non_target_pkgs_to_install = nb_non_target_pkgs_in_queue - \
                                    nb_skipped_pkgs
    nb_pkgs_to_install = nb_jobs - nb_skipped_pkgs
    print "BBS> Job summary:"
    print "BBS> | %d (out of %d) target pkgs are not supporting pkgs" % \
          (nb_not_needed, len(target_pkgs))
    print "BBS> |   => no need to install them"
    print "BBS> |   => they're not going in the installation queue"
    print "BBS> | %d pkgs in the installation queue (all supporting pkgs):" % \
          nb_jobs
    print "BBS> |   o %d are target pkgs" % nb_target_pkgs_in_queue
    print "BBS> |       => will (re-)install them with 'R CMD INSTALL'"
    print "BBS> |   o %d are non-target pkgs:" % nb_non_target_pkgs_in_queue
    print "BBS> |     - %d are already installed" % nb_skipped_pkgs
    print "BBS> |         => won't re-install them (job will be skipped)"
    print "BBS> |     - %d are not already installed" % \
          nb_non_target_pkgs_to_install
    print "BBS> |         => will install them with"
    print "BBS> |              install.packages(pkg, repos=non_target_repos,"
    print "BBS> |                               dep=FALSE, ...)"
    print "BBS> | Total nb of packages to install: %d" % nb_pkgs_to_install
    print "BBS>"
    print "BBS> BEGIN STAGE2 loop."
    t0 = time.time()
    nb_installed = bbs.jobs.processJobQueue(job_queue, pkg_deps_list, nb_cpu,
                                            BBScorevars.r_cmd_timeout, True)
    dt = time.time() - t0
    print "BBS> END STAGE2 loop."
    nb_failures = nb_pkgs_to_install - nb_installed
    print "BBS> -------------------------------------------------------------"
    print "BBS> STAGE2 SUMMARY:"
    print "BBS>   o Working dir: %s" % os.getcwd()
    print "BBS>   o %d pkg dir(s) queued and processed" % nb_jobs
    print "BBS>   o %d pkg(s) to (re-)install: %d successes / %d failures" % \
          (nb_pkgs_to_install, nb_installed, nb_failures)
    print "BBS>   o Total time: %.2f seconds" % dt
    print "BBS> -------------------------------------------------------------"
    return