def STAGE2():
print "BBS> [STAGE2] STARTING STAGE2 at %s..." % time.asctime()
# We want to make sure the target repo is ready before we actually start
# (if it's not ready yet it probably means that the prerun.sh script did
# not finish on the main node, in which case we want to wait before we
# sync the local meat dir with the central MEAT0 dir).
waitForTargetRepoToBeReady()
meat_path = BBSvars.meat_path
BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True)
if BBSvars.MEAT0_type == 2:
srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path)
for srcpkg_file in srcpkg_files:
srcpkg_filepath = os.path.join(meat_path, srcpkg_file)
BBSbase.Untar(srcpkg_filepath, meat_path)
os.remove(srcpkg_filepath)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp"
STAGE2_tmp = os.path.join(BBSvars.work_topdir, "STAGE2_tmp")
bbs.fileutils.remake_dir(STAGE2_tmp)
os.chdir(STAGE2_tmp)
# Re-create architecture-specific Renviron.site files on multi-arch
# build machines.
if ('BBS_STAGE2_MODE' in os.environ
and os.environ['BBS_STAGE2_MODE'] == 'multiarch'
and BBScorevars.mode == "bioc"):
CreateREnvironFiles()
# Try to update all installed packages
CallRfunctionFromSTAGE2Script("updateNonTargetPkgs",
"updateNonTargetPkgs1.Rout")
# Extract list of target packages.
target_pkgs = extractTargetPkgListFromMeatIndex()
# Get 'pkg_deps_list' and 'installed_pkgs'.
pkg_deps_list = make_STAGE2_pkg_deps_list(target_pkgs)
installed_pkgs = get_installed_pkgs()
# Then re-install the supporting packages.
print "BBS> [STAGE2] cd BBS_MEAT_PATH"
os.chdir(BBSvars.meat_path)
BBSvars.install_rdir.RemakeMe(True)
STAGE2_loop(target_pkgs, pkg_deps_list, installed_pkgs, BBSvars.nb_cpu)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp"
os.chdir(STAGE2_tmp)
# Try again to update all installed packages (some updates could have
# failed in the previous attempt because of dependency issues).
CallRfunctionFromSTAGE2Script("updateNonTargetPkgs",
"updateNonTargetPkgs2.Rout")
updateNodeInfo()
print "BBS> [STAGE2] DONE at %s." % time.asctime()
return
def extractSrcPkgTarballs(dest_dir):
MEAT0_path = BBSvars.MEAT0_rdir.path # Hopefully this is local!
srcpkg_files = bbs.fileutils.listSrcPkgFiles(MEAT0_path)
bbs.fileutils.remake_dir(dest_dir)
pkgs = []
for srcpkg_file in srcpkg_files:
pkg = bbs.parse.getPkgFromPath(srcpkg_file)
srcpkg_filepath = os.path.join(MEAT0_path, srcpkg_file)
BBSbase.Untar(srcpkg_filepath, dest_dir)
pkgs.append(pkg)
return pkgs
def GetCranPkgs(work_topdir):
print "BBS> [prerun:get-cran-pkgs] cd BBS_WORK_TOPDIR"
os.chdir(work_topdir)
pkgs_dir = "pkgs"
if os.path.exists(pkgs_dir):
print "BBS> [prerun:get-cran-pkgs] rm -r %s" % pkgs_dir
bbs.fileutils.nuke_tree(pkgs_dir)
print "BBS> [prerun:get-cran-pkgs] Download CRAN packages to BBS_WORK_TOPDIR/pkgs"
cmd = "/usr/bin/rsync -ae ssh webadmin@cobra:'/extra/www/cran-mirror/src/contrib/*.tar.gz' %s" % pkgs_dir
bbs.jobs.doOrDie(cmd)
os.chdir(pkgs_dir)
srcpkg_files = bbs.fileutils.listSrcPkgFiles()
for srcpkg_file in srcpkg_files:
print "BBS> [prerun:get-cran-pkgs] tar zxf %s" % srcpkg_file
BBSbase.Untar(srcpkg_file)
return
def STAGE2():
print "BBS> [STAGE2] STARTING STAGE2 at %s..." % time.asctime()
# We want to make sure the target repo is ready before we actually start
# (if it's not ready yet it probably means that the prerun.sh script did
# not finish on the main node, in which case we want to wait before we
# sync the local meat dir with the central MEAT0 dir).
waitForTargetRepoToBeReady()
meat_path = BBSvars.meat_path
BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True)
if BBSvars.MEAT0_type == 2:
srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path)
for srcpkg_file in srcpkg_files:
srcpkg_filepath = os.path.join(meat_path, srcpkg_file)
BBSbase.Untar(srcpkg_filepath, meat_path)
os.remove(srcpkg_filepath)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/gitlog"
gitlog_path = BBSvars.gitlog_path
BBSvars.GITLOG_rdir.syncLocalDir(gitlog_path, True)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp"
STAGE2_tmp = os.path.join(BBSvars.work_topdir, "STAGE2_tmp")
bbs.fileutils.remake_dir(STAGE2_tmp)
os.chdir(STAGE2_tmp)
# Re-create architecture-specific Renviron.site files on multi-arch
# build machines.
if ('BBS_STAGE2_MODE' in os.environ and
os.environ['BBS_STAGE2_MODE'] == 'multiarch' and
BBScorevars.subbuilds == "bioc"):
CreateREnvironFiles()
# Try to update all installed packages.
CallRfunctionFromSTAGE2Script("updateNonTargetPkgs",
"updateNonTargetPkgs1.Rout")
# Extract list of target packages.
target_pkgs = extractTargetPkgListFromMeatIndex()
# Get 'pkg_deps_list' and 'installed_pkgs'.
pkg_deps_list = make_STAGE2_pkg_deps_list(target_pkgs)
installed_pkgs = get_installed_pkgs()
# Inject additional fields into DESCRIPTION.
print "BBS> [STAGE2] cd BBS_MEAT_PATH"
print "BBS> [STAGE2] Injecting fields into DESCRIPTION"
os.chdir(meat_path)
for pkg in target_pkgs:
gitlog_file = os.path.join(BBSvars.gitlog_path, "git-log-%s.dcf" % pkg)
if not os.path.exists(gitlog_file):
print "BBS> %s file does not exist --> skipping." % gitlog_file
continue
desc_file = os.path.join(BBSvars.meat_path, pkg, 'DESCRIPTION')
if not os.path.exists(desc_file):
print "BBS> %s file does not exist --> skipping." % desc_file
continue
bbs.parse.injectFieldsInDESCRIPTION(desc_file, gitlog_file)
# Then re-install the supporting packages.
print "BBS> [STAGE2] Re-install supporting packages"
os.chdir(meat_path)
BBSvars.install_rdir.RemakeMe(True)
job_queue = prepare_STAGE2_job_queue(target_pkgs, pkg_deps_list,
installed_pkgs)
STAGE2_loop(job_queue, BBSvars.nb_cpu)
print "BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp"
os.chdir(STAGE2_tmp)
# Try again to update all installed packages (some updates could have
# failed in the previous attempt because of dependency issues).
CallRfunctionFromSTAGE2Script("updateNonTargetPkgs",
"updateNonTargetPkgs2.Rout")
print "BBS> [STAGE2] DONE at %s." % time.asctime()
return
def STAGE2():
print('BBS> [STAGE2] STARTING STAGE2 at %s' % time.asctime())
# We want to make sure the target repo is ready before we actually start
# (if it's not ready yet it probably means that the prerun.sh script did
# not finish on the main node, in which case we want to wait before we
# sync the local meat dir with the central MEAT0 dir).
waitForTargetRepoToBeReady()
BBSvars.install_rdir.RemakeMe(True)
if asynchronous_mode:
out_dir = make_stage_out_dir('install')
else:
out_dir = BBSvars.install_rdir
meat_path = BBSvars.meat_path
BBSvars.MEAT0_rdir.syncLocalDir(meat_path, True)
if BBSvars.MEAT0_type == 2:
srcpkg_files = bbs.fileutils.listSrcPkgFiles(meat_path)
for srcpkg_file in srcpkg_files:
srcpkg_filepath = os.path.join(meat_path, srcpkg_file)
BBSbase.Untar(srcpkg_filepath, meat_path)
os.remove(srcpkg_filepath)
print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/gitlog')
gitlog_path = BBSutils.getenv('BBS_GITLOG_PATH')
BBSvars.GITLOG_rdir.syncLocalDir(gitlog_path, True)
print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp')
STAGE2_tmp = os.path.join(BBSvars.work_topdir, 'STAGE2_tmp')
bbs.fileutils.remake_dir(STAGE2_tmp)
os.chdir(STAGE2_tmp)
# Re-create architecture-specific Renviron.site files on multi-arch
# build machines.
#if ('BBS_STAGE2_MODE' in os.environ and
# os.environ['BBS_STAGE2_MODE'] == 'multiarch' and
# BBSvars.buildtype == 'bioc'):
# CreateREnvironFiles()
if BBSvars.buildtype in ['bioc', 'bioc-testing']:
# Update non-target packages.
print('BBS> [STAGE2] Update non-target packages (1st run) ...',
end=' ')
sys.stdout.flush()
cmd = BBSbase.get_update_cmd_for_non_target_pkgs()
bbs.jobs.runJob(cmd, 'updateNonTargetPkgs1.Rout', 3600.0)
print('OK')
sys.stdout.flush()
# Extract list of target packages.
target_pkgs = get_list_of_target_pkgs()
# Get 'pkg_dep_graph' and 'installed_pkgs'.
pkg_dep_graph = build_pkg_dep_graph(target_pkgs)
installed_pkgs = get_installed_pkgs()
# Inject additional fields into DESCRIPTION.
print('BBS> [STAGE2] cd BBS_MEAT_PATH')
print('BBS> [STAGE2] Injecting fields into DESCRIPTION')
os.chdir(meat_path)
for pkg in target_pkgs:
gitlog_file = os.path.join(gitlog_path, 'git-log-%s.dcf' % pkg)
if not os.path.exists(gitlog_file):
print('BBS> %s file does not exist --> skipping.' % gitlog_file)
continue
desc_file = os.path.join(BBSvars.meat_path, pkg, 'DESCRIPTION')
if not os.path.exists(desc_file):
print('BBS> %s file does not exist --> skipping.' % desc_file)
continue
bbs.parse.injectFieldsInDESCRIPTION(desc_file, gitlog_file)
# Then re-install the supporting packages.
print('BBS> [STAGE2] Re-install supporting packages')
os.chdir(meat_path)
job_queue = prepare_STAGE2_job_queue(target_pkgs, pkg_dep_graph,
installed_pkgs, out_dir)
STAGE2_loop(job_queue, BBSvars.install_nb_cpu, out_dir)
print('BBS> [STAGE2] cd BBS_WORK_TOPDIR/STAGE2_tmp')
os.chdir(STAGE2_tmp)
if BBSvars.buildtype in ['bioc', 'bioc-testing']:
# Try again to update non-target packages (some updates could have
# failed in the previous attempt because of dependency issues).
print('BBS> [STAGE2] Update non-target packages (2nd run) ...',
end=' ')
sys.stdout.flush()
cmd = BBSbase.get_update_cmd_for_non_target_pkgs()
bbs.jobs.runJob(cmd, 'updateNonTargetPkgs2.Rout', 3600.0)
print('OK')
sys.stdout.flush()
makeNodeInfo()
print('BBS> [STAGE2] DONE at %s.' % time.asctime())
return
