def stage6_make_OUTGOING():
## Create working directory
OUTGOING_dir = os.path.join(BBScorevars.Central_rdir.path, "OUTGOING")
print "BBS> [stage6] remake_dir %s" % OUTGOING_dir
bbs.fileutils.remake_dir(OUTGOING_dir)
## Loop over each element of the OUTGOING map
OUTGOING_map = BBScorevars.getenv('BBS_OUTGOING_MAP')
map_elts = OUTGOING_map.split(" ")
for map_elt in map_elts:
tmp = map_elt.split(":")
if len(tmp) != 2:
sys.exit("ERROR: Invalid OUTGOING map element %s" % map_elt)
source_node = False
if tmp[0] == "source":
source_node = True
OUTGOING_subdir = os.path.join(OUTGOING_dir, tmp[0])
print "BBS> [stage6] mkdir %s" % OUTGOING_subdir
os.mkdir(OUTGOING_subdir)
print "BBS> [stage6] cd %s/" % OUTGOING_subdir
os.chdir(OUTGOING_subdir)
tmp2 = tmp[1].split("+")
if len(tmp2) == 1:
copy_outgoing_pkgs(tmp[1], source_node)
elif len(tmp2) == 2:
make_outgoing_biarch_pkgs(tmp2[0], tmp2[1])
else:
sys.exit("ERROR: Invalid OUTGOING map element %s" % map_elt)
return
def _BiocGreaterThanOrEqualTo(x, y):
bioc_version = BBScorevars.getenv('BBS_BIOC_VERSION', False)
# If BBS_BIOC_VERSION is not defined, then we assume it's the latest version.
if bioc_version == None:
return True
parts = bioc_version.split('.')
x0 = int(parts[0])
y0 = int(parts[1])
return x0 > x or (x0 == x and y0 >= y)
def _BiocGreaterThanOrEqualTo(x, y):
bioc_version = BBScorevars.getenv('BBS_BIOC_VERSION', False)
# If BBS_BIOC_VERSION is not defined, then we assume it's the latest version.
if bioc_version == None:
return True
parts = bioc_version.split('.')
x0 = int(parts[0])
y0 = int(parts[1])
return x0 > x or (x0 == x and y0 >= y)
def _getMaintainerFromDir(pkg_dir):
desc_file = getDescFile(pkg_dir)
FNULL = open(os.devnull, 'w')
r_home = BBScorevars.getenv('BBS_R_HOME')
Rscript_cmd = os.path.join(r_home, "bin", "Rscript")
script_path = os.path.join(BBScorevars.BBS_home, "utils", "getMaintainer.R")
cmd = [Rscript_cmd, '--vanilla', script_path, desc_file]
maintainer = subprocess.check_output(cmd, stderr=FNULL)
if maintainer == 'NA':
raise DcfFieldNotFoundError(desc_file, 'Maintainer')
return maintainer
def _getMaintainerFromDir(pkg_dir):
desc_file = getDescFile(pkg_dir)
FNULL = open(os.devnull, 'w')
r_home = BBScorevars.getenv('BBS_R_HOME')
Rscript_cmd = os.path.join(r_home, "bin", "Rscript")
script_path = os.path.join(BBScorevars.BBS_home, "utils",
"getMaintainer.R")
cmd = [Rscript_cmd, '--vanilla', script_path, desc_file]
maintainer = subprocess.check_output(cmd, stderr=FNULL)
if maintainer == 'NA':
raise DcfFieldNotFoundError(desc_file, 'Maintainer')
return maintainer
def copy_outgoing_pkgs(fresh_pkgs_subdir, source_node):
tmp = fresh_pkgs_subdir.split("/")
if len(tmp) != 2:
sys.exit("ERROR: Invalid relative path to fresh pkgs %s (must be of the form node/subdir)" % fresh_pkgs_subdir)
node_id = tmp[0]
node_hostname = node_id.split("-")[0]
fileext = BBScorevars.getNodeSpec(node_hostname, 'pkgFileExt')
fresh_pkgs_subdir = os.path.join(BBScorevars.nodes_rdir.path, fresh_pkgs_subdir)
## Workflow packages do not have manuals/ because we do not run
## `R CMD check`.
manuals_dir = "../manuals"
if BBScorevars.is_workflow:
pass
elif (source_node):
print "BBS> [stage6] mkdir %s" % manuals_dir
os.mkdir(manuals_dir)
print "BBS> [stage6] BEGIN copying outgoing packages from %s." % fresh_pkgs_subdir
pkgType = BBScorevars.getNodeSpec(node_hostname, 'pkgType')
meat_index_file = os.path.join(BBScorevars.Central_rdir.path, BBScorevars.meat_index_file)
dcf = open(meat_index_file, 'r')
pkgs = bbs.parse.readPkgsFromDCF(dcf, node_id, pkgType)
dcf.close()
for pkg in pkgs:
if pkgMustBeRejected(node_hostname, node_id, pkg):
continue
dcf = open(meat_index_file, 'r')
version = bbs.parse.getPkgFieldFromDCF(dcf, pkg, 'Version', BBScorevars.meat_index_file)
dcf.close()
## Copy pkg from 'fresh_pkgs_subdir2'.
pkg_file = "%s_%s.%s" % (pkg, version, fileext)
pkg_file = os.path.join(fresh_pkgs_subdir, pkg_file)
print "BBS> [stage6] - copying %s to OUTGOING folder ..." % pkg_file
if os.path.exists(pkg_file):
shutil.copy(pkg_file, ".")
else:
print "BBS> [stage6] SKIPPED (file %s doesn't exist)" % pkg_file
## Get reference manual from pkg.Rcheck directory.
if BBScorevars.is_workflow:
pass
elif source_node:
pdf_file = "%s/meat/%s.Rcheck/%s-manual.pdf" % \
(BBScorevars.getenv('BBS_WORK_TOPDIR'), pkg, pkg)
print "BBS> [stage6] - copying %s manual to OUTGOING/manuals folder..." % pkg
if os.path.exists(pdf_file):
shutil.copy(pdf_file, "%s/%s.pdf" % (manuals_dir, pkg))
else:
print "BBS> [stage6] SKIPPED (file %s doesn't exist)" % pdf_file
print "BBS> [stage6] END copying outgoing packages from %s." % fresh_pkgs_subdir
return
def getSTAGE2cmd(pkg, version):
cmd = '%s CMD INSTALL %s' % (BBSvars.r_cmd, pkg)
if sys.platform == "win32":
win_archs = _supportedWinArchs(pkg)
if _mustRunSTAGE2InMultiarchMode() and len(win_archs) == 2:
curl_cmd = BBScorevars.getenv('BBS_CURL_CMD')
srcpkg_file = pkg + '_' + version + '.tar.gz'
srcpkg_url = BBScorevars.Central_rdir.url + '/src/contrib/' + \
srcpkg_file
cmd = '%s -O %s' % (curl_cmd, srcpkg_url) + ' && ' + \
'%s CMD INSTALL --merge-multiarch %s' % \
(BBSvars.r_cmd, srcpkg_file) + ' && ' + \
'rm %s' % srcpkg_file
else:
cmd = '%s --arch %s CMD INSTALL --no-multiarch %s' % \
(BBSvars.r_cmd, win_archs[0], pkg)
return cmd
def getSTAGE2cmd(pkg, version):
cmd = '%s CMD INSTALL %s' % (BBSvars.r_cmd, pkg)
if sys.platform == "win32":
win_archs = _supportedWinArchs(pkg)
if _mustRunSTAGE2InMultiarchMode() and len(win_archs) == 2:
curl_cmd = BBScorevars.getenv('BBS_CURL_CMD')
srcpkg_file = pkg + '_' + version + '.tar.gz'
srcpkg_url = BBScorevars.Central_rdir.url + '/src/contrib/' + \
srcpkg_file
cmd = '%s -O %s' % (curl_cmd, srcpkg_url) + ' && ' + \
'%s CMD INSTALL --merge-multiarch %s' % \
(BBSvars.r_cmd, srcpkg_file) + ' && ' + \
'rm %s' % srcpkg_file
else:
cmd = '%s --arch %s CMD INSTALL --no-multiarch %s' % \
(BBSvars.r_cmd, win_archs[0], pkg)
return cmd
def getSTAGE2cmd(pkg, version):
prepend = bbs.parse.getBBSoptionFromDir(pkg, 'INSTALLprepend')
if sys.platform != "win32":
cmd = '%s %s' % (_get_RINSTALL_cmd0(), pkg)
else:
prepend_win = bbs.parse.getBBSoptionFromDir(pkg, 'INSTALLprepend.win')
if prepend_win != None:
prepend = prepend_win
win_archs = _supportedWinArchs(pkg)
if _mustRunSTAGE2InMultiarchMode() and len(win_archs) >= 2:
## Here is what Dan's commit message says about why BBS uses this
## very long and complicated compound command to install packages
## in multiarch mode on Windows during STAGE2 (see
## 'git show 87822fb346e04b4301d0c2efd7ec1a2a8762e93a'):
## Install STAGE2 target pkgs to zip+libdir first, then install
## zip. This mitigates the problem where the INSTALL times out
## after installing only one architecture. Now, if the install
## times out, it has only failed to install a package to a
## temporary libdir, and the previous installation is still
## intact (or pkg is not installed at all), so dependent packages
## will not complain about the timed-out package being only
## available for one architecture.
## We still need to figure out why mzR in particular times out
## during INSTALL. When run manually (admittedly not during
## peak load times) the install takes ~ 5-6 minutes, even when
## done via the build system.
curl_cmd = BBScorevars.getenv('BBS_CURL_CMD')
srcpkg_file = pkg + '_' + version + '.tar.gz'
srcpkg_url = BBScorevars.Central_rdir.url + '/src/contrib/' + \
srcpkg_file
zip_file = srcpkg_file.replace(".tar.gz", ".zip")
cmd = '%s -O %s' % (curl_cmd, srcpkg_url) + ' && ' + \
_get_BuildBinPkg_cmd(srcpkg_file, win_archs) + ' && ' + \
'%s %s' % (_get_RINSTALL_cmd0(), zip_file) + ' && ' + \
'rm %s %s' % (srcpkg_file, zip_file)
else:
cmd = '%s %s' % (_get_RINSTALL_cmd0(win_archs), pkg)
if prepend != None:
cmd = '%s %s' % (prepend, cmd)
return cmd
def getSTAGE2cmd(pkg, version):
prepend = bbs.parse.getBBSoptionFromDir(pkg, 'INSTALLprepend')
if sys.platform != "win32":
cmd = '%s %s' % (_get_RINSTALL_cmd0(), pkg)
else:
prepend_win = bbs.parse.getBBSoptionFromDir(pkg, 'INSTALLprepend.win')
if prepend_win != None:
prepend = prepend_win
win_archs = _supportedWinArchs(pkg)
if _mustRunSTAGE2InMultiarchMode() and len(win_archs) >= 2:
## Here is what Dan's commit message says about why BBS uses this
## very long and complicated compound command to install packages
## in multiarch mode on Windows during STAGE2 (see
## 'git show 87822fb346e04b4301d0c2efd7ec1a2a8762e93a'):
## Install STAGE2 target pkgs to zip+libdir first, then install
## zip. This mitigates the problem where the INSTALL times out
## after installing only one architecture. Now, if the install
## times out, it has only failed to install a package to a
## temporary libdir, and the previous installation is still
## intact (or pkg is not installed at all), so dependent packages
## will not complain about the timed-out package being only
## available for one architecture.
## We still need to figure out why mzR in particular times out
## during INSTALL. When run manually (admittedly not during
## peak load times) the install takes ~ 5-6 minutes, even when
## done via the build system.
curl_cmd = BBScorevars.getenv('BBS_CURL_CMD')
srcpkg_file = pkg + '_' + version + '.tar.gz'
srcpkg_url = BBScorevars.Central_rdir.url + '/src/contrib/' + \
srcpkg_file
zip_file = srcpkg_file.replace(".tar.gz", ".zip")
cmd = '%s -O %s' % (curl_cmd, srcpkg_url) + ' && ' + \
_get_BuildBinPkg_cmd(srcpkg_file, win_archs) + ' && ' + \
'%s %s' % (_get_RINSTALL_cmd0(), zip_file) + ' && ' + \
'rm %s %s' % (srcpkg_file, zip_file)
else:
cmd = '%s %s' % (_get_RINSTALL_cmd0(win_archs), pkg)
if prepend != None:
cmd = '%s %s' % (prepend, cmd)
return cmd
def getSTAGE2cmd(pkg, version):
cmd = '%s CMD INSTALL %s' % (BBSvars.r_cmd, pkg)
if sys.platform == "win32":
win_archs = _supportedWinArchs(pkg)
if _mustRunSTAGE2InMultiarchMode() and len(win_archs) == 2:
curl_cmd = BBScorevars.getenv('BBS_CURL_CMD')
srcpkg_file = pkg + '_' + version + '.tar.gz'
srcpkg_url = BBScorevars.Central_rdir.url + '/src/contrib/' + \
srcpkg_file
zipfile = srcpkg_file.replace(".tar.gz", ".zip")
cmd = 'rm -rf %s.buildbin-libdir' % pkg + ' && ' + \
'mkdir %s.buildbin-libdir ' % pkg + ' && ' + \
'%s -O %s' % (curl_cmd, srcpkg_url) + ' && ' + \
'%s CMD INSTALL --build --library=%s.buildbin-libdir --merge-multiarch %s' % \
(BBSvars.r_cmd, pkg, srcpkg_file) + ' && ' + \
'%s CMD INSTALL %s ' % (BBSvars.r_cmd,
zipfile) + ' && ' + \
'rm %s %s' % (srcpkg_file, zipfile)
else:
cmd = '%s --arch %s CMD INSTALL --no-multiarch %s' % \
(BBSvars.r_cmd, win_archs[0], pkg)
return cmd
###
import sys
import bbs.rdir
import bbs.jobs
import BBScorevars
##############################################################################
### BBS GLOBAL VARIABLES
##############################################################################
### Only needed by BBS-prerun.py and BBS-run.py
MEAT0_rdir = bbs.rdir.RemoteDir('BBS_MEAT0_RDIR', None,
BBScorevars.getenv('BBS_MEAT0_RDIR'),
BBScorevars.getenv('BBS_MEAT0_RHOST', False),
BBScorevars.getenv('BBS_MEAT0_RUSER', False),
BBScorevars.ssh_cmd, BBScorevars.rsync_cmd,
BBScorevars.rsync_rsh_cmd)
meat_path = BBScorevars.getenv('BBS_MEAT_PATH')
work_topdir = BBScorevars.getenv('BBS_WORK_TOPDIR')
r_cmd = BBScorevars.getenv('BBS_R_CMD')
### Only needed by BBS-prerun.py
MEAT0_type = int(BBScorevars.getenv('BBS_MEAT0_TYPE'))
return
def send_CRAN_notifications(allpkgs):
print "BBS> [send_CRAN_notifications] BEGIN..."
send_notifications(allpkgs)
print "BBS> [send_CRAN_notifications] END."
return
##############################################################################
### MAIN SECTION
##############################################################################
print "BBS> [stage9] STARTING stage9 at %s..." % time.asctime()
BBSreportutils.data_source = BBScorevars.getenv('BBS_PUBLISHED_REPORT_URL')
notify_nodes = BBScorevars.getenv('BBS_NOTIFY_NODES')
argc = len(sys.argv)
if argc > 1:
arg1 = sys.argv[1]
else:
arg1 = ""
if arg1 != "":
bbs.email.mode = "do-it"
if arg1 != "do-it":
bbs.email.redirect_to_addr = arg1
BBSreportutils.set_NODES(notify_nodes)
if len(BBSreportutils.NODES) != 1:
# BUILD BIN status
if BBSreportutils.is_doing_buildbin(node):
stage = 'buildbin'
if skipped_is_OK:
status = "skipped"
else:
status = get_status_from_summary_file(pkg, node.id, stage)
out.write('%s#%s#%s: %s\n' % (pkg, node.id, stage, status))
out.close()
print "BBS> [make_STATUS_DB] END"
return
##############################################################################
### MAIN SECTION
##############################################################################
print "BBS> [stage7a] STARTING stage7a at %s..." % time.asctime()
central_rdir_path = BBScorevars.Central_rdir.path
if central_rdir_path != None:
print "BBS> [stage7a] cd %s/" % central_rdir_path
os.chdir(central_rdir_path)
report_nodes = BBScorevars.getenv('BBS_REPORT_NODES')
BBSreportutils.set_NODES(report_nodes)
allpkgs = BBSreportutils.get_pkgs_from_meat_index()
make_STATUS_DB(allpkgs)
print "BBS> [stage7a] DONE at %s." % time.asctime()
return
def send_CRAN_notifications(allpkgs):
print "BBS> [send_CRAN_notifications] BEGIN..."
send_notifications(allpkgs)
print "BBS> [send_CRAN_notifications] END."
return
##############################################################################
### MAIN SECTION
##############################################################################
print "BBS> [stage9] STARTING stage9 at %s..." % time.asctime()
BBSreportutils.data_source = BBScorevars.getenv('BBS_PUBLISHED_REPORT_URL')
notify_nodes = BBScorevars.getenv('BBS_NOTIFY_NODES')
argc = len(sys.argv)
if argc > 1:
arg1 = sys.argv[1]
else:
arg1 = ""
if arg1 != "":
bbs.email.mode = "do-it"
if arg1 != "do-it":
bbs.email.redirect_to_addr = arg1
BBSreportutils.set_NODES(notify_nodes)
if len(BBSreportutils.NODES) != 1:
if BBSreportutils.is_doing_buildbin(node):
stage = 'buildbin'
if skipped_is_OK:
status = "skipped"
else:
status = get_status_from_summary_file(pkg, node.id, stage)
out.write('%s#%s#%s: %s\n' % (pkg, node.id, stage, status))
out.close()
print "BBS> [make_STATUS_DB] END"
return
##############################################################################
### MAIN SECTION
##############################################################################
print "BBS> [stage7a] STARTING stage7a at %s..." % time.asctime()
central_rdir_path = BBScorevars.Central_rdir.path
if central_rdir_path != None:
print "BBS> [stage7a] cd %s/" % central_rdir_path
os.chdir(central_rdir_path)
report_nodes = BBScorevars.getenv('BBS_REPORT_NODES')
BBSreportutils.set_NODES(report_nodes)
allpkgs = BBSreportutils.get_pkgs_from_meat_index()
make_STATUS_DB(allpkgs)
print "BBS> [stage7a] DONE at %s." % time.asctime()
## Report title.
if BBScorevars.subbuilds == "bioc-longtests":
REPORT_TITLE = "Long tests"
elif BBScorevars.subbuilds == "workflows":
REPORT_TITLE = "Workflows build"
else:
if len(NODES) != 1:
REPORT_TITLE = "Multiple platform build/check"
else:
REPORT_TITLE = "Build/check"
REPORT_TITLE += " report for "
if BBScorevars.subbuilds == "cran":
REPORT_TITLE += "CRAN"
else:
bioc_version = BBScorevars.getenv('BBS_BIOC_VERSION', False)
REPORT_TITLE += "BioC %s" % bioc_version
if BBScorevars.subbuilds == "data-experiment":
REPORT_TITLE += " experimental data"
## Stages to display on report (as columns in HTML table).
def stages_to_display():
if BBScorevars.subbuilds == "bioc-longtests":
return ['checksrc'] # we run 'buildsrc' but don't display it
if BBScorevars.subbuilds == "workflows":
return ['install', 'buildsrc']
return ['install', 'buildsrc', 'checksrc', 'buildbin']
def stage_label(stage):
stage2label = {
'install': "INSTALL",
import bbs.rdir
import bbs.jobs
import BBScorevars
##############################################################################
### BBS GLOBAL VARIABLES
##############################################################################
### Only needed by BBS-prerun.py and BBS-run.py
MEAT0_rdir = bbs.rdir.RemoteDir('BBS_MEAT0_RDIR',
None,
BBScorevars.getenv('BBS_MEAT0_RDIR'),
BBScorevars.getenv('BBS_MEAT0_RHOST', False),
BBScorevars.getenv('BBS_MEAT0_RUSER', False),
BBScorevars.ssh_cmd,
BBScorevars.rsync_cmd,
BBScorevars.rsync_rsh_cmd)
meat_path = BBScorevars.getenv('BBS_MEAT_PATH')
work_topdir = BBScorevars.getenv('BBS_WORK_TOPDIR')
r_cmd = BBScorevars.getenv('BBS_R_CMD')
r_home = BBScorevars.getenv('BBS_R_HOME')
### Only needed by BBS-prerun.py
import bbs.rdir
import bbs.jobs
import BBScorevars
##############################################################################
### BBS GLOBAL VARIABLES
##############################################################################
### Only needed by BBS-prerun.py and BBS-run.py
MEAT0_rdir = bbs.rdir.RemoteDir('BBS_MEAT0_RDIR',
None,
BBScorevars.getenv('BBS_MEAT0_RDIR'),
BBScorevars.getenv('BBS_MEAT0_RHOST', False),
BBScorevars.getenv('BBS_MEAT0_RUSER', False),
BBScorevars.ssh_cmd,
BBScorevars.rsync_cmd,
BBScorevars.rsync_rsh_cmd)
meat_path = BBScorevars.getenv('BBS_MEAT_PATH')
work_topdir = BBScorevars.getenv('BBS_WORK_TOPDIR')
r_cmd = BBScorevars.getenv('BBS_R_CMD')
### Only needed by BBS-prerun.py
import sys
import os
import bbs.rdir
import bbs.jobs
import BBScorevars
##############################################################################
### BBS GLOBAL VARIABLES
##############################################################################
### Only needed by BBS-prerun.py and BBS-run.py
MEAT0_rdir = bbs.rdir.RemoteDir('BBS_MEAT0_RDIR', None,
BBScorevars.getenv('BBS_MEAT0_RDIR'),
BBScorevars.getenv('BBS_MEAT0_RHOST', False),
BBScorevars.getenv('BBS_MEAT0_RUSER', False),
BBScorevars.ssh_cmd, BBScorevars.rsync_cmd,
BBScorevars.rsync_rsh_cmd)
meat_path = BBScorevars.getenv('BBS_MEAT_PATH')
work_topdir = BBScorevars.getenv('BBS_WORK_TOPDIR')
r_cmd = BBScorevars.getenv('BBS_R_CMD')
r_home = BBScorevars.getenv('BBS_R_HOME')
### Only needed by BBS-prerun.py