def extract_tarball():
package_name = manifest['job_id'].split("_")[0]
# first, log in to the tracker and get a cookie
tracker_url = HOSTS['tracker']
if not "tracker.bioconductor.org" in manifest['svn_url'].lower():
tracker_url = tracker_url + '/roundup/bioc_submit'
br = mechanize.Browser()
br.open(tracker_url)
br.select_form(nr=2)
br["__login_name"] = ENVIR['tracker_user']
br["__login_password"] = ENVIR['tracker_pass']
br.submit()
segs = manifest['svn_url'].split("/")
local_file = segs[len(segs) - 1]
try:
br.retrieve(manifest['svn_url'], local_file)
retcode = 0
except:
retcode = 255
logging.error("extract_tarball() Failed to download '%s' to '%s'.",
manifest['svn_url'], local_file)
raise
send_message({
"status": "post_processing",
"retcode": retcode,
"body": "Download of tarball completed. "
})
if (retcode != 0):
logging.error("extract_tarball() Failed to 'curl' tarball.")
raise
tmp = manifest['svn_url'].split("/")
tarball = tmp[len(tmp) - 1]
package_name = tarball.split("_")[0]
# what if package name does not have version in it? do this:
package_name = package_name.replace(".tar.gz", "")
os.rename(tarball, "%s.orig" % tarball)
extra_flags = ""
if platform.system() == "Windows":
extra_flags = " --no-same-owner "
retcode = subprocess.call("tar %s -zxf %s.orig" % (extra_flags, tarball),
shell=True)
send_message({
"status": "post_processing",
"retcode": retcode,
"body": "Untar of tarball completed. "
})
if (not retcode == 0):
logging.error("extract_tarball() Failed to 'untar' tarball.")
raise
f = open("%s/DESCRIPTION" % package_name)
description = f.read()
f.close()
dcf_file = dcf.DcfRecordParser(description.rstrip().split("\n"))
send_dcf_info(dcf_file)
def get_dcf_info(manifest):
global svn_url_global
svn_url_global = manifest['svn_url']
package_name = manifest['job_id'].split("_")[0]
logging.info("Starting get_dcf_info() '%s'." % package_name)
github_url = re.sub(r'\.git$', '', manifest['svn_url'])
if not github_url.endswith("/"):
github_url += "/"
# We only build the master branch. There had better be one.
# (technically we build whatever the default branch is, but
# this step looks at master because to find out what the
# default branch is at this point we would need octokit here).
github_url += "master/DESCRIPTION"
github_url = github_url.replace("https://github.com",
"https://raw.githubusercontent.com")
try:
f = urllib2.urlopen(github_url)
dcf_text = f.read()
dcf_file = dcf.DcfRecordParser(dcf_text.rstrip().split("\n"))
send_dcf_info(dcf_file)
desc_name = dcf_file.getValue("Package")
except:
logging.error("ERROR: get_dcf_info() failed\n Could not open ",
github_url)
send_message({
"status": "build_complete",
"retcode": 1,
"warnings": False,
"body": "get_dcf_info failed; could not open. ",
"elapsed_time": "NA"
})
send_message({
"status": "post_processing",
"retcode": 1,
"body": "get_dcf_info failed; could not open. "
})
sys.exit("Exiting get_dcf_info check failed")
if package_name != desc_name:
msg = "ERROR: Repository name: '" + package_name + \
"' and DESCRIPTION Package: '" + desc_name + \
"' do not match. "
send_message({
"status": "build_complete",
"retcode": 1,
"warnings": False,
"body": msg,
"elapsed_time": "NA"
})
send_message({"status": "post_processing", "retcode": 1, "body": msg})
sys.exit("Exiting get_dcf_info check failed")
def svn_info():
logging.debug("svn_info() svn_url is %s" % manifest['svn_url'])
svn_info = subprocess.Popen(["svn", "info", manifest['svn_url']],
stdout=subprocess.PIPE).communicate()[0]
logging.debug("svn_info() svn_info is:\n%s" % svn_info)
dcf_records = dcf.DcfRecordParser(svn_info.rstrip().split("\n"))
keys = [
'Path', 'URL', 'Repository Root', 'Repository UUID', 'Revision',
'Node Kind', 'Last Changed Author', 'Last Changed Rev',
'Last Changed Date'
]
svn_hash = {}
for key in keys:
svn_hash[key] = dcf_records.getValue(key)
svn_hash['status'] = "svn_info"
svn_hash['body'] = "svn info"
send_message(svn_hash)
def __init__(self, reportUrl):
reportFile = urllib.urlopen(reportUrl)
data = dcf.DcfRecordParser(reportFile)
self.name = data.getValue("Package")
self.version = data.getValue("Version")
self.status = data.getValue("Status")
## FIXME: we should really define CSS classes for
## these elements and then let the color be defined in
## a CSS style file.
if self.status == "OK":
self.statusColor = "green"
else:
self.statusColor = "red"
self.time = data.getValue("Time")
urlBase = os.path.dirname(reportUrl)
self.outputUrl = urlBase + '/' + self.name + '-out.txt'
def install_pkg_deps():
package_name = manifest['job_id'].split("_")[0]
f = open("%s/%s/DESCRIPTION" % (working_dir, package_name))
description = f.read()
logging.info("DESCRIPTION file loaded for package '%s': \n%s",
package_name, description)
f.close()
desc = dcf.DcfRecordParser(description.rstrip().split("\n"))
fields = ["Depends", "Imports", "Suggests", "Enhances", "LinkingTo"]
args = ""
for field in fields:
try:
args += '%s=@@%s@@; ' % (field, desc.getValue(field))
except KeyError:
pass
r_script = os.path.join(ENVIR['spb_home'], "installPkgDeps.R")
log = "%s/installDeps.log" % working_dir
if args.strip() == "":
args = "None=1"
rscript_dir = os.path.dirname(ENVIR['bbs_R_cmd'])
rscript_binary = os.path.join(rscript_dir, "Rscript")
cmd = "%s --vanilla --no-save --no-restore %s --args \"%s\" > %s 2>&1" % \
(rscript_binary, r_script, args.strip(), log)
send_message({
"body": "Installing dependencies. ",
"status": "preprocessing",
"retcode": 0
})
logging.info("Command to install dependencies:" + "\n %s" % cmd)
retcode = subprocess.call(cmd, shell=True)
send_message({
"body": "Installing dependency status: " + str(retcode) + ". ",
"status": "post_processing",
"retcode": retcode
})
logging.info(
"Finished Installing Dependencies.\n completed with status: " +
str(retcode))
return retcode
def getPackageType():
global longBuild
global pkg_type_views
package_name = manifest['job_id'].split("_")[0]
f = open("%s/%s/DESCRIPTION" % (working_dir, package_name))
description = f.read()
f.close()
desc = dcf.DcfRecordParser(description.rstrip().split("\n"))
try:
isWorkflow = desc.getValue("Workflow")
except KeyError:
pass
isWorkflow = "false"
if (isWorkflow.lower() == "true"):
longBuild = True
pkg_type_views = "Workflow"
logging.info("Package is a workflow.")
else:
try:
views = desc.getValue("biocViews").replace(",", "")
r_script = os.path.join(ENVIR['spb_home'], "getPackageType.R")
rscript_dir = os.path.dirname(ENVIR['bbs_R_cmd'])
rscript_binary = os.path.join(rscript_dir, "Rscript")
cmd = "%s --vanilla --no-save --no-restore %s" % (rscript_binary,
r_script)
cmd = cmd + " " + views
logging.info("Get Package Type command:\n" + cmd)
pkg_type_views = subprocess.check_output(cmd, shell=True)
if (pkg_type_views == "ExperimentData"):
longBuild = True
except KeyError:
pkg_type_views = "Software"
send_message({
"body": "Package type: " + pkg_type_views + ". ",
"status": "post_processing",
"retcode": 0
})
logging.info("Package is of type: " + pkg_type_views)
logging.info("Package gets long build: " + str(longBuild))
def is_build_required(manifest):
global svn_url_global
svn_url_global = manifest['svn_url']
package_name = manifest['job_id'].split("_")[0]
logging.info("Starting is_build_required() '%s'." % package_name)
if (get_package_source() == "svn"):
description_url = manifest['svn_url'].rstrip("/") + "/DESCRIPTION"
logging.debug("is_build_required() package source is svn" +
"\n description_url = " + description_url +
"\n svn_user ="******"\n svn_pass = "******"curl", "-k", "-s", "--user",
"%s:%s" %
(ENVIR['svn_user'], ENVIR['svn_pass']), description_url
],
stdout=subprocess.PIPE).communicate()[0]
# TODO - handle it if description does not exist
except:
logging.error("is_build_required() curl exception: %s.",
sys.exc_info()[0])
raise
logging.debug("is_build_required()" +
"\n description = %s" % description +
"\n length = %d" % len(description))
dcf_file = dcf.DcfRecordParser(description.rstrip().split("\n"))
send_dcf_info(dcf_file)
svn_version = dcf_file.getValue("Version")
elif get_package_source() == "github":
github_url = re.sub(r'\.git$', '', manifest['svn_url'])
if not github_url.endswith("/"):
github_url += "/"
# We only build the master branch. There had better be one.
# (technically we build whatever the default branch is, but
# this step looks at master because to find out what the
# default branch is at this point we would need octokit here).
github_url += "master/DESCRIPTION"
github_url = github_url.replace("https://github.com",
"https://raw.githubusercontent.com")
f = urllib2.urlopen(github_url)
dcf_text = f.read()
dcf_file = dcf.DcfRecordParser(dcf_text.rstrip().split("\n"))
send_dcf_info(dcf_file)
svn_version = dcf_file.getValue("Version")
elif get_package_source() == "tracker":
tmp = manifest["svn_url"].split("/")
pkgname = tmp[len(tmp) - 1].replace(".tar.gz", "")
if (pkgname.find("_") == -1
): # package name doesn't have version in it
return (True
) # TODO - download tarball and examine DESCRIPTION file
svn_version = pkgname.split("_")[1]
if ("force" in manifest.keys()):
if (manifest['force'] is True):
return (True)
r_version = BIOC_R_MAP[ENVIR['bbs_Bioc_version']]
pkg_type = BBScorevars.getNodeSpec(BUILDER_ID, "pkgType")
cran_repo_map = {
'source': "src/contrib",
'win.binary': "bin/windows/contrib/" + r_version,
'win64.binary': "bin/windows64/contrib/" + r_version,
'mac.binary': "bin/macosx/contrib/" + r_version,
'mac.binary.mavericks': "bin/macosx/mavericks/contrib/" + r_version
}
# todo - put repos url in config file (or get it from user)
base_repo_url = HOSTS['bioc']
if (manifest['repository'] == 'course'):
base_repo_url += '/course-packages'
elif (manifest['repository'] == 'scratch'):
base_repo_url += '/scratch_repos/' + manifest['bioc_version']
repository_url = "%s/%s/PACKAGES" % (base_repo_url,
cran_repo_map[pkg_type])
# What if there is no file at this url?
packages = subprocess.Popen(["curl", "-k", "-s", repository_url],
stdout=subprocess.PIPE).communicate()[0]
inpackage = False
repository_version = False
for line in packages.split("\n"):
if line == "Package: %s" % package_name:
inpackage = True
if (line.startswith("Version: ") and inpackage):
repository_version = line.split(": ")[1]
break
if not repository_version:
return True # package hasn't been pushed to repo before
logging.debug("is_build_required()" +
"\n [%s] svn version is %s, repository version is %s" %
(package_name, svn_version, repository_version))
return svn_version != repository_version