def make_STATUS_DB(allpkgs):
print "BBS> [make_STATUS_DB] BEGIN..."
out = open(BBSreportutils.STATUS_DB_file, 'w')
for pkg in allpkgs:
for node in BBSreportutils.supported_nodes(pkg):
# INSTALL status
stagecmd = 'install'
status = get_status_from_summary_file(pkg, node.id, stagecmd)
out.write('%s#%s#%s: %s\n' % (pkg, node.id, stagecmd, status))
# BUILD status
stagecmd = 'buildsrc'
status = get_status_from_summary_file(pkg, node.id, stagecmd)
out.write('%s#%s#%s: %s\n' % (pkg, node.id, stagecmd, status))
skipped_is_OK = status in ["TIMEOUT", "ERROR"]
# CHECK status
stagecmd = 'checksrc'
if skipped_is_OK:
status = "skipped"
else:
status = get_status_from_summary_file(pkg, node.id, stagecmd)
out.write('%s#%s#%s: %s\n' % (pkg, node.id, stagecmd, status))
if BBSreportutils.is_doing_buildbin(node):
# BUILD BIN status
stagecmd = 'buildbin'
if skipped_is_OK:
status = "skipped"
else:
status = get_status_from_summary_file(pkg, node.id, stagecmd)
out.write('%s#%s#%s: %s\n' % (pkg, node.id, stagecmd, status))
out.close()
print "BBS> [make_STATUS_DB] END"
return
def _write_pkg_results_to_BUILD_STATUS_DB(pkg, out):
for node in BBSreportutils.supported_nodes(pkg):
# INSTALL status
if BBSvars.buildtype != 'bioc-longtests':
stage = 'install'
status = _read_status_from_summary_file(pkg, node.node_id, stage)
_write_status_to_BUILD_STATUS_DB(out, pkg, node.node_id, stage,
status)
# BUILD status
stage = 'buildsrc'
status = _read_status_from_summary_file(pkg, node.node_id, stage)
_write_status_to_BUILD_STATUS_DB(out, pkg, node.node_id, stage, status)
skipped_is_OK = status in ['TIMEOUT', 'ERROR']
# CHECK status
if BBSvars.buildtype not in ['workflows', 'books']:
stage = 'checksrc'
if skipped_is_OK:
status = 'skipped'
else:
status = _read_status_from_summary_file(
pkg, node.node_id, stage)
_write_status_to_BUILD_STATUS_DB(out, pkg, node.node_id, stage,
status)
# BUILD BIN status
if BBSreportutils.is_doing_buildbin(node):
stage = 'buildbin'
if skipped_is_OK:
status = 'skipped'
else:
status = _read_status_from_summary_file(
pkg, node.node_id, stage)
_write_status_to_BUILD_STATUS_DB(out, pkg, node.node_id, stage,
status)
return
def send_notification(pkg):
package_status = BBSreportutils.get_pkg_field_from_meat_index(
pkg, 'PackageStatus')
if package_status == 'Deprecated':
return
maintainer_email = BBSreportutils.get_pkg_field_from_meat_index(
pkg, 'MaintainerEmail')
#print "%s %s %s %s" % (pkg, version, maintainer, maintainer_email)
#key = 'Last Changed Date'
#last_changed_date = BBSreportutils.get_vcs_meta(pkg, key)
#key = 'Last Changed Rev'
#last_changed_rev = BBSreportutils.get_vcs_meta(pkg, key)
problem_descs = []
for node in BBSreportutils.supported_nodes(pkg):
stage = 'install'
status = BBSreportutils.get_status_from_db(pkg, node.id, stage)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rel_url(
pkg, node.id, stage)
problem_desc = " o %s for 'R CMD INSTALL' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
stage = 'buildsrc'
status = BBSreportutils.get_status_from_db(pkg, node.id, stage)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rel_url(
pkg, node.id, stage)
problem_desc = " o %s for 'R CMD build' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
stage = 'checksrc'
status = BBSreportutils.get_status_from_db(pkg, node.id, stage)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rel_url(
pkg, node.id, stage)
problem_desc = " o %s for 'R CMD check' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
if len(problem_descs) == 0:
return
if maintainer_email == "*****@*****.**":
to_addrs = ["*****@*****.**"]
else:
to_addrs = [maintainer_email]
subject = "%s problems reported in the %s" % \
(pkg, BBSreportutils.REPORT_TITLE)
msg = "%s\nHi %s maintainer,\n\n" % (msg_head, pkg) \
+ "According to the %s,\n" % BBSreportutils.REPORT_TITLE \
+ "the %s package has the following problem(s):\n\n" % pkg \
+ "%s\n%s\n%s" % ('\n'.join(problem_descs), msg_tail, msg_footnote)
if arg1 == "":
print "###########################################################"
print "maintainer_email: %s\n" % maintainer_email
print "subject: %s\n" % subject
print msg
print ""
print "BBS> Notifying maintainer(s) of %s:" % pkg
sys.stdout.flush()
bbs.email.sendtextmail(from_addr, to_addrs, subject, msg)
return
def send_notification(pkg):
maintainer_email = BBSreportutils.get_pkg_field_from_meat_index(
pkg, 'MaintainerEmail')
#print "%s %s %s %s" % (pkg, version, maintainer, maintainer_email)
#key = 'Last Changed Date'
#last_changed_date = BBSreportutils.get_svn_info(pkg, key)
#key = 'Last Changed Rev'
#last_changed_rev = BBSreportutils.get_svn_info(pkg, key)
problem_descs = []
for node in BBSreportutils.supported_nodes(pkg):
stagecmd = 'install'
status = BBSreportutils.get_status_from_db(pkg, node.id, stagecmd)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rURL(
pkg, node.id, stagecmd)
problem_desc = " o %s for 'R CMD INSTALL' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
stagecmd = 'buildsrc'
status = BBSreportutils.get_status_from_db(pkg, node.id, stagecmd)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rURL(
pkg, node.id, stagecmd)
problem_desc = " o %s for 'R CMD build' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
stagecmd = 'checksrc'
status = BBSreportutils.get_status_from_db(pkg, node.id, stagecmd)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rURL(
pkg, node.id, stagecmd)
problem_desc = " o %s for 'R CMD check' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
if len(problem_descs) != 0:
if maintainer_email == "*****@*****.**":
to_addrs = ["*****@*****.**"]
else:
to_addrs = [maintainer_email]
subject = "%s problems reported by the \"%s\" for %s" \
% (pkg, main_page_title, BBSreportutils.get_build_label())
msg = "%s\nHi %s maintainer,\n\n" % (msg_head, pkg) \
+ "According to the \"%s\" for %s,\n" % (main_page_title, BBSreportutils.get_build_label()) \
+ "the %s package has the following problem(s):\n\n" % pkg \
+ "%s\n%s\n%s" % ('\n'.join(problem_descs), msg_tail, msg_footnote)
if arg1 == "":
print "###########################################################"
print "maintainer_email: %s\n" % maintainer_email
print "subject: %s\n" % subject
print msg
print ""
bbs.email.sendtextmail(from_addr, to_addrs, subject, msg)
return
def send_notification(pkg):
package_status = BBSreportutils.get_pkg_field_from_meat_index(pkg, 'PackageStatus')
if package_status == 'Deprecated':
return
maintainer_email = BBSreportutils.get_pkg_field_from_meat_index(pkg, 'MaintainerEmail')
#print "%s %s %s %s" % (pkg, version, maintainer, maintainer_email)
#key = 'Last Changed Date'
#last_changed_date = BBSreportutils.get_vcs_meta(pkg, key)
#key = 'Last Changed Rev'
#last_changed_rev = BBSreportutils.get_vcs_meta(pkg, key)
problem_descs = []
for node in BBSreportutils.supported_nodes(pkg):
stage = 'install'
status = BBSreportutils.get_status_from_db(pkg, node.id, stage)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rel_url(pkg, node.id, stage)
problem_desc = " o %s for 'R CMD INSTALL' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
stage = 'buildsrc'
status = BBSreportutils.get_status_from_db(pkg, node.id, stage)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rel_url(pkg, node.id, stage)
problem_desc = " o %s for 'R CMD build' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
stage = 'checksrc'
status = BBSreportutils.get_status_from_db(pkg, node.id, stage)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rel_url(pkg, node.id, stage)
problem_desc = " o %s for 'R CMD check' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
if len(problem_descs) == 0:
return
if maintainer_email == "*****@*****.**":
to_addrs = ["*****@*****.**"]
else:
to_addrs = [maintainer_email]
subject = "%s problems reported in the %s" % \
(pkg, BBSreportutils.REPORT_TITLE)
msg = "%s\nHi %s maintainer,\n\n" % (msg_head, pkg) \
+ "According to the %s,\n" % BBSreportutils.REPORT_TITLE \
+ "the %s package has the following problem(s):\n\n" % pkg \
+ "%s\n%s\n%s" % ('\n'.join(problem_descs), msg_tail, msg_footnote)
if arg1 == "":
print "###########################################################"
print "maintainer_email: %s\n" % maintainer_email
print "subject: %s\n" % subject
print msg
print ""
print "BBS> Notifying maintainer(s) of %s:" % pkg
sys.stdout.flush()
bbs.email.sendtextmail(from_addr, to_addrs, subject, msg)
return
def send_notification(pkg):
maintainer_email = BBSreportutils.get_pkg_field_from_meat_index(pkg, 'MaintainerEmail')
#print "%s %s %s %s" % (pkg, version, maintainer, maintainer_email)
#key = 'Last Changed Date'
#last_changed_date = BBSreportutils.get_svn_info(pkg, key)
#key = 'Last Changed Rev'
#last_changed_rev = BBSreportutils.get_svn_info(pkg, key)
problem_descs = []
for node in BBSreportutils.supported_nodes(pkg):
stagecmd = 'install'
status = BBSreportutils.get_status_from_db(pkg, node.id, stagecmd)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rURL(pkg, node.id, stagecmd)
problem_desc = " o %s for 'R CMD INSTALL' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
stagecmd = 'buildsrc'
status = BBSreportutils.get_status_from_db(pkg, node.id, stagecmd)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rURL(pkg, node.id, stagecmd)
problem_desc = " o %s for 'R CMD build' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
stagecmd = 'checksrc'
status = BBSreportutils.get_status_from_db(pkg, node.id, stagecmd)
if status in ["TIMEOUT", "ERROR"]:
leafreport_rURL = BBSreportutils.get_leafreport_rURL(pkg, node.id, stagecmd)
problem_desc = " o %s for 'R CMD check' on %s. See the details here:\n" % (status, node.id) \
+ " %s%s\n" % (BBSreportutils.data_source, leafreport_rURL)
problem_descs.append(problem_desc)
if len(problem_descs) != 0:
if maintainer_email == "*****@*****.**":
to_addrs = ["*****@*****.**"]
else:
to_addrs = [maintainer_email]
subject = "%s problems reported by the \"%s\" for %s" \
% (pkg, main_page_title, BBSreportutils.get_build_label())
msg = "%s\nHi %s maintainer,\n\n" % (msg_head, pkg) \
+ "According to the \"%s\" for %s,\n" % (main_page_title, BBSreportutils.get_build_label()) \
+ "the %s package has the following problem(s):\n\n" % pkg \
+ "%s\n%s\n%s" % ('\n'.join(problem_descs), msg_tail, msg_footnote)
if arg1 == "":
print "###########################################################"
print "maintainer_email: %s\n" % maintainer_email
print "subject: %s\n" % subject
print msg
print ""
bbs.email.sendtextmail(from_addr, to_addrs, subject, msg)
return
def get_status_from_summary_file(pkg, node_id, stage):
file = "%s.%s-summary.dcf" % (pkg, stage)
rdir = BBScorevars.nodes_rdir.subdir('%s/%s' % (node_id, stage))
try:
status = BBSreportutils.WReadDcfVal(rdir, file, 'Status')
except bbs.rdir.WOpenError:
if stage == "install":
return "NotNeeded"
return "NA"
return status
def collect_problems(pkg, node):
stage2command = {
'install': 'R CMD INSTALL',
'buildsrc': 'R CMD build',
'checksrc': 'R CMD check'
}
stages_on_report = BBSreportutils.stages_to_display(BBSvars.buildtype)
stages_to_collect = list(stage2command.keys() & stages_on_report)
problem_descs = []
for stage in stages_to_collect:
status = BBSreportutils.get_pkg_status(pkg, node.node_id, stage)
if status in ["TIMEOUT", "ERROR"]:
problem_desc = "%s for '%s' on %s" % \
(status, stage2command[stage], node.node_id)
rurl = BBSreportutils.get_leafreport_rel_url(
pkg, node.node_id, stage)
problem_desc = " o %s. See the details here:\n %s%s\n" % \
(problem_desc, published_report_url, rurl)
problem_descs.append(problem_desc)
return problem_descs
def get_status_from_summary_file(pkg, node_id, stagecmd):
file = "%s.%s-summary.dcf" % (pkg, stagecmd)
rdir = BBScorevars.nodes_rdir.subdir('%s/%s' % (node_id, stagecmd))
try:
status = BBSreportutils.WReadDcfVal(rdir, file, 'Status')
except urllib2.HTTPError:
if stagecmd == "install":
return "NotNeeded"
print "==> FATAL ERROR: cannot get %s status for package %s on %s" % (stagecmd, pkg, node_id)
sys.exit("==> EXIT")
return status
def send_notification(dcf_record):
pkg = dcf_record['Package']
package_status = dcf_record.get('PackageStatus')
if package_status == 'Deprecated':
return
maintainer_email = dcf_record['MaintainerEmail']
#print("%s %s %s %s" % (pkg, version, maintainer, maintainer_email))
#key = 'Last Changed Date'
#last_changed_date = BBSreportutils.get_vcs_meta(pkg, key)
#key = 'Last Changed Rev'
#last_changed_rev = BBSreportutils.get_vcs_meta(pkg, key)
problem_descs = []
for node in BBSreportutils.supported_nodes(pkg):
problem_descs += collect_problems(pkg, node)
if len(problem_descs) == 0:
return
report_nodes = BBSutils.getenv('BBS_REPORT_NODES')
report_title = BBSreportutils.make_report_title(report_nodes)
if maintainer_email == "*****@*****.**":
to_addrs = ["*****@*****.**"]
else:
to_addrs = [maintainer_email]
subject = "%s problems reported in the %s" % (pkg, report_title)
msg = "%s\nHi %s maintainer,\n\n" % (msg_head, pkg) \
+ "According to the %s,\n" % report_title \
+ "the %s package has the following problem(s):\n\n" % pkg \
+ "%s\n%s\n%s" % ('\n'.join(problem_descs), msg_tail, msg_footnote)
if arg1 == "":
print("###########################################################")
print("maintainer_email: %s\n" % maintainer_email)
print("subject: %s\n" % subject)
print(msg)
print()
print("BBS> Notifying maintainer(s) of %s:" % pkg)
sys.stdout.flush()
bbs.notify.sendtextmail(from_addr, to_addrs, subject, msg)
return
print "BBS> [stage9] STARTING stage9 at %s..." % time.asctime()
BBSreportutils.data_source = BBScorevars.getenv('BBS_PUBLISHED_REPORT_URL')
notify_nodes = BBScorevars.getenv('BBS_NOTIFY_NODES')
argc = len(sys.argv)
if argc > 1:
arg1 = sys.argv[1]
else:
arg1 = ""
if arg1 != "":
bbs.email.mode = "do-it"
if arg1 != "do-it":
bbs.email.redirect_to_addr = arg1
BBSreportutils.set_NODES(notify_nodes)
if len(BBSreportutils.NODES) != 1:
main_page_title = 'Multiple platform build/check report'
else:
main_page_title = 'Build/check report'
allpkgs = BBSreportutils.get_pkgs_from_meat_index()
print "BBS> [stage9] Notifying package maintainers for nodes: %s" % notify_nodes
if BBScorevars.mode in ["bioc", "biocLite", "data-experiment"]:
send_BioC_notifications(allpkgs)
else: # "cran" mode
send_CRAN_notifications(allpkgs)
print "BBS> [stage9] DONE at %s." % time.asctime()
print "BBS> [stage9] STARTING stage9 at %s..." % time.asctime()
BBSreportutils.data_source = BBScorevars.getenv('BBS_PUBLISHED_REPORT_URL')
notify_nodes = BBScorevars.getenv('BBS_NOTIFY_NODES')
argc = len(sys.argv)
if argc > 1:
arg1 = sys.argv[1]
else:
arg1 = ""
if arg1 != "":
bbs.email.mode = "do-it"
if arg1 != "do-it":
bbs.email.redirect_to_addr = arg1
BBSreportutils.set_NODES(notify_nodes)
if len(BBSreportutils.NODES) != 1:
main_page_title = 'Multiple platform build/check report'
else:
main_page_title = 'Build/check report'
allpkgs = BBSreportutils.get_pkgs_from_meat_index()
print "BBS> [stage9] Notifying package maintainers for nodes: %s" % notify_nodes
if BBScorevars.mode in ["bioc", "biocLite", "data-experiment"]:
send_BioC_notifications(allpkgs)
else: # "cran" mode
send_CRAN_notifications(allpkgs)
print "BBS> [stage9] DONE at %s." % time.asctime()
report_path = BBSutils.getenv('BBS_REPORT_PATH')
argc = len(sys.argv)
if argc > 1:
arg1 = sys.argv[1]
else:
arg1 = ""
if arg1 != "":
bbs.notify.mode = "do-it"
if arg1 != "do-it":
bbs.notify.redirect_to_addr = arg1
os.chdir(report_path)
BBSreportutils.set_NODES(notify_nodes)
### Compute 'meat_pkgs' (dict) and 'allpkgs' (list).
meat_index = bbs.parse.parse_DCF(BBSutils.meat_index_file)
meat_pkgs = {}
for dcf_record in meat_index:
meat_pkgs[dcf_record['Package']] = dcf_record
allpkgs = list(meat_pkgs.keys())
allpkgs.sort(key=str.lower)
print("BBS> [stage7] Import package statuses from %s ..." % \
BBSreportutils.BUILD_STATUS_DB_file, end=" ")
sys.stdout.flush()
BBSreportutils.import_BUILD_STATUS_DB(allpkgs)
print("OK")
sys.stdout.flush()
sys.stdout.flush()
out = open(BBSreportutils.BUILD_STATUS_DB_file, 'w')
for pkg in pkgs:
_write_pkg_results_to_BUILD_STATUS_DB(pkg, out)
out.close()
print('OK')
return
##############################################################################
### MAIN SECTION
##############################################################################
if __name__ == "__main__":
print()
if not os.path.isdir('products-in'):
print('mmh.. I don\'t see the \'products-in\' subdirectory ' + \
'in the current directory!')
print('Make sure to be in \'%s/\' ' % BBSvars.Central_rdir.path)
print('before running the BBS-make-BUILD_STATUS_DB.py script.')
sys.exit('=> EXIT.')
print(
'BBS> ==============================================================')
print('BBS> [stage6a] STARTING stage6a on %s...' % time.asctime())
sys.stdout.flush()
report_nodes = BBSutils.getenv('BBS_REPORT_NODES')
BBSreportutils.set_NODES(report_nodes)
pkgs = bbs.parse.get_meat_packages(BBSutils.meat_index_file)
make_BUILD_STATUS_DB(pkgs)
print('BBS> [stage6a] DONE on %s.' % time.asctime())
