def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""):
clp.add_option(
"--extra-repos", dest="extraRepos", type="string", default="",
help="List of names of extra repos to be cloned <extra-repos>" \
" (i.e. \"repo0,repo1,,...\"). When set to empty '' (the default value)" \
" then all repos that match <extra-repos-type> listed in <extra-repos-file>" \
" will be selected. But the repos listed in <extra-repos> must always" \
" be a subset of the repos of type <extra-repos-type> selected from" \
" <extra-repos-file>. (Default '')" )
clp.add_option(
"--not-extra-repos", dest="notExtraRepos", type="string", default="",
help="List of names of extra repos *NOT* to clone (i.e. \"repo0,repo1,...\")." \
" (Default '')" )
clp.add_option(
"--extra-repos-file", dest="extraReposFile", type="string",
default=g_extraRerposFileDefault,
help="The file path <extra-repos-file> for the ExtraRepositoriesList.cmake file." \
" This can be an absolute or relative path." \
" (Default = '"+g_extraRerposFileDefault+"')")
addOptionParserChoiceOption(
"--extra-repos-type", "extraReposType",
g_extraReposTypes , g_extraReposTypesDefaulIdx,
"Type of extra repositories <extra-repos-type> to select from " \
"<extra-repos-file>. When --extra-repos is set, then this argument" \
" is ignored.",
clp )
clp.add_option(
"--gitolite-root", dest="gitoliteRoot", type="string", default=gitoliteRootDefault,
help="Gives the root for a gitolite repos <gitolite-root> (e.g. git@<some-url>)." \
" If specified, then any git repos with the <gitolite-root> listed as their" \
" root will only be selected if they are listed with 'R' permissions returned" \
" from 'ssh <gitolite-root> info'. WARNING: Make sure that you have your" \
" gitoliote SSH registered correctly before using this option by typing" \
" the command 'ssh <gitlite-root> info' and make sure that it does *not*"
" ask for a password! (Default = '"+gitoliteRootDefault+"')" )
clp.add_option(
"--with-cmake", dest="withCmake", type="string", default="cmake",
help="CMake executable to use with cmake -P scripts internally (only set" \
+" by unit testing code). (Default = 'cmake')")
addOptionParserChoiceOption(
"--verbosity", "verbLevel", g_verbosityLevels, g_verbosityLevelDefaultIdx,
"Verbosity of the script (levels are cumulative):" \
" none = no output at all (except for commands with --no-op). " \
" minimal = print script args echo and clone commands." \
" more = print basic repo include/exclude logic and print repo table." \
" most = print output from cmake script called, the output from gitolite," \
" and other detailed info." \
,
clp )
clp.add_option(
"--do-clone", dest="doClone", action="store_true",
help="Do the clone of the selected repos. [default]")
clp.add_option(
"--skip-clone", dest="doClone", action="store_false",
help="Skip the clone of the repos and just show what would be done.",
default=True )
clp.add_option(
"--do-op", dest="doOp", action="store_true",
help="Do the clone of the selected repos. [default]" )
clp.add_option(
"--no-op", dest="doOp", action="store_false",
help="Skip cloning the repos and just show the clone commands.",
default=True )
clp.add_option(
"--create-gitdist-file", dest="createGitdistFile", type="string", default="",
help="If specified, the file <gitdist-file> will get generated with the list" \
" of git repos (the same list that is cloned with --do-clone)." \
" (Default = '')")
clp.add_option(
"--show-defaults", dest="showDefaults", action="store_true",
help="Show the default option values and do nothing at all.",
default=False )
def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""):
clp.add_option(
"--date", dest="date", type="string", default='yesterday',
help="Date for the testing day <YYYY-MM-DD> or special values 'today'"+\
" or 'yesterday'. [default 'yesterday']" )
clp.add_option(
"--cdash-project-testing-day-start-time", dest="cdashProjectTestingDayStartTime",
type="string", default="00:00",
help="The CDash project testing day build star time in UTC in format '<hh>:<mm>'."+\
" [default = '00:00'" )
clp.add_option(
"--cdash-project-name",
dest="cdashProjectName",
type="string",
default="",
help="CDash project name (e.g. 'Trilinos'). [REQUIRED] [default = '']")
clp.add_option(
"--build-set-name", dest="buildSetName", type="string", default="",
help="Name for the set of builds, (e.g. 'Trilinos Nightly Builds)."+\
" This used in the email summary line and in the HTML file body"+\
" to identify the set of builds and tests being examined."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-site-url", dest="cdashSiteUrl", type="string", default="",
help="Base CDash site (e.g. 'https://testing.sandia.gov/cdash')."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-builds-filters", dest="cdashBuildsFilters", type="string",
default="",
help="Partial URL fragment for index.php making of the filters for"+\
" the set of builds (e.g. 'filtercount=1&showfilters=1&field1=groupname&compare1=61&value1=ATDM')."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-nonpassed-tests-filters", dest="cdashNonpassedTestsFilters", type="string",
default="",
help="Partial URL fragment for queryTests.php making of the filters for"+\
" the set of non-passing tests matching this set of builds (e.g."+\
" 'filtercombine=and&filtercount=1&showfilters=1&filtercombine=and&field1=groupname&compare1=61&value1=ATDM')."+\
" This set of filter fields may also filter out extra nonpassing tests"+\
" such for know random system failures to avoid flooding the output. In this"+\
" case, one should also set --require-test-history-match-nonpassing-tests=off."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--expected-builds-file", dest="expectedBuildsFile", type="string",
default="",
help="Path to CSV file that lists the expected builds. Each of these builds"+\
" must have unique 'site' and 'buildname' field pairs or an error will be"+\
" raised and the tool will abort. [default = '']" )
clp.add_option(
"--tests-with-issue-trackers-file", dest="testsWithIssueTrackersFile",
type="string", default="",
help="Path to CSV file that lists tests with issue trackers (and other data)."+\
" Each of these tests must have a unique 'site', 'buildName', and 'testname'"+\
" sets or an error will be raised and the tool will abort. [default = '']" )
cdashQueriesCacheDir_default = os.getcwd()
clp.add_option(
"--cdash-queries-cache-dir", dest="cdashQueriesCacheDir", type="string",
default=cdashQueriesCacheDir_default,
help="Cache CDash query data this directory." \
+" [Default = '"+cdashQueriesCacheDir_default+"']" )
clp.add_option(
"--cdash-base-cache-files-prefix", dest="cdashBaseCacheFilesPrefix", type="string",
default="",
help="Prefix given to the base-level cache files outside of the test_history/"+\
" directory. This is to allow multiple invocations of this script to share"+\
" the same base cache directory and share the test_history/ in case there are"+\
" overrlapping sets of tests where the test history cache could be reused."+\
" [default is derived from the --build-set-name=<build_set_name> argument where"+\
" spaces and punctuation in <build_set_name> are replaced with '_']" )
addOptionParserChoiceOption(
"--use-cached-cdash-data", "useCachedCDashDataStr",
("on", "off"), 1,
"Use data downloaded from CDash already cached. Note that this only"+\
" impacts the reuse of base-level cache files and does not impact the usage"+\
" of test history cache in the <cacheDir>/test_history/ directory."+\
" If a test history file for a given testing day exists under the test_history/"+\
" directory it is used unconditionally.",
clp )
testHistoryDaysDefault = 30
clp.add_option(
"--limit-test-history-days", dest="testHistoryDays",
default=testHistoryDaysDefault, type="int",
help="Number of days to go back in history for each test."+\
" [default = '"+str(testHistoryDaysDefault)+"']" )
limitTableRows = 10
clp.add_option(
"--limit-table-rows", dest="limitTableRows", type="int",
default=limitTableRows,
help="Limit to the number of rows displayed in many of"+\
" the tables. This impacts tables like 'twoif' and 'twoinr'"+\
" that could have thousands of entries for some projects."+\
" This limits the number of tests for which detailed test history"+\
" is downloaded from CDash and is therefore important to ensure the"+\
" tool does not take too long to execute. However, this does NOT"+\
" limit the number of rows in many other tables that should be bounded like"+\
" any of the tables related to the builds or the list of tests with"+\
" issue trackers. (The number of those should never be extremely high.)"+\
" [default '"+str(limitTableRows)+"']" )
addOptionParserChoiceOption(
"--require-test-history-match-nonpassing-tests",
"requireTestHistoryMatchNonpassingTestsStr",
("on", "off"), 0,
"Require that the status for each tracked test listed in the tests with issue"\
+" trackers CSV file match the status of that test returned from the test history"\
+" returned from CDash. In general, these should match up but these may not if extra"\
+" filter criteria has been added to the list on nonpassing tests in the"\
+" --cdash-nonpassed-tests-filters=<filters> set of filters (such as to filter out"\
+" a large number of random system failures). In this case, an error will be"\
+" returned by default and the script will crash. But this can be relaxed by"\
+" setting this to 'off' which will result in these tracked tests being listed in"\
+" the 'twim' table but typically with status 'Failed'.",
clp )
addOptionParserChoiceOption(
"--print-details", "printDetailsStr",
("on", "off"), 1,
"Print more info like the CDash URLs for downloaded data and the cache"+\
" file names.",
clp )
clp.add_option(
"--write-failing-tests-without-issue-trackers-to-file",
dest="writeFailingTestsWithoutIssueTrackersToFile", type="string", default="",
help="Write a CSV file with a list of tets with issue trackers failed 'twif'." \
+" This is to make it easy to add new entires to the file read by" \
+" the option --tests-with-issue-trackers-file=<file>. [default = '']" )
clp.add_option(
"--write-test-data-to-file",
dest="writeTestDataToFile", type="string", default="",
help="Write pretty-printed Python list of dictionaries for tests with" \
+" with issue trackers. This includes the history of the tests for" \
+" --limit-test-history-days=<days> of history. This contains all of the" \
+" information that appears in the generated summary tables for tests with" \
+" associated issue trackers. [default = '']" )
clp.add_option(
"--write-email-to-file",
dest="writeEmailToFile",
type="string",
default="",
help="Write the body of the HTML email to this file. [default = '']")
clp.add_option("--email-from-address=",
dest="emailFromAddress",
type="string",
default="",
help="Address reported in the sent email. [default '']")
clp.add_option(
"--send-email-to=",
dest="sendEmailTo",
type="string",
default="",
help="Send email to 'address1, address2, ...'. [default '']")
def runDriver(self):
# Get the production version out of command-line
productBaseName = self.installObj.getProductBaseName()
versionCmndArgName = "--"+productBaseName+"-version"
for arg in sys.argv[1:]:
#print("arg = '"+arg+"'")
arg_and_value = arg.split("=")
if len(arg_and_value) and arg_and_value[0] == versionCmndArgName:
self.productVersion = arg_and_value[1].strip()
# Get basic info after knowing the version
productName = self.installObj.getProductName(self.productVersion)
baseDirName = self.installObj.getBaseDirName(self.productVersion)
scriptName = self.installObj.getScriptName();
#
# 1) Set up the help text
#
productSupportedVersions = self.installObj.getProductSupportedVersions()
if productSupportedVersions:
supportedVersionsTxt = "Versions supported include:\n "+\
str(self.installObj.getProductSupportedVersions())
else:
supportedVersionsTxt = "Arbitrary versions are supported."
usageHelp = scriptName+\
r""" [OPTIONS] [--install-dir=<install-dir> ...]
This script checks out source, untars, configures, builds, and installs
"""+productName+r""" in one shot. """+supportedVersionsTxt+r"""
The version to install is set with --"""+productBaseName+r"""-version=<version>.
By default, if you just type:
$ """+scriptName+r""" --install-dir=<install-dir> --parallel=<num-procs> --do-all
then the directory """+baseDirName+r""" will be created in the local working directory
and it will contain the source for """+productName+r""" and the build files. NOTE: This
requires that you not run as root or your userid on the download computer
will not be correct. If you want to install as root, see below.
You can control various parts of the process with various options (see below).
If you do not install as root then you must override the option --install-dir
which is set to /usr/local/bin by default. For example, you might just type:
$ """+scriptName+r""" --install-dir=$HOME/install --parallel=8 --do-all
and then it would install """+productName+r""" and the other executables in $HOME/install/bin.
NOTE: You will have to update your PATH variable to include whatever directory
you choose to install """+productName+r""" in.
NOTE: If you need to use sudo to install in /usr/local/bin or some other place
that needs root privileges, do:
$ """+scriptName+r""" --install-dir=$HOME/install --parallel=8 \
--download --untar --configure --build
$ sudo """+scriptName+r""" --install-dir=$HOME/install --parallel=8 \
--install-owner=<owner> --install-group=<group> [--install-for-all] \
--install
This appears to work on most systems.
After you have done a successful install, you might want to do:
$ rm -r """+baseDirName+r"""
in order to remove the intermediate source and build files.
""" + self.installObj.getExtraHelpStr(self.productVersion)
#
# 2) Parse the command-line
#
clp = OptionParser(usage=usageHelp)
supportedVersions = self.installObj.getProductSupportedVersions()
defaultVersion = self.installObj.getProductDefaultVersion()
defaultVersionIdx = findInSequence(supportedVersions, defaultVersion)
if supportedVersions:
addOptionParserChoiceOption(
versionCmndArgName, "version", supportedVersions, defaultVersionIdx,
"Version to install for "+productName+".", clp)
else:
clp.add_option(
versionCmndArgName, dest="version", type="string",
default=defaultVersion,
help="Version to install for "+productName+" (list of versions open-ended).")
clp.add_option(
"--install-dir", dest="installDir", type="string",
default="",
help="The install directory <install-dir> for "+productName+ \
" (default = ''). This can be a relative or absolute path, it can" \
" start with ~/, etc." )
clp.add_option(
"--install-dir-base", dest="installDirBase", type="string",
default="",
help="The base install directory <install-dir> for "+productName+ \
" (default = ''). In this case the subdir "+productName+\
" will be created under this directory for the install prefix." )
insertInstallPermissionsOptions(clp)
clp.add_option(
"--parallel", dest="parallel", type="int", \
default=0, \
help="Uses parallelism in build if set to > 0." )
clp.add_option(
"--make-options", dest="makeOptions", type="string",
default="",
help="The options to pass to make for "+productName+"." )
self.installObj.injectExtraCmndLineOptions(clp, self.productVersion)
clp.add_option(
"--show-defaults", dest="showDefaults", action="store_true", default=False,
help="[ACTION] Show the defaults and exit." )
clp.add_option(
"--download", dest="download", action="store_true", default=False,
help="[ACTION] Do the download of the tarball" )
clp.add_option(
"--untar", dest="untar", action="store_true", default=False,
help="[ACTION] Do the untar of the "+productName+" sources" )
clp.add_option(
"--configure", dest="configure", action="store_true", default=False,
help="[ACTION] Configure "+productName+" to build" )
clp.add_option(
"--build", dest="build", action="store_true", default=False,
help="[Action] Build "+productName+" and related executables" )
clp.add_option(
"--install", dest="install", action="store_true", default=False,
help="[ACTION] Install "+productName )
clp.add_option(
"--show-final-instructions", dest="showFinalInstructions", action="store_true",
default=False,
help="[ACTION] Show final instructions for using "+productName )
clp.add_option(
"--do-all", dest="doAll", action="store_true", default=False,
help="[AGGR ACTION] Same as --download --untar --configure --build --install" \
+" --show-final-instructions")
(options, args) = clp.parse_args()
#
# 3) Echo the command-line options
#
cmndLine = \
"******************************************************************************\n"
cmndLine += scriptName + " \\\n"
cmndLine += " "+versionCmndArgName + "='"+options.version+"' \\\n"
cmndLine += " --install-dir='" + options.installDir + "' \\\n"
cmndLine += " --install-dir-base='" + options.installDirBase + "' \\\n"
cmndLine += echoInsertPermissionsOptions(options)
cmndLine += " --parallel='" + str(options.parallel) + "' \\\n"
cmndLine += " --make-options='" + options.makeOptions + "'\\\n"
cmndLine += self.installObj.echoExtraCmndLineOptions(options)
if options.download:
cmndLine += " --download \\\n"
if options.untar:
cmndLine += " --untar \\\n"
if options.configure:
cmndLine += " --configure \\\n"
if options.build:
cmndLine += " --build \\\n"
if options.install:
cmndLine += " --install \\\n"
if options.showFinalInstructions:
cmndLine += " --show-final-instructions \\\n"
if options.doAll:
cmndLine += " --do-all \\\n"
print(cmndLine)
if options.showDefaults:
return 0;
# Check the options
if options.installDirBase != "" and options.installDir == "":
options.installDir=options.installDirBase+"/"+productName
elif options.installDir == "":
raise Exception("Error, --install-dir=<install-dir> can't be empty!")
options.installDir = os.path.abspath(os.path.expanduser(options.installDir))
#
# 4) Execute the commands
#
if options.doAll:
options.download = True
options.untar = True
options.configure = True
options.build = True
options.install = True
options.showFinalInstructions = True
baseDir = os.getcwd()
productBaseDir = baseDir+"/"+baseDirName
self.installObj.setup(options)
print("")
print("A) Download the source for "+productName+" ...")
print("")
if options.download:
self.installObj.doDownload()
else:
print("Skipping on request ...")
print("")
print("B) Untar the tarball(s) and set up ready to configure ...")
print("")
if options.untar:
self.installObj.doUntar()
else:
print("Skipping on request ...")
print("")
print("C) Configure "+productName+" ...")
print("")
if options.configure:
self.installObj.doConfigure()
else:
print("Skipping on request ...")
print("")
print("D) Build "+productName+" ...")
print("")
if options.build:
self.installObj.doBuild()
else:
print("Skipping on request ...")
print("")
print("E) Install "+productName+" ...")
print("")
if options.install:
self.installObj.doInstall()
fixupInstallPermissions(options, options.installDir)
else:
print("Skipping on request ...")
print("")
print("D) Final instructions for using "+productName+" ...")
print("")
if options.showFinalInstructions:
print(self.installObj.getFinalInstructions())
else:
print("Skipping on request ...")
print("\n[End]")
def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""):
yesterday = (datetime.date.today() +
datetime.timedelta(days=-1)).isoformat()
clp.add_option(
"--date", dest="date", type="string", default=yesterday,
help="Date for the testing day <YYYY-MM-DD>."+\
" [default yesterday '"+yesterday+"']" )
clp.add_option(
"--cdash-project-name",
dest="cdashProjectName",
type="string",
default="",
help="CDash project name (e.g. 'Trilinos'). [REQUIRED] [default = '']")
clp.add_option(
"--build-set-name", dest="buildSetName", type="string", default="",
help="Name for the set of builds, (e.g. 'Trilinos Nightly Builds)."+\
" This used in the email summary line and in the HTML file body"+\
" to identify the set of builds and tests being examined."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-site-url", dest="cdashSiteUrl", type="string", default="",
help="Base CDash site (e.g. 'https://testing.sandia.gov/cdash')."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-builds-filters", dest="cdashBuildsFilters", type="string",
default="",
help="Partial URL fragment for index.php making of the filters for"+\
" the set of builds (e.g. 'filtercount=1&showfilters=1&field1=groupname&compare1=61&value1=ATDM')."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-nonpassed-tests-filters", dest="cdashNonpassedTestsFilters", type="string",
default="",
help="Partial URL fragment for queryTests.php making of the filters for"+\
" the set of non-passing tests matching this set of builds (e.g."+\
" 'filtercombine=and&filtercount=1&showfilters=1&filtercombine=and&field1=groupname&compare1=61&value1=ATDM'"+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--expected-builds-file", dest="expectedBuildsFile", type="string",
default="",
help="Path to CSV file that lists the expected builds. Each of these builds"+\
" must have unique 'site' and 'buildname' field pairs or an error will be"+\
" raised and the tool will abort. [default = '']" )
clp.add_option(
"--tests-with-issue-trackers-file", dest="testsWithIssueTrackersFile",
type="string", default="",
help="Path to CSV file that lists tests with issue trackers (and other data)."+\
" Each of these tests must have a unique 'site', 'buildName', and 'testname'"+\
" sets or an error will be raised and the tool will abort. [default = '']" )
cdashQueriesCacheDir_default = os.getcwd()
clp.add_option(
"--cdash-queries-cache-dir", dest="cdashQueriesCacheDir", type="string",
default=cdashQueriesCacheDir_default,
help="Cache CDash query data this directory." \
+" [Default = '"+cdashQueriesCacheDir_default+"']" )
clp.add_option(
"--cdash-base-cache-files-prefix", dest="cdashBaseCacheFilesPrefix", type="string",
default="",
help="Prefix given to the base-level cache files outside of the test_history/"+\
" directory. This is to allow multiple invocations of this script to share"+\
" the same base cache directory and share the test_history/ in case there are"+\
" overrlapping sets of tests where the test history cache could be reused."+\
" [default is derived from the --build-set-name=<build_set_name> argument where"+\
" spaces and punctuation in <build_set_name> are replaced with '_']" )
addOptionParserChoiceOption(
"--use-cached-cdash-data", "useCachedCDashDataStr",
("on", "off"), 1,
"Use data downloaded from CDash already cached. Note that this only"+\
" impacts the reuse of base-level cache files and does not impact the usage"+\
" of test history cache in the <cacheDir>/test_history/ directory."+\
" If a test history file for a given testing day exists under the test_history/"+\
" directory it is used unconditionally.",
clp )
testHistoryDaysDefault = 30
clp.add_option(
"--limit-test-history-days", dest="testHistoryDays",
default=testHistoryDaysDefault, type="int",
help="Number of days to go back in history for each test."+\
" [default = '"+str(testHistoryDaysDefault)+"']" )
limitTableRows = 10
clp.add_option(
"--limit-table-rows", dest="limitTableRows", type="int",
default=limitTableRows,
help="Limit to the number of rows displayed in many of"+\
" the tables. This impacts tables like 'twoif' and 'twoinr'"+\
" that could have thousands of entries for some projects."+\
" This limits the number of tests for which detailed test history"+\
" is downloaded from CDash and is therefore important to ensure the"+\
" tool does not take too long to execute. However, this does NOT"+\
" limit the number of rows in many other tables that should be bounded like"+\
" any of the tables related to the builds or the list of tests with"+\
" issue trackers. (The number of those should never be extremely high.)"+\
" [default '"+str(limitTableRows)+"']" )
addOptionParserChoiceOption(
"--print-details", "printDetailsStr",
("on", "off"), 1,
"Print more info like the CDash URLs for downloaded data and the cache"+\
" file names.",
clp )
clp.add_option(
"--write-failing-tests-without-issue-trackers-to-file",
dest="writeFailingTestsWithoutIssueTrackersToFile", type="string", default="",
help="Write CSV file with a list of tets with issue trackers failed 'twif'." \
+" This is to make it easy to add new entires to the file read by" \
+" the option --tests-with-issue-trackers-file=<file>. [default = '']" )
clp.add_option(
"--write-email-to-file",
dest="writeEmailToFile",
type="string",
default="",
help="Write the body of the HTML email to this file. [default = '']")
clp.add_option("--email-from-address=",
dest="emailFromAddress",
type="string",
default="",
help="Address reported in the sent email. [default '']")
clp.add_option(
"--send-email-to=",
dest="sendEmailTo",
type="string",
default="",
help="Send email to 'address1, address2, ...'. [default '']")
def runDriver(self):
# Get the production version out of command-line
productBaseName = self.installObj.getProductBaseName()
versionCmndArgName = "--"+productBaseName+"-version"
for arg in sys.argv[1:]:
#print("arg = '"+arg+"'")
arg_and_value = arg.split("=")
if len(arg_and_value) and arg_and_value[0] == versionCmndArgName:
self.productVersion = arg_and_value[1].strip()
# Get basic info after knowing the version
productName = self.installObj.getProductName(self.productVersion)
baseDirName = self.installObj.getBaseDirName(self.productVersion)
scriptName = self.installObj.getScriptName();
#
# 1) Set up the help text
#
usageHelp = scriptName+\
r""" [OPTIONS] [--install-dir=<install-dir> ...]
This script checks out source, untars, configures, builds, and installs
"""+productName+r""" in one shot. Versions supported include:
"""+str(self.installObj.getProductSupportedVersions())+r"""
(set with --"""+productBaseName+r"""-version=<version>)
By default, if you just type:
$ """+scriptName+r""" --install-dir=<install-dir> --parallel=<num-procs> --do-all
then the directory """+baseDirName+r""" will be created in the local working directory
and it will contain the source for """+productName+r""" and the build files. NOTE: This
requires that you not run as root or your userid on the download computer
will not be correct. If you want to install as root, see below.
You can control various parts of the process with various options (see below).
If you do not install as root then you must override the option --install-dir
which is set to /usr/local/bin by default. For example, you might just type:
$ """+scriptName+r""" --install-dir=$HOME/install --parallel=8 --do-all
and then it would install """+productName+r""" and the other executables in $HOME/install/bin.
NOTE: You will have to update your PATH variable to include whatever directory
you choose to install """+productName+r""" in.
NOTE: If you need to use sudo to install in /usr/local/bin or some other place
that needs root privileges, do:
$ """+scriptName+r""" --install-dir=$HOME/install --parallel=8 \
--download --untar --configure --build
$ sudo """+scriptName+r""" --install-dir=$HOME/install --parallel=8 \
--install-owner=<owner> --install-group=<group> [--install-for-all] \
--install
This appears to work on most systems.
After you have done a successful install, you might want to do:
$ rm -r """+baseDirName+r"""
in order to remove the intermediate source and build files.
""" + self.installObj.getExtraHelpStr(self.productVersion)
#
# 2) Parse the command-line
#
clp = OptionParser(usage=usageHelp)
supportedVersions = self.installObj.getProductSupportedVersions()
defaultVersion = self.installObj.getProductDefaultVersion()
defaultVersionIdx = findInSequence(supportedVersions, defaultVersion)
addOptionParserChoiceOption(
versionCmndArgName, "version", supportedVersions, defaultVersionIdx,
"Version to install for "+productName+".", clp)
clp.add_option(
"--install-dir", dest="installDir", type="string",
default="/usr/local",
help="The install directory <install-dir> for "+productName+ \
" (default = /usr/local). This can be a relative or absolute path, it can" \
" start with ~/, etc." )
insertInstallPermissionsOptions(clp)
clp.add_option(
"--parallel", dest="parallel", type="int", \
default=0, \
help="Uses parallelism in build if set to > 0." )
clp.add_option(
"--make-options", dest="makeOptions", type="string",
default="",
help="The options to pass to make for "+productName+"." )
self.installObj.injectExtraCmndLineOptions(clp, self.productVersion)
clp.add_option(
"--show-defaults", dest="showDefaults", action="store_true", default=False,
help="[ACTION] Show the defaults and exit." )
clp.add_option(
"--download", dest="download", action="store_true", default=False,
help="[ACTION] Do the download of the tarball" )
clp.add_option(
"--untar", dest="untar", action="store_true", default=False,
help="[ACTION] Do the untar of the "+productName+" sources" )
clp.add_option(
"--configure", dest="configure", action="store_true", default=False,
help="[ACTION] Configure "+productName+" to build" )
clp.add_option(
"--build", dest="build", action="store_true", default=False,
help="[Action] Build "+productName+" and related executables" )
clp.add_option(
"--install", dest="install", action="store_true", default=False,
help="[ACTION] Install "+productName )
clp.add_option(
"--show-final-instructions", dest="showFinalInstructions", action="store_true",
default=False,
help="[ACTION] Show final instructions for using "+productName )
clp.add_option(
"--do-all", dest="doAll", action="store_true", default=False,
help="[AGGR ACTION] Same as --download --untar --configure --build --install" \
+" --show-final-instructions")
(options, args) = clp.parse_args()
#
# 3) Echo the command-line options
#
cmndLine = "******************************************************************************\n"
cmndLine += scriptName + " \\\n"
cmndLine += " "+versionCmndArgName + "='"+options.version+"' \\\n"
cmndLine += " --install-dir='" + options.installDir + "' \\\n"
cmndLine += echoInsertPermissionsOptions(options)
cmndLine += " --parallel='" + str(options.parallel) + "' \\\n"
cmndLine += " --make-options='" + options.makeOptions + "'\\\n"
cmndLine += self.installObj.echoExtraCmndLineOptions(options)
if options.download:
cmndLine += " --download \\\n"
if options.untar:
cmndLine += " --untar \\\n"
if options.configure:
cmndLine += " --configure \\\n"
if options.build:
cmndLine += " --build \\\n"
if options.install:
cmndLine += " --install \\\n"
if options.showFinalInstructions:
cmndLine += " --show-final-instructions \\\n"
if options.doAll:
cmndLine += " --do-all \\\n"
print(cmndLine)
if options.showDefaults:
return 0;
# Check the options
if options.installDir == "":
raise Exception("Error, --install-dir=<install-dir> can't be empty!")
options.installDir = os.path.abspath(os.path.expanduser(options.installDir))
#
# 4) Execute the commands
#
if options.doAll:
options.download = True
options.untar = True
options.configure = True
options.build = True
options.install = True
options.showFinalInstructions = True
baseDir = os.getcwd()
productBaseDir = baseDir+"/"+baseDirName
self.installObj.setup(options)
print("")
print("A) Download the source for "+productName+" ...")
print("")
if options.download:
self.installObj.doDownload()
else:
print("Skipping on request ...")
print("")
print("B) Untar the tarball(s) and set up ready to configure ...")
print("")
if options.untar:
self.installObj.doUntar()
else:
print("Skipping on request ...")
print("")
print("C) Configure "+productName+" ...")
print("")
if options.configure:
self.installObj.doConfigure()
else:
print("Skipping on request ...")
print("")
print("D) Build "+productName+" ...")
print("")
if options.build:
self.installObj.doBuild()
else:
print("Skipping on request ...")
print("")
print("E) Install "+productName+" ...")
print("")
if options.install:
self.installObj.doInstall()
fixupInstallPermissions(options, options.installDir)
else:
print("Skipping on request ...")
print("")
print("D) Final instructions for using "+productName+" ...")
print("")
if options.showFinalInstructions:
print(self.installObj.getFinalInstructions())
else:
print("Skipping on request ...")
print("\n[End]")
This script returns a comma-separated list of all of the project's top-level
or packages or the full set of SE packages (i.e. parent and subpackages).
"""
from optparse import OptionParser
clp = OptionParser(usage=usageHelp)
clp.add_option(
"--deps-xml-file", dest="depsXmlFile", type="string",
help="File containing TriBITS-generated XML data-structure the listing"+\
" of packages, dir names, dependencies, etc.")
addOptionParserChoiceOption(
"--only-top-level-packages", "onlyTopLevelPackagesStr",
("on", "off"), 0,
"If 'on', then only top-level packages will be included. If 'off', then"+\
" top-level and subpackages will be included in the list (in order).",
clp )
(options, args) = clp.parse_args()
if options.onlyTopLevelPackagesStr == "on":
onlyTopLevelPackages = True
else:
onlyTopLevelPackages = False
trilinosDependencies = getProjectDependenciesFromXmlFile(options.depsXmlFile)
packagesNamesList = trilinosDependencies.getPackagesNamesList(
onlyTopLevelPackages)
def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""):
yesterday = (datetime.date.today()+datetime.timedelta(days=-1)).isoformat()
clp.add_option(
"--date", dest="date", type="string", default=yesterday,
help="Date for the testing day <YYYY-MM-DD>."+\
" [default yesterday '"+yesterday+"']" )
clp.add_option(
"--cdash-project-name", dest="cdashProjectName", type="string", default="",
help="CDash project name (e.g. 'Trilinos'). [REQUIRED] [default = '']" )
clp.add_option(
"--build-set-name", dest="buildSetName", type="string", default="",
help="Name for the set of builds, (e.g. 'Trilinos Nightly Builds)."+\
" This used in the email summary line and in the HTML file body"+\
" to identify the set of builds and tests being examined."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-site-url", dest="cdashSiteUrl", type="string", default="",
help="Base CDash site (e.g. 'https://testing.sandia.gov/cdash')."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-builds-filters", dest="cdashBuildsFilters", type="string",
default="",
help="Partial URL fragment for index.php making of the filters for"+\
" the set of builds (e.g. 'filtercount=1&showfilters=1&field1=groupname&compare1=61&value1=ATDM')."+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--cdash-nonpassed-tests-filters", dest="cdashNonpassedTestsFilters", type="string",
default="",
help="Partial URL fragment for queryTests.php making of the filters for"+\
" the set of non-passing tests matching this set of builds (e.g."+\
" 'filtercombine=and&filtercount=1&showfilters=1&filtercombine=and&field1=groupname&compare1=61&value1=ATDM'"+\
" [REQUIRED] [default = '']" )
clp.add_option(
"--expected-builds-file", dest="expectedBuildsFile", type="string",
default="",
help="Path to CSV file that lists the expected builds. Each of these builds"+\
" must have unique 'site' and 'buildname' field pairs or an error will be"+\
" raised and the tool will abort. [default = '']" )
clp.add_option(
"--tests-with-issue-trackers-file", dest="testsWithIssueTrackersFile",
type="string", default="",
help="Path to CSV file that lists tests with issue trackers (and other data)."+\
" Each of these tests must have a unique 'site', 'buildName', and 'testname'"+\
" sets or an error will be raised and the tool will abort. [default = '']" )
cdashQueriesCacheDir_default=os.getcwd()
clp.add_option(
"--cdash-queries-cache-dir", dest="cdashQueriesCacheDir", type="string",
default=cdashQueriesCacheDir_default,
help="Cache CDash query data this directory." \
+" [Default = '"+cdashQueriesCacheDir_default+"']" )
clp.add_option(
"--cdash-base-cache-files-prefix", dest="cdashBaseCacheFilesPrefix", type="string",
default="",
help="Prefix given to the base-level cache files outside of the test_history/"+\
" directory. This is to allow multiple invocations of this script to share"+\
" the same base cache directory and share the test_history/ in case there are"+\
" overrlapping sets of tests where the test history cache could be reused."+\
" [default is derived from the --build-set-name=<build_set_name> argument where"+\
" spaces and punctuation in <build_set_name> are replaced with '_']" )
addOptionParserChoiceOption(
"--use-cached-cdash-data", "useCachedCDashDataStr",
("on", "off"), 1,
"Use data downloaded from CDash already cached. Note that this only"+\
" impacts the reuse of base-level cache files and does not impact the usage"+\
" of test history cache in the <cacheDir>/test_history/ directory."+\
" If a test history file for a given testing day exists under the test_history/"+\
" directory it is used unconditionally.",
clp )
testHistoryDaysDefault= 30
clp.add_option(
"--limit-test-history-days", dest="testHistoryDays",
default=testHistoryDaysDefault, type="int",
help="Number of days to go back in history for each test."+\
" [default = '"+str(testHistoryDaysDefault)+"']" )
limitTableRows = 10
clp.add_option(
"--limit-table-rows", dest="limitTableRows", type="int",
default=limitTableRows,
help="Limit to the number of rows displayed in many of"+\
" the tables. This impacts tables like 'twoif' and 'twoinr'"+\
" that could have thousands of entries for some projects."+\
" This limits the number of tests for which detailed test history"+\
" is downloaded from CDash and is therefore important to ensure the"+\
" tool does not take too long to execute. However, this does NOT"+\
" limit the number of rows in many other tables that should be bounded like"+\
" any of the tables related to the builds or the list of tests with"+\
" issue trackers. (The number of those should never be extremely high.)"+\
" [default '"+str(limitTableRows)+"']" )
addOptionParserChoiceOption(
"--print-details", "printDetailsStr",
("on", "off"), 1,
"Print more info like the CDash URLs for downloaded data and the cache"+\
" file names.",
clp )
clp.add_option(
"--write-failing-tests-without-issue-trackers-to-file",
dest="writeFailingTestsWithoutIssueTrackersToFile", type="string", default="",
help="Write CSV file with a list of tets with issue trackers failed 'twif'." \
+" This is to make it easy to add new entires to the file read by" \
+" the option --tests-with-issue-trackers-file=<file>. [default = '']" )
clp.add_option(
"--write-email-to-file", dest="writeEmailToFile", type="string", default="",
help="Write the body of the HTML email to this file. [default = '']" )
clp.add_option(
"--email-from-address=", dest="emailFromAddress", type="string", default="",
help="Address reported in the sent email. [default '']" )
clp.add_option(
"--send-email-to=", dest="sendEmailTo", type="string", default="",
help="Send email to 'address1, address2, ...'. [default '']" )