def Verify(self):
conf = self.config
distDir = JoinPaths(GetObjDir(conf), 'dist')
delivsFound = FindDeliverables(distDir, conf)
if delivsFound <= 0:
raise self.SimpleStepError("No deliverables found after installer "
"build?")
else:
print "Deliverables found in %s: %d" % (distDir, delivsFound)
for d in GetAllDeliverables():
print "Name %s -> %s" % (d.name, d.file)
installerObj = GetDeliverable('installer:linux:en-US')
if installerObj is None:
raise self.SimpleStepError("No installer found after installer "
"build")
else:
tmpDir = tempfile.gettempdir()
print "Copying installer %s to %s" % (installerObj, tmpDir)
shutil.copy(installerObj.fileName, tmpDir)
# You probably don't need to do this, but as an example of some
# of the utility functions...
ourInstallerHash = GetSHA1FileHash(installerObj.fileName)
tmpInstallerHash = GetSHA1FileHash(JoinPaths(tmpDir,
installerObj.file))
if ourInstallerHash != tmpInstallerHash:
raise self.SimpleStepError(
"Hashes do not match; orig: %s; copy: %s" %
(ourInstallerHash, tmpInstallerHash))
def VerifyFirefoxDownload(conf):
sourceFile = conf.Get('source_download_file')
sourceFileFullPath = JoinPaths(conf.rootDir, sourceFile)
if not os.path.isfile(sourceFileFullPath):
raise ValueError("Couldn't find downloaded firefox source code: %s" %
(sourceFileFullPath))
sha1SumsFile = os.path.basename(conf.Get('sha1_checksum_download_url'))
sha1SumHandle = open(sha1SumsFile, 'r')
sumFilePath = "./source/%s" % (sourceFile)
sourceSha1 = None
for line in sha1SumHandle.readlines():
(sha1, filename) = line.split(None, 1)
if filename.strip() == sumFilePath:
sourceSha1 = sha1
break
sha1SumHandle.close()
if sourceSha1 is None:
raise ValueError("Couldn't find entry for %s in %s" %
(sumFilePath, sha1SumsFile))
downloadSha1 = GetSHA1FileHash(sourceFileFullPath)
if sourceSha1 != downloadSha1:
raise ValueError("Download doesn't match expected SHA1: expected: %s; "
"download checksum: %s" % (sourceSha1, downloadSha1))
def VerifySuccessfulFirefoxBuild(conf):
firefoxBuildTestFiles = conf.Get('build_test_files', list)
distDir = JoinPaths(GetObjDir(conf), 'dist')
try:
VerifyFileList(firefoxBuildTestFiles, distDir)
except ValueError, ex:
raise ValueError("Expected Firefox build output missing: %s" % (ex))
def VerifyFileList(fileList, commonPrefix=None):
for f in fileList:
testFile = f
if commonPrefix is not None:
testFile = JoinPaths(commonPrefix, f)
if not os.path.isfile(testFile):
raise ValueError(testFile)
def Execute(self):
conf = self.config
sourceTarball = JoinPaths(
conf.rootDir, os.path.basename(conf.Get('source_download_url')))
cmd = [conf.GetConstant('TAR'), '-xvjf', sourceTarball]
rv = LoggedShellCommand(command=cmd, workdir=conf.rootDir)
def Verify(self):
for f in self.dlFiles:
# Probably shouldn't use os.path.basename here for realsies;
# should really use urlparse, but for this simple case, it
# seems to work fine.
fileName = os.path.basename(f)
if not os.path.isfile(fileName):
raise self.SimpleStepError("Key/checksum file %s missing." %
(fileName))
keyFile = JoinPaths(
os.getcwd(),
os.path.basename(
self.config.Get('sha1_checksum_sig_download_url')))
sha1SumsFile = JoinPaths(
os.getcwd(),
os.path.basename(self.config.Get('sha1_checksum_download_url')))
validationReqd = self.config.Get('require_pgp_validation', bool)
# Could (should?) probably do this via PyCrypto, but for example-
# purposes, I'm feeling lazy.
gpgCommand = self.config.GetConstant('GPG')
cmd = [gpgCommand, '--verify', keyFile, sha1SumsFile]
rv = None
try:
rv = LoggedShellCommand(command=cmd, raiseErrors=False)
except ReleaseFrameworkError, ex:
if str(ex) == "Invalid command or working dir":
if validationReqd:
raise self.SimpleStepError(
"%s validation required, but "
"it looks like PGP isn't installed. Please install it."
% (gpgCommand))
print >> sys.stderr, (
"It appears %s isn't installed. "
"Checksum cannot be considered valid. Continuing anyway..."
% (gpgCommand))
def _GetNextStepCommandLogfileName(self):
stepLogDir = GetAppLogger().stepLogDirectory
if stepLogDir is None:
raise RuntimeError("BLAH")
runShellCommandLogs = []
for entry in os.listdir(stepLogDir):
if os.path.isfile(entry) and re.match(
'^%s\d+' % (_COMMAND_LOGFILE_PREFIX), entry):
runShellCommandLogs.append(entry)
return JoinPaths(
stepLogDir, '%s%d.txt' %
(_COMMAND_LOGFILE_PREFIX, len(runShellCommandLogs) + 1))
def _HandleStepLogHandler(self, givenKwargs):
if _DIR_OUTPUT not in self._loggingConfig.keys():
return
# Handles the case where someone grabbed the logger from GetAppLogger(),
# and wants to print an error message; in this case (for now), we'll
# print the log message to wherever the other log output is going,
# which seems reasonable, but may also bury log output.
#
# Hrm...
if 'step' not in givenKwargs:
return
step = givenKwargs['step']
if step.name == self._currentStepName:
return
if self._currentStepLogHandler is not None:
self.logHandle.removeHandler(self._currentStepLogHandler)
self._currentStepLogHandler.close()
self._currentStepLogHandler = None
self._currentStepName = step.name
self._currentStepLogDir = self._GetStepLogDir(step)
if not os.path.exists(stepLogDir):
Makedirs(stepLogDir)
stepLogFile = JoinPaths(self._currentStepLogDir, _STEP_LOG_FILE)
fh = None
try:
fh = logging.FileHandler(stepLogFile, mode='w')
fh.setLevel(self._loggingConfig[_DIR_OUTPUT])
fh.setFormatter(self._formatter)
fh.addFilter(NoCommandOutputFilter())
except IOError, ex:
raise _QuickReleaseError(ex)
def GetObjDir(conf):
return JoinPaths(GetSourceDirRoot(conf), conf.Get('objdir'))
def _GetMozconfigFilename(self):
conf = self.config
return JoinPaths(GetSourceDirRoot(conf), conf.Get('mozconfig_file'))
def GetSourceDirRoot(conf):
return JoinPaths(conf.rootDir, conf.Get('source_root_dir'))
testFile = JoinPaths(commonPrefix, f)
if not os.path.isfile(testFile):
raise ValueError(testFile)
def VerifyFirefoxBuildConfigured(conf):
autoconfTestFiles = conf.Get('autoconf_output_testfiles', list)
objDir = GetObjDir(conf)
try:
VerifyFileList(autoconfTestFiles, objDir)
except ValueError, ex:
raise ValueError("Autoconf test file not present: %s" % (ex))
confStatusFile = JoinPaths(objDir, conf.Get('autoconf_status_file'))
lastLine = None
confStatusFileHandle = open(confStatusFile, 'r')
for l in confStatusFileHandle:
lastLine = l
confStatusFileHandle.close()
if lastLine.strip() != 'exit 0':
raise ValueError("Last %s line didn't match successful exit: %s" %
(confStatusFile, lastLine))
def VerifySuccessfulFirefoxBuild(conf):
firefoxBuildTestFiles = conf.Get('build_test_files', list)
distDir = JoinPaths(GetObjDir(conf), 'dist')
Deliverable.ERROR_STR_NEED_NAME_OR_REGEX %
DeliverableClassFromSectionName(section))
#print "f is %s, name is %s, regex is %s" % (f, delivName, delivRegex)
if ((delivName is not None and f == delivName) or
(delivRegex is not None and re.search(delivRegex, f,
regexFlags))):
if 'subclass' in sectionItems:
subclassType = config.SectionGet(section,
'subclass').strip()
delivClassDescription = {
'type': matchType,
'subclass' : subclassType,
'class' : DeliverableClassFromSectionName(section),
'file' : JoinPaths(root, f),
}
deliverableDescList.append(delivClassDescription)
if len(deliverableDescList) == 0:
if not ignoreUndefinedDeliverables:
assert False, "Should be a release framework error."
else:
continue
if len(deliverableDescList) == 1:
delivDesc = deliverableDescList[0]
if delivDesc['subclass'] is not None:
def GetAvailableProcesses():
"""
Search for and instantiate all of the defined QuickRelease L{Process}es.
B{Note}: for performance reasons, a cache is used to store these L{Process}es after the first call to C{GetAvailableProcesses()}. There is currently no way to clear this cache.
@return: A tuple of references to all defined QuickRelease L{Process}es in the provided "processes" directories.
@rtype: C{tuple} of L{Process}
@raise ReleaseFrameworkError: A L{ReleaseFrameworkError<quickrelease.exception.ReleaseFrameworkError>} is raised in the following circumstances:
1. A L{Process} lists a L{Step<quickrelease.step.Step>}, but that Step is not defined.
2. A L{Process} is attempting to import a module which has not been defined.
3. A L{Process} is attempting to import a L{Step<quickrelease.step.Step>}, but that Step contains a syntax error.
4. Other problems may cause L{NameError}s, L{ImportError}s, or L{SyntaxError}s to be recast into L{ReleaseFrameworkError<quickrelease.exception.ReleaseFrameworkError>}s with the original error message.
"""
if Process._gAvailableProcessList is None:
Process._gAvailableProcessList = []
for path in gProcessAndStepDefnPath:
#print "Checking process/step definition path: %s" % (path)
cwd = os.getcwd()
processModuleFiles = []
try:
os.chdir(JoinPaths(path, QUICKRELEASE_PROCESSES_DIR))
processModuleFiles = glob('*.py')
finally:
os.chdir(cwd)
processModuleFiles.remove(INIT_PY)
if len(processModuleFiles) <= 0:
continue
filenameToModuleName = lambda f: '.'.join([
os.path.basename(path), QUICKRELEASE_PROCESSES_DIR,
os.path.splitext(f)[0]
])
moduleFiles = map(filenameToModuleName, processModuleFiles)
processList = []
for f in moduleFiles:
#print "Importing process module: " + f
try:
mod = ImportModule(f)
except NameError, ex:
nameErrorRegex = "name '(\w+)' is not defined"
nameErrorMatch = re.match(nameErrorRegex, str(ex))
if nameErrorMatch:
raise ReleaseFrameworkError(
"Step %s is specified as "
"part of process %s, but is not defined" %
(nameErrorMatch.group(1), f.split('.')[-1]))
else:
raise ReleaseFrameworkError(
"%s: NameError: %s" % (f, ex), ex)
except ImportError, ex:
importErrorRegex = "No module named (.+)"
importErrorMatch = re.match(importErrorRegex, str(ex))
if importErrorMatch:
raise ReleaseFrameworkError(
"Process %s is trying to "
"import undefined module %s" %
(f, importErrorMatch.group(1)))
else:
raise ReleaseFrameworkError(
"%s: ImportError: %s" % (f, ex), ex)
except SyntaxError, ex:
definitionType = None
parentDir = os.path.basename(os.path.dirname(ex.filename))
processDetailStr = ""
if parentDir == QUICKRELEASE_PROCESSES_DIR:
definitionType = "process"
elif parentDir == QUICKRELEASE_STEPS_DIR:
definitionType = "step"
processDetailStr = " (part of process %s)" % (
f.split('.')[-1])
raise ReleaseFrameworkError(
"Syntax error in %s "
"definition %s%s, line %d:\n%s" %
(definitionType, os.path.basename(ex.filename),
processDetailStr, ex.lineno, ex.text))
for attr in dir(mod):
obj = getattr(mod, attr)
if inspect.isclass(obj):
if obj.__module__ == f and issubclass(obj, Process):
#print "Process class found: %s" % (obj.__name__)
processList.append(obj)
if os.getenv('QUICKRELEASE_DISABLE_IMPORT_DASH_WARNING') is None:
raise ReleaseFrameworkError(IMPORT_PATH_DASH_WARNING_STR)
absPath = os.path.abspath(path)
gProcessAndStepDefnPath.append(absPath)
sys.path.append(os.path.dirname(absPath))
if os.getenv('QUICKRELEASE_OVERRIDE_DEFAULT_DEFINITIONS') is None:
gProcessAndStepDefnPath.append(os.path.dirname(os.path.abspath(__file__)))
INIT_PY_MISSING_ERR_STR = (
"The specified directory '%s' in "
"QUICKRELEASE_DEFINITIONS_PATH is missing an __init__.py file; please add "
"one.")
for path in gProcessAndStepDefnPath:
if not os.path.isfile(JoinPaths(path, INIT_PY)):
raise RuntimeWarning(INIT_PY_MISSING_ERR_STR % (path))
for d in (QUICKRELEASE_PROCESSES_DIR, QUICKRELEASE_STEPS_DIR):
checkDir = JoinPaths(path, d)
if not os.path.isdir(checkDir):
raise RuntimeWarning(
"The specified directory '%s' in "
"QUICKRELEASE_DEFINITIONS_PATH is missing the '%s' directory; "
"bailing." % (path, d))
elif not os.path.isfile(JoinPaths(checkDir, INIT_PY)):
raise RuntimeWarning(INIT_PY_MISSING_ERR_STR % (checkDir))
class Process(object):
"""
An object representing an automated process.