def findItems(self): '''Find all the items verify their checksums, and download anything that is missing''' for thisSectionName in self.packageGroups: for thisItem in self.packageGroups[thisSectionName]: # progressReporter progressReporter = displayTools.statusHandler(taskMessage=' ' + thisItem.displayName + ' -') thisItem.findItem(progressReporter=progressReporter)
def findItems(self):
'''Find all the items verify their checksums, and download anything that is missing'''
for thisSectionName in self.packageGroups:
for thisItem in self.packageGroups[thisSectionName]:
# progressReporter
progressReporter = displayTools.statusHandler(
taskMessage=' ' + thisItem.displayName + ' -')
thisItem.findItem(progressReporter=progressReporter)
elif options.overwriteExisting is False:
choice = raw_input('File already exists: %s\nOverwrite this file? (Y/N):' % targetPath)
if not choice.lower() in ['y', 'yes']:
print("Canceling")
sys.exit()
# Note: at this point it is all-right to overwrite the file if it exists
if options.automaticRun is False:
choice = raw_input('Ready to produce "%s" from the volume "%s".\n\tContinue? (Y/N):' % (chosenFileName, chosenMount.getWorkingPath()))
if not choice.lower() in ['y', 'yes']:
print("Canceling")
sys.exit()
myStatusHandler = displayTools.statusHandler(taskMessage='Creating image from disc at: %s' % chosenMount.getWorkingPath())
diskFormat = 'UDZO' # default to zlib
if options.compressionType == "bzip2":
diskFormat = 'UDBZ'
diskutilArguments = ['/usr/bin/hdiutil', 'create', '-ov', '-srcowners', 'on', '-srcfolder', chosenMount.getWorkingPath(), '-format', diskFormat]
if options.compressionType == "zlib":
# go for more compression
diskutilArguments.append('-imagekey')
diskutilArguments.append('zlib-level=6')
diskutilArguments.append(targetPath)
diskutilProcess = managedSubprocess(diskutilArguments)
myStatusHandler.update(taskMessage='Image "%s" created in %s' % (chosenFileName, displayTools.secondsToReadableTime(time.time() - startTime)))
'Item was not a format that this tool supports: ' + location)
if parsedURL.scheme is '' and location[-1] == '/':
location = location[:-1]
thisOutputLocation = None
if options.outputFolder is not None and parsedURL.scheme in [
'http', 'https'
]:
thisOutputLocation = options.outputFolder
progressReporter = None
if options.reportProgress is True:
if parsedURL.scheme in ['http', 'https']:
progressReporter = statusHandler(
taskMessage=os.path.basename(parsedURL.path) + " ")
else:
progressReporter = statusHandler(
taskMessage=os.path.basename(location) + " ")
data = checksum(location,
checksumType=options.checksumAlgorithm,
progressReporter=progressReporter,
outputFolder=thisOutputLocation,
checksumInFileName=options.checksumInFileName)
dataLine = ""
normalizedPath = pathHelpers.normalizePath(location)
# in the standardCacheFolder
def parseCatalogFile(self, fileLocation=None):
if fileLocation is None:
fileLocation = self.catalogFilePath
# the file passed could be an absolute path, a relative path, or a catalog file name
# the first two are handled without a special section, but the name needs some work
fileLocation = self.getCatalogFullPath(
fileLocation, self.catalogFolders
) # there should not be an error here, since we have already validated it
# note: this last will have taken care of downloading any remote files
assert os.path.isfile(
fileLocation
), "There was no file where it was expected to be: %s" % fileLocation
# check to make sure we are not in a loop
assert fileLocation not in self.parsedFiles, 'Loop detected in catalog files: %s' % fileLocation
self.parsedFiles.append(fileLocation)
inputfile = open(fileLocation, "r")
if inputfile == None:
raise Exception('Unable to open input file: %s' %
inputFilePath) # TODO: improve error handling
currentSection = None
lineNumber = 0
# parse through the file
for line in inputfile.readlines():
lineNumber += 1
if self.emptyLineParser.search(line):
continue
# ---- settings lines
settingLineMatch = self.settingLineParser.search(line)
if settingLineMatch is not None:
# get the variable name this would be
variableName = settingLineMatch.group("variableName").split(
)[0].lower() + "".join(x.capitalize()
for x in settingLineMatch.group(
"variableName").split()[1:])
# sanity check that this variable exists
if not hasattr(self, variableName):
raise InstallerChoicesFileException(
'The %s class does not have a "%s" variable as it should have. This error should never happen.'
% (self.__class__.__name__, variableName),
choicesFile=fileLocation,
lineNumber=lineNumber)
if hasattr(getattr(self, variableName), '__iter__'):
# if this variable as an array, append to it
getattr(self, variableName).append(
settingLineMatch.group(
"variableValue")) # amazingly, this works
elif getattr(self, variableName) is None:
setattr(self, variableName,
settingLineMatch.group("variableValue"))
# note: if the variable is already set, ignore this value
continue
# ---- settings lines with checksums
settingLineMatch = self.settingLineChecksumParser.search(line)
if settingLineMatch is not None:
# get the variable name this would be
variableName = settingLineMatch.group("variableName").split(
)[0].lower() + "".join(x.capitalize()
for x in settingLineMatch.group(
"variableName").split()[1:])
checksumType = settingLineMatch.group('checksumType')
checksumValue = settingLineMatch.group('checksumValue')
# find this item in the caches by name/checksum
progressReporter = displayTools.statusHandler(
taskMessage='\t%s: %s -' %
(settingLineMatch.group("variableName"),
settingLineMatch.group("variableValue")))
itemPath = None
try:
itemPath = cacheController.findItem(
settingLineMatch.group("variableValue"),
checksumType,
checksumValue,
progressReporter=progressReporter)
except commonExceptions.FileNotFoundException:
raise commonExceptions.InstallerChoicesFileException(
'Unable to find %s: %s (checksum: %s)' %
(settingLineMatch.group("variableName"),
settingLineMatch.group("variableValue"),
settingLineMatch.group("fileChecksum")),
choicesFile=fileLocation,
lineNumber=lineNumber)
# sanity check that this variable exists
if not hasattr(self, variableName + 'Path'):
raise Exception(
'The %s class does not have a "%s" variable as it should have. This error should never happen.'
% (self.__class__.__name__, variableName + 'Path'))
if hasattr(getattr(self, variableName + 'Path'), '__iter__'):
# if this variable as an array, append to it
getattr(self, variableName + 'Path').append(
itemPath) # amazingly, this works
elif getattr(self, variableName + 'Path') is None:
setattr(self, variableName + 'Path', itemPath)
# note: if the variable is already set, ignore this value
continue
# ---- file includes lines
includeLineMatch = self.includeLineParser.search(line)
if includeLineMatch:
self.parseCatalogFile(
self.getCatalogFullPath(includeLineMatch.group("location"),
self.catalogFolders))
continue
# ---- section lines
sectionTitleMatch = self.sectionStartParser.search(line)
if sectionTitleMatch:
if sectionTitleMatch.group(
"sectionName"
) not in self.packageGroups and sectionTitleMatch.group(
"sectionName") != "Base OS Disk":
raise Exception(
'Unknown section title: "%s" on line: %i of file: %s\n%s'
% (sectionTitleMatch.group("sectionName"), lineNumber,
fileLocation, line)) # TODO: improve error handling
currentSection = sectionTitleMatch.group("sectionName")
continue
# ---- item lines
packageLineMatch = self.packageLineParser.search(line)
if packageLineMatch:
if currentSection == None:
# we have to have a place to put this
raise Exception('Every item must belong to a section'
) # TODO: improve error handling
thisPackage = installerPackage(
displayName=packageLineMatch.group("displayName"),
sourceLocation=packageLineMatch.group("fileLocation"),
checksumString=packageLineMatch.group("fileChecksum"),
installerChoices=packageLineMatch.group(
"installerChoicesFile"))
print('\t' + packageLineMatch.group("displayName"))
self.packageGroups[currentSection].append(thisPackage)
continue
# if we got here, the line was not good
raise Exception('Error in config file: %s line number: %i\n%s' %
(fileLocation, lineNumber,
line)) # TODO: improve error handling
inputfile.close()
def parseCatalogFile(self, fileLocation=None):
if fileLocation is None:
fileLocation = self.catalogFilePath
# the file passed could be an absolute path, a relative path, or a catalog file name
# the first two are handled without a special section, but the name needs some work
fileLocation = self.getCatalogFullPath(fileLocation, self.catalogFolders) # there should not be an error here, since we have already validated it
# note: this last will have taken care of downloading any remote files
assert os.path.isfile(fileLocation), "There was no file where it was expected to be: %s" % fileLocation
# check to make sure we are not in a loop
assert fileLocation not in self.parsedFiles, 'Loop detected in catalog files: %s' % fileLocation
self.parsedFiles.append(fileLocation)
inputfile = open(fileLocation, "r")
if inputfile == None:
raise Exception('Unable to open input file: %s' % inputFilePath) # TODO: improve error handling
currentSection = None;
lineNumber = 0
# parse through the file
for line in inputfile.readlines():
lineNumber += 1
if self.emptyLineParser.search(line):
continue
# ---- settings lines
settingLineMatch = self.settingLineParser.search(line)
if settingLineMatch is not None:
# get the variable name this would be
variableName = settingLineMatch.group("variableName").split()[0].lower() + "".join(x.capitalize() for x in settingLineMatch.group("variableName").split()[1:])
# sanity check that this variable exists
if not hasattr(self, variableName):
raise InstallerChoicesFileException('The %s class does not have a "%s" variable as it should have. This error should never happen.' % (self.__class__.__name__, variableName), choicesFile=fileLocation, lineNumber=lineNumber)
if hasattr(getattr(self, variableName), '__iter__'):
# if this variable as an array, append to it
getattr(self, variableName).append(settingLineMatch.group("variableValue")) # amazingly, this works
elif getattr(self, variableName) is None:
setattr(self, variableName, settingLineMatch.group("variableValue"))
# note: if the variable is already set, ignore this value
continue
# ---- settings lines with checksums
settingLineMatch = self.settingLineChecksumParser.search(line)
if settingLineMatch is not None:
# get the variable name this would be
variableName = settingLineMatch.group("variableName").split()[0].lower() + "".join(x.capitalize() for x in settingLineMatch.group("variableName").split()[1:])
checksumType = settingLineMatch.group('checksumType')
checksumValue = settingLineMatch.group('checksumValue')
# find this item in the caches by name/checksum
progressReporter = displayTools.statusHandler(taskMessage='\t%s: %s -' % (settingLineMatch.group("variableName"), settingLineMatch.group("variableValue")))
itemPath = None
try:
itemPath = cacheController.findItem(settingLineMatch.group("variableValue"), checksumType, checksumValue, progressReporter=progressReporter)
except commonExceptions.FileNotFoundException:
raise commonExceptions.InstallerChoicesFileException('Unable to find %s: %s (checksum: %s)' % (settingLineMatch.group("variableName"), settingLineMatch.group("variableValue"), settingLineMatch.group("fileChecksum")), choicesFile=fileLocation, lineNumber=lineNumber)
# sanity check that this variable exists
if not hasattr(self, variableName + 'Path'):
raise Exception('The %s class does not have a "%s" variable as it should have. This error should never happen.' % (self.__class__.__name__, variableName + 'Path'))
if hasattr(getattr(self, variableName + 'Path'), '__iter__'):
# if this variable as an array, append to it
getattr(self, variableName + 'Path').append(itemPath) # amazingly, this works
elif getattr(self, variableName + 'Path') is None:
setattr(self, variableName + 'Path', itemPath)
# note: if the variable is already set, ignore this value
continue
# ---- file includes lines
includeLineMatch = self.includeLineParser.search(line)
if includeLineMatch:
self.parseCatalogFile( self.getCatalogFullPath(includeLineMatch.group("location"), self.catalogFolders) )
continue
# ---- section lines
sectionTitleMatch = self.sectionStartParser.search(line)
if sectionTitleMatch:
if sectionTitleMatch.group("sectionName") not in self.packageGroups and sectionTitleMatch.group("sectionName") != "Base OS Disk":
raise Exception('Unknown section title: "%s" on line: %i of file: %s\n%s' % (sectionTitleMatch.group("sectionName"), lineNumber, fileLocation, line) ) # TODO: improve error handling
currentSection = sectionTitleMatch.group("sectionName")
continue
# ---- item lines
packageLineMatch = self.packageLineParser.search(line)
if packageLineMatch:
if currentSection == None:
# we have to have a place to put this
raise Exception('Every item must belong to a section') # TODO: improve error handling
thisPackage = installerPackage(
displayName = packageLineMatch.group("displayName"),
sourceLocation = packageLineMatch.group("fileLocation"),
checksumString = packageLineMatch.group("fileChecksum"),
installerChoices = packageLineMatch.group("installerChoicesFile")
)
print('\t' + packageLineMatch.group("displayName"))
self.packageGroups[currentSection].append(thisPackage)
continue
# if we got here, the line was not good
raise Exception('Error in config file: %s line number: %i\n%s' % (fileLocation, lineNumber, line)) # TODO: improve error handling
inputfile.close()
if parsedURL.scheme not in ['', 'http', 'https']:
optionParser.error('Item was not a format that this tool supports: ' + location)
if parsedURL.scheme is '' and location[-1] == '/':
location = location[:-1]
thisOutputLocation = None
if options.outputFolder is not None and parsedURL.scheme in ['http', 'https']:
thisOutputLocation = options.outputFolder
progressReporter = None
if options.reportProgress is True:
if parsedURL.scheme in ['http', 'https']:
progressReporter = statusHandler(taskMessage=os.path.basename(parsedURL.path) + " ")
else:
progressReporter = statusHandler(taskMessage=os.path.basename(location) + " ")
data = checksum(location, checksumType=options.checksumAlgorithm, progressReporter=progressReporter, outputFolder=thisOutputLocation, checksumInFileName=options.checksumInFileName)
dataLine = ""
normalizedPath = pathHelpers.normalizePath(location)
# in the standardCacheFolder
if parsedURL.scheme is '' and pathHelpers.pathInsideFolder(location, commonConfiguration.standardCacheFolder) and hasattr(os.path, 'relpath'): # relpath is python 2.6
dataLine = "\t".join(["", os.path.splitext(data['name'])[0], os.path.relpath(normalizedPath, commonConfiguration.standardCacheFolder), data['checksumType'] + ":" + data['checksum']])
# in the standardUserItemsFolder
if parsedURL.scheme is '' and pathHelpers.pathInsideFolder(location, commonConfiguration.standardUserItemsFolder) and hasattr(os.path, 'relpath'): # relpath is python 2.6
if not choice.lower() in ['y', 'yes']:
print("Canceling")
sys.exit()
# Note: at this point it is all-right to overwrite the file if it exists
if options.automaticRun is False:
choice = raw_input(
'Ready to produce "%s" from the volume "%s".\n\tContinue? (Y/N):' %
(chosenFileName, chosenMount.getWorkingPath()))
if not choice.lower() in ['y', 'yes']:
print("Canceling")
sys.exit()
myStatusHandler = displayTools.statusHandler(
taskMessage='Creating image from disc at: %s' %
chosenMount.getWorkingPath())
diskFormat = 'UDZO' # default to zlib
if options.compressionType == "bzip2":
diskFormat = 'UDBZ'
diskutilArguments = [
'/usr/bin/hdiutil', 'create', '-ov', '-srcowners', 'on', '-srcfolder',
chosenMount.getWorkingPath(), '-format', diskFormat
]
if options.compressionType == "zlib":
# go for more compression
diskutilArguments.append('-imagekey')
diskutilArguments.append('zlib-level=6')
diskutilArguments.append(targetPath)