Beispiel #1
0
    def getManagedPathForPath(myClass, targetPath):
        '''Find any managed items that already house this path'''

        if targetPath is None or not hasattr(targetPath, 'capitalize'):
            raise ValueError(
                'getManagedPathForPath recieved a target path that it could not understand: '
                + str(targetPath))

        targetPath = pathHelpers.normalizePath(targetPath)

        # rewind back through folders untill we get a path that actually exists
        existingPath = targetPath
        while not os.path.exists(existingPath):
            existingPath = os.path.dirname(existingPath)

        for thisMangedPath in myClass.managedItems:

            if os.path.exists(targetPath) and os.path.samefile(
                    targetPath, thisMangedPath):
                return thisMangedPath

            elif os.path.samefile(existingPath, thisMangedPath):
                return thisMangedPath

            elif os.path.isdir(thisMangedPath) and existingPath.lower(
            ).startswith(
                    pathHelpers.normalizePath(thisMangedPath).lower() +
                    os.sep) and os.lstat(existingPath)[
                        stat.ST_DEV] == os.lstat(thisMangedPath)[stat.ST_DEV]:
                return thisMangedPath

        return None
Beispiel #2
0
	def getManagedPathForPath(myClass, targetPath):
		'''Find any managed items that already house this path'''
		
		if targetPath is None or not hasattr(targetPath, 'capitalize'):
			raise ValueError('getManagedPathForPath recieved a target path that it could not understand: ' + str(targetPath))
		
		targetPath = pathHelpers.normalizePath(targetPath)
		
		# rewind back through folders untill we get a path that actually exists
		existingPath = targetPath
		while not os.path.exists(existingPath):
			existingPath = os.path.dirname(existingPath)
		
		for thisMangedPath in myClass.managedItems:
			
			if os.path.exists(targetPath) and os.path.samefile(targetPath, thisMangedPath):
				return thisMangedPath
			
			elif os.path.samefile(existingPath, thisMangedPath):
				return thisMangedPath
			
			elif os.path.isdir(thisMangedPath) and existingPath.lower().startswith( pathHelpers.normalizePath(thisMangedPath).lower() + os.sep ) and os.lstat(existingPath)[stat.ST_DEV] == os.lstat(thisMangedPath)[stat.ST_DEV]:
				return thisMangedPath
		
		return None
	def test_plainFiles(self):
		'''Test that a random file will be deleted'''
		
		(fileHandle, filePath) = tempfile.mkstemp()
		os.close(fileHandle) # we don't need to write anything to this
		filePath = pathHelpers.normalizePath(filePath, followSymlink=True)
		
		# add it, and confirm that this is managed
		tempFolderManager.addManagedItem(filePath)
		self.assertTrue(filePath in tempFolderManager.managedItems, 'Adding a file using addManagedItem did not result in it being put in managedItems')
		self.assertTrue(tempFolderManager.getManagedPathForPath(filePath) is not None, 'Adding a file using addManagedItem did not make it managed (according to getManagedPathForPath)')
		
		# wipe this out using cleanupItem
		tempFolderManager.cleanupItem(filePath)
		self.assertFalse(os.path.exists(filePath), 'Removing a file added with addManagedItem with cleanupItem did not get rid of the file')
		
		# repeat the exercise for cleanupForExit
		(fileHandle, filePath) = tempfile.mkstemp()
		os.close(fileHandle) # we don't need to write anything to this
		filePath = pathHelpers.normalizePath(filePath, followSymlink=True)
		
		# add it, and confirm that this is managed
		tempFolderManager.addManagedItem(filePath)
		self.assertTrue(filePath in tempFolderManager.managedItems, 'Adding a file using addManagedItem did not result in it being put in managedItems')
		self.assertTrue(tempFolderManager.getManagedPathForPath(filePath) is not None, 'Adding a file using addManagedItem did not make it managed (according to getManagedPathForPath)')
		
		# wipe this out using cleanupItem
		tempFolderManager.cleanupForExit()
		self.assertFalse(os.path.exists(filePath), 'Removing a file added with addManagedItem with cleanupForExit did not get rid of the file')
Beispiel #4
0
	def isManagedItem(myClass, thisItem):
		
		if thisItem is None or not os.path.exists(thisItem):
			raise ValueError('isManagedMount requires a valid item, got: ' + str(thisItem))
		
		thisItem = pathHelpers.normalizePath(thisItem)
		
		for thisMount in myClass.managedMounts:
			if pathHelpers.normalizePath(thisMount) == thisItem:
				return True
		
		return myClass.isManagedMount(thisItem)
Beispiel #5
0
	def isManagedMount(myClass, mountPoint):
		
		if mountPoint is None or not os.path.ismount(mountPoint):
			raise ValueError('isManagedMount requires a valid mount point, got: ' + str(mountPoint))
		
		mountPoint = pathHelpers.normalizePath(mountPoint)
		
		for thisMount in myClass.managedMounts:
			if pathHelpers.normalizePath(thisMount) == mountPoint:
				return True
		
		return False
Beispiel #6
0
    def isManagedItem(myClass, thisItem):

        if thisItem is None or not os.path.exists(thisItem):
            raise ValueError('isManagedMount requires a valid item, got: ' +
                             str(thisItem))

        thisItem = pathHelpers.normalizePath(thisItem)

        for thisMount in myClass.managedMounts:
            if pathHelpers.normalizePath(thisMount) == thisItem:
                return True

        return myClass.isManagedMount(thisItem)
Beispiel #7
0
    def isManagedMount(myClass, mountPoint):

        if mountPoint is None or not os.path.ismount(mountPoint):
            raise ValueError(
                'isManagedMount requires a valid mount point, got: ' +
                str(mountPoint))

        mountPoint = pathHelpers.normalizePath(mountPoint)

        for thisMount in myClass.managedMounts:
            if pathHelpers.normalizePath(thisMount) == mountPoint:
                return True

        return False
	def test_getNewTempFolder(self):
		'''Test the getNewTempFolder method'''
		
		# test without any input
		firstFolderPath = tempFolderManager.getNewTempFolder()
		self.assertTrue(firstFolderPath is not None, 'Called with no options getNewTempFolder gave back None')
		self.assertTrue(os.path.isdir(firstFolderPath), 'Called with no options getNewTempFolder returned a string that was not a path to an existing folder: ' + str(firstFolderPath))
		self.assertTrue(tempFolderManager.getManagedPathForPath(firstFolderPath) is not None, 'Called with no options getNewTempFolder returned a path that was not in any managed path (according to getManagedPathForPath): ' + firstFolderPath)
		
		# test with the parent folder option
		secondParentFolder = tempfile.mkdtemp(dir='/tmp')
		secondFolderPath = tempFolderManager.getNewTempFolder(parentFolder=secondParentFolder)
		self.assertTrue(secondFolderPath is not None, 'Called with a parent folder getNewTempFolder gave back None')
		self.assertTrue(os.path.isdir(secondFolderPath), 'Called with a parent folder getNewTempFolder returned a string that was not a path to an existing folder: ' + str(secondFolderPath))
		self.assertTrue(secondFolderPath.startswith(pathHelpers.normalizePath(secondParentFolder, followSymlink=True)), 'Called with a parent folder (%s) getNewTempFolder returned a path not in the parent folder: %s' % (secondParentFolder, secondFolderPath))
		self.assertTrue(tempFolderManager.getManagedPathForPath(secondFolderPath) is not None, 'Called with a parent folder getNewTempFolder returned a path that was not in any managed path (according to getManagedPathForPath): ' + secondFolderPath)
		
		# test with the prefix option
		prefixOption = "thisIsATest"
		thirdFolderPath = tempFolderManager.getNewTempFolder(prefix=prefixOption)
		self.assertTrue(thirdFolderPath is not None, 'Called with the prefix option (%s) getNewTempFolder gave back None' % prefixOption)
		self.assertTrue(os.path.isdir(thirdFolderPath), 'Called with the prefix option (%s) getNewTempFolder returned a string that was not a path to an existing folder: %s' % (prefixOption, str(thirdFolderPath)))
		self.assertTrue(os.path.basename(thirdFolderPath).startswith(prefixOption), 'Called with the prefix option (%s) getNewTempFolder returned a path not in the parent folder: %s' % (prefixOption, thirdFolderPath))
		self.assertTrue(tempFolderManager.getManagedPathForPath(secondFolderPath) is not None, 'Called with the prefix option (%s) getNewTempFolder returned a path that was not in any managed path (according to getManagedPathForPath): %s' % (prefixOption, thirdFolderPath))
		
		# call cleanupAtExit to clear everything
		tempFolderManager.cleanupForExit()
		
		# verify that the folders dissapeared
		self.assertFalse(os.path.exists(firstFolderPath), 'After being created with getNewTempFolder using no options the folder path was not cleaned properly by cleanupForExit: ' + firstFolderPath)
		self.assertFalse(os.path.exists(secondFolderPath), 'After being created with getNewTempFolder using the parent folder the folder optionthe path was not cleaned properly by cleanupForExit: ' + secondFolderPath)
		self.assertFalse(os.path.exists(thirdFolderPath), 'After being created with getNewTempFolder using the prefix option (%s) the folder path was not cleaned properly by cleanupForExit: %s' % (prefixOption, thirdFolderPath))
		
		# remove the tempdir we made for the parent folder test
		shutil.rmtree(secondParentFolder)
Beispiel #9
0
    def findManagedItemsInsideDirectory(myClass, targetDirectory):
        '''Return a list of managed items below this path in rough order that they should be cleaned'''

        targetDirectory = pathHelpers.normalizePath(targetDirectory)

        if not os.path.isdir(targetDirectory) or os.path.islink(
                targetDirectory):
            raise ValueError(
                'findManagedItemsInsideDirectory requires a directory as the input, and it can not be a symlink'
                + str(targetDirectory))

        itemsToClean = []
        if myClass.managedItems is not None:
            for thisItem in myClass.managedItems:
                if pathHelpers.pathInsideFolder(thisItem, targetDirectory):
                    itemsToClean.append(thisItem)
        # Note: this should mean that once sorted and reversed that mounted items get taken care of before their mount-points
        if myClass.managedMounts is not None:
            for thisMount in myClass.managedMounts:
                if pathHelpers.pathInsideFolder(thisMount, targetDirectory):
                    itemsToClean.append(thisMount)

        itemsToClean.sort(
        )  # items inside others should now be below their parents
        itemsToClean.reverse()  # the opposite should now be true

        return itemsToClean
Beispiel #10
0
    def test_plainFiles(self):
        '''Test that a random file will be deleted'''

        (fileHandle, filePath) = tempfile.mkstemp()
        os.close(fileHandle)  # we don't need to write anything to this
        filePath = pathHelpers.normalizePath(filePath, followSymlink=True)

        # add it, and confirm that this is managed
        tempFolderManager.addManagedItem(filePath)
        self.assertTrue(
            filePath in tempFolderManager.managedItems,
            'Adding a file using addManagedItem did not result in it being put in managedItems'
        )
        self.assertTrue(
            tempFolderManager.getManagedPathForPath(filePath) is not None,
            'Adding a file using addManagedItem did not make it managed (according to getManagedPathForPath)'
        )

        # wipe this out using cleanupItem
        tempFolderManager.cleanupItem(filePath)
        self.assertFalse(
            os.path.exists(filePath),
            'Removing a file added with addManagedItem with cleanupItem did not get rid of the file'
        )

        # repeat the exercise for cleanupForExit
        (fileHandle, filePath) = tempfile.mkstemp()
        os.close(fileHandle)  # we don't need to write anything to this
        filePath = pathHelpers.normalizePath(filePath, followSymlink=True)

        # add it, and confirm that this is managed
        tempFolderManager.addManagedItem(filePath)
        self.assertTrue(
            filePath in tempFolderManager.managedItems,
            'Adding a file using addManagedItem did not result in it being put in managedItems'
        )
        self.assertTrue(
            tempFolderManager.getManagedPathForPath(filePath) is not None,
            'Adding a file using addManagedItem did not make it managed (according to getManagedPathForPath)'
        )

        # wipe this out using cleanupItem
        tempFolderManager.cleanupForExit()
        self.assertFalse(
            os.path.exists(filePath),
            'Removing a file added with addManagedItem with cleanupForExit did not get rid of the file'
        )
Beispiel #11
0
	def removeManagedItem(myClass, targetPath):
		'''Remove an item from the list of managed items'''
		
		targetPath = pathHelpers.normalizePath(targetPath)
		
		if not targetPath in myClass.managedItems:
			raise ValueError('removeManagedItem called with a targetPath that was not a managed path: ' + targetPath)
		
		myClass.managedItems.remove(targetPath)
Beispiel #12
0
    def addManagedItem(myClass, targetPath):
        '''Add an item to be watched over'''

        targetPath = pathHelpers.normalizePath(targetPath)

        if myClass.getManagedPathForPath(targetPath) is not None:
            return  # we are already managing this space

        # the item exists, and is not otherwise accounted for
        myClass.managedItems.append(targetPath)
Beispiel #13
0
	def addManagedItem(myClass, targetPath):
		'''Add an item to be watched over'''
		
		targetPath = pathHelpers.normalizePath(targetPath)
		
		if myClass.getManagedPathForPath(targetPath) is not None:
			return # we are already managing this space

		# the item exists, and is not otherwise accounted for
		myClass.managedItems.append(targetPath)
Beispiel #14
0
    def getNewTempItem(myClass,
                       fileOrFolder,
                       parentFolder=None,
                       prefix=None,
                       suffix=None,
                       supressCreation=False):
        '''Create a new managed file or folder and return the path'''

        if not fileOrFolder in ['file', 'folder']:
            raise ValueError(
                'getNewTempITem only accepts "file" or "folder" for the fileOrFolder attribute, was given: '
                + str(fileOrFolder))

        pathToReturn = None

        if prefix is None:
            if fileOrFolder == 'file':
                prefix = myClass.tempFilePrefix
            else:
                prefix = myClass.tempFolderPrefix

        if suffix is None:
            suffix = ''

        if parentFolder is None:
            # create a new folder/file inside the current default one
            parentFolder = myClass.getDefaultFolder()

        elif not os.path.isdir(str(parentFolder)):
            raise ValueError(
                'getNewTempFolder called with a parentFolder path that does not exist is is not a directory: '
                + str(parentFolder))

        if supressCreation is True:
            pathToReturn = tempfile.mktemp(dir=parentFolder,
                                           prefix=prefix,
                                           suffix=suffix)
        elif fileOrFolder == 'file':
            openFile, pathToReturn = tempfile.mkstemp(dir=parentFolder,
                                                      prefix=prefix,
                                                      suffix=suffix)
            os.close(openFile)
        else:
            # create the new folder
            pathToReturn = tempfile.mkdtemp(dir=parentFolder, prefix=prefix)

        pathToReturn = pathHelpers.normalizePath(pathToReturn)

        # make sure that this file/folder is in managed space
        if myClass.getManagedPathForPath(pathToReturn) is None:
            # this is an unmanaged path, and we need to add it
            myClass.addManagedItem(pathToReturn)

        # return a fully normalized path
        return pathToReturn
Beispiel #15
0
    def removeManagedItem(myClass, targetPath):
        '''Remove an item from the list of managed items'''

        targetPath = pathHelpers.normalizePath(targetPath)

        if not targetPath in myClass.managedItems:
            raise ValueError(
                'removeManagedItem called with a targetPath that was not a managed path: '
                + targetPath)

        myClass.managedItems.remove(targetPath)
Beispiel #16
0
    def addManagedMount(myClass, mountPoint):

        mountPoint = pathHelpers.normalizePath(mountPoint, followSymlink=True)

        myClass.addManagedItem(mountPoint)

        for thisMount in myClass.managedMounts:
            if os.path.samefile(thisMount, mountPoint):
                return

        myClass.managedMounts.append(mountPoint)
Beispiel #17
0
	def addManagedMount(myClass, mountPoint):
		
		mountPoint = pathHelpers.normalizePath(mountPoint, followSymlink=True)
		
		myClass.addManagedItem(mountPoint)
		
		for thisMount in myClass.managedMounts:
			if os.path.samefile(thisMount, mountPoint):
				return
		
		myClass.managedMounts.append(mountPoint)
Beispiel #18
0
def unmountVolume(targetPath):
	'''Unmount a volume or dmg mounted at the path given'''
	
	if not os.path.ismount(targetPath):
		raise ValueError('unmountVolume valled on a path that was not a mount point: ' + targetPath)
	
	targetPath = pathHelpers.normalizePath(targetPath)
	
	# check to see if this is a disk image
	isMountedDMG = False
	command = ['/usr/bin/hdiutil', 'info', '-plist']
	process = managedSubprocess(command, processAsPlist=True)
	plistData = process.getPlistObject()
	
	if not hasattr(plistData, 'has_key') or not plistData.has_key('images') or not hasattr(plistData['images'], '__iter__'):
		raise RuntimeError('The "%s" output does not have an "images" array as expected. Output was:\n%s' % (' '.join(command), output))
	
	for thisImage in plistData['images']:
	
		if not hasattr(thisImage, '__iter__') or not thisImage.has_key('system-entities') or not hasattr(thisImage['system-entities'], '__iter__'):
			raise RuntimeError('The "%s" output had an image entry that was not formed as expected. Output was:\n%s' % (' '.join(command), output))
		
		for thisEntry in thisImage['system-entities']:
			if thisEntry.has_key('mount-point') and os.path.samefile(thisEntry['mount-point'], targetPath):
				isMountedDMG = True
				break
		if isMountedDMG is True: break
	
	if isMountedDMG is True:
		# ToDo: log this
		command = ['/usr/bin/hdiutil', 'eject', targetPath]
		process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		
		if process.wait() != 0 and os.path.ismount(targetPath):
			# try again with a bit more force
			# ToDo: log this
			
			command = ['/usr/bin/hdiutil', 'eject', '-force', targetPath]
			managedSubprocess(command)
	
	else:
		# a non dmg mount point
		# ToDo: log this
		command = ['/usr/sbin/diskutil', 'unmount', targetPath]
		process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		
		if process.wait() != 0 and os.path.ismount(targetPath):
			# try again with a bit more force
			# ToDo: log this
			
			command = ['/usr/sbin/diskutil', 'unmount', 'force', targetPath]
			managedSubprocess(command)
Beispiel #19
0
	def cleanupForExit(myClass):
		'''Clean up everything, should be called by an atexit handler'''
		
		for itemToClean in myClass.findManagedItemsInsideDirectory('/'):
			try:
				if os.path.exists(itemToClean):
					myClass.cleanupItem(pathHelpers.normalizePath(itemToClean))
			
			except ValueError, error:
				pass # means the item does not exist
			
			except Exception, error: # catch everything else
				sys.stderr.write('Unable to process the folder: "%s" got error: %s\n' % (itemToClean, str(error))) # ToDo: logging
Beispiel #20
0
	def __init__(self, sourceLocation, checksumString, displayName=None, installerChoices=None):	
		
		# ---- validate input and set instance variables
		
		# -- source location
		
		if not hasattr(sourceLocation, 'capitalize'):
			raise ValueError("Recieved an empty or invalid sourceLocation: " + str(sourceLocation))
		
		# remove file:// if it is the sourceLocation
		if sourceLocation.startswith('file://'):
			sourceLocation = nameOrLocation[len('file://'):]
	
		self.source = sourceLocation
		
		parsedSourceLocationURL = urlparse.urlparse(sourceLocation)
		
		# fail out if we don't support this method
		if parsedSourceLocationURL.scheme not in ['', 'http', 'https']:
			raise ValueError('findItem can not process urls types other than http, https, or file')
					
		# -- displayName
		if hasattr(displayName, 'capitalize'):
			self.displayName = displayName
		elif displayName is None and sourceLocation is not None:
			if parsedSourceLocationURL.scheme in ['http', 'https']:
				# default to the part that looks like a name in the path
				self.displayName = os.path.basename(parsedSourceLocationURL.path)
			else:
				self.displayName = os.path.basename(sourceLocation)
		else:
			raise ValueError("Recieved an empty or invalid displayName: " + str(displayName))
			
		# -- checksum and checksumType
		if hasattr(checksumString, 'capitalize') and checksumString.count(":") > 0:
			self.checksumType, self.checksumValue = checksumString.split(":", 1)
			
			# confirm that hashlib supports the hash type:
			try:
				hashlib.new(self.checksumType)
			except ValueError:
				raise Exception('Hash type: %s is not supported by hashlib' % self.checksumType)
		else:
			raise ValueError('Recieved an empty or invalid checksumString: ' + str(checksumString))
		
		# -- installerChoices
		if installerChoices is not None and os.path.isfile(installerChoices):
			self.installerChoicesPath = pathHelpers.normalizePath(installerChoices, followSymlink=True)
		elif installerChoices is not None:
			raise ValueError('Recieved an empty or invalid installerChoices: ' + str(installerChoices))
Beispiel #21
0
    def cleanupForExit(myClass):
        '''Clean up everything, should be called by an atexit handler'''

        for itemToClean in myClass.findManagedItemsInsideDirectory('/'):
            try:
                if os.path.exists(itemToClean):
                    myClass.cleanupItem(pathHelpers.normalizePath(itemToClean))

            except ValueError, error:
                pass  # means the item does not exist

            except Exception, error:  # catch everything else
                sys.stderr.write(
                    'Unable to process the folder: "%s" got error: %s\n' %
                    (itemToClean, str(error)))  # ToDo: logging
Beispiel #22
0
	def __init__(self, targetPath=None):
				
		if targetPath is None:
			# create a new folder inside the default temporary folder
			targetPath = tempfile.mkdtemp(prefix=self.tempFolderPrefix, dir=self.getDefaultFolder())
		
		targetPath = pathHelpers.normalizePath(targetPath)
		
		if not os.path.isdir(os.path.dirname(targetPath)):
			raise ValueError('%s called with a targePath whose parent directory does not exist: %s' % (self.__class__.__name__, targetPath))
		
		elif not os.path.lexists(targetPath):
			# create a folder here
			os.mkdir(targetPath)
		
		self.addManagedItem(targetPath)
		self.itemPath = targetPath
Beispiel #23
0
	def getNewTempItem(myClass, fileOrFolder, parentFolder=None, prefix=None, suffix=None, supressCreation=False):
		'''Create a new managed file or folder and return the path'''
		
		if not fileOrFolder in ['file', 'folder']:
			raise ValueError('getNewTempITem only accepts "file" or "folder" for the fileOrFolder attribute, was given: ' + str(fileOrFolder))
		
		pathToReturn = None
		
		if prefix is None:
			if fileOrFolder == 'file':
				prefix = myClass.tempFilePrefix
			else:
				prefix = myClass.tempFolderPrefix
		
		if suffix is None:
			suffix=''
		
		if parentFolder is None:
			# create a new folder/file inside the current default one
			parentFolder = myClass.getDefaultFolder()
		
		elif not os.path.isdir(str(parentFolder)):
			raise ValueError('getNewTempFolder called with a parentFolder path that does not exist is is not a directory: ' + str(parentFolder))
		
		if supressCreation is True:
			pathToReturn = tempfile.mktemp(dir=parentFolder, prefix=prefix, suffix=suffix)
		elif fileOrFolder == 'file':
			openFile, pathToReturn = tempfile.mkstemp(dir=parentFolder, prefix=prefix, suffix=suffix)
			os.close(openFile)
		else:
			# create the new folder
			pathToReturn = tempfile.mkdtemp(dir=parentFolder, prefix=prefix)
		
		pathToReturn = pathHelpers.normalizePath(pathToReturn)
		
		# make sure that this file/folder is in managed space
		if myClass.getManagedPathForPath(pathToReturn) is None:
			# this is an unmanaged path, and we need to add it
			myClass.addManagedItem(pathToReturn)
		
		# return a fully normalized path
		return pathToReturn
Beispiel #24
0
    def __init__(self, targetPath=None):

        if targetPath is None:
            # create a new folder inside the default temporary folder
            targetPath = tempfile.mkdtemp(prefix=self.tempFolderPrefix,
                                          dir=self.getDefaultFolder())

        targetPath = pathHelpers.normalizePath(targetPath)

        if not os.path.isdir(os.path.dirname(targetPath)):
            raise ValueError(
                '%s called with a targePath whose parent directory does not exist: %s'
                % (self.__class__.__name__, targetPath))

        elif not os.path.lexists(targetPath):
            # create a folder here
            os.mkdir(targetPath)

        self.addManagedItem(targetPath)
        self.itemPath = targetPath
Beispiel #25
0
	def setCacheFolder(myClass, newCacheFolder):
		# the first cacheFolder is used to store new files, and must be write-able
		
		if newCacheFolder is None:
			raise ValueError("%s's setCacheFolder was given None as an input" % myClass.__name__)
		elif not hasattr(newCacheFolder, 'capitalize'):
			raise ValueError("%s's setCacheFolder recieved a newCacheFolder value that it did not understand: %s" % (myClass.__name__, newCacheFolder))
		elif not os.path.isdir(newCacheFolder):
			raise ValueError("%s's setCacheFolder given a path that was not a valid directory: %s" % (myClass.__name__, newCacheFolder))
		
		# confirm that this path is writeable
		if not os.access(newCacheFolder, os.W_OK):
			raise ValueError("The value given to %s's setCacheFolder method must be a write-able folder: %s" % (myClass.__name__, newCacheFolder))
		
		# make sure we have a canonical path
		newCacheFolder = pathHelpers.normalizePath(newCacheFolder, followSymlink=True)
		
		# set the cache folder
		myClass.writeableCacheFolder = newCacheFolder
		
		# make sure it is in the list of source folders
		myClass.addSourceFolders(newCacheFolder)
Beispiel #26
0
	def findManagedItemsInsideDirectory(myClass, targetDirectory):
		'''Return a list of managed items below this path in rough order that they should be cleaned'''
		
		targetDirectory = pathHelpers.normalizePath(targetDirectory)
		
		if not os.path.isdir(targetDirectory) or os.path.islink(targetDirectory):
			raise ValueError('findManagedItemsInsideDirectory requires a directory as the input, and it can not be a symlink' + str(targetDirectory))
		
		itemsToClean = []
		if myClass.managedItems is not None:
			for thisItem in myClass.managedItems:
				if pathHelpers.pathInsideFolder(thisItem, targetDirectory):
					itemsToClean.append(thisItem)
		# Note: this should mean that once sorted and reversed that mounted items get taken care of before their mount-points
		if myClass.managedMounts is not None:
			for thisMount in myClass.managedMounts:
				if pathHelpers.pathInsideFolder(thisMount, targetDirectory):
					itemsToClean.append(thisMount)
		
		itemsToClean.sort() # items inside others should now be below their parents
		itemsToClean.reverse() # the opposite should now be true
		
		return itemsToClean
Beispiel #27
0
	def findItemInCaches(myClass, nameOrLocation, checksumType, checksumValue, displayName=None, additionalSourceFolders=None, progressReporter=True, includeRemoteCaches=False): 
		
		# ---- validate input
		
		# nameOrLocation
		if not hasattr(nameOrLocation, 'capitalize') and not nameOrLocation is None:
			raise ValueError('findItem requires a string or none as a nameOrLocation, but got: ' + nameOrLocation)
		if nameOrLocation is not None and nameOrLocation.startswith('file://'):
			nameOrLocation = nameOrLocation[len('file://'):]
		
		if nameOrLocation is not None and urlparse.urlparse(nameOrLocation).scheme != '':
			raise ValueError('findItemInCaches only works on file paths or names, got: ' + str(nameOrLocation))
		
		# checksumType
		if not hasattr(checksumType, 'capitalize'):
			raise ValueError('findItem requires a string as a checksumType, but got: ' + checksumType)
		
		# checksumValue
		if not hasattr(checksumValue, 'capitalize'):
			raise ValueError('findItem requires a string as a checksumValue, but got: ' + checksumValue)
		
		# displayName
		if not hasattr(displayName, 'capitalize') and not displayName is None:
			raise ValueError('findItem requires a string or None as a displayName, but got: ' + displayName)
		
		# additionalSourceFolders
		foldersToSearch = myClass.getSourceFolders()
		if additionalSourceFolders is None:
			pass # nothing to do
		elif hasattr(additionalSourceFolders, 'capitalize') and os.path.isdir(additionalSourceFolders):
			foldersToSearch.append(pathHelpers.normalizePath(additionalSourceFolders, followSymlink=True))
		elif hasattr(additionalSourceFolders, '__iter__'):
			# validate that these are all folders
			for thisFolder in additionalSourceFolders:
				if not os.path.isdir(thisFolder):
					raise ValueError('The folder given to findItemInCaches as an additionalSourceFolders either did not exist or was not a folder: ' + thisFolder)
				foldersToSearch.append(pathHelpers.normalizePath(thisFolder, followSymlink=True))
		else:
			raise ValueError('Unable to understand the additionalSourceFolders given: ' + str(additionalSourceFolders))
		
		# progressReporter
		if progressReporter is True:
			progressReporter = displayTools.statusHandler(statusMessage='Searching cache folders for ' + nameOrLocation)
		elif progressReporter is False:
			progressReporter = None
		
		# ---- search for the items
		
		# absolute paths
		if nameOrLocation is not None and os.path.isabs(nameOrLocation):
			if os.path.exists(nameOrLocation):
				if checksumValue == checksum.checksum(nameOrLocation, checksumType=checksumType, progressReporter=progressReporter)['checksum']:
					return nameOrLocation, False
				else:
					raise FileNotFoundException('The item at the path given does not match the checksum given: ' + nameOrLocation)
			else:
				raise FileNotFoundException('No file/folder existed at the absolute path: ' + nameOrLocation)
		
		# relative path
		elif nameOrLocation is not None and os.path.exists(nameOrLocation):
			if checksumValue == checksum.checksum(nameOrLocation, checksumType=checksumType, progressReporter=progressReporter)['checksum']:
				return pathHelpers.normalizePath(nameOrLocation, followSymlink=True), False
		
		# cache folders
		for thisCacheFolder in foldersToSearch:
			
			parsedLocation = urlparse.urlparse('')
			if nameOrLocation is not None:
				parsedLocation = urlparse.urlparse(thisCacheFolder)
			
			if parsedLocation.scheme in ['http', 'https'] and includeRemoteCaches is False:
				continue
			
			elif parsedLocation.scheme in ['http', 'https'] and includeRemoteCaches is True:
				
				# -- try different paths on the server
				urlsToTry = {}
				
				(scheme, netloc, path, params, query, fragment) = urlparse.urlparse(thisCacheFolder)
				
				# simple name
				urlsToTry[urlparse.urlunparse((scheme, netloc, os.path.join(path, nameOrLocation), params, query, fragment))] = True
				urlsToTry[urlparse.urlunparse((scheme, netloc, os.path.join(path, urllib.quote(nameOrLocation)), params, query, fragment))] = True
				
				# name including checksum
				nameWithChecksum = os.path.splitext(nameOrLocation)[0] + " " + checksumType + "-" + checksumValue + os.path.splitext(nameOrLocation)[1]
				urlsToTry[urlparse.urlunparse((scheme, netloc, os.path.join(path, nameWithChecksum), params, query, fragment))] = True
				urlsToTry[urlparse.urlunparse((scheme, netloc, os.path.join(path, urllib.quote(nameWithChecksum)), params, query, fragment))] = True
				
				for thisURL in urlsToTry.keys():
					try:
						readFile = urllib2.urlopen(thisURL)
					except IOError, error:
						continue
					
					remoteGuessedName	= os.path.basename(urllib.unquote(urlparse.urlparse(thisURL).path))
					targetFileName		= remoteGuessedName
					if checksumType + "-" + checksumValue not in targetFileName:
						targetFileName = os.path.splitext(remoteGuessedName)[0] + " " + checksumType + "-" + checksumValue + os.path.splitext(remoteGuessedName)[1]
					
					# try to get the expected file length
					httpHeader = readFile.info()
					expectedLength = None
					if httpHeader.has_key("content-length"):
						try:
							expectedLength = int(httpHeader.getheader("content-length"))
						except:
							pass
					
					if progressReporter is not None:
						if expectedLength is None:
							progressReporter.update(statusMessage=' downloading from local web cache ')
						else:
							progressReporter.update(statusMessage=' downloading %s from local web cache ' % displayTools.bytesToRedableSize(expectedLength))
					
					# download file
					hashGenerator = hashlib.new(checksumType)
					startTime = time.time()
					targetFilePath = os.path.join(myClass.getCacheFolder(), targetFileName)
					processedBytes, processSeconds = checksum.checksumFileObject(hashGenerator, readFile, remoteGuessedName, expectedLength, copyToPath=targetFilePath, progressReporter=progressReporter)
					
					if hashGenerator.hexdigest() != checksumValue:
						os.unlink(targetFilePath)
					
					else:
						if progressReporter is not None:
							progressReporter.update(statusMessage=' downloaded from local web cache and verified %s in %s (%s/sec)' % (displayTools.bytesToRedableSize(processedBytes), displayTools.secondsToReadableTime(time.time() - startTime), displayTools.bytesToRedableSize(processedBytes/processSeconds)))
							progressReporter.finishLine()
						
						myClass.addItemToVerifiedFiles('%s-%s' % (checksumType, checksumValue), targetFilePath)
						readFile.close()
						return targetFilePath, True
					
					readFile.close()
			
			elif parsedLocation.scheme == '':
			
				# relative paths from the source folders
				if nameOrLocation is not None and nameOrLocation.count(os.sep) > 0 and os.path.exists(os.path.join(thisCacheFolder, nameOrLocation)):
					if checksumValue == checksum.checksum(os.path.join(thisCacheFolder, nameOrLocation), checksumType=checksumType, progressReporter=progressReporter)['checksum']:
						return pathHelpers.normalizePath(os.path.join(thisCacheFolder, nameOrLocation), followSymlink=True), False
				
				# walk up through the whole set
				for currentFolder, dirs, files in os.walk(thisCacheFolder, topdown=True):
					
					# check each file to see if it is what we are looking for
					for thisItemPath, thisItemName in [[os.path.join(currentFolder, internalName), internalName] for internalName in (files + dirs)]:
						
						# checksum in name
						fileNameSearchResults = myClass.fileNameChecksumRegex.search(thisItemName)
						
						nameChecksumType = None
						nameChecksumValue = None
						if fileNameSearchResults is not None:
							nameChecksumType = fileNameSearchResults.group('checksumType')
							nameChecksumValue = fileNameSearchResults.group('checksumValue')
						
							if nameChecksumType is not None and nameChecksumType.lower() == checksumType.lower() and nameChecksumValue is not None and nameChecksumValue == checksumValue:
								if checksumValue == checksum.checksum(thisItemPath, checksumType=checksumType, progressReporter=progressReporter)['checksum']:
									return thisItemPath, False
						
						# file name
						if nameOrLocation is not None:
							if nameOrLocation in [thisItemName, os.path.splitext(thisItemName)[0]] or os.path.splitext(nameOrLocation)[0] in [thisItemName, os.path.splitext(thisItemName)[0]]:
								if checksumValue == checksum.checksum(thisItemPath, checksumType=checksumType, progressReporter=progressReporter)['checksum']:
									return thisItemPath, False
						
						# don't decend into folders that look like bundles or sparce dmg's
						if os.path.isdir(thisItemPath):
							if os.listdir(thisItemPath) == ["Contents"] or os.listdir(thisItemPath) == ["Info.bckup", "Info.plist", "bands", "token"]:
								dirs.remove(thisItemName)
Beispiel #28
0
    def test_getNewTempFolder(self):
        '''Test the getNewTempFolder method'''

        # test without any input
        firstFolderPath = tempFolderManager.getNewTempFolder()
        self.assertTrue(
            firstFolderPath is not None,
            'Called with no options getNewTempFolder gave back None')
        self.assertTrue(
            os.path.isdir(firstFolderPath),
            'Called with no options getNewTempFolder returned a string that was not a path to an existing folder: '
            + str(firstFolderPath))
        self.assertTrue(
            tempFolderManager.getManagedPathForPath(firstFolderPath)
            is not None,
            'Called with no options getNewTempFolder returned a path that was not in any managed path (according to getManagedPathForPath): '
            + firstFolderPath)

        # test with the parent folder option
        secondParentFolder = tempfile.mkdtemp(dir='/tmp')
        secondFolderPath = tempFolderManager.getNewTempFolder(
            parentFolder=secondParentFolder)
        self.assertTrue(
            secondFolderPath is not None,
            'Called with a parent folder getNewTempFolder gave back None')
        self.assertTrue(
            os.path.isdir(secondFolderPath),
            'Called with a parent folder getNewTempFolder returned a string that was not a path to an existing folder: '
            + str(secondFolderPath))
        self.assertTrue(
            secondFolderPath.startswith(
                pathHelpers.normalizePath(secondParentFolder,
                                          followSymlink=True)),
            'Called with a parent folder (%s) getNewTempFolder returned a path not in the parent folder: %s'
            % (secondParentFolder, secondFolderPath))
        self.assertTrue(
            tempFolderManager.getManagedPathForPath(secondFolderPath)
            is not None,
            'Called with a parent folder getNewTempFolder returned a path that was not in any managed path (according to getManagedPathForPath): '
            + secondFolderPath)

        # test with the prefix option
        prefixOption = "thisIsATest"
        thirdFolderPath = tempFolderManager.getNewTempFolder(
            prefix=prefixOption)
        self.assertTrue(
            thirdFolderPath is not None,
            'Called with the prefix option (%s) getNewTempFolder gave back None'
            % prefixOption)
        self.assertTrue(
            os.path.isdir(thirdFolderPath),
            'Called with the prefix option (%s) getNewTempFolder returned a string that was not a path to an existing folder: %s'
            % (prefixOption, str(thirdFolderPath)))
        self.assertTrue(
            os.path.basename(thirdFolderPath).startswith(prefixOption),
            'Called with the prefix option (%s) getNewTempFolder returned a path not in the parent folder: %s'
            % (prefixOption, thirdFolderPath))
        self.assertTrue(
            tempFolderManager.getManagedPathForPath(secondFolderPath)
            is not None,
            'Called with the prefix option (%s) getNewTempFolder returned a path that was not in any managed path (according to getManagedPathForPath): %s'
            % (prefixOption, thirdFolderPath))

        # call cleanupAtExit to clear everything
        tempFolderManager.cleanupForExit()

        # verify that the folders dissapeared
        self.assertFalse(
            os.path.exists(firstFolderPath),
            'After being created with getNewTempFolder using no options the folder path was not cleaned properly by cleanupForExit: '
            + firstFolderPath)
        self.assertFalse(
            os.path.exists(secondFolderPath),
            'After being created with getNewTempFolder using the parent folder the folder optionthe path was not cleaned properly by cleanupForExit: '
            + secondFolderPath)
        self.assertFalse(
            os.path.exists(thirdFolderPath),
            'After being created with getNewTempFolder using the prefix option (%s) the folder path was not cleaned properly by cleanupForExit: %s'
            % (prefixOption, thirdFolderPath))

        # remove the tempdir we made for the parent folder test
        shutil.rmtree(secondParentFolder)
Beispiel #29
0
def checksum(location, tempFolderPrefix="InstaDMGtemp", checksumType="sha1", displayName=None, outputFolder=None, checksumInFileName=True, chunkSize=None, progressReporter=True):
	'''Return the checksum of a given file or folder'''
	
	startReportTime = time.time()
	
	# validate input
	if location is None:
		raise Exception('Checksum called with a empty file location')
	if checksumType is None:
		raise Exception('Checksum called with a empty checksum type')
	if outputFolder is not None and not os.path.isdir(outputFolder):
		raise Exception('The output folder given does not exist, or is not a folder: ' + outputFolder)
	
	# make sure that the location is a string
	location = str(location)
	
	# confirm that hashlib supports the hash type:
	try:
		hashlib.new(checksumType)
	except ValueError:
		raise Exception("Hash type: %s is not supported by hashlib" % checksumType)
	
	# if a local copy is made, this will house the location
	localCopyPath = None
	
	if outputFolder is not None:
		# make sure we have an absolute path to it
		outputFolder = pathHelpers.normalizePath(outputFolder, followSymlink=True)
	
	# warm up the checksummer
	hashGenerator = hashlib.new(checksumType)
	
	# get rid of file:// urls
	if location.startswith('file://'):
		location = location[len('file://'):]
	
	locationURL = urlparse.urlparse(location)
	
	if displayName is None:
		if locationURL.scheme in ['http', 'https']:
			displayName = os.path.basename(locationURL.path)
		else:
			displayName = os.path.basename(location)
	
	# get the progress reporter ready
	if progressReporter is True:
		# create a statusHandler to handle this
		if locationURL.scheme in ['http', 'https']:
			progressReporter = statusHandler(taskMessage="Downloading %s: " % displayName)
		else:
			progressReporter = statusHandler(taskMessage="Checksumming %s: " % displayName)
	elif progressReporter in [False, None]:
		progressReporter = None # need to be consistent
	elif not isinstance(progressReporter, statusHandler):
		raise Exception('Unable to understand what the progressReporter is: ' + str(progressReporter))
	
	if locationURL.scheme is '':
		# a local path, check if it is a folder
		
		# make sure we have the canonical location
		location = pathHelpers.normalizePath(location, followSymlink=True)
		fileName = os.path.basename(location)
		
		if chunkSize is None:
			chunkSize = 1*1024*1024 # 1 MiB chunks for local files
		
		if not os.path.exists(location):
			raise Exception('Checksum called with a file location that does not exist: %s' % location)
		
		elif os.path.isdir(location):
			
			if outputFolder is not None:
				# validate outputFolder if there is one
				if os.path.samefile(location, outputFolder):
					raise ValueError('The output folder (%s) can not be the source item (%s)' % (outputFolder, location))
				if location.startswith(outputFolder + "/"):
					raise ValueError('The output folder (%s) can not be inside the source item (%s)' % (outputFolder, location))
				if os.path.samefile(os.path.dirname(location), outputFolder):
					raise ValueError('The output folder (%s) can not the the same as the source folder (%s)' % (outputFolder, os.path.dirname(location)))
				
				# create a temporary file until we get the checksum
				localCopyPath = tempfile.mkdtemp(prefix='checksumTempFolder.', dir=outputFolder)
				# register it with the tempFolderManager class so it will get cleaned up if something goes wrong
				tempFolderManager.addManagedItem(localCopyPath)
			
			if progressReporter is not None:
				progressReporter.update(statusMessage="building file list ", progressTemplate="%(value)i items", value=0)
			
			# get a quick count
			itemCount = 0
			for thisFolder, subFolders, subFiles in os.walk(location):
				
				for thisFile in subFiles:
					thisFilePath = os.path.join(thisFolder, thisFile)
					# note: we skip anything that is not a link or a file (ie: /dev)
					if os.path.islink(thisFilePath) or os.path.isfile(thisFilePath):
						itemCount += 1
				
				for thisFolder in subFolders:
					itemCount += 1
				
				if progressReporter is not None:
					progressReporter.update(value=itemCount)
			
			# change the status message
			if progressReporter is not None:
				progressReporter.update(statusMessage=" checksumming item ", progressTemplate="%(value)i of %(expectedLength)i (%(progressPercentage)i%%)", expectedLength=itemCount, value=0)
			
			# process the items
			processedCount = 0
			for processFolder, subFolders, subFiles in os.walk(location):
				
				for thisFile in subFiles:
					thisFilePath = os.path.join(processFolder, thisFile)
					relativeFilePath = os.path.join(processFolder.replace(location, '', 1), thisFile)
					if os.path.isabs(relativeFilePath):
						relativeFilePath = relativeFilePath[1:]
					
					if os.path.islink(thisFilePath):
						if localCopyPath is not None:
							os.symlink(os.readlink(thisFilePath), os.path.join(localCopyPath, relativeFilePath))
						
						hashGenerator.update("softlink %s to %s" % (os.readlink(thisFilePath), relativeFilePath))
						
					elif os.path.isfile(thisFilePath):
						
						readFile = open(thisFilePath)
						if readFile == None:
							raise Exception("Unable to open file for checksumming: " + thisFilePath)
						
						targetLength = os.stat(thisFilePath)[stat.ST_SIZE]
						writeTarget = None
						if localCopyPath is not None:
							writeTarget = os.path.join(localCopyPath, relativeFilePath)
						
						# add the path to the checksum
						hashGenerator.update("file " + relativeFilePath)
						
						checksumFileObject(hashGenerator, readFile, thisFile, targetLength, chunkSize, copyToPath=writeTarget)
						readFile.close()
												
					else:
						continue # skip anything that is not a link or a file (ie: /dev)
					
					processedCount += 1
					if progressReporter is not None:
						progressReporter.update(value=processedCount)
				
				for thisFolder in subFolders:
					thisFolderPath = os.path.join(processFolder, thisFolder)
					relativeFolderPath = os.path.join(processFolder.replace(location, '', 1), thisFolder)
					if os.path.isabs(relativeFolderPath):
						relativeFolderPath = relativeFolderPath[1:]
					
					if os.path.islink(thisFolderPath):
						if localCopyPath is not None:
							os.symlink(os.readlink(thisFolderPath), os.path.join(localCopyPath, relativeFolderPath))
						
						hashGenerator.update("softlink %s to %s" % (os.readlink(thisFolderPath), relativeFolderPath))
					
					else:
						if localCopyPath != None:
							os.mkdir( os.path.join(localCopyPath, relativeFolderPath) )
						
						# add this to the hash
						hashGenerator.update("folder %s" % relativeFolderPath)
					
					processedCount += 1
					if progressReporter is not None:
						progressReporter.update(value=processedCount)
			
			if progressReporter is not None:
				progressReporter.update(statusMessage='checksummed %i items in %s' % (processedCount, secondsToReadableTime(time.time() - startReportTime)))
			
			if localCopyPath is not None:
				# check if there is already something there
				targetOutputPath = os.path.join(outputFolder, fileName)
				if os.path.exists(targetOutputPath):
					if os.path.islink(targetOutputPath) or os.path.isfile(targetOutputPath):
						os.unlink(targetOutputPath)
					else:
						shutil.rmtree(targetOutputPath) # ToDo: handle errors
				
				# move the folder into place
				os.rename(localCopyPath, targetOutputPath)
				
				# unregister it from tempFolderManager
				tempFolderManager.removeManagedItem(localCopyPath)
				
				# change the localCopyPath to reflect the new location
				localCopyPath = os.path.basename(targetOutputPath)
			
		elif os.path.isfile(location):
			
			fileName = os.path.basename(location)
			readFile = open(location)
			if readFile == None:
				raise Exception("Unable to open file for checksumming: %s" % location)
			
			targetLength = os.stat(location)[stat.ST_SIZE]
			
			if outputFolder is not None:
				# create a temporary file until we get the checksum
				localCopyFile, localCopyPath = tempfile.mkstemp(prefix='checksumTempFile.', dir=outputFolder)
				os.close(localCopyFile)
				# register it with the tempFolderManager class so it will get cleaned up if something goes wrong
				tempFolderManager.addManagedItem(localCopyPath)
			
			if progressReporter is not None:
				progressReporter.update(statusMessage=" checksumming: ", progressTemplate='%(progressPercentage)i%% (%(recentRateInBytes)s)', expectedLength=targetLength, value=0)
			
			processedBytes, processSeconds = checksumFileObject(hashGenerator, readFile, os.path.basename(location), targetLength, chunkSize=chunkSize, copyToPath=localCopyPath, progressReporter=progressReporter)
			
			if progressReporter is not None:
				progressReporter.update(statusMessage=' checksummed (%s) in %s (%s/sec)' % (bytesToRedableSize(processedBytes), secondsToReadableTime(processSeconds), bytesToRedableSize(processedBytes/processSeconds)))
			
			readFile.close()
			
			# if we are keeping a local copy, move it into place
			if localCopyPath is not None:
				# change the file name to the real one, including the checksum if not suppressed
				realFilePath = None
				if checksumInFileName is True:
					realFilePath = os.path.join(outputFolder, os.path.splitext(fileName)[0] + " " + checksumType + "-" + hashGenerator.hexdigest() + os.path.splitext(fileName)[1])
				else:
					realFilePath = os.path.join(outputFolder, fileName)
				
				# try to move the item to the proper name
				os.rename(localCopyPath, realFilePath) # ToDo: proper error handling for all of the bad things that can happen here
				
				# unregister it from tempFolderManager
				tempFolderManager.removeManagedItem(localCopyPath)
				
				# change the localCopyPath to reflect the new location, and that it will now be pulled from the cache
				localCopyPath = os.path.basename(realFilePath)
			
		else:
			raise Exception('Checksum called on a location that is neither a file or folder: %s' % location)
	
	elif locationURL.scheme in ['http', 'https']:
		
		if chunkSize is None:
			chunkSize = 1024*100 # 100KiB for urls
		
		try:
			readFile = urllib2.urlopen(location)
		except IOError, error:
			if hasattr(error, 'reason'):
				raise Exception('Unable to connect to remote url: %s got error: %s' % (location, error.reason))
			elif hasattr(error, 'code'):
				raise Exception('Got status code: %s while trying to connect to remote url: %s' % (str(error.code), location))
		
		if readFile == None:
			raise Exception("Unable to open file for checksumming: %s" % location)
		
		# default the filename to the last bit of path of the url
				
		fileName = os.path.basename( urllib.unquote(urlparse.urlparse(readFile.geturl()).path) )
		if fileName in [None, '']:
			fileName = 'No_filename_provided'
		targetLength = None
		
		# grab the name of the file and its length from the http headers if avalible
		httpHeader = readFile.info()
		if httpHeader.has_key("content-length"):
			try:
				targetLength = int(httpHeader.getheader("content-length"))
			except:
				pass # 
		
		if httpHeader.has_key("content-disposition"):
			fileName = httpHeader.getheader("content-disposition").strip()
		
		if outputFolder is not None:
			# create a temporary file until we get the checksum
			localCopyFile, localCopyPath = tempfile.mkstemp(prefix='checksumTempFile.', dir=outputFolder)
			os.close(localCopyFile)
			# register it with the tempFolderManager class so it will get cleaned up if something goes wrong
			tempFolderManager.addManagedItem(localCopyPath)
		
		if progressReporter is not None:
			if targetLength is not None:
				progressReporter.update(statusMessage="downloading: ", progressTemplate='%(progressPercentage)i%% (%(recentRateInBytes)s)', expectedLength=targetLength, value=0)
			else:
				progressReporter.update(statusMessage="downloading: ", progressTemplate='%(valueInBytes)s (%(recentRateInBytes)s)', value=0)
		
		processedBytes, processSeconds = checksumFileObject(hashGenerator, readFile, fileName, targetLength, copyToPath=localCopyPath, chunkSize=chunkSize, progressReporter=progressReporter)
		
		if progressReporter is not None:
			progressReporter.update(statusMessage=" downloaded %s (%s) in %s (%s/sec)" % (fileName, bytesToRedableSize(processedBytes), secondsToReadableTime(processSeconds), bytesToRedableSize(processedBytes/processSeconds)))
		
		if localCopyPath is not None:
			# change the file name to the real one, including the checksum if not suppressed
			realFilePath = None
			if checksumInFileName is True:
				realFilePath = os.path.join(outputFolder, os.path.splitext(fileName)[0] + " " + checksumType + "-" + hashGenerator.hexdigest() + os.path.splitext(fileName)[1])
			else:
				realFilePath = os.path.join(outputFolder, fileName)
			
			# try to move the item to the proper name
			os.rename(localCopyPath, realFilePath) # ToDo: proper error handling for all of the bad things that can happen here
			
			# unregister it from tempFolderManager
			tempFolderManager.removeManagedItem(localCopyPath)
			
			# change the localCopyPath to reflect the new location
			localCopyPath = realFilePath
		
		readFile.close()
Beispiel #30
0
    elif options.mountPoint is not None and options.parentFolder is not None:
        optionsParser.error(
            'Only one of the -m/--mount-point or the -p/--parent-folder can be used at one time'
        )
    elif options.mountPoint is not None:
        # mountPoint
        if not os.path.isdir(options.mountPoint):
            optionsParser.error(
                'The path given to the -m/--mount-point option must be a folder, got: '
                + options.mountPoint)
        elif os.path.ismount(options.mountPoint):
            optionsParser.error(
                'The path given to the -m/--mount-point option must not be a mount point, got: '
                + options.mountPoint)

        options.mountPoint = pathHelpers.normalizePath(options.mountPoint)
    else:
        # parent folder
        if not os.path.isdir(options.parentFolder):
            optionsParser.error(
                'The path given to the -p/--parent-folder option must be a folder, got: '
                + options.parentFolder)
        elif os.path.ismount(options.parentFolder):
            optionsParser.error(
                'The path given to the -p/--parent-folder option must not be a mount point, got: '
                + options.parentFolder)

        options.mountPoint = tempFolderManager.getNewMountPoint(
            parentFolder=options.parentFolder)
        tempFolderManager.removeManagedItem(
            options.mountPoint
Beispiel #31
0
def unmountVolume(targetPath):
    '''Unmount a volume or dmg mounted at the path given'''

    if not os.path.ismount(targetPath):
        raise ValueError(
            'unmountVolume valled on a path that was not a mount point: ' +
            targetPath)

    targetPath = pathHelpers.normalizePath(targetPath)

    # check to see if this is a disk image
    isMountedDMG = False
    command = ['/usr/bin/hdiutil', 'info', '-plist']
    process = managedSubprocess(command, processAsPlist=True)
    plistData = process.getPlistObject()

    if not hasattr(plistData, 'has_key') or not plistData.has_key(
            'images') or not hasattr(plistData['images'], '__iter__'):
        raise RuntimeError(
            'The "%s" output does not have an "images" array as expected. Output was:\n%s'
            % (' '.join(command), output))

    for thisImage in plistData['images']:

        if not hasattr(thisImage, '__iter__') or not thisImage.has_key(
                'system-entities') or not hasattr(thisImage['system-entities'],
                                                  '__iter__'):
            raise RuntimeError(
                'The "%s" output had an image entry that was not formed as expected. Output was:\n%s'
                % (' '.join(command), output))

        for thisEntry in thisImage['system-entities']:
            if thisEntry.has_key('mount-point') and os.path.samefile(
                    thisEntry['mount-point'], targetPath):
                isMountedDMG = True
                break
        if isMountedDMG is True: break

    if isMountedDMG is True:
        # ToDo: log this
        command = ['/usr/bin/hdiutil', 'eject', targetPath]
        process = subprocess.Popen(command,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)

        if process.wait() != 0 and os.path.ismount(targetPath):
            # try again with a bit more force
            # ToDo: log this

            command = ['/usr/bin/hdiutil', 'eject', '-force', targetPath]
            managedSubprocess(command)

    else:
        # a non dmg mount point
        # ToDo: log this
        command = ['/usr/sbin/diskutil', 'unmount', targetPath]
        process = subprocess.Popen(command,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)

        if process.wait() != 0 and os.path.ismount(targetPath):
            # try again with a bit more force
            # ToDo: log this

            command = ['/usr/sbin/diskutil', 'unmount', 'force', targetPath]
            managedSubprocess(command)
Beispiel #32
0
		optionsParser.error('There does not seem to be anything at the path provided: ' + imageToMount)
	elif not dmgManager.verifyIsDMG(imageToMount):
		optionsParser.error('The item at the provided path does not seem to be a valid dmg: ' + imageToMount)
	
	if options.mountPoint is None and options.parentFolder is None:
		optionsParser.error('One of either the -m/--mount-point or the -p/--parent-folder options are required')
	elif options.mountPoint is not None and options.parentFolder is not None:
		optionsParser.error('Only one of the -m/--mount-point or the -p/--parent-folder can be used at one time')
	elif options.mountPoint is not None:
		# mountPoint
		if not os.path.isdir(options.mountPoint):
			optionsParser.error('The path given to the -m/--mount-point option must be a folder, got: ' + options.mountPoint)
		elif os.path.ismount(options.mountPoint):
			optionsParser.error('The path given to the -m/--mount-point option must not be a mount point, got: ' + options.mountPoint)
		
		options.mountPoint = pathHelpers.normalizePath(options.mountPoint)
	else:
		# parent folder
		if not os.path.isdir(options.parentFolder):
			optionsParser.error('The path given to the -p/--parent-folder option must be a folder, got: ' + options.parentFolder)
		elif os.path.ismount(options.parentFolder):
			optionsParser.error('The path given to the -p/--parent-folder option must not be a mount point, got: ' + options.parentFolder)
		
		options.mountPoint = tempFolderManager.getNewMountPoint(parentFolder=options.parentFolder)
		tempFolderManager.removeManagedItem(options.mountPoint) # so it does not get cleaned back up immediately
	
	# ---- process
	
	currentMountPoints = dmgManager.getDMGMountPoints(imageToMount)
	if options.allowOtherMount is True and currentMountPoints is not None:
		sys.stdout.write(currentMountPoints[0] + options.lineEnding)
Beispiel #33
0
	def isValidInstaller(myClass, pkgPath, chrootPath=None, installerChoicesFilePath=None):
		'''Use installer to check if a target looks like a valid pkg/mpkg'''
		
		pathToInstaller = '/usr/sbin/installer'
		
		# ---- validate and normalize input
		
		# pkg path
		
		if pkgPath is None:
			raise ValueError('The pkgPath given to isValidInstaller can not be none')
		elif not os.path.isdir(pkgPath) and not os.path.isfile(pkgPath):
			raise ValueError('The pkgPath given to isValidInstaller does not look correct: ' + str(pkgPath))
		pkgPath = pathHelpers.normalizePath(pkgPath, followSymlink=True)
		
		# chroot path
		
		if chrootPath is not None and not os.path.ismount(chrootPath):
			raise ValueError('The chrootPath given to isValidInstaller must be a mount point, this was not: ' + str(chrootPath))
		
		if chrootPath is not None:
			chrootPath = pathHelpers.normalizePath(chrootPath, followSymlink=True)
		
		# installer choices
		
		if installerChoicesFilePath is not None and not os.path.isfile(installerChoicesFilePath):
			raise ValueError('The installerChoicesFilePath given to isValidInstaller must be a file, this was not: ' + str(installerChoicesFilePath))
		installerChoicesFilePath = pathHelpers.normalizePath(installerChoicesFilePath, followSymlink=True)
		
		# ---- validate that the installer command is avalible
		
		if chrootPath is None and not os.access(pathToInstaller, os.F_OK | os.X_OK):
			raise RuntimeError('The installer command was not avalible where it was expected to be, or was not useable: ' + pathToInstaller)
		elif chrootPath is not None and not os.access(os.path.join(chrootPath, pathToInstaller[1:]), os.F_OK | os.X_OK):
			raise RuntimeError('The installer command was not avalible where it was expected to be, or was not useable: ' + os.path.join(chrootPath, pathToInstaller[1:]))
		
		# ---- build the command
		
		command = []
		
		if chrootPath is not None:
			command += ['/usr/sbin/chroot', chrootPath]
		
		command += [pathToInstaller, '-pkginfo']
		
		if installerChoicesFilePath is not None:
			command += ['-applyChoiceChangesXML', installerChoicesFilePath]
		
		command += ['-pkg', pkgPath, '-target', '.']
		
		# ---- run the command
		
		process = None
		try:
			if chrootPath is None:
				process = managedSubprocess(command)
			else:
				process = managedSubprocess(command, cwd=chrootPath)
		except:
			return False
		
		# if installer did not have a problem, then it is very probably good
		return True
Beispiel #34
0
	def mountImage(myClass, dmgFile, mountPoint=None, mountInFolder=None, mountReadWrite=None, shadowFile=None, paranoidMode=False):
		'''Mount an image'''
		
		# -- validate input
		
		# dmgFile
		if dmgFile is None or not os.path.exists(dmgFile) or os.path.ismount(dmgFile):
			raise ValueError('mountImage requires dmgFile be a path to a file, got: ' + dmgFile)
		dmgFile = pathHelpers.normalizePath(dmgFile, followSymlink=True)
		
		# mountPoint/mountInFolder
		if mountPoint is not None and mountInFolder is not None:
			raise ValueError('mountImage can only be called with mountPoint or mountInFolder, not both')
		
		elif mountPoint is not None and os.path.ismount(mountPoint):
			raise ValueError('mountImage called with a mountPoint that is already a mount point: ' + mountPoint)
		
		elif mountPoint is not None and os.path.isdir(mountPoint):
			if len(os.listdir(mountPoint)) != 0:
				raise ValueError('mountImage called with a mountPoint that already has contents: %s (%s)' % (mountPoint, str(os.listdir(mountPoint))))
		
		elif mountPoint is not None and os.path.exists(mountPoint):
			raise ValueError('mountImage called with a mountPoint that already exists and is not a folder: ' + mountPoint)
		
		elif mountPoint is not None:
			tempFolderManager.add
		
		elif mountInFolder is not None and not os.path.isdir(mountInFolder):
			raise ValueError('mountImage called with a mountInFolder path that is not a folder: ' + mountInFolder)
		
		elif mountInFolder is not None:
			mountPoint = tempFolderManager.getNewMountPoint()
		
		else:
			# create a default mount point
			mountPoint = tempFolderManager.getNewMountPoint()
		mountPoint = pathHelpers.normalizePath(mountPoint, followSymlink=True)
		
		# mountReadWrite
		if mountReadWrite not in [None, True, False]:
			raise ValueError('The only valid options for mountReadWrite are True or False')
		
		# shadowFile
		if shadowFile is True:
			# generate a temporary one
			shadowFile = tempFolderManager.getNewTempFile(suffix='.shadow')
			
		elif shadowFile is not None:
			shadowFile = pathHelpers.normalizePath(shadowFile, followSymlink=True)
			
			if os.path.isfile(shadowFile): # work here
				pass # just use the file
			
			elif os.path.isdir(shadowFile):
				# a directory to put the shadow file in
				shadowFile = tempFolderManager.getNewTempFile(parentFolder=shadowFile, suffix='.shadow')
			
			elif os.path.isdir(os.path.dirname(shadowFile)):
				# the path does not exist, but the directory it is in looks good
				pass
			
			else:
				# not valid
				raise ValueError('The path given for the shadow file does not look valid: ' + str(shadowFile))
		
		# paranoidMode
		if paranoidMode not in [True, False]:
			raise ValueError('checksumImage must be either True, or False. Got: ' + str(paranoidMode))		
		
		# -- check to see if it is already mounted
		
		existingMountPoints = myClass.getDMGMountPoints(dmgFile)
		if existingMountPoints is not None:
			raise RuntimeError('The image (%s) was already mounted: %s' % (dmgFile, ", ".join(existingMountPoints)))
		
		# -- construct the command
		
		command = ['/usr/bin/hdiutil', 'attach', dmgFile, '-plist', '-mountpoint', mountPoint, '-nobrowse', '-owners', 'on']
		
		if mountReadWrite is True and shadowFile is None and self.writeable is False:
			shadowFile = tempFolderManager.getNewTempFile(suffix='.shadow')
		elif mountReadWrite is False and shadowFile is None:
			command += ['-readonly']
		
		if paranoidMode is False:
			command += ['-noverify', '-noautofsck']
		else:
			command += ['-verify', '-autofsck']
		
		if shadowFile is not None:
			command += ['-shadow', shadowFile]
		
		# -- run the command
		
		process = managedSubprocess(command, processAsPlist=True)
		mountInfo = process.getPlistObject()
		
		actualMountedPath = None
		
		for thisEntry in mountInfo['system-entities']:
			if 'mount-point' in thisEntry:
				if actualMountedPath != None and os.path.ismount(actualMountedPath):
					# ToDo: think all of this through, prefereabley before it is needed
					raise Exception('This item (%s) seems to be mounted at two places , this is possible, but now that it is happening larkost needs to work on this' % self.filePath)
				
				actualMountedPath = thisEntry['mount-point'] # ToDo: make sure that this is the mount point we requested
				break # assume that there is only one mountable item... otherwise we are hosed already
		
		if actualMountedPath is None:
			raise RuntimeError('Error: image could not be mounted')
		
		return actualMountedPath
Beispiel #35
0
	def normalizePathTestHelper(self, testPath, expectedResult):
		result = pathHelpers.normalizePath(testPath)
		self.assertEqual(expectedResult, result, 'normalizePath did not return "%s" for "%s", but rather: %s' % (expectedResult, testPath, result))
Beispiel #36
0
    def cleanupItem(myClass, targetPath):
        '''Dispose of an item'''

        if targetPath is None:
            raise ValueError('cleanupItem called with an empty targetPath')

        targetPath = pathHelpers.normalizePath(targetPath)

        # -- confirm that this item is a managed item, or in a manged space

        managedItem = False
        managedMount = False
        managedSpace = False

        if targetPath in myClass.managedItems:
            managedItem = True
            managedSpace = True
        else:
            for thisManagedSpace in myClass.managedItems:
                if os.path.isdir(thisManagedSpace) and not os.path.islink(
                        thisManagedSpace) and os.path.lexists(targetPath):
                    if pathHelpers.pathInsideFolder(
                            targetPath, thisManagedSpace) and os.lstat(
                                targetPath)[stat.ST_DEV] == os.lstat(
                                    thisManagedSpace)[stat.ST_DEV]:
                        managedSpace = True
                        break

        if targetPath in myClass.managedMounts:
            managedMount = True

        if managedMount is False and managedSpace is False:
            raise ValueError(
                'cleanupItem handed a path that was not in a managed space or a managed mount: '
                + targetPath)

        if not os.path.lexists(targetPath):
            if True in [managedItem, managedSpace]:
                # the item no longer exists, we just have to clean it out of managedItems and/or managedMount
                if managedItem is True:
                    myClass.managedItems.remove(targetPath)
                if managedMount is True:
                    myClass.managedMounts.remove(targetPath)

                return
            else:
                raise ValueError(
                    'cleanupItem handed a path that does not exist: ' +
                    targetPath)

        # -- find any managed items inside this one, and let them handle their business first
        if os.path.isdir(targetPath) and not os.path.islink(targetPath):
            for thisItem in myClass.findManagedItemsInsideDirectory(
                    targetPath):
                myClass.cleanupItem(thisItem)

        # -- if this is a mount, unmount it
        if os.path.ismount(targetPath):
            volumeTools.unmountVolume(targetPath)

        # -- if this is in controlled space, wipe it
        if managedSpace is True:

            # handle the simple cases of a soft-link or a file
            if os.path.islink(targetPath) or os.path.isfile(targetPath):
                try:
                    os.unlink(targetPath)
                except OSError:
                    # assume that this was a permissions error, and try to chmod it into cooperating
                    os.chmod(thisFile,
                             stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
                    os.unlink(thisFile)

            # handle folders
            else:
                # make sure that the permissions on the root folder are ok
                if not os.access(targetPath, os.R_OK | os.X_OK):
                    os.chmod(
                        targetPath, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
                        | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)

                # walk up the tree to remove any volumes mounted into the path
                for root, dirs, files in os.walk(targetPath, topdown=True):

                    # unmount the directory if it is a volume
                    if os.path.ismount(root):
                        volumeTools.unmountVolume(root)  # ToDo: log this
                        dirs = [
                        ]  # make sure we don't try to decend into folders that are no longer there
                        continue

                    # delete all files, continuing through failures
                    for thisFile in [
                            os.path.join(root, internalName)
                            for internalName in files
                    ]:
                        try:
                            try:
                                os.unlink(thisFile)
                            except OSError:
                                # assume that this was a permissions error, and try to chmod it into cooperating
                                os.chmod(
                                    thisFile,
                                    stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
                                os.unlink(thisFile)

                        except:  # ToDo: make this more specific
                            pass  # ToDo: log this

                    # catch any symlinks
                    for thisFolder in [
                            os.path.join(root, internalName)
                            for internalName in dirs
                    ]:
                        # make sure we can make it into all sub-folders and delete them:
                        if not os.access(thisFolder, os.R_OK | os.X_OK):
                            os.chmod(
                                thisFolder,
                                stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
                                | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)

                        if os.path.islink(thisFolder):
                            try:
                                os.unlink(thisFolder)
                            except:  # ToDo: make this more specific
                                pass  # ToDo: log this

                # now that there are no mounted volumes, there should be no files, so delete the folders
                for root, dirs, files in os.walk(targetPath, topdown=False):
                    try:
                        os.rmdir(root)
                    except Exception, error:  # ToDo: make this more specific
                        sys.stderr.write(
                            'Unable to delete folder: "%s" got error: %s' %
                            (root, str(error)))  # ToDo: logging
Beispiel #37
0
    def setDefaultFolder(myClass, targetFolder=None):

        if myClass.classActivated is False:
            # arm the atexit handler
            atexit.register(myClass.cleanupForExit)

        if targetFolder is None and myClass.defaultFolder is None:
            # setup the system with a default value
            targetFolder = tempfile.mkdtemp(prefix=myClass.tempFolderPrefix,
                                            dir='/private/tmp')
            myClass.managedItems.append(targetFolder)  # ToDo: log this
            myClass.defaultFolder = targetFolder
            return myClass.defaultFolder

        elif targetFolder is None:
            # we are already setup, use the default folder
            return myClass.defaultFolder

        targetFolder = pathHelpers.normalizePath(
            targetFolder)  # this will deal with any trailing slashes

        if not os.path.isdir(targetFolder):
            raise ValueError(
                'setDefaultFolder called with a path that was not an existing folder: '
                + targetFolder)

        # check if we are already managing an enclosing folder
        if myClass.getManagedPathForPath(targetFolder) is not None:
            myClass.defaultFolder = targetFolder
            return myClass.defaultFolder

        # a new item, not one we had before

        if os.path.isdir(targetFolder):
            # the directory already exists, we don't want to potentially kill something

            # search through the folder to see if there is anything to protect (not folders, symlinks, or a few select file types)
            for root, dirs, files in os.walk(targetFolder):
                for thisFile in files:
                    thisFilePath = os.path.join(root, thisFile)
                    if os.path.islink(thisFilePath) or thisFile in [
                            ".svn", ".DS_Store"
                    ]:
                        continue

                    raise ValueError(
                        'setDefaultFolder called with a non-empty folder: %s (%s)'
                        % (targetFolder, thisFilePath))

            # register everything to be cleaned up at exit
            myClass.managedItems.append(targetFolder)
            # ToDo: log this

        elif os.path.lexists(targetFolder):
            raise ValueError(
                'setDefaultFolder called on a path that is not a directory: ' +
                targetFolder)

        else:
            # since there is nothing in our way, check that the enclosing folder exists and create the temp directory
            if not os.path.isdir(os.path.dirname(targetFolder)):
                raise ValueError(
                    'setDefaultFolder can create the target folder, but the enclosing folder must already exist: '
                    + targetFolder)

            # create the directory
            os.mkdir(targetFolder)  # ToDo: think through the permissions

            # register everything to be cleaned up at exit
            myClass.managedItems.append(targetFolder)
            # ToDo: log this

        myClass.defaultFolder = targetFolder
        return myClass.defaultFolder
Beispiel #38
0
class cacheController:
	
	# ------ class variables
	
	writeableCacheFolder	= None		# writeable folder to put downloads into
	sourceFolders			= []		# paths to folders to search when looking for files
	
	verifiedFiles			= {}		# collection of items that have already been found indexed by checksum
	
	fileNameChecksumRegex	= re.compile('^(.+?/)?(?P<fileName>.*)( (?P<checksumType>\S+)-(?P<checksumValue>[^\.]+))(?P<fileExtension>\.[^\.]+)?$')
	
	# ------ class methods
	
	# ---- cacheFolder methods
	
	@classmethod
	def setCacheFolder(myClass, newCacheFolder):
		# the first cacheFolder is used to store new files, and must be write-able
		
		if newCacheFolder is None:
			raise ValueError("%s's setCacheFolder was given None as an input" % myClass.__name__)
		elif not hasattr(newCacheFolder, 'capitalize'):
			raise ValueError("%s's setCacheFolder recieved a newCacheFolder value that it did not understand: %s" % (myClass.__name__, newCacheFolder))
		elif not os.path.isdir(newCacheFolder):
			raise ValueError("%s's setCacheFolder given a path that was not a valid directory: %s" % (myClass.__name__, newCacheFolder))
		
		# confirm that this path is writeable
		if not os.access(newCacheFolder, os.W_OK):
			raise ValueError("The value given to %s's setCacheFolder method must be a write-able folder: %s" % (myClass.__name__, newCacheFolder))
		
		# make sure we have a canonical path
		newCacheFolder = pathHelpers.normalizePath(newCacheFolder, followSymlink=True)
		
		# set the cache folder
		myClass.writeableCacheFolder = newCacheFolder
		
		# make sure it is in the list of source folders
		myClass.addSourceFolders(newCacheFolder)
	
	@classmethod
	def getCacheFolder(myClass):
		if not hasattr(myClass.writeableCacheFolder, 'capitalize'):
			raise RuntimeWarning("The %s class's cacheFolder value was not useable: %s" % (myClass.__name__, str(myClass.writeableCacheFolder)))
		
		return myClass.writeableCacheFolder
	
	@classmethod
	def removeCacheFolder(myClass):
		'''Remove the current class cache folder, usefull mostly in testing'''
		
		myClass.removeSourceFolders(myClass.writeableCacheFolder)
		myClass.writeableCacheFolder = None
	
	# ---- sourceFolder methods
	
	@classmethod
	def addSourceFolders(myClass, newSourceFolders):
		
		# -- sanity check
		if not isinstance(myClass.sourceFolders, list):
			raise RuntimeWarning("The %s class's sourceFolders value was not useable, something has gone wrong prior to this" % myClass.__name__)
		
		foldersToAdd = []
		
		# -- force newSourceFolders to be a list
		if hasattr(newSourceFolders, 'capitalize'):
			foldersToAdd.append(newSourceFolders)
		
		elif hasattr(newSourceFolders, '__iter__'):
			for thisFolder in newSourceFolders:
				if not hasattr(thisFolder, 'capitalize'):
					raise ValueError("One of the items given to %s class's addSourceFolders method was not a string: %s" % (myClass.__name__, str(thisFolder)))
					
				foldersToAdd.append(thisFolder)
		
		else:
			raise ValueError("The value given to %s class's addSourceFolders method was not useable: %s" % (myClass.__name__, str(newSourceFolders)))
		
		# -- process the items
		for thisFolder in foldersToAdd:
			
			# remove file://
			if thisFolder.lower().startswith('file://'):
				thisFolder = thisFolder[len('file://'):]
			
			parsedLocation = urlparse.urlparse(thisFolder)
			
			if parsedLocation.scheme in ['http', 'https']:
				# web caches
				
				# open a connection, and see if we get a responce
				
				try:
					readFile = urllib2.urlopen(thisFolder)
					readFile.close()
				except urllib2.HTTPError, error:
					if error.code in [403]: # these might mean that the directory can't be listed
						myClass.sourceFolders.append(thisFolder)
					else:
						raise Exception('Got status code: %s while trying to connect to remote url: %s' % (str(error.code), thisFolder))
				
				except urllib2.URLError, error:
					# a bad network connection, or url
					raise Exception('Unable to connect to remote url: %s got error: %s' % (thisFolder, error.reason))
				
				# a 200 responce
				myClass.sourceFolders.append(thisFolder)	
			
			elif parsedLocation.scheme == '':
				# local path
				
				if not os.path.isdir(thisFolder):
					raise ValueError("The value given to %s class's addSourceFolders was not a directory as is required: %s" % (myClass.__name__, str(thisFolder)))
				
				# normalize the path
				thisFolder = pathHelpers.normalizePath(thisFolder, followSymlink=True)
				
				if not thisFolder in myClass.sourceFolders:
					myClass.sourceFolders.append(thisFolder)
Beispiel #39
0
	def cleanupItem(myClass, targetPath):
		'''Dispose of an item'''
		
		if targetPath is None:
			raise ValueError('cleanupItem called with an empty targetPath')
		
		targetPath = pathHelpers.normalizePath(targetPath)
		
		# -- confirm that this item is a managed item, or in a manged space
		
		managedItem		= False
		managedMount	= False
		managedSpace	= False
		
		if targetPath in myClass.managedItems:
			managedItem = True
			managedSpace = True
		else:
			for thisManagedSpace in myClass.managedItems:
				if os.path.isdir(thisManagedSpace) and not os.path.islink(thisManagedSpace) and os.path.lexists(targetPath):
					if pathHelpers.pathInsideFolder(targetPath, thisManagedSpace) and os.lstat(targetPath)[stat.ST_DEV] == os.lstat(thisManagedSpace)[stat.ST_DEV]:
						managedSpace = True
						break
		
		if targetPath in myClass.managedMounts:
			managedMount = True
		
		if managedMount is False and managedSpace is False:
			raise ValueError('cleanupItem handed a path that was not in a managed space or a managed mount: ' + targetPath)
		
		if not os.path.lexists(targetPath):
			if True in [managedItem, managedSpace]:
				# the item no longer exists, we just have to clean it out of managedItems and/or managedMount
				if managedItem is True:
					myClass.managedItems.remove(targetPath)
				if managedMount is True:
					myClass.managedMounts.remove(targetPath)
				
				return
			else:
				raise ValueError('cleanupItem handed a path that does not exist: ' + targetPath)
		
		# -- find any managed items inside this one, and let them handle their business first
		if os.path.isdir(targetPath) and not os.path.islink(targetPath):
			for thisItem in myClass.findManagedItemsInsideDirectory(targetPath):
				myClass.cleanupItem(thisItem)
		
		# -- if this is a mount, unmount it
		if os.path.ismount(targetPath):
			volumeTools.unmountVolume(targetPath)
		
		# -- if this is in controlled space, wipe it
		if managedSpace is True:
			
			# handle the simple cases of a soft-link or a file
			if os.path.islink(targetPath) or os.path.isfile(targetPath):
				try:
					os.unlink(targetPath)
				except OSError:
					# assume that this was a permissions error, and try to chmod it into cooperating
					os.chmod(thisFile, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
					os.unlink(thisFile)
			
			# handle folders
			else:
				# make sure that the permissions on the root folder are ok
				if not os.access(targetPath, os.R_OK | os.X_OK):
					os.chmod(targetPath, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
				
				# walk up the tree to remove any volumes mounted into the path
				for root, dirs, files in os.walk(targetPath, topdown=True):
					
					# unmount the directory if it is a volume
					if os.path.ismount(root):
						volumeTools.unmountVolume(root) # ToDo: log this
						dirs = [] # make sure we don't try to decend into folders that are no longer there
						continue
					
					# delete all files, continuing through failures
					for thisFile in [os.path.join(root, internalName) for internalName in files]:
						try:
							try:
								os.unlink(thisFile)
							except OSError:
								# assume that this was a permissions error, and try to chmod it into cooperating
								os.chmod(thisFile, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
								os.unlink(thisFile)
						
						except: # ToDo: make this more specific
							pass # ToDo: log this				
				
					# catch any symlinks
					for thisFolder in [os.path.join(root, internalName) for internalName in dirs]:
						# make sure we can make it into all sub-folders and delete them:
						if not os.access(thisFolder, os.R_OK | os.X_OK):
							os.chmod(thisFolder, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
						
						if os.path.islink(thisFolder):
							try:
								os.unlink(thisFolder)
							except: # ToDo: make this more specific
								pass # ToDo: log this	
				
				# now that there are no mounted volumes, there should be no files, so delete the folders
				for root, dirs, files in os.walk(targetPath, topdown=False):
					try:
						os.rmdir(root)
					except Exception, error: # ToDo: make this more specific
						sys.stderr.write('Unable to delete folder: "%s" got error: %s' % (root, str(error))) # ToDo: logging
Beispiel #40
0
	def setDefaultFolder(myClass, targetFolder=None):
		
		if myClass.classActivated is False:
			# arm the atexit handler
			atexit.register(myClass.cleanupForExit)
		
		if targetFolder is None and myClass.defaultFolder is None:
			# setup the system with a default value
			targetFolder = tempfile.mkdtemp(prefix=myClass.tempFolderPrefix, dir='/private/tmp')
			myClass.managedItems.append(targetFolder) # ToDo: log this
			myClass.defaultFolder = targetFolder
			return myClass.defaultFolder
			
		elif targetFolder is None:
			# we are already setup, use the default folder
			return myClass.defaultFolder
		
		targetFolder = pathHelpers.normalizePath(targetFolder) # this will deal with any trailing slashes
		
		if not os.path.isdir(targetFolder):
			raise ValueError('setDefaultFolder called with a path that was not an existing folder: ' + targetFolder)
		
		# check if we are already managing an enclosing folder
		if myClass.getManagedPathForPath(targetFolder) is not None:
			myClass.defaultFolder = targetFolder
			return myClass.defaultFolder
		
		# a new item, not one we had before
		
		if os.path.isdir(targetFolder):
			# the directory already exists, we don't want to potentially kill something
			
			# search through the folder to see if there is anything to protect (not folders, symlinks, or a few select file types)
			for root, dirs, files in os.walk(targetFolder):
				for thisFile in files:
					thisFilePath = os.path.join(root, thisFile)
					if os.path.islink(thisFilePath) or thisFile in [".svn", ".DS_Store"]:
						continue
					
					raise ValueError('setDefaultFolder called with a non-empty folder: %s (%s)' % (targetFolder, thisFilePath))
			
			# register everything to be cleaned up at exit
			myClass.managedItems.append(targetFolder)
			# ToDo: log this
		
		elif os.path.lexists(targetFolder):
			raise ValueError('setDefaultFolder called on a path that is not a directory: ' + targetFolder)
		
		else:
			# since there is nothing in our way, check that the enclosing folder exists and create the temp directory
			if not os.path.isdir(os.path.dirname(targetFolder)):
				raise ValueError('setDefaultFolder can create the target folder, but the enclosing folder must already exist: ' + targetFolder)
			
			# create the directory
			os.mkdir(targetFolder) # ToDo: think through the permissions
			
			# register everything to be cleaned up at exit
			myClass.managedItems.append(targetFolder)
			# ToDo: log this
		
		myClass.defaultFolder = targetFolder
		return myClass.defaultFolder
Beispiel #41
0
 def normalizePathTestHelper(self, testPath, expectedResult):
     result = pathHelpers.normalizePath(testPath)
     self.assertEqual(
         expectedResult, result,
         'normalizePath did not return "%s" for "%s", but rather: %s' %
         (expectedResult, testPath, result))
Beispiel #42
0
    def mountImage(myClass,
                   dmgFile,
                   mountPoint=None,
                   mountInFolder=None,
                   mountReadWrite=None,
                   shadowFile=None,
                   paranoidMode=False):
        '''Mount an image'''

        # -- validate input

        # dmgFile
        if dmgFile is None or not os.path.exists(dmgFile) or os.path.ismount(
                dmgFile):
            raise ValueError(
                'mountImage requires dmgFile be a path to a file, got: ' +
                dmgFile)
        dmgFile = pathHelpers.normalizePath(dmgFile, followSymlink=True)

        # mountPoint/mountInFolder
        if mountPoint is not None and mountInFolder is not None:
            raise ValueError(
                'mountImage can only be called with mountPoint or mountInFolder, not both'
            )

        elif mountPoint is not None and os.path.ismount(mountPoint):
            raise ValueError(
                'mountImage called with a mountPoint that is already a mount point: '
                + mountPoint)

        elif mountPoint is not None and os.path.isdir(mountPoint):
            if len(os.listdir(mountPoint)) != 0:
                raise ValueError(
                    'mountImage called with a mountPoint that already has contents: %s (%s)'
                    % (mountPoint, str(os.listdir(mountPoint))))

        elif mountPoint is not None and os.path.exists(mountPoint):
            raise ValueError(
                'mountImage called with a mountPoint that already exists and is not a folder: '
                + mountPoint)

        elif mountPoint is not None:
            tempFolderManager.add

        elif mountInFolder is not None and not os.path.isdir(mountInFolder):
            raise ValueError(
                'mountImage called with a mountInFolder path that is not a folder: '
                + mountInFolder)

        elif mountInFolder is not None:
            mountPoint = tempFolderManager.getNewMountPoint()

        else:
            # create a default mount point
            mountPoint = tempFolderManager.getNewMountPoint()
        mountPoint = pathHelpers.normalizePath(mountPoint, followSymlink=True)

        # mountReadWrite
        if mountReadWrite not in [None, True, False]:
            raise ValueError(
                'The only valid options for mountReadWrite are True or False')

        # shadowFile
        if shadowFile is True:
            # generate a temporary one
            shadowFile = tempFolderManager.getNewTempFile(suffix='.shadow')

        elif shadowFile is not None:
            shadowFile = pathHelpers.normalizePath(shadowFile,
                                                   followSymlink=True)

            if os.path.isfile(shadowFile):  # work here
                pass  # just use the file

            elif os.path.isdir(shadowFile):
                # a directory to put the shadow file in
                shadowFile = tempFolderManager.getNewTempFile(
                    parentFolder=shadowFile, suffix='.shadow')

            elif os.path.isdir(os.path.dirname(shadowFile)):
                # the path does not exist, but the directory it is in looks good
                pass

            else:
                # not valid
                raise ValueError(
                    'The path given for the shadow file does not look valid: '
                    + str(shadowFile))

        # paranoidMode
        if paranoidMode not in [True, False]:
            raise ValueError(
                'checksumImage must be either True, or False. Got: ' +
                str(paranoidMode))

        # -- check to see if it is already mounted

        existingMountPoints = myClass.getDMGMountPoints(dmgFile)
        if existingMountPoints is not None:
            raise RuntimeError('The image (%s) was already mounted: %s' %
                               (dmgFile, ", ".join(existingMountPoints)))

        # -- construct the command

        command = [
            '/usr/bin/hdiutil', 'attach', dmgFile, '-plist', '-mountpoint',
            mountPoint, '-nobrowse', '-owners', 'on'
        ]

        if mountReadWrite is True and shadowFile is None and self.writeable is False:
            shadowFile = tempFolderManager.getNewTempFile(suffix='.shadow')
        elif mountReadWrite is False and shadowFile is None:
            command += ['-readonly']

        if paranoidMode is False:
            command += ['-noverify', '-noautofsck']
        else:
            command += ['-verify', '-autofsck']

        if shadowFile is not None:
            command += ['-shadow', shadowFile]

        # -- run the command

        process = managedSubprocess(command, processAsPlist=True)
        mountInfo = process.getPlistObject()

        actualMountedPath = None

        for thisEntry in mountInfo['system-entities']:
            if 'mount-point' in thisEntry:
                if actualMountedPath != None and os.path.ismount(
                        actualMountedPath):
                    # ToDo: think all of this through, prefereabley before it is needed
                    raise Exception(
                        'This item (%s) seems to be mounted at two places , this is possible, but now that it is happening larkost needs to work on this'
                        % self.filePath)

                actualMountedPath = thisEntry[
                    'mount-point']  # ToDo: make sure that this is the mount point we requested
                break  # assume that there is only one mountable item... otherwise we are hosed already

        if actualMountedPath is None:
            raise RuntimeError('Error: image could not be mounted')

        return actualMountedPath