def searchForDependencies(self, projectDirectory):
"""Search for dependencies in a project.
Returns dictionary where the keys are the packages/modules
in the project and the values are packages/modules that
the respective key imported.
Arguments:
projectDirectory -- Absolute path to the root directory
of the project to search for
dependencies in.
"""
if not os.path.isdir(projectDirectory):
raise IOError("'{}' is not a valid directory".format(projectDirectory))
# Important to make the project directory an absolute path
projectDirectory = os.path.abspath(projectDirectory)
# Extract dependencies for the project directory
searcher = FileSearcher(True)
filterers = [ ExtensionFilterer(["py"]) ]
extractor = ModuleDependencyExtractor()
processor = FileProcessor(searcher, filterers, extractor)
dependencies = processor.process(projectDirectory)
# Resolve relative imports
resolver = ImportResolver(projectDirectory)
dependencies = resolver.resolveImports(dependencies)
# Finally, apply a whitelist to the dependencies to only
# include modules that belong to the scanned project
whitelistApplier = WhitelistApplier()
return whitelistApplier.applyToProject(projectDirectory, dependencies)
def main(directory, showAll):
"""Run image resource extraction process.
Arguments:
directoriesToSearch -- List containing all of the
directories containing web page
soure code that need to be scanned
showAll -- If set to True, then all scanned files will
be displayed, even if no image URLs were
extracted from them. This means if this is
False, then any files where no data was
found are omitted.
"""
# Build components to use for file processor
searcher = searchers.FileSearcher(True)
filterer = filterers.ExtensionFilterer( ["html", "htm", "shtml", "php", "css", "js"] )
extractor = ImageURLExtractor()
processor = FileProcessor(searcher, [ filterer ], extractor)
# Perform the URL extraction and display findings
extractedURLs = processor.process(directoriesToSearch)
for filename, imageURLs in extractedURLs.items():
# If nothing was found in this file and the
# approrpiate flag is set, skip this file
if len(imageURLs) == 0 and not showAll:
continue
imageURLLines = ""
for url in imageURLs:
imageURLLines += "\t{}\n".format(url)
# Remove last newline
if len(imageURLLines) > 0:
imageURLLines = imageURLLines[:-1]
print("{}\n{}".format(filename, imageURLLines))
class TestFileProcessor(unittest.TestCase):
def setUp(self):
self.mockSearcher = MockFileSearcher()
self.mockFilterers = [MockFilterer(), MockFilterer2()]
self.mockExtractor = MockDataExtractor()
self.fileProcessor = FileProcessor(self.mockSearcher,
self.mockFilterers, self.mockExtractor)
# Create empty directories just so the file processor
# doesn't comaplin when we test process()
os.mkdir("empty_dir")
os.mkdir("root_dir")
os.mkdir("second_dir")
def tearDown(self):
self.mockSearcher = None
self.mockFilterer = None
self.mockExtractor = None
self.fileProcessor = None
# Delete created directories
os.rmdir("empty_dir")
os.rmdir("root_dir")
os.rmdir("second_dir")
def test_constructor(self):
# Test valid arguments (use constructed object in setUp())
self.assertEqual(self.fileProcessor.searcher, self.mockSearcher)
self.assertEqual(self.fileProcessor.filterers, self.mockFilterers)
self.assertEqual(self.fileProcessor.extractor, self.mockExtractor)
def test_process_failure(self):
# Invalid directory (type)
with self.assertRaises(TypeError):
self.fileProcessor.process(5454)
def test_process_success(self):
EXPECTED_DATA = {
"/path/to/stuff.txt" : "/path/to/stuff.txt: PROCESSED",
"/another_path/test.txt" : "/another_path/test.txt: PROCESSED"
}
EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES = dict(EXPECTED_DATA)
EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES.update({
"two_more_.txt" : "two_more_.txt: PROCESSED",
"files.txt" : "files.txt: PROCESSED"
})
# Test invalid directory (doesn't exist).
# Should just return no data, not raise exception!
self.assertEqual(self.fileProcessor.process("non-existent"), {})
# Test with no files
self.assertEqual(self.fileProcessor.process("empty_dir"), {})
# Test with files
self.assertEqual(self.fileProcessor.process("root_dir"), EXPECTED_DATA)
# Test with multiple directores, with one that doesn't exist
# Should just not search non-existent directory but search the others
self.assertEqual(self.fileProcessor.process( ["root_dir", "non-existent"] ),
EXPECTED_DATA)
# Test with multiple directores, where they all exist
self.assertEqual(self.fileProcessor.process( ["root_dir", "second_dir"] ),
EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES)
def main(directoriesToSearch):
"""Run checksum generation process.
Arguments:
directoriesToSearch -- List containing all of the
directories containing files
to generate checksums for
"""
# Build components to use for file processor
searcher = searchers.FileSearcher(True)
extractor = ChecksumGenerator()
processor = FileProcessor(searcher, [], extractor)
# Perofrm checksum generation and display every checksum
generatedChecksums = processor.process(directoriesToSearch)
for filename, checksum in generatedChecksums.items():
print("{}\n\t{}".format(filename, checksum))
def main(directoriesToSearch):
"""Run checksum generation process.
Arguments:
directoriesToSearch -- List containing all of the
directories containing files
to generate checksums for
"""
# Build components to use for file processor
searcher = searchers.FileSearcher(True)
extractor = ChecksumGenerator()
processor = FileProcessor(searcher, [], extractor)
# Perofrm checksum generation and display every checksum
generatedChecksums = processor.process(directoriesToSearch)
for filename, checksum in generatedChecksums.items():
print("{}\n\t{}".format(filename, checksum))
