def searchForDependencies(self, projectDirectory): """Search for dependencies in a project. Returns dictionary where the keys are the packages/modules in the project and the values are packages/modules that the respective key imported. Arguments: projectDirectory -- Absolute path to the root directory of the project to search for dependencies in. """ if not os.path.isdir(projectDirectory): raise IOError("'{}' is not a valid directory".format(projectDirectory)) # Important to make the project directory an absolute path projectDirectory = os.path.abspath(projectDirectory) # Extract dependencies for the project directory searcher = FileSearcher(True) filterers = [ ExtensionFilterer(["py"]) ] extractor = ModuleDependencyExtractor() processor = FileProcessor(searcher, filterers, extractor) dependencies = processor.process(projectDirectory) # Resolve relative imports resolver = ImportResolver(projectDirectory) dependencies = resolver.resolveImports(dependencies) # Finally, apply a whitelist to the dependencies to only # include modules that belong to the scanned project whitelistApplier = WhitelistApplier() return whitelistApplier.applyToProject(projectDirectory, dependencies)
def main(directory, showAll): """Run image resource extraction process. Arguments: directoriesToSearch -- List containing all of the directories containing web page soure code that need to be scanned showAll -- If set to True, then all scanned files will be displayed, even if no image URLs were extracted from them. This means if this is False, then any files where no data was found are omitted. """ # Build components to use for file processor searcher = searchers.FileSearcher(True) filterer = filterers.ExtensionFilterer( ["html", "htm", "shtml", "php", "css", "js"] ) extractor = ImageURLExtractor() processor = FileProcessor(searcher, [ filterer ], extractor) # Perform the URL extraction and display findings extractedURLs = processor.process(directoriesToSearch) for filename, imageURLs in extractedURLs.items(): # If nothing was found in this file and the # approrpiate flag is set, skip this file if len(imageURLs) == 0 and not showAll: continue imageURLLines = "" for url in imageURLs: imageURLLines += "\t{}\n".format(url) # Remove last newline if len(imageURLLines) > 0: imageURLLines = imageURLLines[:-1] print("{}\n{}".format(filename, imageURLLines))
class TestFileProcessor(unittest.TestCase): def setUp(self): self.mockSearcher = MockFileSearcher() self.mockFilterers = [MockFilterer(), MockFilterer2()] self.mockExtractor = MockDataExtractor() self.fileProcessor = FileProcessor(self.mockSearcher, self.mockFilterers, self.mockExtractor) # Create empty directories just so the file processor # doesn't comaplin when we test process() os.mkdir("empty_dir") os.mkdir("root_dir") os.mkdir("second_dir") def tearDown(self): self.mockSearcher = None self.mockFilterer = None self.mockExtractor = None self.fileProcessor = None # Delete created directories os.rmdir("empty_dir") os.rmdir("root_dir") os.rmdir("second_dir") def test_constructor(self): # Test valid arguments (use constructed object in setUp()) self.assertEqual(self.fileProcessor.searcher, self.mockSearcher) self.assertEqual(self.fileProcessor.filterers, self.mockFilterers) self.assertEqual(self.fileProcessor.extractor, self.mockExtractor) def test_process_failure(self): # Invalid directory (type) with self.assertRaises(TypeError): self.fileProcessor.process(5454) def test_process_success(self): EXPECTED_DATA = { "/path/to/stuff.txt" : "/path/to/stuff.txt: PROCESSED", "/another_path/test.txt" : "/another_path/test.txt: PROCESSED" } EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES = dict(EXPECTED_DATA) EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES.update({ "two_more_.txt" : "two_more_.txt: PROCESSED", "files.txt" : "files.txt: PROCESSED" }) # Test invalid directory (doesn't exist). # Should just return no data, not raise exception! self.assertEqual(self.fileProcessor.process("non-existent"), {}) # Test with no files self.assertEqual(self.fileProcessor.process("empty_dir"), {}) # Test with files self.assertEqual(self.fileProcessor.process("root_dir"), EXPECTED_DATA) # Test with multiple directores, with one that doesn't exist # Should just not search non-existent directory but search the others self.assertEqual(self.fileProcessor.process( ["root_dir", "non-existent"] ), EXPECTED_DATA) # Test with multiple directores, where they all exist self.assertEqual(self.fileProcessor.process( ["root_dir", "second_dir"] ), EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES)
def main(directoriesToSearch): """Run checksum generation process. Arguments: directoriesToSearch -- List containing all of the directories containing files to generate checksums for """ # Build components to use for file processor searcher = searchers.FileSearcher(True) extractor = ChecksumGenerator() processor = FileProcessor(searcher, [], extractor) # Perofrm checksum generation and display every checksum generatedChecksums = processor.process(directoriesToSearch) for filename, checksum in generatedChecksums.items(): print("{}\n\t{}".format(filename, checksum))