def searchForDependencies(self, projectDirectory):
"""Search for dependencies in a project.
Returns dictionary where the keys are the packages/modules
in the project and the values are packages/modules that
the respective key imported.
Arguments:
projectDirectory -- Absolute path to the root directory
of the project to search for
dependencies in.
"""
if not os.path.isdir(projectDirectory):
raise IOError("'{}' is not a valid directory".format(projectDirectory))
# Important to make the project directory an absolute path
projectDirectory = os.path.abspath(projectDirectory)
# Extract dependencies for the project directory
searcher = FileSearcher(True)
filterers = [ ExtensionFilterer(["py"]) ]
extractor = ModuleDependencyExtractor()
processor = FileProcessor(searcher, filterers, extractor)
dependencies = processor.process(projectDirectory)
# Resolve relative imports
resolver = ImportResolver(projectDirectory)
dependencies = resolver.resolveImports(dependencies)
# Finally, apply a whitelist to the dependencies to only
# include modules that belong to the scanned project
whitelistApplier = WhitelistApplier()
return whitelistApplier.applyToProject(projectDirectory, dependencies)
def run(self, header, opts):
proxy = _getProxy(header)
comment = proxy.getComment(self.commentid)
commenttext = "COMMENTID: %s\n" % (self.commentid)
commenttext += "PARENTID: %s\n" % (comment['parent'])
commenttext += "COMMENTSTATUS: %s\n" % (comment['status'])
commenttext += "AUTHOR: %s\n" % (comment['author'])
if comment['author_url']:
commenttext += "AUTHORURL: %s\n" % (comment['author_url'])
commenttext += "AUTHOREMAIL: %s\n" % (comment['author_email'])
commenttext += "\n%s" % (html2md.convert(comment['content']))
fd = utils.edit(commenttext)
if fd == None:
print "Nothing to do with comment."
fp = FileProcessor(**{'addpostcats' : False,
'publish' : True,
'posttime' : None,
'allblogs' : False,
'comment' : True,
'charset': opts.charset,
})
try:
header_text, commenttext = fp.parsePostFile(fd.name, '')
except FileProcessorError, err_msg:
print err_msg
sys.exit()
def build(self):
f = FileProcessor()
manager = ScreenManager()
manager.add_widget(LoginWindow(name="Login"))
manager.add_widget(DashboardWindow(name="Dashboard"))
f.processFiles()
return manager
def main(directory, showAll):
"""Run image resource extraction process.
Arguments:
directoriesToSearch -- List containing all of the
directories containing web page
soure code that need to be scanned
showAll -- If set to True, then all scanned files will
be displayed, even if no image URLs were
extracted from them. This means if this is
False, then any files where no data was
found are omitted.
"""
# Build components to use for file processor
searcher = searchers.FileSearcher(True)
filterer = filterers.ExtensionFilterer( ["html", "htm", "shtml", "php", "css", "js"] )
extractor = ImageURLExtractor()
processor = FileProcessor(searcher, [ filterer ], extractor)
# Perform the URL extraction and display findings
extractedURLs = processor.process(directoriesToSearch)
for filename, imageURLs in extractedURLs.items():
# If nothing was found in this file and the
# approrpiate flag is set, skip this file
if len(imageURLs) == 0 and not showAll:
continue
imageURLLines = ""
for url in imageURLs:
imageURLLines += "\t{}\n".format(url)
# Remove last newline
if len(imageURLLines) > 0:
imageURLLines = imageURLLines[:-1]
print("{}\n{}".format(filename, imageURLLines))
def __init__(self, file_data):
FileProcessor.__init__(self, file_data)
self.uploaded_file_path = gen_file_path(self.client_id,self.filename,'upload')
self.audio_filename = self.filename.split('.')[0]
self.output_filename = self.audio_filename +'.csv'
self.output_file_path = gen_file_path(self.client_id,self.output_filename,'download')
self.u_file.save(self.uploaded_file_path)
self.url = self.output_file_path
self.delete_url = "delete/%s" % self.output_file_path
self.error_dict = {}
self.word_timing_data = {}
def main(directoriesToSearch):
"""Run checksum generation process.
Arguments:
directoriesToSearch -- List containing all of the
directories containing files
to generate checksums for
"""
# Build components to use for file processor
searcher = searchers.FileSearcher(True)
extractor = ChecksumGenerator()
processor = FileProcessor(searcher, [], extractor)
# Perofrm checksum generation and display every checksum
generatedChecksums = processor.process(directoriesToSearch)
for filename, checksum in generatedChecksums.items():
print("{}\n\t{}".format(filename, checksum))
def main(directoriesToSearch):
"""Run checksum generation process.
Arguments:
directoriesToSearch -- List containing all of the
directories containing files
to generate checksums for
"""
# Build components to use for file processor
searcher = searchers.FileSearcher(True)
extractor = ChecksumGenerator()
processor = FileProcessor(searcher, [], extractor)
# Perofrm checksum generation and display every checksum
generatedChecksums = processor.process(directoriesToSearch)
for filename, checksum in generatedChecksums.items():
print("{}\n\t{}".format(filename, checksum))
class TestFileProcessor(unittest.TestCase):
def setUp(self):
self.mockSearcher = MockFileSearcher()
self.mockFilterers = [MockFilterer(), MockFilterer2()]
self.mockExtractor = MockDataExtractor()
self.fileProcessor = FileProcessor(self.mockSearcher,
self.mockFilterers, self.mockExtractor)
# Create empty directories just so the file processor
# doesn't comaplin when we test process()
os.mkdir("empty_dir")
os.mkdir("root_dir")
os.mkdir("second_dir")
def tearDown(self):
self.mockSearcher = None
self.mockFilterer = None
self.mockExtractor = None
self.fileProcessor = None
# Delete created directories
os.rmdir("empty_dir")
os.rmdir("root_dir")
os.rmdir("second_dir")
def test_constructor(self):
# Test valid arguments (use constructed object in setUp())
self.assertEqual(self.fileProcessor.searcher, self.mockSearcher)
self.assertEqual(self.fileProcessor.filterers, self.mockFilterers)
self.assertEqual(self.fileProcessor.extractor, self.mockExtractor)
def test_process_failure(self):
# Invalid directory (type)
with self.assertRaises(TypeError):
self.fileProcessor.process(5454)
def test_process_success(self):
EXPECTED_DATA = {
"/path/to/stuff.txt" : "/path/to/stuff.txt: PROCESSED",
"/another_path/test.txt" : "/another_path/test.txt: PROCESSED"
}
EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES = dict(EXPECTED_DATA)
EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES.update({
"two_more_.txt" : "two_more_.txt: PROCESSED",
"files.txt" : "files.txt: PROCESSED"
})
# Test invalid directory (doesn't exist).
# Should just return no data, not raise exception!
self.assertEqual(self.fileProcessor.process("non-existent"), {})
# Test with no files
self.assertEqual(self.fileProcessor.process("empty_dir"), {})
# Test with files
self.assertEqual(self.fileProcessor.process("root_dir"), EXPECTED_DATA)
# Test with multiple directores, with one that doesn't exist
# Should just not search non-existent directory but search the others
self.assertEqual(self.fileProcessor.process( ["root_dir", "non-existent"] ),
EXPECTED_DATA)
# Test with multiple directores, where they all exist
self.assertEqual(self.fileProcessor.process( ["root_dir", "second_dir"] ),
EXPECTED_DATA_WITH_MULTIPLE_DIRECTORIES)
def run():
fp = FileProcessor(**options.flags())
tmp_fn = None
for filename in filelist:
if tmp_fn != filename:
tmp_fn = filename
print "Processing post file %s..." % filename
try:
header_text, post_text = fp.parsePostFile(filename,
emptyheader_text)
except FileProcessorRetry:
filelist.insert(0, filename)
continue
except FileProcessorError, err_msg:
print err_msg
continue
header.addParms(header_text, fp.allblogs)
for hdr in header:
try:
rval = fp.pushContent(post_text, hdr)
if rval and not fp.comment:
header.postid = rval
print 'Updating post file...'
fp.updateFile(filename, '%s' % header, post_text)
except FileProcessorError, err:
print err
if filename.startswith("/tmp"):
if fp.comment:
filename += '.' + hdr.postid
else:
filename += '.' + hdr.title
print "Saving tmp content file %s" % filename
f = open(filename, 'w')
f.write('%s' % header + '\n' + post_text)
f.close()
# It's possible there are other files to process so rather than
# bailing entirely we'll break out of this loop and move on to
# the next file if there is one. In most cases, this will be
# like exitting the program since typical usage doesn't have
# multiple files to process.
break
def setUp(self):
self.mockSearcher = MockFileSearcher()
self.mockFilterers = [MockFilterer(), MockFilterer2()]
self.mockExtractor = MockDataExtractor()
self.fileProcessor = FileProcessor(self.mockSearcher,
self.mockFilterers, self.mockExtractor)
# Create empty directories just so the file processor
# doesn't comaplin when we test process()
os.mkdir("empty_dir")
os.mkdir("root_dir")
os.mkdir("second_dir")
def ExportToExcel(self, path, expIds):
# Trigger the worker thread unless it's already busy
if not self.worker:
#self.status.SetLabel('Starting computation')
self.worker = FileProcessor(self, path, expIds)
max = 100
self.progress = wx.ProgressDialog(
"Export in Progress",
"Preparing ...",
maximum=max,
parent=self,
style=0
| wx.PD_APP_MODAL
| wx.PD_CAN_ABORT
#| wx.PD_CAN_SKIP
#| wx.PD_ELAPSED_TIME
| wx.PD_ESTIMATED_TIME
| wx.PD_REMAINING_TIME
#| wx.PD_AUTO_HIDE
)
class MainExpList(MagMainListCtrl):
def __init__(self, *args, **kwargs):
ObjectListView.__init__(self, *args, **kwargs)
self.Bind(wx.EVT_LIST_ITEM_RIGHT_CLICK, self.OnRightClick)
self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.OnActivated)
self.Bind(EVT_XLS_WRITE_COMPLETED, self.OnComplete)
self.Bind(EVT_XLS_WRITE_PROGRESS, self.OnProgress)
self.Bind(EVT_XLS_WRITE_CANCELED, self.OnCanceled)
self.SetColumns(ExpListColumnDefn)
self.worker = None
def ExportToExcel(self, path, expIds):
# Trigger the worker thread unless it's already busy
if not self.worker:
#self.status.SetLabel('Starting computation')
self.worker = FileProcessor(self, path, expIds)
max = 100
self.progress = wx.ProgressDialog(
"Export in Progress",
"Preparing ...",
maximum=max,
parent=self,
style=0
| wx.PD_APP_MODAL
| wx.PD_CAN_ABORT
#| wx.PD_CAN_SKIP
#| wx.PD_ELAPSED_TIME
| wx.PD_ESTIMATED_TIME
| wx.PD_REMAINING_TIME
#| wx.PD_AUTO_HIDE
)
def GetExpSelectedExpIds(self, state=wx.LIST_STATE_SELECTED):
exp_ids = []
objs = self.GetSelectedObjects(state)
for o in objs:
exp_ids.append(o.id)
return exp_ids
def OnCanceled(self, event):
self.worker = None
dlg = wx.MessageDialog(
self, event.data['message'], event.data['title'],
wx.OK | wx.ICON_INFORMATION
#wx.YES_NO | wx.NO_DEFAULT | wx.CANCEL | wx.ICON_INFORMATION
)
dlg.ShowModal()
dlg.Destroy()
self.progress.Destroy()
def OnProgress(self, event):
self.progress.Update(event.step, event.message)
def OnStop(self, event):
"""Stop Computation."""
# Flag the worker thread to stop if running
if self.worker:
self.progress.Destroy()
self.worker.abort()
wx.MessageDialog(None, 'Operation Stopped', 'Error',
wx.NO_DEFAULT | wx.ICON_STOP)
def OnComplete(self, event):
"""Show Result status."""
# Process results here
#self.status.SetLabel('Computation Result: %s' % event.data)
# In either event, the worker is done
self.progress.Destroy()
self.worker = None
wx.MessageBox('Operation completed', 'Info',
wx.OK | wx.ICON_INFORMATION)
def OnActivated(self, event):
#frame = GraphFrame()
#frame.Show()
#frame = ExpACCoilPropertyDialog(self)
#frame.ShowModal()
frame = ImpedanceGraphFrame(wx.GetApp().TopWindow,
-1,
'Impedance Graph',
exp_ids=self.GetExpSelectedExpIds())
frame.Show()
def OnRightClick(self, event):
### 2. Launcher creates wxMenu. ###
menu = MainExpListPopup(self, expIds=self.GetExpSelectedExpIds())
### 5. Launcher displays menu with call to PopupMenu, invoked on the source component, passing event's GetPoint. ###
self.PopupMenu(menu, event.GetPoint())
menu.Destroy() # destroy to avoid mem leak
def setData(self, ac_coils=[], src_powers=[], dc_currents=[]):
self.SetObjects(getAllExpsByFilter(ac_coils, src_powers, dc_currents))