def get_res(self):
for repo in self.repo_list:
print(
"______________________________________________________________________________________"
)
print(repo)
self.analysisManager = AnalysisManager(self, repo)
self.analysisManager.set_xml_files_list(repo + "/~Temp/")
self.analysisManager.process_data()
def load_xml_clicked(self):
xml_file = str(QFileDialog.getOpenFileName(self, "Select XML File")[0])
self.progressBar.setValue(0)
self.print_line("Loading data from XML structure. Please wait ...\n")
self.progressBar.update()
xml_dir = os.path.dirname(xml_file)
self.analysisManager = AnalysisManager(self, xml_dir)
self.analysisManager.load_structure_from_xml(xml_file)
self.progressBar.setValue(100)
self.progressBar.update()
def __init__(self, import_name):
super(AudioRec, self).__init__(import_name)
self.route("/audiorec/")(self.audiorec)
self.route("/select")(self.selectArticlePage)
self.route("/loading-results/")(self.loadScreen)
self.route("/selectfunc", methods=['GET','POST'])(self.text_choice_func)
self.fi = STTFileImporter()
self.am = AnalysisManager()
self.dbh = DatabaseHelper().getInstance()
self.tipSelector = FileImporter()
lh = LoggerHelper().getInstance()
self.recordinglogger = lh.createLoggerWorker("recording","DEBUG",2)
self.recordinglogger.info("Audio Rec Instantiated")
def load_files_clicked(self):
file_names = self.model.export_checked()
repo_dir = self.model.rootDir
self.analysisManager = AnalysisManager(self, repo_dir)
self.analysisManager.set_files_list(file_names)
self.print_line("Files loaded:\n")
total = len(file_names)
count = 1
self.progressBar.setValue(0)
self.progressBar.update()
for file in file_names:
self.print_line(file + "\n")
self.progressBar.setValue((count * 100) / total)
self.progressBar.update()
count += 1
self.print_line("Total number of files loaded : {}".format(total))
class AudioRec(flask.Flask):
def __init__(self, import_name):
super(AudioRec, self).__init__(import_name)
self.route("/audiorec/")(self.audiorec)
self.route("/select")(self.selectArticlePage)
self.route("/loading-results/")(self.loadScreen)
self.route("/selectfunc", methods=['GET','POST'])(self.text_choice_func)
self.fi = STTFileImporter()
self.am = AnalysisManager()
self.dbh = DatabaseHelper().getInstance()
self.tipSelector = FileImporter()
lh = LoggerHelper().getInstance()
self.recordinglogger = lh.createLoggerWorker("recording","DEBUG",2)
self.recordinglogger.info("Audio Rec Instantiated")
def text_choice_func(self):
if flask.request.method == "POST":
file = flask.request.form['file']
file += '.txt'
orderNames = self.fi.getOrdering(flask.session['sessionID'])
for val in orderNames:
text_files = fnmatch.filter(os.listdir('files/daily-articles/' + val + '/' ), '*.txt')
if file in text_files:
try:
f = open('files/daily-articles/' + val + '/' + file)
except:
f = codecs.open('files/daily-articles/' + val + '/' + file, "r", encoding="windows-1252")
self.text = f.read()
self.text = self.fi.trimText(self.text)
return flask.jsonify('complete')
return flask.render_template('wrong.html')
@login_required
def selectArticlePage(self):
orderNames = self.fi.getOrdering(flask.session['sessionID'])
orderImages = []
orderhead1 = []
orderhead2 = []
orderhead3 = []
countEmpty = 0
self.flasklogger.debug("{}".format(orderNames))
for val in orderNames:
with open('files/Daily-Articles/ArticleSites.txt') as f:
for line in f:
line = line.split()
if line[0] == val:
orderImages.append(line[3])
text_files = fnmatch.filter(os.listdir('files/daily-articles/' + val + '/' ), '*.txt')
self.flasklogger.debug("text files: {}".format(text_files))
while len(text_files)< 3:
text_files.append("EMPTY")
countEmpty += 1
self.flasklogger.debug("Number of empty values: {}".format( countEmpty))
orderhead1.append(text_files[0].replace(".txt", ''))
orderhead2.append(text_files[1].replace(".txt", ''))
orderhead3.append(text_files[2].replace(".txt", ''))
if countEmpty < 9:
ser_imgs = pd.Series(orderImages, index = [orderNames], name = "images")
ser_head1 = pd.Series(orderhead1, index = [orderNames], name = "head1")
ser_head2 = pd.Series(orderhead2, index = [orderNames], name = "head2")
ser_head3 = pd.Series(orderhead3, index = [orderNames], name = "head3")
frame = pd.concat([ser_imgs,ser_head1,ser_head2,ser_head3],axis=1)
frame.to_json("static/json/prefs.json")
f = open("static/json/prefs.json", 'r')
jsonstring = f.read()
return flask.render_template('selectArticle.html', jsonstring=jsonstring)
else:
self.flasklogger.debug("Number of empty values: {}".format( countEmpty))
self.text = self.fi.loadFile()
return flask.redirect('/audiorec/')
# This function deals with the recording page
@login_required
def audiorec(self):
try:
self.tip = self.tipSelector.loadFile()
self.userID = flask.session['sessionID']
self.username = flask.session['user']
return flask.render_template('recordnew.html', value = self.text, username=self.username,
filename=self.am.generateFilename(self.username))
except Exception as e:
self.recordinglogger.error("Recording Page Failed: Exception:{0}".format(e))
return flask.render_template('wrong.html', error = "{}".format(e))
# This function starts the analysis threads
@login_required
def loadScreen(self):
try:
self.am.startAnalysis(self.userID,self.text)
return flask.render_template('loading.html', value = self.tip)
except Exception as e:
self.recordinglogger.error("Load Screen Failed: Exception:{0}".format(e))
return flask.render_template('wrong.html', error = "{}".format(e))
class Application():
'''
This class is the bridge to the back-end. All function calls in the GUI
are in this class. From here, it will split off into three major
modules. The functions in this class are plain- they send calls down
to sub modules where most of the "heavy lifting" occurs. This structure
allows for separation of distinct functionality/duties of the app. The
three main branches are the PersistentDataManager, the TransactionManager,
and the Analysis Manager.
See each of those classes to get the description on their distinct functionality.
'''
# intitializes 3 managers with specialized tools in each of their layers
def __init__(self):
self.pdm = PersistentDataManager()
self.transactionManager = TransactionManager()
self.analysisManager = AnalysisManager()
def initialize(self):
'''
Retrieves past transactions and categories from xml
files from past uses of the app recreates categories
and transactions.
'''
tList, cList = self.pdm.retrievePersistentData("TransactionData.xml",
"CategoryData.xml")
for c in cList:
self.transactionManager.createCategory(c)
self.analysisManager.categories = self.transactionManager.categories
for t in tList:
self.transactionManager.createTransaction(t)
self.analysisManager.plannedTransactions = self.transactionManager.transactions
def sortCompletedTransactions(self, fileName):
'''
Wrapper function for AnalysisManager's equivalent
call.
@filename: Path to CSV file that will be parsed
and used to create completedTransaction objects
'''
self.fileName = fileName
self.analysisManager.sortCompletedTransactions(fileName)
self.analysisManager.getCompletedTransactionsDataframe()
self.transactionManager.completedTransactions = self.analysisManager.completedTransactions
def saveData(self):
'''
Calls function in PersistentDataManager that writes
planned transactions and category info to an .xml file
@transactionData.xml, categoryData.xml: files on which
planned transactions and category info are stored
@self.transactionManager.transactions.values(): names
of plannedTransaction objects
@self.transactionManager.categories.values(): names
of Category objects
'''
self.pdm.stashPersistentData(
"TransactionData.xml", "CategoryData.xml",
self.transactionManager.transactions.values(),
self.transactionManager.categories.values())
def saveUserSetupData(self, chkAccBal, incomeAmt, incomeFreq, payDate,
ccDate):
self.analysisManager.saveUserSetupData(float(chkAccBal),
float(incomeAmt),
str(incomeFreq), str(payDate),
str(ccDate))
def getCategoryNamesList(self):
'''
This method can be used to get the keys
to look up category information using
other API functions.
@returns: A list of keys for a category
dictionary in the form of category names
'''
return list(self.analysisManager.categories.keys())
def getUnhandledTransactions(self):
'''
Gets all completedTransaction objects in the
"Unhandled" category
@returns: CompletedTransaction objects that
have not been categorized
'''
return self.transactionManager.categories[
"Unhandled"].completedTransactions
def createNewCategory(self, data):
'''
pyCategoryPop.py GUI window sends APIData object through here
to TransactionManager where new Category object is made
@data: APIData object with name, allotted amount, and keywords
to make new Category object in back end
'''
self.transactionManager.createCategory(data)
self.analysisManager.categories = self.transactionManager.categories
def updateCategoryData(self, data):
'''
Same functionality as createNewCategory, but firsr
checks if the name of existing category was changed
@data: APIData object with name, allotted amount, and keywords
to make new Category object in back end
'''
self.transactionManager.updateCategoryData(data)
self.analysisManager.categories = self.transactionManager.categories
def getCompletedTransactionsList(self):
'''
@returns: The list of completed transactions from the
most recently imported csv file
'''
return self.analysisManager.completedTransactions
def getTotalAmountSpent(self):
'''
@returns: The total amount spent from all the transactions
in the most recently imported csv file
'''
return abs(self.analysisManager.getTotalAmountSpent())
def getAmountSpentByCategory(self, categoryName):
'''
Wrapper function for the AnalysisManager's equivalent
call
@categoryName: A string to be used as a dictionary
key to associate to the corresponding category object
@returns: The sum of all transactions that have been
associated with that category
'''
return abs(
round(self.analysisManager.getAmountSpentByCategory(categoryName),
2))
def getDeltaByCategory(self, categoryName):
'''
Wrapper function for the AnalysisManager's equivalent
call
@categoryName: A string to be used as a dictionary
key to associate to the corresponding category object
@returns: The difference between the amount that was
allotted and the amount that was spent for a given category
'''
return round(self.analysisManager.getDeltaByCategory(categoryName), 2)
def getAmountAllottedByCategory(self, categoryName):
'''
Wrapper function for the AnalysisManager's equivalent
call
@categoryName: A string to be used as a dictionary
key to associate to the corresponding category object
@returns: The amount that the user allotted for a given
category
'''
return self.analysisManager.getAmountAllottedByCategory(categoryName)
def getAmountPlannedByCategory(self, categoryName):
'''
Wrapper function for the AnalysisManager's equivalent
call
@categoryName: A string to be used as a dictionary
key to associate to the corresponding category object
@returns: The sum of all planned transactions associated with
the given category
'''
return self.analysisManager.getAmountPlannedByCategory(categoryName)
def getKeywordsByCategory(self, categoryName):
'''
Wrapper function for the AnalysisManager's equivalent
call
@categoryName: A string to be used as a dictionary
key to associate to the corresponding category object
@returns: The list of keywords to identify transactions
'''
return self.analysisManager.getKeywordsByCategory(categoryName)
def deleteCategory(self, categoryName):
'''
Wrapper function for the Transaction Manager's equivalent call.
After the call, the category list of the Analysis Manager is
updated to reflect the change. I am not sure if python is copying the
list over between the 2 managers, or if the Analysis Manager is
getting a pointer/reference to the Transaction Manager's original
list of categories. If the situation is the latter, then the second
line of this method is actually not necessary. For now, I am playing
it safe and making sure the Analysis Manager is updated accordingly.
@categoryName: Category name to be deleted
'''
self.transactionManager.deleteCategory(categoryName)
self.analysisManager.categories = self.transactionManager.categories
def registerCompletedTransaction(self, categoryName, transRefNumber):
'''
Registers a completed transaction with a category so that it can be
properly sorted next time.
@categoryName: Name of the category that the
transaction will be registered with
@transRefNumber: Transaction reference number to be used
as a key by the TransactionManager to locate the Transaction
'''
self.transactionManager.registerCompletedTransaction(
categoryName, transRefNumber)
self.analysisManager.categories[
categoryName] = self.transactionManager.categories[categoryName]
def unregisterCompletedTransaction(self, categoryName, transRefNumber):
'''
Unregisters a completed transaction from a specified category.
@categoryName: Name of the category that the
transaction will be registered with
@transRefNumber: Transaction reference number to be used
as a key by the TransactionManager to locate the Transaction
'''
self.transactionManager.unregisterCompletedTransaction(
categoryName, transRefNumber)
self.analysisManager.categories[
categoryName] = self.transactionManager.categories[categoryName]
def getCompletedTransactionsByCategory(self, categoryName):
'''
@categoryName: Name of the category to query for
completed transactions
@returns: A dictionary of completed transactions that
are registered to the specified category
'''
return self.transactionManager.getCompletedTransactionsByCategory(
categoryName)
def createPlannedTransaction(self, data):
'''
Calls mirror function in TransactionManager. Creates planned transaction within the
user defined category
@data: TransactionData API object with identifying members
'''
self.transactionManager.createTransaction(data)
self.analysisManager.plannedTransactions[
data.name] = self.transactionManager.transactions[data.name]
def getPlannedTransactions(self, category):
'''
Wrapper function for TransactionManager's
equivalent
@category: Name of category of interest
@returns: All PlannedTransaction objects in
respective category
'''
return self.transactionManager.getPlannedTransactions(category)
def removePlannedTransaction(self, category, name):
'''
Wrapper function for removing a planned transaction for the list
of plannedTransactions
@category: specified category of transaction chosen
@name: name of transaction chosen to be deleted
'''
self.transactionManager.removePlannedTransaction(category, name)
def diagnosticDbg(self):
self.transactionManager.categoryDbg()
def getTimeSeriesData(self):
'''
Wrapper function for getting time series data to be plotted
in the PlottingWindow.
'''
return self.analysisManager.getTimeSeriesData()
def createCompletedTransactionsDataFrame(self):
return self.analysisManager.getCompletedTransactionsDataframe()
def getProjectionData(self):
'''
Wrapper function for projection data to be plotted in
ProjectionWindow.
'''
return self.analysisManager.getProjectionData()
def __init__(self):
self.pdm = PersistentDataManager()
self.transactionManager = TransactionManager()
self.analysisManager = AnalysisManager()
class Dialog(QMainWindow):
def __init__(self, app, exePath=None, parent=None):
super(QMainWindow, self).__init__(parent)
self.accepted_suffix = ['cpp', 'h', 'cc', 'c++', 'java', 'cs']
self.repo_list = [
"E:\\faculta\\Master\\TestProjects\\bluecove",
"E:\\faculta\\Master\\TestProjects\\aima-java",
"E:\\faculta\\Master\\TestProjects\\powermock",
"E:\\faculta\\Master\\TestProjects\\restfb",
"E:\\faculta\\Master\\TestProjects\\RxJava",
"E:\\faculta\\Master\\TestProjects\\metro-jax-ws",
"E:\\faculta\\Master\\TestProjects\\mockito",
"E:\\faculta\\Master\\TestProjects\\grizzly",
"E:\\faculta\\Master\\TestProjects\\shipkit",
"E:\\faculta\\Master\\TestProjects\\OpenClinica",
"E:\\faculta\\Master\\TestProjects\\robolectric",
"E:\\faculta\\Master\\TestProjects\\aeron",
"E:\\faculta\\Master\\TestProjects\\antlr4",
"E:\\faculta\\Master\\TestProjects\\mcidasv",
"E:\\faculta\\Master\\TestProjects\\ShareX",
"E:\\faculta\\Master\\TestProjects\\aspnetboilerplate",
"E:\\faculta\\Master\\TestProjects\\orleans",
"E:\\faculta\\Master\\TestProjects\\cli",
"E:\\faculta\\Master\\TestProjects\\cake",
"E:\\faculta\\Master\\TestProjects\\Avalonia",
"E:\\faculta\\Master\\TestProjects\\EntityFrameworkCore",
"E:\\faculta\\Master\\TestProjects\\jellyfin",
"E:\\faculta\\Master\\TestProjects\\PowerShell",
"E:\\faculta\\Master\\TestProjects\\WeiXinMPSDK",
"E:\\faculta\\Master\\TestProjects\\ArchiSteamFarm",
"E:\\faculta\\Master\\TestProjects\\VisualStudio",
"E:\\faculta\\Master\\TestProjects\\CppSharp"
]
self.setWindowFlags(QtCore.Qt.FramelessWindowHint)
self.setAutoFillBackground(True)
self.resize(900, 700)
self.setStyleSheet("""MainWindow { border-radius: 5px; }""")
self.app = app
self.analysisManager = None
self.show()
self.maxNormal = False
self.main_widget = QWidget(self)
self.main_layout = QVBoxLayout(self.main_widget)
self.titleBar = TitleBar(self)
self.setMenuBar(self.titleBar)
self.model = CheckableDirModel()
self.tree = QTreeView()
self.tree.setModel(self.model)
self.tree.resize(640, 480)
self.tree.hideColumn(1)
self.tree.hideColumn(2)
self.tree.hideColumn(3)
self.tree.setHeaderHidden(True)
self.tree.setStyleSheet(
"QTreeView { background-color: transparent; } ")
# Creating a progress bar and setting the value limits
self.progressBar = QProgressBar(self)
self.progressBar.setMaximum(100)
self.progressBar.setMinimum(0)
self.progressLine = QLabel()
self.progressLine.setStyleSheet(
"QLineEdit { background-color: transparent; } ")
# Creating a Horizontal Layout to add all the widgets
self.boxLayout = QVBoxLayout(self)
# Adding the widgets
self.boxLayout.addWidget(self.tree)
self.boxLayout.addWidget(self.progressLine)
self.boxLayout.addWidget(self.progressBar)
self.main_layout.addLayout(self.boxLayout)
self.main_widget.setLayout(self.main_layout)
self.setCentralWidget(self.main_widget)
self.build_tool_bar()
self.titleBar.minimizeWindow.connect(self.minimize_window_slot)
self.titleBar.maximizeWindow.connect(self.maximize_window_slot)
self.titleBar.closeWindow.connect(self.close_window_slot)
def build_tool_bar(self):
self.toolbar = QToolBar()
self.addToolBar(QtCore.Qt.LeftToolBarArea, self.toolbar)
load_action = QAction(QtGui.QIcon('resources/installed.png'),
'Load Files', self)
process_action = QAction(QtGui.QIcon('resources/run.png'),
'Process Files', self)
xml_load_action = QAction(QtGui.QIcon('resources/load.png'),
'XML File', self)
process_action.triggered.connect(self.process_files_clicked)
xml_load_action.triggered.connect(self.load_xml_clicked)
load_action.triggered.connect(self.load_files_clicked)
self.toolbar.addAction(load_action)
self.toolbar.addAction(process_action)
self.toolbar.addAction(xml_load_action)
self.toolbar.setIconSize(QtCore.QSize(65, 65))
self.toolbar.setFocus()
self.toolbar.setToolButtonStyle(QtCore.Qt.ToolButtonTextBesideIcon
| QtCore.Qt.LeftToolBarArea)
self.toolbar.setStyleSheet("""QToolBar {
background-color: rgba(155, 155, 144, 100); }""")
for action in self.toolbar.actions():
widget = self.toolbar.widgetForAction(action)
widget.setFixedSize(70, 70)
def mousePressEvent(self, event):
self.offset = event.pos()
def mouseMoveEvent(self, event):
x = event.globalX()
y = event.globalY()
x_w = self.offset.x()
y_w = self.offset.y()
self.move(x - x_w, y - y_w)
def paintEvent(self, event):
palette = QtGui.QPalette()
palette.setBrush(
QtGui.QPalette.Background,
QtGui.QBrush(
QtGui.QPixmap("resources/theme.png").scaled(self.size())))
self.setPalette(palette)
@QtCore.pyqtSlot()
def close_window_slot(self):
self.deleteLater()
@QtCore.pyqtSlot()
def minimize_window_slot(self):
self.showMinimized()
@QtCore.pyqtSlot()
def maximize_window_slot(self):
if self.maxNormal:
self.resize(800, 600)
self.maxNormal = False
self.titleBar.maximize.setIcon(
QtGui.QIcon('resources/maximize.png'))
frame_gm = self.frameGeometry()
center_point = QDesktopWidget().availableGeometry().center()
frame_gm.moveCenter(center_point)
self.move(frame_gm.topLeft())
else:
screen = QDesktopWidget().screenGeometry()
self.setGeometry(0, 0, screen.width(), screen.height() - 30)
self.maxNormal = True
self.titleBar.maximize.setIcon(
QtGui.QIcon('resources/unmaximize.png'))
def clear_layout(self, layout):
for i in reversed(range(layout.count())):
layout.itemAt(i).widget().setParent(None)
def settings_clicked(self):
self.clear_layout(self.main_layout)
def load_xml_clicked(self):
xml_file = str(QFileDialog.getOpenFileName(self, "Select XML File")[0])
self.progressBar.setValue(0)
self.print_line("Loading data from XML structure. Please wait ...\n")
self.progressBar.update()
xml_dir = os.path.dirname(xml_file)
self.analysisManager = AnalysisManager(self, xml_dir)
self.analysisManager.load_structure_from_xml(xml_file)
self.progressBar.setValue(100)
self.progressBar.update()
def load_files_clicked(self):
file_names = self.model.export_checked()
repo_dir = self.model.rootDir
self.analysisManager = AnalysisManager(self, repo_dir)
self.analysisManager.set_files_list(file_names)
self.print_line("Files loaded:\n")
total = len(file_names)
count = 1
self.progressBar.setValue(0)
self.progressBar.update()
for file in file_names:
self.print_line(file + "\n")
self.progressBar.setValue((count * 100) / total)
self.progressBar.update()
count += 1
self.print_line("Total number of files loaded : {}".format(total))
def print_line(self, text):
self.app.processEvents()
self.progressLine.setText(text)
def change_layout(self):
self.clear_layout(self.main_layout)
self.main_widget.setLayout(self.main_layout)
self.toolbar.setEnabled(True)
def process_files_clicked(self):
self.get_res()
'''
This method is usually called for the entire process of converting to xml, saving git diff files and counting
logical and structural dependencies for a selected repo from UI.
'''
def process_files(self):
self.print_line("Converting to XML .......")
self.analysisManager.convert_to_xml()
self.analysisManager.set_xml_files_list(self.model.rootDir + "/~Temp/")
self.print_line("Getting commits .......")
self.analysisManager.get_git_commits()
self.print_line("Processing data .......")
self.analysisManager.process_data()
'''
This method is usually called for getting git commits statistics for all the repos that I have.
Nr of git commits with less than 5, 10, 20, inf files changed. And average per repo.
'''
def get_commits_statistic(self):
for repo in self.repo_list:
print(
"______________________________________________________________________________________"
)
print(repo)
repo = repo + "\\~diffs"
sum_commits = 0
nr_commits = 0
commits_under_5 = 0
commits_under_10 = 0
commits_under_20 = 0
commits_above_20 = 0
for file in os.listdir(repo):
nr_commits += 1
file = file.replace('.txt', '')
nr_of_commits_str = file.split('FilesChanged_')[1]
commit_size = int(nr_of_commits_str)
if commit_size <= 5:
commits_under_5 += 1
elif commit_size <= 10:
commits_under_10 += 1
elif commit_size <= 20:
commits_under_20 += 1
else:
commits_above_20 += 1
sum_commits += commit_size
print(
str(commits_under_5) + "," + str(commits_under_10) + "," +
str(commits_under_20) + "," + str(commits_above_20) + "," +
str(sum_commits / nr_commits))
def get_res(self):
for repo in self.repo_list:
print(
"______________________________________________________________________________________"
)
print(repo)
self.analysisManager = AnalysisManager(self, repo)
self.analysisManager.set_xml_files_list(repo + "/~Temp/")
self.analysisManager.process_data()
from AnalysisManager import AnalysisManager
import sys
currentPath = sys.path[0]+"/"
Ana=AnalysisManager(currentPath)
Ana.CreateOutTreePaths()
Ana.SetUpSamples()
Ana.SetQueHelper("NAFSL6")
Ana.PrepareTrees()
def get_results(self):
print("***************************************" + self.repo +
"*************************************")
analysis_manager = AnalysisManager(self, self.repo, self.output_dir,
self.threshold, self.jar_file)
if self.get_files:
print("Converting to XML ...")
analysis_manager.load_files_from_repo()
analysis_manager.convert_to_xml()
print("Getting commits ...")
analysis_manager.get_git_commits()
analysis_manager.load_files_from_repo()
analysis_manager.set_xml_files_list(self.repo + "/~Temp/")
print("Processing data ...")
analysis_manager.process_data()