def __init__(self, title, dimensions):
"""
Constructor method for MainWindow
:param title: the title of the MainWindow of the GUI
:param dimensions: the dimensions of the MainWindow of the GUI
"""
QMainWindow.__init__(self)
Ui_MainWindow.__init__(self)
self.setupUi(self)
self.log_file_name = self.features_file_name = self.data = self.search_data = self.categories = self.read_success = ''
self.model = DataModel(pd.DataFrame([["No file selected."]]))
self.search_model = DataModel(pd.DataFrame([["No file selected."]]))
self.thread_pool = QThreadPool()
self.data_table.setModel(self.model)
self.search_table.setModel(self.search_model)
self.log_button.pressed.connect(self.open_log_file)
self.features_button.pressed.connect(self.open_features_file)
self.read_button.pressed.connect(self.read_button_clicked)
self.export_results_button.pressed.connect(self.export_search_results)
self.export_stats_button.triggered.connect(
self.export_stats_button_clicked)
self.search_button.pressed.connect(self.search_log)
self.data_table.setSortingEnabled(True)
self.search_table.setSortingEnabled(True)
self.initial_ui_state()
self.setGeometry(*dimensions)
self.setWindowIcon(QIcon(os.path.abspath('./../icon.png')))
self.setWindowTitle(title)
def read_button_clicked(self):
"""
Handles the click of "Read Log File" button
:return: N.A.
"""
log_file_name = self.log_file_name
features_file_name = self.features_file_name
if log_file_name != "" and features_file_name != "":
self.model = DataModel(pd.DataFrame([["Reading data..."]]))
self.search_model = DataModel(pd.DataFrame([["Reading data..."]]))
self.data_table.setModel(self.model)
self.search_table.setModel(self.search_model)
self.set_all_buttons(False)
log_ext = os.path.splitext(log_file_name)[1]
self.create_worker(self.read, ({
".csv": read_csv,
".xlsx": read_xlsx,
"": self.none_reader
}.get(log_ext), log_file_name, features_file_name),
self.show_read_data)
else:
QMessageBox.warning(
QMessageBox(), "Read Log Error",
"Please select a log file and a features (header describing log) file",
QMessageBox.Ok)
def __init__(self, *args, obj=None, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.setupUi(self)
self.control_panel = ControlPanel()
self.control_panel.updated.connect(self.send_data)
# for table
self.model = DataModel(header)
self.tableView.setModel(self.model)
self.vbar = self.scrollArea.verticalScrollBar()
# for server thread
self.thread_pool = QThreadPool()
self.server = None
self.message_queues = {}
# for plot_1
self.canvas_1 = pg.PlotWidget()
self.canvas_1.setBackground("w")
self.canvas_1.addLegend()
self.plot_1_canvas.addWidget(self.canvas_1)
self.Slider_1.setValue(99)
self.Slider_1.valueChanged.connect(self.slider_handle_1)
self.lines = []
self.y_1 = y_1
self.init_plot_1()
# checkbox for plot 1
self.plot_1_items = QVBoxLayout(self.scrollAreaWidgetContents_2)
self.checkboxes = []
for i in header:
if i in x_1:
continue
checkbox = QCheckBox(i)
if i in y_1:
checkbox.setChecked(True)
checkbox.stateChanged.connect(self.check1state)
self.plot_1_items.addWidget(checkbox)
self.checkboxes.append(checkbox)
# live video
win = pg.GraphicsLayoutWidget()
win.setBackground("w")
self.video.addWidget(win)
view = win.addViewBox()
self.imv = pg.ImageItem()
view.addItem(self.imv)
self.vcount = 0
# ACTIONS
self.actionListening.triggered.connect(self.start_server)
self.actionClose.triggered.connect(self.stop_server)
self.actionStart.triggered.connect(self.enable_recording)
self.actionStop.triggered.connect(self.disable_recording)
self.actionNew.triggered.connect(self.reset_data)
self.actionOpen.triggered.connect(self.file_open)
self.actionSave.triggered.connect(self.file_save)
self.actionTest.triggered.connect(self.send_data)
self.actionAdjust.triggered.connect(self.show_control_panel)
def __init__(self, master):
tk.Frame.__init__(self, master)
self.master = master
self.setup_interface(master)
self.data_model = DataModel()
self.note_key = RPDR_NOTE_KEYWORD
self.patient_key = RPDR_PATIENT_KEYWORD
self.checkvar = False
def queried_data():
date_query = DataModel.select(DataModel.date).distinct().where(DataModel.view_id.in_(VIEW_ID))
dates = [dq.date.strftime("%Y-%m-%d") for dq in date_query]
results = []
for date in dates:
dm = DataModel.select(fn.SUM(DataModel.sessions).alias("sessions"), DataModel.date, fn.SUM(DataModel.bounce_rate).alias("bounce_rate")).where(DataModel.date == date).where(DataModel.view_id.in_(VIEW_ID))
for d in dm:
results.append({'date': d.date, 'sessions': d.sessions, 'sessions_avg': d.sessions//len(VIEW_ID), 'bounce_rate': d.bounce_rate//len(VIEW_ID)})
return results
def show_read_data(self, data):
"""
Displays the log data read to the table in GUI
To be passed to create_worker to handle the output from read
:param data: The read log data, encapsulated into a pandas DataFrame
:return: N.A.
"""
self.data, self.categories, self.read_success = data
self.search_data = self.data.copy()
self.model = DataModel(
data=self.data) if self.read_success else DataModel(
pd.DataFrame([["No file selected."]]))
self.search_model = DataModel(
data=self.search_data) if self.read_success else DataModel(
pd.DataFrame([["No file selected."]]))
self.data_table.setModel(self.model)
self.search_table.setModel(self.search_model)
self.set_all_buttons()
self.set_tabs()
def _refresh_data(self, *args, **kwargs):
"""Called upon button Run is clicked."""
# disable the selectable objects while running
self.button_run.disabled = True
# clear any content in the output
self.widgets['main_box'].children = []
# Retrieve user input to calibrate the Model
# inputs for model: universe, period, asset
universe = self.widgets['universe_select'].value
date = self.widgets['period_select'].value
asset_class = 'Equity'
# query = '''
# let(#amt=sum(group(amt_outstanding(currency='USD'),[year(announce_date()), month(announce_date()),cntry_of_risk()]))/1000000;)
# get(#amt)
# for( filter(members('{idx}'), announce_date() >= '{date}') )
# '''.format(idx=universe, date=date)
query = '''
let(#ret1m = (product(dropna(1.0+day_to_day_total_return(start=-{date},end=0d)))-1)*100;
#ret1m_idx = value(#ret1m,['{idx}']);
#rel_ret1m = #ret1m - #ret1m_idx;
#avg_rel_ret = avg(group(#rel_ret1m,[country_full_name(),gics_sector_name()]));)
get(#avg_rel_ret)
for( members('{idx}'))
'''.format(idx=universe, date=date)
# for better display in logger
_logger.info('Loading the data Model... ({})'.format(asset_class))
self._model = DataModel(query, asset_class=asset_class)
self._model.run()
_logger.info('Refreshing data and chart...')
# calling method that fetch the arranged data and display it
self._build_matrix()
_logger.info('Done.')
# re-enable the selectable objects
self.button_run.disabled = False
def display_search_results(self, results):
"""
Displays the search results to the table in GUI
To be passed to create_worker to handle the output from search
:param results: The search results, encapsulated into a pandas DataFrame
:return: N.A.
"""
if not results.empty:
results.columns = self.categories.tolist()
self.search_model = DataModel(data=results)
search_result_message = 'Search term \'%s\' returned %d results.' % \
(self.search_textedit.text(), self.search_model.get_data().shape[0])
else:
self.search_model = DataModel(data=self.data)
search_result_message = '\'%s\' was not found in data-set.' % self.search_textedit.text(
)
self.search_table.setModel(self.search_model)
self.edit_search_result_label(search_result_message)
self.set_all_buttons()
self.set_tabs()
def search_log(self):
"""
Handles the click of "Search" button
:return: N.A.
"""
search_term = self.search_textedit.text()
if search_term == '':
self.search_model = DataModel(data=self.search_data)
self.search_table.setModel(self.search_model)
self.edit_search_result_label(
'Table Reset Success. You are now viewing the entire data set.'
)
return
self.search_table.setModel(self.search_model)
self.set_all_buttons(False)
self.set_buttons((self.export_stats_button, ))
self.set_tabs(False)
self.search_model = DataModel(pd.DataFrame([["Searching data..."]]))
self.create_worker(self.search,
(search_data_frame, self.data, search_term),
self.display_search_results)
def main():
import argparse
parser = argparse.ArgumentParser(description='RoboGenerator 0.2 - a test data generator for Robot Framework',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('input', type=str, help='input file')
parser.add_argument('--host', '-ip', type=str, default=None,
help='Pate IP address (default is None)')
parser.add_argument('--variable','-v', type=str, default=None,
help='unit type to do operation',action='append')
parser.add_argument('--output', '-o', type=str, default=None,
help='output file (default is input file with NEW prefix)')
parser.add_argument('--tests_max', '-t',
type=int, default=1,
help='''maximum number of test to generate,
only useful in random strategy''')
parser.add_argument('--percentage', '-p',
type=str, default=None,
help='''percentage of data combination to select,
only useful in random strategy''')
parser.add_argument('--nsteps', '-n',
type=int, default=None,
help='maximum number of steps allowed in generated case \n'+\
'only useful in MBT style cases,default is 50')
parser.add_argument('--strategy',
type=str, default='DynamicRandom',
help='used test generation strategy in MBT(default DynamicRandom)\n'+\
'StateCoverage = Select least covered State to test\n'+\
'ActionNameCoverage = Select least covered Action Name To cover\n'+\
'DynamicRandom = Randomly select next State but try to avoid already tested,\n'+
'if all possible transitions tested, randomly select one of them\n'+
'ShortestPath = cover all possible transition in least cost total path,need jython support\n')
parser.add_argument('--style', '-s',
type=str, default='normal',choices =['normal','atdd'],
help='diffrent case style to choose\n'+
'only useful in data driven model cases'+
'atdd for template keyword like cases')
parser.add_argument('--cachedir', '-c',
type=str, default=None,
help='''directory in server to store tested combinations''')
parser.add_argument('--generation_algorithm', '-g',
type=str, default='pairwise', choices=['dfs', 'random','smart-random','pairwise'],
help='data driven generation algorithm (default pairwise)\n'+
'dfs = depth first search \n'+
'pairwise = Generate test in pairwise algorithm\n'+
'random = generate tests randomly\n'+
'smart-random = generate test randomly and\n'+
"don't repeat case already tested in last several rounds\n")
parser.add_argument('--graph',
type=str, default=None,choices=['yes','no'],
help='''directory in server to store tested combinations''')
args = parser.parse_args()
print args.variable
if args.host:
os.environ['HOSTIP']= args.host
config = get_config_from_py_config_file(args.input)
if args.host:
config.hostip = args.host
output = args.output or config.output_filename
algorithm = args.generation_algorithm
nsteps = args.nsteps
strategy = args.strategy
case_count = args.tests_max
percentage = args.percentage
case_style = args.style
graph_option = args.graph
if not args.cachedir:
if os.sep =='\\':
config.cachedir = os.environ['APPDATA']+'\\'+'robogenerator'
else:
config.cachedir = os.environ['HOME']+'/'+'.robogenerator'
else:
config.cachedir = args.cachedir
if not os.path.exists(config.cachedir):
os.makedirs(config.cachedir)
case_instance = CaseModel(config)
data_instance = DataModel(config,algorithm)
if getattr(config,'parameters',None) and not data_instance.get_max_combinations():
return
if percentage:
#print 'calculate percentage'
case_count = len(data_instance.get_max_combinations())* int(percentage)/100
#state_instance = StateModel(config)
output_format = output.split('.')[-1]
output_filename = output.split('.')[0]
raw_output = output_filename + '.' + 'txt'
parameters = getattr(config,'parameters',None)
if args.variable :
for varible in args.variable:
key = varible.split(':')[0]
value = varible.split(':')[1].split()
parameters[key] = value
#print parameters
state_graph = getattr(config,'state_graph',None)
if state_graph:
#print 'mbt only'
state_instance_graph = [StateModel(state) for state in config.state_graph]
#print state_instance_graph
casegenerator = StateMachineCaseGenerator(case_instance,data_instance,state_instance_graph)
casegenerator.generate_case(raw_output,case_count,nsteps,strategy,case_style)
if graph_option:
config.frontier =[]
config.finished =casegenerator.get_tested_nodes()
config.deadend =[]
config.runstarts =[]
config.tested_transitions = casegenerator.get_tested_transitions()
generate_state_machine_graph(config,config.filename)
import Image
im = Image.open('%s.png'%config.filename)
im.show()
elif parameters:
#print 'data-driven only '
casegenerator = DataDrivenCaseGenerator(case_instance,data_instance,case_style)
casegenerator.generate_case(raw_output,case_count)
else:
raise Exception,'no stat_graph or no parameters defined'
print 'Case Generation Succeeded'
if output_format != 'txt':
from tidy import tidy_cli
command_string = '--inplace --format %s %s' % (output_format, raw_output) #print command_string
command_string_list = command_string.split(' ')
retcode = tidy_cli(command_string_list)
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, *args, obj=None, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.setupUi(self)
self.control_panel = ControlPanel()
self.control_panel.updated.connect(self.send_data)
# for table
self.model = DataModel(header)
self.tableView.setModel(self.model)
self.vbar = self.scrollArea.verticalScrollBar()
# for server thread
self.thread_pool = QThreadPool()
self.server = None
self.message_queues = {}
# for plot_1
self.canvas_1 = pg.PlotWidget()
self.canvas_1.setBackground("w")
self.canvas_1.addLegend()
self.plot_1_canvas.addWidget(self.canvas_1)
self.Slider_1.setValue(99)
self.Slider_1.valueChanged.connect(self.slider_handle_1)
self.lines = []
self.y_1 = y_1
self.init_plot_1()
# checkbox for plot 1
self.plot_1_items = QVBoxLayout(self.scrollAreaWidgetContents_2)
self.checkboxes = []
for i in header:
if i in x_1:
continue
checkbox = QCheckBox(i)
if i in y_1:
checkbox.setChecked(True)
checkbox.stateChanged.connect(self.check1state)
self.plot_1_items.addWidget(checkbox)
self.checkboxes.append(checkbox)
# live video
win = pg.GraphicsLayoutWidget()
win.setBackground("w")
self.video.addWidget(win)
view = win.addViewBox()
self.imv = pg.ImageItem()
view.addItem(self.imv)
self.vcount = 0
# ACTIONS
self.actionListening.triggered.connect(self.start_server)
self.actionClose.triggered.connect(self.stop_server)
self.actionStart.triggered.connect(self.enable_recording)
self.actionStop.triggered.connect(self.disable_recording)
self.actionNew.triggered.connect(self.reset_data)
self.actionOpen.triggered.connect(self.file_open)
self.actionSave.triggered.connect(self.file_save)
self.actionTest.triggered.connect(self.send_data)
self.actionAdjust.triggered.connect(self.show_control_panel)
def check1state(self, state):
self.y_1 = [c.text() for c in self.checkboxes if c.isChecked()]
self.statusbar.showMessage('plot ' + str(self.y_1))
self.init_plot_1()
def init_plot_1(self):
self.canvas_1.clear()
self.lines = []
for y in self.y_1:
# plot each line
r = random.randrange(256)
g = random.randrange(256)
b = random.randrange(256)
pen = pg.mkPen(color=(r, g, b))
line = self.canvas_1.plot(name=y, pen=pen)
self.lines.append(line)
def start_server(self):
self.server = Server(self.message_queues, port=port, sep=sep)
self.server.signals.connected.connect(self.client_connect)
self.server.signals.closed.connect(self.client_close)
self.server.signals.result.connect(self.client_data)
self.thread_pool.start(self.server)
hostname = socket.gethostname()
IPAddr = socket.gethostbyname(hostname)
msg = 'IP:' + IPAddr + ' Server is listening at port ' + str(
port) + '!'
self.statusbar.showMessage(msg)
# availability
self.actionListening.setEnabled(False)
self.actionClose.setEnabled(True)
self.actionStart.setEnabled(True)
self.actionOpen.setEnabled(False)
def stop_server(self):
self.server.kill()
self.model.disable()
self.statusbar.showMessage('Server stopped listening.')
# availability
self.actionListening.setEnabled(True)
self.actionClose.setEnabled(False)
self.actionStart.setEnabled(False)
self.actionStop.setEnabled(False)
self.actionOpen.setEnabled(True)
def client_connect(self, s):
msg = 'Connected from ' + s + '!'
self.statusbar.showMessage(msg)
self.actionAdjust.setEnabled(True)
def client_close(self, s):
msg = 'Closed from ' + s + '!'
self.statusbar.showMessage(msg)
self.actionAdjust.setEnabled(False)
self.control_panel.close()
def plot_last(self):
data = self.model.df
for i, y in enumerate(self.y_1):
data_x = data[x_1][-interv_1:].tolist()
data_y = data[y][-interv_1:].tolist()
self.lines[i].setData(x=data_x, y=data_y)
def plot_range(self, a, b):
data = self.model.df
for i, y in enumerate(self.y_1):
data_x = data[x_1][a:b].tolist()
data_y = data[y][a:b].tolist()
self.lines[i].setData(x=data_x, y=data_y)
def client_data(self, s):
data_dict = json.loads(s)
if 'sensor' in data_dict:
sensor_dict = data_dict['sensor']
self.model.append(sensor_dict)
self.tableView.scrollToBottom()
self.plot_last()
if 'parameter' in data_dict:
self.control_panel.parameters = data_dict['parameter']
data_print = json.dumps(data_dict['parameter'], indent=4)
self.plainTextEdit.setPlainText(data_print)
if 'image' in data_dict:
self.vcount += 1
# for performance
if self.vcount % 5 == 0:
img_lst = data_dict['image']
img_np = np.array(img_lst)
self.imv.setImage(img_np)
def slider_handle_1(self, val):
# val 0-99
row_count = self.model.rowCount()
if row_count <= interv_1:
a = 0
b = row_count
self.plot_range(a, b)
else:
total = row_count - interv_1 + 1
a = int(total * val / 99)
b = a + interv_1
self.plot_range(a, b)
def enable_recording(self):
self.model.enable()
self.statusbar.showMessage('Start recording.')
# availability
self.actionStart.setEnabled(False)
self.actionStop.setEnabled(True)
def disable_recording(self):
self.model.disable()
self.statusbar.showMessage('Stop recording.')
# availability
self.actionStop.setEnabled(False)
self.actionStart.setEnabled(True)
def reset_data(self):
self.model.clear()
self.init_plot_1()
# self.data_line_1.setData([], [])
def file_save(self):
path = os.path.split(os.path.realpath(__file__))[0]
expected_filename = 'data_' + datetime.now().strftime(
"%Y%m%d_%H%M%S") + '.csv'
expected_filepath = os.path.join(path, 'data')
if not os.path.exists(expected_filepath):
os.makedirs(expected_filepath)
filename = QFileDialog.getSaveFileName(
self, 'Save CSV file',
os.path.join(expected_filepath, expected_filename),
'CSV files (*.csv)')
if filename[0]:
self.model.df.to_csv(filename[0], index=False)
self.statusbar.showMessage('Save data to ' + filename[0])
def file_open(self):
path = os.path.split(os.path.realpath(__file__))[0]
expected_filepath = os.path.join(path, 'data')
if not os.path.exists(expected_filepath):
os.makedirs(expected_filepath)
filename = QFileDialog.getOpenFileName(self, 'Open CSV file',
expected_filepath,
'CSV files (*.csv)')
if filename[0]:
df = pd.read_csv(filename[0])
self.model.load(df)
self.tableView.scrollToBottom()
self.plot_last()
self.Slider_1.setValue(99)
self.statusbar.showMessage('Load data from ' + filename[0])
def send_data(self, d_dict):
# print(d_dict)
d_str = json.dumps(d_dict) + sep
for i in self.message_queues:
self.message_queues[i].put(d_str.encode())
def closeEvent(self, event):
if self.server is None:
pass
else:
self.stop_server()
del self.control_panel
self.statusbar.showMessage('Quiting...')
time.sleep(0.5)
event.accept()
def show_control_panel(self):
self.control_panel.show()
from __future__ import division
from flask import Flask, render_template, request, jsonify
from math import sqrt
from model import DataModel
app = Flask(__name__)
data_model = DataModel()
@app.route('/')
def index():
equations = data_model.get_prior_equations()
return render_template('quadratic.html', equations = equations)
@app.route('/solve', methods=['POST'])
def solve():
user_data = request.json
a, b, c = int(user_data['xsquare']), int(user_data['x']), int(user_data['constant'])
root_1, root_2 = _solve_quadratic(a, b, c)
return jsonify({'root_1': root_1, 'root_2': root_2})
def _solve_quadratic(a, b, c):
disc = b*b - 4*a*c
root_1 = (-b + sqrt(disc))/(2*a)
root_2 = (-b - sqrt(disc))/(2*a)
return root_1, root_2
if __name__ == '__main__':
app.run(host='0.0.0.0', threaded=True)
def to_dataframe():
query = DataModel.select()
return pd.DataFrame(list(query.dicts()))
class MainApplication(tk.Frame):
def __init__(self, master):
tk.Frame.__init__(self, master)
self.master = master
self.setup_interface(master)
self.data_model = DataModel()
self.note_key = RPDR_NOTE_KEYWORD
self.patient_key = RPDR_PATIENT_KEYWORD
self.checkvar = False
# Set up button click methods
def on_select_file(self):
file = filedialog.askopenfilename(title="Select File")
if file:
self.data_model.input_fname = file
self.file_text.config(
text=self.data_model.input_fname.split('/')[-1])
def on_select_output_file(self):
output_fname = filedialog.askopenfilename(title="Select Output File")
if output_fname:
# Change note-key
rpdr_checkbox = self.rpdr_checkbox.var.get()
if rpdr_checkbox == 0:
self.note_key = self.note_key_entry.get()
self.patient_key = self.patient_id_entry.get()
self.refresh_viewer(output_fname)
def on_run_regex(self):
if not self.data_model.input_fname:
# Warning
messagebox.showerror(
title="Error",
message=
"Please select an input file using the 'Select File' button.")
return
output_fname = '/'.join(self.data_model.input_fname.split('/')
[:-1]) + '/' + self.regex_label.get()
# Retrieve phrases
phrases = self.regex_text.get(1.0, 'end-1c').strip()
if phrases == self.original_regex_text or len(phrases) == 0:
messagebox.showerror(
title="Error",
message="Please input comma-separated phrases to search for. ")
return
rpdr_checkbox = self.rpdr_checkbox.var.get()
if rpdr_checkbox == 0:
note_keyword = self.note_key_entry.get()
patient_keyword = self.patient_id_entry.get()
if not note_keyword:
messagebox.showerror(
title='Error',
message='Please input the column name for notes.')
return
if not patient_keyword:
messagebox.showerror(
title='Error',
message='Please input the column name for patient IDs.')
return
try:
run_regex(self.data_model.input_fname, phrases, output_fname,
False, note_keyword, patient_keyword)
self.note_key = note_keyword
self.patient_key = patient_keyword
except:
messagebox.showerror(
title="Error",
message=
"Something went wrong, did you select an appropriately formatted file to perform the Regex on?"
)
return
else:
try:
run_regex(self.data_model.input_fname, phrases, output_fname)
self.note_key = RPDR_NOTE_KEYWORD
self.patient_key = RPDR_PATIENT_KEYWORD
except:
messagebox.showerror(
title="Error",
message=
"Something went wrong, did you select an appropriately formatted RPDR file to perform the Regex on?"
)
return
self.refresh_viewer(output_fname)
# Functions that change display
def refresh_viewer(self, output_fname):
self.data_model.output_fname = output_fname
self.data_model.output_df = pd.read_csv(self.data_model.output_fname,
index_col=0,
header=0,
dtype=object)
self.refresh_model()
def refresh_model(self):
self.data_model.current_row_index = 0
if self.checkvar:
self.data_model.display_df = self.data_model.output_df[
self.data_model.output_df['EXTRACTED_VALUE'] == '1']
else:
self.data_model.display_df = self.data_model.output_df.copy()
self.data_model.num_notes = self.data_model.display_df.shape[0]
self.regex_file_text.config(
text=self.data_model.output_fname.split('/')[-1])
self.display_output_note()
def display_output_note(self):
current_note_row = self.data_model.display_df.iloc[
self.data_model.current_row_index]
try:
current_note_text = current_note_row[self.note_key]
except:
messagebox.showerror(
title='Error',
message=
'Unable to retrieve note text. Did you select the correct key?'
)
return
try:
current_patient_id = current_note_row[self.patient_key]
except:
messagebox.showerror(
title='Error',
message=
'Unable to retrieve patient ID. Did you select the correct key?'
)
return
self.number_label.config(
text='%d of %d' %
(self.data_model.current_row_index + 1, self.data_model.num_notes))
self.patient_num_label.config(text='Patient ID: %s' %
current_patient_id)
match_indices = ast.literal_eval(current_note_row['MATCHES'])
self.pttext.config(state=tk.NORMAL)
self.pttext.delete(1.0, tk.END)
self.pttext.insert(tk.END, current_note_text)
self.pttext.config(state=tk.DISABLED)
tag_start = '1.0'
# Add highlighting
for start, end in match_indices:
pos_start = '{}+{}c'.format(tag_start, start)
pos_end = '{}+{}c'.format(tag_start, end)
self.pttext.tag_add('highlighted', pos_start, pos_end)
self.show_annotation()
def show_annotation(self):
self.ann_textbox.delete(0, tk.END)
self.ann_textbox.insert(0, self.data_model.get_annotation())
def on_save_annotation(self):
annotation = self.ann_textbox.get()
if len(annotation) > 0:
self.data_model.write_to_annotation(annotation)
def on_prev(self):
self.on_save_annotation()
if self.data_model.current_row_index > 0:
self.data_model.current_row_index -= 1
self.display_output_note()
def on_next(self):
self.on_save_annotation()
if self.data_model.current_row_index < self.data_model.num_notes:
self.data_model.current_row_index += 1
self.display_output_note()
## GUI helper methods
def clear_textbox(self, event, widget, original_text):
if widget.get(1.0, 'end-1c') == original_text:
widget.delete(1.0, 'end-1c')
def on_checkbox_click(self, event, widget):
if widget.var.get() == 0:
self.hide_regex_options()
else:
self.show_regex_options()
def on_positive_checkbox_click(self, event, widget):
if self.checkvar:
self.checkvar = False
else:
self.checkvar = True
self.refresh_model()
def hide_regex_options(self):
self.note_key_entry_label.grid_remove()
self.note_key_entry.grid_remove()
self.patient_id_label.grid_remove()
self.patient_id_entry.grid_remove()
def show_regex_options(self):
self.note_key_entry_label.grid()
self.note_key_entry.grid()
self.patient_id_label.grid()
self.patient_id_entry.grid()
def setup_interface(self, root):
# Define fonts
titlefont = font.Font(family='Open Sans', size=18, weight='bold')
boldfont = font.Font(size=16, family='Open Sans', weight='bold')
textfont = font.Font(family='Roboto', size=15)
labelfont = font.Font(family='Roboto', size=11)
smallfont = font.Font(family='Roboto', size=13)
left_bg_color = 'lightblue1'
right_bg_color = 'azure'
# Creating all main containers
left_frame = tk.Frame(root, bg=left_bg_color)
right_frame = tk.Frame(root, bg=right_bg_color)
# Laying out all main containers
root.grid_rowconfigure(0, weight=1)
root.grid_columnconfigure(0, weight=1)
root.grid_columnconfigure(1, weight=1)
root.grid_columnconfigure(2, weight=1)
left_frame.grid(column=0,
row=0,
columnspan=2,
rowspan=5,
sticky='nsew')
left_frame.grid_rowconfigure(0, weight=1)
left_frame.grid_rowconfigure(1, weight=1)
left_frame.grid_rowconfigure(2, weight=1)
left_frame.grid_rowconfigure(3, weight=1)
left_frame.grid_rowconfigure(4, weight=1)
left_frame.grid_rowconfigure(5, weight=1)
left_frame.grid_columnconfigure(0, weight=1)
right_frame.grid(column=2, row=0, columnspan=1, sticky='nsew')
right_frame.grid_rowconfigure(0, weight=1)
right_frame.grid_rowconfigure(1, weight=1)
right_frame.grid_rowconfigure(2, weight=1)
right_frame.grid_rowconfigure(3, weight=1)
right_frame.grid_rowconfigure(4, weight=1)
right_frame.grid_rowconfigure(5, weight=1)
right_frame.grid_rowconfigure(6, weight=1)
right_frame.grid_rowconfigure(7, weight=1)
right_frame.grid_rowconfigure(8, weight=1)
right_frame.grid_rowconfigure(9, weight=1)
right_frame.grid_rowconfigure(10, weight=1)
right_frame.grid_columnconfigure(0, weight=1)
# Buttons
header_frame = tk.Frame(left_frame, bg=left_bg_color)
header_frame.grid(column=0, row=0, padx=10, pady=10, sticky='nsew')
header_frame.grid_propagate(False)
header_frame.grid_rowconfigure(0, weight=1)
header_frame.grid_rowconfigure(1, weight=1)
header_frame.grid_columnconfigure(0, weight=2)
header_frame.grid_columnconfigure(1, weight=1)
header_frame.grid_columnconfigure(2, weight=1)
title_text = tk.Label(header_frame,
text='Clinical Note',
font=titlefont,
bg=left_bg_color)
title_text.grid(column=0, row=0, sticky='w')
button_frame = tk.Frame(header_frame, bg=left_bg_color)
button_frame.grid(column=0, row=1, sticky='nsew')
button_frame.grid_propagate(False)
button_frame.grid_columnconfigure(0, weight=1)
button_frame.grid_columnconfigure(1, weight=1)
button_frame.grid_columnconfigure(2, weight=1)
button_frame.grid_rowconfigure(0, weight=1)
button_frame.grid_rowconfigure(1, weight=1)
prev_button = tk.Button(button_frame,
text='Prev',
width=5,
command=self.on_prev)
prev_button.grid(column=0, row=0, sticky='sw')
self.number_label = tk.Label(button_frame,
font=smallfont,
text='',
bg=left_bg_color)
self.number_label.grid(column=1, row=0, sticky='sw')
next_button = tk.Button(button_frame,
text='Next',
width=5,
command=self.on_next)
next_button.grid(column=2, row=0, sticky='sw')
# Patient ID
self.patient_num_label = tk.Label(header_frame,
text='',
font=labelfont,
bg=left_bg_color)
self.patient_num_label.grid(column=1, row=1)
# Filter checkbox
positive_checkbox_var = tk.BooleanVar()
self.positive_checkbox = tk.Checkbutton(
header_frame,
text='Display only positive hits',
font=labelfont,
variable=positive_checkbox_var,
bg=left_bg_color,
offvalue=False,
onvalue=True)
self.positive_checkbox.var = positive_checkbox_var
self.positive_checkbox.grid(column=2, row=1, sticky='e')
self.positive_checkbox.bind(
"<Button-1>", lambda event: self.on_positive_checkbox_click(
event, self.positive_checkbox))
# Text frame
text_frame = tk.Frame(left_frame, borderwidth=1, relief="sunken")
text_frame.grid(column=0,
row=1,
rowspan=4,
padx=10,
pady=0,
sticky='nsew')
text_frame.grid_rowconfigure(0, weight=1)
text_frame.grid_columnconfigure(0, weight=1)
text_frame.grid_propagate(False)
# Patient note container (with scrolling)
self.pttext = tk.Text(text_frame,
wrap="word",
font=textfont,
background="white",
borderwidth=0,
highlightthickness=0)
scrollbar = tk.Scrollbar(text_frame)
self.pttext.config(yscrollcommand=scrollbar.set)
self.pttext.config(state=tk.DISABLED)
scrollbar.config(command=self.pttext.yview)
scrollbar.grid(column=1, row=0, sticky='nsw')
self.pttext.grid(column=0, row=0, padx=15, pady=15, sticky='nsew')
self.pttext.tag_config('highlighted', background='gold')
self.pttext.bind("<1>", lambda event: self.pttext.focus_set())
# Right button frame
right_button_frame = tk.Frame(right_frame, bg=right_bg_color)
right_button_frame.grid(column=0,
row=0,
padx=10,
pady=10,
sticky='nsew')
right_button_frame.grid_propagate(False)
right_button_frame.grid_rowconfigure(0, weight=1)
right_button_frame.grid_columnconfigure(0, weight=2)
right_button_frame.grid_columnconfigure(1, weight=1)
self.file_text = tk.Label(right_button_frame,
text='',
bg=right_bg_color,
font=labelfont,
fg='dodgerblue4')
self.file_text.grid(column=0, row=0, sticky='ne')
file_button = tk.Button(right_button_frame,
text='Select File',
width=10,
command=self.on_select_file,
bg=right_bg_color)
file_button.grid(column=1, row=0, sticky='ne')
self.regex_file_text = tk.Label(right_button_frame,
text='',
bg=right_bg_color,
font=labelfont,
fg='dodgerblue4')
self.regex_file_text.grid(column=0, row=1, sticky='ne')
regex_file_button = tk.Button(right_button_frame,
text='Select Output',
width=10,
command=self.on_select_output_file,
bg=right_bg_color)
regex_file_button.grid(column=1, row=1, sticky='ne')
# Right button container
right_regex_frame = tk.Frame(right_frame, bg=right_bg_color)
right_regex_frame.grid(column=0,
row=3,
padx=10,
pady=10,
sticky='nsew')
right_regex_frame.grid_propagate(False)
right_regex_frame.grid_columnconfigure(0, weight=1)
right_regex_frame.grid_columnconfigure(1, weight=3)
right_regex_frame.grid_rowconfigure(0, weight=1)
right_regex_frame.grid_rowconfigure(1, weight=1)
regex_title = tk.Label(right_regex_frame,
text='Regular Expression',
font=boldfont,
bg=right_bg_color)
regex_title.grid(column=0, row=0)
regex_button = tk.Button(right_regex_frame,
text='Run Regex',
width=7,
command=self.on_run_regex)
regex_button.grid(column=0, row=1, sticky='sw')
self.regex_label = tk.Entry(right_regex_frame, font=labelfont)
self.regex_label.insert(0, 'output.csv')
self.regex_label.grid(column=1, row=1, sticky='se')
# Right regex options container
right_options_frame = tk.Frame(right_frame, bg=right_bg_color)
right_options_frame.grid(column=0,
row=1,
rowspan=2,
padx=10,
sticky='nsew')
right_options_frame.grid_propagate(False)
right_options_frame.grid_columnconfigure(0, weight=1)
right_options_frame.grid_columnconfigure(1, weight=1)
right_options_frame.grid_rowconfigure(0, weight=1)
right_options_frame.grid_rowconfigure(1, weight=1)
right_options_frame.grid_rowconfigure(2, weight=1)
checkbox_var = tk.IntVar()
self.rpdr_checkbox = tk.Checkbutton(right_options_frame,
padx=10,
anchor='e',
font=labelfont,
text='RPDR format',
variable=checkbox_var,
bg=right_bg_color)
self.rpdr_checkbox.var = checkbox_var
self.rpdr_checkbox.select()
self.rpdr_checkbox.bind(
"<Button-1>",
lambda event: self.on_checkbox_click(event, self.rpdr_checkbox))
self.rpdr_checkbox.grid(column=1, row=0, sticky='e')
self.note_key_entry_label = tk.Label(right_options_frame,
text='Note column key: ',
font=labelfont,
bg=right_bg_color)
self.note_key_entry_label.grid(column=0, row=1, sticky='e')
self.note_key_entry = tk.Entry(right_options_frame, font=labelfont)
self.note_key_entry.grid(column=1, row=1, sticky='e')
self.patient_id_label = tk.Label(right_options_frame,
text='Patient ID column key: ',
font=labelfont,
bg=right_bg_color)
self.patient_id_label.grid(column=0, row=2, sticky='e')
self.patient_id_entry = tk.Entry(right_options_frame, font=labelfont)
self.patient_id_entry.grid(column=1, row=2, sticky='e')
self.hide_regex_options()
# Regex text box
text_regex_frame = tk.Frame(right_frame,
borderwidth=1,
relief="sunken")
text_regex_frame.grid_rowconfigure(0, weight=1)
text_regex_frame.grid_rowconfigure(1, weight=1)
text_regex_frame.grid_columnconfigure(0, weight=1)
text_regex_frame.grid(column=0,
row=4,
rowspan=2,
padx=10,
pady=10,
sticky='nsew')
text_regex_frame.grid_propagate(False)
self.original_regex_text = "Type comma-separated keywords here."
self.regex_text = tk.Text(text_regex_frame,
font=textfont,
borderwidth=2,
highlightthickness=0,
height=5)
self.regex_text.insert(tk.END, self.original_regex_text)
self.regex_text.grid(column=0, row=0, sticky='nsew')
self.regex_text.bind(
"<Button-1>", lambda event: self.clear_textbox(
event, self.regex_text, self.original_regex_text))
# Right textbox container
entry_frame = tk.Frame(right_frame, bg=right_bg_color)
entry_frame.grid(column=0,
row=6,
rowspan=2,
padx=10,
pady=10,
sticky='nsew')
entry_frame.grid_propagate(False)
entry_frame.grid_rowconfigure(0, weight=1)
entry_frame.grid_rowconfigure(1, weight=1)
entry_frame.grid_rowconfigure(2, weight=1)
ann_text = tk.Label(entry_frame,
text='Annotated Value',
font=boldfont,
bg=right_bg_color)
ann_text.grid(column=0, row=0, sticky='ws')
self.ann_textbox = tk.Entry(entry_frame, font=textfont)
self.ann_textbox.grid(column=0, row=1, sticky='e')
ann_button = tk.Button(entry_frame,
text='Save',
width=8,
command=self.on_save_annotation)
ann_button.grid(column=0, row=2, sticky='nw')
def to_db(output, vid):
for row in output:
data_model = DataModel(sessions=row['ga:sessions'], date=str(pd.to_datetime(row['ga:date'])), bounce_rate=row['ga:bounceRate'], view_id=vid)
data_model.save()
return
class HeatmapApp(object):
def __init__(self):
self.widgets = dict()
def show(self):
"""Construct and return the user interface.
This is the entry method of the app.
Returns : Instance of ipywidgets
"""
ui = self._build_ui()
_logger.info('Select your universe and click on Run to start.')
return ui
def _build_ui(self):
"""Build main frame of the user interface."""
# universe picker definition
self.widgets['universe_select'] = TickerAutoComplete(
description='Select universe:',
yellow_keys=['Index'],
value='BWORLD Index',
style={'description_width': 'initial'})
# period dropdown selector
self.widgets['period_select'] = ipywidgets.Dropdown(
description='Period:',
#options=['2010-01-01','2014-01-01']) # for fixed income request
options=['1m', '3m', '6m', '12m']) # for equity request
# App title definition
self.widgets['app_title'] = ipywidgets.HTML('<h1>Heatmap</h1>')
# Button definition
self.button_run = ipywidgets.Button(description='Run',
button_style='info',
icon='fa-play')
self.button_run.on_click(self._refresh_data)
# creation of the control box with the header
self.widgets['header'] = ipywidgets.VBox([
self.widgets['app_title'],
ipywidgets.HBox([
self.widgets['universe_select'], self.widgets['period_select'],
self.button_run
]),
])
# definition of the box holding the data output (heatmap)
self.widgets['main_box'] = ipywidgets.Box(
layout={'min_height': '100px'})
# Main UI Box
main_box = ipywidgets.VBox(
[self.widgets['header'], self.widgets['main_box']],
layout={'overflow_x': 'hidden'})
ui = ipywidgets.VBox([main_box, _log_widget.get_widget()])
return ui
def _refresh_data(self, *args, **kwargs):
"""Called upon button Run is clicked."""
# disable the selectable objects while running
self.button_run.disabled = True
# clear any content in the output
self.widgets['main_box'].children = []
# Retrieve user input to calibrate the Model
# inputs for model: universe, period, asset
universe = self.widgets['universe_select'].value
date = self.widgets['period_select'].value
asset_class = 'Equity'
# query = '''
# let(#amt=sum(group(amt_outstanding(currency='USD'),[year(announce_date()), month(announce_date()),cntry_of_risk()]))/1000000;)
# get(#amt)
# for( filter(members('{idx}'), announce_date() >= '{date}') )
# '''.format(idx=universe, date=date)
query = '''
let(#ret1m = (product(dropna(1.0+day_to_day_total_return(start=-{date},end=0d)))-1)*100;
#ret1m_idx = value(#ret1m,['{idx}']);
#rel_ret1m = #ret1m - #ret1m_idx;
#avg_rel_ret = avg(group(#rel_ret1m,[country_full_name(),gics_sector_name()]));)
get(#avg_rel_ret)
for( members('{idx}'))
'''.format(idx=universe, date=date)
# for better display in logger
_logger.info('Loading the data Model... ({})'.format(asset_class))
self._model = DataModel(query, asset_class=asset_class)
self._model.run()
_logger.info('Refreshing data and chart...')
# calling method that fetch the arranged data and display it
self._build_matrix()
_logger.info('Done.')
# re-enable the selectable objects
self.button_run.disabled = False
# -------- end main UI ----------
# -------- start graphic (heatmap) ----------
def on_matrix_hover(self, caller, event):
try:
x = event.get('data', {})['row']
y = event.get('data', {})['column']
c = self.data.loc[x][y]
if np.isnan(c):
self.matrix_tooltip.children = [
ipywidgets.HTML(
'<p><span style="font-weight:bold; color:dimgrey;">No data for </span><span>{}</span></p>'
.format(x))
]
else:
self.matrix_tooltip.children = [
ipywidgets.HTML(
'<p><span style="font-weight:bold; color:dimgrey;">{} </span><span>{}</span></p>'
.format(self.data.index.name, x)),
ipywidgets.HTML(
'<p><span style="font-weight:bold; color:dimgrey;">{} </span><span>{}</span></p>'
.format(self.data.columns.name, y)),
ipywidgets.HTML(
'<p><span style="font-weight:bold; color:dimgrey;">{} </span><span>{:.1f}%</span></p>'
.format('Return', c))
]
except Exception as e:
self.matrix_tooltip.children = [
ipywidgets.HTML(
'<p style="font-weight:italic; color:dimgrey;">No data fetched.</p>'
)
]
def _build_matrix(self):
# retrieve the data to be displayed
#data = self._model.build_2dim_dataset(self._model.data, x='Month', y='Year', v='Amount Out', calc_type='sum')
self.data = self._model.build_2dim_dataset(self._model.data,
x='Sector',
y='Country',
v='1m return',
calc_type='median')
# get the heatmap object
self.widgets['heatmap'] = self._build_heatmap(self.data)
# set the heatmap widgets as the child of the main-box element
self.widgets['main_box'].children = [self.widgets['heatmap']]
# Adjust the height of the box hosting the heatmap (~15px/name)
self.widgets['main_box'].layout.height = '{}px'.format(
len(self.data) * 15) if len(self.data) * 15 > 400 else '400px'
def _build_heatmap(self, df):
# create the matrix
x_sc, y_sc, col_sc = OrdinalScale(), OrdinalScale(
reverse=True), ColorScale(scheme='RdYlGr')
# define a tooltip
self.matrix_tooltip = ipywidgets.VBox(layout={
'width': '180px',
'height': '100px'
})
# building the marks for inflow and outflow
grid_map = GridHeatMap(row=df.index,
column=df.columns,
color=df,
scales={
'column': x_sc,
'row': y_sc,
'color': col_sc
},
tooltip=self.matrix_tooltip,
interactions={'hover': 'tooltip'},
stroke='transparent',
null_color='transparent',
selected_style={'opacity': 1.0},
unselected_style={'opacity': 0.4})
ax_x, ax_y = Axis(scale=x_sc, grid_lines='none', label=df.columns.name, tick_rotate=-25, tick_style={'text-anchor': 'end'}), \
Axis(scale=y_sc, grid_lines='none', label=df.index.name, orientation='vertical')
# generating the figures inflow and outflow
grid_ui = Figure(marks=[grid_map],
axes=[ax_x, ax_y],
padding_y=0.0,
title='{} distribution'.format(
self.widgets['universe_select'].value),
fig_margin={
'bottom': 90,
'left': 150,
'right': 10,
'top': 60
},
layout={
'width': '100%',
'height': '100%'
})
# set the callback for the hovering effect
grid_map.on_hover(self.on_matrix_hover)
# define the output object to get displayed
return ipywidgets.VBox([grid_ui],
layout={
'width': '99%',
'min_height': '100%',
'overflow_x': 'hidden'
})
class MainWindow(QMainWindow, Ui_MainWindow):
# Constructor method for MainWindow
def __init__(self, title, dimensions):
"""
Constructor method for MainWindow
:param title: the title of the MainWindow of the GUI
:param dimensions: the dimensions of the MainWindow of the GUI
"""
QMainWindow.__init__(self)
Ui_MainWindow.__init__(self)
self.setupUi(self)
self.log_file_name = self.features_file_name = self.data = self.search_data = self.categories = self.read_success = ''
self.model = DataModel(pd.DataFrame([["No file selected."]]))
self.search_model = DataModel(pd.DataFrame([["No file selected."]]))
self.thread_pool = QThreadPool()
self.data_table.setModel(self.model)
self.search_table.setModel(self.search_model)
self.log_button.pressed.connect(self.open_log_file)
self.features_button.pressed.connect(self.open_features_file)
self.read_button.pressed.connect(self.read_button_clicked)
self.export_results_button.pressed.connect(self.export_search_results)
self.export_stats_button.triggered.connect(
self.export_stats_button_clicked)
self.search_button.pressed.connect(self.search_log)
self.data_table.setSortingEnabled(True)
self.search_table.setSortingEnabled(True)
self.initial_ui_state()
self.setGeometry(*dimensions)
self.setWindowIcon(QIcon(os.path.abspath('./../icon.png')))
self.setWindowTitle(title)
# Setter method for model
def set_model(self, model):
"""
Setter method for model
:param model: the model to set self.model to
:return: N.A.
"""
self.model = model
# Set the buttons and tabs to initial UI state
def initial_ui_state(self):
"""
Set the buttons and tabs to initial UI state
:return: N.A.
"""
self.set_all_buttons(False)
self.set_tabs(False)
self.set_buttons(
(self.log_button, self.features_button, self.read_button))
# Set buttons to enable status
@staticmethod
def set_buttons(buttons, enable=True):
"""
Set buttons to enable status
:param buttons: The tuple of buttons to set the enable status
:param enable: Whether to enable buttons (True for enable, False for disable)
:return: N.A.
"""
for button in buttons:
button.setEnabled(enable)
# Set all buttons to enable status
def set_all_buttons(self, enable=True):
"""
Set all buttons to enable status
:param enable: Whether to enable buttons (True for enable, False for disable)
:return: N.A.
"""
self.log_button.setEnabled(enable)
self.features_button.setEnabled(enable)
self.read_button.setEnabled(enable)
self.export_results_button.setEnabled(enable)
self.search_button.setEnabled(enable)
self.export_stats_button.setEnabled(enable)
# Set non-current tabs to enable status
def set_tabs(self, enable=True):
"""
Set non-current tabs to enable status
:param enable: Whether to enable tabs (True for enable, False for disable)
:return: N.A.
"""
non_current_tabs = list(
filter(lambda x: x != self.tabWidget.currentIndex(),
range(self.tabWidget.count())))
for tab in non_current_tabs:
self.tabWidget.setTabEnabled(tab, enable)
# Edits the search label with message
def edit_search_result_label(self, message):
"""
Edits the search label with message
:param message: The message to write to the search label
:return: N.A.
"""
current_label_text = self.search_result_label.text().split(': ')
current_label_text[-1] = message
self.search_result_label.setText(": ".join(current_label_text))
# Opens File Dialog to ask for the file name to read
@pyqtSlot()
def open_file_name_dialog(self, label, file_type):
"""
Opens File Dialog to ask for the file name to read
:param label: The label to update with the name of the file about to be read
:param file_type: The text to display on the title of the File Dialog
:return: file_name The name of the file to read, as requested by the user, or ''
"""
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
file_name, _ = QFileDialog.getOpenFileName(QFileDialog(),
"Select %s File" %
file_type,
"",
"CSV Files (*.csv)",
options=options)
if file_name:
label.setText(os.path.basename(file_name))
label.adjustSize()
return file_name or ''
# Opens File Dialog to ask for the file name to save to
@pyqtSlot()
def save_file_dialog(self, file_type, file_ext):
"""
Opens File Dialog to ask for the file name to save to
:param file_type: Text to display on the title of the File Dialog
:param file_ext: The file extension to request for
:return: file_name The name of the file to save to, as requested by the user, or ''
"""
file_exts = {
'csv': 'CSV Files (*.csv)',
'xlsx': 'Excel Files (*.xlsx)'
}
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
file_name, _ = QFileDialog.getSaveFileName(
QFileDialog(),
"Select %s File" % file_type,
"",
file_exts.get(file_ext, 'All Files (*.csv *.xlsx)'),
options=options)
return file_name or ''
# Handles the click of "Select Log File" button
@pyqtSlot()
def open_log_file(self):
"""
Handles the click of "Select Log File" button
:return: N.A.
"""
self.log_file_name = self.open_file_name_dialog(self.log_label, "Log")
# Handles the click of "Select Features File" button
@pyqtSlot()
def open_features_file(self):
"""
Handles the click of "Select Features File" button
:return: N.A.
"""
self.features_file_name = self.open_file_name_dialog(
self.features_label, "Features")
@staticmethod
def none_reader(log, features):
return None, None, False
# Read Files
# Read Files - Thread input func - read log function for create_worker
@staticmethod
def read(reader, log, features):
"""
Function to be passed to create_worker for read log function
:param reader: The reader function to read the log and feature file
:param log: The name of the log file to read
:param features: The name of the features file to read
:return: output of reader --> To be accessed by show_read_data
"""
return reader(log, features)
# Read Files - Thread output func - display log data read
def show_read_data(self, data):
"""
Displays the log data read to the table in GUI
To be passed to create_worker to handle the output from read
:param data: The read log data, encapsulated into a pandas DataFrame
:return: N.A.
"""
self.data, self.categories, self.read_success = data
self.search_data = self.data.copy()
self.model = DataModel(
data=self.data) if self.read_success else DataModel(
pd.DataFrame([["No file selected."]]))
self.search_model = DataModel(
data=self.search_data) if self.read_success else DataModel(
pd.DataFrame([["No file selected."]]))
self.data_table.setModel(self.model)
self.search_table.setModel(self.search_model)
self.set_all_buttons()
self.set_tabs()
# Read Files - Handles the click of "Read Log File" button
@pyqtSlot()
def read_button_clicked(self):
"""
Handles the click of "Read Log File" button
:return: N.A.
"""
log_file_name = self.log_file_name
features_file_name = self.features_file_name
if log_file_name != "" and features_file_name != "":
self.model = DataModel(pd.DataFrame([["Reading data..."]]))
self.search_model = DataModel(pd.DataFrame([["Reading data..."]]))
self.data_table.setModel(self.model)
self.search_table.setModel(self.search_model)
self.set_all_buttons(False)
log_ext = os.path.splitext(log_file_name)[1]
self.create_worker(self.read, ({
".csv": read_csv,
".xlsx": read_xlsx,
"": self.none_reader
}.get(log_ext), log_file_name, features_file_name),
self.show_read_data)
else:
QMessageBox.warning(
QMessageBox(), "Read Log Error",
"Please select a log file and a features (header describing log) file",
QMessageBox.Ok)
# Export Statistics
# Export Statistics - Thread input func - exporting statistics function for create_worker
@staticmethod
def export_stats(stats_func, data, stats_file_name):
"""
Function to be passed to create_worker for exporting statistics function
:param stats_func: The export statistics function
:param data: The statistics data to be exported
:param stats_file_name: Name of the file to export to
:return: output of stats_func --> To be accessed by display_export_stats_status
"""
return stats_func(data, stats_file_name)
# Export Statistics - Thread output func - displays the success status of export statistics operation
def display_export_stats_status(self, write_success):
"""
Displays the success status of export statistics operation
To be passed to create_worker to handle the output from export_stats
:param write_success: Boolean value indicating status of exporting of statistics.
(True for success and False for otherwise)
:return: N.A.
"""
self.set_all_buttons()
if write_success:
QMessageBox.information(QMessageBox(), "Export Statistics Success",
"Statistics successfully exported",
QMessageBox.Ok)
else:
QMessageBox.warning(
QMessageBox(), "Export Statistics Failure",
"Statistics could not be exported successfully. Try again later."
)
# Export Statistics - Handles the click of "Export Statistics" button under "Options"
@pyqtSlot()
def export_stats_button_clicked(self):
"""
Handles the click of "Export Statistics" button
:return: N.A.
"""
stats_file_name = self.save_file_dialog('Export Statistics', 'xlsx')
if stats_file_name == '':
QMessageBox.warning(
QMessageBox(), "Export Statistics Failure",
"Please select an Excel File if you wish to export statistics related to your data set."
)
return
self.set_all_buttons()
self.set_buttons(
(self.log_button, self.features_button, self.read_button), False)
self.create_worker(self.export_stats,
(data_to_information, self.data,
os.path.splitext(stats_file_name)[0] + '.xlsx'),
self.display_export_stats_status)
# Search Results
# Search Results - Thread input func - search function for create_worker
@staticmethod
def search(search_func, data_set, search_term):
"""
Function to be passed to create_worker for search function
:param search_func: The search function
:param data_set: The data_set to search from (See search_data_frame in filter.py)
:param search_term: The search term to be used in the search (See search_data_frame in filter.py)
:return: The search results, encapsulated into a pandas DataFrame --> To be accessed by display_search_results
"""
return pd.DataFrame(search_func(data_set, search_term))
# Search Results - Thread output func - display search results
def display_search_results(self, results):
"""
Displays the search results to the table in GUI
To be passed to create_worker to handle the output from search
:param results: The search results, encapsulated into a pandas DataFrame
:return: N.A.
"""
if not results.empty:
results.columns = self.categories.tolist()
self.search_model = DataModel(data=results)
search_result_message = 'Search term \'%s\' returned %d results.' % \
(self.search_textedit.text(), self.search_model.get_data().shape[0])
else:
self.search_model = DataModel(data=self.data)
search_result_message = '\'%s\' was not found in data-set.' % self.search_textedit.text(
)
self.search_table.setModel(self.search_model)
self.edit_search_result_label(search_result_message)
self.set_all_buttons()
self.set_tabs()
# Search Results - Handles the click of "Search" button
@pyqtSlot()
def search_log(self):
"""
Handles the click of "Search" button
:return: N.A.
"""
search_term = self.search_textedit.text()
if search_term == '':
self.search_model = DataModel(data=self.search_data)
self.search_table.setModel(self.search_model)
self.edit_search_result_label(
'Table Reset Success. You are now viewing the entire data set.'
)
return
self.search_table.setModel(self.search_model)
self.set_all_buttons(False)
self.set_buttons((self.export_stats_button, ))
self.set_tabs(False)
self.search_model = DataModel(pd.DataFrame([["Searching data..."]]))
self.create_worker(self.search,
(search_data_frame, self.data, search_term),
self.display_search_results)
# Export Search Results
# Export Search Results - Thread input func - exporting search results function for create_worker
@staticmethod
def start_export_search_results(export_search_results_func, search_results,
file_name):
"""
Function to be passed to create_worker for exporting search results function
:param export_search_results_func: The export search results function
:param search_results: The search results to be passed to export_search_results_func for export
:param file_name: Name of the file to export to
:return: output of export_search_results_func --> To be accessed by display_export_search_results_status
"""
return export_search_results_func(search_results, file_name)
# Export Search Results - Thread output func - displays the success status of export search results operation
def display_export_search_results_status(self, write_success):
"""
Displays the success status of export search results operation
To be passed to create_worker to handle the output from start_export_search_results
:param write_success: Boolean value indicating status of exporting of search results.
(True for success and False for otherwise)
:return: N.A.
"""
export_message = 'Export Success' if write_success else 'Export Failed'
self.edit_search_result_label(export_message)
self.set_all_buttons()
self.set_tabs()
# Export Search Results - Handles the click of "Export Results" button in "Search Data" tab
@pyqtSlot()
def export_search_results(self):
"""
Handles the click of "Export Results" button
:return: N.A.
"""
export_file_name = self.save_file_dialog('Export Results', 'csv')
if export_file_name == '':
QMessageBox.warning(
QMessageBox(), "Export Statistics Failure",
"Please select an Excel File if you wish to export statistics related to your data set."
)
return
self.set_all_buttons(False)
self.set_buttons((self.export_stats_button, ))
self.set_tabs(False)
self.create_worker(self.start_export_search_results,
(write_csv, self.search_model.get_data(),
os.path.splitext(export_file_name)[0]),
self.display_export_search_results_status)
# Generic thread related functions
# Create a new thread to execute a task (function) separately from the main GUI thread
def create_worker(self, func, arguments, result_func):
"""
Create a new thread to execute a task (function) separately from the main GUI thread
This is to avoid the "Not Responding" message that is usually caused by a long
running process clogging the main GUI process.
:param func: The name of the function to execute in the new thread
:param arguments: The *tuple* of arguments to be passed on to the function
:param result_func: The name of the function to handle the data returned from func
:return: N.A.
"""
worker = Worker(func, arguments)
worker.signals.result.connect(result_func)
worker.signals.finished.connect(self.thread_complete)
worker.setAutoDelete(True)
self.thread_pool.start(worker)
# Clears up resources once a thread has finished its task
def thread_complete(self):
"""
Clears up resources once a thread has finished its task
:return: N.A.
"""
self.thread_pool.clear()
# Handles the Close Event
def closeEvent(self, event):
"""
Handles the Close Event
:param event:
:return: N.A.
"""
exit_confirm = QMessageBox.question(
QMessageBox(), "Confirm Exit?",
"Are you sure you want to quit the program?",
QMessageBox.Yes | QMessageBox.No) # , defaultButton=QMessageBox.No
if exit_confirm == QMessageBox.Yes:
event.accept()
else:
event.ignore()
date_query = DataModel.select(DataModel.date).distinct().where(DataModel.view_id.in_(VIEW_ID))
dates = [dq.date.strftime("%Y-%m-%d") for dq in date_query]
results = []
for date in dates:
dm = DataModel.select(fn.SUM(DataModel.sessions).alias("sessions"), DataModel.date, fn.SUM(DataModel.bounce_rate).alias("bounce_rate")).where(DataModel.date == date).where(DataModel.view_id.in_(VIEW_ID))
for d in dm:
results.append({'date': d.date, 'sessions': d.sessions, 'sessions_avg': d.sessions//len(VIEW_ID), 'bounce_rate': d.bounce_rate//len(VIEW_ID)})
return results
def to_dataframe():
query = DataModel.select()
return pd.DataFrame(list(query.dicts()))
def to_dataframe_v2():
return pd.DataFrame(queried_data())
if __name__ == '__main__':
for vid in VIEW_ID:
worker = Thread(target=main, args=(vid, ))
worker.start()
worker.join()
df = to_dataframe_v2()
df.to_csv('{}/sheets/{}.csv'.format(BASE_DIR, 'sammy', vid), index=False)
time.sleep(0.5)
DataModel.drop_table()
def __init__(self, parent):
wx.Panel.__init__(self, parent, -1)
# Create a dataview control
self.dvc = dv.DataViewCtrl(self,
style=wx.BORDER_THEME
| dv.DV_ROW_LINES
| dv.DV_VERT_RULES
| dv.DV_MULTIPLE)
self.model = DataModel([])
self.dvc.AssociateModel(self.model)
self.dvc.AppendTextColumn("path", 1, width=170)
self.dvc.AppendTextColumn("title",
2,
width=300,
mode=dv.DATAVIEW_CELL_EDITABLE)
self.dvc.AppendProgressColumn("progress", 3, width=130)
self.dvc.AppendTextColumn("status",
4,
width=300,
mode=dv.DATAVIEW_CELL_EDITABLE)
# set the Sizer property (same as SetSizer)
self.Sizer = wx.BoxSizer(wx.VERTICAL)
self.Sizer.Add(self.dvc, 1, wx.EXPAND)
b2 = wx.Button(self, label="Add files")
self.Bind(wx.EVT_BUTTON, self.OnAddRow, b2)
self.button_add = b2
b3 = wx.Button(self, label="Delete selected")
b3.Enable(False)
self.Bind(wx.EVT_BUTTON, self.OnDeleteRows, b3)
self.button_delete = b3
b5 = wx.Button(self, label="Start upload")
b5.Enable(False)
self.Bind(wx.EVT_BUTTON, self.start_upload, b5)
self.button_upload_start = b5
b6 = wx.Button(self, label="Stop upload")
b6.Enable(False)
self.Bind(wx.EVT_BUTTON, self.stop_upload, b6)
self.button_upload_stop = b6
self.in_progress = False
self.files_in_progress = 0
btnbox = wx.BoxSizer(wx.HORIZONTAL)
btnbox.Add(b2, 0, wx.LEFT | wx.RIGHT, 5)
btnbox.Add(b3, 0, wx.LEFT | wx.RIGHT, 5)
btnbox.Add(b5, 0, wx.LEFT | wx.RIGHT, 5)
btnbox.Add(b6, 0, wx.LEFT | wx.RIGHT, 5)
self.Sizer.Add(btnbox, 0, wx.TOP | wx.BOTTOM, 5)
# Bind some events so we can see what the DVC sends us
self.Bind(dv.EVT_DATAVIEW_ITEM_START_EDITING, self.on_before_edit,
self.dvc)
# self.Bind(dv.EVT_DATAVIEW_ITEM_EDITING_DONE, self.OnEditingDone, self.dvc)
# self.Bind(dv.EVT_DATAVIEW_ITEM_VALUE_CHANGED, self.OnValueChanged, self.dvc)
self.Bind(dv.EVT_DATAVIEW_ITEM_ACTIVATED, self.RightClick, self.dvc)
parent.Bind(wx.EVT_CLOSE, self.OnClose)
self.parent = parent
# worker stuff
self.enclosure_queue = Queue()
self.worker = None
# drop
file_drop_target = MyFileDropTarget(self.on_drop)
self.SetDropTarget(file_drop_target)
auth = OAuth(self)
self.oauth = auth
if not auth.ShowModal():
self.parent.Close()
class App(wx.Panel):
def __init__(self, parent):
wx.Panel.__init__(self, parent, -1)
# Create a dataview control
self.dvc = dv.DataViewCtrl(self,
style=wx.BORDER_THEME
| dv.DV_ROW_LINES
| dv.DV_VERT_RULES
| dv.DV_MULTIPLE)
self.model = DataModel([])
self.dvc.AssociateModel(self.model)
self.dvc.AppendTextColumn("path", 1, width=170)
self.dvc.AppendTextColumn("title",
2,
width=300,
mode=dv.DATAVIEW_CELL_EDITABLE)
self.dvc.AppendProgressColumn("progress", 3, width=130)
self.dvc.AppendTextColumn("status",
4,
width=300,
mode=dv.DATAVIEW_CELL_EDITABLE)
# set the Sizer property (same as SetSizer)
self.Sizer = wx.BoxSizer(wx.VERTICAL)
self.Sizer.Add(self.dvc, 1, wx.EXPAND)
b2 = wx.Button(self, label="Add files")
self.Bind(wx.EVT_BUTTON, self.OnAddRow, b2)
self.button_add = b2
b3 = wx.Button(self, label="Delete selected")
b3.Enable(False)
self.Bind(wx.EVT_BUTTON, self.OnDeleteRows, b3)
self.button_delete = b3
b5 = wx.Button(self, label="Start upload")
b5.Enable(False)
self.Bind(wx.EVT_BUTTON, self.start_upload, b5)
self.button_upload_start = b5
b6 = wx.Button(self, label="Stop upload")
b6.Enable(False)
self.Bind(wx.EVT_BUTTON, self.stop_upload, b6)
self.button_upload_stop = b6
self.in_progress = False
self.files_in_progress = 0
btnbox = wx.BoxSizer(wx.HORIZONTAL)
btnbox.Add(b2, 0, wx.LEFT | wx.RIGHT, 5)
btnbox.Add(b3, 0, wx.LEFT | wx.RIGHT, 5)
btnbox.Add(b5, 0, wx.LEFT | wx.RIGHT, 5)
btnbox.Add(b6, 0, wx.LEFT | wx.RIGHT, 5)
self.Sizer.Add(btnbox, 0, wx.TOP | wx.BOTTOM, 5)
# Bind some events so we can see what the DVC sends us
self.Bind(dv.EVT_DATAVIEW_ITEM_START_EDITING, self.on_before_edit,
self.dvc)
# self.Bind(dv.EVT_DATAVIEW_ITEM_EDITING_DONE, self.OnEditingDone, self.dvc)
# self.Bind(dv.EVT_DATAVIEW_ITEM_VALUE_CHANGED, self.OnValueChanged, self.dvc)
self.Bind(dv.EVT_DATAVIEW_ITEM_ACTIVATED, self.RightClick, self.dvc)
parent.Bind(wx.EVT_CLOSE, self.OnClose)
self.parent = parent
# worker stuff
self.enclosure_queue = Queue()
self.worker = None
# drop
file_drop_target = MyFileDropTarget(self.on_drop)
self.SetDropTarget(file_drop_target)
auth = OAuth(self)
self.oauth = auth
if not auth.ShowModal():
self.parent.Close()
def open(self, filepath):
if sys.platform.startswith('darwin'):
subprocess.call(('open', filepath))
elif os.name == 'nt': # For Windows
os.startfile(filepath)
elif os.name == 'posix': # For Linux, Mac, etc.
subprocess.call(('xdg-open', filepath))
def RightClick(self, evt):
item = self.model.GetRow(evt.GetItem())
path = ''
if evt.GetColumn() == 1:
path = self.model.GetValueByRow(item, 1)
elif evt.GetColumn() == 4:
path = self.model.GetValueByRow(item, 4)
if path:
self.open(path)
def OnDeleteRows(self, evt):
items = self.dvc.GetSelections()
rows = [self.model.GetRow(item) for item in items]
self.model.delete_rows(rows)
if len(self.model.data) == 0:
self.button_upload_start.Enable(False)
self.button_delete.Enable(False)
def lock_ui(self, lock=True):
self.in_progress = lock
self.button_add.Enable(not lock)
self.button_delete.Enable(not lock)
self.button_upload_start.Enable(not lock)
self.button_upload_stop.Enable(lock)
def start_upload(self, evt):
if len(self.model.data) == 0:
return
self.files_in_progress = len(self.model.data)
self.lock_ui(True)
self.enclosure_queue.empty()
for i in self.model.data:
self.enclosure_queue.put(i)
self.worker = Worker(self.enclosure_queue, self, self.oauth.result)
self.worker.start()
def on_upload_end(self):
self.files_in_progress = self.files_in_progress - 1
if self.files_in_progress == 0:
self.lock_ui(False)
NotificationMessage("Youtube uploader",
message="We're done!",
parent=None,
flags=wx.ICON_INFORMATION).Show()
def stop_upload(self, evt):
self.worker.stop()
self.lock_ui(False)
def open_files(self):
# Create open file dialog
openFileDialog = wx.FileDialog(
self, "Open", "", "",
"Video files (*.mts;*.mp4;*.avi)|*.mts;*.mp4;*.avi",
wx.FD_OPEN | wx.FD_FILE_MUST_EXIST | wx.FD_MULTIPLE)
openFileDialog.ShowModal()
return openFileDialog.GetPaths()
def add_path(self, path):
if self.model.file_exists(path):
return
name, extension = os.path.splitext(os.path.basename(path))
extension = extension.lower()[1:]
if extension not in ['avi', 'mts', 'mp4']:
return
row = Row(path, name)
self.model.AddRow(row)
def on_drop(self, file):
self.add_path(file)
def OnAddRow(self, evt):
for i in self.open_files():
name = os.path.splitext(os.path.basename(i))[0]
row = Row(i, name, 0, '')
self.model.AddRow(row)
self.button_upload_start.Enable(True)
self.button_delete.Enable(True)
def on_before_edit(self, evt):
# todo!
if self.in_progress:
evt.SetEditCanceled(True)
def OnClose(self, e):
self.parent.Destroy()
from autocompleter import get_completions
from model import DataModel
print('Loading the data...')
data = DataModel()
def get_input_from_user(output_str):
val = input(output_str)
return val
def print_completions(completions):
for i, auto_complete_obj in enumerate(completions):
str_ = auto_complete_obj.completed_sentence
path_ = auto_complete_obj.source_text
offset = auto_complete_obj.offset
score = auto_complete_obj.score
print("{}. {} ({}, offset: {}), score: {}".format(i + 1, str_, path_, offset, score))
def get_most_relevant(completions):
sorted_by_score = sorted(completions, key=lambda tup: tup[2], reverse=True)
return sorted_by_score[:5]
def get_best_k_completions(prefix: str):
storage_tree = data.get_completions_tree()
data_dict = data.get_base_data()
completions = get_completions(prefix, storage_tree)