def reset_filter():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return redirect(url_for('home', expired='1'))
if request.method == 'POST':
# predictions_parameters verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
rollback_filter(session['backup'])
source_df = pandas.read_pickle(session['backup']['source_df'])
session['cache'][session['session_id']] = get_analysis_data(
source_df, session['config.ini']['DEFECT_ATTRIBUTES'],
session['config.ini']['MACHINE_LEARNING']['asignee_reporter'] +
session['config.ini']['MACHINE_LEARNING']['weekdays_stop_words'] +
session['config.ini']['MACHINE_LEARNING']['months_stop_words'])
session['cache'][session['session_id']][
'categorical_fields'] = get_categorical_fields(
source_df,
defect_attributes=session['config.ini']['DEFECT_ATTRIBUTES'],
referring_to_fields=session['config.ini']['DEFECT_ATTRIBUTES']
['referring_to'])
session['cache'][session['session_id']]['statistical_info'] = dict(
session['cache'][session['session_id']]['statistical_info'],
**get_records_count(filtered_df=source_df, source_df=source_df))
return jsonify({
'username':
session['username'],
'message':
'filter dropped',
'statistical_info':
session['cache'][session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]['categorical_fields'],
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'plot':
session['cache'][session['session_id']]['charts'],
'markup':
str(session['markup']),
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'freqTop':
session['cache'][session['session_id']]['frequently_terms'],
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
})
def multiple_description_mode():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return redirect(url_for('home', expired='1'), code=302)
if request.method == 'POST':
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
error_message = predictions_parameters_verification['err_message']
return render_template(
'multiplePage.html',
json=json.dumps({
'username':
session['username'],
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'message':
error_message,
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status
}))
def significant_terms():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return jsonify(dict(redirect=url_for('home', expired='1')))
if request.method == 'POST':
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
referr_to = request.form['ReferringTo']
significant_terms = {
referr_to:
calculate_significant_terms(
pandas.read_pickle(session['backup']['filtered_df']),
referr_to,
sw=text.ENGLISH_STOP_WORDS.difference(
('see', 'system', 'call')).union(
session['config.ini']['MACHINE_LEARNING']
['asignee_reporter'], session['config.ini']
['MACHINE_LEARNING']['weekdays_stop_words'],
session['config.ini']['MACHINE_LEARNING']
['months_stop_words']))
}
return jsonify({
'significant_terms': significant_terms[referr_to],
'fields': session['config.ini']['DEFECT_ATTRIBUTES'],
'single_mod': signle_mode_status,
'multiple_mod': multiple_mode_status
})
def single_description_mode():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return redirect(url_for('home', expired='1'), code=302)
try:
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
error_message = predictions_parameters_verification['err_message']
if (multiple_mode_status and signle_mode_status):
fields = {}
fields['Priority'] = session['predictions_parameters.ini'][
'predictions_parameters']['priority_classes']
fields['Resolution'] = unpack_dictionary_val_to_list(
session['config.ini']['DEFECT_ATTRIBUTES']['resolution'])
fields['Testing_areas'] = [
el for el in session['predictions_parameters.ini']
['predictions_parameters']['areas_of_testing_classes']
if el != 'Other'
]
return render_template('resultSinglePage.html',
json=json.dumps({
'username':
session['username'],
'categoric':
fields,
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status
}))
else:
return render_template('resultSinglePage.html',
json=json.dumps({
'username':
session['username'],
'error':
error_message
}))
except Exception as e:
return render_template('resultSinglePage.html',
json=json.dumps({
'username': session['username'],
'error': str(e)
}))
def save_file():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return jsonify(dict(redirect=url_for('home', expired='1')))
if request.method == 'POST':
try:
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification[
'single_mode']
file_name = request.form['fileName']
predictions_table = pandas.read_pickle(
session['backup']['predictions_table'])
save_predictions(
predictions_table,
os.path.join(session['backup']['backup_folder'], file_name))
session['temp_files'].append(
os.path.join(session['backup']['backup_folder'], file_name))
return send_from_directory(session['backup']['backup_folder'],
file_name,
as_attachment=True)
except Exception as e:
return render_template(
'multiplePage.html',
json=json.dumps({
'username':
session['username'],
'message':
str(e),
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
def get_defect_submission_chart():
"""Calculates distribution chart coordinates based on the selected period.
Returns:
stringified json-objects containing calculated coordinates.
"""
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return redirect(url_for('home', expired='1'))
if request.method == 'POST':
period = parse_period(
request.args.get('period', default='W-SUN', type=str))
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
return jsonify({
'username':
session['username'],
'message':
'chart builded',
'statistical_info':
session['cache'][session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]['categorical_fields'],
'plot':
calculate_coordinates(dataframe=pandas.read_pickle(
session['backup']['filtered_df']),
y='Dynamic',
period=period),
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'markup':
str(session['markup']),
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status
})
def upload_file():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
file = request.files['file']
return redirect(url_for('home', expired='1'), code=302)
if request.method == 'POST':
try:
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification[
'single_mode']
file = request.files.getlist("file")
df = convert_to_dataframe(file)
session['backup']['predictions_table'] = '{0}/{1}_{2}.pkl'.format(
session['backup']['backup_folder'],
time.strftime('%y%m%d%H%M%S'), 'predictions_table')
pool = multiprocessing.Pool()
predictions_table = [
pool.apply_async(
get_predictions_table,
args=(
df, index, session['predictions_parameters.ini']
['predictions_parameters']['areas_of_testing_classes'],
unpack_dictionary_val_to_list(
session['config.ini']['DEFECT_ATTRIBUTES']
['resolution']),
session['predictions_parameters.ini']
['predictions_parameters']))
for index in list(range(len(df.index)))
]
predictions_table = dict([el.get() for el in predictions_table])
charts = {}
charts['area_of_testing'] = {
_class: 0
for _class in session['predictions_parameters.ini']
['predictions_parameters']['areas_of_testing_classes']
}
charts['area_of_testing'].update(
calculate_occurrence_percentage(
predictions_table, 'area_of_testing')['area_of_testing'])
charts['ttr'] = {
_class: 0
for _class in session['predictions_parameters.ini']
['predictions_parameters']['ttr_classes']
}
charts['ttr'].update(
calculate_occurrence_percentage(predictions_table,
'ttr')['ttr'])
charts['resolution_pie'] = calculate_occurrence_percentage(
predictions_table,
unpack_dictionary_val_to_list(
session['config.ini']['DEFECT_ATTRIBUTES']['resolution']))
for key in predictions_table:
if isinstance(predictions_table[key]['area_of_testing'], list):
predictions_table[key]['area_of_testing'] = ','.join(
predictions_table[key]['area_of_testing'])
pandas.DataFrame.from_dict(
predictions_table, orient='index').to_pickle(
session['backup']['predictions_table'])
df.to_pickle('{0}/{1}_{2}.pkl'.format(
session['backup']['backup_folder'],
session['frame_store']['frame_multiple'],
session['session_id']))
return render_template(
'multiplePage.html',
json=json.dumps({
'username':
session['username'],
'table':
predictions_table,
'message':
'file uploaded successfully',
'plot':
charts,
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status
}))
except Exception as e:
return render_template('multiplePage.html',
json=json.dumps({
'username':
session['username'],
'message':
str(e),
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status
}))
def train():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return jsonify(dict(redirect=url_for('home', expired='1')))
if request.method == 'POST':
try:
source_df = pandas.read_pickle(session['backup']['filtered_df'])
filtered_df = pandas.read_pickle(session['backup']['filtered_df'])
train_model(
str(Path(__file__).parents[2]) + '/models/selected/',
session['backup']['filtered_df'],
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'],
session['config.ini']['DEFECT_ATTRIBUTES']['resolution'])
load_config_to_session(
str(Path(__file__).parents[2]) + '/models/selected/' +
'predictions_parameters.ini')
create_top_terms_file(
filtered_df,
session['config.ini']['DEFECT_ATTRIBUTES']['resolution'],
session['predictions_parameters.ini']['predictions_parameters']
['priority_classes'], session['predictions_parameters.ini']
['predictions_parameters']['areas_of_testing_classes'])
session['cache'][session['session_id']] = get_analysis_data(
filtered_df, session['config.ini']['DEFECT_ATTRIBUTES'],
session['config.ini']['MACHINE_LEARNING']['asignee_reporter'] +
session['config.ini']['MACHINE_LEARNING']
['weekdays_stop_words'] +
session['config.ini']['MACHINE_LEARNING']['months_stop_words'])
session['cache'][session['session_id']][
'categorical_fields'] = get_categorical_fields(
source_df,
defect_attributes=session['config.ini']
['DEFECT_ATTRIBUTES'],
referring_to_fields=session['config.ini']
['DEFECT_ATTRIBUTES']['referring_to'])
session['cache'][session['session_id']]['statistical_info'] = dict(
session['cache'][session['session_id']]['statistical_info'],
**get_records_count(filtered_df=filtered_df,
source_df=source_df))
path_archived_models = str(
Path(__file__).parents[2]) + r'/models/archived/'
if not os.path.exists(path_archived_models):
create_folder(path_archived_models)
if session['config.ini']['MACHINE_LEARNING'][
'selected_set_models']:
session['config.ini']['MACHINE_LEARNING']['set_models'][
session['config.ini']['MACHINE_LEARNING']
['selected_set_models']] = 'false'
session['config.ini']['MACHINE_LEARNING'][
'selected_set_models'] = 'no_name'
from main.config_processor import Configuration
creator = Configuration('{}/config.ini'.format(
str(Path(__file__).parents[2])))
creator.set_option('MACHINE_LEARNING', 'selected_set_models',
"no_name")
session['config.ini']['MACHINE_LEARNING']['set_models'][
'no_name'] = 'true'
creator.set_option(
'MACHINE_LEARNING', 'set_models',
str(session['config.ini']['MACHINE_LEARNING']['set_models']))
load_config_to_session(
str(Path(__file__).parents[2]) + 'config.ini')
creator = Configuration('{}/predictions_parameters.ini'.format(
str(Path(__file__).parents[2]) + '/models/selected/'))
creator.set_option(
'predictions_parameters', 'mark_up_attributes',
str(session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes']))
with ZipFile(
os.path.join(
str(Path(__file__).parents[2]) + '/models/archived/',
"no_name.zip"), 'w') as zf:
for dirpath, dirnames, filenames in os.walk(
str(Path(__file__).parents[2]) + '/models/selected/'):
for filename in filenames:
zf.write(os.path.join(
str(Path(__file__).parents[2]) +
'/models/selected/', filename),
arcname=filename)
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification[
'single_mode']
return jsonify({
'username':
session['username'],
'message':
'Model has been successfully trained.',
'statistical_info':
session['cache'][session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]['categorical_fields'],
'plot':
session['cache'][session['session_id']]['charts'],
'markup':
False,
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'placeholder':
session['cache'][session['session_id']]['placeholder'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
})
except Exception as e:
print(e)
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification[
'single_mode']
return jsonify({
'message':
str(e),
'username':
session['username'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train'],
'categoric':
session['cache'][session['session_id']]['categorical_fields'],
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'placeholder':
session['cache'][session['session_id']]['placeholder']
})
def filtrate():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return redirect(url_for('home', expired='1'))
if request.method == 'POST':
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
msg = ''
filters = {
k: v[0] if len(v) == 1 else ','.join(v)
for k, v in request.form.to_dict(flat=False).items() if v != ['']
}
source_df = pandas.read_pickle(session['backup']['source_df'])
filtered_df = source_df.to_pickle(session['backup']['filtered_df'])
for field in filters:
if not pandas.read_pickle(session['backup']['filtered_df']).empty:
filtered_df = filter_dataframe(
field,
filters[field],
pandas.read_pickle(session['backup']['filtered_df']),
defect_attributes=session['config.ini']
['DEFECT_ATTRIBUTES'])
filtered_df.to_pickle(session['backup']['filtered_df'])
filtered_df = pandas.read_pickle(session['backup']['filtered_df'])
if pandas.read_pickle(session['backup']['filtered_df']).empty:
rollback_filter(session['backup'])
msg = 'there are no rows corresponding to the condition'
else:
session['cache'][session['session_id']] = get_analysis_data(
pandas.read_pickle(session['backup']['filtered_df']),
session['config.ini']['DEFECT_ATTRIBUTES'],
session['config.ini']['MACHINE_LEARNING']['asignee_reporter'] +
session['config.ini']['MACHINE_LEARNING']
['weekdays_stop_words'] +
session['config.ini']['MACHINE_LEARNING']['months_stop_words'])
session['cache'][session['session_id']]['statistical_info'] = dict(
session['cache'][session['session_id']]['statistical_info'],
**get_records_count(filtered_df=filtered_df,
source_df=source_df))
msg = 'data filtered'
session['cache'][session['session_id']][
'categorical_fields'] = get_categorical_fields(
pandas.read_pickle(session['backup']['source_df']),
session['config.ini']['DEFECT_ATTRIBUTES'],
session['config.ini']['DEFECT_ATTRIBUTES']['referring_to'])
session['cache'][session['session_id']]['categorical_fields'][
'ReferringTo'] = load_categorical_referring_to(
pandas.read_pickle(session['backup']['filtered_df']),
referring_to_fields=session['config.ini']['DEFECT_ATTRIBUTES']
['referring_to'])['ReferringTo']
session['cache'][session['session_id']]['gui_attributes'].update(
filters)
return jsonify({
'username':
session['username'],
'message':
msg,
'statistical_info':
session['cache'][session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]['categorical_fields'],
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'plot':
session['cache'][session['session_id']]['charts'],
'markup':
str(session['markup']),
'freqTop':
session['cache'][session['session_id']]['frequently_terms'],
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'placeholder':
session['cache'][session['session_id']]['placeholder'],
'is_train':
session['is_train']
})
def get_defect_submission_chart_coordinates():
"""Calculates distribution chart coordinates based on the selected period.
Returns:
stringified json-objects containing calculated coordinates.
"""
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return redirect(url_for('home', expired='1'))
if request.method == 'POST':
scale = request.form['scale']
step_size = request.form['stepSize']
x = request.form['x']
y = request.form['y']
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
if check_scale_and_step(
scale, step_size, x,
session['cache'][session['session_id']]['statistical_info']):
return jsonify({
'username':
session['username'],
'message':
'chart builded',
'statistical_info':
session['cache'][session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]['categorical_fields'],
'plot':
calculate_coordinates(
pandas.read_pickle(session['backup']['filtered_df']), x, y,
scale, step_size),
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'markup':
str(session['markup']),
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status
})
else:
return jsonify({
'username':
session['username'],
'message':
'please use Xmax and StepSize value in the array [0,maxValue from stat info]',
'statistical_info':
session['cache'][session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]['categorical_fields'],
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'plot':
calculate_coordinates(
pandas.read_pickle(session['backup']['filtered_df']), x, y,
'', ''),
'markup':
str(session['markup']),
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status
})
def analysis_and_training():
if check_session():
return redirect(url_for('home', expired='1'))
try:
# predictions_parameters.ini config verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
error_message = predictions_parameters_verification['err_message']
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'] = {}
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'] = {
area.strip(): {
'series_name':
area.strip() + '_lab',
'elements':
session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes'][area]['name'].strip(),
'gui_name':
area.strip()
}
for area in session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes']
}
if not is_file_exist(session['backup']['source_df']):
session['is_train'] = False
return render_template(
'filterPage.html',
json=json.dumps({
'username':
session['username'],
'message':
error_message,
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
else:
if session['new_settings']:
df = pandas.read_pickle(session['backup']['source_df'])
if session['markup']:
for area in session['config.ini']['DEFECT_ATTRIBUTES'][
'areas_of_testing']:
df = mark_up_series(
df, session['config.ini']['DEFECT_ATTRIBUTES']
['markup_series'],
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['series_name'],
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['elements'].lower())
df = mark_up_other_data(df, [
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['series_name']
for area in session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'] if area != 'Other'
])
df.to_pickle(session['backup']['source_df'])
df.to_pickle(session['backup']['filtered_df'])
session['cache'][session['session_id']] = get_analysis_data(
df, session['config.ini']['DEFECT_ATTRIBUTES'],
session['config.ini']['MACHINE_LEARNING']
['asignee_reporter'] + session['config.ini']
['MACHINE_LEARNING']['weekdays_stop_words'] +
session['config.ini']['MACHINE_LEARNING']
['months_stop_words'])
session['cache'][session['session_id']][
'categorical_fields'] = get_categorical_fields(
df,
defect_attributes=session['config.ini']
['DEFECT_ATTRIBUTES'],
referring_to_fields=session['config.ini']
['DEFECT_ATTRIBUTES']['referring_to'])
session['cache'][
session['session_id']]['statistical_info'] = dict(
session['cache'][session['session_id']]
['statistical_info'],
**get_records_count(filtered_df=df, source_df=df))
session['config.ini']['DEFECT_ATTRIBUTES'][
'special_attributes']['ttr'] = {
'type': 'Numeric',
'name': 'Time to Resolve (TTR)'
}
session['new_settings'] = False
session['markup'] = 1 if (
'1' if session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes'] else '0') == '1' else 0
session['is_train'] = True if session['markup'] else False
if session['markup']:
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'][
'Other'] = {
'series_name': 'Other_lab',
'elements': 'Other',
'gui_name': 'Other'
}
return render_template(
'filterPage.html',
json=json.dumps({
'username':
session['username'],
'message':
session['filename'] + '\n' + error_message,
'statistical_info':
session['cache'][
session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]
['categorical_fields'],
'plot':
session['cache'][session['session_id']]['charts'],
'markup':
str(session['markup']),
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'placeholder':
session['cache'][session['session_id']]['placeholder'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
except Exception:
return render_template(
'filterPage.html',
json=json.dumps({
'username':
session['username'],
'message':
'Oops! Something went wrong. Please try again later.',
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
def upload_file():
"""
Uploads file from GUI, transforms it to pandas dataframe and saves it as pickle object to HDD.
"""
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if request.method == 'POST':
try:
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification[
'single_mode']
uploaded_files = request.files.getlist("file[]")
df = convert_to_dataframe(uploaded_files)
if not check_bugs_count(df, 1):
raise InconsistentDataError(
'Oops! Uploaded dataset should has at least one defect.')
session['backup']['source_df'] = '{0}/{1}_{2}.pkl'.format(
session['backup']['backup_folder'], strftime('%y%m%d%H%M%S'),
'source_df')
session['backup']['filtered_df'] = '{0}/{1}_{2}.pkl'.format(
session['backup']['backup_folder'], strftime('%y%m%d%H%M%S'),
'filtered_df')
session['config.ini']['MACHINE_LEARNING'][
'asignee_reporter'] = list({
name
for full_name in [
full_name.lower().split()
for full_name in df['Assignee'].tolist() +
df['Reporter'].tolist()
] for name in full_name
})
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'] = {}
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'] = {
area.strip(): {
'series_name':
area.strip() + '_lab',
'elements':
session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes'][area]['name'].strip(),
'gui_name':
area.strip()
}
for area in session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes']
}
if session['markup']:
for area in session['config.ini']['DEFECT_ATTRIBUTES'][
'areas_of_testing']:
df = mark_up_series(
df, session['config.ini']['DEFECT_ATTRIBUTES']
['markup_series'],
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['series_name'],
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['elements'].lower())
df = mark_up_other_data(df, [
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['series_name']
for area in session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'] if area != 'Other'
])
df.to_pickle(session['backup']['source_df'])
df.to_pickle(session['backup']['filtered_df'])
if session['markup']:
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'][
'Other'] = {
'series_name': 'Other_lab',
'elements': 'Other',
'gui_name': 'Other'
}
session['cache'][session['session_id']] = get_analysis_data(
df, session['config.ini']['DEFECT_ATTRIBUTES'],
session['config.ini']['MACHINE_LEARNING']['asignee_reporter'] +
session['config.ini']['MACHINE_LEARNING']
['weekdays_stop_words'] +
session['config.ini']['MACHINE_LEARNING']['months_stop_words'])
session['cache'][session['session_id']][
'categorical_fields'] = get_categorical_fields(
df,
defect_attributes=session['config.ini']
['DEFECT_ATTRIBUTES'],
referring_to_fields=session['config.ini']
['DEFECT_ATTRIBUTES']['referring_to'])
session['cache'][session['session_id']]['statistical_info'] = dict(
session['cache'][session['session_id']]['statistical_info'],
**get_records_count(filtered_df=df, source_df=df))
session['filename'] = '\n'.join(
[str(file.filename) for file in uploaded_files])
df.to_pickle(session['backup']['source_df'])
df.to_pickle(session['backup']['filtered_df'])
session['is_train'] = True if session['markup'] else False
return render_template(
'filterPage.html',
json=json.dumps({
'username':
session['username'],
'message':
'file uploaded successfully',
'statistical_info':
session['cache'][
session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]
['categorical_fields'],
'plot':
session['cache'][session['session_id']]['charts'],
'markup':
str(session['markup']),
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'placeholder':
session['cache'][session['session_id']]['placeholder'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
except (InconsistentDataError, Exception) as err_msg:
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification[
'single_mode']
return render_template(
'filterPage.html',
json=json.dumps({
'username':
session['username'],
'message':
str(err_msg),
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
def predict():
if request.method == 'GET':
session.clear()
return redirect(url_for('home', expired='0'), code=302)
if check_session():
return jsonify(dict(redirect=url_for('home', expired='1')))
try:
# predictions_parameters.ini file verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
orig_descr = request.form['descr']
cleaned_descr = clean_description(orig_descr)
if cleaned_descr.isspace(
) or cleaned_descr == 'aftercleaning' or cleaned_descr is None:
raise Exception(
'Oops! Description can\'t be analyzed. Please check the text.')
session['probabilities'] = {}
session['probabilities']['priority'] = get_probabilities(
cleaned_descr, session['predictions_parameters.ini']
['predictions_parameters']['priority_classes'],
str(Path(__file__).parents[2]) + '/models/selected/' + 'priority')
session['probabilities']['ttr_probability'] = get_probabilities(
cleaned_descr, session['predictions_parameters.ini']
['predictions_parameters']['ttr_classes'],
str(Path(__file__).parents[2]) + '/models/selected/' + 'ttr')
session['probabilities']['resolution_probability'] = {
resolution: get_probabilities(
cleaned_descr, session['predictions_parameters.ini']
['predictions_parameters'][resolution.lower() + '_classes'],
str(Path(__file__).parents[2]) + '/models/selected/' +
secure_filename(resolution))
for resolution in unpack_dictionary_val_to_list(
session['config.ini']['DEFECT_ATTRIBUTES']['resolution'])
}
session['probabilities']['areas_of_testing'] = {
area: get_probabilities(
cleaned_descr, session['predictions_parameters.ini']
['predictions_parameters']['binary_classes'],
str(Path(__file__).parents[2]) + '/models/selected/' +
secure_filename(area))[1]
for area in session['predictions_parameters.ini']
['predictions_parameters']['areas_of_testing_classes']
}
filtered_areas = [
key for key, value in session['probabilities']
['areas_of_testing'].items() if value > 0.5
]
session['probabilities']['recomendation'] = ' '.join(
[area for area in filtered_areas]) if filtered_areas else 'no idea'
return jsonify({
'descr':
orig_descr,
'recom':
session['probabilities']['recomendation'],
'prio':
session['probabilities']['priority'],
'ttr':
session['probabilities']['ttr_probability'],
'resolution_pie':
session['probabilities']['resolution_probability'],
'areas':
session['probabilities']['areas_of_testing'],
'user':
session['username'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status
})
except Exception as err_mssg:
return jsonify({
'username': session['username'],
'error': str(err_mssg)
})