def load_base_loggers_config(func):
""" Setting up logger options.
Parameters:
func (Function): a function to log.
Returns:
logger (Loger): logger object.
"""
logger = logging.getLogger('{func}'.format(func=func.__name__))
level = get_level(
logger,
session['config.ini']['DEFECT_ATTRIBUTES']['logging_level'][0])
if not level:
return None
logger.setLevel(level)
if not len(logger.handlers):
date = datetime.date(datetime.today())
if not is_file_exist('logs/' + str(date) + '/'):
os.makedirs('logs/' + str(date) + '/')
file_handler = handlers.RotatingFileHandler(
filename='logs/' + str(date) + '/' + str(
session['session_id']) + '.log',
maxBytes=50 * 1024 * 1024,
backupCount=10)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
return logger
def check_predictions_parameters_config():
try:
config_exist = os.path.exists(
str(Path(__file__).parents[1]) + '/models/selected/' +
'predictions_parameters.ini')
if not config_exist:
return {
'single_mode': False,
'multiple_mode': False,
'err_message': ''
}
else:
# checking existence of common models
are_models_exist(get_models_names() + ['ttr'])
# checking model's existence required for
# single description mode only
are_models_exist(['priority'])
# checking top_terms.csv file existence which is required
# for single description mode
if not is_file_exist(
str(Path(__file__).parents[1]) + '/models/selected/' +
'top_terms.csv'):
raise FileNotFoundError('Can\'t find top_terms.csv file.')
return {'single_mode': True, 'multiple_mode': True, 'err_message': ''}
except ModelsNotFound as err:
return {
'single_mode': False,
'multiple_mode': False,
'err_message': str(err)
}
except ModelNotFound as err:
return {
'single_mode': False,
'multiple_mode': True,
'err_message': str(err)
}
except FileNotFoundError as err:
return {
'single_mode': False,
'multiple_mode': True,
'err_message': str(err)
}
except (KeyError, SyntaxError) as err:
return {
'single_mode': False,
'multiple_mode': False,
'err_message': str(err)
}
def are_models_exist(models):
""" Checks whether the handled models are exist.
Parameters:
models(list): List of models' names.
"""
not_found_models = []
for model in models:
model = secure_filename(model)
if not is_file_exist(
str(Path(__file__).parents[1]) + '/models/selected/' + model +
'.sav'):
not_found_models.append(model + '.sav')
if len(not_found_models) > 0:
raise ModelsNotFound(
'Oops! Can\'t find the following model(s): {}. Please check whether the models exist'
' and correctness of models names'
' in predictions_parameters.ini'.format(
', '.join(not_found_models)))
def check_defect_attributes(defect_attributes):
mandatory_multiple_mode_fields = [
'Issue_key', 'Summary', 'Priority', 'Description'
]
try:
# checks that received multiple mode's fields are in mandatory
# attributes list
if not is_subset(defect_attributes['multiple_mode_attributes'],
mandatory_multiple_mode_fields):
return False, '{} are mandatory for Multiple Description Mode.'.format(
', '.join(mandatory_multiple_mode_fields))
# checks that mandatory/special attributes have correct data types
allowed_data_types = [
'String', 'Substring', 'Substring_array', 'Numeric', 'Date',
'Categorical', 'Boolean'
]
if not is_subset([
defect_attributes[defect_attributes_group][field]['type']
for defect_attributes_group in
['mandatory_attributes', 'special_attributes']
for field in defect_attributes[defect_attributes_group]
], allowed_data_types):
return False, 'Oops! Incorrect data type specified in mandatory or special attributes. Allowed data types: {}'.format(
', '.join(allowed_data_types))
# checks that quantity of received resolution's elements isn't greater than specified quantity
# because it cannot has more than two values in the list
if not is_greater(defect_attributes['resolution'], 2):
return False, 'Oops! Resolution field cannot has more than two values.'
# checks that mandatory attributes are specified in config.ini
if not is_empty(defect_attributes['mandatory_attributes']):
return False, 'Oops! Mandatory attributes aren\'t specified.'
special_attributes = set(
defect_attributes['special_attributes'].keys()).union({
defect_attributes['special_attributes'][el]['name']
for el in defect_attributes['special_attributes'].keys()
})
mandatory_attributes = set(
defect_attributes['mandatory_attributes'].keys()).union({
defect_attributes['mandatory_attributes'][el]['name']
for el in defect_attributes['mandatory_attributes'].keys()
})
for attribute in special_attributes:
if attribute in mandatory_attributes:
return False, 'Oops! {attribute} is already specified in Mandatory attributes.'.format(
attribute=attribute)
# checks that all received Referring to's fields (except Areas of
# testing) do exist in mandatory/special attributes
if not is_subset(
[
el for el in defect_attributes['referring_to']
if el != 'Areas of testing'
],
list(defect_attributes['mandatory_attributes'].keys()) +
list(defect_attributes['special_attributes'].keys())):
return False, 'Oops! Referring to fields should be presented in mandatory or special attributes.'
if ('Areas of testing' in defect_attributes['referring_to']
) and not (is_empty(defect_attributes['mark_up_attributes'])):
return False, 'Oops! Area of testing section is empty.\
Please specify areas of testing before assigning this element to Referring to section.'
if 'Areas of testing' in defect_attributes['referring_to'] and (
not is_file_exist(str(Path(__file__).parents[1]) +
'/models/selected/' + 'top_terms.csv',
check_size=True)):
return False, 'Oops! Areas of testing couldn\'t be added to Referring to field while models aren\'t trained.' \
'\nPlease train models firstly.'
if not set(defect_attributes['resolution'].keys()).issubset(
set(field for group in
['mandatory_attributes', 'special_attributes']
for field in defect_attributes[group].keys())):
return False, 'Oops! Resolution fields should be presented in Mandatory or Special fields.'
if not all([
field_type == 'Categorical' for field_type in {
defect_attributes[group][field]['type']
for group in
['mandatory_attributes', 'special_attributes']
for field in defect_attributes['resolution'].keys()
if field in defect_attributes[group]
}
]):
return False, 'Oops! Resolution fields should have Categorical datatype.'
return True, None
except Exception as e:
return False, str(e)
def analysis_and_training():
if check_session():
return redirect(url_for('home', expired='1'))
try:
# predictions_parameters.ini config verification
predictions_parameters_verification = check_predictions_parameters_config(
)
multiple_mode_status = predictions_parameters_verification[
'multiple_mode']
signle_mode_status = predictions_parameters_verification['single_mode']
error_message = predictions_parameters_verification['err_message']
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'] = {}
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'] = {
area.strip(): {
'series_name':
area.strip() + '_lab',
'elements':
session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes'][area]['name'].strip(),
'gui_name':
area.strip()
}
for area in session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes']
}
if not is_file_exist(session['backup']['source_df']):
session['is_train'] = False
return render_template(
'filterPage.html',
json=json.dumps({
'username':
session['username'],
'message':
error_message,
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
else:
if session['new_settings']:
df = pandas.read_pickle(session['backup']['source_df'])
if session['markup']:
for area in session['config.ini']['DEFECT_ATTRIBUTES'][
'areas_of_testing']:
df = mark_up_series(
df, session['config.ini']['DEFECT_ATTRIBUTES']
['markup_series'],
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['series_name'],
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['elements'].lower())
df = mark_up_other_data(df, [
session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'][area]['series_name']
for area in session['config.ini']['DEFECT_ATTRIBUTES']
['areas_of_testing'] if area != 'Other'
])
df.to_pickle(session['backup']['source_df'])
df.to_pickle(session['backup']['filtered_df'])
session['cache'][session['session_id']] = get_analysis_data(
df, session['config.ini']['DEFECT_ATTRIBUTES'],
session['config.ini']['MACHINE_LEARNING']
['asignee_reporter'] + session['config.ini']
['MACHINE_LEARNING']['weekdays_stop_words'] +
session['config.ini']['MACHINE_LEARNING']
['months_stop_words'])
session['cache'][session['session_id']][
'categorical_fields'] = get_categorical_fields(
df,
defect_attributes=session['config.ini']
['DEFECT_ATTRIBUTES'],
referring_to_fields=session['config.ini']
['DEFECT_ATTRIBUTES']['referring_to'])
session['cache'][
session['session_id']]['statistical_info'] = dict(
session['cache'][session['session_id']]
['statistical_info'],
**get_records_count(filtered_df=df, source_df=df))
session['config.ini']['DEFECT_ATTRIBUTES'][
'special_attributes']['ttr'] = {
'type': 'Numeric',
'name': 'Time to Resolve (TTR)'
}
session['new_settings'] = False
session['markup'] = 1 if (
'1' if session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes'] else '0') == '1' else 0
session['is_train'] = True if session['markup'] else False
if session['markup']:
session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'][
'Other'] = {
'series_name': 'Other_lab',
'elements': 'Other',
'gui_name': 'Other'
}
return render_template(
'filterPage.html',
json=json.dumps({
'username':
session['username'],
'message':
session['filename'] + '\n' + error_message,
'statistical_info':
session['cache'][
session['session_id']]['statistical_info'],
'categoric':
session['cache'][session['session_id']]
['categorical_fields'],
'plot':
session['cache'][session['session_id']]['charts'],
'markup':
str(session['markup']),
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'attributes':
session['cache'][session['session_id']]['gui_attributes'],
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'placeholder':
session['cache'][session['session_id']]['placeholder'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
except Exception:
return render_template(
'filterPage.html',
json=json.dumps({
'username':
session['username'],
'message':
'Oops! Something went wrong. Please try again later.',
'fields':
session['config.ini']['DEFECT_ATTRIBUTES'],
'file_size':
session['config.ini']['REQUIREMENTS']['max_file_size'],
'single_mod':
signle_mode_status,
'multiple_mod':
multiple_mode_status,
'is_train':
session['is_train']
}))
def home():
page_name = 'loginPage_portable.html'
try:
load_base_config()
load_config_to_session('config.ini')
if is_file_exist(str(Path(__file__).parents[0]) + '/models/selected/' +
'predictions_parameters.ini',
check_size=True):
load_config_to_session(
str(Path(__file__).parents[0]) + '/models/selected/' +
'predictions_parameters.ini')
if request.method == 'GET':
if is_session_expired():
return render_template(page_name,
json=json.dumps({'error': ''}))
else:
return render_template(
page_name,
json=json.dumps(
{'error': 'Session expired. Please login again.'}))
if request.method == 'POST':
# checks required file's existence
regular_expressions = str(
Path(__file__
).parents[0]) + '/extensions/' + 'regularExpression.csv'
config_path = str(Path(__file__).parents[0]) + '/config.ini'
if not is_file_exist(
regular_expressions, config_path, check_size=True):
raise NotExistFile(
'Please check existence of config.ini and regularExpression.csv files.'
)
session['is_train'] = False
session['session_id'] = session.sid[:32]
session['backup'] = {}
session['backup'][
'backup_folder'] = '{0}/backup/{1}/files/'.format(
os.path.abspath(os.curdir), session['session_id'])
session['cache'] = {}
session['cache'][session['session_id']] = {}
session['temp_files'] = [] # temp files' paths storage
session['temp_files'].append("{cur}/files/{id}".format(
cur=os.curdir, id=session['session_id']))
if not os.path.exists(session['backup']['backup_folder']):
create_folder(session['backup']['backup_folder'])
app.config['UPLOAD_FOLDER'] = session['backup']['backup_folder']
session.permanent = False
session['username'] = '******'
# vectorizer settings creation
session['tfidf'] = StemmedTfidfVectorizer(
norm='l2',
sublinear_tf=True,
min_df=1,
stop_words=text.ENGLISH_STOP_WORDS.difference(
('see', 'system', 'call')).union(
session['config.ini']['MACHINE_LEARNING']
['asignee_reporter'], session['config.ini']
['MACHINE_LEARNING']['weekdays_stop_words'],
session['config.ini']['MACHINE_LEARNING']
['months_stop_words'], ['having', 'couldn']),
analyzer='word')
session['markup'] = 1 if (
'1' if session['config.ini']['DEFECT_ATTRIBUTES']
['mark_up_attributes'] else '0') == '1' else 0
session['backup']['source_df'] = ''
session['backup']['filtered_df'] = ''
session['backup']['predictions_table'] = ''
session['new_settings'] = False
# defect attributes verification
verification_status, error_message = check_defect_attributes(
session['config.ini']['DEFECT_ATTRIBUTES'])
if verification_status:
return redirect('/analysis_and_training', code=307)
else:
return render_template(
'setting.html',
json=json.dumps({
'error':
error_message,
'mandatory_attributes':
session['config.ini']['DEFECT_ATTRIBUTES']
['mandatory_attributes'],
'data_types':
session['config.ini']['REQUIREMENTS']
['allowed_data_types'],
'referring_to':
session['config.ini']['DEFECT_ATTRIBUTES']
['referring_to'],
'logging_level':
session['config.ini']['DEFECT_ATTRIBUTES']
['logging_level'],
'config_data':
session['config.ini']['DEFECT_ATTRIBUTES']
}))
except Exception as e:
return render_template(page_name, json=json.dumps({'error': str(e)}))