def load_base_loggers_config(func): """ Setting up logger options. Parameters: func (Function): a function to log. Returns: logger (Loger): logger object. """ logger = logging.getLogger('{func}'.format(func=func.__name__)) level = get_level( logger, session['config.ini']['DEFECT_ATTRIBUTES']['logging_level'][0]) if not level: return None logger.setLevel(level) if not len(logger.handlers): date = datetime.date(datetime.today()) if not is_file_exist('logs/' + str(date) + '/'): os.makedirs('logs/' + str(date) + '/') file_handler = handlers.RotatingFileHandler( filename='logs/' + str(date) + '/' + str( session['session_id']) + '.log', maxBytes=50 * 1024 * 1024, backupCount=10) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') file_handler.setFormatter(formatter) logger.addHandler(file_handler) return logger
def check_predictions_parameters_config(): try: config_exist = os.path.exists( str(Path(__file__).parents[1]) + '/models/selected/' + 'predictions_parameters.ini') if not config_exist: return { 'single_mode': False, 'multiple_mode': False, 'err_message': '' } else: # checking existence of common models are_models_exist(get_models_names() + ['ttr']) # checking model's existence required for # single description mode only are_models_exist(['priority']) # checking top_terms.csv file existence which is required # for single description mode if not is_file_exist( str(Path(__file__).parents[1]) + '/models/selected/' + 'top_terms.csv'): raise FileNotFoundError('Can\'t find top_terms.csv file.') return {'single_mode': True, 'multiple_mode': True, 'err_message': ''} except ModelsNotFound as err: return { 'single_mode': False, 'multiple_mode': False, 'err_message': str(err) } except ModelNotFound as err: return { 'single_mode': False, 'multiple_mode': True, 'err_message': str(err) } except FileNotFoundError as err: return { 'single_mode': False, 'multiple_mode': True, 'err_message': str(err) } except (KeyError, SyntaxError) as err: return { 'single_mode': False, 'multiple_mode': False, 'err_message': str(err) }
def are_models_exist(models): """ Checks whether the handled models are exist. Parameters: models(list): List of models' names. """ not_found_models = [] for model in models: model = secure_filename(model) if not is_file_exist( str(Path(__file__).parents[1]) + '/models/selected/' + model + '.sav'): not_found_models.append(model + '.sav') if len(not_found_models) > 0: raise ModelsNotFound( 'Oops! Can\'t find the following model(s): {}. Please check whether the models exist' ' and correctness of models names' ' in predictions_parameters.ini'.format( ', '.join(not_found_models)))
def check_defect_attributes(defect_attributes): mandatory_multiple_mode_fields = [ 'Issue_key', 'Summary', 'Priority', 'Description' ] try: # checks that received multiple mode's fields are in mandatory # attributes list if not is_subset(defect_attributes['multiple_mode_attributes'], mandatory_multiple_mode_fields): return False, '{} are mandatory for Multiple Description Mode.'.format( ', '.join(mandatory_multiple_mode_fields)) # checks that mandatory/special attributes have correct data types allowed_data_types = [ 'String', 'Substring', 'Substring_array', 'Numeric', 'Date', 'Categorical', 'Boolean' ] if not is_subset([ defect_attributes[defect_attributes_group][field]['type'] for defect_attributes_group in ['mandatory_attributes', 'special_attributes'] for field in defect_attributes[defect_attributes_group] ], allowed_data_types): return False, 'Oops! Incorrect data type specified in mandatory or special attributes. Allowed data types: {}'.format( ', '.join(allowed_data_types)) # checks that quantity of received resolution's elements isn't greater than specified quantity # because it cannot has more than two values in the list if not is_greater(defect_attributes['resolution'], 2): return False, 'Oops! Resolution field cannot has more than two values.' # checks that mandatory attributes are specified in config.ini if not is_empty(defect_attributes['mandatory_attributes']): return False, 'Oops! Mandatory attributes aren\'t specified.' special_attributes = set( defect_attributes['special_attributes'].keys()).union({ defect_attributes['special_attributes'][el]['name'] for el in defect_attributes['special_attributes'].keys() }) mandatory_attributes = set( defect_attributes['mandatory_attributes'].keys()).union({ defect_attributes['mandatory_attributes'][el]['name'] for el in defect_attributes['mandatory_attributes'].keys() }) for attribute in special_attributes: if attribute in mandatory_attributes: return False, 'Oops! {attribute} is already specified in Mandatory attributes.'.format( attribute=attribute) # checks that all received Referring to's fields (except Areas of # testing) do exist in mandatory/special attributes if not is_subset( [ el for el in defect_attributes['referring_to'] if el != 'Areas of testing' ], list(defect_attributes['mandatory_attributes'].keys()) + list(defect_attributes['special_attributes'].keys())): return False, 'Oops! Referring to fields should be presented in mandatory or special attributes.' if ('Areas of testing' in defect_attributes['referring_to'] ) and not (is_empty(defect_attributes['mark_up_attributes'])): return False, 'Oops! Area of testing section is empty.\ Please specify areas of testing before assigning this element to Referring to section.' if 'Areas of testing' in defect_attributes['referring_to'] and ( not is_file_exist(str(Path(__file__).parents[1]) + '/models/selected/' + 'top_terms.csv', check_size=True)): return False, 'Oops! Areas of testing couldn\'t be added to Referring to field while models aren\'t trained.' \ '\nPlease train models firstly.' if not set(defect_attributes['resolution'].keys()).issubset( set(field for group in ['mandatory_attributes', 'special_attributes'] for field in defect_attributes[group].keys())): return False, 'Oops! Resolution fields should be presented in Mandatory or Special fields.' if not all([ field_type == 'Categorical' for field_type in { defect_attributes[group][field]['type'] for group in ['mandatory_attributes', 'special_attributes'] for field in defect_attributes['resolution'].keys() if field in defect_attributes[group] } ]): return False, 'Oops! Resolution fields should have Categorical datatype.' return True, None except Exception as e: return False, str(e)
def analysis_and_training(): if check_session(): return redirect(url_for('home', expired='1')) try: # predictions_parameters.ini config verification predictions_parameters_verification = check_predictions_parameters_config( ) multiple_mode_status = predictions_parameters_verification[ 'multiple_mode'] signle_mode_status = predictions_parameters_verification['single_mode'] error_message = predictions_parameters_verification['err_message'] session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'] = {} session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'] = { area.strip(): { 'series_name': area.strip() + '_lab', 'elements': session['config.ini']['DEFECT_ATTRIBUTES'] ['mark_up_attributes'][area]['name'].strip(), 'gui_name': area.strip() } for area in session['config.ini']['DEFECT_ATTRIBUTES'] ['mark_up_attributes'] } if not is_file_exist(session['backup']['source_df']): session['is_train'] = False return render_template( 'filterPage.html', json=json.dumps({ 'username': session['username'], 'message': error_message, 'fields': session['config.ini']['DEFECT_ATTRIBUTES'], 'file_size': session['config.ini']['REQUIREMENTS']['max_file_size'], 'single_mod': signle_mode_status, 'multiple_mod': multiple_mode_status, 'is_train': session['is_train'] })) else: if session['new_settings']: df = pandas.read_pickle(session['backup']['source_df']) if session['markup']: for area in session['config.ini']['DEFECT_ATTRIBUTES'][ 'areas_of_testing']: df = mark_up_series( df, session['config.ini']['DEFECT_ATTRIBUTES'] ['markup_series'], session['config.ini']['DEFECT_ATTRIBUTES'] ['areas_of_testing'][area]['series_name'], session['config.ini']['DEFECT_ATTRIBUTES'] ['areas_of_testing'][area]['elements'].lower()) df = mark_up_other_data(df, [ session['config.ini']['DEFECT_ATTRIBUTES'] ['areas_of_testing'][area]['series_name'] for area in session['config.ini']['DEFECT_ATTRIBUTES'] ['areas_of_testing'] if area != 'Other' ]) df.to_pickle(session['backup']['source_df']) df.to_pickle(session['backup']['filtered_df']) session['cache'][session['session_id']] = get_analysis_data( df, session['config.ini']['DEFECT_ATTRIBUTES'], session['config.ini']['MACHINE_LEARNING'] ['asignee_reporter'] + session['config.ini'] ['MACHINE_LEARNING']['weekdays_stop_words'] + session['config.ini']['MACHINE_LEARNING'] ['months_stop_words']) session['cache'][session['session_id']][ 'categorical_fields'] = get_categorical_fields( df, defect_attributes=session['config.ini'] ['DEFECT_ATTRIBUTES'], referring_to_fields=session['config.ini'] ['DEFECT_ATTRIBUTES']['referring_to']) session['cache'][ session['session_id']]['statistical_info'] = dict( session['cache'][session['session_id']] ['statistical_info'], **get_records_count(filtered_df=df, source_df=df)) session['config.ini']['DEFECT_ATTRIBUTES'][ 'special_attributes']['ttr'] = { 'type': 'Numeric', 'name': 'Time to Resolve (TTR)' } session['new_settings'] = False session['markup'] = 1 if ( '1' if session['config.ini']['DEFECT_ATTRIBUTES'] ['mark_up_attributes'] else '0') == '1' else 0 session['is_train'] = True if session['markup'] else False if session['markup']: session['config.ini']['DEFECT_ATTRIBUTES']['areas_of_testing'][ 'Other'] = { 'series_name': 'Other_lab', 'elements': 'Other', 'gui_name': 'Other' } return render_template( 'filterPage.html', json=json.dumps({ 'username': session['username'], 'message': session['filename'] + '\n' + error_message, 'statistical_info': session['cache'][ session['session_id']]['statistical_info'], 'categoric': session['cache'][session['session_id']] ['categorical_fields'], 'plot': session['cache'][session['session_id']]['charts'], 'markup': str(session['markup']), 'file_size': session['config.ini']['REQUIREMENTS']['max_file_size'], 'attributes': session['cache'][session['session_id']]['gui_attributes'], 'fields': session['config.ini']['DEFECT_ATTRIBUTES'], 'placeholder': session['cache'][session['session_id']]['placeholder'], 'single_mod': signle_mode_status, 'multiple_mod': multiple_mode_status, 'is_train': session['is_train'] })) except Exception: return render_template( 'filterPage.html', json=json.dumps({ 'username': session['username'], 'message': 'Oops! Something went wrong. Please try again later.', 'fields': session['config.ini']['DEFECT_ATTRIBUTES'], 'file_size': session['config.ini']['REQUIREMENTS']['max_file_size'], 'single_mod': signle_mode_status, 'multiple_mod': multiple_mode_status, 'is_train': session['is_train'] }))
def home(): page_name = 'loginPage_portable.html' try: load_base_config() load_config_to_session('config.ini') if is_file_exist(str(Path(__file__).parents[0]) + '/models/selected/' + 'predictions_parameters.ini', check_size=True): load_config_to_session( str(Path(__file__).parents[0]) + '/models/selected/' + 'predictions_parameters.ini') if request.method == 'GET': if is_session_expired(): return render_template(page_name, json=json.dumps({'error': ''})) else: return render_template( page_name, json=json.dumps( {'error': 'Session expired. Please login again.'})) if request.method == 'POST': # checks required file's existence regular_expressions = str( Path(__file__ ).parents[0]) + '/extensions/' + 'regularExpression.csv' config_path = str(Path(__file__).parents[0]) + '/config.ini' if not is_file_exist( regular_expressions, config_path, check_size=True): raise NotExistFile( 'Please check existence of config.ini and regularExpression.csv files.' ) session['is_train'] = False session['session_id'] = session.sid[:32] session['backup'] = {} session['backup'][ 'backup_folder'] = '{0}/backup/{1}/files/'.format( os.path.abspath(os.curdir), session['session_id']) session['cache'] = {} session['cache'][session['session_id']] = {} session['temp_files'] = [] # temp files' paths storage session['temp_files'].append("{cur}/files/{id}".format( cur=os.curdir, id=session['session_id'])) if not os.path.exists(session['backup']['backup_folder']): create_folder(session['backup']['backup_folder']) app.config['UPLOAD_FOLDER'] = session['backup']['backup_folder'] session.permanent = False session['username'] = '******' # vectorizer settings creation session['tfidf'] = StemmedTfidfVectorizer( norm='l2', sublinear_tf=True, min_df=1, stop_words=text.ENGLISH_STOP_WORDS.difference( ('see', 'system', 'call')).union( session['config.ini']['MACHINE_LEARNING'] ['asignee_reporter'], session['config.ini'] ['MACHINE_LEARNING']['weekdays_stop_words'], session['config.ini']['MACHINE_LEARNING'] ['months_stop_words'], ['having', 'couldn']), analyzer='word') session['markup'] = 1 if ( '1' if session['config.ini']['DEFECT_ATTRIBUTES'] ['mark_up_attributes'] else '0') == '1' else 0 session['backup']['source_df'] = '' session['backup']['filtered_df'] = '' session['backup']['predictions_table'] = '' session['new_settings'] = False # defect attributes verification verification_status, error_message = check_defect_attributes( session['config.ini']['DEFECT_ATTRIBUTES']) if verification_status: return redirect('/analysis_and_training', code=307) else: return render_template( 'setting.html', json=json.dumps({ 'error': error_message, 'mandatory_attributes': session['config.ini']['DEFECT_ATTRIBUTES'] ['mandatory_attributes'], 'data_types': session['config.ini']['REQUIREMENTS'] ['allowed_data_types'], 'referring_to': session['config.ini']['DEFECT_ATTRIBUTES'] ['referring_to'], 'logging_level': session['config.ini']['DEFECT_ATTRIBUTES'] ['logging_level'], 'config_data': session['config.ini']['DEFECT_ATTRIBUTES'] })) except Exception as e: return render_template(page_name, json=json.dumps({'error': str(e)}))