def dashboard_page(state): st.title("PROSPECTOR") st.subheader( "The search engine for fix-commits for security vulnerabilities in OSS" ) st.write('By SAP - Antonino SABETTA & Daan HOMMERSOM') st.write(''' How to use Prospector: \n1) Provide a vulnerability description, (GitHub) repository URL and a release date (or pick a CVE). \n2) Check whether Prospector fills in the rest correctly, and provide additional information if needed. \n3) Find security fixes! ''') # with st.beta_expander(label="Find out more", expanded=False): st.write(''' The objective of Prospector is to minimize the (manual) effort needed for finding the fix commit of a known vulnerability in an open-source software project. Since these repositories can contain hundreds thousands commits, the commits are firstly filtered by only selecting all commits within two years before and one hundred days after the release date with a maximum of respectively 5215 and 100 commits. A study has shown that this selection has 93% recall. \n Firstly, an advisory record is created containing information on the vulnerability. This advisory record is used to select candidate commits. For these candidate commits, ranking vectors are computed. These ranking vectors consist of several components that can be used to predict whether a candidate commit is the fix commit we are looking for. These candidates are then ranked on this probability score. In 77.68% of the cases, the fix is in the top 5. In 84.03% in the top 10, and in 88.59% in the top 20. ''') st.subheader("ADVISORY RECORD") state.vulnerability_id = st.text_input( "Vulnerability identifyer:", value=state.vulnerability_id if state.vulnerability_id else '').upper() if state.vulnerability_id: try: cve_description, cve_published_timestamp, preprocessed_description, references = get_vulnerability_data( state.vulnerability_id, state.vulnerabilities_df, state.db_references_df) except: references = st.text_input( "Please provide useful references (separated by commas)") references = references.split(',') cve_description, cve_published_timestamp, preprocessed_description = '', time.time( ), None else: cve_description, cve_published_timestamp, preprocessed_description, references = '', time.time( ), None, [] vulnerability_description = st.text_area("Vulnerability description", value=cve_description) project_name = st.text_input( "Project name", value=' '.join([ token.text for token in nlp(vulnerability_description) if token.tag_ == 'NNP' ])) repo_url = st.text_input("Repository URL", value=map_description_to_repository_url( vulnerability_id=state.vulnerability_id, description=project_name, vulnerabilities_df=state.vulnerabilities_df, repository_url_df=state.repository_url_df) if project_name != '' else '') published_date = st.date_input("Vulnerability published date", value=datetime.fromtimestamp( int(cve_published_timestamp))) published_timestamp = int(time.mktime(published_date.timetuple())) state.advisory_record_confirmed = st.button( "CONFIRM ADVISORY RECORD" ) if not state.advisory_record_confirmed else True if state.advisory_record_confirmed: # option to clear the state if st.button("CLEAR FIELDS"): state.clear() # if it was a new vulnerability, add it to the DB if type(state.vulnerabilities_df) == type( None) or state.vulnerability_id not in list( state.vulnerabilities_df.index): vulnerabilities_connection, vulnerabilities_cursor = database.connect_with_vulnerabilities_database( vulnerabilities_db_path) database.add_vulnerability_to_database(vulnerabilities_connection, state.vulnerability_id, repo_url, vulnerability_description, published_timestamp) # if it was not an NVD CVE, or the extraction failed if len(references) == 0: try: cve_description, cve_published_timestamp, references = database.extract_nvd_content( state.vulnerability_id) references = [reference for reference in references] except: references = st.text_input( "Please provide useful references (separated by commas)" ) references = references.split(',') database.add_vulnerability_references_to_database( vulnerabilities_connection, state.vulnerability_id, references, driver=None) prospector_connection, prospector_cursor = connect_with_commits_database( commits_db_path) database.add_tags_to_database(prospector_connection, tags=None, git_repo=None, repo_url=repo_url, verbose=True) state.vulnerabilities_df, state.db_references_df, state.advisory_references_df, state.tags_df, state.repository_url_df, state.fixes_df = load_vulnerabilities( ) # gather values repository_tags = gather_tags(repo_url, state.tags_df) versions_in_description = filter.retreive_all_versions_from_description( vulnerability_description) tags_in_description = list( dict.fromkeys([ tag for version in versions_in_description for tag in filter.get_tag_for_version(repository_tags, version) ])) references = [ state.db_references_df.at[index, 'url'] for index in state.db_references_df[state.db_references_df.vulnerability_id == state.vulnerability_id].index ] advisory_references = list(state.advisory_references_df[ state.advisory_references_df.vulnerability_id == state.vulnerability_id].url) # allow the user to influence the filtering state.advanced_settings = st.checkbox("Show advanced settings", state.advanced_settings) if state.advanced_settings: # the adding of references can be gone wrong first_commit_timestamp = rank.get_first_commit_timestamp( repo_url ) #@TODO: add a column to the database containing this value first_commit_date, today = datetime.fromtimestamp( int(first_commit_timestamp)).date(), datetime.fromtimestamp( int(time.time())).date() lower_bound = published_date - timedelta( days=730) if published_date - timedelta( days=730) > first_commit_date else first_commit_date upper_bound = published_date + timedelta( days=100) if published_date + timedelta( days=100) < today else today since, until = st.slider("Published date based interval", min_value=first_commit_date, max_value=today, value=(lower_bound, upper_bound)) since, until = int(time.mktime(since.timetuple())), int( time.mktime(until.timetuple())) # references additional_references = st.text_input( "Additional references (separated by commas)") if additional_references: references += additional_references.split(',') vulnerabilities_connection, vulnerabilities_cursor = database.connect_with_vulnerabilities_database( vulnerabilities_db_path) database.add_vulnerability_references_to_database( vulnerabilities_connection, state.vulnerability_id, references, driver=None) selected_references = st.multiselect('Advisory references', tuple(references), default=tuple(references)) # affected versions relevant_tags = st.multiselect( 'Relevant tags', tuple(repository_tags), default=tuple(tags_in_description) if len(tags_in_description) != 0 else None) # st input int k k = st.number_input("The number of results to show", min_value=1, max_value=50, value=10, step=1) else: selected_references = references relevant_tags = tags_in_description since, until = None, None k = 10 # st.write('vulnerability_description:', vulnerability_description) # st.write('references_content:', references_content) # st.write('vulnerability_id:', state.vulnerability_id) # st.write('since - published_timestamp - until:', since, published_timestamp, until) # st.write('repo_url:', repo_url) # # st.write('references:', references) # # st.write('advisory_references:', advisory_references) # st.write('relevant_tags:', relevant_tags) if st.button("Search prospects!"): model, min_max_scaler = load_model() prospector_connection, prospector_cursor = connect_with_commits_database( commits_db_path) preprocessed_description = rank.simpler_filter_text( vulnerability_description) references_content = tuple(state.db_references_df[ (state.db_references_df.vulnerability_id == state.vulnerability_id) & (state.db_references_df.url.isin(selected_references))]. preprocessed_content) references_content = rank.extract_n_most_occurring_words( rank.remove_forbidden_words_from_string( string=' '.join(references_content), forbidden_words=rank.reference_stopwords + project_name.split(' ')), n=20) st.write(references_content) advisory_record = rank.Advisory_record( state.vulnerability_id, published_timestamp, repo_url, selected_references, references_content, advisory_references, vulnerability_description, prospector_connection, preprocessed_vulnerability_description=preprocessed_description, relevant_tags=relevant_tags, verbose=True, since=since, until=until) print( "\nGathering candidate commits and computing ranking vectors.") advisory_record.gather_candidate_commits() advisory_record.compute_ranking_vectors() # scaling some columns using the pretrained scaler, and some vulnerability specific advisory_record.ranking_vectors[ vulnerability_specific_columns] = MinMaxScaler().fit_transform( advisory_record. ranking_vectors[vulnerability_specific_columns]) advisory_record.ranking_vectors[ universal_columns] = min_max_scaler.transform( advisory_record.ranking_vectors[universal_columns]) advisory_record.ranking_vectors.drop(columns=columns_to_drop, inplace=True) advisory_record.ranked_candidate_commits = rank.rank_candidates( model, advisory_record.ranking_vectors) advisory_record.ranking_vectors.set_index('commit_id', inplace=True) output = prospector_main.advisory_record_to_output( advisory_record, model, prospector_cursor, k=k) tmp_download_link = download_link( output, 'Prospector_results-{}.txt'.format(state.vulnerability_id), "Click here to download Prospector's results as a txt file!") st.header("Results") st.markdown(tmp_download_link, unsafe_allow_html=True) st.write( "Showing the top {} candidates from {} candidates considered". format(k, len(advisory_record.ranking_vectors))) st.write(output)
def test_get_tag_for_version_errors(version, error, tags): with pytest.raises(error): filter.get_tag_for_version(tags, version)
def test_get_tag_for_version_errors_2(tags=[]): with pytest.raises(ValueError): filter.get_tag_for_version(tags, 'version-1.8')
def test_get_tag_for_version(version, tag, tags): # returns a list of tags that could be corresponding to the version assert tag in filter.get_tag_for_version(tags, version)