def put(self, id): arguments = parser.parse_args() data = json.loads(arguments['data']) DatabaseInterface.update_product_by_id(id, data['name'], data['description'], data['prices'])
def __init__(self): readxml_obj = XmlReading() self.ip = readxml_obj.get_cluster_ip() self.src_path = readxml_obj.get_src_path() self.dst_path = readxml_obj.get_dst_path() self.tar_type = readxml_obj.get_tar_type() self.database_obj = DatabaseInterface() self.tar_obj = Tar()
def post(self): arguments = parser.parse_args() data = json.loads(arguments['data']) product = Product(name=data['name'], description=data['description'], prices=[ Price(pr['amount'], PriceType(pr['price_type'])) for pr in data['prices'] ]) DatabaseInterface.add_product(product) return {'task': arguments['data']}
def setup(): with suppress(OSError): remove('cve_cpe.db') sr.QUERIES.update(TEST_QUERIES) cpe_base = sr.setup_cpe_table(dp.extract_cpe(PATH_TO_TEST + EXTRACT_CPE_XML)) cve_base, summary_base = dp.extract_cve(PATH_TO_TEST + EXTRACT_CVE_JSON) cve_base = sr.setup_cve_feeds_table(cve_list=cve_base) summary_base = sr.setup_cve_summary_table(summary_list=summary_base) with DatabaseInterface(PATH_TO_TEST + 'test_update.db') as db: db.execute_query(query=QUERIES['create_cpe_table'].format('cpe_table')) db.insert_rows(query=QUERIES['insert_cpe'].format('cpe_table'), input_data=cpe_base) db.execute_query(query=QUERIES['create_cve_table'].format('cve_table')) db.execute_query(query=QUERIES['create_summary_table'].format('summary_table')) db.insert_rows(query=QUERIES['insert_cve'].format('cve_table'), input_data=cve_base) db.insert_rows(query=QUERIES['insert_summary'].format('summary_table'), input_data=summary_base) db.execute_query(query=TEST_QUERIES['test_create_update'].format('outdated')) db.execute_query(query=TEST_QUERIES['test_create_update'].format('new')) db.insert_rows(query=TEST_QUERIES['test_insert_cve_id'].format('outdated'), input_data=[('CVE-2018-0001', 2018), ('CVE-2018-0002', 2018)]) db.insert_rows(query=TEST_QUERIES['test_insert_cve_id'].format('new'), input_data=[('CVE-2018-0002', 2018), ('CVE-2018-0003', 2018)]) yield with suppress(OSError): remove(PATH_TO_TEST + 'test_update.db') remove(PATH_TO_TEST + 'test_import.db') remove(PATH_TO_TEST + 'test_output.db')
def match_cpe(db: DatabaseInterface, product_search_terms: list) -> List[Product]: return list({ Product(vendor, product, version) for vendor, product, version in db.fetch_multiple(QUERIES['cpe_lookup']) for product_term in product_search_terms if terms_match(product_term, product) })
def test_insert_functionality(): with DatabaseInterface(TEST_DB_PATH) as db: db.insert_rows(TEST_QUERIES['test_insert'].format('test_table'), [(34, )]) test_insert_output = list( db.fetch_multiple( query=QUERIES['select_all'].format('test_table'))) assert test_insert_output == [(23, ), (34, )]
def test_get_basket(): basket = BasketModel(reference_number='test_number', owner_name='owner') db.session.add(basket) db.session.commit() result = DatabaseInterface.get_basket_by_reference_number( 'test_number') assert isinstance(result, Basket)
def get(self, name): """ Possible to search with wild card also /products/na*. If star is is missing then the exact match will be searched """ try: if name.endswith('*'): result = [ i for i in DatabaseInterface.search_product_by_substring( name[:-1]) ] if not result: abort(404, message="Product with name {} does not exist".format( name)) return result else: return DatabaseInterface.search_product_by_name(name) except ProductNotFoundException: abort(404, message="Product with name {} does not exist".format(name))
def search_cve_summary(db: DatabaseInterface, product: namedtuple) -> dict: return { cve_id: { 'score2': cvss_v2_score, 'score3': cvss_v3_score } for cve_id, summary, cvss_v2_score, cvss_v3_score in db.fetch_multiple( QUERIES['summary_lookup']) if product_is_mentioned_in_summary(product, summary) }
def search_cve(db: DatabaseInterface, product: Product) -> dict: result = {} for query_result in db.fetch_multiple(QUERIES['cve_lookup']): cve_entry = CveDbEntry(*query_result) if _product_matches_cve(product, cve_entry): result[cve_entry.cve_id] = { 'score2': cve_entry.cvss_v2_score, 'score3': cve_entry.cvss_v3_score, 'cpe_version': build_version_string(cve_entry) } return result
def test_execute_query(): with DatabaseInterface(TEST_DB_PATH) as db: db.execute_query( query=TEST_QUERIES['test_create'].format('test_table_2')) assert list( db.fetch_multiple( query=QUERIES['exist'].format('test_table_2'))) == [ ('test_table_2', ) ] db.execute_query(query=QUERIES['drop'].format('test_table_2')) assert list( db.fetch_multiple( query=QUERIES['exist'].format('test_table_2'))) == []
def look_up_vulnerabilities(product_name: str, requested_version: str) -> Optional[dict]: with DatabaseInterface() as db: product_terms, version = replace_characters_and_wildcards(generate_search_terms(product_name)), replace_characters_and_wildcards([requested_version])[0] matched_cpe = match_cpe(db, product_terms) if len(matched_cpe) == 0: logging.debug(f'No CPEs were found for product {product_name}') return None try: matched_product = find_matching_cpe_product(matched_cpe, version) except IndexError: return None cve_candidates = search_cve(db, matched_product) cve_candidates.update(search_cve_summary(db, matched_product)) return cve_candidates
def test_search_product_by_name(cls): cls._create_dummy_basket() result = DatabaseInterface.search_product_by_name('name') assert result
def delete(self, id): DatabaseInterface.remove_product_by_id(id) return {'message': 'Deleted product with id: {}'.format(id)}
class Sender(): def __init__(self): self.joblist = [] self.upload_obj = Upload() self.dcider_flag = True self.database_obj = DatabaseInterface() # |----------------------------------------------------------------------------| # job_queue # |----------------------------------------------------------------------------| def job_queue(self, slide, metadata): print('job_queue ', slide) data = {"slide_name": slide, "metadata": metadata} self.joblist.append(data) #print(len(self.joblist)) self.database_obj.log(slide, metadata, 'http://localhost:8025/transfer/', '200', 'transfer') if self.dcider_flag is True: self.dcider_flag = False self.decider() # |----------------------End of job_queue-------------------------------| # |----------------------------------------------------------------------------| # call_restoration_service # |----------------------------------------------------------------------------| def call_restoration_service(self, slide_folder_name, slide_metadata): restoration_service_url = "http://10.20.0.1:8026/restore/{}".\ format(slide_folder_name) print(restoration_service_url) print(slide_metadata) try: resp = requests.post(url=restoration_service_url, data=json.dumps(slide_metadata), timeout=5) if resp.status_code == 200: ServiceLogger.get().log_info( "Request to restore service for " "slide {} " "succeeded.".format(slide_folder_name)) else: ServiceLogger.get().log_info( "Request to restore service for " "slide {}" "failed with response code: {}.".format( slide_folder_name, resp.status_code)) self.database_obj.log(slide_folder_name, slide_metadata, restoration_service_url, resp.status_code, 'restore') except Exception as msg: print("Request failed due to: ", msg) ServiceLogger.get().log_error( "Request to transfer service for slide " "{} failed due to: {}".format(slide_folder_name, msg)) self.database_obj.log(slide_folder_name, slide_metadata, restoration_service_url, '500', 'restore') # |----------------------End of call_restoration_service-------------------------| # |----------------------------------------------------------------------------| # decider # |----------------------------------------------------------------------------| def decider(self): if (len(self.joblist) > 0): slide_name = self.joblist[0]["slide_name"] metadata = copy.deepcopy(self.joblist[0]["metadata"]) status, error = self.upload_obj.transfer_slide_folder_to_cluster( slide_name, metadata) #time.sleep(10) print(status) #if status is True or error == 'Yes': if status is True: print("----------Finish-------------------") self.call_restoration_service(slide_name, self.joblist[0]["metadata"]) self.joblist.pop(0) self.decider() else: self.dcider_flag = True
def test_search_product_by_substring_not_found(cls): cls._create_dummy_basket() result = [ p for p in DatabaseInterface.search_product_by_substring('ka') ] assert not result
from typing import List, Tuple try: from ..internal import data_parsing as dp from ..internal.database_interface import DB_PATH, QUERIES, DatabaseInterface from ..internal.helper_functions import (CveEntry, CveLookupException, CveSummaryEntry, replace_characters_and_wildcards) except (ImportError, ValueError, SystemError): sys.path.append(str(Path(__file__).parent.parent / 'internal')) import data_parsing as dp from database_interface import DB_PATH, QUERIES, DatabaseInterface from helper_functions import CveEntry, CveLookupException, CveSummaryEntry, replace_characters_and_wildcards CURRENT_YEAR = datetime.now().year DATABASE = DatabaseInterface() CPE_SPLIT_REGEX = r'(?<![\\:]):(?!:)|(?<=\\:):' # don't split on '::' or '\:' but split on '\::' Years = namedtuple('Years', 'start_year end_year') def overlap(requested_years: namedtuple, years_in_cve_database: list) -> list: set_of_requested_years = set( range(requested_years.start_year, requested_years.end_year + 1)) return list(set_of_requested_years.difference(set(years_in_cve_database))) def table_exists(table_name: str) -> bool: return bool( list(DATABASE.fetch_multiple(QUERIES['exist'].format(table_name))))
def __init__(self): self.joblist = [] self.upload_obj = Upload() self.dcider_flag = True self.database_obj = DatabaseInterface()
class Upload(): def __init__(self): readxml_obj = XmlReading() self.ip = readxml_obj.get_cluster_ip() self.src_path = readxml_obj.get_src_path() self.dst_path = readxml_obj.get_dst_path() self.tar_type = readxml_obj.get_tar_type() self.database_obj = DatabaseInterface() self.tar_obj = Tar() # |----------------------------------------------------------------------------| # get_folder_size # |----------------------------------------------------------------------------| def get_folder_size(self, folder_path): folder_size_content = subprocess.Popen(["du -sh {}".format(folder_path)], stdout=subprocess.PIPE, shell=True) content_list = str(folder_size_content.stdout.read()) content_list = re.findall(r'([-+]?\d*\.\d+|\d+)(\w+)', content_list) #print("content_list : ", content_list) first_tuple = content_list[0] folder_size = float(first_tuple[0]) if "M" in first_tuple: folder_size = folder_size / 1000 if "K" in first_tuple: folder_size = folder_size / (1000 * 1000) print(folder_size) return folder_size # |----------------------End of get_folder_size-------------------------------| # |----------------------------------------------------------------------------| # update_status_to_node # |----------------------------------------------------------------------------| def update_status_to_node(self,slide_name,slide_metadata,activity_status,error_info): basket_status_url = "http://10.20.0.1:1337/basketslot/status/" slide_metadata['data'] = {} slide_metadata['data']['slide_name'] = slide_name slide_metadata['data']['activity_status'] = activity_status slide_metadata['data']['error_info'] = error_info #print(slide_metadata) try: resp = requests.post(url=basket_status_url, data=json.dumps(slide_metadata), timeout=20) if resp.status_code == 200: ServiceLogger.get().log_info("Request to node for " "folder name {} " "succeeded.". format(slide_name)) else: ServiceLogger.get().log_info("Request to node for " "folder name {} " "failed with response code: {}.". format(slide_name,resp.status_code)) self.database_obj.log(slide_name,slide_metadata,basket_status_url,resp.status_code,'node') except Exception as msg: ServiceLogger.get().log_error("Request to node for " "folder name {} failed due to: {} ". format( slide_name, msg)) self.database_obj.log(slide_name,slide_metadata,basket_status_url,'500','node') # |----------------------End of update_status_to_node-------------------------| # |----------------------------------------------------------------------------| # tar_slide_folder # |----------------------------------------------------------------------------| def tar_slide(self,slide_path,slide_name): ''' print('slide_path ',slide_path) os.chdir(slide_path) tarcmd = "tar -cf {}.tar {}".format(slide_name,slide_name) print(tarcmd) status = os.system(tarcmd) ''' status = 123 if self.tar_type == '1': status = self.tar_obj.tar_slide_folder(slide_path,slide_name) elif self.tar_type == '2': status = self.tar_obj.tar_complete_slide_folder(slide_path,slide_name) elif self.tar_type == '3': status = self.tar_obj.tar_grid_folder(slide_path,slide_name) return status # |----------------------End of tar_slide_folder-------------------------| # |----------------------------------------------------------------------------| # transfer_slide_folder_to_cluster # |----------------------------------------------------------------------------| def transfer_slide_folder_to_cluster(self, slide_name,metadata): move_status = False err_msg = '' status = 123 #slide_path ='' try: slide_path = join(self.src_path,slide_name) if exists(slide_path) is True: ServiceLogger.get().log_info("*** CALL for tar ***") self. update_status_to_node(slide_name,metadata,'transfering','') tar_status = self.tar_slide(self.src_path,slide_name) print("Move src_path: ", slide_path, " to: ", self.dst_path) if tar_status == 0: command = "sshpass -p 'adminspin#123' rsync -e 'ssh -o \ StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' "\ "-av {} adminspin@{}:{}".format(slide_path, self.ip, self.dst_path) #print("command: ",command) ServiceLogger.get().log_info("rsync command: {}". format(command)) startTime = time.time() status = os.system(command) endTime = time.time() total_time = endTime - startTime folder_size = self.get_folder_size(slide_path) print("Total time to move {} of size {}GB is: {}".format(slide_path, folder_size, total_time)) ServiceLogger.get().log_info("Total time to move {} of size {}GB is: {}". format(slide_path,folder_size,total_time)) if status == 0: move_status = True rmcmd = "rm -rf {} {}".format(slide_path,slide_path) #print("rmcmd ",rmcmd) ServiceLogger.get().log_info("rsync rmcmd: {}". format(rmcmd)) os.system(rmcmd) else: move_status = False print('Fail to transfer') ServiceLogger.get().log_error("Fail to transfer {}". format(slide_name)) self. update_status_to_node(slide_name,metadata,'error','Fail to transfer the data') else: self. update_status_to_node(slide_name,metadata,'error','Fail to tar the data') else: err_msg ='Yes' print("Slide does not exits") ServiceLogger.get().log_error("Slide: {} does not exits". format(slide_name)) except Exception as msg: ServiceLogger.get().log_error("exception during transfer slide :{} " "failed due to: {}". format(slide_name, msg)) return move_status,err_msg
def get(self): arguments = parser.parse_args() return DatabaseInterface.get_all_products(arguments['sort'])
def test_select_functionality(): with DatabaseInterface(TEST_DB_PATH) as db: assert list( db.fetch_multiple( query=QUERIES['select_all'].format('test_table'))) == [(23, )]
def test_db_connection(): with DatabaseInterface(TEST_DB_PATH) as db: assert db.connection is not None
def test_get_basket_with_correct_content(cls): cls._create_dummy_basket() result = DatabaseInterface.get_basket_by_reference_number( 'test_number') result.get_all_prices()