def put(self, id):
arguments = parser.parse_args()
data = json.loads(arguments['data'])
DatabaseInterface.update_product_by_id(id, data['name'],
data['description'],
data['prices'])
def __init__(self): readxml_obj = XmlReading() self.ip = readxml_obj.get_cluster_ip() self.src_path = readxml_obj.get_src_path() self.dst_path = readxml_obj.get_dst_path() self.tar_type = readxml_obj.get_tar_type() self.database_obj = DatabaseInterface() self.tar_obj = Tar()
def post(self):
arguments = parser.parse_args()
data = json.loads(arguments['data'])
product = Product(name=data['name'],
description=data['description'],
prices=[
Price(pr['amount'], PriceType(pr['price_type']))
for pr in data['prices']
])
DatabaseInterface.add_product(product)
return {'task': arguments['data']}
def setup():
with suppress(OSError):
remove('cve_cpe.db')
sr.QUERIES.update(TEST_QUERIES)
cpe_base = sr.setup_cpe_table(dp.extract_cpe(PATH_TO_TEST + EXTRACT_CPE_XML))
cve_base, summary_base = dp.extract_cve(PATH_TO_TEST + EXTRACT_CVE_JSON)
cve_base = sr.setup_cve_feeds_table(cve_list=cve_base)
summary_base = sr.setup_cve_summary_table(summary_list=summary_base)
with DatabaseInterface(PATH_TO_TEST + 'test_update.db') as db:
db.execute_query(query=QUERIES['create_cpe_table'].format('cpe_table'))
db.insert_rows(query=QUERIES['insert_cpe'].format('cpe_table'), input_data=cpe_base)
db.execute_query(query=QUERIES['create_cve_table'].format('cve_table'))
db.execute_query(query=QUERIES['create_summary_table'].format('summary_table'))
db.insert_rows(query=QUERIES['insert_cve'].format('cve_table'), input_data=cve_base)
db.insert_rows(query=QUERIES['insert_summary'].format('summary_table'), input_data=summary_base)
db.execute_query(query=TEST_QUERIES['test_create_update'].format('outdated'))
db.execute_query(query=TEST_QUERIES['test_create_update'].format('new'))
db.insert_rows(query=TEST_QUERIES['test_insert_cve_id'].format('outdated'), input_data=[('CVE-2018-0001', 2018), ('CVE-2018-0002', 2018)])
db.insert_rows(query=TEST_QUERIES['test_insert_cve_id'].format('new'), input_data=[('CVE-2018-0002', 2018), ('CVE-2018-0003', 2018)])
yield
with suppress(OSError):
remove(PATH_TO_TEST + 'test_update.db')
remove(PATH_TO_TEST + 'test_import.db')
remove(PATH_TO_TEST + 'test_output.db')
def match_cpe(db: DatabaseInterface, product_search_terms: list) -> List[Product]:
return list({
Product(vendor, product, version)
for vendor, product, version in db.fetch_multiple(QUERIES['cpe_lookup'])
for product_term in product_search_terms
if terms_match(product_term, product)
})
def test_insert_functionality():
with DatabaseInterface(TEST_DB_PATH) as db:
db.insert_rows(TEST_QUERIES['test_insert'].format('test_table'),
[(34, )])
test_insert_output = list(
db.fetch_multiple(
query=QUERIES['select_all'].format('test_table')))
assert test_insert_output == [(23, ), (34, )]
def test_get_basket():
basket = BasketModel(reference_number='test_number',
owner_name='owner')
db.session.add(basket)
db.session.commit()
result = DatabaseInterface.get_basket_by_reference_number(
'test_number')
assert isinstance(result, Basket)
def get(self, name):
""" Possible to search with wild card also /products/na*. If star is is missing then the exact
match will be searched """
try:
if name.endswith('*'):
result = [
i for i in DatabaseInterface.search_product_by_substring(
name[:-1])
]
if not result:
abort(404,
message="Product with name {} does not exist".format(
name))
return result
else:
return DatabaseInterface.search_product_by_name(name)
except ProductNotFoundException:
abort(404,
message="Product with name {} does not exist".format(name))
def search_cve_summary(db: DatabaseInterface, product: namedtuple) -> dict:
return {
cve_id: {
'score2': cvss_v2_score,
'score3': cvss_v3_score
}
for cve_id, summary, cvss_v2_score, cvss_v3_score in db.fetch_multiple(
QUERIES['summary_lookup'])
if product_is_mentioned_in_summary(product, summary)
}
def search_cve(db: DatabaseInterface, product: Product) -> dict:
result = {}
for query_result in db.fetch_multiple(QUERIES['cve_lookup']):
cve_entry = CveDbEntry(*query_result)
if _product_matches_cve(product, cve_entry):
result[cve_entry.cve_id] = {
'score2': cve_entry.cvss_v2_score,
'score3': cve_entry.cvss_v3_score,
'cpe_version': build_version_string(cve_entry)
}
return result
def test_execute_query():
with DatabaseInterface(TEST_DB_PATH) as db:
db.execute_query(
query=TEST_QUERIES['test_create'].format('test_table_2'))
assert list(
db.fetch_multiple(
query=QUERIES['exist'].format('test_table_2'))) == [
('test_table_2', )
]
db.execute_query(query=QUERIES['drop'].format('test_table_2'))
assert list(
db.fetch_multiple(
query=QUERIES['exist'].format('test_table_2'))) == []
def look_up_vulnerabilities(product_name: str, requested_version: str) -> Optional[dict]:
with DatabaseInterface() as db:
product_terms, version = replace_characters_and_wildcards(generate_search_terms(product_name)), replace_characters_and_wildcards([requested_version])[0]
matched_cpe = match_cpe(db, product_terms)
if len(matched_cpe) == 0:
logging.debug(f'No CPEs were found for product {product_name}')
return None
try:
matched_product = find_matching_cpe_product(matched_cpe, version)
except IndexError:
return None
cve_candidates = search_cve(db, matched_product)
cve_candidates.update(search_cve_summary(db, matched_product))
return cve_candidates
def test_search_product_by_name(cls):
cls._create_dummy_basket()
result = DatabaseInterface.search_product_by_name('name')
assert result
def delete(self, id):
DatabaseInterface.remove_product_by_id(id)
return {'message': 'Deleted product with id: {}'.format(id)}
class Sender():
def __init__(self):
self.joblist = []
self.upload_obj = Upload()
self.dcider_flag = True
self.database_obj = DatabaseInterface()
# |----------------------------------------------------------------------------|
# job_queue
# |----------------------------------------------------------------------------|
def job_queue(self, slide, metadata):
print('job_queue ', slide)
data = {"slide_name": slide, "metadata": metadata}
self.joblist.append(data)
#print(len(self.joblist))
self.database_obj.log(slide, metadata,
'http://localhost:8025/transfer/', '200',
'transfer')
if self.dcider_flag is True:
self.dcider_flag = False
self.decider()
# |----------------------End of job_queue-------------------------------|
# |----------------------------------------------------------------------------|
# call_restoration_service
# |----------------------------------------------------------------------------|
def call_restoration_service(self, slide_folder_name, slide_metadata):
restoration_service_url = "http://10.20.0.1:8026/restore/{}".\
format(slide_folder_name)
print(restoration_service_url)
print(slide_metadata)
try:
resp = requests.post(url=restoration_service_url,
data=json.dumps(slide_metadata),
timeout=5)
if resp.status_code == 200:
ServiceLogger.get().log_info(
"Request to restore service for "
"slide {} "
"succeeded.".format(slide_folder_name))
else:
ServiceLogger.get().log_info(
"Request to restore service for "
"slide {}"
"failed with response code: {}.".format(
slide_folder_name, resp.status_code))
self.database_obj.log(slide_folder_name, slide_metadata,
restoration_service_url, resp.status_code,
'restore')
except Exception as msg:
print("Request failed due to: ", msg)
ServiceLogger.get().log_error(
"Request to transfer service for slide "
"{} failed due to: {}".format(slide_folder_name, msg))
self.database_obj.log(slide_folder_name, slide_metadata,
restoration_service_url, '500', 'restore')
# |----------------------End of call_restoration_service-------------------------|
# |----------------------------------------------------------------------------|
# decider
# |----------------------------------------------------------------------------|
def decider(self):
if (len(self.joblist) > 0):
slide_name = self.joblist[0]["slide_name"]
metadata = copy.deepcopy(self.joblist[0]["metadata"])
status, error = self.upload_obj.transfer_slide_folder_to_cluster(
slide_name, metadata)
#time.sleep(10)
print(status)
#if status is True or error == 'Yes':
if status is True:
print("----------Finish-------------------")
self.call_restoration_service(slide_name,
self.joblist[0]["metadata"])
self.joblist.pop(0)
self.decider()
else:
self.dcider_flag = True
def test_search_product_by_substring_not_found(cls):
cls._create_dummy_basket()
result = [
p for p in DatabaseInterface.search_product_by_substring('ka')
]
assert not result
from typing import List, Tuple
try:
from ..internal import data_parsing as dp
from ..internal.database_interface import DB_PATH, QUERIES, DatabaseInterface
from ..internal.helper_functions import (CveEntry, CveLookupException,
CveSummaryEntry,
replace_characters_and_wildcards)
except (ImportError, ValueError, SystemError):
sys.path.append(str(Path(__file__).parent.parent / 'internal'))
import data_parsing as dp
from database_interface import DB_PATH, QUERIES, DatabaseInterface
from helper_functions import CveEntry, CveLookupException, CveSummaryEntry, replace_characters_and_wildcards
CURRENT_YEAR = datetime.now().year
DATABASE = DatabaseInterface()
CPE_SPLIT_REGEX = r'(?<![\\:]):(?!:)|(?<=\\:):' # don't split on '::' or '\:' but split on '\::'
Years = namedtuple('Years', 'start_year end_year')
def overlap(requested_years: namedtuple, years_in_cve_database: list) -> list:
set_of_requested_years = set(
range(requested_years.start_year, requested_years.end_year + 1))
return list(set_of_requested_years.difference(set(years_in_cve_database)))
def table_exists(table_name: str) -> bool:
return bool(
list(DATABASE.fetch_multiple(QUERIES['exist'].format(table_name))))
def __init__(self):
self.joblist = []
self.upload_obj = Upload()
self.dcider_flag = True
self.database_obj = DatabaseInterface()
class Upload():
def __init__(self):
readxml_obj = XmlReading()
self.ip = readxml_obj.get_cluster_ip()
self.src_path = readxml_obj.get_src_path()
self.dst_path = readxml_obj.get_dst_path()
self.tar_type = readxml_obj.get_tar_type()
self.database_obj = DatabaseInterface()
self.tar_obj = Tar()
# |----------------------------------------------------------------------------|
# get_folder_size
# |----------------------------------------------------------------------------|
def get_folder_size(self, folder_path):
folder_size_content = subprocess.Popen(["du -sh {}".format(folder_path)],
stdout=subprocess.PIPE, shell=True)
content_list = str(folder_size_content.stdout.read())
content_list = re.findall(r'([-+]?\d*\.\d+|\d+)(\w+)', content_list)
#print("content_list : ", content_list)
first_tuple = content_list[0]
folder_size = float(first_tuple[0])
if "M" in first_tuple:
folder_size = folder_size / 1000
if "K" in first_tuple:
folder_size = folder_size / (1000 * 1000)
print(folder_size)
return folder_size
# |----------------------End of get_folder_size-------------------------------|
# |----------------------------------------------------------------------------|
# update_status_to_node
# |----------------------------------------------------------------------------|
def update_status_to_node(self,slide_name,slide_metadata,activity_status,error_info):
basket_status_url = "http://10.20.0.1:1337/basketslot/status/"
slide_metadata['data'] = {}
slide_metadata['data']['slide_name'] = slide_name
slide_metadata['data']['activity_status'] = activity_status
slide_metadata['data']['error_info'] = error_info
#print(slide_metadata)
try:
resp = requests.post(url=basket_status_url,
data=json.dumps(slide_metadata),
timeout=20)
if resp.status_code == 200:
ServiceLogger.get().log_info("Request to node for "
"folder name {} "
"succeeded.".
format(slide_name))
else:
ServiceLogger.get().log_info("Request to node for "
"folder name {} "
"failed with response code: {}.".
format(slide_name,resp.status_code))
self.database_obj.log(slide_name,slide_metadata,basket_status_url,resp.status_code,'node')
except Exception as msg:
ServiceLogger.get().log_error("Request to node for "
"folder name {} failed due to: {} ".
format( slide_name, msg))
self.database_obj.log(slide_name,slide_metadata,basket_status_url,'500','node')
# |----------------------End of update_status_to_node-------------------------|
# |----------------------------------------------------------------------------|
# tar_slide_folder
# |----------------------------------------------------------------------------|
def tar_slide(self,slide_path,slide_name):
'''
print('slide_path ',slide_path)
os.chdir(slide_path)
tarcmd = "tar -cf {}.tar {}".format(slide_name,slide_name)
print(tarcmd)
status = os.system(tarcmd)
'''
status = 123
if self.tar_type == '1':
status = self.tar_obj.tar_slide_folder(slide_path,slide_name)
elif self.tar_type == '2':
status = self.tar_obj.tar_complete_slide_folder(slide_path,slide_name)
elif self.tar_type == '3':
status = self.tar_obj.tar_grid_folder(slide_path,slide_name)
return status
# |----------------------End of tar_slide_folder-------------------------|
# |----------------------------------------------------------------------------|
# transfer_slide_folder_to_cluster
# |----------------------------------------------------------------------------|
def transfer_slide_folder_to_cluster(self, slide_name,metadata):
move_status = False
err_msg = ''
status = 123
#slide_path =''
try:
slide_path = join(self.src_path,slide_name)
if exists(slide_path) is True:
ServiceLogger.get().log_info("*** CALL for tar ***")
self. update_status_to_node(slide_name,metadata,'transfering','')
tar_status = self.tar_slide(self.src_path,slide_name)
print("Move src_path: ", slide_path, " to: ", self.dst_path)
if tar_status == 0:
command = "sshpass -p 'adminspin#123' rsync -e 'ssh -o \
StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' "\
"-av {} adminspin@{}:{}".format(slide_path, self.ip, self.dst_path)
#print("command: ",command)
ServiceLogger.get().log_info("rsync command: {}".
format(command))
startTime = time.time()
status = os.system(command)
endTime = time.time()
total_time = endTime - startTime
folder_size = self.get_folder_size(slide_path)
print("Total time to move {} of size {}GB is: {}".format(slide_path,
folder_size,
total_time))
ServiceLogger.get().log_info("Total time to move {} of size {}GB is: {}".
format(slide_path,folder_size,total_time))
if status == 0:
move_status = True
rmcmd = "rm -rf {} {}".format(slide_path,slide_path)
#print("rmcmd ",rmcmd)
ServiceLogger.get().log_info("rsync rmcmd: {}".
format(rmcmd))
os.system(rmcmd)
else:
move_status = False
print('Fail to transfer')
ServiceLogger.get().log_error("Fail to transfer {}".
format(slide_name))
self. update_status_to_node(slide_name,metadata,'error','Fail to transfer the data')
else:
self. update_status_to_node(slide_name,metadata,'error','Fail to tar the data')
else:
err_msg ='Yes'
print("Slide does not exits")
ServiceLogger.get().log_error("Slide: {} does not exits".
format(slide_name))
except Exception as msg:
ServiceLogger.get().log_error("exception during transfer slide :{} "
"failed due to: {}".
format(slide_name, msg))
return move_status,err_msg
def get(self):
arguments = parser.parse_args()
return DatabaseInterface.get_all_products(arguments['sort'])
def test_select_functionality():
with DatabaseInterface(TEST_DB_PATH) as db:
assert list(
db.fetch_multiple(
query=QUERIES['select_all'].format('test_table'))) == [(23, )]
def test_db_connection():
with DatabaseInterface(TEST_DB_PATH) as db:
assert db.connection is not None
def test_get_basket_with_correct_content(cls):
cls._create_dummy_basket()
result = DatabaseInterface.get_basket_by_reference_number(
'test_number')
result.get_all_prices()
