def test_compressed_text_from_url(self, mock_url_call, mock_gzip_read):
mock_gzip_read.return_value = "Hello World"
mock_url_call.return_value = mock.MagicMock(status_code=200,
headers={'content-type':"application/x-gzip"})
resource = Resource("http://simple.com")
data = resource.read()
self.assertEqual("Hello World", data)
def test_compressed_json_from_url(self, mock_url_call, mock_gzip_read):
mock_gzip_read.return_value = "{ \"name\":\"John\", \"age\":31, \"city\":\"New York\" }"
mock_url_call.return_value = mock.MagicMock(status_code=200,
headers={'content-type':"application/x-gzip"})
resource = Resource("http://simple.com")
data = resource.read()
self.assertEqual(json.loads("{ \"name\":\"John\", \"age\":31, \"city\":\"New York\" }"), data)
def test_text_from_url(self, mock_url_call):
mock_url_call.return_value = mock.MagicMock(status_code=200,
headers={'content-type':"text/plain"},
content="Hello World")
resource = Resource("http://simple.com")
data = resource.read()
self.assertEqual("Hello World", data)
def update_cves(self):
cves = []
update_resoure = Resource(self.after_url())
new_data = update_resoure.read()
for cve in new_data:
cves.append(cve['CVE'])
return cves
def test_open_from_directory(self):
test_data = ['files.csv',
'platforms/linux/local/one_cve.py',
'platforms/linux/remote/invalid_cve.py',
'platforms/linux/remote/no_cve.php',
'platforms/linux/remote/two_different_cves.txt',
'platforms/linux/remote/two_different_one_same_cve.txt',
'platforms/linux/remote/two_same_cves.c']
resource = Resource(os.path.join(self.test_data_path, 'exploit-source'))
data = resource.read()
common_prefix = os.path.commonprefix(data)
self.assertTrue(isinstance(data, list))
for filename in data:
self.assertIn(filename.replace(common_prefix, ''), test_data)
def setUp(self):
self.test_data_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', 'test_data')
self.url = 'https://access.redhat.com/labs/securitydataapi/cves.json'
self.mock_data_file = os.path.join(self.test_data_path, 'security_api',
'redhat_security_api.json')
self.mock_resource = Resource(self.mock_data_file)
self.update_url = 'https://access.redhat.com/labs/securitydataapi/cves.json?after=2017-10-17'
self.mock_update_file = os.path.join(
self.test_data_path, 'security_api',
'red_hat_security_api_after_2017_10_17.json')
self.mock_update_resource = Resource(self.mock_update_file)
self.sapi = SecurityApiSource('redhat', self.url)
def __init__(self, **data):
location = data['location']
try:
tlsverify = data['tlsverify']
logger = data['logger']
mbox_resource_connector = data['resource_connector']
Resource.__init__(self, location=location,
tlsverify=tlsverify,
logger=logger,
resource_connector=mbox_resource_connector)
except KeyError:
pass
Document.__init__(self)
self.location = location
self.last_updated = datetime.now()
def __init__(self, *args, **kwargs):
TransformableDict.__init__(self, *args, **kwargs)
try:
location = kwargs.pop('location')
try:
logger = kwargs.pop('logger')
except KeyError:
logger = None
Resource.__init__(self, location, logger=logger, transform_cls=JSONTransformableDictEncoder)
except KeyError:
pass
try:
self.name = kwargs['name']
except KeyError:
pass
def write(self):
if self.location.endswith('.json'):
Resource.write(self, dict(self))
elif self.location.endswith('.html'):
Resource.write(self, self.html())
elif self.location.endswith('.xml'):
Resource.write(self, self.xml())
def exploits(self):
exploits = []
if isinstance(self.data, list):
for location in self.data:
data = Resource(location).read()
exploit = ExploitSource.generate_exploit(
data, location, self.source_name)
exploits.append(exploit)
elif isinstance(self.data, str):
exploits.append(
ExploitSource.generate_exploit(self.data, self.location,
self.source_name))
return exploits
def __init__(self,
location,
cache_location=None,
tlsverify=True,
resource_connector=None,
transform_cls=None):
super(ResourceWithCache,
self).__init__(location, tlsverify, resource_connector,
transform_cls)
if cache_location:
if self.connector_type == MBOX_CONNECTOR:
self.cache = Resource(cache_location,
resource_connector=MBoxResouceConnector(
cache_location, tlsverify=tlsverify))
else:
self.cache = Resource(cache_location)
if not os.path.isdir(self.cache_path):
os.makedirs(self.cache_path)
else:
self.cache = None
def test_init_two(self):
test_cache_file = Resource(self.location)
self.assertEqual(self.location, test_cache_file.location)
self.assertEqual(self.cache_file, test_cache_file.filename)
self.assertEqual(self.cache_path, test_cache_file.filepath)
class TestSecurityApi(unittest.TestCase):
def setUp(self):
self.test_data_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..', 'test_data')
self.url = 'https://access.redhat.com/labs/securitydataapi/cves.json'
self.mock_data_file = os.path.join(self.test_data_path, 'security_api',
'redhat_security_api.json')
self.mock_resource = Resource(self.mock_data_file)
self.update_url = 'https://access.redhat.com/labs/securitydataapi/cves.json?after=2017-10-17'
self.mock_update_file = os.path.join(
self.test_data_path, 'security_api',
'red_hat_security_api_after_2017_10_17.json')
self.mock_update_resource = Resource(self.mock_update_file)
self.sapi = SecurityApiSource('redhat', self.url)
@mock.patch('requests.get')
def test_from_url(self, mock_url_call):
mock_url_call.return_value = mock.MagicMock(
status_code=200,
headers={'content-type': "application/json; charset=utf-8"},
content=self.mock_resource.read())
self.assertEqual(self.url, self.sapi.location)
self.assertEqual('redhat', self.sapi.name)
data = self.sapi.read()
self.assertTrue(isinstance(data, list))
self.assertEqual(self.sapi.latest_date(),
dateutil.parser.parse('10-16-2017').date())
# test cases for not yet implemented functionality in security api source
# @mock.patch('requests.get')
# def test_caching(self, mock_url_call):
# mock_url_call.return_value = mock.MagicMock(status_code=200,
# headers={'content-type':"application/json; charset=utf-8"},
# content=self.mock_resource.read())
# cache_file = os.path.join(self.test_data_path, 'cache/redhat.json')
# self.sapi.configure_cache(cache_file)
# self.assertTrue(self.sapi.cache)
#
# self.assertFalse(os.path.isfile(cache_file))
# self.assertNotEqual(self.url, self.sapi.location)
# self.assertEqual('redhat', self.sapi.name)
# data = self.sapi.read()
# self.assertTrue(os.path.isdir(self.sapi.cache_path))
# self.assertTrue(isinstance(data, list))
# data_from_cache = self.sapi.read()
# self.assertEqual(data, data_from_cache)
#
# @mock.patch('requests.get')
# def test_detele_cache(self, mock_url_call):
# mock_url_call.return_value = mock.MagicMock(status_code=200,
# headers={'content-type':"application/json; charset=utf-8"},
# content=self.mock_resource.read())
# cache_file = os.path.join(self.test_data_path, 'cache/redhat.json')
# self.sapi.configure_cache(cache_file)
#
# self.sapi.read()
# self.assertTrue(os.path.isfile(cache_file))
# self.sapi.delete_cache()
# self.assertFalse(os.path.isfile(cache_file))
@mock.patch('requests.get')
def test_updates(self, mock_url_call):
mock_url_call.side_effect = [
mock.MagicMock(
status_code=200,
headers={'content-type': "application/json; charset=utf-8"},
content=self.mock_resource.read()),
mock.MagicMock(
status_code=200,
headers={'content-type': "application/json; charset=utf-8"},
content=self.mock_update_resource.read())
]
self.assertEqual(self.sapi.after_date(),
dateutil.parser.parse('10-17-2017').date())
self.assertEqual(self.sapi.after_url(), self.update_url)
cves = self.sapi.update_cves()
self.assertEqual(65, len(cves))
@mock.patch('requests.get')
def test_cves(self, mock_url_call):
mock_url_call.return_value = mock.MagicMock(
status_code=200,
headers={'content-type': "application/json; charset=utf-8"},
content=self.mock_resource.read())
cves_to_test = self.sapi.cves()
for cve in cves_to_test:
self.assertEqual(len(cves_to_test[cve].keys()), 4)
self.assertIn('published_date', cves_to_test[cve].keys())
self.assertIn('source_name', cves_to_test[cve].keys())
self.assertIn('source_type', cves_to_test[cve].keys())
self.assertIsInstance(cves_to_test[cve]['source_type'], int)
self.assertEqual(cves_to_test[cve]['source_type'], 0)
def tearDown(self):
cache_file = os.path.join(self.test_data_path, 'cache/redhat.json')
if os.path.isfile(cache_file):
os.remove(cache_file)
def test_open_text_from_file(self):
file_resource = os.path.join(self.test_data_path, 'simple.txt')
resource = Resource(file_resource)
data = resource.read()
self.assertEqual("This is a simple test.\n", data)
class ResourceWithCache(Resource):
def __init__(self,
location,
cache_location=None,
tlsverify=True,
resource_connector=None,
transform_cls=None):
super(ResourceWithCache,
self).__init__(location, tlsverify, resource_connector,
transform_cls)
if cache_location:
if self.connector_type == MBOX_CONNECTOR:
self.cache = Resource(cache_location,
resource_connector=MBoxResouceConnector(
cache_location, tlsverify=tlsverify))
else:
self.cache = Resource(cache_location)
if not os.path.isdir(self.cache_path):
os.makedirs(self.cache_path)
else:
self.cache = None
def configure_cache(self, cachepath):
self.cache = Resource(cachepath)
def delete_cache(self):
FRTLogger.debug("Resource - deleting cache at %s", self.location)
if self.cache:
self.cache.delete()
@property
def location(self):
if self.cache:
return self.cache.location
return self.connector.location
@property
def cache_path(self):
if self.cache:
return self.cache.filepath
return ''
def update(self):
if self.cache:
self.cache.delete()
data = self.connector.open()
if self.cache:
self.cache.write(data)
return data
def read(self):
data = None
if self.cache and self.cache.exists:
FRTLogger.debug("Resource - reading from cache %s",
self.cache.location)
data = self.cache.data
else:
data = self.update()
FRTLogger.debug("Resource - reading from source %s", self.location)
if isinstance(data, str):
try:
return json.loads(data)
except ValueError:
pass
return data
def configure_cache(self, cachepath):
self.cache = Resource(cachepath)
def test_open_json_from_file(self):
file_resource = os.path.join(self.test_data_path, 'simple.json')
resource = Resource(file_resource)
data = resource.read()
self.assertEqual(json.loads("{ \"name\":\"John\", \"age\":31, \"city\":\"New York\" }"), data)
def test_delete(self):
test_cache_file = Resource(self.location)
self.assertTrue(test_cache_file.exists)
test_cache_file.delete()
self.assertFalse(test_cache_file.exists)
def test_exists(self):
test_cache_file = Resource(self.location)
self.assertTrue(test_cache_file.exists)