Esempio n. 1
0
    def test_url(self):
        # main storage not instantiated with a base url so should raise not
        # implemented.
        storage = InMemoryStorage()

        self.assertRaises(ValueError, storage.url, ('file0', ))

        storage = InMemoryStorage(base_url='http://www.example.com')

        self.assertEqual(storage.url('file0'), 'http://www.example.com/file0')
Esempio n. 2
0
    def test_url(self):
        # main storage not instantiated with a base url so should raise not
        # implemented.
        storage = InMemoryStorage()

        self.assertRaises(ValueError, storage.url, ('file0',))

        storage = InMemoryStorage(base_url='http://www.example.com')

        self.assertEqual(storage.url('file0'), 'http://www.example.com/file0')
Esempio n. 3
0
    def __init__(self):
        self.cache = InMemoryStorage()
        self.start_year = SETTINGS.get("start_year", 2002)
        self.sources_settings = SETTINGS.get("sources", {})
        self.queue_settings = SETTINGS.get("queue", {})

        self.cve_base = self.sources_settings.get(
            "cve_base", "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-")
        self.cve_base_postfix = self.sources_settings.get(
            "cve_base_postfix", ".json.gz")
        self.channel_to_publish = self.queue_settings.get(
            "channel", "start_processing")
        self.cve_modified = self.sources_settings.get(
            "cve_modified",
            "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.gz"
        )
        self.cve_recent = self.sources_settings.get(
            "cve_recent",
            "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.gz"
        )
        self.new_queue = self.queue_settings.get(
            "new_queue", "VulnerabilityManagement-CVE:new")
        self.modified_queue = self.queue_settings.get(
            "modified_queue", "VulnerabilityManagement-CVE:modified")
Esempio n. 4
0
    def test_no_persistance_without_setting(self):
        storage_a = InMemoryStorage()
        storage_b = InMemoryStorage()

        # Can't use self.assertIs because it isn't available in Python 2.6
        self.assertTrue(storage_a.filesystem is not storage_b.filesystem)
Esempio n. 5
0
 def setUp(self):
     if not settings.configured:
         settings.configure(MEDIA_URL='')
     self.storage = InMemoryStorage()
     self.filesystem = self.storage.filesystem
Esempio n. 6
0
class MemoryStorageTests(unittest.TestCase):
    def setUp(self):
        if not settings.configured:
            settings.configure(MEDIA_URL='')
        self.storage = InMemoryStorage()
        self.filesystem = self.storage.filesystem

    def test_listdir(self):
        self.assertEqual(self.storage.listdir(''), [[], []])

        self.filesystem.add_child('dir0', InMemoryDir())
        self.filesystem.add_child('file0', InMemoryFile())

        self.assertEqual(self.storage.listdir(''), [['dir0'], ['file0']])
        self.assertEqual(self.storage.listdir('dir0'), [[], []])

        self.filesystem.resolve('dir0').add_child('subdir', InMemoryDir())
        self.assertEqual(self.storage.listdir('dir0'), [['subdir'], []])
        self.assertEqual(self.storage.listdir('dir0/subdir'), [[], []])

    def test_delete(self):
        self.filesystem.add_child('dir0', InMemoryDir())
        self.filesystem.resolve('dir0').add_child('nested_file',
                                                  InMemoryFile())
        self.filesystem.add_child('file0', InMemoryFile())
        self.assertEqual(self.filesystem.resolve('dir0').ls(), ['nested_file'])

        self.storage.delete('dir0/nested_file')
        self.assertEqual(self.filesystem.resolve('dir0').ls(), [])
        self.assertEqual(set(self.filesystem.ls()), set(['dir0', 'file0']))

        self.storage.delete('dir0')
        self.assertEqual(set(self.filesystem.ls()), set(['file0']))

    def test_exists(self):
        self.filesystem.add_child('file0', InMemoryFile())
        self.assertTrue(self.storage.exists('file0'))
        self.assertFalse(self.storage.exists('file1'))
        self.storage.delete('file0')
        self.assertFalse(self.storage.exists('file0'))

    def test_size(self):
        self.filesystem.add_child('file0', InMemoryFile('test'))
        self.assertEqual(self.storage.size('file0'), 4)

    def test_save(self):
        self.storage.save('file', ContentFile('test'))
        self.assertEqual(self.storage.size('file'), 4)
        self.storage.save('subdir/file', ContentFile('test'))
        self.assertEqual(self.storage.size('subdir/file'), 4)
        self.storage.save('bytes_file', ContentFile(b'test'))
        self.assertEqual(self.storage.size('bytes_file'), 4)

    def test_write(self):
        with self.storage.open("file", "w") as f:
            f.write("hello")
        with self.storage.open("file", "r") as f:
            self.assertEqual(f.read(), "hello")

    def test_missing(self):
        missing_file_name = "missing-file"
        with self.assertRaises(PathDoesNotExist) as context_manager:
            self.storage.open(missing_file_name, "r")
            self.assertEqual(context_manager.exception.args,
                             (missing_file_name, ))

    def test_all(self):
        self.assertEqual(self.storage.listdir('/'), [[], []])
        self.assertEqual(
            self.storage.save('dir/subdir/file', ContentFile('testing')),
            'dir/subdir/file')
        self.assertEqual(self.storage.listdir('/'), [['dir'], []])
        self.assertEqual(self.storage.listdir('dir/'), [['subdir'], []])
        self.assertEqual(
            self.storage.save('dir/subdir/file2', ContentFile('testing2')),
            'dir/subdir/file2')
        self.assertEqual(self.storage.save('file', ContentFile('testing3')),
                         'file')
        self.assertEqual(self.storage.listdir('/'), [['dir'], ['file']])
        self.assertEqual(self.storage.listdir('dir/'), [['subdir'], []])
        self.assertEqual(
            self.storage.open('dir/subdir/file').read(), 'testing')
        self.assertEqual(self.storage.size('dir/subdir/file'), 7)
        self.assertEqual(self.storage.size('dir/subdir/file2'), 8)
        self.assertEqual(self.storage.size('file'), 8)
        self.assertEqual(self.storage.delete('file'), None)
        self.assertEqual(self.storage.listdir('/'), [['dir'], []])
        self.assertEqual(self.storage.delete('dir/subdir/file'), None)
        self.assertEqual(self.storage.listdir('dir/subdir'), [[], ['file2']])
        self.assertEqual(self.storage.exists('dir/subdir/file2'), True)
        self.assertEqual(self.storage.delete('dir/subdir/file2'), None)
        self.assertEqual(self.storage.exists('dir/subdir/file2'), False)
        self.assertEqual(self.storage.listdir('dir/subdir'), [[], []])

    @override_settings(MEDIA_URL=None)
    def test_url(self):
        # main storage not instantiated with a base url so should raise not
        # implemented.
        storage = InMemoryStorage()

        self.assertRaises(ValueError, storage.url, ('file0', ))

        storage = InMemoryStorage(base_url='http://www.example.com')

        self.assertEqual(storage.url('file0'), 'http://www.example.com/file0')

    def test_modified_time(self):
        self.storage.save('file', ContentFile('test'))
        modified_time = self.storage.modified_time('file')

        self.storage.delete('file')

        time.sleep(0.1)

        self.storage.save('file', ContentFile('test-again'))
        new_modified_time = self.storage.modified_time('file')
        self.assertTrue(new_modified_time > modified_time)

    def test_accessed_time(self):
        self.storage.save('file', ContentFile('test'))

        self.storage.open('file', 'r')
        accessed_time = self.storage.accessed_time('file')

        time.sleep(0.1)

        self.storage.open('file', 'r')
        new_accessed_time = self.storage.accessed_time('file')
        self.assertTrue(new_accessed_time > accessed_time)

    def test_created_time(self):
        self.storage.save('file', ContentFile('test'))
        created_time = self.storage.created_time('file')

        time.sleep(0.1)

        # make sure opening it doesn't change creation time.
        file = self.storage.open('file', 'r')

        after_open_created_time = self.storage.created_time('file')
        self.assertEqual(after_open_created_time, created_time)

        # make sure writing to it doesn't change creation time.
        file.write('test-test-test')
        self.storage.save('file', file)

        after_write_created_time = self.storage.created_time('file')
        self.assertEqual(after_write_created_time, created_time)

    @override_settings(INMEMORYSTORAGE_PERSIST=True)
    def test_persistance(self):
        storage_a = InMemoryStorage()
        storage_b = InMemoryStorage()

        # Can't use self.assertIs because it isn't available in Python 2.6
        self.assertTrue(storage_a.filesystem is storage_b.filesystem)

    def test_no_persistance_without_setting(self):
        storage_a = InMemoryStorage()
        storage_b = InMemoryStorage()

        # Can't use self.assertIs because it isn't available in Python 2.6
        self.assertTrue(storage_a.filesystem is not storage_b.filesystem)
Esempio n. 7
0
 def setUp(self):
     self.storage = InMemoryStorage()
     self.filesystem = self.storage.filesystem
Esempio n. 8
0
class MemoryStorageTests(unittest.TestCase):
    def setUp(self):
        self.storage = InMemoryStorage()
        self.filesystem = self.storage.filesystem

    def test_listdir(self):
        self.assertEqual(self.storage.listdir(''), [[], []])

        self.filesystem.add_child('dir0', InMemoryDir())
        self.filesystem.add_child('file0', InMemoryFile())

        self.assertEqual(self.storage.listdir(''), [['dir0'], ['file0']])
        self.assertEqual(self.storage.listdir('dir0'), [[], []])

        self.filesystem.resolve('dir0').add_child('subdir', InMemoryDir())
        self.assertEqual(self.storage.listdir('dir0'), [['subdir'], []])

    def test_delete(self):
        self.filesystem.add_child('dir0', InMemoryDir())
        self.filesystem.resolve('dir0').add_child('nested_file',
                                                  InMemoryFile())
        self.filesystem.add_child('file0', InMemoryFile())
        self.assertEqual(self.filesystem.resolve('dir0').ls(), ['nested_file'])

        self.storage.delete('dir0/nested_file')
        self.assertEqual(self.filesystem.resolve('dir0').ls(), [])
        self.assertEqual(set(self.filesystem.ls()), set(['dir0', 'file0']))

        self.storage.delete('dir0')
        self.assertEqual(set(self.filesystem.ls()), set(['file0']))

    def test_exists(self):
        self.filesystem.add_child('file0', InMemoryFile())
        self.assertTrue(self.storage.exists('file0'))
        self.assertFalse(self.storage.exists('file1'))
        self.storage.delete('file0')
        self.assertFalse(self.storage.exists('file0'))

    def test_size(self):
        self.filesystem.add_child('file0', InMemoryFile('test'))
        self.assertEqual(self.storage.size('file0'), 4)

    def test_save(self):
        self.storage.save('file', ContentFile('test'))
        self.assertEqual(self.storage.size('file'), 4)
        self.storage.save('subdir/file', ContentFile('test'))
        self.assertEqual(self.storage.size('subdir/file'), 4)

    def test_all(self):
        self.assertEqual(self.storage.listdir('/'), [[], []])
        self.assertEqual(
            self.storage.save('dir/subdir/file', ContentFile('testing')),
            'dir/subdir/file')
        self.assertEqual(self.storage.listdir('/'), [['dir'], []])
        self.assertEqual(self.storage.listdir('dir/'), [['subdir'], []])
        self.assertEqual(
            self.storage.save('dir/subdir/file2', ContentFile('testing2')),
            'dir/subdir/file2')
        self.assertEqual(self.storage.save('file', ContentFile('testing3')),
                         'file')
        self.assertEqual(self.storage.listdir('/'), [['dir'], ['file']])
        self.assertEqual(self.storage.listdir('dir/'), [['subdir'], []])
        self.assertEqual(
            self.storage.open('dir/subdir/file').read(), 'testing')
        self.assertEqual(self.storage.size('dir/subdir/file'), 7)
        self.assertEqual(self.storage.size('dir/subdir/file2'), 8)
        self.assertEqual(self.storage.size('file'), 8)
        self.assertEqual(self.storage.delete('file'), None)
        self.assertEqual(self.storage.listdir('/'), [['dir'], []])
        self.assertEqual(self.storage.delete('dir/subdir/file'), None)
        self.assertEqual(self.storage.listdir('dir/subdir'), [[], ['file2']])
        self.assertEqual(self.storage.exists('dir/subdir/file2'), True)
        self.assertEqual(self.storage.delete('dir/subdir/file2'), None)
        self.assertEqual(self.storage.exists('dir/subdir/file2'), False)
        self.assertEqual(self.storage.listdir('dir/subdir'), [[], []])
Esempio n. 9
0
class MemoryStorageTests(unittest.TestCase):
    def setUp(self):
        if not settings.configured:
            settings.configure(MEDIA_URL='')
        self.storage = InMemoryStorage()
        self.filesystem = self.storage.filesystem

    def test_listdir(self):
        self.assertEqual(self.storage.listdir(''), [[], []])

        self.filesystem.add_child('dir0', InMemoryDir())
        self.filesystem.add_child('file0', InMemoryFile())

        self.assertEqual(self.storage.listdir(''), [['dir0'], ['file0']])
        self.assertEqual(self.storage.listdir('dir0'), [[], []])

        self.filesystem.resolve('dir0').add_child('subdir', InMemoryDir())
        self.assertEqual(self.storage.listdir('dir0'), [['subdir'], []])
        self.assertEqual(self.storage.listdir('dir0/subdir'), [[], []])

    def test_delete(self):
        self.filesystem.add_child('dir0', InMemoryDir())
        self.filesystem.resolve('dir0').add_child('nested_file', InMemoryFile())
        self.filesystem.add_child('file0', InMemoryFile())
        self.assertEqual(self.filesystem.resolve('dir0').ls(), ['nested_file'])

        self.storage.delete('dir0/nested_file')
        self.assertEqual(self.filesystem.resolve('dir0').ls(), [])
        self.assertEqual(set(self.filesystem.ls()), set(['dir0', 'file0']))

        self.storage.delete('dir0')
        self.assertEqual(set(self.filesystem.ls()), set(['file0']))

    def test_exists(self):
        self.filesystem.add_child('file0', InMemoryFile())
        self.assertTrue(self.storage.exists('file0'))
        self.assertFalse(self.storage.exists('file1'))
        self.storage.delete('file0')
        self.assertFalse(self.storage.exists('file0'))

    def test_size(self):
        self.filesystem.add_child('file0', InMemoryFile('test'))
        self.assertEqual(self.storage.size('file0'), 4)

    def test_save(self):
        self.storage.save('file', ContentFile('test'))
        self.assertEqual(self.storage.size('file'), 4)
        self.storage.save('subdir/file', ContentFile('test'))
        self.assertEqual(self.storage.size('subdir/file'), 4)
        self.storage.save('bytes_file', ContentFile(b'test'))
        self.assertEqual(self.storage.size('bytes_file'), 4)

    def test_write(self):
        with self.storage.open("file", "w") as f:
            f.write("hello")
        with self.storage.open("file", "r") as f:
            self.assertEqual(f.read(), "hello")

    def test_all(self):
        self.assertEqual(self.storage.listdir('/'), [[], []])
        self.assertEqual(self.storage.save('dir/subdir/file', ContentFile('testing')), 'dir/subdir/file')
        self.assertEqual(self.storage.listdir('/'), [['dir'], []])
        self.assertEqual(self.storage.listdir('dir/'), [['subdir'], []])
        self.assertEqual(self.storage.save('dir/subdir/file2', ContentFile('testing2')), 'dir/subdir/file2')
        self.assertEqual(self.storage.save('file', ContentFile('testing3')), 'file')
        self.assertEqual(self.storage.listdir('/'), [['dir'], ['file']])
        self.assertEqual(self.storage.listdir('dir/'), [['subdir'], []])
        self.assertEqual(self.storage.open('dir/subdir/file').read(), 'testing')
        self.assertEqual(self.storage.size('dir/subdir/file'), 7)
        self.assertEqual(self.storage.size('dir/subdir/file2'), 8)
        self.assertEqual(self.storage.size('file'), 8)
        self.assertEqual(self.storage.delete('file'), None)
        self.assertEqual(self.storage.listdir('/'), [['dir'], []])
        self.assertEqual(self.storage.delete('dir/subdir/file'), None)
        self.assertEqual(self.storage.listdir('dir/subdir'), [[], ['file2']])
        self.assertEqual(self.storage.exists('dir/subdir/file2'), True)
        self.assertEqual(self.storage.delete('dir/subdir/file2'), None)
        self.assertEqual(self.storage.exists('dir/subdir/file2'), False)
        self.assertEqual(self.storage.listdir('dir/subdir'), [[], []])

    @override_settings(MEDIA_URL=None)
    def test_url(self):
        # main storage not instantiated with a base url so should raise not
        # implemented.
        storage = InMemoryStorage()

        self.assertRaises(ValueError, storage.url, ('file0',))

        storage = InMemoryStorage(base_url='http://www.example.com')

        self.assertEqual(storage.url('file0'), 'http://www.example.com/file0')

    def test_modified_time(self):
        self.storage.save('file', ContentFile('test'))
        modified_time = self.storage.modified_time('file')

        self.storage.delete('file')

        time.sleep(0.1)

        self.storage.save('file', ContentFile('test-again'))
        new_modified_time = self.storage.modified_time('file')
        self.assertTrue(new_modified_time > modified_time)

    def test_accessed_time(self):
        self.storage.save('file', ContentFile('test'))

        self.storage.open('file', 'r')
        accessed_time = self.storage.accessed_time('file')

        time.sleep(0.1)

        self.storage.open('file', 'r')
        new_accessed_time = self.storage.accessed_time('file')
        self.assertTrue(new_accessed_time > accessed_time)

    def test_created_time(self):
        self.storage.save('file', ContentFile('test'))
        created_time = self.storage.created_time('file')

        time.sleep(0.1)

        # make sure opening it doesn't change creation time.
        file = self.storage.open('file', 'r')

        after_open_created_time = self.storage.created_time('file')
        self.assertEqual(after_open_created_time, created_time)

        # make sure writing to it doesn't change creation time.
        file.write('test-test-test')
        self.storage.save('file', file)

        after_write_created_time = self.storage.created_time('file')
        self.assertEqual(after_write_created_time, created_time)

    @override_settings(INMEMORYSTORAGE_PERSIST=True)
    def test_persistance(self):
        storage_a = InMemoryStorage()
        storage_b = InMemoryStorage()

        # Can't use self.assertIs because it isn't available in Python 2.6
        self.assertTrue(storage_a.filesystem is storage_b.filesystem)

    def test_no_persistance_without_setting(self):
        storage_a = InMemoryStorage()
        storage_b = InMemoryStorage()

        # Can't use self.assertIs because it isn't available in Python 2.6
        self.assertTrue(storage_a.filesystem is not storage_b.filesystem)
Esempio n. 10
0
 def setUp(self):
     if not settings.configured:
         settings.configure(MEDIA_URL='')
     self.storage = InMemoryStorage()
     self.filesystem = self.storage.filesystem
Esempio n. 11
0
class Updater(object):
    def __init__(self):
        self.cache = InMemoryStorage()
        self.start_year = SETTINGS.get("start_year", 2002)
        self.sources_settings = SETTINGS.get("sources", {})
        self.queue_settings = SETTINGS.get("queue", {})

        self.cve_base = self.sources_settings.get(
            "cve_base", "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-")
        self.cve_base_postfix = self.sources_settings.get(
            "cve_base_postfix", ".json.gz")
        self.channel_to_publish = self.queue_settings.get(
            "channel", "start_processing")
        self.cve_modified = self.sources_settings.get(
            "cve_modified",
            "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.gz"
        )
        self.cve_recent = self.sources_settings.get(
            "cve_recent",
            "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.gz"
        )
        self.new_queue = self.queue_settings.get(
            "new_queue", "VulnerabilityManagement-CVE:new")
        self.modified_queue = self.queue_settings.get(
            "modified_queue", "VulnerabilityManagement-CVE:modified")

    #
    # Files
    #

    def download_cve_file(self, source):
        """
        Download CVE file from internet database.
        :param source:
        :return: CVE data, CVE data timestamp and response or None if error
        """
        file_stream, response_info = get_file(source)
        try:
            if isinstance(file_stream, str):
                result = json.loads(file_stream)
            else:
                result = json.load(file_stream)
            if "CVE_Items" in result:
                CVE_data_timestamp = result.get("CVE_data_timestamp",
                                                unify_time(datetime.utcnow()))
                return result["CVE_Items"], CVE_data_timestamp, response_info
            return None
        except json.JSONDecodeError as json_error:
            print('Get an JSON decode error: {}'.format(json_error))
            return None

    def parse_cve_file(self,
                       items=None,
                       CVE_data_timestamp=unify_time(datetime.utcnow())):
        """
        Parse CVE file
        :param items:
        :param CVE_data_timestamp:
        :return: list of json
        """
        if items is None:
            items = []
        parsed_items = []
        for item in items:
            element = json.loads(CVEItem(item).to_json())
            element["cvss_time"] = CVE_data_timestamp
            parsed_items.append(element)
        return parsed_items

    #
    # Filters
    #

    @staticmethod
    def filter_cpe_string__json(element):
        """
        Filter CPE strings in CVE Items for valid component and version values
        :param element:
        :return: json
        """
        result = {"component": None, "version": None}
        try:
            c22 = cpe_module.CPE(element, cpe_module.CPE.VERSION_2_2)
        except ValueError:
            try:
                c22 = cpe_module.CPE(element, cpe_module.CPE.VERSION_2_3)
            except ValueError:
                try:
                    c22 = cpe_module.CPE(element,
                                         cpe_module.CPE.VERSION_UNDEFINED)
                except NotImplementedError:
                    c22 = None
        c22_product = c22.get_product() if c22 is not None else []
        c22_version = c22.get_version() if c22 is not None else []
        result["component"] = c22_product[0] if isinstance(
            c22_product, list) and len(c22_product) > 0 else None
        result["version"] = c22_version[0] if isinstance(
            c22_version, list) and len(c22_version) > 0 else None
        return result

    def filter_items_to_update(self,
                               items_fo_filter,
                               unquote=True,
                               only_digits_and_dot_in_version=True):
        """
        Filter Vulners items and create more items for database - one by one item
        for filtered element in cpe strings in CVE Item
        :param items_fo_filter:
        :param unquote:
        :param only_digits_and_dot_in_version:
        :return: list of items
        """
        filtered_items = []
        # for item in items_fo_filter:
        for item in progressbar(items_fo_filter, prefix='Filtering  '):
            # For every item in downloaded update
            # Get cpe strings
            list_of_cpe_strings = item.get("vulnerable_configuration", [])
            # If list not empty
            if len(list_of_cpe_strings) > 0:
                # For every cpe string
                for one_cpe_string in list_of_cpe_strings:
                    # Get one string and check it
                    filtered_cpe_string = self.filter_cpe_string__json(
                        one_cpe_string)
                    version = filtered_cpe_string.get("version", "")
                    component = filtered_cpe_string.get("component", "")
                    if version is not None and not str(version).__eq__(
                            "") and component is not None and not str(
                                component).__eq__(""):
                        # Copy item into filtered items
                        new_item = item.copy()
                        new_item["component"] = filtered_cpe_string[
                            "component"]
                        new_item["version"] = filtered_cpe_string["version"]
                        if unquote:
                            try:
                                new_item["version"] = urllib.parse.unquote(
                                    new_item["version"])
                            except Exception:
                                pass
                        if only_digits_and_dot_in_version:
                            allow = string.digits + '.' + '(' + ')'
                            new_item["version"] = re.sub(
                                '[^%s]' % allow, '', new_item["version"])
                        new_item[
                            "vulnerable_configuration"] = list_of_cpe_strings
                        if not str(new_item["version"]).__eq__(""):
                            filtered_items.append(new_item)
                        del new_item
        return filtered_items

    def if_item_already_exists_in_vulnerabilities_table(
            self, component, version, cve_id):
        """
        Check, if items already exist in Vulnerabilities table by component,
        version and CVE ID. If exists - return list of its IDs
        :param component:
        :param version:
        :param cve_id:
        :return: list of IDs
        """
        # Get IDs of records
        list_of_elements = list(VULNERABILITIES.select().where(
            (VULNERABILITIES.component == component)
            & (VULNERABILITIES.version == version)
            & (VULNERABILITIES.cve_id == cve_id)))
        list_of_ids = []
        for element in list_of_elements:
            list_of_ids.append(element.id)
        return list_of_ids

    #
    # Creators
    #

    def create_record_in_vulnerabilities_table__id(self, item_to_create):
        """
        Create record in vulnerabilities table and return its ID
        :param item_to_create:
        :return: vulner id
        """
        cwe_list = item_to_create.get("cwe", [])
        capec_list = []

        for cwe in cwe_list:
            capec = list(CAPEC.select().where(
                (CAPEC.related_weakness.contains(cwe))))
            for capec_element in capec:
                capec_list.append(
                    json.dumps(
                        dict(id=re.sub("\D", "", str(capec_element.capec_id)),
                             name=capec_element.name,
                             summary=capec_element.summary,
                             prerequisites=capec_element.prerequisites,
                             solutions=capec_element.solutions,
                             related_weakness=capec_element.related_weakness)))

        vulner = VULNERABILITIES(
            component=item_to_create.get("component", ""),
            version=item_to_create.get("version", ""),
            data_type=item_to_create.get("data_type", ""),
            data_format=item_to_create.get("data_format", ""),
            data_version=item_to_create.get("data_version", ""),
            cve_id=item_to_create.get("cve_id", ""),
            cwe=item_to_create.get("cwe", []),
            references=item_to_create.get("references", []),
            description=item_to_create.get("description", ""),
            # cpe=item_to_create.get("cpe", ""),
            vulnerable_configuration=item_to_create.get(
                "vulnerable_configuration", []),
            published=item_to_create.get("published", str(datetime.utcnow())),
            modified=item_to_create.get("modified", str(datetime.utcnow())),
            access=item_to_create.get("access", '{}'),
            impact=item_to_create.get("impact", '{}'),
            vector_string=item_to_create.get("vector_string", ""),
            cvss_time=item_to_create.get("cvss_time", str(datetime.utcnow())),
            cvss=item_to_create.get("cvss", 0.0),
            capec=capec_list)
        vulner.save()
        return vulner.id

    #
    # Updaters
    #

    def update_vulnerabilities_table_in_memory__counts(self, items_to_update):
        # Populate CVEs Items In Memory
        for item in progressbar(items_to_update):
            self.cache.append_item(item)
        print('Append {} keys.'.format(self.cache.size))

    def update_record_in_vulnerabilities_table__id(self, item_to_update,
                                                   item_id_in_database):
        """
        Update record in vulnerabilities table and return its ID
        :param item_to_update:
        :param item_id_in_database:
        :return: vulner_id
        """
        was_modified = False
        vulner_from_database = VULNERABILITIES.get(
            VULNERABILITIES.id == item_id_in_database)
        vulner = vulner_from_database.to_json

        if vulner["data_type"] != item_to_update["data_type"]:
            was_modified = True
            vulner_from_database.data_type = item_to_update["data_type"]
        if vulner["data_format"] != item_to_update["data_format"]:
            was_modified = True
            vulner_from_database.data_format = item_to_update["data_format"]
        if vulner["data_version"] != item_to_update["data_version"]:
            was_modified = True
            vulner_from_database.data_version = item_to_update["data_version"]
        if vulner["cwe"] != item_to_update["cwe"]:
            was_modified = True
            vulner_from_database.cwe = item_to_update["cwe"]
        if vulner["references"] != item_to_update["references"]:
            was_modified = True
            vulner_from_database.references = item_to_update["references"]
        if vulner["description"] != item_to_update["description"]:
            was_modified = True
            vulner_from_database.description = item_to_update["description"]
        if vulner["vulnerable_configuration"] != item_to_update[
                "vulnerable_configuration"]:
            was_modified = True
            vulner_from_database.vulnerable_configuration = item_to_update[
                "vulnerable_configuration"]
        if unify_time(vulner["published"]) != unify_time(
                item_to_update["published"]):
            was_modified = True
            vulner_from_database.published = unify_time(
                item_to_update["published"])
        if unify_time(vulner["modified"]) != unify_time(
                item_to_update["modified"]):
            was_modified = True
            vulner_from_database.modified = unify_time(
                item_to_update["modified"])
        if was_modified:
            vulner_from_database.save()

        return vulner_from_database.id

    def update_vulnerabilities_table_for_modified_and_recent_cves(
            self, items_to_update):
        """
        Update vulnerabilities table
        :param items_to_update:
        :return: count of new records, count of updated records and time delta
        """
        start_time = time.time()
        count_of_new_records = 0
        count_of_updated_records = 0

        connect_database()

        # For every item in items to update
        # for one_item in items_to_update:
        for one_item in progressbar(items_to_update):
            # Check if exists
            component = one_item.get("component", None)
            version = one_item.get("version", None)
            cve_id = one_item.get("cve_id", None)
            if component is not None and \
                    version is not None and \
                    cve_id is not None:
                if_records_exists_in_database__ids = self.if_item_already_exists_in_vulnerabilities_table(
                    component, version, cve_id)
                if len(if_records_exists_in_database__ids) > 0:
                    for item_id_in_database in if_records_exists_in_database__ids:
                        _id = self.update_record_in_vulnerabilities_table__id(
                            one_item, item_id_in_database)
                        count_of_updated_records += 1
                else:
                    _id = self.create_record_in_vulnerabilities_table__id(
                        one_item)
                    count_of_new_records += 1
            one_item["id"] = _id
            pass

        disconnect_database()

        return count_of_new_records, count_of_updated_records, time.time(
        ) - start_time

    def update_vulnerabilities_table_in_postgres_for_cached_items(self):
        database.connect()
        keys = self.cache.cache.keys()

        for key in progressbar(keys):
            items = self.cache.get(key)
            for item in items:
                self.create_record_in_vulnerabilities_table__id(item)

        database.close()

    def update_modified_vulners_from_source(self):
        """
        Update modified elements in vulnerabilities table
        :return: count of new records to update, count of updated records and time delta
        """
        start_time = time.time()
        count_of_parsed_cve_items = 0
        count_of_updated_items = 0

        modified_items, CVE_data_timestamp, response = self.download_cve_file(
            self.cve_modified)
        modified_parsed = self.parse_cve_file(modified_items,
                                              CVE_data_timestamp)

        last_modified = parse_datetime(response.headers["last-modified"],
                                       ignoretz=True)

        info, created = INFO.get_or_create(name="cve-modified")
        if not created:
            if info.last_modified != "":
                info_last_modified = datetime.strptime(info.last_modified,
                                                       '%Y-%m-%d %H:%M:%S')
            else:
                info_last_modified = datetime.strptime(
                    datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                    '%Y-%m-%d %H:%M:%S')
        else:
            info_last_modified = datetime.strptime(
                datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                '%Y-%m-%d %H:%M:%S')

        if info_last_modified != last_modified:
            info.last_modified = last_modified
            info.save()

            items_to_update = self.filter_items_to_update(modified_parsed)

            # Update vulners in Postgres
            self.update_vulnerabilities_table_for_modified_and_recent_cves(
                items_to_update)

            # Push ~modified~ items into collection ~modified~ in Redis

            for one_item_to_update in items_to_update:
                try:
                    queue.rpush(
                        self.modified_queue,
                        serialize_as_json__for_cache(
                            reformat_vulner_for_output(one_item_to_update)))
                except Exception as ex:
                    sys.exit("Redis exception: {}".format(ex))
            try:
                # Publish message
                queue.publish(self.channel_to_publish, self.modified_queue)
            except Exception as ex:
                sys.exit("Redis exception: {}".format(ex))

            count_of_parsed_cve_items = len(modified_parsed)
            count_of_updated_items = len(items_to_update)

        return count_of_parsed_cve_items, count_of_updated_items, time.time(
        ) - start_time

    def update_recent_vulners_from_source(self):
        """
        Update recent elements in vulnerabilities table
        :return: count of new records to update, count of updated records and time delta
        """
        start_time = time.time()
        count_of_parsed_cve_items = 0
        count_of_updated_items = 0

        recent_items, CVE_data_timestamp, response = self.download_cve_file(
            self.cve_recent)
        recent_parsed = self.parse_cve_file(recent_items, CVE_data_timestamp)

        last_modified = parse_datetime(response.headers["last-modified"],
                                       ignoretz=True)

        info, created = INFO.get_or_create(name="cve-recent")
        if not created:
            if info.last_modified != "":
                info_last_modified = datetime.strptime(info.last_modified,
                                                       '%Y-%m-%d %H:%M:%S')
            else:
                info_last_modified = datetime.strptime(
                    datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                    '%Y-%m-%d %H:%M:%S')
        else:
            info_last_modified = datetime.strptime(
                datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                '%Y-%m-%d %H:%M:%S')

        if info_last_modified != last_modified:
            info.last_modified = last_modified
            info.save()

            items_to_update = self.filter_items_to_update(recent_parsed)

            # Update vulners in Postgres
            self.update_vulnerabilities_table_for_modified_and_recent_cves(
                items_to_update)

            # Push ~recent~ items into collection ~new~ in Redis
            for one_item_to_update in items_to_update:
                try:
                    queue.rpush(
                        self.new_queue,
                        serialize_as_json__for_cache(
                            reformat_vulner_for_output(one_item_to_update)))
                except Exception as ex:
                    sys.exit("Redis exception: {}".format(ex))
            try:
                # Publish message
                queue.publish(self.channel_to_publish, self.new_queue)
            except Exception as ex:
                sys.exit("Redis exception: {}".format(ex))

            count_of_parsed_cve_items = len(recent_parsed)
            count_of_updated_items = len(items_to_update)

        return count_of_parsed_cve_items, count_of_updated_items, time.time(
        ) - start_time

    #
    # Populaters
    #

    def populate_vulners_in_memory_than_postgres__counts(self):
        start_time = time.time()
        # start_year = SETTINGS.get("start_year", 2002)
        current_year = datetime.now().year
        count_of_parsed_cve_items = 0
        count_of_populated_items = 0
        for year in range(self.start_year, current_year + 1):
            print("Populate CVE-{}".format(year))
            source = self.cve_base + str(year) + self.cve_base_postfix
            cve_item, cve_data_timestamp, response = self.download_cve_file(
                source)

            parsed_cve_items = self.parse_cve_file(cve_item,
                                                   cve_data_timestamp)

            items_to_populate = self.filter_items_to_update(parsed_cve_items)

            self.update_vulnerabilities_table_in_memory__counts(
                items_to_populate)

            count_of_parsed_cve_items += len(parsed_cve_items)
            count_of_populated_items += len(items_to_populate)

        self.update_vulnerabilities_table_in_postgres_for_cached_items()

        return count_of_parsed_cve_items, count_of_populated_items, time.time(
        ) - start_time

    #
    # Checkers
    #

    def check_if_redis_working(self):
        try:
            queue.ping()
        except exceptions.ConnectionError:
            print("Redis is not working")
            sys.exit(1)

    #
    # Actions
    #

    def drop_all_tables_in_postgres(self):
        print('Tables will be drop from PostgresQL')
        drop_all_tables_in_postgres()

    def create_tables_in_postgres(self):
        print('Tables will be created in PostgresQL')
        create_tables_in_postgres()

    def action_update_cwe(self):
        action_update_cwe()

    def action_update_capec(self):
        action_update_capec()

    def action_populate_databases(self):
        print("Start population of database")
        count_of_parsed_cve_items, count_of_populated_items, time_delta = self.populate_vulners_in_memory_than_postgres__counts(
        )
        print("Cache stats: {} elements".format(self.cache.stats))
        print('Dumped into: {}'.format(
            self.cache.dump_cache_into_json_file__with_ts()))
        print("Get        {} populated elements from source".format(
            count_of_parsed_cve_items))
        print(
            "Append     {} populated elements from source in database".format(
                count_of_populated_items))
        print("TimeDelta  %.2f sec." % (time_delta))

    def action_update_modified_elements(self):
        print("Start update modified of database")
        count_of_parsed_cve_items, count_of_updated_items, time_delta = self.update_modified_vulners_from_source(
        )
        print("Get        {} modified elements from source".format(
            count_of_parsed_cve_items))
        print("Append     {} modified elements from source in database".format(
            count_of_updated_items))
        print("TimeDelta  %.2f sec." % (time_delta))

    def action_update_recent_elements(self):
        print("Start update recent of database")
        count_of_parsed_cve_items, count_of_updated_items, time_delta = self.update_recent_vulners_from_source(
        )
        print("Get        {} recent elements from source".format(
            count_of_parsed_cve_items))
        print("Append     {} recent elements from souce in database".format(
            count_of_updated_items))
        print("TimeDelta  %.2f sec." % (time_delta))

    def run(self):
        self.check_if_redis_working()
        self.drop_all_tables_in_postgres()
        self.create_tables_in_postgres()
        self.action_update_cwe()
        self.action_update_capec()
        self.action_populate_databases()
        self.action_update_modified_elements()
        self.action_update_recent_elements()
        drop_cache()
        return 0
Esempio n. 12
0
class MemoryStorageTests(unittest.TestCase):
    def setUp(self):
        if not settings.configured:
            settings.configure(MEDIA_URL='')
        self.storage = InMemoryStorage()
        self.filesystem = self.storage.filesystem

    def test_listdir(self):
        self.assertEqual(self.storage.listdir(''), [[], []])

        self.filesystem.add_child('dir0', InMemoryDir())
        self.filesystem.add_child('file0', InMemoryFile())

        self.assertEqual(self.storage.listdir(''), [['dir0'], ['file0']])
        self.assertEqual(self.storage.listdir('dir0'), [[], []])

        self.filesystem.resolve('dir0').add_child('subdir', InMemoryDir())
        self.assertEqual(self.storage.listdir('dir0'), [['subdir'], []])

    def test_delete(self):
        self.filesystem.add_child('dir0', InMemoryDir())
        self.filesystem.resolve('dir0').add_child('nested_file', InMemoryFile())
        self.filesystem.add_child('file0', InMemoryFile())
        self.assertEqual(self.filesystem.resolve('dir0').ls(), ['nested_file'])

        self.storage.delete('dir0/nested_file')
        self.assertEqual(self.filesystem.resolve('dir0').ls(), [])
        self.assertEqual(set(self.filesystem.ls()), set(['dir0', 'file0']))

        self.storage.delete('dir0')
        self.assertEqual(set(self.filesystem.ls()), set(['file0']))

    def test_exists(self):
        self.filesystem.add_child('file0', InMemoryFile())
        self.assertTrue(self.storage.exists('file0'))
        self.assertFalse(self.storage.exists('file1'))
        self.storage.delete('file0')
        self.assertFalse(self.storage.exists('file0'))

    def test_size(self):
        self.filesystem.add_child('file0', InMemoryFile('test'))
        self.assertEqual(self.storage.size('file0'), 4)

    def test_save(self):
        self.storage.save('file', ContentFile('test'))
        self.assertEqual(self.storage.size('file'), 4)
        self.storage.save('subdir/file', ContentFile('test'))
        self.assertEqual(self.storage.size('subdir/file'), 4)

    def test_write(self):
        with self.storage.open("file", "w") as f:
            f.write("hello")
        with self.storage.open("file", "r") as f:
            self.assertEqual(f.read(), "hello")

    def test_all(self):
        self.assertEqual(self.storage.listdir('/'), [[], []])
        self.assertEqual(self.storage.save('dir/subdir/file', ContentFile('testing')), 'dir/subdir/file')
        self.assertEqual(self.storage.listdir('/'), [['dir'], []])
        self.assertEqual(self.storage.listdir('dir/'), [['subdir'], []])
        self.assertEqual(self.storage.save('dir/subdir/file2', ContentFile('testing2')), 'dir/subdir/file2')
        self.assertEqual(self.storage.save('file', ContentFile('testing3')), 'file')
        self.assertEqual(self.storage.listdir('/'), [['dir'], ['file']])
        self.assertEqual(self.storage.listdir('dir/'), [['subdir'], []])
        self.assertEqual(self.storage.open('dir/subdir/file').read(), 'testing')
        self.assertEqual(self.storage.size('dir/subdir/file'), 7)
        self.assertEqual(self.storage.size('dir/subdir/file2'), 8)
        self.assertEqual(self.storage.size('file'), 8)
        self.assertEqual(self.storage.delete('file'), None)
        self.assertEqual(self.storage.listdir('/'), [['dir'], []])
        self.assertEqual(self.storage.delete('dir/subdir/file'), None)
        self.assertEqual(self.storage.listdir('dir/subdir'), [[], ['file2']])
        self.assertEqual(self.storage.exists('dir/subdir/file2'), True)
        self.assertEqual(self.storage.delete('dir/subdir/file2'), None)
        self.assertEqual(self.storage.exists('dir/subdir/file2'), False)
        self.assertEqual(self.storage.listdir('dir/subdir'), [[], []])
Esempio n. 13
0
 def setUp(self):
     self.storage = InMemoryStorage()
     self.filesystem = self.storage.filesystem