def test_last_revision_number_do_not_depend_on_store_name(self):
        with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
            ds_manager = DataStorageManager(self._func_name() + '1')
            with ds_manager.save('file1.xml') as buffer:
                buffer.write(b'data part 1')

            self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER)

            ds_manager = DataStorageManager(self._func_name() + '2')
            with ds_manager.save('file1.xml') as buffer:
                buffer.write(b'data part 1')

            self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER)
    def test_last_revision_number_increase_after_each_save(self):
        with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
            ds_manager = DataStorageManager(self._func_name())

            with ds_manager.save('file.xml') as buffer:
                buffer.write(b'data part 1')

            self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER)

            with ds_manager.save('file.xml') as buffer:
                buffer.write(b'data part 1')
                buffer.write(b'data part 2')

            self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER + 1)
Exemple #3
0
    def test_last_revision_number_do_not_depend_on_store_name(self):
        with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
            ds_manager = DataStorageManager(self._func_name() + '1')
            with ds_manager.save('file1.xml') as buffer:
                buffer.write(b'data part 1')

            self.assertEqual(ds_manager.last_revision_number(),
                             DataStorageManager.FIRST_REV_NUMBER)

            ds_manager = DataStorageManager(self._func_name() + '2')
            with ds_manager.save('file1.xml') as buffer:
                buffer.write(b'data part 1')

            self.assertEqual(ds_manager.last_revision_number(),
                             DataStorageManager.FIRST_REV_NUMBER)
Exemple #4
0
    def test_last_revision_number_increase_after_each_save(self):
        with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
            ds_manager = DataStorageManager(self._func_name())

            with ds_manager.save('file.xml') as buffer:
                buffer.write(b'data part 1')

            self.assertEqual(ds_manager.last_revision_number(),
                             DataStorageManager.FIRST_REV_NUMBER)

            with ds_manager.save('file.xml') as buffer:
                buffer.write(b'data part 1')
                buffer.write(b'data part 2')

            self.assertEqual(ds_manager.last_revision_number(),
                             DataStorageManager.FIRST_REV_NUMBER + 1)
Exemple #5
0
class DataSource:

    def __init__(self, store_config):
        self.name = store_config['name']
        self.store_url = store_config['url']
        self.url = store_config['data_source']['url']
        self.type = store_config['data_source']['type']
        self.ds_manager = DataStorageManager(self.name)

    @staticmethod
    def get_all_subclasses():
        """
        Returns all subclasses, not only direct subclasses, but also
        all subclasses of subclasses, and so on.
        """
        subclasses = {}

        def get_subclasses(subclasses, cls):
            subclasses[cls.__name__] = cls
            for subclass in cls.__subclasses__():
                get_subclasses(subclasses, subclass)

        get_subclasses(subclasses, DataSource)

        return subclasses

    def fetch(self):
        pass

    def _extract(self, revision):
        pass

    def _filter(self, products, prev_rev_number):
        pass

    def update(self):
        print('Updating {} products...'.format(self.name))
        available_revision = self.ds_manager.last_revision_number()
        store, new = Store.objects.get_or_create(name=self.name, url=self.store_url)

        if new or store.last_update_revision is None:
            products = self._extract(available_revision)
            store.update_products(revision_number=available_revision, added=products)
        elif store.last_update_revision < available_revision:
            products = self._extract(available_revision)
            filtered = self._filter(products, store.last_update_revision)
            store.update_products(
                revision_number=available_revision,
                added=filtered['added'],
                deleted=filtered['deleted'],
                modified=filtered['modified']
            )
Exemple #6
0
class DataSourceImpl:

    def __init__(self, store):
        self.store = store
        self.ds_manager = DataStorageManager(self.store.name)

    @staticmethod
    def get_all_subclasses():
        """
        Returns all subclasses, not only direct subclasses, but also
        all subclasses of subclasses, and so on.
        """
        subclasses = {}

        def get_subclasses(subclasses, cls):
            if cls.__name__ != DataSourceImpl.__name__:
                subclasses[cls.__name__] = cls

            for subclass in cls.__subclasses__():
                get_subclasses(subclasses, subclass)

        get_subclasses(subclasses, DataSourceImpl)

        return subclasses

    def fetch(self):
        pass

    def _extract(self, revision):
        pass

    def _filter(self, offers, prev_rev_number):
        pass

    def update(self):
        logger.info('[Store:{}] Updating offers...'.format(self.store.name))
        available_revision = self.ds_manager.last_revision_number()

        if self.store.last_update_revision is None:
            offers = self._extract(available_revision)
            self.store.update_offers(revision_number=available_revision, added=offers)
        elif self.store.last_update_revision < available_revision:
            offers = self._extract(available_revision)
            filtered = self._filter(offers, self.store.last_update_revision)
            self.store.update_offers(
                revision_number=available_revision,
                added=filtered['added'],
                deleted=filtered['deleted'],
                modified=filtered['modified']
            )
        else:
            logger.info('[Store:{}] There are no new revision available!')
Exemple #7
0
class DataSource:

    def __init__(self, store_config):
        self.name = store_config['name']
        self.store_url = store_config['url']
        self.url = store_config['data_source']['url']
        self.type = store_config['data_source']['type']
        self.ds_manager = DataStorageManager(self.name)

    @staticmethod
    def get_all_subclasses():
        """
        Returns all subclasses, not only direct subclasses, but also
        all subclasses of subclasses, and so on.
        """
        subclasses = {}

        def get_subclasses(subclasses, cls):
            subclasses[cls.__name__] = cls
            for subclass in cls.__subclasses__():
                get_subclasses(subclasses, subclass)

        get_subclasses(subclasses, DataSource)

        return subclasses

    def fetch(self):
        pass

    def _extract(self, revision):
        pass

    def _filter(self, products, prev_rev_number):
        pass

    def update(self):
        available_revision = self.ds_manager.last_revision_number()
        store, new = Store.objects.get_or_create(name=self.name, url=self.store_url)

        if new or store.last_update_revision is None:
            products = self._extract(available_revision)
            store.update_products(revision_number=available_revision, added=products)
        elif store.last_update_revision < available_revision:
            products = self._extract(available_revision)
            filtered = self._filter(products, store.last_update_revision)
            store.update_products(
                revision_number=available_revision,
                added=filtered['added'],
                deleted=filtered['deleted'],
                modified=filtered['modified']
            )
    def test_get_returns_proper_revision(self):
        with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
            ds_manager = DataStorageManager(self._func_name())

            for rev in range(10):
                with ds_manager.save('file.xml') as buffer:
                    buffer.write('file content {}'.format(rev).encode())

                self.assertEqual(ds_manager.last_revision_number(), rev)

            for rev in range(10):
                self.assertEqual(
                    'file content {}'.format(rev),
                    ds_manager.get('file.xml', rev)
                )

            # without rev argument, last revision should be returned
            self.assertEqual('file content 9', ds_manager.get('file.xml'))
Exemple #9
0
 def test_last_revision_number_raises_no_revisions_at_the_beginning(self):
     with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
         ds_manager = DataStorageManager(self._func_name())
         with self.assertRaises(DataStorageManager.NoRevision):
             ds_manager.last_revision_number()
 def test_last_revision_number_raises_no_revisions_at_the_beginning(self):
     with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
         ds_manager = DataStorageManager(self._func_name())
         with self.assertRaises(DataStorageManager.NoRevision):
             ds_manager.last_revision_number()