def test_get_raises_no_file_when_there_is_no_file(self): with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name): ds_manager = DataStorageManager(self._func_name()) with ds_manager.save('file.xml') as buffer: buffer.write(b'data part 1') with self.assertRaises(DataStorageManager.NoFile): ds_manager.get('other_file.xml')
def test_get_raises_no_revision_for_not_existing_revision(self): with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name): ds_manager = DataStorageManager(self._func_name()) with ds_manager.save('file.xml') as buffer: buffer.write(b'data part 1') with self.assertRaises(DataStorageManager.NoRevision): ds_manager.get('file.xml', revision=999)
def test_data_are_saved(self): with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name): ds_manager = DataStorageManager(self._func_name()) with ds_manager.save('file.xml') as buffer: buffer.write(b'data part 1') buffer.write(b'data part 2') content = ds_manager.get('file.xml') self.assertEqual(content, 'data part 1data part 2')
def test_last_revision_number_do_not_depend_on_filename(self): with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name): ds_manager = DataStorageManager(self._func_name()) with ds_manager.save('file1.xml') as buffer: buffer.write(b'data part 1') self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER) with ds_manager.save('file2.xml') as buffer: buffer.write(b'data part 1') self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER + 1)
def test_update_passes_no_revision_exception_if_there_is_no_data_in_ds( self, data_storage_manager): data_storage_manager.return_value.last_revision_number.side_effect = DataStorageManager.NoRevision( ) data_source = StoreManager().get_store('xmldatasource') with self.assertRaises(DataStorageManager.NoRevision): data_source.update()
def test_get_returns_proper_revision(self): with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name): ds_manager = DataStorageManager(self._func_name()) for rev in range(10): with ds_manager.save('file.xml') as buffer: buffer.write('file content {}'.format(rev).encode()) self.assertEqual(ds_manager.last_revision_number(), rev) for rev in range(10): self.assertEqual( 'file content {}'.format(rev), ds_manager.get('file.xml', rev) ) # without rev argument, last revision should be returned self.assertEqual('file content 9', ds_manager.get('file.xml'))
class DataSource: def __init__(self, store_config): self.name = store_config['name'] self.store_url = store_config['url'] self.url = store_config['data_source']['url'] self.type = store_config['data_source']['type'] self.ds_manager = DataStorageManager(self.name) @staticmethod def get_all_subclasses(): """ Returns all subclasses, not only direct subclasses, but also all subclasses of subclasses, and so on. """ subclasses = {} def get_subclasses(subclasses, cls): subclasses[cls.__name__] = cls for subclass in cls.__subclasses__(): get_subclasses(subclasses, subclass) get_subclasses(subclasses, DataSource) return subclasses def fetch(self): pass def _extract(self, revision): pass def _filter(self, products, prev_rev_number): pass def update(self): print('Updating {} products...'.format(self.name)) available_revision = self.ds_manager.last_revision_number() store, new = Store.objects.get_or_create(name=self.name, url=self.store_url) if new or store.last_update_revision is None: products = self._extract(available_revision) store.update_products(revision_number=available_revision, added=products) elif store.last_update_revision < available_revision: products = self._extract(available_revision) filtered = self._filter(products, store.last_update_revision) store.update_products( revision_number=available_revision, added=filtered['added'], deleted=filtered['deleted'], modified=filtered['modified'] )
class DataSourceImpl: def __init__(self, store): self.store = store self.ds_manager = DataStorageManager(self.store.name) @staticmethod def get_all_subclasses(): """ Returns all subclasses, not only direct subclasses, but also all subclasses of subclasses, and so on. """ subclasses = {} def get_subclasses(subclasses, cls): if cls.__name__ != DataSourceImpl.__name__: subclasses[cls.__name__] = cls for subclass in cls.__subclasses__(): get_subclasses(subclasses, subclass) get_subclasses(subclasses, DataSourceImpl) return subclasses def fetch(self): pass def _extract(self, revision): pass def _filter(self, offers, prev_rev_number): pass def update(self): logger.info('[Store:{}] Updating offers...'.format(self.store.name)) available_revision = self.ds_manager.last_revision_number() if self.store.last_update_revision is None: offers = self._extract(available_revision) self.store.update_offers(revision_number=available_revision, added=offers) elif self.store.last_update_revision < available_revision: offers = self._extract(available_revision) filtered = self._filter(offers, self.store.last_update_revision) self.store.update_offers( revision_number=available_revision, added=filtered['added'], deleted=filtered['deleted'], modified=filtered['modified'] ) else: logger.info('[Store:{}] There are no new revision available!')
class DataSource: def __init__(self, store_config): self.name = store_config['name'] self.store_url = store_config['url'] self.url = store_config['data_source']['url'] self.type = store_config['data_source']['type'] self.ds_manager = DataStorageManager(self.name) @staticmethod def get_all_subclasses(): """ Returns all subclasses, not only direct subclasses, but also all subclasses of subclasses, and so on. """ subclasses = {} def get_subclasses(subclasses, cls): subclasses[cls.__name__] = cls for subclass in cls.__subclasses__(): get_subclasses(subclasses, subclass) get_subclasses(subclasses, DataSource) return subclasses def fetch(self): pass def _extract(self, revision): pass def _filter(self, products, prev_rev_number): pass def update(self): available_revision = self.ds_manager.last_revision_number() store, new = Store.objects.get_or_create(name=self.name, url=self.store_url) if new or store.last_update_revision is None: products = self._extract(available_revision) store.update_products(revision_number=available_revision, added=products) elif store.last_update_revision < available_revision: products = self._extract(available_revision) filtered = self._filter(products, store.last_update_revision) store.update_products( revision_number=available_revision, added=filtered['added'], deleted=filtered['deleted'], modified=filtered['modified'] )
def test_last_revision_number_do_not_depend_on_store_name(self): with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name): ds_manager = DataStorageManager(self._func_name() + '1') with ds_manager.save('file1.xml') as buffer: buffer.write(b'data part 1') self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER) ds_manager = DataStorageManager(self._func_name() + '2') with ds_manager.save('file1.xml') as buffer: buffer.write(b'data part 1') self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER)
def test_last_revision_number_increase_after_each_save(self): with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name): ds_manager = DataStorageManager(self._func_name()) with ds_manager.save('file.xml') as buffer: buffer.write(b'data part 1') self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER) with ds_manager.save('file.xml') as buffer: buffer.write(b'data part 1') buffer.write(b'data part 2') self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER + 1)
def __init__(self, store): self.store = store self.ds_manager = DataStorageManager(self.store.name)
def test_last_revision_number_raises_no_revisions_at_the_beginning(self): with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name): ds_manager = DataStorageManager(self._func_name()) with self.assertRaises(DataStorageManager.NoRevision): ds_manager.last_revision_number()
def __init__(self, store_config): self.name = store_config['name'] self.store_url = store_config['url'] self.url = store_config['data_source']['url'] self.type = store_config['data_source']['type'] self.ds_manager = DataStorageManager(self.name)