def test_get_raises_no_file_when_there_is_no_file(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with ds_manager.save('file.xml') as buffer:
buffer.write(b'data part 1')
with self.assertRaises(DataStorageManager.NoFile):
ds_manager.get('other_file.xml')
def test_get_raises_no_revision_for_not_existing_revision(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with ds_manager.save('file.xml') as buffer:
buffer.write(b'data part 1')
with self.assertRaises(DataStorageManager.NoRevision):
ds_manager.get('file.xml', revision=999)
def test_get_raises_no_file_when_there_is_no_file(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with ds_manager.save('file.xml') as buffer:
buffer.write(b'data part 1')
with self.assertRaises(DataStorageManager.NoFile):
ds_manager.get('other_file.xml')
def test_get_raises_no_revision_for_not_existing_revision(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with ds_manager.save('file.xml') as buffer:
buffer.write(b'data part 1')
with self.assertRaises(DataStorageManager.NoRevision):
ds_manager.get('file.xml', revision=999)
def test_data_are_saved(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with ds_manager.save('file.xml') as buffer:
buffer.write(b'data part 1')
buffer.write(b'data part 2')
content = ds_manager.get('file.xml')
self.assertEqual(content, 'data part 1data part 2')
def test_data_are_saved(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with ds_manager.save('file.xml') as buffer:
buffer.write(b'data part 1')
buffer.write(b'data part 2')
content = ds_manager.get('file.xml')
self.assertEqual(content, 'data part 1data part 2')
def test_last_revision_number_do_not_depend_on_filename(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with ds_manager.save('file1.xml') as buffer:
buffer.write(b'data part 1')
self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER)
with ds_manager.save('file2.xml') as buffer:
buffer.write(b'data part 1')
self.assertEqual(ds_manager.last_revision_number(), DataStorageManager.FIRST_REV_NUMBER + 1)
def test_update_passes_no_revision_exception_if_there_is_no_data_in_ds(
self, data_storage_manager):
data_storage_manager.return_value.last_revision_number.side_effect = DataStorageManager.NoRevision(
)
data_source = StoreManager().get_store('xmldatasource')
with self.assertRaises(DataStorageManager.NoRevision):
data_source.update()
def test_get_returns_proper_revision(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
for rev in range(10):
with ds_manager.save('file.xml') as buffer:
buffer.write('file content {}'.format(rev).encode())
self.assertEqual(ds_manager.last_revision_number(), rev)
for rev in range(10):
self.assertEqual(
'file content {}'.format(rev),
ds_manager.get('file.xml', rev)
)
# without rev argument, last revision should be returned
self.assertEqual('file content 9', ds_manager.get('file.xml'))
class DataSource:
def __init__(self, store_config):
self.name = store_config['name']
self.store_url = store_config['url']
self.url = store_config['data_source']['url']
self.type = store_config['data_source']['type']
self.ds_manager = DataStorageManager(self.name)
@staticmethod
def get_all_subclasses():
"""
Returns all subclasses, not only direct subclasses, but also
all subclasses of subclasses, and so on.
"""
subclasses = {}
def get_subclasses(subclasses, cls):
subclasses[cls.__name__] = cls
for subclass in cls.__subclasses__():
get_subclasses(subclasses, subclass)
get_subclasses(subclasses, DataSource)
return subclasses
def fetch(self):
pass
def _extract(self, revision):
pass
def _filter(self, products, prev_rev_number):
pass
def update(self):
print('Updating {} products...'.format(self.name))
available_revision = self.ds_manager.last_revision_number()
store, new = Store.objects.get_or_create(name=self.name, url=self.store_url)
if new or store.last_update_revision is None:
products = self._extract(available_revision)
store.update_products(revision_number=available_revision, added=products)
elif store.last_update_revision < available_revision:
products = self._extract(available_revision)
filtered = self._filter(products, store.last_update_revision)
store.update_products(
revision_number=available_revision,
added=filtered['added'],
deleted=filtered['deleted'],
modified=filtered['modified']
)
class DataSourceImpl:
def __init__(self, store):
self.store = store
self.ds_manager = DataStorageManager(self.store.name)
@staticmethod
def get_all_subclasses():
"""
Returns all subclasses, not only direct subclasses, but also
all subclasses of subclasses, and so on.
"""
subclasses = {}
def get_subclasses(subclasses, cls):
if cls.__name__ != DataSourceImpl.__name__:
subclasses[cls.__name__] = cls
for subclass in cls.__subclasses__():
get_subclasses(subclasses, subclass)
get_subclasses(subclasses, DataSourceImpl)
return subclasses
def fetch(self):
pass
def _extract(self, revision):
pass
def _filter(self, offers, prev_rev_number):
pass
def update(self):
logger.info('[Store:{}] Updating offers...'.format(self.store.name))
available_revision = self.ds_manager.last_revision_number()
if self.store.last_update_revision is None:
offers = self._extract(available_revision)
self.store.update_offers(revision_number=available_revision, added=offers)
elif self.store.last_update_revision < available_revision:
offers = self._extract(available_revision)
filtered = self._filter(offers, self.store.last_update_revision)
self.store.update_offers(
revision_number=available_revision,
added=filtered['added'],
deleted=filtered['deleted'],
modified=filtered['modified']
)
else:
logger.info('[Store:{}] There are no new revision available!')
class DataSource:
def __init__(self, store_config):
self.name = store_config['name']
self.store_url = store_config['url']
self.url = store_config['data_source']['url']
self.type = store_config['data_source']['type']
self.ds_manager = DataStorageManager(self.name)
@staticmethod
def get_all_subclasses():
"""
Returns all subclasses, not only direct subclasses, but also
all subclasses of subclasses, and so on.
"""
subclasses = {}
def get_subclasses(subclasses, cls):
subclasses[cls.__name__] = cls
for subclass in cls.__subclasses__():
get_subclasses(subclasses, subclass)
get_subclasses(subclasses, DataSource)
return subclasses
def fetch(self):
pass
def _extract(self, revision):
pass
def _filter(self, products, prev_rev_number):
pass
def update(self):
available_revision = self.ds_manager.last_revision_number()
store, new = Store.objects.get_or_create(name=self.name, url=self.store_url)
if new or store.last_update_revision is None:
products = self._extract(available_revision)
store.update_products(revision_number=available_revision, added=products)
elif store.last_update_revision < available_revision:
products = self._extract(available_revision)
filtered = self._filter(products, store.last_update_revision)
store.update_products(
revision_number=available_revision,
added=filtered['added'],
deleted=filtered['deleted'],
modified=filtered['modified']
)
def test_last_revision_number_do_not_depend_on_store_name(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name() + '1')
with ds_manager.save('file1.xml') as buffer:
buffer.write(b'data part 1')
self.assertEqual(ds_manager.last_revision_number(),
DataStorageManager.FIRST_REV_NUMBER)
ds_manager = DataStorageManager(self._func_name() + '2')
with ds_manager.save('file1.xml') as buffer:
buffer.write(b'data part 1')
self.assertEqual(ds_manager.last_revision_number(),
DataStorageManager.FIRST_REV_NUMBER)
def test_last_revision_number_increase_after_each_save(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with ds_manager.save('file.xml') as buffer:
buffer.write(b'data part 1')
self.assertEqual(ds_manager.last_revision_number(),
DataStorageManager.FIRST_REV_NUMBER)
with ds_manager.save('file.xml') as buffer:
buffer.write(b'data part 1')
buffer.write(b'data part 2')
self.assertEqual(ds_manager.last_revision_number(),
DataStorageManager.FIRST_REV_NUMBER + 1)
def __init__(self, store):
self.store = store
self.ds_manager = DataStorageManager(self.store.name)
def test_last_revision_number_raises_no_revisions_at_the_beginning(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with self.assertRaises(DataStorageManager.NoRevision):
ds_manager.last_revision_number()
def test_last_revision_number_raises_no_revisions_at_the_beginning(self):
with override_settings(ST_STORES_DATA_DIR=self.temp_dir.name):
ds_manager = DataStorageManager(self._func_name())
with self.assertRaises(DataStorageManager.NoRevision):
ds_manager.last_revision_number()
def __init__(self, store_config):
self.name = store_config['name']
self.store_url = store_config['url']
self.url = store_config['data_source']['url']
self.type = store_config['data_source']['type']
self.ds_manager = DataStorageManager(self.name)
def __init__(self, store_config):
self.name = store_config['name']
self.store_url = store_config['url']
self.url = store_config['data_source']['url']
self.type = store_config['data_source']['type']
self.ds_manager = DataStorageManager(self.name)
