def test_get_data(self):
service = "th_rss"
self.assertTrue(service.startswith('th_'))
service_long = MyService.full_name(service)
self.assertTrue(service_long in settings.TH_SERVICES)
self.assertTrue('publishing_limit' in settings.DJANGO_TH)
def test_get_data(self):
service = "th_rss"
self.assertTrue(service.startswith('th_'))
service_long = MyService.full_name(service)
self.assertTrue(service_long in settings.TH_SERVICES)
self.assertTrue('publishing_limit' in settings.DJANGO_TH)
def get_outside_cache():
"""
the purpose of this tasks is to recycle the data from the cache
with version=2 in the main cache
"""
all_packages = MyService.all_packages()
for package in all_packages:
cache = caches[package]
# http://niwinz.github.io/django-redis/latest/#_scan_delete_keys_in_bulk
for service in cache.iter_keys('th_*'):
try:
cache.get(service, version=2)
except ValueError:
pass
def get_data(service, cache_data, trigger_id):
"""
get the data from the cache
:param service: the service name
:param cache_data: the data from the cache
:type trigger_id: integer
:return: Return the data from the cache
:rtype: object
"""
# rebuild the string
# th_<service>.my_<service>.Service<Service>
if service.startswith('th_'):
service_long = MyService.full_name(service)
# service_name = service.split('_')[1]
# service_long = ''.join((service, ".my_", service_name, ".Service",
# service_name.title()))
# ... and check it
if service_long in settings.TH_SERVICES:
cache = caches[service]
if 'publishing_limit' in settings.DJANGO_TH:
limit = settings.DJANGO_TH['publishing_limit']
# publishing of all the data
if limit == 0:
return cache_data
# or just a set of them
if cache_data is not None and len(cache_data) > limit:
for data in cache_data[limit:]:
service_str = ''.join(
(service, '_', str(trigger_id)))
# put that data in a version 2 of the cache
cache.set(service_str, data, version=2)
# delete data from cache version=1
# https://niwinz.github.io/django-redis/latest/#_scan_delete_keys_in_bulk
cache.delete_pattern(service_str)
# put in cache unpublished data
cache_data = cache_data[:limit]
return cache_data
def get_data(service, cache_data, trigger_id):
"""
get the data from the cache
:param service: the service name
:param cache_data: the data from the cache
:type trigger_id: integer
:return: Return the data from the cache
:rtype: object
"""
# rebuild the string
# th_<service>.my_<service>.Service<Service>
if service.startswith('th_'):
service_long = MyService.full_name(service)
# service_name = service.split('_')[1]
# service_long = ''.join((service, ".my_", service_name, ".Service",
# service_name.title()))
# ... and check it
if service_long in settings.TH_SERVICES:
cache = caches[service]
if 'publishing_limit' in settings.DJANGO_TH:
limit = settings.DJANGO_TH['publishing_limit']
# publishing of all the data
if limit == 0:
return cache_data
# or just a set of them
if cache_data is not None and len(cache_data) > limit:
for data in cache_data[limit:]:
service_str = ''.join((service, '_',
str(trigger_id)))
# put that data in a version 2 of the cache
cache.set(service_str, data, version=2)
# delete data from cache version=1
# https://niwinz.github.io/django-redis/latest/#_scan_delete_keys_in_bulk
cache.delete_pattern(service_str)
# put in cache unpublished data
cache_data = cache_data[:limit]
return cache_data
def recycle():
"""
the purpose of this tasks is to recycle the data from the cache
with version=2 in the main cache
"""
all_packages = MyService.all_packages()
for package in all_packages:
cache = caches[package]
# http://niwinz.github.io/django-redis/latest/#_scan_delete_keys_in_bulk
for service in cache.iter_keys('th_*'):
try:
# get the value from the cache version=2
service_value = cache.get(service, version=2)
# put it in the version=1
cache.set(service, service_value)
# remote version=2
cache.delete_pattern(service, version=2)
except ValueError:
pass
logger.info('recycle of cache done!')
def get_outside_cache(result=''):
"""
the purpose of this tasks is to recycle the data from the cache
with version=2 in the main cache
:param result: the result of the previous tasks when cascading all of them
:type result: string
"""
all_packages = MyService.all_packages()
for package in all_packages:
cache = caches[package]
# http://niwinz.github.io/django-redis/latest/#_scan_delete_keys_in_bulk
for service in cache.iter_keys('th_*'):
try:
# get the value from the cache version=2
service_value = cache.get(service, version=2)
# put it in the version=1
cache.set(service, service_value)
# remote version=2
cache.delete_pattern(service, version=2)
except ValueError:
pass
def get_outside_cache(result=''):
"""
the purpose of this tasks is to recycle the data from the cache
with version=2 in the main cache
:param result: the result of the previous tasks when cascading all of them
:type result: string
"""
all_packages = MyService.all_packages()
for package in all_packages:
cache = caches[package]
# http://niwinz.github.io/django-redis/latest/#_scan_delete_keys_in_bulk
for service in cache.iter_keys('th_*'):
try:
# get the value from the cache version=2
service_value = cache.get(service, version=2)
# put it in the version=1
cache.set(service, service_value)
# remote version=2
cache.delete_pattern(service, version=2)
except ValueError:
pass
def recycle():
"""
the purpose of this tasks is to recycle the data from the cache
with version=2 in the main cache
"""
logger = getLogger('django_th.trigger_happy')
all_packages = MyService.all_packages()
for package in all_packages:
if package in settings.DJANGO_TH.get('services_wo_cache'):
continue
cache = caches['django_th']
# http://niwinz.github.io/django-redis/latest/#_scan_delete_keys_in_bulk
for service in cache.iter_keys('th_*'):
try:
# get the value from the cache version=2
service_value = cache.get(service, version=2)
# put it in the version=1
cache.set(service, service_value)
# remote version=2
cache.delete_pattern(service, version=2)
except ValueError:
pass
logger.info('recycle of cache done!')
def test_all_packages(self):
all_packages = MyService.all_packages()
self.assertIsInstance(all_packages, list)
def test_service_name(self):
service_name = MyService.service_name('th_rss')
self.assertEqual(self.service_name, service_name)
def test_module_name(self):
module_name = MyService.module_name('th_rss')
self.assertEqual(self.module_name, module_name)
def test_full_name(self):
service_long = MyService.full_name('th_rss')
self.assertEqual(self.full_name, service_long)
self.assertTrue(service_long in settings.TH_SERVICES)
