def __init__(self, *args, **kwargs):
super(LTACachedService, self).__init__(*args, **kwargs)
# TODO: need to profile how much data we are caching
one_hour = 3600 # seconds
self.MC_KEY_FMT = '({resource})'
self.MD_KEY_FMT = '({resource},{id})'
self.cache = CachingProvider(timeout=one_hour)
class LTACachedService(LTAService):
"""
Wrapper on top of the cache, with helper functions which balance requests
to the external service when needed.
"""
def __init__(self, *args, **kwargs):
super(LTACachedService, self).__init__(*args, **kwargs)
# TODO: need to profile how much data we are caching
one_hour = 3600 # seconds
self.MC_KEY_FMT = '({resource})'
self.cache = CachingProvider(timeout=one_hour)
def get_login(self):
cache_key = self.MC_KEY_FMT.format(resource='login')
token = self.cache.get(cache_key)
return token
def set_login(self, token):
cache_key = self.MC_KEY_FMT.format(resource='login')
success = self.cache.set(cache_key, token)
if not success:
logger.error('LTACachedService: Token not cached')
def cached_login(self):
token = self.get_login()
if token is None:
token = self.login()
self.set_login(token)
return token
from api.domain.user import User
from api.providers.configuration.configuration_provider import ConfigurationProvider
from api.providers.caching.caching_provider import CachingProvider
from api.util.sshcmd import RemoteHost
import socket
cache = CachingProvider()
config = ConfigurationProvider()
class HadoopHandler(object):
def list_jobs(self):
return self._remote_cmd('hadoop job -list')
def kill_job(self, jobid):
return self._remote_cmd('hadoop job -kill {}'.format(jobid))
def kill_user_jobs(self, username):
_response = dict()
try:
for job_name in User.by_username(
username).active_hadoop_job_names():
_response[job_name] = self.kill_job(
self.job_names_ids()[job_name])
except AttributeError, e:
if "object has no attribute 'active_hadoop_job_names'" in e.message:
_response['msg'] = 'user not found'
else:
raise e
return _response
class LTACachedService(LTAService):
"""
Wrapper on top of the cache, with helper functions which balance requests
to the external service when needed.
"""
def __init__(self, *args, **kwargs):
super(LTACachedService, self).__init__(*args, **kwargs)
# TODO: need to profile how much data we are caching
one_hour = 3600 # seconds
self.MC_KEY_FMT = '({resource})'
self.MD_KEY_FMT = '({resource},{id})'
self.cache = CachingProvider(timeout=one_hour)
# -----------------------------------------------------------------------+
# Handlers to format cache keys and perform bulk value fetching/setting |
def get_login(self):
cache_key = self.MC_KEY_FMT.format(resource='login')
token = self.cache.get(cache_key)
return token
def set_login(self, token):
cache_key = self.MC_KEY_FMT.format(resource='login')
success = self.cache.set(cache_key, token)
if not success:
raise LTAError('Token not cached')
def get_lookup(self, id_list):
cache_keys = [
self.MD_KEY_FMT.format(resource='idLookup', id=i) for i in id_list
]
entries = self.cache.get_multi(cache_keys)
entries = {k.split(',')[1][:-1]: v for k, v in entries.items()}
return entries
def set_lookup(self, id_pairs):
cache_entries = {
self.MD_KEY_FMT.format(resource='idLookup', id=i): e
for i, e in id_pairs.items()
}
success = self.cache.set_multi(cache_entries)
if not success:
raise LTAError('ID conversion not cached')
# ---------------------------------------------------------------+
# Handlers to balance fetching cached/external values as needed |
def cached_login(self):
token = self.get_login()
if token is None:
token = self.login()
self.set_login(token)
return token
def cached_id_lookup(self, id_list):
entities = self.get_lookup(id_list)
if len(entities) > 0:
diff = set(id_list) - set(entities)
if diff:
fetched = self.id_lookup(list(diff))
self.set_lookup(entities)
entities.update(fetched)
else:
entities = self.id_lookup(id_list)
self.set_lookup(entities)
return entities
def cached_verify_scenes(self, id_list):
entities = self.get_lookup(id_list)
if len(entities) > 0:
diff = set(id_list) - set(entities)
if diff:
fetched = self.id_lookup(list(diff))
self.set_lookup(entities)
entities.update(fetched)
else:
entities = self.id_lookup(id_list)
self.set_lookup(entities)
results = {k: entities.get(k) for k in id_list}
return results
