class GenericService(object): """ Generic cuadrnt service class Shared properties between services: Contact a web service using a base url and some key:value parameters Services require a valid cert and key Want to cache results in a document-oriented database """ def __init__(self, config=dict()): self.logger = logging.getLogger(__name__) self.config = config self.storage = StorageManager(self.config) self.SERVICE = 'generic' self.TARGET_URL = '' def fetch(self, api, params=dict(), method='get', secure=True, cache=True, cache_only=False, force_cache=False): """ Get data from url using parameters params If param cache is true update cache on cache miss If param cache_only is true just update the cache, don't return any data. Use this parameter to spawn external thread to update cache in background """ if cache: json_data = dict() if not force_cache: json_data = self.storage.get_cache(self.SERVICE, api, params) if not json_data: if secure: json_data = get_secure_data(target_url=self.TARGET_URL, api=api, params=params, method=method) else: json_data = get_data(target_url=self.TARGET_URL, api=api, file_=params) if type(json_data) is not dict: json_data = {'data':json_data} self.storage.insert_cache(self.SERVICE, api, params, json_data) if not cache_only: return json_data else: if secure: json_data = get_secure_data(target_url=self.TARGET_URL, api=api, params=params, method=method) else: json_data = get_data(target_url=self.TARGET_URL, api=api, file_=params) if type(json_data) is not dict: json_data = {'data':json_data} return json_data
class StorageTests(unittest.TestCase): """ A test class for service classes """ def setUp(self): "Set up for test" self.config = get_config(path=opt_path, file_name='test.cfg') self.storage = StorageManager(config=self.config) self.storage.drop_db() def tearDown(self): "Clean up" coll = 'test' query = dict() self.storage.delete_data(coll=coll, query=query) pipeline = list() match = {'$match':{}} pipeline.append(match) expected = list() result = self.storage.get_data(coll=coll, pipeline=pipeline) self.assertEqual(result, expected) self.storage.drop_db() #@unittest.skip("Skip Test") def test_cache(self): "Test storage cache" print "" phedex = PhEDExService(config=self.config) api = 'data' params = {'level':'block', 'dataset':'/DoubleElectron/Run2012D-22Jan2013-v1/AOD'} expected = '/DoubleElectron/Run2012D-22Jan2013-v1/AOD' phedex.fetch(api=api, params=params, cache_only=True, force_cache=True) cache_data = self.storage.get_cache(coll='phedex', api=api, params=params) try: result = cache_data['phedex']['dbs'][0]['dataset'][0]['name'] except KeyError: self.assertTrue(False) else: self.assertEqual(result, expected) #@unittest.skip("Skip Test") def test_data(self): "Test general collection manipulation functions" coll = 'test' # insert data = [{'foo':'bar_1'}, {'foo':'bar_2'}] self.storage.insert_data(coll=coll, data=data) # get pipeline = list() match = {'$match':{'foo':'bar_2'}} pipeline.append(match) data = self.storage.get_data(coll=coll, pipeline=pipeline) expected = 'bar_2' result = data[0]['foo'] self.assertEqual(result, expected) # update query = {'foo':'bar_1'} data = {'$set':{'foo':'bar_3'}} self.storage.update_data(coll=coll, query=query, data=data) pipeline = list() match = {'$match':{'foo':'bar_3'}} pipeline.append(match) data = self.storage.get_data(coll=coll, pipeline=pipeline) expected = 'bar_3' result = data[0]['foo'] self.assertEqual(result, expected) # last insert timestamp data = [{'foo':'bar_4'}] datetime_1 = datetime.utcnow().replace(microsecond=0) self.storage.insert_data(coll=coll, data=data) datetime_2 = self.storage.get_last_insert_time(coll) self.assertTrue(datetime_1 <= datetime_2)