def __init__(self,params,queue):
'''DataUpdater base initialiser.
@param params: List of configuration parameters
@type params: List<?>
@param queue: Response queue
@type queues: Queue.Queue
'''
super(DataUpdater,self).__init__()
self.ref,self.conf,self.factory = params
self.queue = queue
#self._stop = threading.Event()
self.api = AimsApi(self.conf)
class DataUpdater(Observable):
'''Mantenence thread comtrolling data updates and api interaction.
Instantiates an amisapi instance with wrappers for initialisation of local data store
and change/resolution feed updating
'''
#et = FeatureType.ADDRESS
#ft = FeedType.FEATURES
address = None
pno = 0
changeId = 0
global aimslog
aimslog = Logger.setup()
getfeat = None
def __init__(self,params,queue):
'''DataUpdater base initialiser.
@param params: List of configuration parameters
@type params: List<?>
@param queue: Response queue
@type queues: Queue.Queue
'''
super(DataUpdater,self).__init__()
self.ref,self.conf,self.factory = params
self.queue = queue
#self._stop = threading.Event()
self.api = AimsApi(self.conf)
def setup(self,etft,sw,ne,pno):
'''Parameter setup.
@param etft: Feed/Feature identifier
@type etft: FeedRef
@param sw: South-West corner, coordinate value pair
@type sw: List<Double>{2}
@param ne: North-East corner, coordinate value pair
@type ne: List<Double>{2}
@param pno: Feed page number
@type pno: Integer
'''
self.etft = etft
self.sw,self.ne = sw,ne
self.pno = pno
def run(self):
'''Main updater run method fetching single page of addresses from API'''
aimslog.info('GET.{} {} - Page{}'.format(self.ref,self.etft,self.pno))
featlist = []
#for page in self.api.getOnePage(self.etft,self.sw,self.ne,self.pno):
# featlist.append(self.processPage(page,self.etft))
ce,pages = self.api.getOnePage(self.etft,self.sw,self.ne,self.pno)
if any(ce.values()): aimslog.error('Single-page request failure {}'.format(ce))
if pages.has_key('entities'):
for page in pages['entities']:
featlist.append(self.processPage(page,self.etft))
else:
aimslog.error('Single-page response missing entities')
self.queue.put(featlist)
self.notify(self.ref)
def processPage(self,page,etft):
'''Process an individual page. If page is resolution type optionally re-query at individual level
@param page: Processed results from pno request
@type page: Dict<?>
@param etft: Feed/Feature identifier
@type etft: FeedRef
'''
if etft.ft == FeedType.RESOLUTIONFEED and ENABLE_ENTITY_EVALUATION:
cid = self.cid(page)
ce,feat = self.api.getOneFeature(etft,cid)
if any(ce.values()): aimslog.error('Single-feature request failure {}'.format(ce))
if feat == {u'class': [u'error']}:
#if the pno request returns the not-supposed-to-happen error, it gets special treatment
aimslog.error('Invalid API response {}'.format(feat))
#return self.factory.get(model=pno['properties'])
else:
return self._processEntity(feat,cid,etft)
else:
#just return the main feedlevel address objects
return self.factory.get(model=page['properties'])
def _processEntity(self,feat,cid,etft):
'''Identify and select group, address or entity processing for a reolutionfeed feature
@param feat: dict representation of feature before object processing
@type feat: Dict
@param cid: Change ID or group change ID
@type cid: Integer
@param etft: Feed/Feature identifier
@type etft: FeedRef
@return: Instantiated feature object
'''
if feat['class'][0] == 'validation':
return self._processValidationEntity(feat)
#e = EntityValidation.getInstance(feat)# self.getEntityInstance()
#-------------------------------
# Resolution Group
elif feat['class'][0] == 'resolutiongroup':
return self._processResolutionGroup(feat,cid,etft)
# Address List
elif feat['class'][0] == 'addressresolution':
return self._processAddressResolution(feat)
#--------------------------------
# Simple Entity object
else:
return self._processSimpleEntity(self.factory.get,feat)
def _processValidationEntity(self,feat):
'''Wraps call to validation entity instantiator
@param feat: dict representation of feature before object processing
@type feat: Dict
@return: Instantiated validation Entity
'''
return EntityValidation.getInstance(feat)
def _processAddressEntity(self,feat):
'''Processes feature data into address object
@param feat: dict representation of feature before object processing
@type feat: Dict
@return: Instantiated Address entity
'''
#return EntityAddress.getInstance(feat)
return self._processSimpleEntity(FeatureFactory.getInstance(FeedRef((FeatureType.ADDRESS,FeedType.RESOLUTIONFEED))).get,feat)
def _processSimpleEntity(self,fact,feat):
'''Default processor for generic entities but the same as address resolution processor (below).
@param fact: Link to factory, object instantiation method
@type fact: <Function>
@param feat: dict representation of feature before object processing
@type feat: Dict
@return: Instantiated Address entity
'''
featurelist = []
a = fact(model=feat['properties'])
if feat.has_key('entities'):
for e in feat['entities']:
featurelist.append(self._populateEntity(e))
a._setEntities(featurelist)
return a
def _processAddressResolution(self,feat):
'''Processes entries in the addressresolution entities list
@param feat: dict representation of feature before object processing
@type feat: Dict
@return: Instantiated Address entity
'''
featurelist = []
a = self.factory.get(model=feat['properties'])
for e in feat['entities']:
featurelist.append(self._populateEntity(e))
a._setEntities(featurelist)
return a
def _processResolutionGroup(self,feat,cid,etft):
'''Processes the res-address objects in a res-group. Subsequently populates the sub entities as feature-addresses.
@param feat: dict representation of feature before object processing
@type feat: Dict
@param cid: Change ID or group change ID
@type cid: Integer
@param etft: Feed/Feature identifier
@type etft: FeedRef
@return: Instantiated feature object
'''
featurelist = []
g = self.factory.get(model=feat['properties'])#group
#HACK subst cid for cid+count string
ce,feat2 = self.api.getOneFeature(etft,'{}/address?count={}'.format(cid,MAX_FEATURE_COUNT))#group entity/adr list
if any(ce.values()): aimslog.error('Single-feature request failure {}'.format(ce))
etft2 = FeedRef((FeatureType.ADDRESS,FeedType.RESOLUTIONFEED))
factory2 = FeatureFactory.getInstance(etft2)
for f in feat2['entities']:
a = factory2.get(model=f['properties'])
elist2 = []
for e in f['entities']:
elist2.append(self._populateEntity(e))
a._setEntities(elist2)
featurelist.append(a)
g._setEntities(featurelist)
return g
def _populateEntity(self,ent):
'''Selects type and instantiates appropriate entity object.
@param ent: dict representation of feature before object processing
@type ent: Dict
'''
if ent['class'][0] == 'validation':
return self._processValidationEntity(ent)
elif ent['class'][0] == 'address':
###res factory might work here instead
#etft3 = FeedRef((FeatureType.ADDRESS,FeedType.FEATURES))
#factory3 = FeatureFactory.getInstance(etft3)
#return factory3.get(model=e['properties'])
return self._processAddressEntity(ent)
else:
return Entity.getInstance(ent)
@staticmethod
def getInstance(etft):
'''Based on the provided FeedRef this getInstance returns a group,address or user updater object
@param etft: Feed/Feature identifier
@type etft: FeedRef
'''
if etft.et == FeatureType.GROUPS: return DataUpdaterGroup
elif etft.et == FeatureType.ADDRESS: return DataUpdaterAddress
elif etft.et == FeatureType.USERS: return DataUpdaterUser
else: raise DataUpdaterSelectionException('Select Address,Groups or Users')
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def close(self):
aimslog.info('Queue {} stopped'.format(self.queue.qsize()))
self.queue.task_done()
#executed by subclass
def cid(self,_): pass
