class CFProcessor(service.Service): def __init__(self, name, conf): _log.info("CF_INIT %s", name) self.name, self.conf = name, conf self.channel_dict = defaultdict(list) self.iocs = dict() self.client = None self.currentTime = getCurrentTime self.lock = DeferredLock() def startService(self): service.Service.startService(self) self.running = 1 _log.info("CF_START") if self.client is None: # For setting up mock test client """ Using the default python cf-client. The url, username, and password are provided by the channelfinder._conf module. """ from channelfinder import ChannelFinderClient self.client = ChannelFinderClient() try: cf_props = [ prop['name'] for prop in self.client.getAllProperties() ] reqd_props = { 'hostName', 'iocName', 'pvStatus', 'time', 'iocid' } wl = self.conf.get('infotags', list()) whitelist = [s.strip(', ') for s in wl.split()] \ if wl else wl # Are any required properties not already present on CF? properties = reqd_props - set(cf_props) # Are any whitelisted properties not already present on CF? # If so, add them too. properties.update(set(whitelist) - set(cf_props)) owner = self.conf.get('username', 'cfstore') for prop in properties: self.client.set(property={u'name': prop, u'owner': owner}) self.whitelist = set(whitelist) _log.debug('WHITELIST = {}'.format(self.whitelist)) except ConnectionError: _log.exception("Cannot connect to Channelfinder service") raise else: self.clean_service() def stopService(self): service.Service.stopService(self) # Set channels to inactive and close connection to client self.running = 0 self.clean_service() _log.info("CF_STOP") @defer.inlineCallbacks def commit(self, transaction_record): yield self.lock.acquire() try: yield deferToThread(self.__commit__, transaction_record) finally: self.lock.release() def __commit__(self, TR): _log.debug("CF_COMMIT %s", TR.infos.items()) """ a dictionary with a list of records with their associated property info pvInfo {rid: { "pvName":"recordName", "infoProperties":{propName:value, ...}}} """ iocName = TR.infos.get('IOCNAME') or TR.src.port hostName = TR.infos.get('HOSTNAME') or TR.src.host owner = TR.infos.get('ENGINEER') or TR.infos.get( 'CF_USERNAME') or self.conf.get('username', 'cfstore') time = self.currentTime() pvInfo = {} for rid, (rname, rtype) in TR.addrec.items(): pvInfo[rid] = {"pvName": rname} for rid, (recinfos) in TR.recinfos.items(): # find intersection of these sets recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()] if recinfo_wl: pvInfo[rid]['infoProperties'] = list() for infotag in recinfo_wl: _log.debug('INFOTAG = {}'.format(infotag)) property = { u'name': infotag, u'owner': owner, u'value': recinfos[infotag] } pvInfo[rid]['infoProperties'].append(property) _log.debug(pvInfo) pvNames = [info["pvName"] for rid, (info) in pvInfo.items()] delrec = list(TR.delrec) _log.info("DELETED records " + str(delrec)) host = TR.src.host port = TR.src.port """The unique identifier for a particular IOC""" iocid = host + ":" + str(port) _log.info("CF_COMMIT: " + iocid) if TR.initial: """Add IOC to source list """ self.iocs[iocid] = { "iocname": iocName, "hostname": hostName, "owner": owner, "time": time, "channelcount": 0 } if not TR.connected: delrec.extend(self.channel_dict.keys()) for pv in pvNames: self.channel_dict[pv].append( iocid) # add iocname to pvName in dict self.iocs[iocid]["channelcount"] += 1 for pv in delrec: if iocid in self.channel_dict[pv]: self.channel_dict[pv].remove(iocid) if iocid in self.iocs: self.iocs[iocid]["channelcount"] -= 1 if self.iocs[iocid]['channelcount'] == 0: self.iocs.pop(iocid, None) elif self.iocs[iocid]['channelcount'] < 0: _log.error("channel count negative!") if len(self.channel_dict[pv] ) <= 0: # case: channel has no more iocs del self.channel_dict[pv] poll(__updateCF__, self.client, pvInfo, delrec, self.channel_dict, self.iocs, hostName, iocName, iocid, owner, time) dict_to_file(self.channel_dict, self.iocs, self.conf) def clean_service(self): """ Marks all channels as "Inactive" until the recsync server is back up """ sleep = 1 retry_limit = 5 owner = self.conf.get('username', 'cfstore') while 1: try: _log.debug("Cleaning service...") channels = self.client.findByArgs([('pvStatus', 'Active')]) if channels is not None: new_channels = [] for ch in channels or []: new_channels.append(ch[u'name']) if len(new_channels) > 0: self.client.update(property={ u'name': 'pvStatus', u'owner': owner, u'value': "Inactive" }, channelNames=new_channels) _log.debug("Service clean.") return except RequestException: _log.exception("cleaning failed, retrying: ") time.sleep(min(60, sleep)) sleep *= 1.5 if self.running == 0 and sleep >= retry_limit: _log.debug("Abandoning clean.") return
class CFProcessor(service.Service): def __init__(self, name, conf): _log.info("CF_INIT %s", name) self.name, self.conf = name, conf self.channel_dict = defaultdict(list) self.iocs = dict() self.client = None self.currentTime = getCurrentTime self.lock = DeferredLock() def startService(self): service.Service.startService(self) # Returning a Deferred is not supported by startService(), # so instead attempt to acquire the lock synchonously! d = self.lock.acquire() if not d.called: d.cancel() service.Service.stopService(self) raise RuntimeError( 'Failed to acquired CF Processor lock for service start') try: self._startServiceWithLock() except: service.Service.stopService(self) raise finally: self.lock.release() def _startServiceWithLock(self): _log.info("CF_START") if self.client is None: # For setting up mock test client """ Using the default python cf-client. The url, username, and password are provided by the channelfinder._conf module. """ from channelfinder import ChannelFinderClient self.client = ChannelFinderClient() try: cf_props = [ prop['name'] for prop in self.client.getAllProperties() ] if (self.conf.get('alias', 'default') == 'on'): reqd_props = { 'hostName', 'iocName', 'pvStatus', 'time', 'iocid', 'alias' } else: reqd_props = { 'hostName', 'iocName', 'pvStatus', 'time', 'iocid' } wl = self.conf.get('infotags', list()) whitelist = [s.strip(', ') for s in wl.split()] \ if wl else wl # Are any required properties not already present on CF? properties = reqd_props - set(cf_props) # Are any whitelisted properties not already present on CF? # If so, add them too. properties.update(set(whitelist) - set(cf_props)) owner = self.conf.get('username', 'cfstore') for prop in properties: self.client.set(property={u'name': prop, u'owner': owner}) self.whitelist = set(whitelist) _log.debug('WHITELIST = {}'.format(self.whitelist)) except ConnectionError: _log.exception("Cannot connect to Channelfinder service") raise else: if self.conf.getboolean('cleanOnStart', True): self.clean_service() def stopService(self): service.Service.stopService(self) return self.lock.run(self._stopServiceWithLock) def _stopServiceWithLock(self): # Set channels to inactive and close connection to client if self.conf.getboolean('cleanOnStop', True): self.clean_service() _log.info("CF_STOP") # @defer.inlineCallbacks # Twisted v16 does not support cancellation! def commit(self, transaction_record): return self.lock.run(self._commitWithLock, transaction_record) def _commitWithLock(self, TR): self.cancelled = False t = deferToThread(self._commitWithThread, TR) def cancelCommit(d): self.cancelled = True d.callback(None) d = defer.Deferred(cancelCommit) def waitForThread(_ignored): if self.cancelled: return t d.addCallback(waitForThread) def chainError(err): if not err.check(defer.CancelledError): _log.error("CF_COMMIT FAILURE: %s", err) if self.cancelled: if not err.check(defer.CancelledError): raise defer.CancelledError() return err else: d.callback(None) def chainResult(_ignored): if self.cancelled: raise defer.CancelledError() else: d.callback(None) t.addCallbacks(chainResult, chainError) return d def _commitWithThread(self, TR): if not self.running: raise defer.CancelledError( 'CF Processor is not running (TR: %s:%s)', TR.src.host, TR.src.port) _log.info("CF_COMMIT: %s", TR) """ a dictionary with a list of records with their associated property info pvInfo {rid: { "pvName":"recordName", "infoProperties":{propName:value, ...}}} """ host = TR.src.host port = TR.src.port iocName = TR.infos.get('IOCNAME') or TR.src.port hostName = TR.infos.get('HOSTNAME') or TR.src.host owner = TR.infos.get('ENGINEER') or TR.infos.get( 'CF_USERNAME') or self.conf.get('username', 'cfstore') time = self.currentTime() """The unique identifier for a particular IOC""" iocid = host + ":" + str(port) pvInfo = {} for rid, (rname, rtype) in TR.addrec.items(): pvInfo[rid] = {"pvName": rname} for rid, (recinfos) in TR.recinfos.items(): # find intersection of these sets if rid not in pvInfo: _log.warn('IOC: %s: PV not found for recinfo with RID: %s', iocid, rid) continue recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()] if recinfo_wl: pvInfo[rid]['infoProperties'] = list() for infotag in recinfo_wl: property = { u'name': infotag, u'owner': owner, u'value': recinfos[infotag] } pvInfo[rid]['infoProperties'].append(property) for rid, alias in TR.aliases.items(): if rid not in pvInfo: _log.warn('IOC: %s: PV not found for alias with RID: %s', iocid, rid) continue pvInfo[rid]['aliases'] = alias delrec = list(TR.delrec) _log.debug("Delete records: %s", delrec) pvInfoByName = {} for rid, (info) in pvInfo.items(): if info["pvName"] in pvInfoByName: _log.warn( "Commit contains multiple records with PV name: %s (%s)", pv, iocid) continue pvInfoByName[info["pvName"]] = info _log.debug("Add record: %s: %s", rid, info) if TR.initial: """Add IOC to source list """ self.iocs[iocid] = { "iocname": iocName, "hostname": hostName, "owner": owner, "time": time, "channelcount": 0 } if not TR.connected: delrec.extend(self.channel_dict.keys()) for pv in pvInfoByName.keys(): self.channel_dict[pv].append( iocid) # add iocname to pvName in dict self.iocs[iocid]["channelcount"] += 1 """In case, alias exists""" if (self.conf.get('alias', 'default' == 'on')): if pv in pvInfoByName and "aliases" in pvInfoByName[pv]: for a in pvInfoByName[pv]["aliases"]: self.channel_dict[a].append( iocid) # add iocname to pvName in dict self.iocs[iocid]["channelcount"] += 1 for pv in delrec: if iocid in self.channel_dict[pv]: self.channel_dict[pv].remove(iocid) if iocid in self.iocs: self.iocs[iocid]["channelcount"] -= 1 if self.iocs[iocid]['channelcount'] == 0: self.iocs.pop(iocid, None) elif self.iocs[iocid]['channelcount'] < 0: _log.error("Channel count negative: %s", iocid) if len(self.channel_dict[pv] ) <= 0: # case: channel has no more iocs del self.channel_dict[pv] """In case, alias exists""" if (self.conf.get('alias', 'default' == 'on')): if pv in pvInfoByName and "aliases" in pvInfoByName[pv]: for a in pvInfoByName[pv]["aliases"]: self.channel_dict[a].remove(iocid) if iocid in self.iocs: self.iocs[iocid]["channelcount"] -= 1 if self.iocs[iocid]['channelcount'] == 0: self.iocs.pop(iocid, None) elif self.iocs[iocid]['channelcount'] < 0: _log.error("Channel count negative: %s", iocid) if len(self.channel_dict[a] ) <= 0: # case: channel has no more iocs del self.channel_dict[a] poll(__updateCF__, self, pvInfoByName, delrec, hostName, iocName, iocid, owner, time) dict_to_file(self.channel_dict, self.iocs, self.conf) def clean_service(self): """ Marks all channels as "Inactive" until the recsync server is back up """ sleep = 1 retry_limit = 5 owner = self.conf.get('username', 'cfstore') while 1: try: _log.info("CF Clean Started") channels = self.client.findByArgs( prepareFindArgs(self.conf, [('pvStatus', 'Active')])) if channels is not None: new_channels = [] for ch in channels or []: new_channels.append(ch[u'name']) _log.info("Total channels to update: %s", len(new_channels)) while len(new_channels) > 0: _log.debug( 'Update "pvStatus" property to "Inactive" for %s channels', min(len(new_channels), 10000)) self.client.update(property={ u'name': 'pvStatus', u'owner': owner, u'value': "Inactive" }, channelNames=new_channels[:10000]) new_channels = new_channels[10000:] _log.info("CF Clean Completed") return except RequestException as e: _log.error("Clean service failed: %s", e) _log.info("Clean service retry in %s seconds", min(60, sleep)) time.sleep(min(60, sleep)) sleep *= 1.5 if self.running == 0 and sleep >= retry_limit: _log.info("Abandoning clean after %s seconds", retry_limit) return
def updateChannelFinder(pvNames, hostName, iocName, time, owner, \ service=None, username=None, password=None): ''' pvNames = list of pvNames ([] permitted will effectively remove the hostname, iocname from all channels) hostName = pv hostName (None not permitted) iocName = pv iocName (None not permitted) owner = the owner of the channels and properties being added, this can be different from the user e.g. user = abc might create a channel with owner = group-abc time = the time at which these channels are being created/modified [optional] if not specified the default values are used by the channelfinderapi lib service = channelfinder service URL username = channelfinder username password = channelfinder password ''' if hostName == None or iocName == None: raise Exception, 'missing hostName or iocName' channels = [] try: client = ChannelFinderClient(BaseURL=service, username=username, password=password) except: raise Exception, 'Unable to create a valid webResourceClient' checkPropertiesExist(client, owner) previousChannelsList = client.findByArgs([('hostName', hostName), ('iocName', iocName)]) if previousChannelsList != None: for ch in previousChannelsList: if pvNames != None and ch[u'name'] in pvNames: '''''' channels.append(updateChannel(ch,\ owner=owner, \ hostName=hostName, \ iocName=iocName, \ pvStatus='Active', \ time=time)) pvNames.remove(ch[u'name']) elif pvNames == None or ch[u'name'] not in pvNames: '''Orphan the channel : mark as inactive, keep the old hostName and iocName''' channels.append(updateChannel(ch, \ owner=owner, \ hostName=ch.getProperties()['hostName'], \ iocName=ch.getProperties()['iocName'], \ pvStatus='InActive', \ time=ch.getProperties()['time'])) # now pvNames contains a list of pv's new on this host/ioc for pv in pvNames: ch = client.findByArgs([('~name',pv)]) if ch == None: '''New channel''' channels.append(createChannel(pv, \ chOwner=owner, \ hostName=hostName, \ iocName=iocName, \ pvStatus='Active', \ time=time)) elif ch[0] != None: '''update existing channel: exists but with a different hostName and/or iocName''' channels.append(updateChannel(ch[0], \ owner=owner, \ hostName=hostName, \ iocName=iocName, \ pvStatus='Active', \ time=time)) client.set(channels=channels)
class CFProcessor(service.Service): implements(interfaces.IProcessor) def __init__(self, name, conf): _log.info("CF_INIT %s", name) self.name, self.conf = name, conf self.channel_dict = defaultdict(list) self.iocs = dict() self.client = None self.currentTime = getCurrentTime self.lock = DeferredLock() def startService(self): service.Service.startService(self) self.running = 1 _log.info("CF_START") from channelfinder import ChannelFinderClient # Using the default python cf-client. # The usr, username, and password are provided by the channelfinder._conf module. if self.client is None: # For setting up mock test client self.client = ChannelFinderClient() self.clean_service() def stopService(self): service.Service.stopService(self) #Set channels to inactive and close connection to client self.running = 0 self.clean_service() _log.info("CF_STOP") @defer.inlineCallbacks def commit(self, transaction_record): yield self.lock.acquire() try: yield deferToThread(self.__commit__, transaction_record) finally: self.lock.release() def __commit__(self, TR): _log.debug("CF_COMMIT %s", TR.infos.items()) pvNames = [unicode(rname, "utf-8") for rid, (rname, rtype) in TR.addrec.iteritems()] delrec = list(TR.delrec) iocName = TR.src.port hostName = TR.src.host iocid = hostName + ":" + str(iocName) owner = TR.infos.get('CF_USERNAME') or TR.infos.get('ENGINEER') or self.conf.get('username', 'cfstore') time = self.currentTime() if TR.initial: self.iocs[iocid] = {"iocname": iocName, "hostname": hostName, "owner": owner, "channelcount": 0} # add IOC to source list if not TR.connected: delrec.extend(self.channel_dict.keys()) for pv in pvNames: self.channel_dict[pv].append(iocid) # add iocname to pvName in dict self.iocs[iocid]["channelcount"] += 1 for pv in delrec: if iocid in self.channel_dict[pv]: self.channel_dict[pv].remove(iocid) self.iocs[iocid]["channelcount"] -= 1 if self.iocs[iocid]['channelcount'] == 0: self.iocs.pop(iocid, None) elif self.iocs[iocid]['channelcount'] < 0: _log.error("channel count negative!") if len(self.channel_dict[pv]) <= 0: # case: channel has no more iocs del self.channel_dict[pv] poll(__updateCF__, self.client, pvNames, delrec, self.channel_dict, self.iocs, hostName, iocName, time, owner) dict_to_file(self.channel_dict, self.iocs, self.conf) def clean_service(self): sleep = 1 retry_limit = 5 owner = self.conf.get('username', 'cfstore') while 1: try: _log.debug("Cleaning service...") channels = self.client.findByArgs([('pvStatus', 'Active')]) if channels is not None: new_channels = [] for ch in channels or []: new_channels.append(ch[u'name']) if len(new_channels) > 0: self.client.update(property={u'name': 'pvStatus', u'owner': owner, u'value': "Inactive"}, channelNames=new_channels) _log.debug("Service clean.") return except RequestException: _log.exception("cleaning failed, retrying: ") time.sleep(min(60, sleep)) sleep *= 1.5 if self.running == 0 and sleep >= retry_limit: _log.debug("Abandoning clean.") return
class CFProcessor(service.Service): def __init__(self, name, conf): _log.info("CF_INIT %s", name) self.name, self.conf = name, conf self.channel_dict = defaultdict(list) self.iocs = dict() self.client = None self.currentTime = getCurrentTime self.lock = DeferredLock() def startService(self): service.Service.startService(self) self.running = 1 _log.info("CF_START") if self.client is None: # For setting up mock test client """ Using the default python cf-client. The url, username, and password are provided by the channelfinder._conf module. """ from channelfinder import ChannelFinderClient self.client = ChannelFinderClient() try: cf_props = [prop['name'] for prop in self.client.getAllProperties()] if (self.conf.get('alias', 'default') == 'on'): reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid', 'alias'} else: reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid'} wl = self.conf.get('infotags', list()) whitelist = [s.strip(', ') for s in wl.split()] \ if wl else wl # Are any required properties not already present on CF? properties = reqd_props - set(cf_props) # Are any whitelisted properties not already present on CF? # If so, add them too. properties.update(set(whitelist) - set(cf_props)) owner = self.conf.get('username', 'cfstore') for prop in properties: self.client.set(property={u'name': prop, u'owner': owner}) self.whitelist = set(whitelist) _log.debug('WHITELIST = {}'.format(self.whitelist)) except ConnectionError: _log.exception("Cannot connect to Channelfinder service") raise else: self.clean_service() def stopService(self): service.Service.stopService(self) # Set channels to inactive and close connection to client self.running = 0 self.clean_service() _log.info("CF_STOP") @defer.inlineCallbacks def commit(self, transaction_record): yield self.lock.acquire() try: yield deferToThread(self.__commit__, transaction_record) finally: self.lock.release() def __commit__(self, TR): _log.debug("CF_COMMIT %s", TR.infos.items()) """ a dictionary with a list of records with their associated property info pvInfo {rid: { "pvName":"recordName", "infoProperties":{propName:value, ...}}} """ iocName = TR.infos.get('IOCNAME') or TR.src.port hostName = TR.infos.get('HOSTNAME') or TR.src.host owner = TR.infos.get('ENGINEER') or TR.infos.get('CF_USERNAME') or self.conf.get('username', 'cfstore') time = self.currentTime() pvInfo = {} for rid, (rname, rtype) in TR.addrec.items(): pvInfo[rid] = {"pvName": rname} for rid, (recinfos) in TR.recinfos.items(): # find intersection of these sets recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()] if recinfo_wl: pvInfo[rid]['infoProperties'] = list() for infotag in recinfo_wl: _log.debug('INFOTAG = {}'.format(infotag)) property = {u'name': infotag, u'owner': owner, u'value': recinfos[infotag]} pvInfo[rid]['infoProperties'].append(property) for rid, alias in TR.aliases.items(): pvInfo[rid]['aliases'] = alias _log.debug(pvInfo) pvNames = [info["pvName"] for rid, (info) in pvInfo.items()] delrec = list(TR.delrec) _log.info("DELETED records " + str(delrec)) host = TR.src.host port = TR.src.port """The unique identifier for a particular IOC""" iocid = host + ":" + str(port) _log.info("CF_COMMIT: " + iocid) if TR.initial: """Add IOC to source list """ self.iocs[iocid] = {"iocname": iocName, "hostname": hostName, "owner": owner, "time": time, "channelcount": 0} if not TR.connected: delrec.extend(self.channel_dict.keys()) for pv in pvNames: self.channel_dict[pv].append(iocid) # add iocname to pvName in dict self.iocs[iocid]["channelcount"] += 1 """In case, alias exists""" if (self.conf.get('alias', 'default' == 'on')): al = [info["aliases"] for rid, (info) in pvInfo.items() if info["pvName"] == pv and "aliases" in info ] if len(al) == 1: ali = al[0] for a in ali: self.channel_dict[a].append(iocid) # add iocname to pvName in dict self.iocs[iocid]["channelcount"] += 1 for pv in delrec: if iocid in self.channel_dict[pv]: self.channel_dict[pv].remove(iocid) if iocid in self.iocs: self.iocs[iocid]["channelcount"] -= 1 if self.iocs[iocid]['channelcount'] == 0: self.iocs.pop(iocid, None) elif self.iocs[iocid]['channelcount'] < 0: _log.error("channel count negative!") if len(self.channel_dict[pv]) <= 0: # case: channel has no more iocs del self.channel_dict[pv] """In case, alias exists""" if (self.conf.get('alias', 'default' == 'on')): al = [info["aliases"] for rid, (info) in pvInfo.items() if info["pvName"] == pv and "aliases" in info ] if len(al) == 1: ali = al[0] for a in ali: self.channel_dict[a].remove(iocid) if iocid in self.iocs: self.iocs[iocid]["channelcount"] -= 1 if self.iocs[iocid]['channelcount'] == 0: self.iocs.pop(iocid, None) elif self.iocs[iocid]['channelcount'] < 0: _log.error("channel count negative!") if len(self.channel_dict[a]) <= 0: # case: channel has no more iocs del self.channel_dict[a] poll(__updateCF__, self.client, pvInfo, delrec, self.channel_dict, self.iocs, self.conf, hostName, iocName, iocid, owner, time) dict_to_file(self.channel_dict, self.iocs, self.conf) def clean_service(self): """ Marks all channels as "Inactive" until the recsync server is back up """ sleep = 1 retry_limit = 5 owner = self.conf.get('username', 'cfstore') while 1: try: _log.debug("Cleaning service...") channels = self.client.findByArgs([('pvStatus', 'Active')]) if channels is not None: new_channels = [] for ch in channels or []: new_channels.append(ch[u'name']) if len(new_channels) > 0: self.client.update(property={u'name': 'pvStatus', u'owner': owner, u'value': "Inactive"}, channelNames=new_channels) _log.debug("Service clean.") return except RequestException: _log.exception("cleaning failed, retrying: ") time.sleep(min(60, sleep)) sleep *= 1.5 if self.running == 0 and sleep >= retry_limit: _log.debug("Abandoning clean.") return