Ejemplo n.º 1
0
def simple_test():
    cf = ChannelFinderClient(**cfinput)
    #cf.set(property=Property('test-prop1', 'lyyang', 'v1'))
    #cf.set(property=Property('test-prop2', 'cf-asd', 'v2'))
    #cf.set(property=Property('test-prop3', 'lyyang'))
    #cf.set(property=Property('test-prop4', 'lyyang'))
    #cf.delete(propertyName='test-prop1')
    #cf.delete(propertyName='test-prop3')

    pvname1 = 'SR:C00-BI:G00{DCCT:00}CUR-RB'
    pvname2 = 'SR:C00-Glb:G00{CHROM:00}RB-X'
    pvname3 = 'SR:C00-Glb:G00{CHROM:00}RB-Y'
    cf.update(property=Property('test-prop2', 'cf-asd', 'v1'),
              channelNames=[pvname1, pvname2, pvname3])
    #chs = cf.find(property=[('test-prop2', '*')])
    #if chs is not None:
    #    for ch in chs:
    #        print ch.Name
    #        for p in ch.Properties:
    #            print "  ", p.Name,"=",p.Value, p.Owner, ", "
    #        print "/"

    cf.delete(property=Property('test-prop2', 'cf-asd', 'v1'),
              channelNames=[pvname2, pvname3])
    chs = cf.find(property=[('test-prop2', '*')])
    if chs is not None:
        for ch in chs:
            print ch.Name,
            for p in ch.Properties:
                print p.Name, "=", p.Value, ", ",
            print " /"
Ejemplo n.º 2
0
def simple_test():
    cf = ChannelFinderClient(**cfinput)
    #cf.set(property=Property('test-prop1', 'lyyang', 'v1'))
    #cf.set(property=Property('test-prop2', 'cf-asd', 'v2'))
    #cf.set(property=Property('test-prop3', 'lyyang'))
    #cf.set(property=Property('test-prop4', 'lyyang'))
    #cf.delete(propertyName='test-prop1')
    #cf.delete(propertyName='test-prop3')

    pvname1='SR:C00-BI:G00{DCCT:00}CUR-RB'
    pvname2='SR:C00-Glb:G00{CHROM:00}RB-X'
    pvname3='SR:C00-Glb:G00{CHROM:00}RB-Y'
    cf.update(property=Property('test-prop2', 'cf-asd', 'v1'),
              channelNames=[pvname1, pvname2, pvname3])
    #chs = cf.find(property=[('test-prop2', '*')])
    #if chs is not None:
    #    for ch in chs:
    #        print ch.Name
    #        for p in ch.Properties: 
    #            print "  ", p.Name,"=",p.Value, p.Owner, ", "
    #        print "/"

    cf.delete(property=Property('test-prop2', 'cf-asd', 'v1'), 
              channelNames=[pvname2, pvname3])
    chs = cf.find(property=[('test-prop2', '*')])
    if chs is not None:
        for ch in chs:
            print ch.Name,
            for p in ch.Properties: 
                print p.Name,"=",p.Value,", ", 
            print " /"
Ejemplo n.º 3
0
def update():
    cf = ChannelFinderClient(**cfinput)

    #pv="SR:C30-MG{PS:QH1A}I:Ps1DCCT1-I"
    #cf.update(property=Property('elemType', PRPTOWNER, 'SEXT'),
    #          channelNames = [pv])
    #chs = cf.find(name=pv)
    #print "updating {0}".format([ch.Name for ch in chs])
    #print "updating {0} elemField=b0".format([ch.Name for ch in chs])
    #cf.update(property=Property('elemType', PRPTOWNER, 'SEXT'),
    #          channelNames = [ch.Name for ch in chs])
    #cf.update(property=Property('system', PRPTOWNER, 'SR'),
    #              channelNames = [ch.Name for ch in chs])
    cf.update(property=Property('elemGroups', PRPTOWNER), originalPropertyName='elemGroup')
Ejemplo n.º 4
0
def initCfs(fpv, sep=","):
    dat = {}
    for line in open(fpv, 'r').readlines():
        #pv, hdl, name, idx, fam, fld
        rec = [v.strip() for v in line.split(sep)]
        pv, prpt = rec[0], rec[1:]
        dat.setdefault(pv, [])
        dat[pv].append(tuple(rec[1:]))

    import conf
    cfinput = {
        'BaseURL': cfsurl,
        'username': conf.username,
        'password': conf.password
    }
    cf = ChannelFinderClient(**cfinput)
    prpts = [p.Name for p in cf.getAllProperties()]
    for p in ["elemName", "elemIndex", "elemType", "elemField"]:
        if p in prpts: continue
        # property owner is cf-asd
        logging.info("add new property '%s'" % p)
        cf.set(property=Property(p, "cf-asd"))

    for pv,prptsets in dat.items():
        if len(prptsets) == 1:
            hdl, name, idx, fam, fld = prptsets[0]
            #cf.update(channel=Channel(pv, "cf-update", properties=[
            #    Property("elemHandle", "cf-asd", hdl),
            #    Property("elemName",   "cf-asd", name),
            #    Property("elemIndex",  "cf-asd", idx),
            #    Property("elemType",   "cf-asd", fam),
            #    Property("elemField",  "cf-asd", fld)]))
            cf.update(property=Property("elemHandle", "cf-asd", hdl),
                      channelName=pv)
            cf.update(property=Property("elemName",   "cf-asd", name),
                      channelName=pv)
            cf.update(property=Property("elemIndex",  "cf-asd", idx),
                      channelName=pv)
            cf.update(property=Property("elemType",   "cf-asd", fam),
                      channelName=pv)
            cf.update(property=Property("elemField",  "cf-asd", fld),
                      channelName=pv)
            logging.info("Done with %s" % pv)
Ejemplo n.º 5
0
def initCfs(fpv, sep=","):
    dat = {}
    for line in open(fpv, 'r').readlines():
        #pv, hdl, name, idx, fam, fld
        rec = [v.strip() for v in line.split(sep)]
        pv, prpt = rec[0], rec[1:]
        dat.setdefault(pv, [])
        dat[pv].append(tuple(rec[1:]))

    import conf
    cfinput = {
        'BaseURL': cfsurl,
        'username': conf.username,
        'password': conf.password
    }
    cf = ChannelFinderClient(**cfinput)
    prpts = [p.Name for p in cf.getAllProperties()]
    for p in ["elemName", "elemIndex", "elemType", "elemField"]:
        if p in prpts: continue
        # property owner is cf-asd
        logging.info("add new property '%s'" % p)
        cf.set(property=Property(p, "cf-asd"))

    for pv, prptsets in dat.items():
        if len(prptsets) == 1:
            hdl, name, idx, fam, fld = prptsets[0]
            #cf.update(channel=Channel(pv, "cf-update", properties=[
            #    Property("elemHandle", "cf-asd", hdl),
            #    Property("elemName",   "cf-asd", name),
            #    Property("elemIndex",  "cf-asd", idx),
            #    Property("elemType",   "cf-asd", fam),
            #    Property("elemField",  "cf-asd", fld)]))
            cf.update(property=Property("elemHandle", "cf-asd", hdl),
                      channelName=pv)
            cf.update(property=Property("elemName", "cf-asd", name),
                      channelName=pv)
            cf.update(property=Property("elemIndex", "cf-asd", idx),
                      channelName=pv)
            cf.update(property=Property("elemType", "cf-asd", fam),
                      channelName=pv)
            cf.update(property=Property("elemField", "cf-asd", fld),
                      channelName=pv)
            logging.info("Done with %s" % pv)
Ejemplo n.º 6
0
class CFProcessor(service.Service):
    def __init__(self, name, conf):
        _log.info("CF_INIT %s", name)
        self.name, self.conf = name, conf
        self.channel_dict = defaultdict(list)
        self.iocs = dict()
        self.client = None
        self.currentTime = getCurrentTime
        self.lock = DeferredLock()

    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [
                    prop['name'] for prop in self.client.getAllProperties()
                ]

                reqd_props = {
                    'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
                }

                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                    if wl else wl
                # Are any required properties not already present on CF?
                properties = reqd_props - set(cf_props)
                # Are any whitelisted properties not already present on CF?
                # If so, add them too.
                properties.update(set(whitelist) - set(cf_props))

                owner = self.conf.get('username', 'cfstore')
                for prop in properties:
                    self.client.set(property={u'name': prop, u'owner': owner})

                self.whitelist = set(whitelist)
                _log.debug('WHITELIST = {}'.format(self.whitelist))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                self.clean_service()

    def stopService(self):
        service.Service.stopService(self)
        # Set channels to inactive and close connection to client
        self.running = 0
        self.clean_service()
        _log.info("CF_STOP")

    @defer.inlineCallbacks
    def commit(self, transaction_record):
        yield self.lock.acquire()
        try:
            yield deferToThread(self.__commit__, transaction_record)
        finally:
            self.lock.release()

    def __commit__(self, TR):
        _log.debug("CF_COMMIT %s", TR.infos.items())
        """
        a dictionary with a list of records with their associated property info  
        pvInfo 
        {rid: { "pvName":"recordName",
                "infoProperties":{propName:value, ...}}}
        """

        iocName = TR.infos.get('IOCNAME') or TR.src.port
        hostName = TR.infos.get('HOSTNAME') or TR.src.host
        owner = TR.infos.get('ENGINEER') or TR.infos.get(
            'CF_USERNAME') or self.conf.get('username', 'cfstore')
        time = self.currentTime()

        pvInfo = {}
        for rid, (rname, rtype) in TR.addrec.items():
            pvInfo[rid] = {"pvName": rname}
        for rid, (recinfos) in TR.recinfos.items():
            # find intersection of these sets
            recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
            if recinfo_wl:
                pvInfo[rid]['infoProperties'] = list()
                for infotag in recinfo_wl:
                    _log.debug('INFOTAG = {}'.format(infotag))
                    property = {
                        u'name': infotag,
                        u'owner': owner,
                        u'value': recinfos[infotag]
                    }
                    pvInfo[rid]['infoProperties'].append(property)
        _log.debug(pvInfo)

        pvNames = [info["pvName"] for rid, (info) in pvInfo.items()]

        delrec = list(TR.delrec)
        _log.info("DELETED records " + str(delrec))

        host = TR.src.host
        port = TR.src.port
        """The unique identifier for a particular IOC"""
        iocid = host + ":" + str(port)
        _log.info("CF_COMMIT: " + iocid)

        if TR.initial:
            """Add IOC to source list """
            self.iocs[iocid] = {
                "iocname": iocName,
                "hostname": hostName,
                "owner": owner,
                "time": time,
                "channelcount": 0
            }
        if not TR.connected:
            delrec.extend(self.channel_dict.keys())
        for pv in pvNames:
            self.channel_dict[pv].append(
                iocid)  # add iocname to pvName in dict
            self.iocs[iocid]["channelcount"] += 1
        for pv in delrec:
            if iocid in self.channel_dict[pv]:
                self.channel_dict[pv].remove(iocid)
                if iocid in self.iocs:
                    self.iocs[iocid]["channelcount"] -= 1
                if self.iocs[iocid]['channelcount'] == 0:
                    self.iocs.pop(iocid, None)
                elif self.iocs[iocid]['channelcount'] < 0:
                    _log.error("channel count negative!")
                if len(self.channel_dict[pv]
                       ) <= 0:  # case: channel has no more iocs
                    del self.channel_dict[pv]
        poll(__updateCF__, self.client, pvInfo, delrec, self.channel_dict,
             self.iocs, hostName, iocName, iocid, owner, time)
        dict_to_file(self.channel_dict, self.iocs, self.conf)

    def clean_service(self):
        """
        Marks all channels as "Inactive" until the recsync server is back up
        """
        sleep = 1
        retry_limit = 5
        owner = self.conf.get('username', 'cfstore')
        while 1:
            try:
                _log.debug("Cleaning service...")
                channels = self.client.findByArgs([('pvStatus', 'Active')])
                if channels is not None:
                    new_channels = []
                    for ch in channels or []:
                        new_channels.append(ch[u'name'])
                    if len(new_channels) > 0:
                        self.client.update(property={
                            u'name': 'pvStatus',
                            u'owner': owner,
                            u'value': "Inactive"
                        },
                                           channelNames=new_channels)
                    _log.debug("Service clean.")
                    return
            except RequestException:
                _log.exception("cleaning failed, retrying: ")

            time.sleep(min(60, sleep))
            sleep *= 1.5
            if self.running == 0 and sleep >= retry_limit:
                _log.debug("Abandoning clean.")
                return
Ejemplo n.º 7
0
class CFProcessor(service.Service):
    def __init__(self, name, conf):
        _log.info("CF_INIT %s", name)
        self.name, self.conf = name, conf
        self.channel_dict = defaultdict(list)
        self.iocs = dict()
        self.client = None
        self.currentTime = getCurrentTime
        self.lock = DeferredLock()

    def startService(self):
        service.Service.startService(self)
        # Returning a Deferred is not supported by startService(),
        # so instead attempt to acquire the lock synchonously!
        d = self.lock.acquire()
        if not d.called:
            d.cancel()
            service.Service.stopService(self)
            raise RuntimeError(
                'Failed to acquired CF Processor lock for service start')

        try:
            self._startServiceWithLock()
        except:
            service.Service.stopService(self)
            raise
        finally:
            self.lock.release()

    def _startServiceWithLock(self):
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [
                    prop['name'] for prop in self.client.getAllProperties()
                ]
                if (self.conf.get('alias', 'default') == 'on'):
                    reqd_props = {
                        'hostName', 'iocName', 'pvStatus', 'time', 'iocid',
                        'alias'
                    }
                else:
                    reqd_props = {
                        'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
                    }
                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                    if wl else wl
                # Are any required properties not already present on CF?
                properties = reqd_props - set(cf_props)
                # Are any whitelisted properties not already present on CF?
                # If so, add them too.
                properties.update(set(whitelist) - set(cf_props))

                owner = self.conf.get('username', 'cfstore')
                for prop in properties:
                    self.client.set(property={u'name': prop, u'owner': owner})

                self.whitelist = set(whitelist)
                _log.debug('WHITELIST = {}'.format(self.whitelist))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                if self.conf.getboolean('cleanOnStart', True):
                    self.clean_service()

    def stopService(self):
        service.Service.stopService(self)
        return self.lock.run(self._stopServiceWithLock)

    def _stopServiceWithLock(self):
        # Set channels to inactive and close connection to client
        if self.conf.getboolean('cleanOnStop', True):
            self.clean_service()
        _log.info("CF_STOP")

    # @defer.inlineCallbacks # Twisted v16 does not support cancellation!
    def commit(self, transaction_record):
        return self.lock.run(self._commitWithLock, transaction_record)

    def _commitWithLock(self, TR):
        self.cancelled = False

        t = deferToThread(self._commitWithThread, TR)

        def cancelCommit(d):
            self.cancelled = True
            d.callback(None)

        d = defer.Deferred(cancelCommit)

        def waitForThread(_ignored):
            if self.cancelled:
                return t

        d.addCallback(waitForThread)

        def chainError(err):
            if not err.check(defer.CancelledError):
                _log.error("CF_COMMIT FAILURE: %s", err)
            if self.cancelled:
                if not err.check(defer.CancelledError):
                    raise defer.CancelledError()
                return err
            else:
                d.callback(None)

        def chainResult(_ignored):
            if self.cancelled:
                raise defer.CancelledError()
            else:
                d.callback(None)

        t.addCallbacks(chainResult, chainError)
        return d

    def _commitWithThread(self, TR):
        if not self.running:
            raise defer.CancelledError(
                'CF Processor is not running (TR: %s:%s)', TR.src.host,
                TR.src.port)

        _log.info("CF_COMMIT: %s", TR)
        """
        a dictionary with a list of records with their associated property info  
        pvInfo 
        {rid: { "pvName":"recordName",
                "infoProperties":{propName:value, ...}}}
        """

        host = TR.src.host
        port = TR.src.port
        iocName = TR.infos.get('IOCNAME') or TR.src.port
        hostName = TR.infos.get('HOSTNAME') or TR.src.host
        owner = TR.infos.get('ENGINEER') or TR.infos.get(
            'CF_USERNAME') or self.conf.get('username', 'cfstore')
        time = self.currentTime()
        """The unique identifier for a particular IOC"""
        iocid = host + ":" + str(port)

        pvInfo = {}
        for rid, (rname, rtype) in TR.addrec.items():
            pvInfo[rid] = {"pvName": rname}
        for rid, (recinfos) in TR.recinfos.items():
            # find intersection of these sets
            if rid not in pvInfo:
                _log.warn('IOC: %s: PV not found for recinfo with RID: %s',
                          iocid, rid)
                continue
            recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
            if recinfo_wl:
                pvInfo[rid]['infoProperties'] = list()
                for infotag in recinfo_wl:
                    property = {
                        u'name': infotag,
                        u'owner': owner,
                        u'value': recinfos[infotag]
                    }
                    pvInfo[rid]['infoProperties'].append(property)
        for rid, alias in TR.aliases.items():
            if rid not in pvInfo:
                _log.warn('IOC: %s: PV not found for alias with RID: %s',
                          iocid, rid)
                continue
            pvInfo[rid]['aliases'] = alias

        delrec = list(TR.delrec)
        _log.debug("Delete records: %s", delrec)

        pvInfoByName = {}
        for rid, (info) in pvInfo.items():
            if info["pvName"] in pvInfoByName:
                _log.warn(
                    "Commit contains multiple records with PV name: %s (%s)",
                    pv, iocid)
                continue
            pvInfoByName[info["pvName"]] = info
            _log.debug("Add record: %s: %s", rid, info)

        if TR.initial:
            """Add IOC to source list """
            self.iocs[iocid] = {
                "iocname": iocName,
                "hostname": hostName,
                "owner": owner,
                "time": time,
                "channelcount": 0
            }
        if not TR.connected:
            delrec.extend(self.channel_dict.keys())
        for pv in pvInfoByName.keys():
            self.channel_dict[pv].append(
                iocid)  # add iocname to pvName in dict
            self.iocs[iocid]["channelcount"] += 1
            """In case, alias exists"""
            if (self.conf.get('alias', 'default' == 'on')):
                if pv in pvInfoByName and "aliases" in pvInfoByName[pv]:
                    for a in pvInfoByName[pv]["aliases"]:
                        self.channel_dict[a].append(
                            iocid)  # add iocname to pvName in dict
                        self.iocs[iocid]["channelcount"] += 1
        for pv in delrec:
            if iocid in self.channel_dict[pv]:
                self.channel_dict[pv].remove(iocid)
                if iocid in self.iocs:
                    self.iocs[iocid]["channelcount"] -= 1
                if self.iocs[iocid]['channelcount'] == 0:
                    self.iocs.pop(iocid, None)
                elif self.iocs[iocid]['channelcount'] < 0:
                    _log.error("Channel count negative: %s", iocid)
                if len(self.channel_dict[pv]
                       ) <= 0:  # case: channel has no more iocs
                    del self.channel_dict[pv]
                """In case, alias exists"""
                if (self.conf.get('alias', 'default' == 'on')):
                    if pv in pvInfoByName and "aliases" in pvInfoByName[pv]:
                        for a in pvInfoByName[pv]["aliases"]:
                            self.channel_dict[a].remove(iocid)
                            if iocid in self.iocs:
                                self.iocs[iocid]["channelcount"] -= 1
                            if self.iocs[iocid]['channelcount'] == 0:
                                self.iocs.pop(iocid, None)
                            elif self.iocs[iocid]['channelcount'] < 0:
                                _log.error("Channel count negative: %s", iocid)
                            if len(self.channel_dict[a]
                                   ) <= 0:  # case: channel has no more iocs
                                del self.channel_dict[a]
        poll(__updateCF__, self, pvInfoByName, delrec, hostName, iocName,
             iocid, owner, time)
        dict_to_file(self.channel_dict, self.iocs, self.conf)

    def clean_service(self):
        """
        Marks all channels as "Inactive" until the recsync server is back up
        """
        sleep = 1
        retry_limit = 5
        owner = self.conf.get('username', 'cfstore')
        while 1:
            try:
                _log.info("CF Clean Started")
                channels = self.client.findByArgs(
                    prepareFindArgs(self.conf, [('pvStatus', 'Active')]))
                if channels is not None:
                    new_channels = []
                    for ch in channels or []:
                        new_channels.append(ch[u'name'])
                    _log.info("Total channels to update: %s",
                              len(new_channels))
                    while len(new_channels) > 0:
                        _log.debug(
                            'Update "pvStatus" property to "Inactive" for %s channels',
                            min(len(new_channels), 10000))
                        self.client.update(property={
                            u'name': 'pvStatus',
                            u'owner': owner,
                            u'value': "Inactive"
                        },
                                           channelNames=new_channels[:10000])
                        new_channels = new_channels[10000:]
                    _log.info("CF Clean Completed")
                    return
            except RequestException as e:
                _log.error("Clean service failed: %s", e)

            _log.info("Clean service retry in %s seconds", min(60, sleep))
            time.sleep(min(60, sleep))
            sleep *= 1.5
            if self.running == 0 and sleep >= retry_limit:
                _log.info("Abandoning clean after %s seconds", retry_limit)
                return
Ejemplo n.º 8
0
class CFProcessor(service.Service):
    implements(interfaces.IProcessor)

    def __init__(self, name, conf):
        _log.info("CF_INIT %s", name)
        self.name, self.conf = name, conf
        self.channel_dict = defaultdict(list)
        self.iocs = dict()
        self.client = None
        self.currentTime = getCurrentTime
        self.lock = DeferredLock()

    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")
        from channelfinder import ChannelFinderClient
        # Using the default python cf-client.
        # The usr, username, and password are provided by the channelfinder._conf module.
        if self.client is None:  # For setting up mock test client
            self.client = ChannelFinderClient()
        self.clean_service()

    def stopService(self):
        service.Service.stopService(self)
        #Set channels to inactive and close connection to client
        self.running = 0
        self.clean_service()
        _log.info("CF_STOP")

    @defer.inlineCallbacks
    def commit(self, transaction_record):
        yield self.lock.acquire()
        try:
            yield deferToThread(self.__commit__, transaction_record)
        finally:
            self.lock.release()

    def __commit__(self, TR):
        _log.debug("CF_COMMIT %s", TR.infos.items())
        pvNames = [unicode(rname, "utf-8") for rid, (rname, rtype) in TR.addrec.iteritems()]
        delrec = list(TR.delrec)
        iocName = TR.src.port
        hostName = TR.src.host
        iocid = hostName + ":" + str(iocName)
        owner = TR.infos.get('CF_USERNAME') or TR.infos.get('ENGINEER') or self.conf.get('username', 'cfstore')
        time = self.currentTime()
        if TR.initial:
            self.iocs[iocid] = {"iocname": iocName, "hostname": hostName, "owner": owner, "channelcount": 0}  # add IOC to source list
        if not TR.connected:
            delrec.extend(self.channel_dict.keys())
        for pv in pvNames:
            self.channel_dict[pv].append(iocid)  # add iocname to pvName in dict
            self.iocs[iocid]["channelcount"] += 1
        for pv in delrec:
            if iocid in self.channel_dict[pv]:
                self.channel_dict[pv].remove(iocid)
                self.iocs[iocid]["channelcount"] -= 1
                if self.iocs[iocid]['channelcount'] == 0:
                    self.iocs.pop(iocid, None)
                elif self.iocs[iocid]['channelcount'] < 0:
                    _log.error("channel count negative!")
                if len(self.channel_dict[pv]) <= 0:  # case: channel has no more iocs
                    del self.channel_dict[pv]
        poll(__updateCF__, self.client, pvNames, delrec, self.channel_dict, self.iocs, hostName, iocName, time, owner)
        dict_to_file(self.channel_dict, self.iocs, self.conf)

    def clean_service(self):
        sleep = 1
        retry_limit = 5
        owner = self.conf.get('username', 'cfstore')
        while 1:
            try:
                _log.debug("Cleaning service...")
                channels = self.client.findByArgs([('pvStatus', 'Active')])
                if channels is not None:
                    new_channels = []
                    for ch in channels or []:
                        new_channels.append(ch[u'name'])
                    if len(new_channels) > 0:
                        self.client.update(property={u'name': 'pvStatus', u'owner': owner, u'value': "Inactive"},
                                           channelNames=new_channels)
                    _log.debug("Service clean.")
                    return
            except RequestException:
                _log.exception("cleaning failed, retrying: ")

            time.sleep(min(60, sleep))
            sleep *= 1.5
            if self.running == 0 and sleep >= retry_limit:
                _log.debug("Abandoning clean.")
                return
Ejemplo n.º 9
0
def cfs_append_from_csv1(rec_list, update_only):
    cf = ChannelFinderClient(**cfinput)
    all_prpts = [p.Name for p in cf.getAllProperties()]
    all_tags = [t.Name for t in cf.getAllTags()]
    ignore_prpts = ['hostName', 'iocName']
    import csv
    rd = csv.reader(rec_list)
    # header line
    header = rd.next()
    # lower case of header
    hlow = [s.lower() for s in header]
    # number of headers, pv + properties
    nheader = len(header)
    # the index of PV, properties and tags
    ipv = hlow.index('pv')
    # the code did not rely on it, but it is a good practice
    if ipv != 0:
        raise RuntimeError("the first column should be pv")

    iprpt, itags = [], []
    for i, h in enumerate(header):
        if i == ipv: continue
        # if the header is empty, it is a tag
        if len(h.strip()) == 0:
            itags.append(i)
        else:
            iprpt.append(i)

    tag_owner = OWNER
    prpt_owner = PRPTOWNER
    tags = {}
    # the data body
    for s in rd:
        prpts = [Property(header[i], prpt_owner, s[i]) for i in iprpt if s[i]]
        # itags could be empty if we put all tags in the end columns
        for i in itags + range(nheader, len(s)):
            rec = tags.setdefault(s[i].strip(), [])
            rec.append(s[ipv].strip())

        #print s[ipv], prpts, tags
        ch = cf.find(name=s[ipv])
        if ch is None:
            logging.warning("pv {0} does not exist".format(s[ipv]))
        elif len(ch) > 1:
            logging.warning("pv {0} is not unique ({1})".format(
                s[ipv], len(ch)))
        else:
            for p in prpts:
                #continue
                if p.Name in ignore_prpts: continue
                #if p.Name != 'symmetry': continue
                logging.info("updating '{0}' with property, {1}={2}".format(
                    s[ipv], p.Name, p.Value))
                cf.update(channelName=s[ipv], property=p)

    logging.info("finished updating properties")
    for t, pvs in tags.items():
        if not hasTag(cf, t): cf.set(tag=Tag(t, tag_owner))
        if 'V:1-SR-BI{BETA}X-I' in pvs: continue
        if 'V:1-SR-BI{BETA}Y-I' in pvs: continue
        try:
            cf.update(tag=Tag(t, tag_owner), channelNames=pvs)
        except:
            print(t, pvs)
            raise

        logging.info("update '{0}' for {1} pvs".format(t, len(pvs)))
    logging.info("finished updating tags")
def _export_to_cfweb(channels, uri, username, password):
    # Improve performance by reducing the number
    # of HTTP server requests by restructuring
    # the channel data from channel oriented:
    #
    # {
    #    "CH:NAME1": {
    #           "properties":{
    #                 "system":"SEG1",
    #                 "device":"DEV1"
    #            },
    #            "tags":[
    #                "phyutil.sys.LS1",
    #                "phyutil.sub.CA01"
    #            ]
    #    },
    #    "CH:NAME2: {
    #            "properties":{
    #                 "system":"SEG1",
    #                 "device":"DEV2"
    #            }
    #            "tags":[
    #                "phyutil.sys.LS1",
    #                "phyutil.sub.BTS"
    #            ]
    #    }
    # }
    #
    # To property oriented:
    #
    # {
    #    "system" : {
    #        "SEG1": [ "CH:NAME1", "CH:NAME2" ]
    #    },
    #    "device" : {
    #        "DEV1": [ "CH:NAME1" ],
    #        "DEV2": [ "CH:NAME2" ]
    #    }
    # }
    #
    # And tag oriented:
    #
    # {
    #     "phyutil.sys.LS1": [ "CH:NAME1", "CH:NAME2" ],
    #     "phyutil.sub.CA01": [ "CH:NAME1" ],
    #     "phyutil.sub.BTS": [ "CH:NAME2" ]
    # }
    #
    tags = {}
    properties = {}
    channelnames = set()
    for name, ps, ts in channels:
        for p, value in ps.iteritems():
            if p not in properties:
                properties[p] = {}

            v = str(value)
            if v not in properties[p]:
                properties[p][v] = set()

            properties[p][v].add(name)

        for t in ts:
            if t not in tags:
                tags[t] = set()

            tags[t].add(name)

        channelnames.add(name)

    client = ChannelFinderClient(BaseURL=uri, username=username, password=password)

    channels = []
    for cname in channelnames:
        c = Channel(cname, username)
        channels.append(c)
        _LOGGER.debug("Export to Channel Finder: Set channel: %s", _Chan(c))
    client.set(channels=channels)
    _LOGGER.info("Export to ChannelFinder: Set channels: %s", len(channels))

    for prop, values in properties.iteritems():
        p = Property(prop, username)
        client.set(property=p)
        _LOGGER.info("Export to ChannelFinder: Set property: %s", _Prop(p))
        for propvalue, channelnames in values.iteritems():
            p.Value = propvalue
            client.update(property=p, channelNames=channelnames)
            _LOGGER.debug("Export to ChannelFinder: Update property: %s, for channels: %s", _Prop(p), len(channelnames))

    for tag, channelnames in tags.iteritems():
        t = Tag(tag, username)
        client.set(tag=t)
        _LOGGER.info("Export to ChannelFinder: Set tag: %s", _Tag(t))
        client.update(tag=t, channelNames=channelnames)
        _LOGGER.debug("Export to ChannelFinder: Update tag: %s, for channels: %s", _Tag(t), len(channelnames))
Ejemplo n.º 11
0
class CFProcessor(service.Service):
    def __init__(self, name, conf):
        _log.info("CF_INIT %s", name)
        self.name, self.conf = name, conf
        self.channel_dict = defaultdict(list)
        self.iocs = dict()
        self.client = None
        self.currentTime = getCurrentTime
        self.lock = DeferredLock()

    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [prop['name'] for prop in self.client.getAllProperties()]
                if (self.conf.get('alias', 'default') == 'on'):
                    reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid', 'alias'}
                else:
                    reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid'}
                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                    if wl else wl
                # Are any required properties not already present on CF?
                properties = reqd_props - set(cf_props)
                # Are any whitelisted properties not already present on CF?
                # If so, add them too.
                properties.update(set(whitelist) - set(cf_props))

                owner = self.conf.get('username', 'cfstore')
                for prop in properties:
                    self.client.set(property={u'name': prop, u'owner': owner})

                self.whitelist = set(whitelist)
                _log.debug('WHITELIST = {}'.format(self.whitelist))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                self.clean_service()

    def stopService(self):
        service.Service.stopService(self)
        # Set channels to inactive and close connection to client
        self.running = 0
        self.clean_service()
        _log.info("CF_STOP")

    @defer.inlineCallbacks
    def commit(self, transaction_record):
        yield self.lock.acquire()
        try:
            yield deferToThread(self.__commit__, transaction_record)
        finally:
            self.lock.release()

    def __commit__(self, TR):
        _log.debug("CF_COMMIT %s", TR.infos.items())
        """
        a dictionary with a list of records with their associated property info  
        pvInfo 
        {rid: { "pvName":"recordName",
                "infoProperties":{propName:value, ...}}}
        """

        iocName = TR.infos.get('IOCNAME') or TR.src.port
        hostName = TR.infos.get('HOSTNAME') or TR.src.host
        owner = TR.infos.get('ENGINEER') or TR.infos.get('CF_USERNAME') or self.conf.get('username', 'cfstore')
        time = self.currentTime()

        pvInfo = {}
        for rid, (rname, rtype) in TR.addrec.items():
            pvInfo[rid] = {"pvName": rname}
        for rid, (recinfos) in TR.recinfos.items():
            # find intersection of these sets
            recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
            if recinfo_wl:
                pvInfo[rid]['infoProperties'] = list()
                for infotag in recinfo_wl:
                    _log.debug('INFOTAG = {}'.format(infotag))
                    property = {u'name': infotag, u'owner': owner,
                                u'value': recinfos[infotag]}
                    pvInfo[rid]['infoProperties'].append(property)
        for rid, alias in TR.aliases.items():
            pvInfo[rid]['aliases'] = alias
        _log.debug(pvInfo)

        pvNames = [info["pvName"] for rid, (info) in pvInfo.items()]

        delrec = list(TR.delrec)
        _log.info("DELETED records " + str(delrec))

        host = TR.src.host
        port = TR.src.port

        """The unique identifier for a particular IOC"""
        iocid = host + ":" + str(port)
        _log.info("CF_COMMIT: " + iocid)

        if TR.initial:
            """Add IOC to source list """
            self.iocs[iocid] = {"iocname": iocName, "hostname": hostName, "owner": owner, "time": time,
                                "channelcount": 0}
        if not TR.connected:
            delrec.extend(self.channel_dict.keys())
        for pv in pvNames:
            self.channel_dict[pv].append(iocid)  # add iocname to pvName in dict
            self.iocs[iocid]["channelcount"] += 1
            """In case, alias exists"""
            if (self.conf.get('alias', 'default' == 'on')):
                al = [info["aliases"] for rid, (info) in pvInfo.items() if info["pvName"] == pv and "aliases" in info ]
                if len(al) == 1:
                    ali = al[0]
                    for a in ali:
                        self.channel_dict[a].append(iocid)  # add iocname to pvName in dict
                        self.iocs[iocid]["channelcount"] += 1
        for pv in delrec:
            if iocid in self.channel_dict[pv]:
                self.channel_dict[pv].remove(iocid)
                if iocid in self.iocs:
                    self.iocs[iocid]["channelcount"] -= 1
                if self.iocs[iocid]['channelcount'] == 0:
                    self.iocs.pop(iocid, None)
                elif self.iocs[iocid]['channelcount'] < 0:
                    _log.error("channel count negative!")
                if len(self.channel_dict[pv]) <= 0:  # case: channel has no more iocs
                    del self.channel_dict[pv]
                """In case, alias exists"""
                if (self.conf.get('alias', 'default' == 'on')):
                    al = [info["aliases"] for rid, (info) in pvInfo.items() if info["pvName"] == pv and "aliases" in info ]
                    if len(al) == 1:
                        ali = al[0]
                        for a in ali:
                            self.channel_dict[a].remove(iocid)
                            if iocid in self.iocs:
                                self.iocs[iocid]["channelcount"] -= 1
                            if self.iocs[iocid]['channelcount'] == 0:
                                self.iocs.pop(iocid, None)
                            elif self.iocs[iocid]['channelcount'] < 0:
                                _log.error("channel count negative!")
                            if len(self.channel_dict[a]) <= 0:  # case: channel has no more iocs
                                del self.channel_dict[a]
        poll(__updateCF__, self.client, pvInfo, delrec, self.channel_dict, self.iocs, self.conf, hostName, iocName, iocid,
             owner, time)
        dict_to_file(self.channel_dict, self.iocs, self.conf)

    def clean_service(self):
        """
        Marks all channels as "Inactive" until the recsync server is back up
        """
        sleep = 1
        retry_limit = 5
        owner = self.conf.get('username', 'cfstore')
        while 1:
            try:
                _log.debug("Cleaning service...")
                channels = self.client.findByArgs([('pvStatus', 'Active')])
                if channels is not None:
                    new_channels = []
                    for ch in channels or []:
                        new_channels.append(ch[u'name'])
                    if len(new_channels) > 0:
                        self.client.update(property={u'name': 'pvStatus', u'owner': owner, u'value': "Inactive"},
                                           channelNames=new_channels)
                    _log.debug("Service clean.")
                    return
            except RequestException:
                _log.exception("cleaning failed, retrying: ")

            time.sleep(min(60, sleep))
            sleep *= 1.5
            if self.running == 0 and sleep >= retry_limit:
                _log.debug("Abandoning clean.")
                return
Ejemplo n.º 12
0
def cfs_append_from_csv1(rec_list, update_only):
    cf = ChannelFinderClient(**cfinput)
    all_prpts = [p.Name for p in cf.getAllProperties()]
    all_tags  = [t.Name for t in cf.getAllTags()]
    ignore_prpts = ['hostName', 'iocName']
    import csv
    rd = csv.reader(rec_list)
    # header line
    header = rd.next()
    # lower case of header
    hlow = [s.lower() for s in header]
    # number of headers, pv + properties
    nheader = len(header)
    # the index of PV, properties and tags
    ipv = hlow.index('pv')
    # the code did not rely on it, but it is a good practice
    if ipv != 0:
        raise RuntimeError("the first column should be pv")

    iprpt, itags = [], []
    for i, h in enumerate(header):
        if i == ipv: continue
        # if the header is empty, it is a tag
        if len(h.strip()) == 0: 
            itags.append(i)
        else:
            iprpt.append(i)

    tag_owner = OWNER
    prpt_owner = PRPTOWNER
    tags = {}
    # the data body
    for s in rd:
        prpts = [Property(header[i], prpt_owner, s[i]) for i in iprpt if s[i]]
        # itags could be empty if we put all tags in the end columns
        for i in itags + range(nheader, len(s)):
            rec = tags.setdefault(s[i].strip(), [])
            rec.append(s[ipv].strip())

        #print s[ipv], prpts, tags
        ch = cf.find(name=s[ipv])
        if ch is None:
            logging.warning("pv {0} does not exist".format(s[ipv]))
        elif len(ch) > 1:
            logging.warning("pv {0} is not unique ({1})".format(s[ipv], len(ch)))
        else:
            for p in prpts:
                #continue
                if p.Name in ignore_prpts: continue
                #if p.Name != 'symmetry': continue
                logging.info("updating '{0}' with property, {1}={2}".format(
                        s[ipv], p.Name, p.Value))
                cf.update(channelName=s[ipv], property=p)

    logging.info("finished updating properties")
    for t,pvs in tags.iteritems():
        if not hasTag(cf, t): cf.set(tag=Tag(t, tag_owner))
        if 'V:1-SR-BI{BETA}X-I' in pvs: continue
        if 'V:1-SR-BI{BETA}Y-I' in pvs: continue
        try:
            cf.update(tag=Tag(t, tag_owner), channelNames=pvs)
        except:
            print t, pvs
            raise

        logging.info("update '{0}' for {1} pvs".format(t, len(pvs)))
    logging.info("finished updating tags")