Ejemplo n.º 1
0
def init_cor(elems, url, patl, fname):
    cf = ChannelFinderClient(BaseURL=url)
    f = open(fname, "w")
    for pvpat, hdl, name, idx, etp, fld in patl:
        chs = cf.find(name=pvpat, property=[("hostName", "*"),
                                            ("iocName","*")])
        for ch in chs:
            pv = ch.Name
            g = re.match(r"SR:(C[0-9][0-9]?)-MG{PS:([A-Z]+[0-9]+)([AB]?)}.*",
                         ch.Name)
            if not g: 
                raise RuntimeError("unknown PV: '{0}'".format(ch.Name))
            cell, fam = g.group(1), g.group(2)
            matched = []
            for e in elems:
                if e[6] != cell: continue
                if e[9].find(fam) < 0: continue
                if g.group(3) and e[8] != g.group(3): continue
                matched.append(e)

            logging.info("found {0} COR records for '{1}', cell={2}, fam={3}".format(
                len(matched), pv, cell, fam))
            if not matched:
                logging.warn("element not found for {0}".format(pv))
                raise RuntimeError("unmatched quad PV: {0}".format(ch.Name))

            if len(matched) > 1:
                msg = "duplicate data for {0}: {1}".format(pv, matched)
                logging.error(msg)
                raise RuntimeError(msg)

            e = matched[0]
            logging.info("pick {0}".format(e))
            f.write("%s,%s,%s,%s,COR,%s\n" % (ch.Name, hdl, e[0], e[1], fld))
    f.close()
Ejemplo n.º 2
0
    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [prop['name'] for prop in self.client.getAllProperties()]

                reqd_props = set(('hostName', 'iocName', 'pvStatus', 'time',
                                  'iocid'))

                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                                 if wl else wl
                self.prop_cache = (reqd_props - set(cf_props)) \
                                  | set(whitelist)

                owner = self.conf.get('username', 'cfstore')
                for prop in self.prop_cache:
                    self.client.set(property={u'name': prop, u'owner': owner})

                _log.debug('PROP_CACHE = {}'.format(self.prop_cache))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                self.clean_service()
Ejemplo n.º 3
0
    def testAddUpdateChannels(self):
        # Check the method finds the error conditions and raises exceptions
        self.assertRaises(Exception, updateChannelFinder, [[], None, None])
        self.assertRaises(Exception, updateChannelFinder,
                          [[], None, 'iocname'])
        self.assertRaises(Exception, updateChannelFinder,
                          [[], 'hostName', None])
        # create default client
        client = ChannelFinderClient()

        # add new pv's
        t1 = str(time())
        hostName1 = 'update-test-hostname' + t1
        iocName1 = 'update-test-iocName' + t1
        channels = client.find(property=[('hostName',
                                          hostName1), ('iocName', iocName1)])
        self.assertTrue(channels == None or len(channels) == 0,
                        'channels already present')
        # New Channels added
        updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                            hostName1, \
                            iocName1, \
                            owner=self.owner, \
                            time=t1, \
                            service=self.baseURL , \
                            username=self.username, \
                            password=self.password)
        channels = client.find(property=[('hostName',
                                          hostName1), ('iocName',
                                                       iocName1), ('time',
                                                                   t1)])
        self.assertTrue(
            len(channels) == 2,
            'failed to create the channels with appropriate properties')
        t2 = str(time())
        hostName2 = 'update-test-hostname' + t2
        iocName2 = 'update-test-iocName' + t2
        # Existing channels are updated
        updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                            hostName2, \
                            iocName2, \
                            owner=self.owner, \
                            time=t2, \
                            service=self.baseURL , \
                            username=self.username, \
                            password=self.password)
        # no channels should have the old proerty values
        self.assertTrue(not client.find(property=[('hostName', hostName1), ('iocName', iocName1), ('time', t1)]), \
                        'failed to update the channels with appropriate properties, old values found')
        # channels should be updated to the new values
        self.assertTrue(len(client.find(property=[('hostName', hostName2), ('iocName', iocName2), ('time', t2)])) == 2, \
                        'failed to update the channels with appropriate properties')
        # Cleanup
        client = ChannelFinderClient(BaseURL=self.baseURL,
                                     username=self.username,
                                     password=self.password)
        client.delete(channelName='cf-update-pv1')
        client.delete(channelName='cf-update-pv2')
        pass
Ejemplo n.º 4
0
 def test_run(self):
     '''
     Tests main program full sweep.
     '''
     CFPropertyManager.run("cf-property-manager-test-dbl",
                           "cf-property-manager-test-cfg")
     os.remove("cf-property-manager-test-cfg")
     os.remove("cf-property-manager-test-dbl")
     client = ChannelFinderClient()
     client.delete(channelName="UT:RF-Cu:1{LD}Time:ShtDwn-I")
Ejemplo n.º 5
0
 def startService(self):
     service.Service.startService(self)
     self.running = 1
     _log.info("CF_START")
     from channelfinder import ChannelFinderClient
     # Using the default python cf-client.
     # The usr, username, and password are provided by the channelfinder._conf module.
     if self.client is None:  # For setting up mock test client
         self.client = ChannelFinderClient()
     self.clean_service()
Ejemplo n.º 6
0
def download_cfs(cfsurl, **cfinput):
    cf = ChannelFinderClient(BaseURL = cfsurl)
    chs = cf.find(**cfinput)
    ret = chanData.ChannelFinderData()
    for ch in chs:
        pv = ch.Name
        prpt = dict([(p.Name, p.Value) for p in ch.Properties])
        tags = [t.Name for t in ch.Tags]

        ret.update(pv = pv, properties = prpt, tags = tags)
    return ret
Ejemplo n.º 7
0
def download_cfs(cfsurl, **cfinput):
    cf = ChannelFinderClient(BaseURL = cfsurl)
    chs = cf.find(**cfinput)
    ret = chanData.ChannelFinderData()
    for ch in chs:
        pv = ch.Name
        prpt = dict([(p.Name, p.Value) for p in ch.Properties])
        tags = [t.Name for t in ch.Tags]

        ret.update(pv = pv, properties = prpt, tags = tags)
    return ret
    def _downloadCfs(self, cfsurl, **kwargs):
        """Get daya from channel finder web service

        :param cfsurl:
        :param kwargs:
        :return:
        """
        keep_prpts = kwargs.pop('keep', None)
        converter  = kwargs.pop('converter', {})
        from channelfinder import ChannelFinderClient
        cf = ChannelFinderClient(BaseURL = cfsurl)
        if len(kwargs) == 0:
            chs = cf.find(name='*')
        else:
            #print kwargs
            chs = cf.find(**kwargs)
        if not chs:
            return

        if keep_prpts is None:
            # use all possible property names
            keep_prpts = [p.Name for p in cf.getAllProperties()]
            
        #print "# include properties", properties
        for ch in chs:
            # keep only known properties
            prptdict = ch.getProperties()
            # prpts is known part from prptdict, otherwise empty dict
            if prptdict is not None:
                prpts = dict([v for v in prptdict.iteritems()])
                # convert the data type
            else:
                prpts = None
            # the empty tags could be None
            if self.use_unicode:
                self.results.append([unicode(ch.Name),
                                  dict([(unicode(k), unicode(v))
                                        for k,v in prpts.iteritems()]),
                                  [unicode(v) for v in ch.getTags()]])
            else:
                self.results.append([ch.Name.encode('ascii'),
                                  dict([(k.encode('ascii'), v.encode('ascii'))
                                        for k,v in prpts.iteritems()]),
                                  [v.encode('ascii') for v in ch.getTags()]])
            if self.results[-1][1]:
                for k in converter:
                    self.results[-1][1][k] = converter[k](prpts[k])
            # warn if hostName or iocName does not present
            if self.host_name not in self.results[-1][1]:
                _logger.warn("no 'hostName' for {0}".format(self.results[-1]))
            if self.ioc_name not in self.results[-1][1]:
                _logger.warn("no 'iocName' for {0}".format(self.results[-1]))

            del prptdict
Ejemplo n.º 9
0
def update():
    cf = ChannelFinderClient(**cfinput)

    #pv="SR:C30-MG{PS:QH1A}I:Ps1DCCT1-I"
    #cf.update(property=Property('elemType', PRPTOWNER, 'SEXT'),
    #          channelNames = [pv])
    #chs = cf.find(name=pv)
    #print "updating {0}".format([ch.Name for ch in chs])
    #print "updating {0} elemField=b0".format([ch.Name for ch in chs])
    #cf.update(property=Property('elemType', PRPTOWNER, 'SEXT'),
    #          channelNames = [ch.Name for ch in chs])
    #cf.update(property=Property('system', PRPTOWNER, 'SR'),
    #              channelNames = [ch.Name for ch in chs])
    cf.update(property=Property('elemGroups', PRPTOWNER), originalPropertyName='elemGroup')
Ejemplo n.º 10
0
def main():
    if len(sys.argv) < 2:
        parser.print_help()
        return 1

    args = parser.parse_args(sys.argv[2:])

    if args.channelsfile is None:
        parser.print_help()
        return 1

    if args.channelsfile is None:
        parser.print_help()
        return 1

    try:
        if args.url is None:
            cfc = ChannelFinderClient()
            args.url = "https://127.0.0.1:8181/ChannelFinder"
        else:
            if args.username is None:
                args.username = r_input("Enter username: "******"Enter password: "******"Cannot connect to channelfinder service.")
        return 1

    ds = DataSource(source=args.channelsfile)
    pv_data = ds.get_data()
    if args.op == 'del':
        try:
            [cfc.delete(channelName=pv['name']) for pv in pv_data]
        except:
            print(
                "Channels deleting ERROR, probably channel(s) does not exist.")
    else:  # args.op == 'add'
        if args.username is None:
            write_cfs(pv_data, args.url, force=True)
        else:
            write_cfs(pv_data,
                      args.url,
                      username=args.username,
                      password=args.password)
    return 0
Ejemplo n.º 11
0
def startClient():
    '''
    Initiates client using default values if none are changed.
    '''
    global client
    client = ChannelFinderClient(BaseURL=SERVICE_URL,
                                 username=username,
                                 password=password)
Ejemplo n.º 12
0
def run_simple_task():
    cf = ChannelFinderClient(**cfinput)

    chs = cf.find(property=[('elemType', 'RFCAVITY')])
    if chs is not None:
        for ch in chs:
            print ch.Name,
            for p in ch.Properties:
                print p.Name,"=",p.Value,", ",
            print " /"
            #addPvProperty(cf, ch.Name, "elemPosition", "633.566", PRPTOWNER)
            #addPvProperty(cf, ch.Name, "elemIndex", "2928", PRPTOWNER)
            #addPvProperty(cf, ch.Name, "elemLength", "0.0", PRPTOWNER)
            #addPvProperty(cf, ch.Name, "cell", "C24", PRPTOWNER)
            addPvProperty(cf, ch.Name, "girder", "G", PRPTOWNER)
    else:
        print "did not found RFCAVITY"
Ejemplo n.º 13
0
def run_simple_task():
    cf = ChannelFinderClient(**cfinput)

    chs = cf.find(property=[('elemType', 'RFCAVITY')])
    if chs is not None:
        for ch in chs:
            print ch.Name,
            for p in ch.Properties:
                print p.Name, "=", p.Value, ", ",
            print " /"
            #addPvProperty(cf, ch.Name, "elemPosition", "633.566", PRPTOWNER)
            #addPvProperty(cf, ch.Name, "elemIndex", "2928", PRPTOWNER)
            #addPvProperty(cf, ch.Name, "elemLength", "0.0", PRPTOWNER)
            #addPvProperty(cf, ch.Name, "cell", "C24", PRPTOWNER)
            addPvProperty(cf, ch.Name, "girder", "G", PRPTOWNER)
    else:
        print "did not found RFCAVITY"
Ejemplo n.º 14
0
    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [
                    prop['name'] for prop in self.client.getAllProperties()
                ]
                if (self.conf.get('alias', 'default') == 'on'):
                    reqd_props = {
                        'hostName', 'iocName', 'pvStatus', 'time', 'iocid',
                        'alias'
                    }
                else:
                    reqd_props = {
                        'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
                    }
                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                    if wl else wl
                # Are any required properties not already present on CF?
                properties = reqd_props - set(cf_props)
                # Are any whitelisted properties not already present on CF?
                # If so, add them too.
                properties.update(set(whitelist) - set(cf_props))

                owner = self.conf.get('username', 'cfstore')
                for prop in properties:
                    self.client.set(property={u'name': prop, u'owner': owner})

                self.whitelist = set(whitelist)
                _log.debug('WHITELIST = {}'.format(self.whitelist))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                self.clean_service()
Ejemplo n.º 15
0
def simple_test():
    cf = ChannelFinderClient(**cfinput)
    #cf.set(property=Property('test-prop1', 'lyyang', 'v1'))
    #cf.set(property=Property('test-prop2', 'cf-asd', 'v2'))
    #cf.set(property=Property('test-prop3', 'lyyang'))
    #cf.set(property=Property('test-prop4', 'lyyang'))
    #cf.delete(propertyName='test-prop1')
    #cf.delete(propertyName='test-prop3')

    pvname1 = 'SR:C00-BI:G00{DCCT:00}CUR-RB'
    pvname2 = 'SR:C00-Glb:G00{CHROM:00}RB-X'
    pvname3 = 'SR:C00-Glb:G00{CHROM:00}RB-Y'
    cf.update(property=Property('test-prop2', 'cf-asd', 'v1'),
              channelNames=[pvname1, pvname2, pvname3])
    #chs = cf.find(property=[('test-prop2', '*')])
    #if chs is not None:
    #    for ch in chs:
    #        print ch.Name
    #        for p in ch.Properties:
    #            print "  ", p.Name,"=",p.Value, p.Owner, ", "
    #        print "/"

    cf.delete(property=Property('test-prop2', 'cf-asd', 'v1'),
              channelNames=[pvname2, pvname3])
    chs = cf.find(property=[('test-prop2', '*')])
    if chs is not None:
        for ch in chs:
            print ch.Name,
            for p in ch.Properties:
                print p.Name, "=", p.Value, ", ",
            print " /"
Ejemplo n.º 16
0
def convCfsToSqlite(url, prefix = '', ignore = []):
    """
    url - channel finder server URL
    prefix - output DB file name prefix
    ignore - list of ignored lattice name 
    """
    cf = ChannelFinderClient(BaseURL=url)
    tagprefx = "aphla.sys."
    tags = [tag.Name for tag in cf.getAllTags() 
            if tag.Name.startswith(tagprefx)]
    for tag in tags:
        latname = tag[len(tagprefx):]
        if latname in ignore: continue
        cfa = ChannelFinderAgent()
        cfa.downloadCfs(url, property=[('hostName', '*')], tagName=tag)
        #cfa.splitPropertyValue('elemGroups')
        cfa.splitChainedElement('elemName')
        cfa.saveSqlite("%s%s.sqlite" % (prefix, latname))
Ejemplo n.º 17
0
def convCfsToSqlite(url, prefix = '', ignore = []):
    """
    url - channel finder server URL
    prefix - output DB file name prefix
    ignore - list of ignored lattice name
    """
    cf = ChannelFinderClient(BaseURL=url)
    tagprefx = "aphla.sys."
    tags = [tag.Name for tag in cf.getAllTags()
            if tag.Name.startswith(tagprefx)]
    for tag in tags:
        latname = tag[len(tagprefx):]
        if latname in ignore: continue
        cfa = ChannelFinderAgent()
        cfa.downloadCfs(url, property=[('hostName', '*')], tagName=tag)
        #cfa.splitPropertyValue('elemGroups')
        cfa.splitChainedElement('elemName')
        cfa.saveSqlite("%s%s.sqlite" % (prefix, latname))
Ejemplo n.º 18
0
        def _init_cfs_data(self, **kws):
            username = kws.get('username', None)
            password = kws.get('password', None)
            cfc = ChannelFinderClient(BaseURL=self.source,
                                      username=username,
                                      password=password)
            c_url = cfc.get_resource('channel')
            t_url = cfc.get_resource('tag')
            p_url = cfc.get_resource('property')

            resource_data = _cofetch_data([c_url, t_url, p_url])

            prop_list = sorted([p['name'] for p in resource_data[p_url]])
            tag_list = sorted([t['name'] for t in resource_data[t_url]])
            raw_data = resource_data[c_url]

            self._prop_list = prop_list
            self._tag_list = tag_list
            self._pvdata = raw_data
Ejemplo n.º 19
0
def init_bpm(elems, url, patl, fname):
    # sort BPM names in lattice
    bpms = {}
    for e in elems:
        if e[2] != "BPM": continue
        # ignore User BPM
        if e[0].startswith("pu"): continue

        k = "C%d" % (int(e[6][1:]), )
        bpms.setdefault(k, []).append((
            e[0],
            e[1],
        ))
    for k, v in bpms.items():
        print k, v

    cf = ChannelFinderClient(BaseURL=url)
    f = open(fname, "w")
    for pvpat, hdl, name, idx, etp, fld in patl:
        # for each pattern, sort PV and check against with elements.
        bpmpvs = {}
        chs = cf.find(name=pvpat,
                      property=[("hostName", "*"), ("iocName", "*")])
        for ch in chs:
            g = re.match(r"SR:(C[0-9][0-9]?)-BI{BPM:([0-9]+)}.*", ch.Name)
            if not g:
                raise RuntimeError("unknown PV: '{0}'".format(ch.Name))
            bpmpvs.setdefault(g.group(1), []).append(ch.Name)

        for k, pvs in bpmpvs.items():
            #print k, pvs
            names = bpms.get(k, [])
            if len(names) != len(pvs):
                #raise RuntimeError("BPM in {0} does not agree {1} != {2}".format(k, len(pvs), len(names)))
                print "ERROR: BPM in {0} does not agree {1} != {2}".format(
                    k, len(pvs), len(names))
            for pv in pvs:
                g = re.match(r"SR:(C[0-9][0-9]?)-BI{BPM:([0-9]+)}.*", pv)
                r = bpms[k][int(g.group(2)) - 1]
                f.write("%s,%s,%s,%s,BPM,%s\n" % (pv, hdl, r[0], r[1], fld))
    f.close()
Ejemplo n.º 20
0
def simple_test():
    cf = ChannelFinderClient(**cfinput)
    #cf.set(property=Property('test-prop1', 'lyyang', 'v1'))
    #cf.set(property=Property('test-prop2', 'cf-asd', 'v2'))
    #cf.set(property=Property('test-prop3', 'lyyang'))
    #cf.set(property=Property('test-prop4', 'lyyang'))
    #cf.delete(propertyName='test-prop1')
    #cf.delete(propertyName='test-prop3')

    pvname1='SR:C00-BI:G00{DCCT:00}CUR-RB'
    pvname2='SR:C00-Glb:G00{CHROM:00}RB-X'
    pvname3='SR:C00-Glb:G00{CHROM:00}RB-Y'
    cf.update(property=Property('test-prop2', 'cf-asd', 'v1'),
              channelNames=[pvname1, pvname2, pvname3])
    #chs = cf.find(property=[('test-prop2', '*')])
    #if chs is not None:
    #    for ch in chs:
    #        print ch.Name
    #        for p in ch.Properties: 
    #            print "  ", p.Name,"=",p.Value, p.Owner, ", "
    #        print "/"

    cf.delete(property=Property('test-prop2', 'cf-asd', 'v1'), 
              channelNames=[pvname2, pvname3])
    chs = cf.find(property=[('test-prop2', '*')])
    if chs is not None:
        for ch in chs:
            print ch.Name,
            for p in ch.Properties: 
                print p.Name,"=",p.Value,", ", 
            print " /"
Ejemplo n.º 21
0
def init_skquad(elems, url, patl, fname):
    cf = ChannelFinderClient(BaseURL=url)
    f = open(fname, "w")
    for pvpat, hdl, name, idx, etp, fld in patl:
        chs = cf.find(name=pvpat,
                      property=[("hostName", "*"), ("iocName", "*")])
        for ch in chs:
            pv = ch.Name
            g = re.match(r"SR:(C[0-9][0-9]?)-MG{PS:SQK([A-Z][0-9])([AB]?)}.*",
                         ch.Name)
            if not g:
                raise RuntimeError("unknown PV: '{0}'".format(ch.Name))
            cell, fam = g.group(1), g.group(2)
            matched = []
            for e in elems:
                if e[2] != "SKQUAD": continue
                # cell
                if e[6] != cell: continue
                # symmetry
                if g.group(3) and e[8] != g.group(3): continue
                matched.append(e)

            logging.info(
                "found {0} SKQUAD records for '{1}', cell={2}, fam={3}".format(
                    len(matched), pv, cell, fam))
            if not matched:
                logging.warn("element not found for {0}".format(pv))
                raise RuntimeError("unmatched quad PV: {0}".format(ch.Name))

            if len(matched) > 1:
                msg = "duplicate data for {0}: {1}".format(pv, matched)
                logging.error(msg)
                raise RuntimeError(msg)

            e = matched[0]
            logging.info("pick {0}".format(e))
            f.write("%s,%s,%s,%s,SKQUAD,%s\n" %
                    (ch.Name, hdl, e[0], e[1], fld))
    f.close()
Ejemplo n.º 22
0
    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            # This module is always imported by twistd, even when 'proc = cf'
            # isn't given. This way the ChannelFinderClient import is only
            # attempted if it is actually needed.
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                self._setup_properties()
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                self.clean_service()
Ejemplo n.º 23
0
 def testAddUpdateChannels(self):
     # Check the method finds the error conditions and raises exceptions
     self.assertRaises(Exception, updateChannelFinder, [[], None, None])
     self.assertRaises(Exception, updateChannelFinder, [[], None, 'iocname'])
     self.assertRaises(Exception, updateChannelFinder, [[], 'hostName', None])
     # create default client
     client = ChannelFinderClient()
     
     # add new pv's
     t1 = str(time())
     hostName1 = 'update-test-hostname' + t1
     iocName1 = 'update-test-iocName' + t1
     channels = client.find(property=[('hostName', hostName1), ('iocName', iocName1)])
     self.assertTrue(channels == None or len(channels) == 0, 'channels already present')
     # New Channels added
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName1, \
                         iocName1, \
                         owner=self.owner, \
                         time=t1, \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(property=[('hostName', hostName1), ('iocName', iocName1), ('time', t1)])
     self.assertTrue(len(channels) == 2, 'failed to create the channels with appropriate properties')
     t2 = str(time())
     hostName2 = 'update-test-hostname' + t2
     iocName2 = 'update-test-iocName' + t2
     # Existing channels are updated
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName2, \
                         iocName2, \
                         owner=self.owner, \
                         time=t2, \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     # no channels should have the old proerty values 
     self.assertTrue(client.find(property=[('hostName', hostName1), ('iocName', iocName1), ('time', t1)]) == None, \
                     'failed to update the channels with appropriate properties, old values found')
     # channels should be updated to the new values
     self.assertTrue(len(client.find(property=[('hostName', hostName2), ('iocName', iocName2), ('time', t2)])) == 2, \
                     'failed to update the channels with appropriate properties')
     # Cleanup
     client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
     client.delete(channelName='cf-update-pv1')
     client.delete(channelName='cf-update-pv2')
     pass
Ejemplo n.º 24
0
def init_sext(elems, url, patl, fname):
    cf = ChannelFinderClient(BaseURL=url)
    skquad   = re.compile(r"SR:(C[0-9]+)-MG{PS:(SQ[A-Z0-9]+)}.*")
    sxt_dw   = re.compile(r"SR:(C[0-9]+)-MG{PS:(S[HLM][0-9])-(DW[0-9]+)}.*")
    f = open(fname, "w")
    for pvpat, hdl, name, idx, etp, fld in patl:
        chs = cf.find(name=pvpat, property=[("hostName", "*"),
                                            ("iocName","*")])
        for ch in chs:
            if skquad.match(ch.Name): continue
            if sxt_dw.match(ch.Name): continue
            matched = match_sextupoles(elems, ch.Name)

            logging.info("elements found for pv '{0}': {1}".format(
                        ch.Name, [elems[i][0] for i in matched]))
            if not matched:
                raise RuntimeError("unknown Sextupole PV: '{0}'".format(pv))
            if len(matched) != 6:
                raise RuntimeError("should be 6 sextupoles, {0} found".format(
                    len(matched)))
            for e in [elems[i] for i in matched]:
                f.write("%s,%s,%s,%s,SEXT,%s\n" % (ch.Name, hdl, e[0], e[1], fld))
    f.close()
Ejemplo n.º 25
0
def init_bpm(elems, url, patl, fname):
    # sort BPM names in lattice
    bpms = {}
    for e in elems:
        if e[2] != "BPM": continue
        # ignore User BPM
        if e[0].startswith("pu"): continue

        k = "C%d" % (int(e[6][1:]),)
        bpms.setdefault(k, []).append((e[0], e[1],))
    for k,v in bpms.items(): print k, v
    
    cf = ChannelFinderClient(BaseURL=url)
    f = open(fname, "w")
    for pvpat, hdl, name, idx, etp, fld in patl:
        # for each pattern, sort PV and check against with elements.
        bpmpvs = {}
        chs = cf.find(name=pvpat, property=[("hostName", "*"),
                                            ("iocName","*")])
        for ch in chs:
            g = re.match(r"SR:(C[0-9][0-9]?)-BI{BPM:([0-9]+)}.*", ch.Name)
            if not g: 
                raise RuntimeError("unknown PV: '{0}'".format(ch.Name))
            bpmpvs.setdefault(g.group(1), []).append(ch.Name)

        for k,pvs in bpmpvs.items():
            #print k, pvs
            names = bpms.get(k, [])
            if len(names) != len(pvs):
                #raise RuntimeError("BPM in {0} does not agree {1} != {2}".format(k, len(pvs), len(names)))
                print "ERROR: BPM in {0} does not agree {1} != {2}".format(k, len(pvs), len(names))
            for pv in pvs:
                g = re.match(r"SR:(C[0-9][0-9]?)-BI{BPM:([0-9]+)}.*", pv)
                r = bpms[k][int(g.group(2)) - 1]
                f.write("%s,%s,%s,%s,BPM,%s\n" % (pv, hdl, r[0], r[1], fld))
    f.close()
Ejemplo n.º 26
0
def init_sext(elems, url, patl, fname):
    cf = ChannelFinderClient(BaseURL=url)
    skquad = re.compile(r"SR:(C[0-9]+)-MG{PS:(SQ[A-Z0-9]+)}.*")
    sxt_dw = re.compile(r"SR:(C[0-9]+)-MG{PS:(S[HLM][0-9])-(DW[0-9]+)}.*")
    f = open(fname, "w")
    for pvpat, hdl, name, idx, etp, fld in patl:
        chs = cf.find(name=pvpat,
                      property=[("hostName", "*"), ("iocName", "*")])
        for ch in chs:
            if skquad.match(ch.Name): continue
            if sxt_dw.match(ch.Name): continue
            matched = match_sextupoles(elems, ch.Name)

            logging.info("elements found for pv '{0}': {1}".format(
                ch.Name, [elems[i][0] for i in matched]))
            if not matched:
                raise RuntimeError("unknown Sextupole PV: '{0}'".format(pv))
            if len(matched) != 6:
                raise RuntimeError("should be 6 sextupoles, {0} found".format(
                    len(matched)))
            for e in [elems[i] for i in matched]:
                f.write("%s,%s,%s,%s,SEXT,%s\n" %
                        (ch.Name, hdl, e[0], e[1], fld))
    f.close()
Ejemplo n.º 27
0
 def testPVUpdate(self):
     '''
     Test condition 
     IOC turned on with ch1, ch2
     IOC turned on with ch1 only
     IOC turned on with ch1, ch2
     '''
     try:
         updateChannelFinder(['ch1', 'ch2'], \
                             'testHost', \
                             'testIOC', \
                             owner=self.owner, \
                             time=time(), \
                             service=self.baseURL, \
                             username=self.username, password=self.password)
         client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
         chs = client.find(property=[('hostName', 'testHost'), ('iocName', 'testIOC'), ('pvStatus', 'Active')])
         self.assertEqual(len(chs), 2, 'Expected 2 positive matches but found ' + str(len(chs)))
         updateChannelFinder(['ch1'], \
                             'testHost', \
                             'testIOC', \
                             owner=self.owner, \
                             time=time(), \
                             service=self.baseURL, \
                             username=self.username, password=self.password)
         chs = client.find(property=[('hostName', 'testHost'), ('iocName', 'testIOC'), ('pvStatus', 'Active')])
         self.assertEqual(len(chs), 1, 'Expected 1 positive matches but found ' + str(len(chs)))
         self.assertTrue(chs[0].Name == 'ch1', 'channel with name ch1 not found')
         chs = client.find(property=[('hostName', 'testHost'), ('iocName', 'testIOC'), ('pvStatus', 'InActive')])
         self.assertEqual(len(chs), 1, 'Expected 1 positive matches but found ' + str(len(chs)))
         self.assertTrue(chs[0].Name == 'ch2', 'channel with name ch2 not found')            
         updateChannelFinder(['ch1', 'ch2'], \
                             'testHost', \
                             'testIOC', \
                             owner=self.owner, \
                             time=time(), \
                             service=self.baseURL, \
                             username=self.username, password=self.password)
         chs = client.find(property=[('hostName', 'testHost'), ('iocName', 'testIOC'), ('pvStatus', 'Active')])
         self.assertEqual(len(chs), 2, 'Expected 2 positive matches but found ' + str(len(chs)))
     finally:
         client.delete(channelName='ch1')
         client.delete(channelName='ch2')
Ejemplo n.º 28
0
    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [prop['name'] for prop in self.client.getAllProperties()]
                if (self.conf.get('alias', 'default') == 'on'):
                    reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid', 'alias'}
                else:
                    reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid'}
                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                    if wl else wl
                # Are any required properties not already present on CF?
                properties = reqd_props - set(cf_props)
                # Are any whitelisted properties not already present on CF?
                # If so, add them too.
                properties.update(set(whitelist) - set(cf_props))

                owner = self.conf.get('username', 'cfstore')
                for prop in properties:
                    self.client.set(property={u'name': prop, u'owner': owner})

                self.whitelist = set(whitelist)
                _log.debug('WHITELIST = {}'.format(self.whitelist))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                self.clean_service()
Ejemplo n.º 29
0
def cfs_append_from_cmd(cmd_list, update_only=False):
    """
    update the cfs from command file:

        addtag tag pv1,pv2,pv3
        removetag tag pv1,pv2,...
        updateproperty prpt=val pv1,pv2
        appendproperty prpt=val pv1,pv2
        removeproperty prpt pv1,pv2

    The properties are updated as 'cf-asd' account, and tags in 'cf-aphla'
    """
    print("# Updating CFS from text file")

    cf = ChannelFinderClient(**cfinput)

    pvs = []

    # read each line of the txt command file
    for i, line in enumerate(cmd_list.readlines()):
        if not line.strip(): continue
        if line.strip().find('#') == 0: continue

        rec = [v.strip() for v in line.strip().split()]
        if len(rec) <= 2:
            logging.warning("skiping line %d: %s" % (i, line))
            continue

        if rec[0] == 'addtag':
            pvs = [pv.strip() for pv in rec[2].split(',')]
            addTagPvs(cf, rec[1].strip(), pvs, OWNER)
            continue
        elif rec[0] == 'removetag':
            pvs = [pv.strip() for pv in rec[2].split(',')]
            removeTagPvs(cf, rec[1].strip(), pvs, OWNER)
            continue
        elif rec[0] in ('updateproperty', ):
            pvs = [pv.strip() for pv in rec[2].split(',')]
            prpt, val = rec[1].split('=')
            if val == "''": val = ''
            updatePropertyPvs(cf, prpt, PRPTOWNER, val, pvs)
            continue
        elif rec[0] in ('appendproperty', ):
            pvs = [pv.strip() for pv in rec[2].split(',')]
            prpt, val = rec[1].split('=')
            if val == "''": val = ''
            appendPropertyPvs(cf, prpt, PRPTOWNER, val, pvs)
            continue
        elif rec[0] == 'removeproperty':
            pvs = [pv.strip() for pv in rec[2].split(',')]
            prpt = rec[1].strip()
            removePropertyPvs(cf, prpt, PRPTOWNER, pvs)

        pv = rec[0]
        cmd = rec[1]
        newval = ''.join(rec[2:])
        for v in [v.strip() for v in newval.split(',')]:
            if v.find('=') >= 0:
                # it is a property
                prpt, val = v.split('=')
                if cmd == 'add':
                    print("add", prpt, val)
                    addPvProperty(cf, pv, prpt, val, PRPTOWNER)
            else:
                # tag:
                if cmd == 'add': addPvTag(cf, pv, v, OWNER)
                elif cmd == 'remove': removePvTag(cf, pv, v, OWNER)
Ejemplo n.º 30
0
def cfs_append_from_sqlite(fname, update_only):
    sq = ap.chanfinder.ChannelFinderAgent()
    sq.loadSqlite(fname)

    cf = ChannelFinderClient(**cfinput)
    all_prpts = [p.Name for p in cf.getAllProperties()]
    all_tags = [t.Name for t in cf.getAllTags()]
    ignore_prpts = ['hostName', 'iocName']

    allpvs = []
    tag_owner = OWNER
    prpt_owner = PRPTOWNER
    prpt_data, tag_data = {}, {}
    # the data body
    for pv, prpts, tags in sq.rows:
        if not pv: continue
        if pv in allpvs: continue
        if pv.find("SR:") != 0: continue
        logging.info("updating '{0}'".format(pv))

        allpvs.append(pv)
        prpt_list, tag_list = [], []
        for k, v in prpts.items():
            if k not in [
                    "elemIndex", "system", "elemType", "elemHandle",
                    "elemName", "elemField"
            ]:
                continue
            prpt_data.setdefault((k, v), [])
            prpt_data[(k, v)].append(pv)
        for tag in tags:
            if not tag.startswith("aphla."): continue
            tag_data.setdefault(tag, [])
            tag_data[tag].append(pv)
            #tag_list.append(Tag(r.strip()), tag_owner)
            logging.info("{0}: {1} ({2})".format(pv, tag, tag_owner))
            #addPvTag(cf, pv, tag, tag_owner)

    errpvs = []
    for pv in allpvs:
        chs = cf.find(name=pv)
        if not chs:
            errpvs.append(pv)
            print("PV '%s' does not exist" % pv)
            continue
        elif len(chs) != 1:
            print("Find two results for pv=%s" % pv)
            continue
        prpts = chs[0].getProperties()
        if not prpts: continue
        for prpt, val in prpts.items():
            pvlst = prpt_data.get((prpt, val), [])
            if not pvlst: continue
            try:
                j = pvlst.index(pv)
                prpt_data[(prpt, val)].pop(j)
            except:
                # the existing data is not in the update list, skip
                pass

    #if errpvs:
    #    #raise RuntimeError("PVs '{0}' are missing".format(errpvs))
    #    print

    logging.warn("{0} does not exist in DB".format(errpvs))

    for k, v in prpt_data.items():
        vf = [pv for pv in v if pv not in errpvs]
        if not vf:
            logging.info("no valid PVs for {0}".format(k))
            continue
        updatePropertyPvs(cf, k[0], prpt_owner, k[1], vf)
        logging.info("add property {0} for pvs {1}".format(k, vf))
    for k, v in tag_data.items():
        vf = [pv for pv in v if pv not in errpvs]
        addTagPvs(cf, k, vf, tag_owner)
        logging.info("add tag {0} for pvs {1}".format(k, vf))
Ejemplo n.º 31
0
def cfs_append_from_csv2(rec_list, update_only):
    cf = ChannelFinderClient(**cfinput)
    all_prpts = [p.Name for p in cf.getAllProperties()]
    all_tags = [t.Name for t in cf.getAllTags()]
    ignore_prpts = ['hostName', 'iocName']
    import csv
    rd = csv.reader(rec_list)

    allpvs = []
    tag_owner = OWNER
    prpt_owner = PRPTOWNER
    prpt_data, tag_data = {}, {}
    # the data body
    for s in rd:
        if not s: continue
        if not s[0].strip(): continue
        if s[0].strip().startswith('#'): continue

        pv = s[0].strip()
        logging.info("updating '{0}'".format(pv))
        #cf.update(property=Property('elemType', PRPTOWNER, 'QUAD'),
        #          channelNames = [pv])
        #chs = cf.find(name=pv)
        #sys.exit(0)
        #logging.info("{0} and {1}".format(chs[0].Name, type(chs[0].Name)))
        #logging.info("{0} and {1}".format(pv, type(pv)))

        allpvs.append(pv)
        prpt_list, tag_list = [], []
        for r in s[1:]:
            if r.find('=') > 0:
                prpt, val = [v.strip() for v in r.split('=')]
                prpt_data.setdefault((prpt, val), [])
                prpt_data[(prpt, val)].append(pv)
            else:
                # it is a tag
                tag = r.strip()
                if not tag: continue
                tag_data.setdefault(tag, [])
                tag_data[tag].append(pv)
                #tag_list.append(Tag(r.strip()), tag_owner)
                logging.info("{0}: {1} ({2})".format(pv, tag, tag_owner))
                #addPvTag(cf, pv, tag, tag_owner)

    errpvs = []
    for pv in allpvs:
        chs = cf.find(name=pv)
        if not chs:
            errpvs.append(pv)
            print("PV '%s' does not exist" % pv)
            continue
        elif len(chs) != 1:
            print("Find two results for pv=%s" % pv)
            continue
        for prpt, val in chs[0].getProperties().items():
            pvlst = prpt_data.get((prpt, val), [])
            if not pvlst: continue
            try:
                j = pvlst.index(pv)
                prpt_data[(prpt, val)].pop(j)
            except:
                # the existing data is not in the update list, skip
                pass

    #if errpvs:
    #    #raise RuntimeError("PVs '{0}' are missing".format(errpvs))
    #    print

    logging.warn("{0} does not exist in DB".format(errpvs))

    for k, v in prpt_data.items():
        vf = [pv for pv in v if pv not in errpvs]
        if not vf:
            logging.info("no valid PVs for {0}".format(k))
            continue
        updatePropertyPvs(cf, k[0], prpt_owner, k[1], vf)
        logging.info("add property {0} for pvs {1}".format(k, vf))
    for k, v in tag_data.items():
        vf = [pv for pv in v if pv not in errpvs]
        addTagPvs(cf, k, vf, tag_owner)
        logging.info("add tag {0} for pvs {1}".format(k, vf))
Ejemplo n.º 32
0
def cfs_append_from_csv1(rec_list, update_only):
    cf = ChannelFinderClient(**cfinput)
    all_prpts = [p.Name for p in cf.getAllProperties()]
    all_tags = [t.Name for t in cf.getAllTags()]
    ignore_prpts = ['hostName', 'iocName']
    import csv
    rd = csv.reader(rec_list)
    # header line
    header = rd.next()
    # lower case of header
    hlow = [s.lower() for s in header]
    # number of headers, pv + properties
    nheader = len(header)
    # the index of PV, properties and tags
    ipv = hlow.index('pv')
    # the code did not rely on it, but it is a good practice
    if ipv != 0:
        raise RuntimeError("the first column should be pv")

    iprpt, itags = [], []
    for i, h in enumerate(header):
        if i == ipv: continue
        # if the header is empty, it is a tag
        if len(h.strip()) == 0:
            itags.append(i)
        else:
            iprpt.append(i)

    tag_owner = OWNER
    prpt_owner = PRPTOWNER
    tags = {}
    # the data body
    for s in rd:
        prpts = [Property(header[i], prpt_owner, s[i]) for i in iprpt if s[i]]
        # itags could be empty if we put all tags in the end columns
        for i in itags + range(nheader, len(s)):
            rec = tags.setdefault(s[i].strip(), [])
            rec.append(s[ipv].strip())

        #print s[ipv], prpts, tags
        ch = cf.find(name=s[ipv])
        if ch is None:
            logging.warning("pv {0} does not exist".format(s[ipv]))
        elif len(ch) > 1:
            logging.warning("pv {0} is not unique ({1})".format(
                s[ipv], len(ch)))
        else:
            for p in prpts:
                #continue
                if p.Name in ignore_prpts: continue
                #if p.Name != 'symmetry': continue
                logging.info("updating '{0}' with property, {1}={2}".format(
                    s[ipv], p.Name, p.Value))
                cf.update(channelName=s[ipv], property=p)

    logging.info("finished updating properties")
    for t, pvs in tags.items():
        if not hasTag(cf, t): cf.set(tag=Tag(t, tag_owner))
        if 'V:1-SR-BI{BETA}X-I' in pvs: continue
        if 'V:1-SR-BI{BETA}Y-I' in pvs: continue
        try:
            cf.update(tag=Tag(t, tag_owner), channelNames=pvs)
        except:
            print(t, pvs)
            raise

        logging.info("update '{0}' for {1} pvs".format(t, len(pvs)))
    logging.info("finished updating tags")
Ejemplo n.º 33
0
def cfs_append_from_csv2(rec_list, update_only):
    cf = ChannelFinderClient(**cfinput)
    all_prpts = [p.Name for p in cf.getAllProperties()]
    all_tags  = [t.Name for t in cf.getAllTags()]
    ignore_prpts = ['hostName', 'iocName']
    import csv
    rd = csv.reader(rec_list)

    allpvs = []
    tag_owner = OWNER
    prpt_owner = PRPTOWNER
    prpt_data, tag_data = {}, {}
    # the data body
    for s in rd:
        if not s: continue
        if not s[0].strip(): continue
        if s[0].strip().startswith('#'): continue
        
        pv = s[0].strip()
        logging.info("updating '{0}'".format(pv))
        #cf.update(property=Property('elemType', PRPTOWNER, 'QUAD'),
        #          channelNames = [pv])
        #chs = cf.find(name=pv)
        #sys.exit(0)
        #logging.info("{0} and {1}".format(chs[0].Name, type(chs[0].Name)))
        #logging.info("{0} and {1}".format(pv, type(pv)))

        allpvs.append(pv)
        prpt_list, tag_list = [], []
        for r in s[1:]:
            if r.find('=') > 0:
                prpt, val = [v.strip() for v in r.split('=')]
                prpt_data.setdefault((prpt, val), [])
                prpt_data[(prpt, val)].append(pv)
            else:
                # it is a tag
                tag = r.strip()
                if not tag: continue
                tag_data.setdefault(tag, [])
                tag_data[tag].append(pv)
                #tag_list.append(Tag(r.strip()), tag_owner)
                logging.info("{0}: {1} ({2})".format(pv, tag, tag_owner))
                #addPvTag(cf, pv, tag, tag_owner)
        
    errpvs = []
    for pv in allpvs:
        chs = cf.find(name=pv)
        if not chs:
            errpvs.append(pv)
            print "PV '%s' does not exist" % pv
            continue
        elif len(chs) != 1:
            print "Find two results for pv=%s" % pv
            continue
        for prpt,val in chs[0].getProperties().items():
            pvlst = prpt_data.get((prpt, val), [])
            if not pvlst: continue
            try:
                j = pvlst.index(pv)
                prpt_data[(prpt,val)].pop(j)
            except:
                # the existing data is not in the update list, skip
                pass

    #if errpvs: 
    #    #raise RuntimeError("PVs '{0}' are missing".format(errpvs))
    #    print 

    logging.warn("{0} does not exist in DB".format(errpvs))

    for k,v in prpt_data.iteritems():
        vf = [pv for pv in v if pv not in errpvs]
        if not vf: 
            logging.info("no valid PVs for {0}".format(k))
            continue
        updatePropertyPvs(cf, k[0], prpt_owner, k[1], vf)
        logging.info("add property {0} for pvs {1}".format(k, vf))
    for k,v in tag_data.iteritems():
        vf = [pv for pv in v if pv not in errpvs]
        addTagPvs(cf, k, vf, tag_owner)
        logging.info("add tag {0} for pvs {1}".format(k, vf))
Ejemplo n.º 34
0
class CFProcessor(service.Service):
    def __init__(self, name, conf):
        _log.info("CF_INIT %s", name)
        self.name, self.conf = name, conf
        self.channel_dict = defaultdict(list)
        self.iocs = dict()
        self.client = None
        self.currentTime = getCurrentTime
        self.lock = DeferredLock()

    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [prop['name'] for prop in self.client.getAllProperties()]
                if (self.conf.get('alias', 'default') == 'on'):
                    reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid', 'alias'}
                else:
                    reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid'}
                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                    if wl else wl
                # Are any required properties not already present on CF?
                properties = reqd_props - set(cf_props)
                # Are any whitelisted properties not already present on CF?
                # If so, add them too.
                properties.update(set(whitelist) - set(cf_props))

                owner = self.conf.get('username', 'cfstore')
                for prop in properties:
                    self.client.set(property={u'name': prop, u'owner': owner})

                self.whitelist = set(whitelist)
                _log.debug('WHITELIST = {}'.format(self.whitelist))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                self.clean_service()

    def stopService(self):
        service.Service.stopService(self)
        # Set channels to inactive and close connection to client
        self.running = 0
        self.clean_service()
        _log.info("CF_STOP")

    @defer.inlineCallbacks
    def commit(self, transaction_record):
        yield self.lock.acquire()
        try:
            yield deferToThread(self.__commit__, transaction_record)
        finally:
            self.lock.release()

    def __commit__(self, TR):
        _log.debug("CF_COMMIT %s", TR.infos.items())
        """
        a dictionary with a list of records with their associated property info  
        pvInfo 
        {rid: { "pvName":"recordName",
                "infoProperties":{propName:value, ...}}}
        """

        iocName = TR.infos.get('IOCNAME') or TR.src.port
        hostName = TR.infos.get('HOSTNAME') or TR.src.host
        owner = TR.infos.get('ENGINEER') or TR.infos.get('CF_USERNAME') or self.conf.get('username', 'cfstore')
        time = self.currentTime()

        pvInfo = {}
        for rid, (rname, rtype) in TR.addrec.items():
            pvInfo[rid] = {"pvName": rname}
        for rid, (recinfos) in TR.recinfos.items():
            # find intersection of these sets
            recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
            if recinfo_wl:
                pvInfo[rid]['infoProperties'] = list()
                for infotag in recinfo_wl:
                    _log.debug('INFOTAG = {}'.format(infotag))
                    property = {u'name': infotag, u'owner': owner,
                                u'value': recinfos[infotag]}
                    pvInfo[rid]['infoProperties'].append(property)
        for rid, alias in TR.aliases.items():
            pvInfo[rid]['aliases'] = alias
        _log.debug(pvInfo)

        pvNames = [info["pvName"] for rid, (info) in pvInfo.items()]

        delrec = list(TR.delrec)
        _log.info("DELETED records " + str(delrec))

        host = TR.src.host
        port = TR.src.port

        """The unique identifier for a particular IOC"""
        iocid = host + ":" + str(port)
        _log.info("CF_COMMIT: " + iocid)

        if TR.initial:
            """Add IOC to source list """
            self.iocs[iocid] = {"iocname": iocName, "hostname": hostName, "owner": owner, "time": time,
                                "channelcount": 0}
        if not TR.connected:
            delrec.extend(self.channel_dict.keys())
        for pv in pvNames:
            self.channel_dict[pv].append(iocid)  # add iocname to pvName in dict
            self.iocs[iocid]["channelcount"] += 1
            """In case, alias exists"""
            if (self.conf.get('alias', 'default' == 'on')):
                al = [info["aliases"] for rid, (info) in pvInfo.items() if info["pvName"] == pv and "aliases" in info ]
                if len(al) == 1:
                    ali = al[0]
                    for a in ali:
                        self.channel_dict[a].append(iocid)  # add iocname to pvName in dict
                        self.iocs[iocid]["channelcount"] += 1
        for pv in delrec:
            if iocid in self.channel_dict[pv]:
                self.channel_dict[pv].remove(iocid)
                if iocid in self.iocs:
                    self.iocs[iocid]["channelcount"] -= 1
                if self.iocs[iocid]['channelcount'] == 0:
                    self.iocs.pop(iocid, None)
                elif self.iocs[iocid]['channelcount'] < 0:
                    _log.error("channel count negative!")
                if len(self.channel_dict[pv]) <= 0:  # case: channel has no more iocs
                    del self.channel_dict[pv]
                """In case, alias exists"""
                if (self.conf.get('alias', 'default' == 'on')):
                    al = [info["aliases"] for rid, (info) in pvInfo.items() if info["pvName"] == pv and "aliases" in info ]
                    if len(al) == 1:
                        ali = al[0]
                        for a in ali:
                            self.channel_dict[a].remove(iocid)
                            if iocid in self.iocs:
                                self.iocs[iocid]["channelcount"] -= 1
                            if self.iocs[iocid]['channelcount'] == 0:
                                self.iocs.pop(iocid, None)
                            elif self.iocs[iocid]['channelcount'] < 0:
                                _log.error("channel count negative!")
                            if len(self.channel_dict[a]) <= 0:  # case: channel has no more iocs
                                del self.channel_dict[a]
        poll(__updateCF__, self.client, pvInfo, delrec, self.channel_dict, self.iocs, self.conf, hostName, iocName, iocid,
             owner, time)
        dict_to_file(self.channel_dict, self.iocs, self.conf)

    def clean_service(self):
        """
        Marks all channels as "Inactive" until the recsync server is back up
        """
        sleep = 1
        retry_limit = 5
        owner = self.conf.get('username', 'cfstore')
        while 1:
            try:
                _log.debug("Cleaning service...")
                channels = self.client.findByArgs([('pvStatus', 'Active')])
                if channels is not None:
                    new_channels = []
                    for ch in channels or []:
                        new_channels.append(ch[u'name'])
                    if len(new_channels) > 0:
                        self.client.update(property={u'name': 'pvStatus', u'owner': owner, u'value': "Inactive"},
                                           channelNames=new_channels)
                    _log.debug("Service clean.")
                    return
            except RequestException:
                _log.exception("cleaning failed, retrying: ")

            time.sleep(min(60, sleep))
            sleep *= 1.5
            if self.running == 0 and sleep >= retry_limit:
                _log.debug("Abandoning clean.")
                return
Ejemplo n.º 35
0
def cfs_append_from_sqlite(fname, update_only):
    sq = ap.chanfinder.ChannelFinderAgent()
    sq.loadSqlite(fname)

    cf = ChannelFinderClient(**cfinput)
    all_prpts = [p.Name for p in cf.getAllProperties()]
    all_tags  = [t.Name for t in cf.getAllTags()]
    ignore_prpts = ['hostName', 'iocName']

    allpvs = []
    tag_owner = OWNER
    prpt_owner = PRPTOWNER
    prpt_data, tag_data = {}, {}
    # the data body
    for pv, prpts, tags in sq.rows:
        if not pv: continue
        if pv in allpvs: continue
        if pv.find("SR:") != 0: continue
        logging.info("updating '{0}'".format(pv))
        
        allpvs.append(pv)
        prpt_list, tag_list = [], []
        for k,v in prpts.items():
            if k not in ["elemIndex", "system", "elemType", "elemHandle",
                         "elemName", "elemField"]: continue
            prpt_data.setdefault((k, v), [])
            prpt_data[(k, v)].append(pv)
        for tag in tags:
            if not tag.startswith("aphla."): continue
            tag_data.setdefault(tag, [])
            tag_data[tag].append(pv)
            #tag_list.append(Tag(r.strip()), tag_owner)
            logging.info("{0}: {1} ({2})".format(pv, tag, tag_owner))
            #addPvTag(cf, pv, tag, tag_owner)
        
    errpvs = []
    for pv in allpvs:
        chs = cf.find(name=pv)
        if not chs:
            errpvs.append(pv)
            print "PV '%s' does not exist" % pv
            continue
        elif len(chs) != 1:
            print "Find two results for pv=%s" % pv
            continue
        prpts = chs[0].getProperties()
        if not prpts: continue
        for prpt,val in prpts.items():
            pvlst = prpt_data.get((prpt, val), [])
            if not pvlst: continue
            try:
                j = pvlst.index(pv)
                prpt_data[(prpt,val)].pop(j)
            except:
                # the existing data is not in the update list, skip
                pass

    #if errpvs: 
    #    #raise RuntimeError("PVs '{0}' are missing".format(errpvs))
    #    print 

    logging.warn("{0} does not exist in DB".format(errpvs))

    for k,v in prpt_data.iteritems():
        vf = [pv for pv in v if pv not in errpvs]
        if not vf: 
            logging.info("no valid PVs for {0}".format(k))
            continue
        updatePropertyPvs(cf, k[0], prpt_owner, k[1], vf)
        logging.info("add property {0} for pvs {1}".format(k, vf))
    for k,v in tag_data.iteritems():
        vf = [pv for pv in v if pv not in errpvs]
        addTagPvs(cf, k, vf, tag_owner)
        logging.info("add tag {0} for pvs {1}".format(k, vf))
Ejemplo n.º 36
0
    def testPreservingOfAttributes(self):
        '''
        This test is to ensure that existing properties and tags are left untouched.
        Case1:
        first time the cf-update comes across these channels and adds hostName and iocName
        Case2:
        the hostName is changed
        Case3:
        the iocName is changed
        Case4:
        both hostName and iocName are changed
        Case5:
        the channel is removed
        in all cases the existing unaffected* property and tag should remain with the channel               
        '''
        unaffectedProperty = {
            u'name': u'unaffectedProperty',
            u'owner': self.owner,
            u'value': u'unchanged'
        }
        unaffectedTag = {u'name': u'unaffectedTag', u'owner': self.owner}
        # create default client
        client = ChannelFinderClient(BaseURL=self.baseURL,
                                     username=self.username,
                                     password=self.password)
        client.set(property=unaffectedProperty)
        client.set(tag=unaffectedTag)

        client.set(
            channel={
                u'name': u'cf-update-pv1',
                u'owner': u'cf-update',
                u'properties': [unaffectedProperty],
                u'tags': [unaffectedTag]
            })
        client.set(
            channel={
                u'name': u'cf-update-pv2',
                u'owner': u'cf-update',
                u'properties': [unaffectedProperty],
                u'tags': [unaffectedTag]
            })

        # Case1:
        hostName = 'initialHost'
        iocName = 'initialIoc'
        updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                            hostName, \
                            iocName, \
                            owner=self.owner, \
                            time=time(), \
                            service=self.baseURL , \
                            username=self.username, \
                            password=self.password)
        channels = client.find(name='cf-update-pv*')
        for channel in channels:
            self.assertTrue(unaffectedProperty in channel['properties']
                            and unaffectedTag in channel['tags'])
            self.assertTrue(
                self.__check4properties(
                    {
                        u'name': u'hostName',
                        u'value': hostName
                    }, channel['properties']) and self.__check4properties(
                        {
                            u'name': u'iocName',
                            u'value': iocName
                        }, channel['properties']) and self.__check4properties(
                            {
                                u'name': u'pvStatus',
                                u'value': u'Active'
                            }, channel['properties']),
                'Failed to update channels with the correct hostName and/or iocName'
            )
        # Case2:
        hostName = 'newHost'
        updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                            hostName, \
                            iocName, \
                            owner=self.owner, \
                            time=time(), \
                            service=self.baseURL , \
                            username=self.username, \
                            password=self.password)
        channels = client.find(name='cf-update-pv*')
        for channel in channels:
            self.assertTrue(unaffectedProperty in channel['properties']
                            and unaffectedTag in channel['tags'])
            self.assertTrue(
                self.__check4properties(
                    {
                        u'name': u'hostName',
                        u'value': hostName
                    }, channel['properties']) and self.__check4properties(
                        {
                            u'name': u'iocName',
                            u'value': iocName
                        }, channel['properties']) and self.__check4properties(
                            {
                                u'name': u'pvStatus',
                                u'value': u'Active'
                            }, channel['properties']),
                'Failed to update channels with the correct hostName and/or iocName'
            )
        self.assertTrue(
            not client.find(property=[('hostName', 'initialHost')]),
            'Failed to cleanup old property')
        # Case 3:
        iocName = 'newIoc'
        updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                            hostName, \
                            iocName, \
                            owner=self.owner, \
                            time=time(), \
                            service=self.baseURL , \
                            username=self.username, \
                            password=self.password)
        channels = client.find(name='cf-update-pv*')
        for channel in channels:
            self.assertTrue(unaffectedProperty in channel['properties']
                            and unaffectedTag in channel['tags'])
            self.assertTrue(
                self.__check4properties(
                    {
                        u'name': u'hostName',
                        u'value': hostName
                    }, channel['properties']) and self.__check4properties(
                        {
                            u'name': u'iocName',
                            u'value': iocName
                        }, channel['properties']) and self.__check4properties(
                            {
                                u'name': u'pvStatus',
                                u'value': u'Active'
                            }, channel['properties']),
                'Failed to update channels with the correct hostName and/or iocName'
            )
        self.assertTrue(
            not client.find(property=[('hostName', 'initialHost')]),
            'Failed to cleanup old property')
        self.assertTrue(not client.find(property=[('iocName', 'initialIoc')]),
                        'Failed to cleanup old property')
        # Case 4:
        updateChannelFinder([], \
                            hostName, \
                            iocName, \
                            owner=self.owner, \
                            time=time(), \
                            service=self.baseURL , \
                            username=self.username, \
                            password=self.password)
        channels = client.find(name='cf-update-pv*')
        for channel in channels:
            self.assertTrue(unaffectedProperty in channel['properties']
                            and unaffectedTag in channel['tags'])
            self.assertTrue(
                self.__check4properties(
                    {
                        u'name': u'hostName',
                        u'value': hostName
                    }, channel['properties']) and self.__check4properties(
                        {
                            u'name': u'iocName',
                            u'value': iocName
                        }, channel['properties']) and self.__check4properties(
                            {
                                u'name': u'pvStatus',
                                u'value': u'Inactive'
                            }, channel['properties']),
                'Failed to update channels with the correct hostName and/or iocName'
            )
        self.assertTrue(
            not client.find(property=[('hostName', 'initialHost')]),
            'Failed to cleanup old property')
        self.assertTrue(not client.find(property=[('iocName', 'initialIoc')]),
                        'Failed to cleanup old property')

        # Cleanup
        '''
        TODO this cleanup code should not be contingent to the successful completion of all checks...
        This could pollute CF 
        '''
        client.delete(channelName='cf-update-pv1')
        client.delete(channelName='cf-update-pv2')
        client.delete(propertyName=unaffectedProperty[u'name'])
        client.delete(tagName=unaffectedTag[u'name'])
Ejemplo n.º 37
0
 def testPVMove(self):
     '''
     ch1, ch2 on host1, ioc1
     ch1 on host1, ioc1; ch2 on host1, ioc2 (case1)
     ch1, ch2 on host1, ioc1 (reset)
     ch1 on host1, ioc1; ch2 on host2, ioc2 (case2)
     ch1, ch2 on host1, ioc1 (reset)
     '''
     try:
         updateChannelFinder(['ch1', 'ch2'], \
                             'host1', \
                             'ioc1', \
                             owner=self.owner, \
                             time=time(),\
                             service=self.baseURL, \
                             username=self.username, password=self.password)
         client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
         chs = client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])
         self.assertEqual(len(chs), 2, 'Expected 2 positive matches but found ' + str(len(chs)))
         '''CASE1'''
         updateChannelFinder(['ch1'], \
                             'host1', \
                             'ioc1', \
                             time=time(), \
                             owner=self.owner, service=self.baseURL, \
                             username=self.username, password=self.password)
         updateChannelFinder(['ch2'], \
                             'host1', \
                             'ioc2', \
                             time=time(), \
                             owner=self.owner, service=self.baseURL, \
                             username=self.username, password=self.password)
         chs = client.find(property=[('hostName', 'host1')])
         self.assertEqual(len(chs), 2, 'Expected 1 positive matches but found ' + str(len(chs)))
         self.assertEqual(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])[0].Name, 'ch1', \
                          'Failed to find the expected channel _ch1_ with prop host1, ioc1')
         self.assertEqual(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc2')])[0].Name, 'ch2', \
                          'Failed to find the expected channel _ch2_ with prop host1, ioc2')
         '''RESET'''
         updateChannelFinder(['ch1', 'ch2'], \
                             'host1', \
                             'ioc1', \
                             time=time(), \
                             owner=self.owner, service=self.baseURL, \
                             username=self.username, password=self.password)
         self.assertEqual(len(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])), 2, \
                          'Failed to reset the channels')            
         '''CASE2'''
         updateChannelFinder(['ch1'], \
                             'host1', \
                             'ioc1', \
                             owner=self.owner, \
                             time=time(), \
                             service=self.baseURL, username=self.username, \
                             password=self.password)
         updateChannelFinder(['ch2'], \
                             'host2', \
                             'ioc2', \
                             owner=self.owner, service=self.baseURL, \
                             time=time(), \
                             username=self.username, password=self.password)
         self.assertEqual(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])[0].Name, 'ch1', \
                          'Failed to find the expected channel _ch1_ with prop host1, ioc1')
         self.assertEqual(client.find(property=[('hostName', 'host2'), ('iocName', 'ioc2')])[0].Name, 'ch2', \
                          'Failed to find the expected channel _ch2_ with prop host1, ioc2')
         '''RESET'''
         updateChannelFinder(['ch1', 'ch2'], \
                             'host1', \
                             'ioc1', \
                             owner=self.owner, \
                             time=time(), \
                             service=self.baseURL, \
                             username=self.username, password=self.password)
         self.assertEqual(len(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])), 2, \
                          'Failed to reset the channels')
     finally:
         client.delete(channelName='ch1')
         client.delete(channelName='ch2')
Ejemplo n.º 38
0
 def testPreservingOfAttributes(self):
     '''
     This test is to ensure that existing properties and tags are left untouched.
     Case1:
     first time the cf-update comes across these channels and adds hostName and iocName
     Case2:
     the hostName is changed
     Case3:
     the iocName is changed
     Case4:
     both hostName and iocName are changed
     Case5:
     the channel is removed
     in all cases the existing unaffected* property and tag should remain with the channel               
     '''
     unaffectedProperty = {u'name':u'unaffectedProperty', u'owner':self.owner, u'value':u'unchanged'}
     unaffectedTag = {u'name':u'unaffectedTag', u'owner':self.owner}
     # create default client
     client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
     client.set(property=unaffectedProperty)
     client.set(tag=unaffectedTag)
     
     client.set(channel={u'name':u'cf-update-pv1', u'owner':u'cf-update', u'properties':[unaffectedProperty], u'tags':[unaffectedTag]})
     client.set(channel={u'name':u'cf-update-pv2', u'owner':u'cf-update', u'properties':[unaffectedProperty], u'tags':[unaffectedTag]})
     
     # Case1:
     hostName = 'initialHost'
     iocName = 'initialIoc'
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName, \
                         iocName, \
                         owner=self.owner, \
                         time=time(), \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(name='cf-update-pv*')
     for channel in channels:
         self.assertTrue(unaffectedProperty in channel['properties'] and unaffectedTag in channel['tags'])
         self.assertTrue(self.__check4properties({u'name':u'hostName', u'value':hostName}, channel['properties']) and 
                         self.__check4properties({u'name':u'iocName', u'value':iocName}, channel['properties']) and 
                         self.__check4properties({u'name':u'pvStatus', u'value':u'Active'}, channel['properties']),
                         'Failed to update channels with the correct hostName and/or iocName')
     # Case2:
     hostName = 'newHost'
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName, \
                         iocName, \
                         owner=self.owner, \
                         time=time(), \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(name='cf-update-pv*')
     for channel in channels:
         self.assertTrue(unaffectedProperty in channel['properties'] and unaffectedTag in channel['tags'])
         self.assertTrue(self.__check4properties({u'name':u'hostName', u'value':hostName}, channel['properties']) and 
                         self.__check4properties({u'name':u'iocName', u'value':iocName}, channel['properties']) and 
                         self.__check4properties({u'name':u'pvStatus', u'value':u'Active'}, channel['properties']),
                         'Failed to update channels with the correct hostName and/or iocName')
     self.assertTrue(not client.find(property=[('hostName', 'initialHost')]), 'Failed to cleanup old property')
     # Case 3:
     iocName = 'newIoc'
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName, \
                         iocName, \
                         owner=self.owner, \
                         time=time(), \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(name='cf-update-pv*')
     for channel in channels:
         self.assertTrue(unaffectedProperty in channel['properties'] and unaffectedTag in channel['tags'])
         self.assertTrue(self.__check4properties({u'name':u'hostName', u'value':hostName}, channel['properties']) and 
                         self.__check4properties({u'name':u'iocName', u'value':iocName}, channel['properties']) and 
                         self.__check4properties({u'name':u'pvStatus', u'value':u'Active'}, channel['properties']),
                         'Failed to update channels with the correct hostName and/or iocName')
     self.assertTrue(not client.find(property=[('hostName', 'initialHost')]), 'Failed to cleanup old property')
     self.assertTrue(not client.find(property=[('iocName', 'initialIoc')]), 'Failed to cleanup old property')
     # Case 4:
     updateChannelFinder([], \
                         hostName, \
                         iocName, \
                         owner=self.owner, \
                         time=time(), \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(name='cf-update-pv*')
     for channel in channels:
         self.assertTrue(unaffectedProperty in channel['properties'] and unaffectedTag in channel['tags'])
         self.assertTrue(self.__check4properties({u'name':u'hostName', u'value':hostName}, channel['properties']) and 
                         self.__check4properties({u'name':u'iocName', u'value':iocName}, channel['properties']) and 
                         self.__check4properties({u'name':u'pvStatus', u'value':u'Inactive'}, channel['properties']),
                         'Failed to update channels with the correct hostName and/or iocName')
     self.assertTrue(not client.find(property=[('hostName', 'initialHost')]), 'Failed to cleanup old property')
     self.assertTrue(not client.find(property=[('iocName', 'initialIoc')]), 'Failed to cleanup old property')
     
     # Cleanup
     '''
     TODO this cleanup code should not be contingent to the successful completion of all checks...
     This could pollute CF 
     '''
     client.delete(channelName='cf-update-pv1')
     client.delete(channelName='cf-update-pv2')
     client.delete(propertyName=unaffectedProperty[u'name'])
     client.delete(tagName=unaffectedTag[u'name'])
def _export_to_cfweb(channels, uri, username, password):
    # Improve performance by reducing the number
    # of HTTP server requests by restructuring
    # the channel data from channel oriented:
    #
    # {
    #    "CH:NAME1": {
    #           "properties":{
    #                 "system":"SEG1",
    #                 "device":"DEV1"
    #            },
    #            "tags":[
    #                "phyutil.sys.LS1",
    #                "phyutil.sub.CA01"
    #            ]
    #    },
    #    "CH:NAME2: {
    #            "properties":{
    #                 "system":"SEG1",
    #                 "device":"DEV2"
    #            }
    #            "tags":[
    #                "phyutil.sys.LS1",
    #                "phyutil.sub.BTS"
    #            ]
    #    }
    # }
    #
    # To property oriented:
    #
    # {
    #    "system" : {
    #        "SEG1": [ "CH:NAME1", "CH:NAME2" ]
    #    },
    #    "device" : {
    #        "DEV1": [ "CH:NAME1" ],
    #        "DEV2": [ "CH:NAME2" ]
    #    }
    # }
    #
    # And tag oriented:
    #
    # {
    #     "phyutil.sys.LS1": [ "CH:NAME1", "CH:NAME2" ],
    #     "phyutil.sub.CA01": [ "CH:NAME1" ],
    #     "phyutil.sub.BTS": [ "CH:NAME2" ]
    # }
    #
    tags = {}
    properties = {}
    channelnames = set()
    for name, ps, ts in channels:
        for p, value in ps.iteritems():
            if p not in properties:
                properties[p] = {}

            v = str(value)
            if v not in properties[p]:
                properties[p][v] = set()

            properties[p][v].add(name)

        for t in ts:
            if t not in tags:
                tags[t] = set()

            tags[t].add(name)

        channelnames.add(name)

    client = ChannelFinderClient(BaseURL=uri, username=username, password=password)

    channels = []
    for cname in channelnames:
        c = Channel(cname, username)
        channels.append(c)
        _LOGGER.debug("Export to Channel Finder: Set channel: %s", _Chan(c))
    client.set(channels=channels)
    _LOGGER.info("Export to ChannelFinder: Set channels: %s", len(channels))

    for prop, values in properties.iteritems():
        p = Property(prop, username)
        client.set(property=p)
        _LOGGER.info("Export to ChannelFinder: Set property: %s", _Prop(p))
        for propvalue, channelnames in values.iteritems():
            p.Value = propvalue
            client.update(property=p, channelNames=channelnames)
            _LOGGER.debug("Export to ChannelFinder: Update property: %s, for channels: %s", _Prop(p), len(channelnames))

    for tag, channelnames in tags.iteritems():
        t = Tag(tag, username)
        client.set(tag=t)
        _LOGGER.info("Export to ChannelFinder: Set tag: %s", _Tag(t))
        client.update(tag=t, channelNames=channelnames)
        _LOGGER.debug("Export to ChannelFinder: Update tag: %s, for channels: %s", _Tag(t), len(channelnames))
Ejemplo n.º 40
0
from channelfinder import ChannelFinderClient

"""
Simple script for adding active channels to Channel Finder Service for testing cf-store clean
If it gives a 500 error, run it again. Glassfish and CFS must be set up and running.
"""


def abbr(name, hostname, iocname, status):
    return {
        u"owner": "cf-update",
        u"name": name,
        u"properties": [
            {u"owner": "cf-update", u"name": "hostName", u"value": hostname},
            {u"owner": "cf-update", u"name": "iocName", u"value": iocname},
            {u"owner": "cf-update", u"name": "pvStatus", u"value": status},
            {u"owner": "cf-update", u"name": "time", u"value": "2016-08-18 12:33:09.953985"},
        ],
    }


client = ChannelFinderClient()
client.set(channels=[abbr(u"ch1", "testhosta", 1111, "Active"), abbr(u"test_channel", "testhosta", 1111, "Active")])
Ejemplo n.º 41
0
def updateChannelFinder(pvNames, hostName, iocName, time, owner, \
                        service=None, username=None, password=None):
    '''
    pvNames = list of pvNames 
    ([] permitted will effectively remove the hostname, iocname from all channels)
    hostName = pv hostName (None not permitted)
    iocName = pv iocName (None not permitted)
    owner = the owner of the channels and properties being added, this can be different from the user
    e.g. user = abc might create a channel with owner = group-abc
    time = the time at which these channels are being created/modified
    [optional] if not specified the default values are used by the 
    channelfinderapi lib
    service = channelfinder service URL
    username = channelfinder username
    password = channelfinder password
    '''
    if hostName == None or iocName == None:
        raise Exception, 'missing hostName or iocName'
    channels = []
    try:
        client = ChannelFinderClient(BaseURL=service, username=username, password=password)
    except:
        raise Exception, 'Unable to create a valid webResourceClient'
    checkPropertiesExist(client, owner)
    previousChannelsList = client.findByArgs([('hostName', hostName), ('iocName', iocName)])
    if previousChannelsList != None:
        for ch in previousChannelsList:
            if pvNames != None and ch[u'name'] in pvNames:
                ''''''
                channels.append(updateChannel(ch,\
                                              owner=owner, \
                                              hostName=hostName, \
                                              iocName=iocName, \
                                              pvStatus='Active', \
                                              time=time))
                pvNames.remove(ch[u'name'])
            elif pvNames == None or ch[u'name'] not in pvNames:
                '''Orphan the channel : mark as inactive, keep the old hostName and iocName'''
                channels.append(updateChannel(ch, \
                                              owner=owner, \
                                              hostName=ch.getProperties()['hostName'], \
                                              iocName=ch.getProperties()['iocName'], \
                                              pvStatus='InActive', \
                                              time=ch.getProperties()['time']))
    # now pvNames contains a list of pv's new on this host/ioc
    for pv in pvNames:
        ch = client.findByArgs([('~name',pv)])
        if ch == None:
            '''New channel'''
            channels.append(createChannel(pv, \
                                          chOwner=owner, \
                                          hostName=hostName, \
                                          iocName=iocName, \
                                          pvStatus='Active', \
                                          time=time))
        elif ch[0] != None:
            '''update existing channel: exists but with a different hostName and/or iocName'''
            channels.append(updateChannel(ch[0], \
                                          owner=owner, \
                                          hostName=hostName, \
                                          iocName=iocName, \
                                          pvStatus='Active', \
                                          time=time))
    client.set(channels=channels)
Ejemplo n.º 42
0
    def downloadCfs(self, cfsurl, **kwargs):
        """
        downloads data from channel finder service.
        
        :param cfsurl: the URL of channel finder service.
        :type cfsurl: str
        :param keep: if present, it only downloads specified properties.
        :type keep: list
        :param converter: convert properties from string to other format.
        :type converter: dict

        :Example:

            >>> prpt_list = ['elemName', 'sEnd']
            >>> conv_dict = {'sEnd', float}
            >>> downloadCfs(URL, keep = prpt_list, converter = conv_dict)
            >>> downloadCfs(URL, property=[('hostName', 'virtac2')])
            >>> downloadCfs(URL, property=[('hostName', 'virtac')], tagName='aphla.*')

        The channel finder client API provides *property* and *tagName* as
        keywords parameters. 
        """
        keep_prpts = kwargs.pop('keep', None)
        converter = kwargs.pop('converter', {})
        self.source = cfsurl

        from channelfinder import ChannelFinderClient
        cf = ChannelFinderClient(BaseURL=cfsurl)
        if len(kwargs) == 0:
            chs = cf.find(name='*')
        else:
            #print kwargs
            chs = cf.find(**kwargs)
        if not chs: return

        if keep_prpts is None:
            # use all possible property names
            keep_prpts = [p.Name for p in cf.getAllProperties()]

        #print "# include properties", properties
        for ch in chs:
            # keep only known properties
            prptdict = ch.getProperties()
            # prpts is known part from prptdict, otherwise empty dict
            if prptdict is not None:
                prpts = dict([v for v in prptdict.iteritems()])
                # convert the data type
            else:
                prpts = None
            # the empty tags could be None
            if self.use_unicode:
                self.rows.append([
                    unicode(ch.Name),
                    dict([(unicode(k), unicode(v))
                          for k, v in prpts.iteritems()]),
                    [unicode(v) for v in ch.getTags()]
                ])
            else:
                self.rows.append([
                    ch.Name.encode('ascii'),
                    dict([(k.encode('ascii'), v.encode('ascii'))
                          for k, v in prpts.iteritems()]),
                    [v.encode('ascii') for v in ch.getTags()]
                ])
            if self.rows[-1][1]:
                for k in converter:
                    self.rows[-1][1][k] = converter[k](prpts[k])
            # warn if hostName or iocName does not present
            if "hostName" not in self.rows[-1][1]:
                _logger.warn("no 'hostName' for {0}".format(self.rows[-1]))
            if "iocName" not in self.rows[-1][1]:
                _logger.warn("no 'iocName' for {0}".format(self.rows[-1]))

            del prptdict
Ejemplo n.º 43
0
class CFProcessor(service.Service):
    implements(interfaces.IProcessor)

    def __init__(self, name, conf):
        _log.info("CF_INIT %s", name)
        self.name, self.conf = name, conf
        self.channel_dict = defaultdict(list)
        self.iocs = dict()
        self.client = None
        self.currentTime = getCurrentTime
        self.lock = DeferredLock()

    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")
        from channelfinder import ChannelFinderClient
        # Using the default python cf-client.
        # The usr, username, and password are provided by the channelfinder._conf module.
        if self.client is None:  # For setting up mock test client
            self.client = ChannelFinderClient()
        self.clean_service()

    def stopService(self):
        service.Service.stopService(self)
        #Set channels to inactive and close connection to client
        self.running = 0
        self.clean_service()
        _log.info("CF_STOP")

    @defer.inlineCallbacks
    def commit(self, transaction_record):
        yield self.lock.acquire()
        try:
            yield deferToThread(self.__commit__, transaction_record)
        finally:
            self.lock.release()

    def __commit__(self, TR):
        _log.debug("CF_COMMIT %s", TR.infos.items())
        pvNames = [unicode(rname, "utf-8") for rid, (rname, rtype) in TR.addrec.iteritems()]
        delrec = list(TR.delrec)
        iocName = TR.src.port
        hostName = TR.src.host
        iocid = hostName + ":" + str(iocName)
        owner = TR.infos.get('CF_USERNAME') or TR.infos.get('ENGINEER') or self.conf.get('username', 'cfstore')
        time = self.currentTime()
        if TR.initial:
            self.iocs[iocid] = {"iocname": iocName, "hostname": hostName, "owner": owner, "channelcount": 0}  # add IOC to source list
        if not TR.connected:
            delrec.extend(self.channel_dict.keys())
        for pv in pvNames:
            self.channel_dict[pv].append(iocid)  # add iocname to pvName in dict
            self.iocs[iocid]["channelcount"] += 1
        for pv in delrec:
            if iocid in self.channel_dict[pv]:
                self.channel_dict[pv].remove(iocid)
                self.iocs[iocid]["channelcount"] -= 1
                if self.iocs[iocid]['channelcount'] == 0:
                    self.iocs.pop(iocid, None)
                elif self.iocs[iocid]['channelcount'] < 0:
                    _log.error("channel count negative!")
                if len(self.channel_dict[pv]) <= 0:  # case: channel has no more iocs
                    del self.channel_dict[pv]
        poll(__updateCF__, self.client, pvNames, delrec, self.channel_dict, self.iocs, hostName, iocName, time, owner)
        dict_to_file(self.channel_dict, self.iocs, self.conf)

    def clean_service(self):
        sleep = 1
        retry_limit = 5
        owner = self.conf.get('username', 'cfstore')
        while 1:
            try:
                _log.debug("Cleaning service...")
                channels = self.client.findByArgs([('pvStatus', 'Active')])
                if channels is not None:
                    new_channels = []
                    for ch in channels or []:
                        new_channels.append(ch[u'name'])
                    if len(new_channels) > 0:
                        self.client.update(property={u'name': 'pvStatus', u'owner': owner, u'value': "Inactive"},
                                           channelNames=new_channels)
                    _log.debug("Service clean.")
                    return
            except RequestException:
                _log.exception("cleaning failed, retrying: ")

            time.sleep(min(60, sleep))
            sleep *= 1.5
            if self.running == 0 and sleep >= retry_limit:
                _log.debug("Abandoning clean.")
                return
Ejemplo n.º 44
0
    def downloadCfs(self, cfsurl, **kwargs):
        """
        downloads data from channel finder service.
        
        :param cfsurl: the URL of channel finder service.
        :type cfsurl: str
        :param keep: if present, it only downloads specified properties.
        :type keep: list
        :param converter: convert properties from string to other format.
        :type converter: dict

        :Example:

            >>> prpt_list = ['elemName', 'sEnd']
            >>> conv_dict = {'sEnd', float}
            >>> downloadCfs(URL, keep = prpt_list, converter = conv_dict)
            >>> downloadCfs(URL, property=[('hostName', 'virtac2')])
            >>> downloadCfs(URL, property=[('hostName', 'virtac')], tagName='aphla.*')

        The channel finder client API provides *property* and *tagName* as
        keywords parameters. 
        """
        keep_prpts = kwargs.pop('keep', None)
        converter  = kwargs.pop('converter', {})
        self.source = cfsurl

        from channelfinder import ChannelFinderClient
        cf = ChannelFinderClient(BaseURL = cfsurl)
        if len(kwargs) == 0:
            chs = cf.find(name='*')
        else:
            #print kwargs
            chs = cf.find(**kwargs)
        if not chs: return

        if keep_prpts is None:
            # use all possible property names
            keep_prpts = [p.Name for p in cf.getAllProperties()]
            
        #print "# include properties", properties
        for ch in chs:
            # keep only known properties
            prptdict = ch.getProperties()
            # prpts is known part from prptdict, otherwise empty dict
            if prptdict is not None:
                prpts = dict([v for v in prptdict.iteritems()])
                # convert the data type
            else:
                prpts = None
            # the empty tags could be None
            if self.use_unicode:
                self.rows.append([unicode(ch.Name), 
                                  dict([(unicode(k), unicode(v))
                                        for k,v in prpts.iteritems()]),
                                  [unicode(v) for v in ch.getTags()]])
            else:
                self.rows.append([ch.Name.encode('ascii'), 
                                  dict([(k.encode('ascii'), v.encode('ascii'))
                                        for k,v in prpts.iteritems()]),
                                  [v.encode('ascii') for v in ch.getTags()]])
            if self.rows[-1][1]:
                for k in converter:
                    self.rows[-1][1][k] = converter[k](prpts[k])
            # warn if hostName or iocName does not present
            if "hostName" not in self.rows[-1][1]:
                _logger.warn("no 'hostName' for {0}".format(self.rows[-1]))
            if "iocName" not in self.rows[-1][1]:
                _logger.warn("no 'iocName' for {0}".format(self.rows[-1]))

            del prptdict
Ejemplo n.º 45
0
 def testAddUpdateChannelsWithProperties(self):
     '''
     This is to check that existing properties of channels are not affected.
     '''
     unaffectedProperty = Property('unaffectedProperty', self.owner, 'unchanged')
     # create default client
     client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
     client.set(property=unaffectedProperty)
     
     # add new pv's
     t1 = str(time())
     hostName1 = 'update-test-hostname' + t1
     iocName1 = 'update-test-iocName' + t1
     # New Channels added
     client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password);
     client.set(channel=Channel('cf-update-pv1', 'cf-update', properties=[unaffectedProperty]))
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName1, \
                         iocName1, \
                         owner=self.owner, \
                         time=t1, \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(property=[('hostName', hostName1), ('iocName', iocName1), ('time', t1)])
     self.assertTrue(len(channels) == 2, 'failed to create the channels with appropriate properties')
     channels = client.find(name='cf-update-pv1')
     self.assertTrue(len(channels) == 1)
     self.assertTrue(len(channels[0].Properties) == 5)
     # Cleanup
     client.delete(channelName='cf-update-pv1')
     client.delete(channelName='cf-update-pv2')
     client.delete(propertyName=unaffectedProperty.Name)
Ejemplo n.º 46
0
def initCfs(fpv, sep=","):
    dat = {}
    for line in open(fpv, 'r').readlines():
        #pv, hdl, name, idx, fam, fld
        rec = [v.strip() for v in line.split(sep)]
        pv, prpt = rec[0], rec[1:]
        dat.setdefault(pv, [])
        dat[pv].append(tuple(rec[1:]))

    import conf
    cfinput = {
        'BaseURL': cfsurl,
        'username': conf.username,
        'password': conf.password
    }
    cf = ChannelFinderClient(**cfinput)
    prpts = [p.Name for p in cf.getAllProperties()]
    for p in ["elemName", "elemIndex", "elemType", "elemField"]:
        if p in prpts: continue
        # property owner is cf-asd
        logging.info("add new property '%s'" % p)
        cf.set(property=Property(p, "cf-asd"))

    for pv,prptsets in dat.items():
        if len(prptsets) == 1:
            hdl, name, idx, fam, fld = prptsets[0]
            #cf.update(channel=Channel(pv, "cf-update", properties=[
            #    Property("elemHandle", "cf-asd", hdl),
            #    Property("elemName",   "cf-asd", name),
            #    Property("elemIndex",  "cf-asd", idx),
            #    Property("elemType",   "cf-asd", fam),
            #    Property("elemField",  "cf-asd", fld)]))
            cf.update(property=Property("elemHandle", "cf-asd", hdl),
                      channelName=pv)
            cf.update(property=Property("elemName",   "cf-asd", name),
                      channelName=pv)
            cf.update(property=Property("elemIndex",  "cf-asd", idx),
                      channelName=pv)
            cf.update(property=Property("elemType",   "cf-asd", fam),
                      channelName=pv)
            cf.update(property=Property("elemField",  "cf-asd", fld),
                      channelName=pv)
            logging.info("Done with %s" % pv)
Ejemplo n.º 47
0
class CFProcessor(service.Service):
    def __init__(self, name, conf):
        _log.info("CF_INIT %s", name)
        self.name, self.conf = name, conf
        self.channel_dict = defaultdict(list)
        self.iocs = dict()
        self.client = None
        self.currentTime = getCurrentTime
        self.lock = DeferredLock()

    def startService(self):
        service.Service.startService(self)
        # Returning a Deferred is not supported by startService(),
        # so instead attempt to acquire the lock synchonously!
        d = self.lock.acquire()
        if not d.called:
            d.cancel()
            service.Service.stopService(self)
            raise RuntimeError(
                'Failed to acquired CF Processor lock for service start')

        try:
            self._startServiceWithLock()
        except:
            service.Service.stopService(self)
            raise
        finally:
            self.lock.release()

    def _startServiceWithLock(self):
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [
                    prop['name'] for prop in self.client.getAllProperties()
                ]
                if (self.conf.get('alias', 'default') == 'on'):
                    reqd_props = {
                        'hostName', 'iocName', 'pvStatus', 'time', 'iocid',
                        'alias'
                    }
                else:
                    reqd_props = {
                        'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
                    }
                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                    if wl else wl
                # Are any required properties not already present on CF?
                properties = reqd_props - set(cf_props)
                # Are any whitelisted properties not already present on CF?
                # If so, add them too.
                properties.update(set(whitelist) - set(cf_props))

                owner = self.conf.get('username', 'cfstore')
                for prop in properties:
                    self.client.set(property={u'name': prop, u'owner': owner})

                self.whitelist = set(whitelist)
                _log.debug('WHITELIST = {}'.format(self.whitelist))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                if self.conf.getboolean('cleanOnStart', True):
                    self.clean_service()

    def stopService(self):
        service.Service.stopService(self)
        return self.lock.run(self._stopServiceWithLock)

    def _stopServiceWithLock(self):
        # Set channels to inactive and close connection to client
        if self.conf.getboolean('cleanOnStop', True):
            self.clean_service()
        _log.info("CF_STOP")

    # @defer.inlineCallbacks # Twisted v16 does not support cancellation!
    def commit(self, transaction_record):
        return self.lock.run(self._commitWithLock, transaction_record)

    def _commitWithLock(self, TR):
        self.cancelled = False

        t = deferToThread(self._commitWithThread, TR)

        def cancelCommit(d):
            self.cancelled = True
            d.callback(None)

        d = defer.Deferred(cancelCommit)

        def waitForThread(_ignored):
            if self.cancelled:
                return t

        d.addCallback(waitForThread)

        def chainError(err):
            if not err.check(defer.CancelledError):
                _log.error("CF_COMMIT FAILURE: %s", err)
            if self.cancelled:
                if not err.check(defer.CancelledError):
                    raise defer.CancelledError()
                return err
            else:
                d.callback(None)

        def chainResult(_ignored):
            if self.cancelled:
                raise defer.CancelledError()
            else:
                d.callback(None)

        t.addCallbacks(chainResult, chainError)
        return d

    def _commitWithThread(self, TR):
        if not self.running:
            raise defer.CancelledError(
                'CF Processor is not running (TR: %s:%s)', TR.src.host,
                TR.src.port)

        _log.info("CF_COMMIT: %s", TR)
        """
        a dictionary with a list of records with their associated property info  
        pvInfo 
        {rid: { "pvName":"recordName",
                "infoProperties":{propName:value, ...}}}
        """

        host = TR.src.host
        port = TR.src.port
        iocName = TR.infos.get('IOCNAME') or TR.src.port
        hostName = TR.infos.get('HOSTNAME') or TR.src.host
        owner = TR.infos.get('ENGINEER') or TR.infos.get(
            'CF_USERNAME') or self.conf.get('username', 'cfstore')
        time = self.currentTime()
        """The unique identifier for a particular IOC"""
        iocid = host + ":" + str(port)

        pvInfo = {}
        for rid, (rname, rtype) in TR.addrec.items():
            pvInfo[rid] = {"pvName": rname}
        for rid, (recinfos) in TR.recinfos.items():
            # find intersection of these sets
            if rid not in pvInfo:
                _log.warn('IOC: %s: PV not found for recinfo with RID: %s',
                          iocid, rid)
                continue
            recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
            if recinfo_wl:
                pvInfo[rid]['infoProperties'] = list()
                for infotag in recinfo_wl:
                    property = {
                        u'name': infotag,
                        u'owner': owner,
                        u'value': recinfos[infotag]
                    }
                    pvInfo[rid]['infoProperties'].append(property)
        for rid, alias in TR.aliases.items():
            if rid not in pvInfo:
                _log.warn('IOC: %s: PV not found for alias with RID: %s',
                          iocid, rid)
                continue
            pvInfo[rid]['aliases'] = alias

        delrec = list(TR.delrec)
        _log.debug("Delete records: %s", delrec)

        pvInfoByName = {}
        for rid, (info) in pvInfo.items():
            if info["pvName"] in pvInfoByName:
                _log.warn(
                    "Commit contains multiple records with PV name: %s (%s)",
                    pv, iocid)
                continue
            pvInfoByName[info["pvName"]] = info
            _log.debug("Add record: %s: %s", rid, info)

        if TR.initial:
            """Add IOC to source list """
            self.iocs[iocid] = {
                "iocname": iocName,
                "hostname": hostName,
                "owner": owner,
                "time": time,
                "channelcount": 0
            }
        if not TR.connected:
            delrec.extend(self.channel_dict.keys())
        for pv in pvInfoByName.keys():
            self.channel_dict[pv].append(
                iocid)  # add iocname to pvName in dict
            self.iocs[iocid]["channelcount"] += 1
            """In case, alias exists"""
            if (self.conf.get('alias', 'default' == 'on')):
                if pv in pvInfoByName and "aliases" in pvInfoByName[pv]:
                    for a in pvInfoByName[pv]["aliases"]:
                        self.channel_dict[a].append(
                            iocid)  # add iocname to pvName in dict
                        self.iocs[iocid]["channelcount"] += 1
        for pv in delrec:
            if iocid in self.channel_dict[pv]:
                self.channel_dict[pv].remove(iocid)
                if iocid in self.iocs:
                    self.iocs[iocid]["channelcount"] -= 1
                if self.iocs[iocid]['channelcount'] == 0:
                    self.iocs.pop(iocid, None)
                elif self.iocs[iocid]['channelcount'] < 0:
                    _log.error("Channel count negative: %s", iocid)
                if len(self.channel_dict[pv]
                       ) <= 0:  # case: channel has no more iocs
                    del self.channel_dict[pv]
                """In case, alias exists"""
                if (self.conf.get('alias', 'default' == 'on')):
                    if pv in pvInfoByName and "aliases" in pvInfoByName[pv]:
                        for a in pvInfoByName[pv]["aliases"]:
                            self.channel_dict[a].remove(iocid)
                            if iocid in self.iocs:
                                self.iocs[iocid]["channelcount"] -= 1
                            if self.iocs[iocid]['channelcount'] == 0:
                                self.iocs.pop(iocid, None)
                            elif self.iocs[iocid]['channelcount'] < 0:
                                _log.error("Channel count negative: %s", iocid)
                            if len(self.channel_dict[a]
                                   ) <= 0:  # case: channel has no more iocs
                                del self.channel_dict[a]
        poll(__updateCF__, self, pvInfoByName, delrec, hostName, iocName,
             iocid, owner, time)
        dict_to_file(self.channel_dict, self.iocs, self.conf)

    def clean_service(self):
        """
        Marks all channels as "Inactive" until the recsync server is back up
        """
        sleep = 1
        retry_limit = 5
        owner = self.conf.get('username', 'cfstore')
        while 1:
            try:
                _log.info("CF Clean Started")
                channels = self.client.findByArgs(
                    prepareFindArgs(self.conf, [('pvStatus', 'Active')]))
                if channels is not None:
                    new_channels = []
                    for ch in channels or []:
                        new_channels.append(ch[u'name'])
                    _log.info("Total channels to update: %s",
                              len(new_channels))
                    while len(new_channels) > 0:
                        _log.debug(
                            'Update "pvStatus" property to "Inactive" for %s channels',
                            min(len(new_channels), 10000))
                        self.client.update(property={
                            u'name': 'pvStatus',
                            u'owner': owner,
                            u'value': "Inactive"
                        },
                                           channelNames=new_channels[:10000])
                        new_channels = new_channels[10000:]
                    _log.info("CF Clean Completed")
                    return
            except RequestException as e:
                _log.error("Clean service failed: %s", e)

            _log.info("Clean service retry in %s seconds", min(60, sleep))
            time.sleep(min(60, sleep))
            sleep *= 1.5
            if self.running == 0 and sleep >= retry_limit:
                _log.info("Abandoning clean after %s seconds", retry_limit)
                return
Ejemplo n.º 48
0
 def testPreservingOfAttributes(self):
     '''
     This test is to ensure that existing properties and tags are left untouched.
     Case1:
     first time the cf-update comes across these channels and adds hostName and iocName
     Case2:
     the hostName is changed
     Case3:
     the iocName is changed
     Case4:
     both hostName and iocName are changed
     Case5:
     the channel is removed
     in all cases the existing unaffected* property and tag should remain with the channel               
     '''
     unaffectedProperty = Property('unaffectedProperty', self.owner, 'unchanged')
     unaffectedTag = Tag('unaffectedTag', self.owner)
     # create default client
     client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
     client.set(property=unaffectedProperty)
     client.set(tag=unaffectedTag)
     
     client.set(channel=Channel('cf-update-pv1', 'cf-update', properties=[unaffectedProperty], tags=[unaffectedTag]))     
     client.set(channel=Channel('cf-update-pv2', 'cf-update', properties=[unaffectedProperty], tags=[unaffectedTag]))
     
     # Case1:
     hostName = 'initialHost'
     iocName = 'initialIoc'
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName, \
                         iocName, \
                         owner=self.owner, \
                         time=time(), \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(name='cf-update-pv*')
     for channel in channels:
         self.assertTrue(unaffectedProperty in channel.Properties and unaffectedTag in channel.Tags)
         self.assertTrue(channel.getProperties()['hostName'] == hostName and \
                         channel.getProperties()['iocName'] == iocName and \
                         channel.getProperties()['pvStatus'] == 'Active', \
                         'Failed to update channels with the correct hostName and/or iocName')
     # Case2:
     hostName = 'newHost'
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName, \
                         iocName, \
                         owner=self.owner, \
                         time=time(), \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(name='cf-update-pv*')
     for channel in channels:
         self.assertTrue(unaffectedProperty in channel.Properties and unaffectedTag in channel.Tags)
         self.assertTrue(channel.getProperties()['hostName'] == hostName and \
                         channel.getProperties()['iocName'] == iocName and \
                         channel.getProperties()['pvStatus'] == 'Active', \
                         'Failed to update channels with the correct hostName and/or iocName')
     self.assertTrue(client.find(property=[('hostName', 'initialHost')]) == None, 'Failed to cleanup old property')
     # Case 3:
     iocName = 'newIoc'
     updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
                         hostName, \
                         iocName, \
                         owner=self.owner, \
                         time=time(), \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(name='cf-update-pv*')
     for channel in channels:
         self.assertTrue(unaffectedProperty in channel.Properties and unaffectedTag in channel.Tags)
         self.assertTrue(channel.getProperties()['hostName'] == hostName and \
                         channel.getProperties()['iocName'] == iocName and \
                         channel.getProperties()['pvStatus'] == 'Active', 'Failed to update channels with the correct hostName and/or iocName')
     self.assertTrue(client.find(property=[('hostName', 'initialHost')]) == None, 'Failed to cleanup old property')
     self.assertTrue(client.find(property=[('iocName', 'initialIoc')]) == None, 'Failed to cleanup old property')
     # Case 4:
     updateChannelFinder([], \
                         hostName, \
                         iocName, \
                         owner=self.owner, \
                         time=time(), \
                         service=self.baseURL , \
                         username=self.username, \
                         password=self.password)
     channels = client.find(name='cf-update-pv*')
     for channel in channels:
         self.assertTrue(unaffectedProperty in channel.Properties and unaffectedTag in channel.Tags)
         self.assertTrue(channel.getProperties()['hostName'] == hostName and \
                         channel.getProperties()['iocName'] == iocName and \
                         channel.getProperties()['pvStatus'] == 'InActive', \
                         'Failed to update channels with the correct hostName and/or iocName')
     self.assertTrue(client.find(property=[('hostName', 'initialHost')]) == None, 'Failed to cleanup old property')
     self.assertTrue(client.find(property=[('iocName', 'initialIoc')]) == None, 'Failed to cleanup old property')
     
     # Cleanup
     client.delete(channelName='cf-update-pv1')
     client.delete(channelName='cf-update-pv2')
     client.delete(propertyName=unaffectedProperty.Name)
     client.delete(tagName=unaffectedTag.Name)
Ejemplo n.º 49
0
class CFProcessor(service.Service):
    def __init__(self, name, conf):
        _log.info("CF_INIT %s", name)
        self.name, self.conf = name, conf
        self.channel_dict = defaultdict(list)
        self.iocs = dict()
        self.client = None
        self.currentTime = getCurrentTime
        self.lock = DeferredLock()

    def startService(self):
        service.Service.startService(self)
        self.running = 1
        _log.info("CF_START")

        if self.client is None:  # For setting up mock test client
            """
            Using the default python cf-client.  The url, username, and
            password are provided by the channelfinder._conf module.
            """
            from channelfinder import ChannelFinderClient
            self.client = ChannelFinderClient()
            try:
                cf_props = [
                    prop['name'] for prop in self.client.getAllProperties()
                ]

                reqd_props = {
                    'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
                }

                wl = self.conf.get('infotags', list())
                whitelist = [s.strip(', ') for s in wl.split()] \
                    if wl else wl
                # Are any required properties not already present on CF?
                properties = reqd_props - set(cf_props)
                # Are any whitelisted properties not already present on CF?
                # If so, add them too.
                properties.update(set(whitelist) - set(cf_props))

                owner = self.conf.get('username', 'cfstore')
                for prop in properties:
                    self.client.set(property={u'name': prop, u'owner': owner})

                self.whitelist = set(whitelist)
                _log.debug('WHITELIST = {}'.format(self.whitelist))
            except ConnectionError:
                _log.exception("Cannot connect to Channelfinder service")
                raise
            else:
                self.clean_service()

    def stopService(self):
        service.Service.stopService(self)
        # Set channels to inactive and close connection to client
        self.running = 0
        self.clean_service()
        _log.info("CF_STOP")

    @defer.inlineCallbacks
    def commit(self, transaction_record):
        yield self.lock.acquire()
        try:
            yield deferToThread(self.__commit__, transaction_record)
        finally:
            self.lock.release()

    def __commit__(self, TR):
        _log.debug("CF_COMMIT %s", TR.infos.items())
        """
        a dictionary with a list of records with their associated property info  
        pvInfo 
        {rid: { "pvName":"recordName",
                "infoProperties":{propName:value, ...}}}
        """

        iocName = TR.infos.get('IOCNAME') or TR.src.port
        hostName = TR.infos.get('HOSTNAME') or TR.src.host
        owner = TR.infos.get('ENGINEER') or TR.infos.get(
            'CF_USERNAME') or self.conf.get('username', 'cfstore')
        time = self.currentTime()

        pvInfo = {}
        for rid, (rname, rtype) in TR.addrec.items():
            pvInfo[rid] = {"pvName": rname}
        for rid, (recinfos) in TR.recinfos.items():
            # find intersection of these sets
            recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
            if recinfo_wl:
                pvInfo[rid]['infoProperties'] = list()
                for infotag in recinfo_wl:
                    _log.debug('INFOTAG = {}'.format(infotag))
                    property = {
                        u'name': infotag,
                        u'owner': owner,
                        u'value': recinfos[infotag]
                    }
                    pvInfo[rid]['infoProperties'].append(property)
        _log.debug(pvInfo)

        pvNames = [info["pvName"] for rid, (info) in pvInfo.items()]

        delrec = list(TR.delrec)
        _log.info("DELETED records " + str(delrec))

        host = TR.src.host
        port = TR.src.port
        """The unique identifier for a particular IOC"""
        iocid = host + ":" + str(port)
        _log.info("CF_COMMIT: " + iocid)

        if TR.initial:
            """Add IOC to source list """
            self.iocs[iocid] = {
                "iocname": iocName,
                "hostname": hostName,
                "owner": owner,
                "time": time,
                "channelcount": 0
            }
        if not TR.connected:
            delrec.extend(self.channel_dict.keys())
        for pv in pvNames:
            self.channel_dict[pv].append(
                iocid)  # add iocname to pvName in dict
            self.iocs[iocid]["channelcount"] += 1
        for pv in delrec:
            if iocid in self.channel_dict[pv]:
                self.channel_dict[pv].remove(iocid)
                if iocid in self.iocs:
                    self.iocs[iocid]["channelcount"] -= 1
                if self.iocs[iocid]['channelcount'] == 0:
                    self.iocs.pop(iocid, None)
                elif self.iocs[iocid]['channelcount'] < 0:
                    _log.error("channel count negative!")
                if len(self.channel_dict[pv]
                       ) <= 0:  # case: channel has no more iocs
                    del self.channel_dict[pv]
        poll(__updateCF__, self.client, pvInfo, delrec, self.channel_dict,
             self.iocs, hostName, iocName, iocid, owner, time)
        dict_to_file(self.channel_dict, self.iocs, self.conf)

    def clean_service(self):
        """
        Marks all channels as "Inactive" until the recsync server is back up
        """
        sleep = 1
        retry_limit = 5
        owner = self.conf.get('username', 'cfstore')
        while 1:
            try:
                _log.debug("Cleaning service...")
                channels = self.client.findByArgs([('pvStatus', 'Active')])
                if channels is not None:
                    new_channels = []
                    for ch in channels or []:
                        new_channels.append(ch[u'name'])
                    if len(new_channels) > 0:
                        self.client.update(property={
                            u'name': 'pvStatus',
                            u'owner': owner,
                            u'value': "Inactive"
                        },
                                           channelNames=new_channels)
                    _log.debug("Service clean.")
                    return
            except RequestException:
                _log.exception("cleaning failed, retrying: ")

            time.sleep(min(60, sleep))
            sleep *= 1.5
            if self.running == 0 and sleep >= retry_limit:
                _log.debug("Abandoning clean.")
                return
Ejemplo n.º 50
0
def cfs_append_from_csv1(rec_list, update_only):
    cf = ChannelFinderClient(**cfinput)
    all_prpts = [p.Name for p in cf.getAllProperties()]
    all_tags  = [t.Name for t in cf.getAllTags()]
    ignore_prpts = ['hostName', 'iocName']
    import csv
    rd = csv.reader(rec_list)
    # header line
    header = rd.next()
    # lower case of header
    hlow = [s.lower() for s in header]
    # number of headers, pv + properties
    nheader = len(header)
    # the index of PV, properties and tags
    ipv = hlow.index('pv')
    # the code did not rely on it, but it is a good practice
    if ipv != 0:
        raise RuntimeError("the first column should be pv")

    iprpt, itags = [], []
    for i, h in enumerate(header):
        if i == ipv: continue
        # if the header is empty, it is a tag
        if len(h.strip()) == 0: 
            itags.append(i)
        else:
            iprpt.append(i)

    tag_owner = OWNER
    prpt_owner = PRPTOWNER
    tags = {}
    # the data body
    for s in rd:
        prpts = [Property(header[i], prpt_owner, s[i]) for i in iprpt if s[i]]
        # itags could be empty if we put all tags in the end columns
        for i in itags + range(nheader, len(s)):
            rec = tags.setdefault(s[i].strip(), [])
            rec.append(s[ipv].strip())

        #print s[ipv], prpts, tags
        ch = cf.find(name=s[ipv])
        if ch is None:
            logging.warning("pv {0} does not exist".format(s[ipv]))
        elif len(ch) > 1:
            logging.warning("pv {0} is not unique ({1})".format(s[ipv], len(ch)))
        else:
            for p in prpts:
                #continue
                if p.Name in ignore_prpts: continue
                #if p.Name != 'symmetry': continue
                logging.info("updating '{0}' with property, {1}={2}".format(
                        s[ipv], p.Name, p.Value))
                cf.update(channelName=s[ipv], property=p)

    logging.info("finished updating properties")
    for t,pvs in tags.iteritems():
        if not hasTag(cf, t): cf.set(tag=Tag(t, tag_owner))
        if 'V:1-SR-BI{BETA}X-I' in pvs: continue
        if 'V:1-SR-BI{BETA}Y-I' in pvs: continue
        try:
            cf.update(tag=Tag(t, tag_owner), channelNames=pvs)
        except:
            print t, pvs
            raise

        logging.info("update '{0}' for {1} pvs".format(t, len(pvs)))
    logging.info("finished updating tags")