def init_cor(elems, url, patl, fname):
cf = ChannelFinderClient(BaseURL=url)
f = open(fname, "w")
for pvpat, hdl, name, idx, etp, fld in patl:
chs = cf.find(name=pvpat, property=[("hostName", "*"),
("iocName","*")])
for ch in chs:
pv = ch.Name
g = re.match(r"SR:(C[0-9][0-9]?)-MG{PS:([A-Z]+[0-9]+)([AB]?)}.*",
ch.Name)
if not g:
raise RuntimeError("unknown PV: '{0}'".format(ch.Name))
cell, fam = g.group(1), g.group(2)
matched = []
for e in elems:
if e[6] != cell: continue
if e[9].find(fam) < 0: continue
if g.group(3) and e[8] != g.group(3): continue
matched.append(e)
logging.info("found {0} COR records for '{1}', cell={2}, fam={3}".format(
len(matched), pv, cell, fam))
if not matched:
logging.warn("element not found for {0}".format(pv))
raise RuntimeError("unmatched quad PV: {0}".format(ch.Name))
if len(matched) > 1:
msg = "duplicate data for {0}: {1}".format(pv, matched)
logging.error(msg)
raise RuntimeError(msg)
e = matched[0]
logging.info("pick {0}".format(e))
f.write("%s,%s,%s,%s,COR,%s\n" % (ch.Name, hdl, e[0], e[1], fld))
f.close()
def startService(self):
service.Service.startService(self)
self.running = 1
_log.info("CF_START")
if self.client is None: # For setting up mock test client
"""
Using the default python cf-client. The url, username, and
password are provided by the channelfinder._conf module.
"""
from channelfinder import ChannelFinderClient
self.client = ChannelFinderClient()
try:
cf_props = [prop['name'] for prop in self.client.getAllProperties()]
reqd_props = set(('hostName', 'iocName', 'pvStatus', 'time',
'iocid'))
wl = self.conf.get('infotags', list())
whitelist = [s.strip(', ') for s in wl.split()] \
if wl else wl
self.prop_cache = (reqd_props - set(cf_props)) \
| set(whitelist)
owner = self.conf.get('username', 'cfstore')
for prop in self.prop_cache:
self.client.set(property={u'name': prop, u'owner': owner})
_log.debug('PROP_CACHE = {}'.format(self.prop_cache))
except ConnectionError:
_log.exception("Cannot connect to Channelfinder service")
raise
else:
self.clean_service()
def testAddUpdateChannels(self):
# Check the method finds the error conditions and raises exceptions
self.assertRaises(Exception, updateChannelFinder, [[], None, None])
self.assertRaises(Exception, updateChannelFinder,
[[], None, 'iocname'])
self.assertRaises(Exception, updateChannelFinder,
[[], 'hostName', None])
# create default client
client = ChannelFinderClient()
# add new pv's
t1 = str(time())
hostName1 = 'update-test-hostname' + t1
iocName1 = 'update-test-iocName' + t1
channels = client.find(property=[('hostName',
hostName1), ('iocName', iocName1)])
self.assertTrue(channels == None or len(channels) == 0,
'channels already present')
# New Channels added
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName1, \
iocName1, \
owner=self.owner, \
time=t1, \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(property=[('hostName',
hostName1), ('iocName',
iocName1), ('time',
t1)])
self.assertTrue(
len(channels) == 2,
'failed to create the channels with appropriate properties')
t2 = str(time())
hostName2 = 'update-test-hostname' + t2
iocName2 = 'update-test-iocName' + t2
# Existing channels are updated
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName2, \
iocName2, \
owner=self.owner, \
time=t2, \
service=self.baseURL , \
username=self.username, \
password=self.password)
# no channels should have the old proerty values
self.assertTrue(not client.find(property=[('hostName', hostName1), ('iocName', iocName1), ('time', t1)]), \
'failed to update the channels with appropriate properties, old values found')
# channels should be updated to the new values
self.assertTrue(len(client.find(property=[('hostName', hostName2), ('iocName', iocName2), ('time', t2)])) == 2, \
'failed to update the channels with appropriate properties')
# Cleanup
client = ChannelFinderClient(BaseURL=self.baseURL,
username=self.username,
password=self.password)
client.delete(channelName='cf-update-pv1')
client.delete(channelName='cf-update-pv2')
pass
def test_run(self):
'''
Tests main program full sweep.
'''
CFPropertyManager.run("cf-property-manager-test-dbl",
"cf-property-manager-test-cfg")
os.remove("cf-property-manager-test-cfg")
os.remove("cf-property-manager-test-dbl")
client = ChannelFinderClient()
client.delete(channelName="UT:RF-Cu:1{LD}Time:ShtDwn-I")
def startService(self):
service.Service.startService(self)
self.running = 1
_log.info("CF_START")
from channelfinder import ChannelFinderClient
# Using the default python cf-client.
# The usr, username, and password are provided by the channelfinder._conf module.
if self.client is None: # For setting up mock test client
self.client = ChannelFinderClient()
self.clean_service()
def download_cfs(cfsurl, **cfinput):
cf = ChannelFinderClient(BaseURL = cfsurl)
chs = cf.find(**cfinput)
ret = chanData.ChannelFinderData()
for ch in chs:
pv = ch.Name
prpt = dict([(p.Name, p.Value) for p in ch.Properties])
tags = [t.Name for t in ch.Tags]
ret.update(pv = pv, properties = prpt, tags = tags)
return ret
def download_cfs(cfsurl, **cfinput):
cf = ChannelFinderClient(BaseURL = cfsurl)
chs = cf.find(**cfinput)
ret = chanData.ChannelFinderData()
for ch in chs:
pv = ch.Name
prpt = dict([(p.Name, p.Value) for p in ch.Properties])
tags = [t.Name for t in ch.Tags]
ret.update(pv = pv, properties = prpt, tags = tags)
return ret
def _downloadCfs(self, cfsurl, **kwargs):
"""Get daya from channel finder web service
:param cfsurl:
:param kwargs:
:return:
"""
keep_prpts = kwargs.pop('keep', None)
converter = kwargs.pop('converter', {})
from channelfinder import ChannelFinderClient
cf = ChannelFinderClient(BaseURL = cfsurl)
if len(kwargs) == 0:
chs = cf.find(name='*')
else:
#print kwargs
chs = cf.find(**kwargs)
if not chs:
return
if keep_prpts is None:
# use all possible property names
keep_prpts = [p.Name for p in cf.getAllProperties()]
#print "# include properties", properties
for ch in chs:
# keep only known properties
prptdict = ch.getProperties()
# prpts is known part from prptdict, otherwise empty dict
if prptdict is not None:
prpts = dict([v for v in prptdict.iteritems()])
# convert the data type
else:
prpts = None
# the empty tags could be None
if self.use_unicode:
self.results.append([unicode(ch.Name),
dict([(unicode(k), unicode(v))
for k,v in prpts.iteritems()]),
[unicode(v) for v in ch.getTags()]])
else:
self.results.append([ch.Name.encode('ascii'),
dict([(k.encode('ascii'), v.encode('ascii'))
for k,v in prpts.iteritems()]),
[v.encode('ascii') for v in ch.getTags()]])
if self.results[-1][1]:
for k in converter:
self.results[-1][1][k] = converter[k](prpts[k])
# warn if hostName or iocName does not present
if self.host_name not in self.results[-1][1]:
_logger.warn("no 'hostName' for {0}".format(self.results[-1]))
if self.ioc_name not in self.results[-1][1]:
_logger.warn("no 'iocName' for {0}".format(self.results[-1]))
del prptdict
def update():
cf = ChannelFinderClient(**cfinput)
#pv="SR:C30-MG{PS:QH1A}I:Ps1DCCT1-I"
#cf.update(property=Property('elemType', PRPTOWNER, 'SEXT'),
# channelNames = [pv])
#chs = cf.find(name=pv)
#print "updating {0}".format([ch.Name for ch in chs])
#print "updating {0} elemField=b0".format([ch.Name for ch in chs])
#cf.update(property=Property('elemType', PRPTOWNER, 'SEXT'),
# channelNames = [ch.Name for ch in chs])
#cf.update(property=Property('system', PRPTOWNER, 'SR'),
# channelNames = [ch.Name for ch in chs])
cf.update(property=Property('elemGroups', PRPTOWNER), originalPropertyName='elemGroup')
def main():
if len(sys.argv) < 2:
parser.print_help()
return 1
args = parser.parse_args(sys.argv[2:])
if args.channelsfile is None:
parser.print_help()
return 1
if args.channelsfile is None:
parser.print_help()
return 1
try:
if args.url is None:
cfc = ChannelFinderClient()
args.url = "https://127.0.0.1:8181/ChannelFinder"
else:
if args.username is None:
args.username = r_input("Enter username: "******"Enter password: "******"Cannot connect to channelfinder service.")
return 1
ds = DataSource(source=args.channelsfile)
pv_data = ds.get_data()
if args.op == 'del':
try:
[cfc.delete(channelName=pv['name']) for pv in pv_data]
except:
print(
"Channels deleting ERROR, probably channel(s) does not exist.")
else: # args.op == 'add'
if args.username is None:
write_cfs(pv_data, args.url, force=True)
else:
write_cfs(pv_data,
args.url,
username=args.username,
password=args.password)
return 0
def startClient():
'''
Initiates client using default values if none are changed.
'''
global client
client = ChannelFinderClient(BaseURL=SERVICE_URL,
username=username,
password=password)
def run_simple_task():
cf = ChannelFinderClient(**cfinput)
chs = cf.find(property=[('elemType', 'RFCAVITY')])
if chs is not None:
for ch in chs:
print ch.Name,
for p in ch.Properties:
print p.Name,"=",p.Value,", ",
print " /"
#addPvProperty(cf, ch.Name, "elemPosition", "633.566", PRPTOWNER)
#addPvProperty(cf, ch.Name, "elemIndex", "2928", PRPTOWNER)
#addPvProperty(cf, ch.Name, "elemLength", "0.0", PRPTOWNER)
#addPvProperty(cf, ch.Name, "cell", "C24", PRPTOWNER)
addPvProperty(cf, ch.Name, "girder", "G", PRPTOWNER)
else:
print "did not found RFCAVITY"
def run_simple_task():
cf = ChannelFinderClient(**cfinput)
chs = cf.find(property=[('elemType', 'RFCAVITY')])
if chs is not None:
for ch in chs:
print ch.Name,
for p in ch.Properties:
print p.Name, "=", p.Value, ", ",
print " /"
#addPvProperty(cf, ch.Name, "elemPosition", "633.566", PRPTOWNER)
#addPvProperty(cf, ch.Name, "elemIndex", "2928", PRPTOWNER)
#addPvProperty(cf, ch.Name, "elemLength", "0.0", PRPTOWNER)
#addPvProperty(cf, ch.Name, "cell", "C24", PRPTOWNER)
addPvProperty(cf, ch.Name, "girder", "G", PRPTOWNER)
else:
print "did not found RFCAVITY"
def startService(self):
service.Service.startService(self)
self.running = 1
_log.info("CF_START")
if self.client is None: # For setting up mock test client
"""
Using the default python cf-client. The url, username, and
password are provided by the channelfinder._conf module.
"""
from channelfinder import ChannelFinderClient
self.client = ChannelFinderClient()
try:
cf_props = [
prop['name'] for prop in self.client.getAllProperties()
]
if (self.conf.get('alias', 'default') == 'on'):
reqd_props = {
'hostName', 'iocName', 'pvStatus', 'time', 'iocid',
'alias'
}
else:
reqd_props = {
'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
}
wl = self.conf.get('infotags', list())
whitelist = [s.strip(', ') for s in wl.split()] \
if wl else wl
# Are any required properties not already present on CF?
properties = reqd_props - set(cf_props)
# Are any whitelisted properties not already present on CF?
# If so, add them too.
properties.update(set(whitelist) - set(cf_props))
owner = self.conf.get('username', 'cfstore')
for prop in properties:
self.client.set(property={u'name': prop, u'owner': owner})
self.whitelist = set(whitelist)
_log.debug('WHITELIST = {}'.format(self.whitelist))
except ConnectionError:
_log.exception("Cannot connect to Channelfinder service")
raise
else:
self.clean_service()
def simple_test():
cf = ChannelFinderClient(**cfinput)
#cf.set(property=Property('test-prop1', 'lyyang', 'v1'))
#cf.set(property=Property('test-prop2', 'cf-asd', 'v2'))
#cf.set(property=Property('test-prop3', 'lyyang'))
#cf.set(property=Property('test-prop4', 'lyyang'))
#cf.delete(propertyName='test-prop1')
#cf.delete(propertyName='test-prop3')
pvname1 = 'SR:C00-BI:G00{DCCT:00}CUR-RB'
pvname2 = 'SR:C00-Glb:G00{CHROM:00}RB-X'
pvname3 = 'SR:C00-Glb:G00{CHROM:00}RB-Y'
cf.update(property=Property('test-prop2', 'cf-asd', 'v1'),
channelNames=[pvname1, pvname2, pvname3])
#chs = cf.find(property=[('test-prop2', '*')])
#if chs is not None:
# for ch in chs:
# print ch.Name
# for p in ch.Properties:
# print " ", p.Name,"=",p.Value, p.Owner, ", "
# print "/"
cf.delete(property=Property('test-prop2', 'cf-asd', 'v1'),
channelNames=[pvname2, pvname3])
chs = cf.find(property=[('test-prop2', '*')])
if chs is not None:
for ch in chs:
print ch.Name,
for p in ch.Properties:
print p.Name, "=", p.Value, ", ",
print " /"
def convCfsToSqlite(url, prefix = '', ignore = []):
"""
url - channel finder server URL
prefix - output DB file name prefix
ignore - list of ignored lattice name
"""
cf = ChannelFinderClient(BaseURL=url)
tagprefx = "aphla.sys."
tags = [tag.Name for tag in cf.getAllTags()
if tag.Name.startswith(tagprefx)]
for tag in tags:
latname = tag[len(tagprefx):]
if latname in ignore: continue
cfa = ChannelFinderAgent()
cfa.downloadCfs(url, property=[('hostName', '*')], tagName=tag)
#cfa.splitPropertyValue('elemGroups')
cfa.splitChainedElement('elemName')
cfa.saveSqlite("%s%s.sqlite" % (prefix, latname))
def convCfsToSqlite(url, prefix = '', ignore = []):
"""
url - channel finder server URL
prefix - output DB file name prefix
ignore - list of ignored lattice name
"""
cf = ChannelFinderClient(BaseURL=url)
tagprefx = "aphla.sys."
tags = [tag.Name for tag in cf.getAllTags()
if tag.Name.startswith(tagprefx)]
for tag in tags:
latname = tag[len(tagprefx):]
if latname in ignore: continue
cfa = ChannelFinderAgent()
cfa.downloadCfs(url, property=[('hostName', '*')], tagName=tag)
#cfa.splitPropertyValue('elemGroups')
cfa.splitChainedElement('elemName')
cfa.saveSqlite("%s%s.sqlite" % (prefix, latname))
def _init_cfs_data(self, **kws):
username = kws.get('username', None)
password = kws.get('password', None)
cfc = ChannelFinderClient(BaseURL=self.source,
username=username,
password=password)
c_url = cfc.get_resource('channel')
t_url = cfc.get_resource('tag')
p_url = cfc.get_resource('property')
resource_data = _cofetch_data([c_url, t_url, p_url])
prop_list = sorted([p['name'] for p in resource_data[p_url]])
tag_list = sorted([t['name'] for t in resource_data[t_url]])
raw_data = resource_data[c_url]
self._prop_list = prop_list
self._tag_list = tag_list
self._pvdata = raw_data
def init_bpm(elems, url, patl, fname):
# sort BPM names in lattice
bpms = {}
for e in elems:
if e[2] != "BPM": continue
# ignore User BPM
if e[0].startswith("pu"): continue
k = "C%d" % (int(e[6][1:]), )
bpms.setdefault(k, []).append((
e[0],
e[1],
))
for k, v in bpms.items():
print k, v
cf = ChannelFinderClient(BaseURL=url)
f = open(fname, "w")
for pvpat, hdl, name, idx, etp, fld in patl:
# for each pattern, sort PV and check against with elements.
bpmpvs = {}
chs = cf.find(name=pvpat,
property=[("hostName", "*"), ("iocName", "*")])
for ch in chs:
g = re.match(r"SR:(C[0-9][0-9]?)-BI{BPM:([0-9]+)}.*", ch.Name)
if not g:
raise RuntimeError("unknown PV: '{0}'".format(ch.Name))
bpmpvs.setdefault(g.group(1), []).append(ch.Name)
for k, pvs in bpmpvs.items():
#print k, pvs
names = bpms.get(k, [])
if len(names) != len(pvs):
#raise RuntimeError("BPM in {0} does not agree {1} != {2}".format(k, len(pvs), len(names)))
print "ERROR: BPM in {0} does not agree {1} != {2}".format(
k, len(pvs), len(names))
for pv in pvs:
g = re.match(r"SR:(C[0-9][0-9]?)-BI{BPM:([0-9]+)}.*", pv)
r = bpms[k][int(g.group(2)) - 1]
f.write("%s,%s,%s,%s,BPM,%s\n" % (pv, hdl, r[0], r[1], fld))
f.close()
def simple_test():
cf = ChannelFinderClient(**cfinput)
#cf.set(property=Property('test-prop1', 'lyyang', 'v1'))
#cf.set(property=Property('test-prop2', 'cf-asd', 'v2'))
#cf.set(property=Property('test-prop3', 'lyyang'))
#cf.set(property=Property('test-prop4', 'lyyang'))
#cf.delete(propertyName='test-prop1')
#cf.delete(propertyName='test-prop3')
pvname1='SR:C00-BI:G00{DCCT:00}CUR-RB'
pvname2='SR:C00-Glb:G00{CHROM:00}RB-X'
pvname3='SR:C00-Glb:G00{CHROM:00}RB-Y'
cf.update(property=Property('test-prop2', 'cf-asd', 'v1'),
channelNames=[pvname1, pvname2, pvname3])
#chs = cf.find(property=[('test-prop2', '*')])
#if chs is not None:
# for ch in chs:
# print ch.Name
# for p in ch.Properties:
# print " ", p.Name,"=",p.Value, p.Owner, ", "
# print "/"
cf.delete(property=Property('test-prop2', 'cf-asd', 'v1'),
channelNames=[pvname2, pvname3])
chs = cf.find(property=[('test-prop2', '*')])
if chs is not None:
for ch in chs:
print ch.Name,
for p in ch.Properties:
print p.Name,"=",p.Value,", ",
print " /"
def init_skquad(elems, url, patl, fname):
cf = ChannelFinderClient(BaseURL=url)
f = open(fname, "w")
for pvpat, hdl, name, idx, etp, fld in patl:
chs = cf.find(name=pvpat,
property=[("hostName", "*"), ("iocName", "*")])
for ch in chs:
pv = ch.Name
g = re.match(r"SR:(C[0-9][0-9]?)-MG{PS:SQK([A-Z][0-9])([AB]?)}.*",
ch.Name)
if not g:
raise RuntimeError("unknown PV: '{0}'".format(ch.Name))
cell, fam = g.group(1), g.group(2)
matched = []
for e in elems:
if e[2] != "SKQUAD": continue
# cell
if e[6] != cell: continue
# symmetry
if g.group(3) and e[8] != g.group(3): continue
matched.append(e)
logging.info(
"found {0} SKQUAD records for '{1}', cell={2}, fam={3}".format(
len(matched), pv, cell, fam))
if not matched:
logging.warn("element not found for {0}".format(pv))
raise RuntimeError("unmatched quad PV: {0}".format(ch.Name))
if len(matched) > 1:
msg = "duplicate data for {0}: {1}".format(pv, matched)
logging.error(msg)
raise RuntimeError(msg)
e = matched[0]
logging.info("pick {0}".format(e))
f.write("%s,%s,%s,%s,SKQUAD,%s\n" %
(ch.Name, hdl, e[0], e[1], fld))
f.close()
def startService(self):
service.Service.startService(self)
self.running = 1
_log.info("CF_START")
if self.client is None: # For setting up mock test client
"""
Using the default python cf-client. The url, username, and
password are provided by the channelfinder._conf module.
"""
# This module is always imported by twistd, even when 'proc = cf'
# isn't given. This way the ChannelFinderClient import is only
# attempted if it is actually needed.
from channelfinder import ChannelFinderClient
self.client = ChannelFinderClient()
try:
self._setup_properties()
except ConnectionError:
_log.exception("Cannot connect to Channelfinder service")
raise
else:
self.clean_service()
def testAddUpdateChannels(self):
# Check the method finds the error conditions and raises exceptions
self.assertRaises(Exception, updateChannelFinder, [[], None, None])
self.assertRaises(Exception, updateChannelFinder, [[], None, 'iocname'])
self.assertRaises(Exception, updateChannelFinder, [[], 'hostName', None])
# create default client
client = ChannelFinderClient()
# add new pv's
t1 = str(time())
hostName1 = 'update-test-hostname' + t1
iocName1 = 'update-test-iocName' + t1
channels = client.find(property=[('hostName', hostName1), ('iocName', iocName1)])
self.assertTrue(channels == None or len(channels) == 0, 'channels already present')
# New Channels added
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName1, \
iocName1, \
owner=self.owner, \
time=t1, \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(property=[('hostName', hostName1), ('iocName', iocName1), ('time', t1)])
self.assertTrue(len(channels) == 2, 'failed to create the channels with appropriate properties')
t2 = str(time())
hostName2 = 'update-test-hostname' + t2
iocName2 = 'update-test-iocName' + t2
# Existing channels are updated
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName2, \
iocName2, \
owner=self.owner, \
time=t2, \
service=self.baseURL , \
username=self.username, \
password=self.password)
# no channels should have the old proerty values
self.assertTrue(client.find(property=[('hostName', hostName1), ('iocName', iocName1), ('time', t1)]) == None, \
'failed to update the channels with appropriate properties, old values found')
# channels should be updated to the new values
self.assertTrue(len(client.find(property=[('hostName', hostName2), ('iocName', iocName2), ('time', t2)])) == 2, \
'failed to update the channels with appropriate properties')
# Cleanup
client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
client.delete(channelName='cf-update-pv1')
client.delete(channelName='cf-update-pv2')
pass
def init_sext(elems, url, patl, fname):
cf = ChannelFinderClient(BaseURL=url)
skquad = re.compile(r"SR:(C[0-9]+)-MG{PS:(SQ[A-Z0-9]+)}.*")
sxt_dw = re.compile(r"SR:(C[0-9]+)-MG{PS:(S[HLM][0-9])-(DW[0-9]+)}.*")
f = open(fname, "w")
for pvpat, hdl, name, idx, etp, fld in patl:
chs = cf.find(name=pvpat, property=[("hostName", "*"),
("iocName","*")])
for ch in chs:
if skquad.match(ch.Name): continue
if sxt_dw.match(ch.Name): continue
matched = match_sextupoles(elems, ch.Name)
logging.info("elements found for pv '{0}': {1}".format(
ch.Name, [elems[i][0] for i in matched]))
if not matched:
raise RuntimeError("unknown Sextupole PV: '{0}'".format(pv))
if len(matched) != 6:
raise RuntimeError("should be 6 sextupoles, {0} found".format(
len(matched)))
for e in [elems[i] for i in matched]:
f.write("%s,%s,%s,%s,SEXT,%s\n" % (ch.Name, hdl, e[0], e[1], fld))
f.close()
def init_bpm(elems, url, patl, fname):
# sort BPM names in lattice
bpms = {}
for e in elems:
if e[2] != "BPM": continue
# ignore User BPM
if e[0].startswith("pu"): continue
k = "C%d" % (int(e[6][1:]),)
bpms.setdefault(k, []).append((e[0], e[1],))
for k,v in bpms.items(): print k, v
cf = ChannelFinderClient(BaseURL=url)
f = open(fname, "w")
for pvpat, hdl, name, idx, etp, fld in patl:
# for each pattern, sort PV and check against with elements.
bpmpvs = {}
chs = cf.find(name=pvpat, property=[("hostName", "*"),
("iocName","*")])
for ch in chs:
g = re.match(r"SR:(C[0-9][0-9]?)-BI{BPM:([0-9]+)}.*", ch.Name)
if not g:
raise RuntimeError("unknown PV: '{0}'".format(ch.Name))
bpmpvs.setdefault(g.group(1), []).append(ch.Name)
for k,pvs in bpmpvs.items():
#print k, pvs
names = bpms.get(k, [])
if len(names) != len(pvs):
#raise RuntimeError("BPM in {0} does not agree {1} != {2}".format(k, len(pvs), len(names)))
print "ERROR: BPM in {0} does not agree {1} != {2}".format(k, len(pvs), len(names))
for pv in pvs:
g = re.match(r"SR:(C[0-9][0-9]?)-BI{BPM:([0-9]+)}.*", pv)
r = bpms[k][int(g.group(2)) - 1]
f.write("%s,%s,%s,%s,BPM,%s\n" % (pv, hdl, r[0], r[1], fld))
f.close()
def init_sext(elems, url, patl, fname):
cf = ChannelFinderClient(BaseURL=url)
skquad = re.compile(r"SR:(C[0-9]+)-MG{PS:(SQ[A-Z0-9]+)}.*")
sxt_dw = re.compile(r"SR:(C[0-9]+)-MG{PS:(S[HLM][0-9])-(DW[0-9]+)}.*")
f = open(fname, "w")
for pvpat, hdl, name, idx, etp, fld in patl:
chs = cf.find(name=pvpat,
property=[("hostName", "*"), ("iocName", "*")])
for ch in chs:
if skquad.match(ch.Name): continue
if sxt_dw.match(ch.Name): continue
matched = match_sextupoles(elems, ch.Name)
logging.info("elements found for pv '{0}': {1}".format(
ch.Name, [elems[i][0] for i in matched]))
if not matched:
raise RuntimeError("unknown Sextupole PV: '{0}'".format(pv))
if len(matched) != 6:
raise RuntimeError("should be 6 sextupoles, {0} found".format(
len(matched)))
for e in [elems[i] for i in matched]:
f.write("%s,%s,%s,%s,SEXT,%s\n" %
(ch.Name, hdl, e[0], e[1], fld))
f.close()
def testPVUpdate(self):
'''
Test condition
IOC turned on with ch1, ch2
IOC turned on with ch1 only
IOC turned on with ch1, ch2
'''
try:
updateChannelFinder(['ch1', 'ch2'], \
'testHost', \
'testIOC', \
owner=self.owner, \
time=time(), \
service=self.baseURL, \
username=self.username, password=self.password)
client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
chs = client.find(property=[('hostName', 'testHost'), ('iocName', 'testIOC'), ('pvStatus', 'Active')])
self.assertEqual(len(chs), 2, 'Expected 2 positive matches but found ' + str(len(chs)))
updateChannelFinder(['ch1'], \
'testHost', \
'testIOC', \
owner=self.owner, \
time=time(), \
service=self.baseURL, \
username=self.username, password=self.password)
chs = client.find(property=[('hostName', 'testHost'), ('iocName', 'testIOC'), ('pvStatus', 'Active')])
self.assertEqual(len(chs), 1, 'Expected 1 positive matches but found ' + str(len(chs)))
self.assertTrue(chs[0].Name == 'ch1', 'channel with name ch1 not found')
chs = client.find(property=[('hostName', 'testHost'), ('iocName', 'testIOC'), ('pvStatus', 'InActive')])
self.assertEqual(len(chs), 1, 'Expected 1 positive matches but found ' + str(len(chs)))
self.assertTrue(chs[0].Name == 'ch2', 'channel with name ch2 not found')
updateChannelFinder(['ch1', 'ch2'], \
'testHost', \
'testIOC', \
owner=self.owner, \
time=time(), \
service=self.baseURL, \
username=self.username, password=self.password)
chs = client.find(property=[('hostName', 'testHost'), ('iocName', 'testIOC'), ('pvStatus', 'Active')])
self.assertEqual(len(chs), 2, 'Expected 2 positive matches but found ' + str(len(chs)))
finally:
client.delete(channelName='ch1')
client.delete(channelName='ch2')
def startService(self):
service.Service.startService(self)
self.running = 1
_log.info("CF_START")
if self.client is None: # For setting up mock test client
"""
Using the default python cf-client. The url, username, and
password are provided by the channelfinder._conf module.
"""
from channelfinder import ChannelFinderClient
self.client = ChannelFinderClient()
try:
cf_props = [prop['name'] for prop in self.client.getAllProperties()]
if (self.conf.get('alias', 'default') == 'on'):
reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid', 'alias'}
else:
reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid'}
wl = self.conf.get('infotags', list())
whitelist = [s.strip(', ') for s in wl.split()] \
if wl else wl
# Are any required properties not already present on CF?
properties = reqd_props - set(cf_props)
# Are any whitelisted properties not already present on CF?
# If so, add them too.
properties.update(set(whitelist) - set(cf_props))
owner = self.conf.get('username', 'cfstore')
for prop in properties:
self.client.set(property={u'name': prop, u'owner': owner})
self.whitelist = set(whitelist)
_log.debug('WHITELIST = {}'.format(self.whitelist))
except ConnectionError:
_log.exception("Cannot connect to Channelfinder service")
raise
else:
self.clean_service()
def cfs_append_from_cmd(cmd_list, update_only=False):
"""
update the cfs from command file:
addtag tag pv1,pv2,pv3
removetag tag pv1,pv2,...
updateproperty prpt=val pv1,pv2
appendproperty prpt=val pv1,pv2
removeproperty prpt pv1,pv2
The properties are updated as 'cf-asd' account, and tags in 'cf-aphla'
"""
print("# Updating CFS from text file")
cf = ChannelFinderClient(**cfinput)
pvs = []
# read each line of the txt command file
for i, line in enumerate(cmd_list.readlines()):
if not line.strip(): continue
if line.strip().find('#') == 0: continue
rec = [v.strip() for v in line.strip().split()]
if len(rec) <= 2:
logging.warning("skiping line %d: %s" % (i, line))
continue
if rec[0] == 'addtag':
pvs = [pv.strip() for pv in rec[2].split(',')]
addTagPvs(cf, rec[1].strip(), pvs, OWNER)
continue
elif rec[0] == 'removetag':
pvs = [pv.strip() for pv in rec[2].split(',')]
removeTagPvs(cf, rec[1].strip(), pvs, OWNER)
continue
elif rec[0] in ('updateproperty', ):
pvs = [pv.strip() for pv in rec[2].split(',')]
prpt, val = rec[1].split('=')
if val == "''": val = ''
updatePropertyPvs(cf, prpt, PRPTOWNER, val, pvs)
continue
elif rec[0] in ('appendproperty', ):
pvs = [pv.strip() for pv in rec[2].split(',')]
prpt, val = rec[1].split('=')
if val == "''": val = ''
appendPropertyPvs(cf, prpt, PRPTOWNER, val, pvs)
continue
elif rec[0] == 'removeproperty':
pvs = [pv.strip() for pv in rec[2].split(',')]
prpt = rec[1].strip()
removePropertyPvs(cf, prpt, PRPTOWNER, pvs)
pv = rec[0]
cmd = rec[1]
newval = ''.join(rec[2:])
for v in [v.strip() for v in newval.split(',')]:
if v.find('=') >= 0:
# it is a property
prpt, val = v.split('=')
if cmd == 'add':
print("add", prpt, val)
addPvProperty(cf, pv, prpt, val, PRPTOWNER)
else:
# tag:
if cmd == 'add': addPvTag(cf, pv, v, OWNER)
elif cmd == 'remove': removePvTag(cf, pv, v, OWNER)
def cfs_append_from_sqlite(fname, update_only):
sq = ap.chanfinder.ChannelFinderAgent()
sq.loadSqlite(fname)
cf = ChannelFinderClient(**cfinput)
all_prpts = [p.Name for p in cf.getAllProperties()]
all_tags = [t.Name for t in cf.getAllTags()]
ignore_prpts = ['hostName', 'iocName']
allpvs = []
tag_owner = OWNER
prpt_owner = PRPTOWNER
prpt_data, tag_data = {}, {}
# the data body
for pv, prpts, tags in sq.rows:
if not pv: continue
if pv in allpvs: continue
if pv.find("SR:") != 0: continue
logging.info("updating '{0}'".format(pv))
allpvs.append(pv)
prpt_list, tag_list = [], []
for k, v in prpts.items():
if k not in [
"elemIndex", "system", "elemType", "elemHandle",
"elemName", "elemField"
]:
continue
prpt_data.setdefault((k, v), [])
prpt_data[(k, v)].append(pv)
for tag in tags:
if not tag.startswith("aphla."): continue
tag_data.setdefault(tag, [])
tag_data[tag].append(pv)
#tag_list.append(Tag(r.strip()), tag_owner)
logging.info("{0}: {1} ({2})".format(pv, tag, tag_owner))
#addPvTag(cf, pv, tag, tag_owner)
errpvs = []
for pv in allpvs:
chs = cf.find(name=pv)
if not chs:
errpvs.append(pv)
print("PV '%s' does not exist" % pv)
continue
elif len(chs) != 1:
print("Find two results for pv=%s" % pv)
continue
prpts = chs[0].getProperties()
if not prpts: continue
for prpt, val in prpts.items():
pvlst = prpt_data.get((prpt, val), [])
if not pvlst: continue
try:
j = pvlst.index(pv)
prpt_data[(prpt, val)].pop(j)
except:
# the existing data is not in the update list, skip
pass
#if errpvs:
# #raise RuntimeError("PVs '{0}' are missing".format(errpvs))
# print
logging.warn("{0} does not exist in DB".format(errpvs))
for k, v in prpt_data.items():
vf = [pv for pv in v if pv not in errpvs]
if not vf:
logging.info("no valid PVs for {0}".format(k))
continue
updatePropertyPvs(cf, k[0], prpt_owner, k[1], vf)
logging.info("add property {0} for pvs {1}".format(k, vf))
for k, v in tag_data.items():
vf = [pv for pv in v if pv not in errpvs]
addTagPvs(cf, k, vf, tag_owner)
logging.info("add tag {0} for pvs {1}".format(k, vf))
def cfs_append_from_csv2(rec_list, update_only):
cf = ChannelFinderClient(**cfinput)
all_prpts = [p.Name for p in cf.getAllProperties()]
all_tags = [t.Name for t in cf.getAllTags()]
ignore_prpts = ['hostName', 'iocName']
import csv
rd = csv.reader(rec_list)
allpvs = []
tag_owner = OWNER
prpt_owner = PRPTOWNER
prpt_data, tag_data = {}, {}
# the data body
for s in rd:
if not s: continue
if not s[0].strip(): continue
if s[0].strip().startswith('#'): continue
pv = s[0].strip()
logging.info("updating '{0}'".format(pv))
#cf.update(property=Property('elemType', PRPTOWNER, 'QUAD'),
# channelNames = [pv])
#chs = cf.find(name=pv)
#sys.exit(0)
#logging.info("{0} and {1}".format(chs[0].Name, type(chs[0].Name)))
#logging.info("{0} and {1}".format(pv, type(pv)))
allpvs.append(pv)
prpt_list, tag_list = [], []
for r in s[1:]:
if r.find('=') > 0:
prpt, val = [v.strip() for v in r.split('=')]
prpt_data.setdefault((prpt, val), [])
prpt_data[(prpt, val)].append(pv)
else:
# it is a tag
tag = r.strip()
if not tag: continue
tag_data.setdefault(tag, [])
tag_data[tag].append(pv)
#tag_list.append(Tag(r.strip()), tag_owner)
logging.info("{0}: {1} ({2})".format(pv, tag, tag_owner))
#addPvTag(cf, pv, tag, tag_owner)
errpvs = []
for pv in allpvs:
chs = cf.find(name=pv)
if not chs:
errpvs.append(pv)
print("PV '%s' does not exist" % pv)
continue
elif len(chs) != 1:
print("Find two results for pv=%s" % pv)
continue
for prpt, val in chs[0].getProperties().items():
pvlst = prpt_data.get((prpt, val), [])
if not pvlst: continue
try:
j = pvlst.index(pv)
prpt_data[(prpt, val)].pop(j)
except:
# the existing data is not in the update list, skip
pass
#if errpvs:
# #raise RuntimeError("PVs '{0}' are missing".format(errpvs))
# print
logging.warn("{0} does not exist in DB".format(errpvs))
for k, v in prpt_data.items():
vf = [pv for pv in v if pv not in errpvs]
if not vf:
logging.info("no valid PVs for {0}".format(k))
continue
updatePropertyPvs(cf, k[0], prpt_owner, k[1], vf)
logging.info("add property {0} for pvs {1}".format(k, vf))
for k, v in tag_data.items():
vf = [pv for pv in v if pv not in errpvs]
addTagPvs(cf, k, vf, tag_owner)
logging.info("add tag {0} for pvs {1}".format(k, vf))
def cfs_append_from_csv1(rec_list, update_only):
cf = ChannelFinderClient(**cfinput)
all_prpts = [p.Name for p in cf.getAllProperties()]
all_tags = [t.Name for t in cf.getAllTags()]
ignore_prpts = ['hostName', 'iocName']
import csv
rd = csv.reader(rec_list)
# header line
header = rd.next()
# lower case of header
hlow = [s.lower() for s in header]
# number of headers, pv + properties
nheader = len(header)
# the index of PV, properties and tags
ipv = hlow.index('pv')
# the code did not rely on it, but it is a good practice
if ipv != 0:
raise RuntimeError("the first column should be pv")
iprpt, itags = [], []
for i, h in enumerate(header):
if i == ipv: continue
# if the header is empty, it is a tag
if len(h.strip()) == 0:
itags.append(i)
else:
iprpt.append(i)
tag_owner = OWNER
prpt_owner = PRPTOWNER
tags = {}
# the data body
for s in rd:
prpts = [Property(header[i], prpt_owner, s[i]) for i in iprpt if s[i]]
# itags could be empty if we put all tags in the end columns
for i in itags + range(nheader, len(s)):
rec = tags.setdefault(s[i].strip(), [])
rec.append(s[ipv].strip())
#print s[ipv], prpts, tags
ch = cf.find(name=s[ipv])
if ch is None:
logging.warning("pv {0} does not exist".format(s[ipv]))
elif len(ch) > 1:
logging.warning("pv {0} is not unique ({1})".format(
s[ipv], len(ch)))
else:
for p in prpts:
#continue
if p.Name in ignore_prpts: continue
#if p.Name != 'symmetry': continue
logging.info("updating '{0}' with property, {1}={2}".format(
s[ipv], p.Name, p.Value))
cf.update(channelName=s[ipv], property=p)
logging.info("finished updating properties")
for t, pvs in tags.items():
if not hasTag(cf, t): cf.set(tag=Tag(t, tag_owner))
if 'V:1-SR-BI{BETA}X-I' in pvs: continue
if 'V:1-SR-BI{BETA}Y-I' in pvs: continue
try:
cf.update(tag=Tag(t, tag_owner), channelNames=pvs)
except:
print(t, pvs)
raise
logging.info("update '{0}' for {1} pvs".format(t, len(pvs)))
logging.info("finished updating tags")
def cfs_append_from_csv2(rec_list, update_only):
cf = ChannelFinderClient(**cfinput)
all_prpts = [p.Name for p in cf.getAllProperties()]
all_tags = [t.Name for t in cf.getAllTags()]
ignore_prpts = ['hostName', 'iocName']
import csv
rd = csv.reader(rec_list)
allpvs = []
tag_owner = OWNER
prpt_owner = PRPTOWNER
prpt_data, tag_data = {}, {}
# the data body
for s in rd:
if not s: continue
if not s[0].strip(): continue
if s[0].strip().startswith('#'): continue
pv = s[0].strip()
logging.info("updating '{0}'".format(pv))
#cf.update(property=Property('elemType', PRPTOWNER, 'QUAD'),
# channelNames = [pv])
#chs = cf.find(name=pv)
#sys.exit(0)
#logging.info("{0} and {1}".format(chs[0].Name, type(chs[0].Name)))
#logging.info("{0} and {1}".format(pv, type(pv)))
allpvs.append(pv)
prpt_list, tag_list = [], []
for r in s[1:]:
if r.find('=') > 0:
prpt, val = [v.strip() for v in r.split('=')]
prpt_data.setdefault((prpt, val), [])
prpt_data[(prpt, val)].append(pv)
else:
# it is a tag
tag = r.strip()
if not tag: continue
tag_data.setdefault(tag, [])
tag_data[tag].append(pv)
#tag_list.append(Tag(r.strip()), tag_owner)
logging.info("{0}: {1} ({2})".format(pv, tag, tag_owner))
#addPvTag(cf, pv, tag, tag_owner)
errpvs = []
for pv in allpvs:
chs = cf.find(name=pv)
if not chs:
errpvs.append(pv)
print "PV '%s' does not exist" % pv
continue
elif len(chs) != 1:
print "Find two results for pv=%s" % pv
continue
for prpt,val in chs[0].getProperties().items():
pvlst = prpt_data.get((prpt, val), [])
if not pvlst: continue
try:
j = pvlst.index(pv)
prpt_data[(prpt,val)].pop(j)
except:
# the existing data is not in the update list, skip
pass
#if errpvs:
# #raise RuntimeError("PVs '{0}' are missing".format(errpvs))
# print
logging.warn("{0} does not exist in DB".format(errpvs))
for k,v in prpt_data.iteritems():
vf = [pv for pv in v if pv not in errpvs]
if not vf:
logging.info("no valid PVs for {0}".format(k))
continue
updatePropertyPvs(cf, k[0], prpt_owner, k[1], vf)
logging.info("add property {0} for pvs {1}".format(k, vf))
for k,v in tag_data.iteritems():
vf = [pv for pv in v if pv not in errpvs]
addTagPvs(cf, k, vf, tag_owner)
logging.info("add tag {0} for pvs {1}".format(k, vf))
class CFProcessor(service.Service):
def __init__(self, name, conf):
_log.info("CF_INIT %s", name)
self.name, self.conf = name, conf
self.channel_dict = defaultdict(list)
self.iocs = dict()
self.client = None
self.currentTime = getCurrentTime
self.lock = DeferredLock()
def startService(self):
service.Service.startService(self)
self.running = 1
_log.info("CF_START")
if self.client is None: # For setting up mock test client
"""
Using the default python cf-client. The url, username, and
password are provided by the channelfinder._conf module.
"""
from channelfinder import ChannelFinderClient
self.client = ChannelFinderClient()
try:
cf_props = [prop['name'] for prop in self.client.getAllProperties()]
if (self.conf.get('alias', 'default') == 'on'):
reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid', 'alias'}
else:
reqd_props = {'hostName', 'iocName', 'pvStatus', 'time', 'iocid'}
wl = self.conf.get('infotags', list())
whitelist = [s.strip(', ') for s in wl.split()] \
if wl else wl
# Are any required properties not already present on CF?
properties = reqd_props - set(cf_props)
# Are any whitelisted properties not already present on CF?
# If so, add them too.
properties.update(set(whitelist) - set(cf_props))
owner = self.conf.get('username', 'cfstore')
for prop in properties:
self.client.set(property={u'name': prop, u'owner': owner})
self.whitelist = set(whitelist)
_log.debug('WHITELIST = {}'.format(self.whitelist))
except ConnectionError:
_log.exception("Cannot connect to Channelfinder service")
raise
else:
self.clean_service()
def stopService(self):
service.Service.stopService(self)
# Set channels to inactive and close connection to client
self.running = 0
self.clean_service()
_log.info("CF_STOP")
@defer.inlineCallbacks
def commit(self, transaction_record):
yield self.lock.acquire()
try:
yield deferToThread(self.__commit__, transaction_record)
finally:
self.lock.release()
def __commit__(self, TR):
_log.debug("CF_COMMIT %s", TR.infos.items())
"""
a dictionary with a list of records with their associated property info
pvInfo
{rid: { "pvName":"recordName",
"infoProperties":{propName:value, ...}}}
"""
iocName = TR.infos.get('IOCNAME') or TR.src.port
hostName = TR.infos.get('HOSTNAME') or TR.src.host
owner = TR.infos.get('ENGINEER') or TR.infos.get('CF_USERNAME') or self.conf.get('username', 'cfstore')
time = self.currentTime()
pvInfo = {}
for rid, (rname, rtype) in TR.addrec.items():
pvInfo[rid] = {"pvName": rname}
for rid, (recinfos) in TR.recinfos.items():
# find intersection of these sets
recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
if recinfo_wl:
pvInfo[rid]['infoProperties'] = list()
for infotag in recinfo_wl:
_log.debug('INFOTAG = {}'.format(infotag))
property = {u'name': infotag, u'owner': owner,
u'value': recinfos[infotag]}
pvInfo[rid]['infoProperties'].append(property)
for rid, alias in TR.aliases.items():
pvInfo[rid]['aliases'] = alias
_log.debug(pvInfo)
pvNames = [info["pvName"] for rid, (info) in pvInfo.items()]
delrec = list(TR.delrec)
_log.info("DELETED records " + str(delrec))
host = TR.src.host
port = TR.src.port
"""The unique identifier for a particular IOC"""
iocid = host + ":" + str(port)
_log.info("CF_COMMIT: " + iocid)
if TR.initial:
"""Add IOC to source list """
self.iocs[iocid] = {"iocname": iocName, "hostname": hostName, "owner": owner, "time": time,
"channelcount": 0}
if not TR.connected:
delrec.extend(self.channel_dict.keys())
for pv in pvNames:
self.channel_dict[pv].append(iocid) # add iocname to pvName in dict
self.iocs[iocid]["channelcount"] += 1
"""In case, alias exists"""
if (self.conf.get('alias', 'default' == 'on')):
al = [info["aliases"] for rid, (info) in pvInfo.items() if info["pvName"] == pv and "aliases" in info ]
if len(al) == 1:
ali = al[0]
for a in ali:
self.channel_dict[a].append(iocid) # add iocname to pvName in dict
self.iocs[iocid]["channelcount"] += 1
for pv in delrec:
if iocid in self.channel_dict[pv]:
self.channel_dict[pv].remove(iocid)
if iocid in self.iocs:
self.iocs[iocid]["channelcount"] -= 1
if self.iocs[iocid]['channelcount'] == 0:
self.iocs.pop(iocid, None)
elif self.iocs[iocid]['channelcount'] < 0:
_log.error("channel count negative!")
if len(self.channel_dict[pv]) <= 0: # case: channel has no more iocs
del self.channel_dict[pv]
"""In case, alias exists"""
if (self.conf.get('alias', 'default' == 'on')):
al = [info["aliases"] for rid, (info) in pvInfo.items() if info["pvName"] == pv and "aliases" in info ]
if len(al) == 1:
ali = al[0]
for a in ali:
self.channel_dict[a].remove(iocid)
if iocid in self.iocs:
self.iocs[iocid]["channelcount"] -= 1
if self.iocs[iocid]['channelcount'] == 0:
self.iocs.pop(iocid, None)
elif self.iocs[iocid]['channelcount'] < 0:
_log.error("channel count negative!")
if len(self.channel_dict[a]) <= 0: # case: channel has no more iocs
del self.channel_dict[a]
poll(__updateCF__, self.client, pvInfo, delrec, self.channel_dict, self.iocs, self.conf, hostName, iocName, iocid,
owner, time)
dict_to_file(self.channel_dict, self.iocs, self.conf)
def clean_service(self):
"""
Marks all channels as "Inactive" until the recsync server is back up
"""
sleep = 1
retry_limit = 5
owner = self.conf.get('username', 'cfstore')
while 1:
try:
_log.debug("Cleaning service...")
channels = self.client.findByArgs([('pvStatus', 'Active')])
if channels is not None:
new_channels = []
for ch in channels or []:
new_channels.append(ch[u'name'])
if len(new_channels) > 0:
self.client.update(property={u'name': 'pvStatus', u'owner': owner, u'value': "Inactive"},
channelNames=new_channels)
_log.debug("Service clean.")
return
except RequestException:
_log.exception("cleaning failed, retrying: ")
time.sleep(min(60, sleep))
sleep *= 1.5
if self.running == 0 and sleep >= retry_limit:
_log.debug("Abandoning clean.")
return
def cfs_append_from_sqlite(fname, update_only):
sq = ap.chanfinder.ChannelFinderAgent()
sq.loadSqlite(fname)
cf = ChannelFinderClient(**cfinput)
all_prpts = [p.Name for p in cf.getAllProperties()]
all_tags = [t.Name for t in cf.getAllTags()]
ignore_prpts = ['hostName', 'iocName']
allpvs = []
tag_owner = OWNER
prpt_owner = PRPTOWNER
prpt_data, tag_data = {}, {}
# the data body
for pv, prpts, tags in sq.rows:
if not pv: continue
if pv in allpvs: continue
if pv.find("SR:") != 0: continue
logging.info("updating '{0}'".format(pv))
allpvs.append(pv)
prpt_list, tag_list = [], []
for k,v in prpts.items():
if k not in ["elemIndex", "system", "elemType", "elemHandle",
"elemName", "elemField"]: continue
prpt_data.setdefault((k, v), [])
prpt_data[(k, v)].append(pv)
for tag in tags:
if not tag.startswith("aphla."): continue
tag_data.setdefault(tag, [])
tag_data[tag].append(pv)
#tag_list.append(Tag(r.strip()), tag_owner)
logging.info("{0}: {1} ({2})".format(pv, tag, tag_owner))
#addPvTag(cf, pv, tag, tag_owner)
errpvs = []
for pv in allpvs:
chs = cf.find(name=pv)
if not chs:
errpvs.append(pv)
print "PV '%s' does not exist" % pv
continue
elif len(chs) != 1:
print "Find two results for pv=%s" % pv
continue
prpts = chs[0].getProperties()
if not prpts: continue
for prpt,val in prpts.items():
pvlst = prpt_data.get((prpt, val), [])
if not pvlst: continue
try:
j = pvlst.index(pv)
prpt_data[(prpt,val)].pop(j)
except:
# the existing data is not in the update list, skip
pass
#if errpvs:
# #raise RuntimeError("PVs '{0}' are missing".format(errpvs))
# print
logging.warn("{0} does not exist in DB".format(errpvs))
for k,v in prpt_data.iteritems():
vf = [pv for pv in v if pv not in errpvs]
if not vf:
logging.info("no valid PVs for {0}".format(k))
continue
updatePropertyPvs(cf, k[0], prpt_owner, k[1], vf)
logging.info("add property {0} for pvs {1}".format(k, vf))
for k,v in tag_data.iteritems():
vf = [pv for pv in v if pv not in errpvs]
addTagPvs(cf, k, vf, tag_owner)
logging.info("add tag {0} for pvs {1}".format(k, vf))
def testPreservingOfAttributes(self):
'''
This test is to ensure that existing properties and tags are left untouched.
Case1:
first time the cf-update comes across these channels and adds hostName and iocName
Case2:
the hostName is changed
Case3:
the iocName is changed
Case4:
both hostName and iocName are changed
Case5:
the channel is removed
in all cases the existing unaffected* property and tag should remain with the channel
'''
unaffectedProperty = {
u'name': u'unaffectedProperty',
u'owner': self.owner,
u'value': u'unchanged'
}
unaffectedTag = {u'name': u'unaffectedTag', u'owner': self.owner}
# create default client
client = ChannelFinderClient(BaseURL=self.baseURL,
username=self.username,
password=self.password)
client.set(property=unaffectedProperty)
client.set(tag=unaffectedTag)
client.set(
channel={
u'name': u'cf-update-pv1',
u'owner': u'cf-update',
u'properties': [unaffectedProperty],
u'tags': [unaffectedTag]
})
client.set(
channel={
u'name': u'cf-update-pv2',
u'owner': u'cf-update',
u'properties': [unaffectedProperty],
u'tags': [unaffectedTag]
})
# Case1:
hostName = 'initialHost'
iocName = 'initialIoc'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel['properties']
and unaffectedTag in channel['tags'])
self.assertTrue(
self.__check4properties(
{
u'name': u'hostName',
u'value': hostName
}, channel['properties']) and self.__check4properties(
{
u'name': u'iocName',
u'value': iocName
}, channel['properties']) and self.__check4properties(
{
u'name': u'pvStatus',
u'value': u'Active'
}, channel['properties']),
'Failed to update channels with the correct hostName and/or iocName'
)
# Case2:
hostName = 'newHost'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel['properties']
and unaffectedTag in channel['tags'])
self.assertTrue(
self.__check4properties(
{
u'name': u'hostName',
u'value': hostName
}, channel['properties']) and self.__check4properties(
{
u'name': u'iocName',
u'value': iocName
}, channel['properties']) and self.__check4properties(
{
u'name': u'pvStatus',
u'value': u'Active'
}, channel['properties']),
'Failed to update channels with the correct hostName and/or iocName'
)
self.assertTrue(
not client.find(property=[('hostName', 'initialHost')]),
'Failed to cleanup old property')
# Case 3:
iocName = 'newIoc'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel['properties']
and unaffectedTag in channel['tags'])
self.assertTrue(
self.__check4properties(
{
u'name': u'hostName',
u'value': hostName
}, channel['properties']) and self.__check4properties(
{
u'name': u'iocName',
u'value': iocName
}, channel['properties']) and self.__check4properties(
{
u'name': u'pvStatus',
u'value': u'Active'
}, channel['properties']),
'Failed to update channels with the correct hostName and/or iocName'
)
self.assertTrue(
not client.find(property=[('hostName', 'initialHost')]),
'Failed to cleanup old property')
self.assertTrue(not client.find(property=[('iocName', 'initialIoc')]),
'Failed to cleanup old property')
# Case 4:
updateChannelFinder([], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel['properties']
and unaffectedTag in channel['tags'])
self.assertTrue(
self.__check4properties(
{
u'name': u'hostName',
u'value': hostName
}, channel['properties']) and self.__check4properties(
{
u'name': u'iocName',
u'value': iocName
}, channel['properties']) and self.__check4properties(
{
u'name': u'pvStatus',
u'value': u'Inactive'
}, channel['properties']),
'Failed to update channels with the correct hostName and/or iocName'
)
self.assertTrue(
not client.find(property=[('hostName', 'initialHost')]),
'Failed to cleanup old property')
self.assertTrue(not client.find(property=[('iocName', 'initialIoc')]),
'Failed to cleanup old property')
# Cleanup
'''
TODO this cleanup code should not be contingent to the successful completion of all checks...
This could pollute CF
'''
client.delete(channelName='cf-update-pv1')
client.delete(channelName='cf-update-pv2')
client.delete(propertyName=unaffectedProperty[u'name'])
client.delete(tagName=unaffectedTag[u'name'])
def testPVMove(self):
'''
ch1, ch2 on host1, ioc1
ch1 on host1, ioc1; ch2 on host1, ioc2 (case1)
ch1, ch2 on host1, ioc1 (reset)
ch1 on host1, ioc1; ch2 on host2, ioc2 (case2)
ch1, ch2 on host1, ioc1 (reset)
'''
try:
updateChannelFinder(['ch1', 'ch2'], \
'host1', \
'ioc1', \
owner=self.owner, \
time=time(),\
service=self.baseURL, \
username=self.username, password=self.password)
client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
chs = client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])
self.assertEqual(len(chs), 2, 'Expected 2 positive matches but found ' + str(len(chs)))
'''CASE1'''
updateChannelFinder(['ch1'], \
'host1', \
'ioc1', \
time=time(), \
owner=self.owner, service=self.baseURL, \
username=self.username, password=self.password)
updateChannelFinder(['ch2'], \
'host1', \
'ioc2', \
time=time(), \
owner=self.owner, service=self.baseURL, \
username=self.username, password=self.password)
chs = client.find(property=[('hostName', 'host1')])
self.assertEqual(len(chs), 2, 'Expected 1 positive matches but found ' + str(len(chs)))
self.assertEqual(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])[0].Name, 'ch1', \
'Failed to find the expected channel _ch1_ with prop host1, ioc1')
self.assertEqual(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc2')])[0].Name, 'ch2', \
'Failed to find the expected channel _ch2_ with prop host1, ioc2')
'''RESET'''
updateChannelFinder(['ch1', 'ch2'], \
'host1', \
'ioc1', \
time=time(), \
owner=self.owner, service=self.baseURL, \
username=self.username, password=self.password)
self.assertEqual(len(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])), 2, \
'Failed to reset the channels')
'''CASE2'''
updateChannelFinder(['ch1'], \
'host1', \
'ioc1', \
owner=self.owner, \
time=time(), \
service=self.baseURL, username=self.username, \
password=self.password)
updateChannelFinder(['ch2'], \
'host2', \
'ioc2', \
owner=self.owner, service=self.baseURL, \
time=time(), \
username=self.username, password=self.password)
self.assertEqual(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])[0].Name, 'ch1', \
'Failed to find the expected channel _ch1_ with prop host1, ioc1')
self.assertEqual(client.find(property=[('hostName', 'host2'), ('iocName', 'ioc2')])[0].Name, 'ch2', \
'Failed to find the expected channel _ch2_ with prop host1, ioc2')
'''RESET'''
updateChannelFinder(['ch1', 'ch2'], \
'host1', \
'ioc1', \
owner=self.owner, \
time=time(), \
service=self.baseURL, \
username=self.username, password=self.password)
self.assertEqual(len(client.find(property=[('hostName', 'host1'), ('iocName', 'ioc1')])), 2, \
'Failed to reset the channels')
finally:
client.delete(channelName='ch1')
client.delete(channelName='ch2')
def testPreservingOfAttributes(self):
'''
This test is to ensure that existing properties and tags are left untouched.
Case1:
first time the cf-update comes across these channels and adds hostName and iocName
Case2:
the hostName is changed
Case3:
the iocName is changed
Case4:
both hostName and iocName are changed
Case5:
the channel is removed
in all cases the existing unaffected* property and tag should remain with the channel
'''
unaffectedProperty = {u'name':u'unaffectedProperty', u'owner':self.owner, u'value':u'unchanged'}
unaffectedTag = {u'name':u'unaffectedTag', u'owner':self.owner}
# create default client
client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
client.set(property=unaffectedProperty)
client.set(tag=unaffectedTag)
client.set(channel={u'name':u'cf-update-pv1', u'owner':u'cf-update', u'properties':[unaffectedProperty], u'tags':[unaffectedTag]})
client.set(channel={u'name':u'cf-update-pv2', u'owner':u'cf-update', u'properties':[unaffectedProperty], u'tags':[unaffectedTag]})
# Case1:
hostName = 'initialHost'
iocName = 'initialIoc'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel['properties'] and unaffectedTag in channel['tags'])
self.assertTrue(self.__check4properties({u'name':u'hostName', u'value':hostName}, channel['properties']) and
self.__check4properties({u'name':u'iocName', u'value':iocName}, channel['properties']) and
self.__check4properties({u'name':u'pvStatus', u'value':u'Active'}, channel['properties']),
'Failed to update channels with the correct hostName and/or iocName')
# Case2:
hostName = 'newHost'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel['properties'] and unaffectedTag in channel['tags'])
self.assertTrue(self.__check4properties({u'name':u'hostName', u'value':hostName}, channel['properties']) and
self.__check4properties({u'name':u'iocName', u'value':iocName}, channel['properties']) and
self.__check4properties({u'name':u'pvStatus', u'value':u'Active'}, channel['properties']),
'Failed to update channels with the correct hostName and/or iocName')
self.assertTrue(not client.find(property=[('hostName', 'initialHost')]), 'Failed to cleanup old property')
# Case 3:
iocName = 'newIoc'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel['properties'] and unaffectedTag in channel['tags'])
self.assertTrue(self.__check4properties({u'name':u'hostName', u'value':hostName}, channel['properties']) and
self.__check4properties({u'name':u'iocName', u'value':iocName}, channel['properties']) and
self.__check4properties({u'name':u'pvStatus', u'value':u'Active'}, channel['properties']),
'Failed to update channels with the correct hostName and/or iocName')
self.assertTrue(not client.find(property=[('hostName', 'initialHost')]), 'Failed to cleanup old property')
self.assertTrue(not client.find(property=[('iocName', 'initialIoc')]), 'Failed to cleanup old property')
# Case 4:
updateChannelFinder([], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel['properties'] and unaffectedTag in channel['tags'])
self.assertTrue(self.__check4properties({u'name':u'hostName', u'value':hostName}, channel['properties']) and
self.__check4properties({u'name':u'iocName', u'value':iocName}, channel['properties']) and
self.__check4properties({u'name':u'pvStatus', u'value':u'Inactive'}, channel['properties']),
'Failed to update channels with the correct hostName and/or iocName')
self.assertTrue(not client.find(property=[('hostName', 'initialHost')]), 'Failed to cleanup old property')
self.assertTrue(not client.find(property=[('iocName', 'initialIoc')]), 'Failed to cleanup old property')
# Cleanup
'''
TODO this cleanup code should not be contingent to the successful completion of all checks...
This could pollute CF
'''
client.delete(channelName='cf-update-pv1')
client.delete(channelName='cf-update-pv2')
client.delete(propertyName=unaffectedProperty[u'name'])
client.delete(tagName=unaffectedTag[u'name'])
def _export_to_cfweb(channels, uri, username, password):
# Improve performance by reducing the number
# of HTTP server requests by restructuring
# the channel data from channel oriented:
#
# {
# "CH:NAME1": {
# "properties":{
# "system":"SEG1",
# "device":"DEV1"
# },
# "tags":[
# "phyutil.sys.LS1",
# "phyutil.sub.CA01"
# ]
# },
# "CH:NAME2: {
# "properties":{
# "system":"SEG1",
# "device":"DEV2"
# }
# "tags":[
# "phyutil.sys.LS1",
# "phyutil.sub.BTS"
# ]
# }
# }
#
# To property oriented:
#
# {
# "system" : {
# "SEG1": [ "CH:NAME1", "CH:NAME2" ]
# },
# "device" : {
# "DEV1": [ "CH:NAME1" ],
# "DEV2": [ "CH:NAME2" ]
# }
# }
#
# And tag oriented:
#
# {
# "phyutil.sys.LS1": [ "CH:NAME1", "CH:NAME2" ],
# "phyutil.sub.CA01": [ "CH:NAME1" ],
# "phyutil.sub.BTS": [ "CH:NAME2" ]
# }
#
tags = {}
properties = {}
channelnames = set()
for name, ps, ts in channels:
for p, value in ps.iteritems():
if p not in properties:
properties[p] = {}
v = str(value)
if v not in properties[p]:
properties[p][v] = set()
properties[p][v].add(name)
for t in ts:
if t not in tags:
tags[t] = set()
tags[t].add(name)
channelnames.add(name)
client = ChannelFinderClient(BaseURL=uri, username=username, password=password)
channels = []
for cname in channelnames:
c = Channel(cname, username)
channels.append(c)
_LOGGER.debug("Export to Channel Finder: Set channel: %s", _Chan(c))
client.set(channels=channels)
_LOGGER.info("Export to ChannelFinder: Set channels: %s", len(channels))
for prop, values in properties.iteritems():
p = Property(prop, username)
client.set(property=p)
_LOGGER.info("Export to ChannelFinder: Set property: %s", _Prop(p))
for propvalue, channelnames in values.iteritems():
p.Value = propvalue
client.update(property=p, channelNames=channelnames)
_LOGGER.debug("Export to ChannelFinder: Update property: %s, for channels: %s", _Prop(p), len(channelnames))
for tag, channelnames in tags.iteritems():
t = Tag(tag, username)
client.set(tag=t)
_LOGGER.info("Export to ChannelFinder: Set tag: %s", _Tag(t))
client.update(tag=t, channelNames=channelnames)
_LOGGER.debug("Export to ChannelFinder: Update tag: %s, for channels: %s", _Tag(t), len(channelnames))
from channelfinder import ChannelFinderClient
"""
Simple script for adding active channels to Channel Finder Service for testing cf-store clean
If it gives a 500 error, run it again. Glassfish and CFS must be set up and running.
"""
def abbr(name, hostname, iocname, status):
return {
u"owner": "cf-update",
u"name": name,
u"properties": [
{u"owner": "cf-update", u"name": "hostName", u"value": hostname},
{u"owner": "cf-update", u"name": "iocName", u"value": iocname},
{u"owner": "cf-update", u"name": "pvStatus", u"value": status},
{u"owner": "cf-update", u"name": "time", u"value": "2016-08-18 12:33:09.953985"},
],
}
client = ChannelFinderClient()
client.set(channels=[abbr(u"ch1", "testhosta", 1111, "Active"), abbr(u"test_channel", "testhosta", 1111, "Active")])
def updateChannelFinder(pvNames, hostName, iocName, time, owner, \
service=None, username=None, password=None):
'''
pvNames = list of pvNames
([] permitted will effectively remove the hostname, iocname from all channels)
hostName = pv hostName (None not permitted)
iocName = pv iocName (None not permitted)
owner = the owner of the channels and properties being added, this can be different from the user
e.g. user = abc might create a channel with owner = group-abc
time = the time at which these channels are being created/modified
[optional] if not specified the default values are used by the
channelfinderapi lib
service = channelfinder service URL
username = channelfinder username
password = channelfinder password
'''
if hostName == None or iocName == None:
raise Exception, 'missing hostName or iocName'
channels = []
try:
client = ChannelFinderClient(BaseURL=service, username=username, password=password)
except:
raise Exception, 'Unable to create a valid webResourceClient'
checkPropertiesExist(client, owner)
previousChannelsList = client.findByArgs([('hostName', hostName), ('iocName', iocName)])
if previousChannelsList != None:
for ch in previousChannelsList:
if pvNames != None and ch[u'name'] in pvNames:
''''''
channels.append(updateChannel(ch,\
owner=owner, \
hostName=hostName, \
iocName=iocName, \
pvStatus='Active', \
time=time))
pvNames.remove(ch[u'name'])
elif pvNames == None or ch[u'name'] not in pvNames:
'''Orphan the channel : mark as inactive, keep the old hostName and iocName'''
channels.append(updateChannel(ch, \
owner=owner, \
hostName=ch.getProperties()['hostName'], \
iocName=ch.getProperties()['iocName'], \
pvStatus='InActive', \
time=ch.getProperties()['time']))
# now pvNames contains a list of pv's new on this host/ioc
for pv in pvNames:
ch = client.findByArgs([('~name',pv)])
if ch == None:
'''New channel'''
channels.append(createChannel(pv, \
chOwner=owner, \
hostName=hostName, \
iocName=iocName, \
pvStatus='Active', \
time=time))
elif ch[0] != None:
'''update existing channel: exists but with a different hostName and/or iocName'''
channels.append(updateChannel(ch[0], \
owner=owner, \
hostName=hostName, \
iocName=iocName, \
pvStatus='Active', \
time=time))
client.set(channels=channels)
def downloadCfs(self, cfsurl, **kwargs):
"""
downloads data from channel finder service.
:param cfsurl: the URL of channel finder service.
:type cfsurl: str
:param keep: if present, it only downloads specified properties.
:type keep: list
:param converter: convert properties from string to other format.
:type converter: dict
:Example:
>>> prpt_list = ['elemName', 'sEnd']
>>> conv_dict = {'sEnd', float}
>>> downloadCfs(URL, keep = prpt_list, converter = conv_dict)
>>> downloadCfs(URL, property=[('hostName', 'virtac2')])
>>> downloadCfs(URL, property=[('hostName', 'virtac')], tagName='aphla.*')
The channel finder client API provides *property* and *tagName* as
keywords parameters.
"""
keep_prpts = kwargs.pop('keep', None)
converter = kwargs.pop('converter', {})
self.source = cfsurl
from channelfinder import ChannelFinderClient
cf = ChannelFinderClient(BaseURL=cfsurl)
if len(kwargs) == 0:
chs = cf.find(name='*')
else:
#print kwargs
chs = cf.find(**kwargs)
if not chs: return
if keep_prpts is None:
# use all possible property names
keep_prpts = [p.Name for p in cf.getAllProperties()]
#print "# include properties", properties
for ch in chs:
# keep only known properties
prptdict = ch.getProperties()
# prpts is known part from prptdict, otherwise empty dict
if prptdict is not None:
prpts = dict([v for v in prptdict.iteritems()])
# convert the data type
else:
prpts = None
# the empty tags could be None
if self.use_unicode:
self.rows.append([
unicode(ch.Name),
dict([(unicode(k), unicode(v))
for k, v in prpts.iteritems()]),
[unicode(v) for v in ch.getTags()]
])
else:
self.rows.append([
ch.Name.encode('ascii'),
dict([(k.encode('ascii'), v.encode('ascii'))
for k, v in prpts.iteritems()]),
[v.encode('ascii') for v in ch.getTags()]
])
if self.rows[-1][1]:
for k in converter:
self.rows[-1][1][k] = converter[k](prpts[k])
# warn if hostName or iocName does not present
if "hostName" not in self.rows[-1][1]:
_logger.warn("no 'hostName' for {0}".format(self.rows[-1]))
if "iocName" not in self.rows[-1][1]:
_logger.warn("no 'iocName' for {0}".format(self.rows[-1]))
del prptdict
class CFProcessor(service.Service):
implements(interfaces.IProcessor)
def __init__(self, name, conf):
_log.info("CF_INIT %s", name)
self.name, self.conf = name, conf
self.channel_dict = defaultdict(list)
self.iocs = dict()
self.client = None
self.currentTime = getCurrentTime
self.lock = DeferredLock()
def startService(self):
service.Service.startService(self)
self.running = 1
_log.info("CF_START")
from channelfinder import ChannelFinderClient
# Using the default python cf-client.
# The usr, username, and password are provided by the channelfinder._conf module.
if self.client is None: # For setting up mock test client
self.client = ChannelFinderClient()
self.clean_service()
def stopService(self):
service.Service.stopService(self)
#Set channels to inactive and close connection to client
self.running = 0
self.clean_service()
_log.info("CF_STOP")
@defer.inlineCallbacks
def commit(self, transaction_record):
yield self.lock.acquire()
try:
yield deferToThread(self.__commit__, transaction_record)
finally:
self.lock.release()
def __commit__(self, TR):
_log.debug("CF_COMMIT %s", TR.infos.items())
pvNames = [unicode(rname, "utf-8") for rid, (rname, rtype) in TR.addrec.iteritems()]
delrec = list(TR.delrec)
iocName = TR.src.port
hostName = TR.src.host
iocid = hostName + ":" + str(iocName)
owner = TR.infos.get('CF_USERNAME') or TR.infos.get('ENGINEER') or self.conf.get('username', 'cfstore')
time = self.currentTime()
if TR.initial:
self.iocs[iocid] = {"iocname": iocName, "hostname": hostName, "owner": owner, "channelcount": 0} # add IOC to source list
if not TR.connected:
delrec.extend(self.channel_dict.keys())
for pv in pvNames:
self.channel_dict[pv].append(iocid) # add iocname to pvName in dict
self.iocs[iocid]["channelcount"] += 1
for pv in delrec:
if iocid in self.channel_dict[pv]:
self.channel_dict[pv].remove(iocid)
self.iocs[iocid]["channelcount"] -= 1
if self.iocs[iocid]['channelcount'] == 0:
self.iocs.pop(iocid, None)
elif self.iocs[iocid]['channelcount'] < 0:
_log.error("channel count negative!")
if len(self.channel_dict[pv]) <= 0: # case: channel has no more iocs
del self.channel_dict[pv]
poll(__updateCF__, self.client, pvNames, delrec, self.channel_dict, self.iocs, hostName, iocName, time, owner)
dict_to_file(self.channel_dict, self.iocs, self.conf)
def clean_service(self):
sleep = 1
retry_limit = 5
owner = self.conf.get('username', 'cfstore')
while 1:
try:
_log.debug("Cleaning service...")
channels = self.client.findByArgs([('pvStatus', 'Active')])
if channels is not None:
new_channels = []
for ch in channels or []:
new_channels.append(ch[u'name'])
if len(new_channels) > 0:
self.client.update(property={u'name': 'pvStatus', u'owner': owner, u'value': "Inactive"},
channelNames=new_channels)
_log.debug("Service clean.")
return
except RequestException:
_log.exception("cleaning failed, retrying: ")
time.sleep(min(60, sleep))
sleep *= 1.5
if self.running == 0 and sleep >= retry_limit:
_log.debug("Abandoning clean.")
return
def downloadCfs(self, cfsurl, **kwargs):
"""
downloads data from channel finder service.
:param cfsurl: the URL of channel finder service.
:type cfsurl: str
:param keep: if present, it only downloads specified properties.
:type keep: list
:param converter: convert properties from string to other format.
:type converter: dict
:Example:
>>> prpt_list = ['elemName', 'sEnd']
>>> conv_dict = {'sEnd', float}
>>> downloadCfs(URL, keep = prpt_list, converter = conv_dict)
>>> downloadCfs(URL, property=[('hostName', 'virtac2')])
>>> downloadCfs(URL, property=[('hostName', 'virtac')], tagName='aphla.*')
The channel finder client API provides *property* and *tagName* as
keywords parameters.
"""
keep_prpts = kwargs.pop('keep', None)
converter = kwargs.pop('converter', {})
self.source = cfsurl
from channelfinder import ChannelFinderClient
cf = ChannelFinderClient(BaseURL = cfsurl)
if len(kwargs) == 0:
chs = cf.find(name='*')
else:
#print kwargs
chs = cf.find(**kwargs)
if not chs: return
if keep_prpts is None:
# use all possible property names
keep_prpts = [p.Name for p in cf.getAllProperties()]
#print "# include properties", properties
for ch in chs:
# keep only known properties
prptdict = ch.getProperties()
# prpts is known part from prptdict, otherwise empty dict
if prptdict is not None:
prpts = dict([v for v in prptdict.iteritems()])
# convert the data type
else:
prpts = None
# the empty tags could be None
if self.use_unicode:
self.rows.append([unicode(ch.Name),
dict([(unicode(k), unicode(v))
for k,v in prpts.iteritems()]),
[unicode(v) for v in ch.getTags()]])
else:
self.rows.append([ch.Name.encode('ascii'),
dict([(k.encode('ascii'), v.encode('ascii'))
for k,v in prpts.iteritems()]),
[v.encode('ascii') for v in ch.getTags()]])
if self.rows[-1][1]:
for k in converter:
self.rows[-1][1][k] = converter[k](prpts[k])
# warn if hostName or iocName does not present
if "hostName" not in self.rows[-1][1]:
_logger.warn("no 'hostName' for {0}".format(self.rows[-1]))
if "iocName" not in self.rows[-1][1]:
_logger.warn("no 'iocName' for {0}".format(self.rows[-1]))
del prptdict
def testAddUpdateChannelsWithProperties(self):
'''
This is to check that existing properties of channels are not affected.
'''
unaffectedProperty = Property('unaffectedProperty', self.owner, 'unchanged')
# create default client
client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
client.set(property=unaffectedProperty)
# add new pv's
t1 = str(time())
hostName1 = 'update-test-hostname' + t1
iocName1 = 'update-test-iocName' + t1
# New Channels added
client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password);
client.set(channel=Channel('cf-update-pv1', 'cf-update', properties=[unaffectedProperty]))
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName1, \
iocName1, \
owner=self.owner, \
time=t1, \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(property=[('hostName', hostName1), ('iocName', iocName1), ('time', t1)])
self.assertTrue(len(channels) == 2, 'failed to create the channels with appropriate properties')
channels = client.find(name='cf-update-pv1')
self.assertTrue(len(channels) == 1)
self.assertTrue(len(channels[0].Properties) == 5)
# Cleanup
client.delete(channelName='cf-update-pv1')
client.delete(channelName='cf-update-pv2')
client.delete(propertyName=unaffectedProperty.Name)
def initCfs(fpv, sep=","):
dat = {}
for line in open(fpv, 'r').readlines():
#pv, hdl, name, idx, fam, fld
rec = [v.strip() for v in line.split(sep)]
pv, prpt = rec[0], rec[1:]
dat.setdefault(pv, [])
dat[pv].append(tuple(rec[1:]))
import conf
cfinput = {
'BaseURL': cfsurl,
'username': conf.username,
'password': conf.password
}
cf = ChannelFinderClient(**cfinput)
prpts = [p.Name for p in cf.getAllProperties()]
for p in ["elemName", "elemIndex", "elemType", "elemField"]:
if p in prpts: continue
# property owner is cf-asd
logging.info("add new property '%s'" % p)
cf.set(property=Property(p, "cf-asd"))
for pv,prptsets in dat.items():
if len(prptsets) == 1:
hdl, name, idx, fam, fld = prptsets[0]
#cf.update(channel=Channel(pv, "cf-update", properties=[
# Property("elemHandle", "cf-asd", hdl),
# Property("elemName", "cf-asd", name),
# Property("elemIndex", "cf-asd", idx),
# Property("elemType", "cf-asd", fam),
# Property("elemField", "cf-asd", fld)]))
cf.update(property=Property("elemHandle", "cf-asd", hdl),
channelName=pv)
cf.update(property=Property("elemName", "cf-asd", name),
channelName=pv)
cf.update(property=Property("elemIndex", "cf-asd", idx),
channelName=pv)
cf.update(property=Property("elemType", "cf-asd", fam),
channelName=pv)
cf.update(property=Property("elemField", "cf-asd", fld),
channelName=pv)
logging.info("Done with %s" % pv)
class CFProcessor(service.Service):
def __init__(self, name, conf):
_log.info("CF_INIT %s", name)
self.name, self.conf = name, conf
self.channel_dict = defaultdict(list)
self.iocs = dict()
self.client = None
self.currentTime = getCurrentTime
self.lock = DeferredLock()
def startService(self):
service.Service.startService(self)
# Returning a Deferred is not supported by startService(),
# so instead attempt to acquire the lock synchonously!
d = self.lock.acquire()
if not d.called:
d.cancel()
service.Service.stopService(self)
raise RuntimeError(
'Failed to acquired CF Processor lock for service start')
try:
self._startServiceWithLock()
except:
service.Service.stopService(self)
raise
finally:
self.lock.release()
def _startServiceWithLock(self):
_log.info("CF_START")
if self.client is None: # For setting up mock test client
"""
Using the default python cf-client. The url, username, and
password are provided by the channelfinder._conf module.
"""
from channelfinder import ChannelFinderClient
self.client = ChannelFinderClient()
try:
cf_props = [
prop['name'] for prop in self.client.getAllProperties()
]
if (self.conf.get('alias', 'default') == 'on'):
reqd_props = {
'hostName', 'iocName', 'pvStatus', 'time', 'iocid',
'alias'
}
else:
reqd_props = {
'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
}
wl = self.conf.get('infotags', list())
whitelist = [s.strip(', ') for s in wl.split()] \
if wl else wl
# Are any required properties not already present on CF?
properties = reqd_props - set(cf_props)
# Are any whitelisted properties not already present on CF?
# If so, add them too.
properties.update(set(whitelist) - set(cf_props))
owner = self.conf.get('username', 'cfstore')
for prop in properties:
self.client.set(property={u'name': prop, u'owner': owner})
self.whitelist = set(whitelist)
_log.debug('WHITELIST = {}'.format(self.whitelist))
except ConnectionError:
_log.exception("Cannot connect to Channelfinder service")
raise
else:
if self.conf.getboolean('cleanOnStart', True):
self.clean_service()
def stopService(self):
service.Service.stopService(self)
return self.lock.run(self._stopServiceWithLock)
def _stopServiceWithLock(self):
# Set channels to inactive and close connection to client
if self.conf.getboolean('cleanOnStop', True):
self.clean_service()
_log.info("CF_STOP")
# @defer.inlineCallbacks # Twisted v16 does not support cancellation!
def commit(self, transaction_record):
return self.lock.run(self._commitWithLock, transaction_record)
def _commitWithLock(self, TR):
self.cancelled = False
t = deferToThread(self._commitWithThread, TR)
def cancelCommit(d):
self.cancelled = True
d.callback(None)
d = defer.Deferred(cancelCommit)
def waitForThread(_ignored):
if self.cancelled:
return t
d.addCallback(waitForThread)
def chainError(err):
if not err.check(defer.CancelledError):
_log.error("CF_COMMIT FAILURE: %s", err)
if self.cancelled:
if not err.check(defer.CancelledError):
raise defer.CancelledError()
return err
else:
d.callback(None)
def chainResult(_ignored):
if self.cancelled:
raise defer.CancelledError()
else:
d.callback(None)
t.addCallbacks(chainResult, chainError)
return d
def _commitWithThread(self, TR):
if not self.running:
raise defer.CancelledError(
'CF Processor is not running (TR: %s:%s)', TR.src.host,
TR.src.port)
_log.info("CF_COMMIT: %s", TR)
"""
a dictionary with a list of records with their associated property info
pvInfo
{rid: { "pvName":"recordName",
"infoProperties":{propName:value, ...}}}
"""
host = TR.src.host
port = TR.src.port
iocName = TR.infos.get('IOCNAME') or TR.src.port
hostName = TR.infos.get('HOSTNAME') or TR.src.host
owner = TR.infos.get('ENGINEER') or TR.infos.get(
'CF_USERNAME') or self.conf.get('username', 'cfstore')
time = self.currentTime()
"""The unique identifier for a particular IOC"""
iocid = host + ":" + str(port)
pvInfo = {}
for rid, (rname, rtype) in TR.addrec.items():
pvInfo[rid] = {"pvName": rname}
for rid, (recinfos) in TR.recinfos.items():
# find intersection of these sets
if rid not in pvInfo:
_log.warn('IOC: %s: PV not found for recinfo with RID: %s',
iocid, rid)
continue
recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
if recinfo_wl:
pvInfo[rid]['infoProperties'] = list()
for infotag in recinfo_wl:
property = {
u'name': infotag,
u'owner': owner,
u'value': recinfos[infotag]
}
pvInfo[rid]['infoProperties'].append(property)
for rid, alias in TR.aliases.items():
if rid not in pvInfo:
_log.warn('IOC: %s: PV not found for alias with RID: %s',
iocid, rid)
continue
pvInfo[rid]['aliases'] = alias
delrec = list(TR.delrec)
_log.debug("Delete records: %s", delrec)
pvInfoByName = {}
for rid, (info) in pvInfo.items():
if info["pvName"] in pvInfoByName:
_log.warn(
"Commit contains multiple records with PV name: %s (%s)",
pv, iocid)
continue
pvInfoByName[info["pvName"]] = info
_log.debug("Add record: %s: %s", rid, info)
if TR.initial:
"""Add IOC to source list """
self.iocs[iocid] = {
"iocname": iocName,
"hostname": hostName,
"owner": owner,
"time": time,
"channelcount": 0
}
if not TR.connected:
delrec.extend(self.channel_dict.keys())
for pv in pvInfoByName.keys():
self.channel_dict[pv].append(
iocid) # add iocname to pvName in dict
self.iocs[iocid]["channelcount"] += 1
"""In case, alias exists"""
if (self.conf.get('alias', 'default' == 'on')):
if pv in pvInfoByName and "aliases" in pvInfoByName[pv]:
for a in pvInfoByName[pv]["aliases"]:
self.channel_dict[a].append(
iocid) # add iocname to pvName in dict
self.iocs[iocid]["channelcount"] += 1
for pv in delrec:
if iocid in self.channel_dict[pv]:
self.channel_dict[pv].remove(iocid)
if iocid in self.iocs:
self.iocs[iocid]["channelcount"] -= 1
if self.iocs[iocid]['channelcount'] == 0:
self.iocs.pop(iocid, None)
elif self.iocs[iocid]['channelcount'] < 0:
_log.error("Channel count negative: %s", iocid)
if len(self.channel_dict[pv]
) <= 0: # case: channel has no more iocs
del self.channel_dict[pv]
"""In case, alias exists"""
if (self.conf.get('alias', 'default' == 'on')):
if pv in pvInfoByName and "aliases" in pvInfoByName[pv]:
for a in pvInfoByName[pv]["aliases"]:
self.channel_dict[a].remove(iocid)
if iocid in self.iocs:
self.iocs[iocid]["channelcount"] -= 1
if self.iocs[iocid]['channelcount'] == 0:
self.iocs.pop(iocid, None)
elif self.iocs[iocid]['channelcount'] < 0:
_log.error("Channel count negative: %s", iocid)
if len(self.channel_dict[a]
) <= 0: # case: channel has no more iocs
del self.channel_dict[a]
poll(__updateCF__, self, pvInfoByName, delrec, hostName, iocName,
iocid, owner, time)
dict_to_file(self.channel_dict, self.iocs, self.conf)
def clean_service(self):
"""
Marks all channels as "Inactive" until the recsync server is back up
"""
sleep = 1
retry_limit = 5
owner = self.conf.get('username', 'cfstore')
while 1:
try:
_log.info("CF Clean Started")
channels = self.client.findByArgs(
prepareFindArgs(self.conf, [('pvStatus', 'Active')]))
if channels is not None:
new_channels = []
for ch in channels or []:
new_channels.append(ch[u'name'])
_log.info("Total channels to update: %s",
len(new_channels))
while len(new_channels) > 0:
_log.debug(
'Update "pvStatus" property to "Inactive" for %s channels',
min(len(new_channels), 10000))
self.client.update(property={
u'name': 'pvStatus',
u'owner': owner,
u'value': "Inactive"
},
channelNames=new_channels[:10000])
new_channels = new_channels[10000:]
_log.info("CF Clean Completed")
return
except RequestException as e:
_log.error("Clean service failed: %s", e)
_log.info("Clean service retry in %s seconds", min(60, sleep))
time.sleep(min(60, sleep))
sleep *= 1.5
if self.running == 0 and sleep >= retry_limit:
_log.info("Abandoning clean after %s seconds", retry_limit)
return
def testPreservingOfAttributes(self):
'''
This test is to ensure that existing properties and tags are left untouched.
Case1:
first time the cf-update comes across these channels and adds hostName and iocName
Case2:
the hostName is changed
Case3:
the iocName is changed
Case4:
both hostName and iocName are changed
Case5:
the channel is removed
in all cases the existing unaffected* property and tag should remain with the channel
'''
unaffectedProperty = Property('unaffectedProperty', self.owner, 'unchanged')
unaffectedTag = Tag('unaffectedTag', self.owner)
# create default client
client = ChannelFinderClient(BaseURL=self.baseURL, username=self.username, password=self.password)
client.set(property=unaffectedProperty)
client.set(tag=unaffectedTag)
client.set(channel=Channel('cf-update-pv1', 'cf-update', properties=[unaffectedProperty], tags=[unaffectedTag]))
client.set(channel=Channel('cf-update-pv2', 'cf-update', properties=[unaffectedProperty], tags=[unaffectedTag]))
# Case1:
hostName = 'initialHost'
iocName = 'initialIoc'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel.Properties and unaffectedTag in channel.Tags)
self.assertTrue(channel.getProperties()['hostName'] == hostName and \
channel.getProperties()['iocName'] == iocName and \
channel.getProperties()['pvStatus'] == 'Active', \
'Failed to update channels with the correct hostName and/or iocName')
# Case2:
hostName = 'newHost'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel.Properties and unaffectedTag in channel.Tags)
self.assertTrue(channel.getProperties()['hostName'] == hostName and \
channel.getProperties()['iocName'] == iocName and \
channel.getProperties()['pvStatus'] == 'Active', \
'Failed to update channels with the correct hostName and/or iocName')
self.assertTrue(client.find(property=[('hostName', 'initialHost')]) == None, 'Failed to cleanup old property')
# Case 3:
iocName = 'newIoc'
updateChannelFinder(['cf-update-pv1', 'cf-update-pv2'], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel.Properties and unaffectedTag in channel.Tags)
self.assertTrue(channel.getProperties()['hostName'] == hostName and \
channel.getProperties()['iocName'] == iocName and \
channel.getProperties()['pvStatus'] == 'Active', 'Failed to update channels with the correct hostName and/or iocName')
self.assertTrue(client.find(property=[('hostName', 'initialHost')]) == None, 'Failed to cleanup old property')
self.assertTrue(client.find(property=[('iocName', 'initialIoc')]) == None, 'Failed to cleanup old property')
# Case 4:
updateChannelFinder([], \
hostName, \
iocName, \
owner=self.owner, \
time=time(), \
service=self.baseURL , \
username=self.username, \
password=self.password)
channels = client.find(name='cf-update-pv*')
for channel in channels:
self.assertTrue(unaffectedProperty in channel.Properties and unaffectedTag in channel.Tags)
self.assertTrue(channel.getProperties()['hostName'] == hostName and \
channel.getProperties()['iocName'] == iocName and \
channel.getProperties()['pvStatus'] == 'InActive', \
'Failed to update channels with the correct hostName and/or iocName')
self.assertTrue(client.find(property=[('hostName', 'initialHost')]) == None, 'Failed to cleanup old property')
self.assertTrue(client.find(property=[('iocName', 'initialIoc')]) == None, 'Failed to cleanup old property')
# Cleanup
client.delete(channelName='cf-update-pv1')
client.delete(channelName='cf-update-pv2')
client.delete(propertyName=unaffectedProperty.Name)
client.delete(tagName=unaffectedTag.Name)
class CFProcessor(service.Service):
def __init__(self, name, conf):
_log.info("CF_INIT %s", name)
self.name, self.conf = name, conf
self.channel_dict = defaultdict(list)
self.iocs = dict()
self.client = None
self.currentTime = getCurrentTime
self.lock = DeferredLock()
def startService(self):
service.Service.startService(self)
self.running = 1
_log.info("CF_START")
if self.client is None: # For setting up mock test client
"""
Using the default python cf-client. The url, username, and
password are provided by the channelfinder._conf module.
"""
from channelfinder import ChannelFinderClient
self.client = ChannelFinderClient()
try:
cf_props = [
prop['name'] for prop in self.client.getAllProperties()
]
reqd_props = {
'hostName', 'iocName', 'pvStatus', 'time', 'iocid'
}
wl = self.conf.get('infotags', list())
whitelist = [s.strip(', ') for s in wl.split()] \
if wl else wl
# Are any required properties not already present on CF?
properties = reqd_props - set(cf_props)
# Are any whitelisted properties not already present on CF?
# If so, add them too.
properties.update(set(whitelist) - set(cf_props))
owner = self.conf.get('username', 'cfstore')
for prop in properties:
self.client.set(property={u'name': prop, u'owner': owner})
self.whitelist = set(whitelist)
_log.debug('WHITELIST = {}'.format(self.whitelist))
except ConnectionError:
_log.exception("Cannot connect to Channelfinder service")
raise
else:
self.clean_service()
def stopService(self):
service.Service.stopService(self)
# Set channels to inactive and close connection to client
self.running = 0
self.clean_service()
_log.info("CF_STOP")
@defer.inlineCallbacks
def commit(self, transaction_record):
yield self.lock.acquire()
try:
yield deferToThread(self.__commit__, transaction_record)
finally:
self.lock.release()
def __commit__(self, TR):
_log.debug("CF_COMMIT %s", TR.infos.items())
"""
a dictionary with a list of records with their associated property info
pvInfo
{rid: { "pvName":"recordName",
"infoProperties":{propName:value, ...}}}
"""
iocName = TR.infos.get('IOCNAME') or TR.src.port
hostName = TR.infos.get('HOSTNAME') or TR.src.host
owner = TR.infos.get('ENGINEER') or TR.infos.get(
'CF_USERNAME') or self.conf.get('username', 'cfstore')
time = self.currentTime()
pvInfo = {}
for rid, (rname, rtype) in TR.addrec.items():
pvInfo[rid] = {"pvName": rname}
for rid, (recinfos) in TR.recinfos.items():
# find intersection of these sets
recinfo_wl = [p for p in self.whitelist if p in recinfos.keys()]
if recinfo_wl:
pvInfo[rid]['infoProperties'] = list()
for infotag in recinfo_wl:
_log.debug('INFOTAG = {}'.format(infotag))
property = {
u'name': infotag,
u'owner': owner,
u'value': recinfos[infotag]
}
pvInfo[rid]['infoProperties'].append(property)
_log.debug(pvInfo)
pvNames = [info["pvName"] for rid, (info) in pvInfo.items()]
delrec = list(TR.delrec)
_log.info("DELETED records " + str(delrec))
host = TR.src.host
port = TR.src.port
"""The unique identifier for a particular IOC"""
iocid = host + ":" + str(port)
_log.info("CF_COMMIT: " + iocid)
if TR.initial:
"""Add IOC to source list """
self.iocs[iocid] = {
"iocname": iocName,
"hostname": hostName,
"owner": owner,
"time": time,
"channelcount": 0
}
if not TR.connected:
delrec.extend(self.channel_dict.keys())
for pv in pvNames:
self.channel_dict[pv].append(
iocid) # add iocname to pvName in dict
self.iocs[iocid]["channelcount"] += 1
for pv in delrec:
if iocid in self.channel_dict[pv]:
self.channel_dict[pv].remove(iocid)
if iocid in self.iocs:
self.iocs[iocid]["channelcount"] -= 1
if self.iocs[iocid]['channelcount'] == 0:
self.iocs.pop(iocid, None)
elif self.iocs[iocid]['channelcount'] < 0:
_log.error("channel count negative!")
if len(self.channel_dict[pv]
) <= 0: # case: channel has no more iocs
del self.channel_dict[pv]
poll(__updateCF__, self.client, pvInfo, delrec, self.channel_dict,
self.iocs, hostName, iocName, iocid, owner, time)
dict_to_file(self.channel_dict, self.iocs, self.conf)
def clean_service(self):
"""
Marks all channels as "Inactive" until the recsync server is back up
"""
sleep = 1
retry_limit = 5
owner = self.conf.get('username', 'cfstore')
while 1:
try:
_log.debug("Cleaning service...")
channels = self.client.findByArgs([('pvStatus', 'Active')])
if channels is not None:
new_channels = []
for ch in channels or []:
new_channels.append(ch[u'name'])
if len(new_channels) > 0:
self.client.update(property={
u'name': 'pvStatus',
u'owner': owner,
u'value': "Inactive"
},
channelNames=new_channels)
_log.debug("Service clean.")
return
except RequestException:
_log.exception("cleaning failed, retrying: ")
time.sleep(min(60, sleep))
sleep *= 1.5
if self.running == 0 and sleep >= retry_limit:
_log.debug("Abandoning clean.")
return
def cfs_append_from_csv1(rec_list, update_only):
cf = ChannelFinderClient(**cfinput)
all_prpts = [p.Name for p in cf.getAllProperties()]
all_tags = [t.Name for t in cf.getAllTags()]
ignore_prpts = ['hostName', 'iocName']
import csv
rd = csv.reader(rec_list)
# header line
header = rd.next()
# lower case of header
hlow = [s.lower() for s in header]
# number of headers, pv + properties
nheader = len(header)
# the index of PV, properties and tags
ipv = hlow.index('pv')
# the code did not rely on it, but it is a good practice
if ipv != 0:
raise RuntimeError("the first column should be pv")
iprpt, itags = [], []
for i, h in enumerate(header):
if i == ipv: continue
# if the header is empty, it is a tag
if len(h.strip()) == 0:
itags.append(i)
else:
iprpt.append(i)
tag_owner = OWNER
prpt_owner = PRPTOWNER
tags = {}
# the data body
for s in rd:
prpts = [Property(header[i], prpt_owner, s[i]) for i in iprpt if s[i]]
# itags could be empty if we put all tags in the end columns
for i in itags + range(nheader, len(s)):
rec = tags.setdefault(s[i].strip(), [])
rec.append(s[ipv].strip())
#print s[ipv], prpts, tags
ch = cf.find(name=s[ipv])
if ch is None:
logging.warning("pv {0} does not exist".format(s[ipv]))
elif len(ch) > 1:
logging.warning("pv {0} is not unique ({1})".format(s[ipv], len(ch)))
else:
for p in prpts:
#continue
if p.Name in ignore_prpts: continue
#if p.Name != 'symmetry': continue
logging.info("updating '{0}' with property, {1}={2}".format(
s[ipv], p.Name, p.Value))
cf.update(channelName=s[ipv], property=p)
logging.info("finished updating properties")
for t,pvs in tags.iteritems():
if not hasTag(cf, t): cf.set(tag=Tag(t, tag_owner))
if 'V:1-SR-BI{BETA}X-I' in pvs: continue
if 'V:1-SR-BI{BETA}Y-I' in pvs: continue
try:
cf.update(tag=Tag(t, tag_owner), channelNames=pvs)
except:
print t, pvs
raise
logging.info("update '{0}' for {1} pvs".format(t, len(pvs)))
logging.info("finished updating tags")
