def phedex_files(phedex_url, kwds):
"Get file information from Phedex"
params = dict(kwds) # parameters to be send to Phedex
site = kwds.get('site', None)
if site and phedex_node_pattern.match(site):
if not site.endswith('*'):
# this will account to look-up site names w/o _Buffer or _MSS
site += '*'
params.update({'node': site})
params.pop('site')
elif site and se_pattern.match(site):
params.update({'se': site})
params.pop('site')
else:
return
expire = 600 # set some expire since we're not going to use it
headers = {'Accept': 'text/xml'}
source, expire = \
getdata(phedex_url, params, headers, expire, ckey=CKEY, cert=CERT,
system='phedex')
tags = 'block.file.name'
prim_key = 'block'
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
files = ddict.get('block.file')
if not isinstance(files, list):
files = [files]
for row in files:
yield row['name']
def files4site(phedex_url, files, site):
"Find site for given files"
params = {}
if site and phedex_node_pattern.match(site):
if not site.endswith('*'):
# this will account to look-up site names w/o _Buffer or _MSS
site += '*'
params.update({'node': site})
elif site and se_pattern.match(site):
params.update({'se': site})
else:
return
sname = urllib.urlencode(params)
urls = []
for fname in files:
url = '%s?lfn=%s&%s' % (phedex_url, fname, sname)
urls.append(url)
tags = 'block.replica.node'
prim_key = 'block'
gen = urlfetch_getdata(urls, CKEY, CERT)
for rec in gen:
if 'error' in rec.keys():
yield rec
else:
# convert record string into StringIO for xml_parser
source = StringIO.StringIO(rec['data'])
for row in xml_parser(source, prim_key, tags):
fobj = row['block']['file']
fname = fobj['name']
replica = fobj['replica']
for item in replica:
yield fname
def phedex_files(phedex_url, kwds):
"Get file information from Phedex"
params = dict(kwds) # parameters to be send to Phedex
site = kwds.get('site', None)
if site and phedex_node_pattern.match(site):
if not site.endswith('*'):
# this will account to look-up site names w/o _Buffer or _MSS
site += '*'
params.update({'node': site})
params.pop('site')
elif site and se_pattern.match(site):
params.update({'se': site})
params.pop('site')
else:
return
expire = 600 # set some expire since we're not going to use it
headers = {'Accept': 'text/xml'}
source, expire = \
getdata(phedex_url, params, headers, expire, ckey=CKEY, cert=CERT,
system='phedex')
tags = 'block.file.name'
prim_key = 'block'
for rec in xml_parser(source, prim_key, tags):
ddict = DotDict(rec)
files = ddict.get('block.file')
if not isinstance(files, list):
files = [files]
for row in files:
yield row['name']
def files4site(phedex_url, files, site):
"Find site for given files"
params = {}
if site and phedex_node_pattern.match(site):
if not site.endswith('*'):
# this will account to look-up site names w/o _Buffer or _MSS
site += '*'
params.update({'node': site})
elif site and se_pattern.match(site):
params.update({'se': site})
else:
return
sname = urllib.urlencode(params)
urls = []
for fname in files:
url = '%s?lfn=%s&%s' % (phedex_url, fname, sname)
urls.append(url)
tags = 'block.replica.node'
prim_key = 'block'
gen = urlfetch_getdata(urls, CKEY, CERT)
for rec in gen:
if 'error' in rec.keys():
yield rec
else:
# convert record string into StringIO for xml_parser
source = StringIO.StringIO(rec['data'])
for row in xml_parser(source, prim_key, tags):
fobj = row['block']['file']
fname = fobj['name']
replica = fobj['replica']
for item in replica:
yield fname
def site_info(self, site):
"Get information about given site"
url = None
site_names = self.map.get('site_names', None)
if site_names:
url = site_names.get('url', None)
if not url:
return
params = {}
expire = 3600 # not important
data, expire = self.getdata(url, params, expire)
for row in data['result']:
if se_pattern.match(site) and row['resources']:
for rec in row['resources']:
if rec['fqdn'] == site:
return row
if phedex_node_pattern.match(site) and row['name'] == site:
return row
def site_info(self, site):
"Get information about given site"
url = None
site_names = self.map.get('site_names', None)
if site_names:
url = site_names.get('url', None)
if not url:
return
params = {}
expire = 3600 # not important
data, expire = self.getdata(url, params, expire)
for row in data['result']:
if se_pattern.match(site) and row['resources']:
for rec in row['resources']:
if rec['fqdn'] == site:
return row
if phedex_node_pattern.match(site) and row['name'] == site:
return row