def adjust_params(self, api, kwds, inst=None):
"""
Adjust Phedex parameters for specific query requests
"""
if api.find('blockReplicas') != -1 or \
api.find('fileReplicas') != -1:
delete_keys(kwds, '*')
if kwds.has_key('node') and kwds['node'].find('*') == -1:
if not api == 'tfc' and kwds['node'] != 'required':
kwds['node'] = kwds['node'] + '*'
def adjust_params(self, api, kwds, inst=None):
"""
Adjust Phedex parameters for specific query requests
"""
if api.find('blockReplicas') != -1 or \
api.find('fileReplicas') != -1:
delete_keys(kwds, '*')
if 'node' in kwds and kwds['node'].find('*') == -1:
if not api == 'tfc' and kwds['node'] != 'required':
kwds['node'] = kwds['node'] + '*'
def test_delete_keys(self):
"""Tests gen_counter function"""
rec = {'a':1, 'b':1, 'c':2}
delete_keys(rec, 1)
self.assertEqual(rec.keys(), ['c'])
def apimap(self, dasquery):
"""
Analyze input query and yield url, api, args, format, expire
for further processing.
"""
srv = self.name # get local copy to avoid threading issues
cond = getarg(dasquery.mongo_query, 'spec', {})
instance = dasquery.mongo_query.get('instance', self.dbs_global)
skeys = getarg(dasquery.mongo_query, 'fields', [])
if not skeys:
skeys = []
self.logger.info("\n")
for api, value in self.map.items():
expire = value['expire']
iformat = value['format']
url = self.adjust_url(value['url'], instance)
if not url:
msg = '--- rejects API %s, no URL' % api
self.logger.info(msg)
continue
args = dict(value['params']) # make new copy, since we'll adjust
wild = value.get('wild_card', '*')
found = 0
# check if input parameters are covered by API
if not self.dasmapping.check_api_match(srv, api, cond):
msg = '--- rejects API %s, does not cover input condition keys' \
% api
self.logger.info(msg)
continue
# once we now that API covers input set of parameters we check
# every input parameter for pattern matching
for key, val in cond.items():
# check if keys from conditions are accepted by API
# need to convert key (which is daskeys.map) into
# input api parameter
for apiparam in self.dasmapping.das2api(srv, api, key, val):
if apiparam in args:
args[apiparam] = val
found += 1
# VK 20160708, wrong statement, it caused to pass
# datasets API for query dataset in [path1, path2]
# I'll leave block here until I test and verify that
# commented out block will not cause other issues
#
# check the case when we only have single condition key
# and it is the key we look-up
# if not found and skeys == [k.split('.')[0] for k in cond.keys()]:
# found = 1
# check if number of keys on cond and args are the same
if len(cond.keys()) != found:
msg = "--- reject API %s, not all condition keys are covered" \
% api
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
continue
if not found:
msg = "--- rejects API %s, parameters don't match" % api
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
continue
self.adjust_params(api, args, instance)
# delete args keys whose value is optional
delete_keys(args, 'optional')
# check that there is no "required" parameter left in args,
# since such api will not work
if 'required' in args.values():
msg = '--- rejects API %s, parameter is required' % api
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
continue
# adjust pattern symbols in arguments
if wild != '*':
for key, val in args.items():
if isinstance(val, str) or isinstance(val, unicode):
val = val.replace('*', wild)
args[key] = val
# compare query selection keys with API look-up keys
api_lkeys = self.dasmapping.api_lkeys(srv, api)
if set(api_lkeys) != set(skeys):
msg = "--- rejects API %s, api_lkeys(%s)!=skeys(%s)"\
% (api, api_lkeys, skeys)
self.logger.info(msg)
continue
msg = '+++ %s passes API %s' % (srv, api)
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
msg = "yield "
msg += "system ***%s***, url=%s, api=%s, args=%s, format=%s, " \
% (srv, url, api, args, iformat)
msg += "expire=%s, wild_card=%s" \
% (expire, wild)
self.logger.debug(msg)
yield url, api, args, iformat, expire
def apimap(self, dasquery):
"""
Analyze input query and yield url, api, args, format, expire
for further processing.
"""
srv = self.name # get local copy to avoid threading issues
cond = getarg(dasquery.mongo_query, "spec", {})
instance = dasquery.mongo_query.get("instance", self.dbs_global)
skeys = getarg(dasquery.mongo_query, "fields", [])
if not skeys:
skeys = []
self.logger.info("\n")
for api, value in self.map.iteritems():
expire = value["expire"]
iformat = value["format"]
url = self.adjust_url(value["url"], instance)
if not url:
msg = "--- rejects API %s, no URL" % api
self.logger.info(msg)
continue
args = dict(value["params"]) # make new copy, since we'll adjust
wild = value.get("wild_card", "*")
found = 0
# check if input parameters are covered by API
if not self.dasmapping.check_api_match(srv, api, cond):
msg = "--- rejects API %s, does not cover input condition keys" % api
self.logger.info(msg)
continue
# once we now that API covers input set of parameters we check
# every input parameter for pattern matching
for key, val in cond.iteritems():
# check if keys from conditions are accepted by API
# need to convert key (which is daskeys.map) into
# input api parameter
for apiparam in self.dasmapping.das2api(srv, api, key, val):
if apiparam in args:
args[apiparam] = val
found += 1
# check if number of keys on cond and args are the same
if len(cond.keys()) != found:
msg = "--- reject API %s, not all condition keys are covered" % api
self.logger.info(msg)
msg = "args=%s" % args
self.logger.debug(msg)
continue
if not found:
msg = "--- rejects API %s, parameters don't match" % api
self.logger.info(msg)
msg = "args=%s" % args
self.logger.debug(msg)
continue
self.adjust_params(api, args, instance)
# delete args keys whose value is optional
delete_keys(args, "optional")
# check that there is no "required" parameter left in args,
# since such api will not work
if "required" in args.values():
msg = "--- rejects API %s, parameter is required" % api
self.logger.info(msg)
msg = "args=%s" % args
self.logger.debug(msg)
continue
# adjust pattern symbols in arguments
if wild != "*":
for key, val in args.iteritems():
if isinstance(val, str) or isinstance(val, unicode):
val = val.replace("*", wild)
args[key] = val
# compare query selection keys with API look-up keys
api_lkeys = self.dasmapping.api_lkeys(srv, api)
if set(api_lkeys) != set(skeys):
msg = "--- rejects API %s, api_lkeys(%s)!=skeys(%s)" % (api, api_lkeys, skeys)
self.logger.info(msg)
continue
msg = "+++ %s passes API %s" % (srv, api)
self.logger.info(msg)
msg = "args=%s" % args
self.logger.debug(msg)
msg = "yield "
msg += "system ***%s***, url=%s, api=%s, args=%s, format=%s, " % (srv, url, api, args, iformat)
msg += "expire=%s, wild_card=%s" % (expire, wild)
self.logger.debug(msg)
yield url, api, args, iformat, expire
def test_delete_keys(self):
"""Tests gen_counter function"""
rec = {'a': 1, 'b': 1, 'c': 2}
delete_keys(rec, 1)
self.assertEqual(list(rec.keys()), ['c'])
def apimap(self, dasquery):
"""
Analyze input query and yield url, api, args, format, expire
for further processing.
"""
srv = self.name # get local copy to avoid threading issues
cond = getarg(dasquery.mongo_query, 'spec', {})
instance = dasquery.mongo_query.get('instance', self.dbs_global)
skeys = getarg(dasquery.mongo_query, 'fields', [])
if not skeys:
skeys = []
self.logger.info("\n")
for api, value in self.map.items():
expire = value['expire']
iformat = value['format']
url = self.adjust_url(value['url'], instance)
if not url:
msg = '--- rejects API %s, no URL' % api
self.logger.info(msg)
continue
args = dict(value['params']) # make new copy, since we'll adjust
wild = value.get('wild_card', '*')
found = 0
# check if input parameters are covered by API
if not self.dasmapping.check_api_match(srv, api, cond):
msg = '--- rejects API %s, does not cover input condition keys' \
% api
self.logger.info(msg)
continue
# once we now that API covers input set of parameters we check
# every input parameter for pattern matching
for key, val in cond.items():
# check if keys from conditions are accepted by API
# need to convert key (which is daskeys.map) into
# input api parameter
for apiparam in self.dasmapping.das2api(srv, api, key, val):
if apiparam in args:
args[apiparam] = val
found += 1
# VK 20160708, wrong statement, it caused to pass
# datasets API for query dataset in [path1, path2]
# I'll leave block here until I test and verify that
# commented out block will not cause other issues
#
# check the case when we only have single condition key
# and it is the key we look-up
# if not found and skeys == [k.split('.')[0] for k in cond.keys()]:
# found = 1
# check if number of keys on cond and args are the same
if len(cond.keys()) != found:
msg = "--- reject API %s, not all condition keys are covered" \
% api
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
continue
if not found:
msg = "--- rejects API %s, parameters don't match" % api
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
continue
self.adjust_params(api, args, instance)
# delete args keys whose value is optional
delete_keys(args, 'optional')
# check that there is no "required" parameter left in args,
# since such api will not work
if 'required' in args.values():
msg = '--- rejects API %s, parameter is required' % api
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
continue
# adjust pattern symbols in arguments
if wild != '*':
for key, val in args.items():
if isinstance(val, str) or isinstance(val, unicode):
val = val.replace('*', wild)
args[key] = val
# compare query selection keys with API look-up keys
api_lkeys = self.dasmapping.api_lkeys(srv, api)
if set(api_lkeys) != set(skeys):
msg = "--- rejects API %s, api_lkeys(%s)!=skeys(%s)"\
% (api, api_lkeys, skeys)
self.logger.info(msg)
continue
msg = '+++ %s passes API %s' % (srv, api)
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
msg = "yield "
msg += "system ***%s***, url=%s, api=%s, args=%s, format=%s, " \
% (srv, url, api, args, iformat)
msg += "expire=%s, wild_card=%s" \
% (expire, wild)
self.logger.debug(msg)
yield url, api, args, iformat, expire
def apimap(self, dasquery):
"""
Analyze input query and yield url, api, args, format, expire
for further processing.
"""
cond = getarg(dasquery.mongo_query, 'spec', {})
instance = dasquery.mongo_query.get('instance', self.dbs_global)
skeys = getarg(dasquery.mongo_query, 'fields', [])
if not skeys:
skeys = []
self.logger.info("\n")
for api, value in self.map.iteritems():
expire = value['expire']
iformat = value['format']
url = self.adjust_url(value['url'], instance)
args = dict(value['params']) # make new copy, since we'll adjust
wild = value.get('wild_card', '*')
found = 0
for key, val in cond.iteritems():
# check if key is a special one
if key in das_special_keys():
found += 1
# check if keys from conditions are accepted by API.
if self.dasmapping.check_dasmap(self.name, api, key, val):
# need to convert key (which is daskeys.map) into
# input api parameter
for apiparam in \
self.dasmapping.das2api(self.name, key, val, api):
if args.has_key(apiparam):
args[apiparam] = val
found += 1
else:
found = 0
break # condition key does not map into API params
self.adjust_params(api, args, instance)
if not found:
msg = "--- rejects API %s, parameters don't match" % api
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
continue
# delete args keys whose value is optional
delete_keys(args, 'optional')
# check that there is no "required" parameter left in args,
# since such api will not work
if 'required' in args.values():
msg = '--- rejects API %s, parameter is required' % api
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
continue
# adjust pattern symbols in arguments
if wild != '*':
for key, val in args.iteritems():
if isinstance(val, str) or isinstance(val, unicode):
val = val.replace('*', wild)
args[key] = val
prim_key = self.dasmapping.primary_key(self.name, api)
if prim_key not in skeys:
msg = "--- rejects API %s, primary_key %s is not selected"\
% (api, prim_key)
self.logger.info(msg)
continue
msg = '+++ %s passes API %s' % (self.name, api)
self.logger.info(msg)
msg = 'args=%s' % args
self.logger.debug(msg)
msg = "yield "
msg += "system ***%s***, url=%s, api=%s, args=%s, format=%s, " \
% (self.name, url, api, args, iformat)
msg += "expire=%s, wild_card=%s" \
% (expire, wild)
self.logger.debug(msg)
yield url, api, args, iformat, expire
