def run(facets_groups,parallel=True,index_host=None,dry_run=False):
"""Set 'searchapi_host' parameter.
Args:
facets_groups (list of dict): facets groups
index_host: index host from arguments (overwrite 'searchapi_host' in facets_groups, if any)
Notes
- If 'searchapi_host' parameter is missing both in func arguments and in 'facets_groups', a random index node is used.
- 'searchapi_host' or 'index_host' is the index host used as url HOST (i.e. the search-API service host where the query will be submitted)
- 'index_node' is the index host used as url PARAMETER (i.e. the SolR parameter)
"""
#verbose=sdstream.get_scalar(facets_groups,'verbose',default=False,type_=bool) # we cast here as verbose can be str (set from parameter) or bool (set from '-v' option)
verbose=False
for facets_group in facets_groups:
# index host from args take precedence over one from selection file (if any)
if index_host is not None:
facets_group['searchapi_host']=index_host
# if searchapi_host is set, we force parallel to False
# (it's not possible to use a specific index in parallel mode so we need to fallback to sequential mode)
if 'searchapi_host' in facets_group:
parallel=False
if dry_run:
# in dry_run mode, we want searchapi_host to be set, not matter if parallel is enabled or not
# (i.e. the query is not executed in dry_run mode, we just want to
# print a working query on stdout, not a query with fake host)
if 'searchapi_host' not in facets_group:
facets_group['searchapi_host']=sdindex.get_default_index()
else:
if parallel:
# in parallel mode, we just check that index host is not set (should always be the case)
assert 'searchapi_host' not in facets_group
else:
if 'searchapi_host' not in facets_group:
facets_group['searchapi_host']=sdindex.get_default_index()
#sdlog.debug("SDIDXHST-001","Using %s"%facets_group['searchapi_host'],stderr=False,logfile=True)
if verbose:
sdtools.print_stderr("Using %s"%facets_group['searchapi_host'])
return facets_groups
def run(pname=None, host=None, facets_group=None, dry_run=False):
"""
Returns:
Dict of list of 'Item' object
"""
if facets_group is None:
facets_group = {}
assert isinstance(facets_group, dict)
# keep only search-API parameter
facets_group = sddquery.search_api_parameters(facets_group)
# set parameter name
if pname is None:
facets_group['facets'] = ['*']
else:
facets_group['facets'] = [pname]
# force 'limit' to 0
facets_group['limit'] = ['0']
# set default type
if 'type' not in facets_group:
facets_group['type'] = ['File']
# force 'fields' to '*'
facets_group['fields'] = [
'*'
] # TODO: maybe this is not needed to retrieve parameter (maybe we can set 'fields' only to 'id', or something like that)
# set index host
host = sdindex.get_default_index() if host is None else host
# build url
url = sdremotequtils.build_url(facets_group, host)
if dry_run:
print url
return {}
# retrieve parameters
params = sdnetutils.call_param_web_service(url, 60)
return params
def run(pname=None,host=None,facets_group=None,dry_run=False):
"""
Returns:
Dict of list of 'Item' object
"""
if facets_group is None:
facets_group={}
assert isinstance(facets_group,dict)
# keep only search-API parameter
facets_group=sddquery.search_api_parameters(facets_group)
# set parameter name
if pname is None:
facets_group['facets']=['*']
else:
facets_group['facets']=[pname]
# force 'limit' to 0
facets_group['limit']=['0']
# set default type
if 'type' not in facets_group:
facets_group['type']=['File']
# force 'fields' to '*'
facets_group['fields']=['*'] # TODO: maybe this is not needed to retrieve parameter (maybe we can set 'fields' only to 'id', or something like that)
# set index host
host=sdindex.get_default_index() if host is None else host
# build url
url=sdremotequtils.build_url(facets_group,host)
if dry_run:
print url
return {}
# retrieve parameters
params=sdnetutils.call_param_web_service(url,60)
return params
def get_one_file(host=sdindex.get_default_index(),project=None,query=None,dry_run=None):
"""Return one sample file with all attributes."""
project_filter='' if project is None else "&project=%s"%project
query_filter='' if query is None else "&query=%s"%query
fmt=sdurlutils.get_solr_output_format(sdconfig.searchapi_output_format)
url='http://%s/esg-search/search?type=File%s%s&fields=*&format=%s'%(host,project_filter,query_filter,fmt)
if dry_run:
print url
return None
else:
# this code cannot use SearchAPIProxy as we need to modify the offset in a non usual way
request=Request(url=url,pagination=False,limit=50) # limit is arbitrary
result=sdnetutils.call_web_service(request.get_url(),timeout=60) # return Response object
# we loop until we find a correct file (i.e. well-formed)
well_formed_file=None
for i in range(20): # arbitrary
for file_ in result.get_files():
if len(file_['variable'])==1:
well_formed_file=file_
break
else:
#print "WARNING: 'variable' attribute contains too much values ('%s')."%file_['title']
pass
if well_formed_file is not None:
break
request.offset+=500
if request.offset<result.num_found:
result=sdnetutils.call_web_service(request.get_url(),timeout=60) # return Response object
else:
break
return well_formed_file
def run(host=None,reload=False,project=None):
# default
if host is None:
host=sdindex.get_default_index()
if reload:
sdsqlutils.truncate_table('param')
parameters=get_parameters_from_searchapi(host,project)
parameters=remove_unused_parameters(parameters)
parameters=add_system_parameters(parameters) # overwrite occurs here, it's normal (e.g. for 'type' parameter)
if reload:
sdlog.info("SDDCACHE-001","Reloading parameters (index=%s)"%host)
_reload_parameters(parameters)
else:
sdlog.info("SDDCACHE-002","Updating parameters (index=%s)"%host)
_update_parameters(parameters)
def run(host=None, reload=False, project=None):
# default
if host is None:
host = sdindex.get_default_index()
if reload:
sdsqlutils.truncate_table('param')
parameters = get_parameters_from_searchapi(host, project)
parameters = remove_unused_parameters(parameters)
parameters = add_system_parameters(
parameters
) # overwrite occurs here, it's normal (e.g. for 'type' parameter)
add_special_parameters(parameters)
if reload:
sdlog.info("SDDCACHE-001", "Reloading parameters (index=%s)" % host)
_reload_parameters(parameters)
else:
sdlog.info("SDDCACHE-002", "Updating parameters (index=%s)" % host)
_update_parameters(parameters)